From 6c4d151cdb1ccfbcb26c705fe15136a0dde2a74c Mon Sep 17 00:00:00 2001 From: Derek Xu Date: Fri, 19 Sep 2025 13:58:59 -0700 Subject: [PATCH 1/7] fix test --- tests/pytest/test_mcp_session_autocreate.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/pytest/test_mcp_session_autocreate.py b/tests/pytest/test_mcp_session_autocreate.py index 6cc76f50..208f65d0 100644 --- a/tests/pytest/test_mcp_session_autocreate.py +++ b/tests/pytest/test_mcp_session_autocreate.py @@ -15,9 +15,14 @@ def _run_airline_server(): import os + import sys - python_version = os.environ.get("PYTHON_VERSION", "3.10").replace(".", "") - port = str(9780 + int(python_version[-1:])) + # Get Python version directly from sys.version_info + minor_version = sys.version_info.minor # 10, 11, 12 + + # Map Python versions to port offsets: 3.10->0, 3.11->1, 3.12->2 + port_offset = minor_version - 10 + port = str(9780 + port_offset) os.environ["PORT"] = port from eval_protocol.mcp_servers.tau2.tau2_mcp import AirlineDomainMcp @@ -27,14 +32,18 @@ def _run_airline_server(): @pytest.mark.asyncio async def test_tool_call_returns_json_without_prior_initial_state(): - import os + import sys proc = Process(target=_run_airline_server, daemon=True) proc.start() try: - python_version = os.environ.get("PYTHON_VERSION", "3.10").replace(".", "") - port = str(9780 + int(python_version[-1:])) + # Get Python version directly from sys.version_info + minor_version = sys.version_info.minor # 10, 11, 12 + + # Map Python versions to port offsets: 3.10->0, 3.11->1, 3.12->2 + port_offset = minor_version - 10 + port = str(9780 + port_offset) base_url = f"http://127.0.0.1:{port}/mcp" client = httpx.Client(timeout=1.0) From 107190d81560cda4b735631f83a0cc7a6a2ffd1e Mon Sep 17 00:00:00 2001 From: Derek Xu Date: Fri, 19 Sep 2025 14:21:13 -0700 Subject: [PATCH 2/7] debug --- tests/pytest/test_mcp_session_autocreate.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/pytest/test_mcp_session_autocreate.py b/tests/pytest/test_mcp_session_autocreate.py index 208f65d0..1265b226 100644 --- a/tests/pytest/test_mcp_session_autocreate.py +++ b/tests/pytest/test_mcp_session_autocreate.py @@ -23,10 +23,14 @@ def _run_airline_server(): # Map Python versions to port offsets: 3.10->0, 3.11->1, 3.12->2 port_offset = minor_version - 10 port = str(9780 + port_offset) + print(f"[SERVER DEBUG] Python 3.{minor_version} -> Setting PORT={port}") os.environ["PORT"] = port from eval_protocol.mcp_servers.tau2.tau2_mcp import AirlineDomainMcp + print(f"[SERVER DEBUG] About to create AirlineDomainMcp with PORT={os.environ.get('PORT')}") server = AirlineDomainMcp(seed=None) + print(f"[SERVER DEBUG] Server created, FastMCP port={server.mcp.settings.port}") + print(f"[SERVER DEBUG] About to run on port {port}") server.run(transport="streamable-http") @@ -44,8 +48,10 @@ async def test_tool_call_returns_json_without_prior_initial_state(): # Map Python versions to port offsets: 3.10->0, 3.11->1, 3.12->2 port_offset = minor_version - 10 port = str(9780 + port_offset) + print(f"[TEST DEBUG] Python 3.{minor_version} -> Looking for server on port {port}") base_url = f"http://127.0.0.1:{port}/mcp" + print(f"[TEST DEBUG] base_url = {base_url}") client = httpx.Client(timeout=1.0) start_time = time.time() deadline = start_time + 20 From 1ba4bcb37319efc6b72c30c81f565bed94700829 Mon Sep 17 00:00:00 2001 From: Derek Xu Date: Fri, 19 Sep 2025 14:28:36 -0700 Subject: [PATCH 3/7] test --- .github/workflows/debug-mcp-test.yml | 50 ++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/debug-mcp-test.yml diff --git a/.github/workflows/debug-mcp-test.yml b/.github/workflows/debug-mcp-test.yml new file mode 100644 index 00000000..acdfe807 --- /dev/null +++ b/.github/workflows/debug-mcp-test.yml @@ -0,0 +1,50 @@ +name: Debug MCP Session Test + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: # Allow manual triggering + +jobs: + debug-mcp-session-test: + name: Debug MCP Session Test (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for all tags and branches + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Install the project + run: uv sync --locked --all-extras --dev + + - name: Install tau2 for testing + run: uv pip install git+https://github.com/sierra-research/tau2-bench.git@main + + - name: Debug Python version info + run: | + echo "Python version: $(python --version)" + python -c "import sys; print(f'sys.version_info: {sys.version_info}')" + python -c "import sys; minor = sys.version_info.minor; port = 9780 + (minor - 10); print(f'Expected port: {port}')" + + - name: Run MCP Session Autocreate Test + env: + PYTHONWARNINGS: "ignore::DeprecationWarning,ignore::RuntimeWarning" + run: | + uv run pytest tests/pytest/test_mcp_session_autocreate.py::test_tool_call_returns_json_without_prior_initial_state -v -s --tb=short From 283082970e65d7c5a515b8892cd14d029d6209a4 Mon Sep 17 00:00:00 2001 From: Derek Xu Date: Fri, 19 Sep 2025 14:33:45 -0700 Subject: [PATCH 4/7] sanity check --- tests/pytest/test_mcp_session_autocreate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/test_mcp_session_autocreate.py b/tests/pytest/test_mcp_session_autocreate.py index 1265b226..da1a9332 100644 --- a/tests/pytest/test_mcp_session_autocreate.py +++ b/tests/pytest/test_mcp_session_autocreate.py @@ -22,7 +22,7 @@ def _run_airline_server(): # Map Python versions to port offsets: 3.10->0, 3.11->1, 3.12->2 port_offset = minor_version - 10 - port = str(9780 + port_offset) + port = str(9780) print(f"[SERVER DEBUG] Python 3.{minor_version} -> Setting PORT={port}") os.environ["PORT"] = port from eval_protocol.mcp_servers.tau2.tau2_mcp import AirlineDomainMcp @@ -47,7 +47,7 @@ async def test_tool_call_returns_json_without_prior_initial_state(): # Map Python versions to port offsets: 3.10->0, 3.11->1, 3.12->2 port_offset = minor_version - 10 - port = str(9780 + port_offset) + port = str(9780) print(f"[TEST DEBUG] Python 3.{minor_version} -> Looking for server on port {port}") base_url = f"http://127.0.0.1:{port}/mcp" From c27a9b7e8a398c63846bcc7e244a93decb278a61 Mon Sep 17 00:00:00 2001 From: Derek Xu Date: Fri, 19 Sep 2025 14:40:31 -0700 Subject: [PATCH 5/7] test this --- tests/pytest/test_mcp_session_autocreate.py | 67 +++++++++++---------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/tests/pytest/test_mcp_session_autocreate.py b/tests/pytest/test_mcp_session_autocreate.py index da1a9332..1dc00607 100644 --- a/tests/pytest/test_mcp_session_autocreate.py +++ b/tests/pytest/test_mcp_session_autocreate.py @@ -3,8 +3,10 @@ without requiring a prior initial state fetch, and returns JSON. """ +import asyncio +import subprocess +import sys import time -from multiprocessing import Process import httpx import pytest @@ -13,43 +15,46 @@ from eval_protocol.types import MCPSession -def _run_airline_server(): - import os - import sys - +@pytest.mark.asyncio +async def test_tool_call_returns_json_without_prior_initial_state(): # Get Python version directly from sys.version_info minor_version = sys.version_info.minor # 10, 11, 12 # Map Python versions to port offsets: 3.10->0, 3.11->1, 3.12->2 port_offset = minor_version - 10 - port = str(9780) - print(f"[SERVER DEBUG] Python 3.{minor_version} -> Setting PORT={port}") - os.environ["PORT"] = port - from eval_protocol.mcp_servers.tau2.tau2_mcp import AirlineDomainMcp - - print(f"[SERVER DEBUG] About to create AirlineDomainMcp with PORT={os.environ.get('PORT')}") - server = AirlineDomainMcp(seed=None) - print(f"[SERVER DEBUG] Server created, FastMCP port={server.mcp.settings.port}") - print(f"[SERVER DEBUG] About to run on port {port}") - server.run(transport="streamable-http") - + port = str(9780 + port_offset) + print(f"[TEST DEBUG] Python 3.{minor_version} -> Looking for server on port {port}") + + # Create server script to run as subprocess instead of multiprocessing + server_script = """ +import sys +import os + +# Get Python version directly from sys.version_info +minor_version = sys.version_info.minor # 10, 11, 12 + +# Map Python versions to port offsets: 3.10->0, 3.11->1, 3.12->2 +port_offset = minor_version - 10 +port = str(9780 + port_offset) +print(f"[SERVER DEBUG] Python 3.{minor_version} -> Setting PORT={port}") +os.environ["PORT"] = port + +from eval_protocol.mcp_servers.tau2.tau2_mcp import AirlineDomainMcp + +print(f"[SERVER DEBUG] About to create AirlineDomainMcp with PORT={os.environ.get('PORT')}") +server = AirlineDomainMcp(seed=None) +print(f"[SERVER DEBUG] Server created, FastMCP port={server.mcp.settings.port}") +print(f"[SERVER DEBUG] About to run on port {port}") +server.run(transport="streamable-http") +""" -@pytest.mark.asyncio -async def test_tool_call_returns_json_without_prior_initial_state(): - import sys + # Start server as subprocess instead of multiprocessing.Process + proc = subprocess.Popen([sys.executable, "-c", server_script]) - proc = Process(target=_run_airline_server, daemon=True) - proc.start() + # Give server time to start + await asyncio.sleep(3) try: - # Get Python version directly from sys.version_info - minor_version = sys.version_info.minor # 10, 11, 12 - - # Map Python versions to port offsets: 3.10->0, 3.11->1, 3.12->2 - port_offset = minor_version - 10 - port = str(9780) - print(f"[TEST DEBUG] Python 3.{minor_version} -> Looking for server on port {port}") - base_url = f"http://127.0.0.1:{port}/mcp" print(f"[TEST DEBUG] base_url = {base_url}") client = httpx.Client(timeout=1.0) @@ -66,7 +71,7 @@ async def test_tool_call_returns_json_without_prior_initial_state(): pass time.sleep(0.2) else: - pytest.fail("Server did not start on port 9780 in time") + pytest.fail(f"Server did not start on port {port} in time") assert ready_time is not None, "Server did not return a successful status before exiting loop" assert ready_time - start_time < 20, f"Server took too long to respond: {ready_time - start_time:.2f}s" @@ -86,4 +91,4 @@ async def test_tool_call_returns_json_without_prior_initial_state(): await mgr.close_session(session) finally: proc.terminate() - proc.join(timeout=5) + proc.wait(timeout=5) From 18d7659a489b3c6ae937624cecfdac71d5d145f6 Mon Sep 17 00:00:00 2001 From: Derek Xu Date: Fri, 19 Sep 2025 14:56:18 -0700 Subject: [PATCH 6/7] test just 9780 --- tests/pytest/test_mcp_session_autocreate.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/tests/pytest/test_mcp_session_autocreate.py b/tests/pytest/test_mcp_session_autocreate.py index 1dc00607..5bc4b5a3 100644 --- a/tests/pytest/test_mcp_session_autocreate.py +++ b/tests/pytest/test_mcp_session_autocreate.py @@ -17,34 +17,19 @@ @pytest.mark.asyncio async def test_tool_call_returns_json_without_prior_initial_state(): - # Get Python version directly from sys.version_info - minor_version = sys.version_info.minor # 10, 11, 12 - - # Map Python versions to port offsets: 3.10->0, 3.11->1, 3.12->2 - port_offset = minor_version - 10 - port = str(9780 + port_offset) - print(f"[TEST DEBUG] Python 3.{minor_version} -> Looking for server on port {port}") + port = "9780" # Create server script to run as subprocess instead of multiprocessing server_script = """ import sys import os -# Get Python version directly from sys.version_info -minor_version = sys.version_info.minor # 10, 11, 12 - -# Map Python versions to port offsets: 3.10->0, 3.11->1, 3.12->2 -port_offset = minor_version - 10 -port = str(9780 + port_offset) -print(f"[SERVER DEBUG] Python 3.{minor_version} -> Setting PORT={port}") +port = "9780" os.environ["PORT"] = port from eval_protocol.mcp_servers.tau2.tau2_mcp import AirlineDomainMcp -print(f"[SERVER DEBUG] About to create AirlineDomainMcp with PORT={os.environ.get('PORT')}") server = AirlineDomainMcp(seed=None) -print(f"[SERVER DEBUG] Server created, FastMCP port={server.mcp.settings.port}") -print(f"[SERVER DEBUG] About to run on port {port}") server.run(transport="streamable-http") """ @@ -56,7 +41,6 @@ async def test_tool_call_returns_json_without_prior_initial_state(): try: base_url = f"http://127.0.0.1:{port}/mcp" - print(f"[TEST DEBUG] base_url = {base_url}") client = httpx.Client(timeout=1.0) start_time = time.time() deadline = start_time + 20 From a9f5f415c03d900488f45935ec128d7a56a8fcbf Mon Sep 17 00:00:00 2001 From: Derek Xu Date: Fri, 19 Sep 2025 14:57:49 -0700 Subject: [PATCH 7/7] delete --- .github/workflows/debug-mcp-test.yml | 50 ---------------------------- 1 file changed, 50 deletions(-) delete mode 100644 .github/workflows/debug-mcp-test.yml diff --git a/.github/workflows/debug-mcp-test.yml b/.github/workflows/debug-mcp-test.yml deleted file mode 100644 index acdfe807..00000000 --- a/.github/workflows/debug-mcp-test.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Debug MCP Session Test - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - workflow_dispatch: # Allow manual triggering - -jobs: - debug-mcp-session-test: - name: Debug MCP Session Test (Python ${{ matrix.python-version }}) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.10", "3.11", "3.12"] - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch all history for all tags and branches - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install uv - uses: astral-sh/setup-uv@v6 - with: - enable-cache: true - - - name: Install the project - run: uv sync --locked --all-extras --dev - - - name: Install tau2 for testing - run: uv pip install git+https://github.com/sierra-research/tau2-bench.git@main - - - name: Debug Python version info - run: | - echo "Python version: $(python --version)" - python -c "import sys; print(f'sys.version_info: {sys.version_info}')" - python -c "import sys; minor = sys.version_info.minor; port = 9780 + (minor - 10); print(f'Expected port: {port}')" - - - name: Run MCP Session Autocreate Test - env: - PYTHONWARNINGS: "ignore::DeprecationWarning,ignore::RuntimeWarning" - run: | - uv run pytest tests/pytest/test_mcp_session_autocreate.py::test_tool_call_returns_json_without_prior_initial_state -v -s --tb=short