From 868c7ab80f2997cc4cab270eef33fb19192f72e5 Mon Sep 17 00:00:00 2001 From: adityamehra Date: Wed, 10 Jun 2026 14:58:49 -0700 Subject: [PATCH 01/12] ci: fix Python version mismatch and add Python 3.14 to matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit actions/setup-python@v6 with Poetry caching was using the runner's ambient `python` (e.g. 3.12 on Ubuntu, 3.14 on macOS) to create the venv instead of the matrix-specified version. This caused jobs labelled "Python 3.10/3.11/3.13" to silently execute under the wrong interpreter, even on a cold cache. Fix: - Add `id: setup-python` to capture the versioned python-path output. - Change `cache-dependency-path` from pyproject.toml → poetry.lock so the cache key is tied to exact locked dependencies. - Add "Configure Poetry Python" step that wipes any stale venv with `poetry env remove --all` and then calls `poetry env use ` to pin the venv to the exact binary installed by setup-python. - Add "Verify Poetry Python version" assertion that fails fast if the active venv ever drifts from the matrix target again. - Extend the matrix to include Python 3.14 (crewai/litellm are already gated to python_version < '3.14' in pyproject.toml so they are skipped cleanly on that version). Verified locally: all 2015 tests pass on 3.13; 1925 pass / 95 skip on 3.14 (expected — crewai/litellm extras excluded). Co-authored-by: Cursor --- .github/workflows/ci-tests.yaml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index e6f1f530..ec0fb092 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -22,7 +22,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] runs-on: ${{ matrix.os }} # Hard cap per matrix job — bail out fast on real hangs instead of @@ -41,11 +41,27 @@ jobs: run: pipx install poetry==2.1.3 - name: Set up Python ${{ matrix.python-version }} + id: setup-python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: cache: "poetry" python-version: ${{ matrix.python-version }} - cache-dependency-path: "pyproject.toml" + cache-dependency-path: "poetry.lock" + + - name: Configure Poetry Python + run: | + poetry env remove --all || true + poetry env use "${{ steps.setup-python.outputs.python-path }}" + + - name: Verify Poetry Python version + run: | + poetry run python -c " + import sys + expected = tuple(map(int, '${{ matrix.python-version }}'.split('.'))) + actual = sys.version_info[:len(expected)] + print('Python:', sys.version) + assert actual == expected, f'Expected Python {expected}, got {actual}' + " - name: Install invoke run: pipx install invoke From 665ba508f0e5b72c2d4c002dd972551d19849af2 Mon Sep 17 00:00:00 2001 From: adityamehra Date: Wed, 10 Jun 2026 15:08:34 -0700 Subject: [PATCH 02/12] ci: fix venv-before-install ordering and drop Poetry venv cache Root cause of the failure: `cache: "poetry"` in setup-python restores a potentially-wrong venv, and `poetry env remove --all` then deletes it. At that point `poetry env use ` only _records_ the Python choice without creating the venv. When `poetry run python` fired in the next step there was no venv yet, so Poetry fell through to the system Python (3.14.5 on macOS), causing every job to report the wrong version. Changes: - Remove `cache: "poetry"` and `cache-dependency-path` from setup-python. Without a restored cache there is nothing to wipe, so the venv is always freshly created by `poetry install` using the PATH Python that setup-python puts first (the matrix-specified version). - Drop `poetry env remove --all` (no stale venv to clear). - Simplify "Configure Poetry Python" to a single `poetry env use ` call. - Move "Install invoke" and "Install Dependencies" _before_ the verify step so the venv is fully populated before `poetry run python` executes. Co-authored-by: Cursor --- .github/workflows/ci-tests.yaml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index ec0fb092..752a7e0c 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -44,14 +44,16 @@ jobs: id: setup-python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: - cache: "poetry" python-version: ${{ matrix.python-version }} - cache-dependency-path: "poetry.lock" - name: Configure Poetry Python - run: | - poetry env remove --all || true - poetry env use "${{ steps.setup-python.outputs.python-path }}" + run: poetry env use "${{ steps.setup-python.outputs.python-path }}" + + - name: Install invoke + run: pipx install invoke + + - name: Install Dependencies + run: invoke install - name: Verify Poetry Python version run: | @@ -63,12 +65,6 @@ jobs: assert actual == expected, f'Expected Python {expected}, got {actual}' " - - name: Install invoke - run: pipx install invoke - - - name: Install Dependencies - run: invoke install - - name: Validate Types if: always() run: invoke type-check From e8989d20f193419c5825206eceea11a689591be4 Mon Sep 17 00:00:00 2001 From: adityamehra Date: Wed, 10 Jun 2026 15:25:07 -0700 Subject: [PATCH 03/12] =?UTF-8?q?ci:=20upgrade=20Poetry=202.1.3=20?= =?UTF-8?q?=E2=86=92=202.4.1=20to=20fix=20virtualenv=20Python=20selection?= =?UTF-8?q?=20bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Poetry 2.1.3 installs the latest virtualenv (>=20.33) which has a known bug: it ignores `poetry env use ` and always creates the venv with the runner's ambient Python instead of the matrix-specified one. This was confirmed by observing that only the jobs whose matrix Python matched the runner default (3.12 on Ubuntu, 3.14 on macOS) passed the version assertion, while all others failed. Timeline of the fix upstream: - 2.1.4: added `virtualenv<20.33` as a temporary workaround (#10491) - 2.4.1: re-enabled `virtualenv>=20.33` after the underlying issue was fixed properly in virtualenv itself (#10506) Upgrading to 2.4.1 (latest stable) resolves the root cause cleanly without needing to pin virtualenv separately. Co-authored-by: Cursor --- .github/workflows/ci-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index 752a7e0c..de83e8d7 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -38,7 +38,7 @@ jobs: run: git config --system core.longpaths true - name: Install poetry - run: pipx install poetry==2.1.3 + run: pipx install poetry==2.4.1 - name: Set up Python ${{ matrix.python-version }} id: setup-python From 965e64eea01b5dc468b670aea69a20f2672191c0 Mon Sep 17 00:00:00 2001 From: adityamehra Date: Wed, 10 Jun 2026 15:36:44 -0700 Subject: [PATCH 04/12] ci: remove poetry env use and id: setup-python (no longer needed with 2.4.1) Co-authored-by: Cursor --- .github/workflows/ci-tests.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index de83e8d7..1505c3c9 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -41,14 +41,10 @@ jobs: run: pipx install poetry==2.4.1 - name: Set up Python ${{ matrix.python-version }} - id: setup-python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: ${{ matrix.python-version }} - - name: Configure Poetry Python - run: poetry env use "${{ steps.setup-python.outputs.python-path }}" - - name: Install invoke run: pipx install invoke From 1cdcc21379634ea019c13cb04998297c6b833fd4 Mon Sep 17 00:00:00 2001 From: adityamehra Date: Wed, 10 Jun 2026 15:42:24 -0700 Subject: [PATCH 05/12] ci: restore cache: poetry and cache-dependency-path Co-authored-by: Cursor --- .github/workflows/ci-tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index 1505c3c9..62c45b42 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -43,7 +43,9 @@ jobs: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: + cache: "poetry" python-version: ${{ matrix.python-version }} + cache-dependency-path: "pyproject.toml" - name: Install invoke run: pipx install invoke From 47cccc148923832e6511ea90c298ecab126c7b0c Mon Sep 17 00:00:00 2001 From: adityamehra Date: Wed, 10 Jun 2026 15:55:04 -0700 Subject: [PATCH 06/12] ci: switch cache-dependency-path to poetry.lock to bust poisoned cache The first CI run on this branch used Poetry 2.1.3 + virtualenv>=20.33 (the wrong-Python bug), which cached venvs whose names said py3.10/py3.11 but whose executables were the runner default (3.12 on Ubuntu, 3.14 on macOS). Poetry matches cached venvs by name, not by the actual Python inside, so even Poetry 2.4.1 restores and reuses the poisoned venv. Changing cache-dependency-path from pyproject.toml to poetry.lock changes the cache key, forcing a cold build on next run. Poetry 2.4.1 will then create a correct venv and persist it under the new key. All subsequent runs get the right venv from cache. poetry.lock is also a better cache-dependency-path in general: it captures exact resolved versions, so the cache is invalidated whenever any transitive dependency is updated, not just direct-dep spec changes. Co-authored-by: Cursor --- .github/workflows/ci-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index 62c45b42..376db89a 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -45,7 +45,7 @@ jobs: with: cache: "poetry" python-version: ${{ matrix.python-version }} - cache-dependency-path: "pyproject.toml" + cache-dependency-path: "poetry.lock" - name: Install invoke run: pipx install invoke From dd1659f790158cffcd5baa447c028254fc30428b Mon Sep 17 00:00:00 2001 From: adityamehra Date: Wed, 10 Jun 2026 16:05:01 -0700 Subject: [PATCH 07/12] trigger ci From 1d12c49acc79333e8c67b1e32cbc4ca96f91b850 Mon Sep 17 00:00:00 2001 From: adityamehra Date: Fri, 12 Jun 2026 11:11:53 -0700 Subject: [PATCH 08/12] deps: pin grpcio>=1.80.0 to guarantee Python 3.14 pre-built wheels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit grpcio is a transitive dependency pulled in by opentelemetry-exporter-otlp → opentelemetry-exporter-otlp-proto-grpc, which only requires grpcio>=1.63.2/1.66.2. Any resolver (Poetry, uv, pip) could legitimately pick an older version that has no cp314 wheels, forcing source compilation (~20 min) on Python 3.14 CI runners and causing the 30-minute job timeout. Changes: - Add grpcio (>=1.80.0,<2.0.0) as an explicit optional dependency in both [project.optional-dependencies] (PEP 621 / uv) and [tool.poetry.dependencies] (Poetry), included in the otel and all extras. - Regenerate poetry.lock via `poetry lock`: resolves to grpcio 1.81.1 (cp314 wheels available for macOS arm64, Linux x86_64/aarch64, Windows) while keeping grpcio 1.74.0 pinned for the crewai extra on Python <=3.13 where the looser constraint still applies. Verified locally on macOS Python 3.14: install completes in ~3s (pre-built wheel), 1925 tests pass, 95 skipped. Co-authored-by: Cursor --- poetry.lock | 109 +++++++++++++++++++++++++++++++++++++++++++------ pyproject.toml | 8 +++- 2 files changed, 102 insertions(+), 15 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8c6872ac..a1acafdf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -484,7 +484,7 @@ files = [ {file = "cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662"}, {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"}, ] -markers = {main = "(extra == \"openai\" or extra == \"all\" or extra == \"crewai\") and python_version <= \"3.13\" or platform_python_implementation == \"PyPy\" or extra == \"openai\" or extra == \"all\"", test = "platform_python_implementation == \"PyPy\""} +markers = {main = "(python_version <= \"3.13\" or extra == \"langchain\" or extra == \"all\" or extra == \"openai\") and (platform_python_implementation != \"PyPy\" or extra == \"langchain\" or extra == \"all\") and (python_version <= \"3.13\" or platform_python_implementation == \"PyPy\" or extra == \"openai\" or extra == \"all\") and (extra == \"openai\" or extra == \"all\" or extra == \"crewai\" or extra == \"langchain\") and (extra == \"openai\" or extra == \"all\" or extra == \"crewai\" or platform_python_implementation == \"PyPy\")", test = "platform_python_implementation == \"PyPy\""} [package.dependencies] pycparser = "*" @@ -589,6 +589,7 @@ files = [ {file = "charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a"}, {file = "charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14"}, ] +markers = {main = "python_version < \"3.13\" and (extra == \"crewai\" or extra == \"all\" or extra == \"langchain\" or extra == \"openai\" or extra == \"otel\") or extra == \"langchain\" or extra == \"all\" or extra == \"openai\" or extra == \"otel\" or python_version <= \"3.13\" and (extra == \"langchain\" or extra == \"all\" or extra == \"openai\" or extra == \"otel\" or extra == \"crewai\")"} [[package]] name = "chromadb" @@ -933,6 +934,7 @@ files = [ {file = "distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2"}, {file = "distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed"}, ] +markers = {main = "(extra == \"crewai\" or extra == \"all\" or extra == \"openai\") and python_version <= \"3.13\" or extra == \"openai\" or extra == \"all\""} [[package]] name = "docstring-parser" @@ -1404,7 +1406,7 @@ description = "HTTP/2-based RPC framework" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "(extra == \"crewai\" or extra == \"all\" or extra == \"otel\") and python_version <= \"3.13\" or extra == \"otel\" or extra == \"all\"" +markers = "python_version <= \"3.13\" and (extra == \"crewai\" or extra == \"all\")" files = [ {file = "grpcio-1.74.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907"}, {file = "grpcio-1.74.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb"}, @@ -1462,6 +1464,74 @@ files = [ [package.extras] protobuf = ["grpcio-tools (>=1.74.0)"] +[[package]] +name = "grpcio" +version = "1.81.1" +description = "HTTP/2-based RPC framework" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "(extra == \"crewai\" or extra == \"all\" or extra == \"otel\") and python_version <= \"3.13\" or extra == \"otel\" or extra == \"all\"" +files = [ + {file = "grpcio-1.81.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:6f9a0c9c1cc15c112d1c053064fd032b64917062292c3d70aea280e02ae10b77"}, + {file = "grpcio-1.81.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:69ef28e54fc85397f91b8c19592b8ef3d81952080366914823bd8572a2958120"}, + {file = "grpcio-1.81.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:15641444eca4a29358107b3dceb74c1c6305c55c822fd199b458aaea4068a7fb"}, + {file = "grpcio-1.81.1-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:d4b2dddfc219f54f956ccd53cf76a1d338ffe68fc7f2849ec9c7feb9927ff692"}, + {file = "grpcio-1.81.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ca1cc11d82677b9662082e5478b7528e2b7db7beaa6bdff42bd62789d81be399"}, + {file = "grpcio-1.81.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:aa2ba7d2ad6df4d80127cea65e5b8d5e2c3adbf153ff4804452836328aca7c54"}, + {file = "grpcio-1.81.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:592b5fee597faa91cce2dd294dd7d9a1c83d76c4dbf877e33ec1adb866b2fbed"}, + {file = "grpcio-1.81.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:62481553b1793a27e9b9c3cf9e5bd483ef045ca72462592074b46d42b0c4d9b9"}, + {file = "grpcio-1.81.1-cp310-cp310-win32.whl", hash = "sha256:bb693b1e3d9a2f3fd228e2110daf4b5aeedb36761ca1e4282f74725f6d89f611"}, + {file = "grpcio-1.81.1-cp310-cp310-win_amd64.whl", hash = "sha256:88268ca418cacea64cecb0d1d600d3c6b3a8038fcba02e1e205178c5b1f47661"}, + {file = "grpcio-1.81.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:d71d30f2d92f67d944631c523713934fee37292469e182ebcd2c1dd8a64ce53f"}, + {file = "grpcio-1.81.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b137f4bf3ada9dc44d411478decc6ff09a79ed30b306cd2abaa98408c3588137"}, + {file = "grpcio-1.81.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a3acb384427816dd5d470f47e62137b87f74da694faa8a50147012cf40df276a"}, + {file = "grpcio-1.81.1-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:f9a0ebbe45c29b5e5866593c12b78bd9035f0f0f0d4bc8361680cd580d99db49"}, + {file = "grpcio-1.81.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a37165cc80b1a368384b383e63a4c38116a10467ae44c904d2d7468c4470ec2"}, + {file = "grpcio-1.81.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6282caffb41ec326d4cb67ca9cf53b739d1b2f975a2acb498c7418e9f7d9a416"}, + {file = "grpcio-1.81.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a35009284d0d3d5c2c9601c164a911b8b4331608d98a9a66d47d97bb2f522b70"}, + {file = "grpcio-1.81.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1b22c80559854b789a01fd89e8929b3798a156c0829b5282a8939f33ad4115ad"}, + {file = "grpcio-1.81.1-cp311-cp311-win32.whl", hash = "sha256:428bec0161b48d8cf583c068591bc0016d0d9cfff52462b72b3884861ea768c5"}, + {file = "grpcio-1.81.1-cp311-cp311-win_amd64.whl", hash = "sha256:30e825f6848d9f18bba350ed6c75c1b02a0b5184474a31db9a32b1fa66fd8c79"}, + {file = "grpcio-1.81.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:8b39472beafc0bdcafc4c8c73ad082ebfdb449d566897a61e7acb4fa88089115"}, + {file = "grpcio-1.81.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:12b7524c88d4026d3dcb7b0ebe16b6714f3b4af402ddd0f0639ab064a00c87c3"}, + {file = "grpcio-1.81.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1e123f9b37edb8375fd74130d1f69c944bbf0a7b06761ae7211154b8759e94d2"}, + {file = "grpcio-1.81.1-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2c2e2ae6867c2966b8daccc836d54a13218e0007e9a490aeb81dd05be64d22d7"}, + {file = "grpcio-1.81.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:766bc7c9a9c340342f4c864ccbda8e78111e4751f13b895812b9c148fb79e9d0"}, + {file = "grpcio-1.81.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b259a04a737cb3496be0901328eb8b7552ed8df4865d8c8f1cf1bffcfc0776a3"}, + {file = "grpcio-1.81.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:85b10a45b8993d195c4f3ff57025b8d1e11834909ee475c403bfa60cb4caefaf"}, + {file = "grpcio-1.81.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8ea1936c26b99999b27479853039a7f34713f56c49375ad52b38535ec93a796c"}, + {file = "grpcio-1.81.1-cp312-cp312-win32.whl", hash = "sha256:a185a04039df6cae8648bc8ab6d6fde7bf94f7188ecf7828e76ac52eef1e41d6"}, + {file = "grpcio-1.81.1-cp312-cp312-win_amd64.whl", hash = "sha256:3ad74f8bb1a18963914c5452d289422830b39459e8776ebbcd207be1fbfb1d94"}, + {file = "grpcio-1.81.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:b10e1ff4756ed27d5a29d7fc79cfce7ef1ff56ad20025b89bac7cf79e09abbbe"}, + {file = "grpcio-1.81.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:819edbdcb42ab8598b494bcf0222684bbb7a3c772bd1b1f0be7e029a6063c28e"}, + {file = "grpcio-1.81.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5bf2dc311127d91230cc79b92188c082634a06cf66c5234db49a43b910183b0"}, + {file = "grpcio-1.81.1-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:e8ca6a1fcdb2943c9cbc1804a1baf3acb6071d72a471591678ded84218006e14"}, + {file = "grpcio-1.81.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e64dd101d380a115cc5a0c7856788adb535f1a4e21fc543775602f8be95180ae"}, + {file = "grpcio-1.81.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:98a07f9bf591e3a8919797bee1c53f026ba4acd587e5a4404c8e57c9ec36b2a5"}, + {file = "grpcio-1.81.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c261d74b1a945cf895a9d6eccd1685a8e837531beaab782da4d630a8d12deffb"}, + {file = "grpcio-1.81.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:58ad1131c300d3c9b933802b3cc4dc69d380822935ba50b28703156ea826fbf7"}, + {file = "grpcio-1.81.1-cp313-cp313-win32.whl", hash = "sha256:78e29211f26da2fdd0e9c6d2b79f489476140cf7029b6a64808ade7ca4156a42"}, + {file = "grpcio-1.81.1-cp313-cp313-win_amd64.whl", hash = "sha256:edb59506291b647a30884b1d51a599d605f40b20af4a7dc3d33786a47a31de60"}, + {file = "grpcio-1.81.1-cp314-cp314-linux_armv7l.whl", hash = "sha256:506f48f2f9c29b143fca3dad7b0d518c188b6c9648c75a2ae6e2d9f2c13a060b"}, + {file = "grpcio-1.81.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d865db4a6318e1c1bea83292e0ed231090538fc4ca45425b0f0480eb338bbc6e"}, + {file = "grpcio-1.81.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e2aa72e3ce1770317ef534f63d397b55e130725f5149bd36077c3b539019db27"}, + {file = "grpcio-1.81.1-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:0490c30c261eded63f3f354979f9dc4502a9fb944cccb60cd9dc85f5a7349854"}, + {file = "grpcio-1.81.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:410482da976329fe5f4067270401b12cf2bd552ff8020f054ecfaddb5475f9d6"}, + {file = "grpcio-1.81.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3657301562ac3cb8018d30d0d3ebfa39932239f7b5703422057ef14b69949f5"}, + {file = "grpcio-1.81.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:24c8e57504c8f45b237e40b99262d181071e5099a07053695b75d97bb53053a0"}, + {file = "grpcio-1.81.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b427c19380991a4eaab2f6144b64b99b412043314c6bf4ab544f97bb31ee4190"}, + {file = "grpcio-1.81.1-cp314-cp314-win32.whl", hash = "sha256:61233fe8951e5c85dff81c2458b6528624760166946b5b47ea150a589168411f"}, + {file = "grpcio-1.81.1-cp314-cp314-win_amd64.whl", hash = "sha256:3768a5ff1b2125e6f552e561b6b2dca0e64982d8949689b4df145cf8b98d7821"}, + {file = "grpcio-1.81.1.tar.gz", hash = "sha256:6fa10a767143a5e82e8eaab53918af0cd8909a57a27f8cb2288b80a613ac671b"}, +] + +[package.dependencies] +typing-extensions = ">=4.12,<5.0" + +[package.extras] +protobuf = ["grpcio-tools (>=1.81.1)"] + [[package]] name = "h11" version = "0.16.0" @@ -1913,6 +1983,7 @@ files = [ {file = "jiter-0.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:1b28302349dc65703a9e4ead16f163b1c339efffbe1049c30a44b001a2a4fff9"}, {file = "jiter-0.10.0.tar.gz", hash = "sha256:07a7142c38aacc85194391108dc91b5b57093c978a9932bd86a36862759d9500"}, ] +markers = {main = "(extra == \"crewai\" or extra == \"all\" or extra == \"openai\") and python_version <= \"3.13\" or extra == \"openai\" or extra == \"all\""} [[package]] name = "json-repair" @@ -1954,6 +2025,7 @@ files = [ {file = "jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade"}, {file = "jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c"}, ] +markers = {main = "extra == \"langchain\" or extra == \"all\""} [package.dependencies] jsonpointer = ">=1.9" @@ -1969,6 +2041,7 @@ files = [ {file = "jsonpointer-3.0.0-py2.py3-none-any.whl", hash = "sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942"}, {file = "jsonpointer-3.0.0.tar.gz", hash = "sha256:2b2d729f2091522d61c3b31f82e11870f60b68f43fbc705cb76bf4b832af59ef"}, ] +markers = {main = "extra == \"langchain\" or extra == \"all\""} [[package]] name = "jsonref" @@ -1998,7 +2071,7 @@ files = [ [package.dependencies] attrs = ">=22.2.0" -jsonschema-specifications = ">=2023.03.6" +jsonschema-specifications = ">=2023.3.6" referencing = ">=0.28.4" rpds-py = ">=0.7.1" @@ -2036,7 +2109,7 @@ files = [ ] [package.dependencies] -certifi = ">=14.05.14" +certifi = ">=14.5.14" durationpy = ">=0.7" google-auth = ">=1.0.1" oauthlib = ">=3.2.2" @@ -2098,6 +2171,7 @@ files = [ {file = "langchain_core-1.2.7-py3-none-any.whl", hash = "sha256:452f4fef7a3d883357b22600788d37e3d8854ef29da345b7ac7099f33c31828b"}, {file = "langchain_core-1.2.7.tar.gz", hash = "sha256:e1460639f96c352b4a41c375f25aeb8d16ffc1769499fb1c20503aad59305ced"}, ] +markers = {main = "extra == \"langchain\" or extra == \"all\""} [package.dependencies] jsonpatch = ">=1.33.0,<2.0.0" @@ -2192,6 +2266,7 @@ files = [ {file = "langsmith-0.4.14-py3-none-any.whl", hash = "sha256:b6d070ac425196947d2a98126fb0e35f3b8c001a2e6e5b7049dd1c56f0767d0b"}, {file = "langsmith-0.4.14.tar.gz", hash = "sha256:4d29c7a9c85b20ba813ab9c855407bccdf5eb4f397f512ffa89959b2a2cb83ed"}, ] +markers = {main = "extra == \"langchain\" or extra == \"all\""} [package.dependencies] httpx = ">=0.23.0,<1" @@ -2893,6 +2968,7 @@ files = [ {file = "openai-2.32.0-py3-none-any.whl", hash = "sha256:4dcc9badeb4bf54ad0d187453742f290226d30150890b7890711bda4f32f192f"}, {file = "openai-2.32.0.tar.gz", hash = "sha256:c54b27a9e4cb8d51f0dd94972ffd1a04437efeb259a9e60d8922b8bd26fe55e0"}, ] +markers = {main = "(extra == \"crewai\" or extra == \"all\" or extra == \"openai\") and python_version <= \"3.13\" or extra == \"openai\" or extra == \"all\""} [package.dependencies] anyio = ">=3.5.0,<5" @@ -3236,7 +3312,7 @@ files = [ {file = "orjson-3.11.2-cp39-cp39-win_amd64.whl", hash = "sha256:c9ec0cc0d4308cad1e38a1ee23b64567e2ff364c2a3fe3d6cbc69cf911c45712"}, {file = "orjson-3.11.2.tar.gz", hash = "sha256:91bdcf5e69a8fd8e8bdb3de32b31ff01d2bd60c1e8d5fe7d5afabdcf19920309"}, ] -markers = {main = "platform_python_implementation != \"PyPy\" or extra == \"langchain\" or extra == \"all\" or (extra == \"langchain\" or extra == \"all\" or extra == \"crewai\") and python_version <= \"3.13\"", test = "platform_python_implementation != \"PyPy\""} +markers = {main = "extra == \"langchain\" or extra == \"all\" or (extra == \"crewai\" or extra == \"all\" or extra == \"langchain\") and python_version <= \"3.13\"", test = "platform_python_implementation != \"PyPy\""} [[package]] name = "ormsgpack" @@ -3321,6 +3397,7 @@ files = [ {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, ] +markers = {main = "python_version < \"3.13\" and (extra == \"crewai\" or extra == \"all\" or extra == \"langchain\" or extra == \"openai\") or extra == \"langchain\" or extra == \"all\" or extra == \"openai\" or python_version <= \"3.13\" and (extra == \"langchain\" or extra == \"all\" or extra == \"openai\" or extra == \"crewai\")"} [[package]] name = "pathspec" @@ -3986,7 +4063,7 @@ files = [ {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, ] -markers = {main = "(extra == \"openai\" or extra == \"all\" or extra == \"crewai\") and python_version <= \"3.13\" or platform_python_implementation == \"PyPy\" or extra == \"openai\" or extra == \"all\"", test = "platform_python_implementation == \"PyPy\""} +markers = {main = "(python_version <= \"3.13\" or extra == \"langchain\" or extra == \"all\" or extra == \"openai\") and (platform_python_implementation != \"PyPy\" or extra == \"langchain\" or extra == \"all\") and (python_version <= \"3.13\" or platform_python_implementation == \"PyPy\" or extra == \"openai\" or extra == \"all\") and (extra == \"openai\" or extra == \"all\" or extra == \"crewai\" or extra == \"langchain\") and (extra == \"openai\" or extra == \"all\" or extra == \"crewai\" or platform_python_implementation == \"PyPy\")", test = "platform_python_implementation == \"PyPy\""} [[package]] name = "pydantic" @@ -4576,6 +4653,7 @@ files = [ {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, ] +markers = {main = "(extra == \"crewai\" or extra == \"all\" or extra == \"langchain\") and python_version <= \"3.13\" or extra == \"langchain\" or extra == \"all\""} [[package]] name = "referencing" @@ -4704,6 +4782,7 @@ files = [ {file = "requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c"}, {file = "requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422"}, ] +markers = {main = "python_version < \"3.13\" and (extra == \"crewai\" or extra == \"all\" or extra == \"langchain\" or extra == \"openai\" or extra == \"otel\") or extra == \"langchain\" or extra == \"all\" or extra == \"openai\" or extra == \"otel\" or python_version <= \"3.13\" and (extra == \"langchain\" or extra == \"all\" or extra == \"openai\" or extra == \"otel\" or extra == \"crewai\")"} [package.dependencies] certifi = ">=2017.4.17" @@ -4764,6 +4843,7 @@ files = [ {file = "requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6"}, {file = "requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"}, ] +markers = {main = "extra == \"langchain\" or extra == \"all\""} [package.dependencies] requests = ">=2.0.1,<3.0.0" @@ -5197,6 +5277,7 @@ files = [ {file = "tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138"}, {file = "tenacity-9.1.2.tar.gz", hash = "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb"}, ] +markers = {main = "(extra == \"crewai\" or extra == \"all\" or extra == \"langchain\") and python_version <= \"3.13\" or extra == \"langchain\" or extra == \"all\""} [package.extras] doc = ["reno", "sphinx"] @@ -5593,11 +5674,11 @@ description = "HTTP library with thread-safe connection pooling, file post, and optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" groups = ["main", "test"] -markers = "platform_python_implementation == \"PyPy\"" files = [ {file = "urllib3-1.26.20-py2.py3-none-any.whl", hash = "sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e"}, {file = "urllib3-1.26.20.tar.gz", hash = "sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32"}, ] +markers = {main = "platform_python_implementation == \"PyPy\" and (extra == \"langchain\" or extra == \"all\" or extra == \"openai\" or extra == \"otel\") or platform_python_implementation == \"PyPy\" and (extra == \"crewai\" or extra == \"all\" or extra == \"langchain\" or extra == \"openai\" or extra == \"otel\") and python_version <= \"3.13\"", test = "platform_python_implementation == \"PyPy\""} [package.extras] brotli = ["brotli (==1.0.9) ; os_name != \"nt\" and python_version < \"3\" and platform_python_implementation == \"CPython\"", "brotli (>=1.0.9) ; python_version >= \"3\" and platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; (os_name != \"nt\" or python_version >= \"3\") and platform_python_implementation != \"CPython\"", "brotlipy (>=0.6.0) ; os_name == \"nt\" and python_version < \"3\""] @@ -5611,11 +5692,11 @@ description = "HTTP library with thread-safe connection pooling, file post, and optional = false python-versions = ">=3.9" groups = ["main", "test"] -markers = "platform_python_implementation != \"PyPy\"" files = [ {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, ] +markers = {main = "platform_python_implementation != \"PyPy\" and (extra == \"langchain\" or extra == \"all\" or extra == \"openai\" or extra == \"otel\") or platform_python_implementation != \"PyPy\" and (extra == \"crewai\" or extra == \"all\" or extra == \"langchain\" or extra == \"openai\" or extra == \"otel\") and python_version <= \"3.13\"", test = "platform_python_implementation != \"PyPy\""} [package.extras] brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] @@ -5654,6 +5735,7 @@ files = [ {file = "uuid_utils-0.13.0-pp311-pypy311_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:b7ccaa20e24c5f60f41a69ef571ed820737f9b0ade4cbeef56aaa8f80f5aa475"}, {file = "uuid_utils-0.13.0.tar.gz", hash = "sha256:4c17df6427a9e23a4cd7fb9ee1efb53b8abb078660b9bdb2524ca8595022dfe1"}, ] +markers = {main = "extra == \"langchain\" or extra == \"all\""} [[package]] name = "uv" @@ -5781,8 +5863,8 @@ files = [ [package.dependencies] PyYAML = "*" urllib3 = [ - {version = "<2", markers = "platform_python_implementation == \"PyPy\""}, {version = "*", markers = "platform_python_implementation != \"PyPy\" and python_version >= \"3.10\""}, + {version = "<2", markers = "platform_python_implementation == \"PyPy\""}, ] wrapt = "*" yarl = "*" @@ -6552,6 +6634,7 @@ files = [ {file = "zstandard-0.23.0-cp39-cp39-win_amd64.whl", hash = "sha256:f8346bfa098532bc1fb6c7ef06783e969d87a99dd1d2a5a18a892c1d7a643c58"}, {file = "zstandard-0.23.0.tar.gz", hash = "sha256:b2d8c62d08e7255f68f7a740bae85b3c9b8e5466baa9cbf7f57f1cde0ac6bc09"}, ] +markers = {main = "extra == \"langchain\" or extra == \"all\""} [package.dependencies] cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} @@ -6560,14 +6643,14 @@ cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\ cffi = ["cffi (>=1.11)"] [extras] -all = ["crewai", "langchain", "langchain-core", "litellm", "openai", "openai-agents", "opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-sdk", "packaging", "starlette"] +all = ["crewai", "grpcio", "langchain", "langchain-core", "litellm", "openai", "openai-agents", "opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-sdk", "packaging", "starlette"] crewai = ["crewai", "litellm"] langchain = ["langchain", "langchain-core"] middleware = ["starlette"] openai = ["openai", "openai-agents", "packaging"] -otel = ["opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-sdk"] +otel = ["grpcio", "opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-sdk"] [metadata] lock-version = "2.1" python-versions = "^3.10,<3.15" -content-hash = "e63f12d6124d0c2e9cda7935d20551745a78b68a4de05c872f45a54ede6824f7" +content-hash = "1546a6ef1f5e5adcccb5539279c40cfdd09b2e201412bc49c9dba94295631cd1" diff --git a/pyproject.toml b/pyproject.toml index 708956d7..779fceff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,8 +16,8 @@ langchain = ["langchain-core", "langchain"] openai = ["openai (>=2.8.0,<3.0.0)", "packaging (>=24.2,<25.0)", "openai-agents (>=0.4.0,<1.0.0)"] crewai = ["crewai (>=0.152.0,<2.0.0); python_version < '3.14'", "litellm (>=1.83.14,<2.0.0); python_version < '3.14'"] middleware = ["starlette"] -otel = ["opentelemetry-sdk (>=1.38.0,<2.0.0)", "opentelemetry-api (>=1.38.0,<2.0.0)", "opentelemetry-exporter-otlp (>=1.38.0,<2.0.0)"] -all = ["langchain-core", "langchain", "openai (>=2.8.0,<3.0.0)", "packaging (>=24.2,<25.0)", "openai-agents (>=0.4.0,<1.0.0)", "opentelemetry-sdk (>=1.38.0,<2.0.0)", "opentelemetry-api (>=1.38.0,<2.0.0)", "opentelemetry-exporter-otlp (>=1.38.0,<2.0.0)", "crewai (>=0.152.0,<2.0.0); python_version < '3.14'", "starlette", "litellm (>=1.83.14,<2.0.0); python_version < '3.14'"] +otel = ["opentelemetry-sdk (>=1.38.0,<2.0.0)", "opentelemetry-api (>=1.38.0,<2.0.0)", "opentelemetry-exporter-otlp (>=1.38.0,<2.0.0)", "grpcio (>=1.80.0,<2.0.0)"] +all = ["langchain-core", "langchain", "openai (>=2.8.0,<3.0.0)", "packaging (>=24.2,<25.0)", "openai-agents (>=0.4.0,<1.0.0)", "opentelemetry-sdk (>=1.38.0,<2.0.0)", "opentelemetry-api (>=1.38.0,<2.0.0)", "opentelemetry-exporter-otlp (>=1.38.0,<2.0.0)", "grpcio (>=1.80.0,<2.0.0)", "crewai (>=0.152.0,<2.0.0); python_version < '3.14'", "starlette", "litellm (>=1.83.14,<2.0.0); python_version < '3.14'"] @@ -42,6 +42,10 @@ typing-extensions = { version = ">=4.5.0" } opentelemetry-sdk = { version = "^1.38.0", optional = true } opentelemetry-api = { version = "^1.38.0", optional = true } opentelemetry-exporter-otlp = { version = "^1.38.0", optional = true } +# Explicit lower bound ensures pre-built cp314 wheels are available (1.80.0+). +# Without this, resolvers could pick grpcio<1.80.0 which has no cp314 wheels, +# forcing source compilation (~20 min) on Python 3.14 CI runners. +grpcio = { version = ">=1.80.0,<2.0.0", optional = true } [tool.poetry.group.test.dependencies] pytest = "^8.4.0" From c260afe3e58bcc1d92c54a8977ccd83c543c5856 Mon Sep 17 00:00:00 2001 From: Fernando Correia Date: Fri, 12 Jun 2026 12:42:50 -0700 Subject: [PATCH 09/12] ci: [DO NOT MERGE] Windows test-duration instrumentation probe Investigation branch (not for merge). Trims the CI matrix to windows-latest x [3.10, 3.11] and replaces the test steps with layered, timestamped diagnostics to localize the ~1s/test silent overhead seen on Python 3.11+ Windows but not 3.10: - env + `poetry show` dump (catch a pure-Python wheel fallback / dep diff) - microbenchmarks: getaddrinfo("localtest"), asyncio loop churn, config init - subset isolation variants toggling one factor each: xdist, coverage, and the pytest-socket plugin (decisive for the getaddrinfo theory) - serial subset under cProfile with DEBUG logs + ResourceWarnings - full instrumented suite with a conftest phase-timing plugin that attributes wall time to setup/call/teardown and prints live timestamped [PHASE] lines timeout-minutes raised 30 -> 45 so the full run isn't truncated. Co-Authored-By: Claude Opus 4.8 (1M context) --- .ci/probe_microbench.py | 91 +++++++++++++++++++++++++++ .github/workflows/ci-tests.yaml | 105 +++++++++++++++++++++++++++++--- tests/conftest.py | 45 ++++++++++++++ 3 files changed, 231 insertions(+), 10 deletions(-) create mode 100644 .ci/probe_microbench.py diff --git a/.ci/probe_microbench.py b/.ci/probe_microbench.py new file mode 100644 index 00000000..eeb309ce --- /dev/null +++ b/.ci/probe_microbench.py @@ -0,0 +1,91 @@ +"""Windows test-duration probe microbenchmarks (investigation branch). + +Times, in isolation (outside pytest), the operations most likely to explain the +~1s-per-test silent overhead seen on Python 3.11+ Windows but not 3.10: + + 1. socket.getaddrinfo() for the bogus host in GALILEO_CONSOLE_URL ("localtest") + 2. asyncio event-loop create/close churn (Windows ProactorEventLoop cost) + 3. GalileoPythonConfig.get() — replicates the autouse `set_validated_config` + fixture that runs on EVERY test. + +Everything is timestamped and flushed so output can be correlated with the rest +of the CI log. +""" + +import asyncio +import contextlib +import datetime +import os +import socket +import sys +import time +from collections.abc import Callable + + +def _ts() -> str: + return datetime.datetime.now().strftime("%H:%M:%S.%f")[:-3] + + +def log(msg: str) -> None: + # Write straight to stdout (not print()) so ruff's T201 autofix can't strip it. + sys.stdout.write(f"[BENCH {_ts()}] {msg}\n") + sys.stdout.flush() + + +def bench(label: str, fn: Callable[[], object], n: int = 5) -> list[float]: + samples = [] + last_exc = None + for _ in range(n): + t = time.perf_counter() + try: + fn() + except Exception as e: + last_exc = e + samples.append(time.perf_counter() - t) + summary = ", ".join(f"{x * 1000:.1f}ms" for x in samples) + total = sum(samples) * 1000 + avg = total / len(samples) + log(f"{label}: avg={avg:.1f}ms total={total:.1f}ms exc={type(last_exc).__name__ if last_exc else None}") + log(f" samples=[{summary}]") + return samples + + +log(f"python {sys.version}") +log(f"platform {sys.platform}") +log(f"asyncio policy {type(asyncio.get_event_loop_policy()).__name__}") + +# 1) Name resolution — the prime suspect. "localtest" is intentionally bogus. +log("--- getaddrinfo ---") +for host in ("localtest", "localhost", "127.0.0.1"): + bench(f"getaddrinfo({host!r}, 8088)", lambda host=host: socket.getaddrinfo(host, 8088), n=5) + +# 2) asyncio event-loop churn. +log("--- asyncio loop churn ---") + + +def _loop_cycle() -> None: + loop = asyncio.new_event_loop() + loop.close() + + +bench("asyncio new+close", _loop_cycle, n=50) + +# 3) GalileoPythonConfig.get — the per-test autouse fixture, with a REAL network +# attempt (no pytest mocks here). If this is ~1s and dominated by getaddrinfo, +# that's the per-test cost. +log("--- GalileoPythonConfig.get ---") +os.environ.setdefault("GALILEO_CONSOLE_URL", "http://localtest:8088") +os.environ.setdefault("GALILEO_API_KEY", "api-1234567890") +try: + from galileo.config import GalileoPythonConfig + + def _config_get() -> None: + cfg = GalileoPythonConfig.get(console_url="http://localtest:8088", api_key="api-1234567890") + with contextlib.suppress(Exception): + cfg.reset() + + bench("GalileoPythonConfig.get+reset", _config_get, n=5) +except Exception as e: + log(f"config import/get failed: {type(e).__name__}: {e}") + +log("done") diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index 376db89a..c939fb43 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -21,13 +21,15 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + # INVESTIGATION BRANCH: trimmed to the two cells that bracket the + # Windows slowdown — 3.10 (fast: ~6 min) vs 3.11 (slow: ~28 min). + os: [windows-latest] + python-version: ["3.10", "3.11"] runs-on: ${{ matrix.os }} - # Hard cap per matrix job — bail out fast on real hangs instead of - # burning CI minutes up to the GitHub-default 6h ceiling. - timeout-minutes: 30 + # Raised from 30 -> 45 so the instrumented full run (~23 min on 3.11) plus + # the extra diagnostic steps don't get truncated by the timeout. + timeout-minutes: 45 steps: - name: Checkout @@ -53,7 +55,26 @@ jobs: - name: Install Dependencies run: invoke install - - name: Verify Poetry Python version + # All diagnostic steps below run in Git-bash (present on Windows runners) + # so heredocs, pipes, and `head` behave consistently regardless of pwsh. + # Ordered cheap -> expensive: the fast probes land their data even if the + # full instrumented run later hits the timeout. + + - name: "[probe] Environment + dependency dump" + if: always() + shell: bash + run: | + echo "::group::interpreter" + poetry run python -VV + poetry run python -c "import sys, asyncio, platform; print('platform:', platform.platform()); print('loop_policy:', type(asyncio.get_event_loop_policy()).__name__)" + echo "::endgroup::" + echo "::group::poetry show" + poetry show + echo "::endgroup::" + + - name: "[probe] Verify Poetry Python version" + if: always() + shell: bash run: | poetry run python -c " import sys @@ -63,10 +84,74 @@ jobs: assert actual == expected, f'Expected Python {expected}, got {actual}' " - - name: Validate Types + - name: "[probe] Microbenchmarks (getaddrinfo / asyncio / config)" if: always() - run: invoke type-check + shell: bash + run: poetry run python .ci/probe_microbench.py + + # Isolation variants on a tiny, all-trivial file. addopts is cleared with + # `-o addopts=` and rebuilt explicitly so each variant toggles exactly ONE + # factor vs the baseline. Compare per-test durations across 3.10 vs 3.11. + - name: "[probe] Subset: baseline (xdist + cov + socket-disabled)" + if: always() + shell: bash + run: | + echo "[probe $(date -u +%H:%M:%S)] baseline" + poetry run pytest tests/test_configuration.py -o addopts= \ + -n auto --disable-socket --allow-hosts=127.0.0.1,localhost --cov=galileo \ + -p no:cacheprovider --durations=0 -q + + - name: "[probe] Subset: no-xdist (serial)" + if: always() + shell: bash + run: | + echo "[probe $(date -u +%H:%M:%S)] no-xdist" + poetry run pytest tests/test_configuration.py -o addopts= \ + -p no:xdist --disable-socket --allow-hosts=127.0.0.1,localhost --cov=galileo \ + -p no:cacheprovider --durations=0 -q - - name: Run Tests + - name: "[probe] Subset: no-cov" if: always() - run: invoke test-report-xml + shell: bash + run: | + echo "[probe $(date -u +%H:%M:%S)] no-cov" + poetry run pytest tests/test_configuration.py -o addopts= \ + -n auto --disable-socket --allow-hosts=127.0.0.1,localhost \ + -p no:cacheprovider --durations=0 -q + + - name: "[probe] Subset: no-socket-plugin (sockets enabled) — decisive for getaddrinfo theory" + if: always() + shell: bash + run: | + echo "[probe $(date -u +%H:%M:%S)] no-socket-plugin" + poetry run pytest tests/test_configuration.py -o addopts= \ + -n auto --cov=galileo \ + -p no:cacheprovider --durations=0 -q + + - name: "[probe] Subset: serial + DEBUG logs + ResourceWarnings (cProfile)" + if: always() + shell: bash + run: | + echo "[probe $(date -u +%H:%M:%S)] cProfile serial run" + poetry run python -m cProfile -o profile.out -m pytest tests/test_configuration.py -o addopts= \ + --disable-socket --allow-hosts=127.0.0.1,localhost \ + -p no:cacheprovider -o log_cli=true --log-cli-level=DEBUG -W default -q 2>&1 | head -300 + echo "::group::cumulative profile (top 45 by cumtime)" + poetry run python -c "import pstats; p=pstats.Stats('profile.out'); p.sort_stats('cumulative'); p.print_stats(45)" + echo "::endgroup::" + echo "::group::profile (top 30 by total/internal time)" + poetry run python -c "import pstats; p=pstats.Stats('profile.out'); p.sort_stats('tottime'); p.print_stats(30)" + echo "::endgroup::" + + # Realistic full run last. The appended conftest timing plugin prints live + # [PHASE ...] lines and an end-of-run setup/call/teardown aggregate. + - name: "[probe] Full suite (all durations + phase aggregate)" + if: always() + shell: bash + run: | + echo "[probe $(date -u +%H:%M:%S)] full suite start" + # Keeps ini addopts (-n auto, --disable-socket, --timeout=120, ...) and + # adds --cov to mirror the real (slow) CI path; --durations=0 wins over + # the ini --durations=10. + poetry run pytest tests --cov=galileo --durations=0 -ra -q + echo "[probe $(date -u +%H:%M:%S)] full suite end" diff --git a/tests/conftest.py b/tests/conftest.py index 9e92abee..e4bd00c6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -553,3 +553,48 @@ def mock_collaborator() -> MagicMock: mock_collab.last_name = "Collaborator" mock_collab.permissions = [] return mock_collab + + +# --------------------------------------------------------------------------- +# TIMING INSTRUMENTATION (investigation branch ci/windows-timing-probe). +# Localizes the ~1s/test silent overhead on Windows by attributing wall time to +# setup / call / teardown phases, with live timestamped lines for slow phases +# and a per-phase aggregate at the end. xdist-safe: pytest replays worker +# reports through pytest_runtest_logreport on the controller. Remove before any +# merge — this is diagnostic only. +# --------------------------------------------------------------------------- +_PHASE_DURATIONS: dict[str, list[tuple[float, str]]] = {"setup": [], "call": [], "teardown": []} + + +def _probe_ts() -> str: + return datetime.datetime.now().strftime("%H:%M:%S.%f")[:-3] + + +def pytest_runtest_logreport(report) -> None: + when = getattr(report, "when", None) + if when not in _PHASE_DURATIONS: + return + dur = getattr(report, "duration", 0.0) or 0.0 + _PHASE_DURATIONS[when].append((dur, report.nodeid)) + # Live line for anything slow, timestamped so we can see WHEN in the run the + # cost accrues and correlate with anything else interleaved in the log. + if dur >= 0.25: + print(f"[PHASE {_probe_ts()}] {when:8s} {dur:7.3f}s {report.nodeid}", flush=True) + + +def pytest_terminal_summary(terminalreporter, exitstatus, config) -> None: + print(f"\n[PHASE {_probe_ts()}] ==================== TIMING SUMMARY ====================", flush=True) + grand = 0.0 + for phase in ("setup", "call", "teardown"): + durs = _PHASE_DURATIONS[phase] + total = sum(d for d, _ in durs) + grand += total + n = len(durs) + avg = (total / n * 1000) if n else 0.0 + print(f" {phase:8s} total={total:9.1f}s count={n:5d} avg={avg:8.2f}ms", flush=True) + print(f" {'GRAND':8s} total={grand:9.1f}s", flush=True) + for phase in ("setup", "call", "teardown"): + print(f" --- slowest {phase} phases ---", flush=True) + for dur, nodeid in sorted(_PHASE_DURATIONS[phase], reverse=True)[:10]: + print(f" {dur:7.3f}s {nodeid}", flush=True) + print(" ========================================================", flush=True) From 4338d179f7951e499a7343c3d5d13afcf615de0d Mon Sep 17 00:00:00 2001 From: Fernando Correia Date: Fri, 12 Jun 2026 14:50:23 -0700 Subject: [PATCH 10/12] =?UTF-8?q?ci:=20[probe]=20CPython=20verification=20?= =?UTF-8?q?=E2=80=94=20Proactor=20vs=20Selector=20loop=20A/B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the (thread-blind) cProfile step and the 24-min full suite with a causal experiment, since cProfile only profiles the calling thread while the async validation work runs in galileo_core's background EventLoopThread: - probe_microbench.py now measures cross-thread dispatch latency (run_coroutine_threadsafe round-trip) — the exact shape of async_run — plus timer-granularity probes, and honors PROBE_EVENT_LOOP=selector. - conftest forces WindowsSelectorEventLoopPolicy when PROBE_EVENT_LOOP=selector. - Workflow runs microbench + the trivial subset twice (Proactor vs Selector); compare the per-test "setup avg" in each TIMING SUMMARY. Co-Authored-By: Claude Opus 4.8 (1M context) --- .ci/probe_microbench.py | 95 +++++++++++++++++++++++++++------ .github/workflows/ci-tests.yaml | 74 ++++++++----------------- tests/conftest.py | 9 ++++ 3 files changed, 110 insertions(+), 68 deletions(-) diff --git a/.ci/probe_microbench.py b/.ci/probe_microbench.py index eeb309ce..356856ff 100644 --- a/.ci/probe_microbench.py +++ b/.ci/probe_microbench.py @@ -1,15 +1,17 @@ """Windows test-duration probe microbenchmarks (investigation branch). -Times, in isolation (outside pytest), the operations most likely to explain the -~1s-per-test silent overhead seen on Python 3.11+ Windows but not 3.10: +Times, in isolation (outside pytest), the operations involved in per-test config +validation, to explain the ~19x slower fixture *setup* on Python 3.11 Windows. 1. socket.getaddrinfo() for the bogus host in GALILEO_CONSOLE_URL ("localtest") - 2. asyncio event-loop create/close churn (Windows ProactorEventLoop cost) - 3. GalileoPythonConfig.get() — replicates the autouse `set_validated_config` - fixture that runs on EVERY test. - -Everything is timestamped and flushed so output can be correlated with the rest -of the CI log. + 2. asyncio event-loop create/close churn + 3. cross-thread dispatch latency: run_coroutine_threadsafe round-trip onto a + background run_forever loop — this is exactly what galileo_core's async_run / + EventLoopThreadPool does for every validation request. THE key measurement. + 4. GalileoPythonConfig.get() — the per-test autouse fixture (real, unmocked). + +Set PROBE_EVENT_LOOP=selector to force the WindowsSelectorEventLoopPolicy so the +default Proactor loop can be A/B'd against it. Everything is timestamped/flushed. """ import asyncio @@ -18,9 +20,25 @@ import os import socket import sys +import threading import time from collections.abc import Callable +# Force the selector loop BEFORE any asyncio object is created, if requested. +_FORCED = "default(Proactor on win32)" +if ( + os.environ.get("PROBE_EVENT_LOOP") == "selector" + and sys.platform == "win32" + and hasattr(asyncio, "WindowsSelectorEventLoopPolicy") +): + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + _FORCED = "forced WindowsSelectorEventLoopPolicy" + +# Make `galileo` importable (installed with --no-root; pytest uses pythonpath=src). +_src = os.path.join(os.getcwd(), "src") +if os.path.isdir(_src): + sys.path.insert(0, _src) + def _ts() -> str: return datetime.datetime.now().strftime("%H:%M:%S.%f")[:-3] @@ -52,14 +70,14 @@ def bench(label: str, fn: Callable[[], object], n: int = 5) -> list[float]: log(f"python {sys.version}") log(f"platform {sys.platform}") -log(f"asyncio policy {type(asyncio.get_event_loop_policy()).__name__}") +log(f"event loop policy: {type(asyncio.get_event_loop_policy()).__name__} ({_FORCED})") # 1) Name resolution — the prime suspect. "localtest" is intentionally bogus. log("--- getaddrinfo ---") for host in ("localtest", "localhost", "127.0.0.1"): bench(f"getaddrinfo({host!r}, 8088)", lambda host=host: socket.getaddrinfo(host, 8088), n=5) -# 2) asyncio event-loop churn. +# 2) asyncio event-loop churn (create + close). log("--- asyncio loop churn ---") @@ -70,10 +88,57 @@ def _loop_cycle() -> None: bench("asyncio new+close", _loop_cycle, n=50) -# 3) GalileoPythonConfig.get — the per-test autouse fixture, with a REAL network -# attempt (no pytest mocks here). If this is ~1s and dominated by getaddrinfo, -# that's the per-test cost. -log("--- GalileoPythonConfig.get ---") +# 3) Cross-thread dispatch latency. A background thread runs run_forever(); we +# submit coroutines from the main thread via run_coroutine_threadsafe and +# block on the result — the exact shape of galileo_core's async_run. This +# isolates the per-call wakeup cost of the event loop, which is what differs +# between the Proactor and Selector loops on Windows. +log("--- cross-thread dispatch (run_coroutine_threadsafe round-trip) ---") +_bg_loop = asyncio.new_event_loop() +log(f"background loop type: {type(_bg_loop).__name__}") +_bg_thread = threading.Thread(target=_bg_loop.run_forever, daemon=True) +_bg_thread.start() + + +async def _noop() -> int: + return 1 + + +def _dispatch_noop() -> None: + asyncio.run_coroutine_threadsafe(_noop(), _bg_loop).result() + + +bench("dispatch noop (pure wakeup, no I/O)", _dispatch_noop, n=50) + + +async def _yield_chain() -> None: + for _ in range(20): + await asyncio.sleep(0) + + +def _dispatch_yield() -> None: + asyncio.run_coroutine_threadsafe(_yield_chain(), _bg_loop).result() + + +bench("dispatch 20x await sleep(0) (ready-callback iterations)", _dispatch_yield, n=50) + + +async def _tiny_sleeps() -> None: + # 1ms requested x10. On Windows the ~15.6ms timer tick rounds each up. + for _ in range(10): + await asyncio.sleep(0.001) + + +def _dispatch_tiny() -> None: + asyncio.run_coroutine_threadsafe(_tiny_sleeps(), _bg_loop).result() + + +bench("dispatch 10x await sleep(0.001) (timer granularity)", _dispatch_tiny, n=20) + +_bg_loop.call_soon_threadsafe(_bg_loop.stop) + +# 4) GalileoPythonConfig.get — the per-test autouse fixture, REAL (no mocks). +log("--- GalileoPythonConfig.get (unmocked: hits real localtest resolution) ---") os.environ.setdefault("GALILEO_CONSOLE_URL", "http://localtest:8088") os.environ.setdefault("GALILEO_API_KEY", "api-1234567890") try: @@ -84,7 +149,7 @@ def _config_get() -> None: with contextlib.suppress(Exception): cfg.reset() - bench("GalileoPythonConfig.get+reset", _config_get, n=5) + bench("GalileoPythonConfig.get+reset", _config_get, n=2) except Exception as e: log(f"config import/get failed: {type(e).__name__}: {e}") diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index c939fb43..36b9ad5b 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -84,74 +84,42 @@ jobs: assert actual == expected, f'Expected Python {expected}, got {actual}' " - - name: "[probe] Microbenchmarks (getaddrinfo / asyncio / config)" - if: always() - shell: bash - run: poetry run python .ci/probe_microbench.py + # CPython verification: A/B the Windows default Proactor loop against the + # Selector loop, both at the raw-primitive level (microbench: cross-thread + # dispatch latency) and at the suite level (per-test setup avg via the + # conftest timing plugin). The full 24-min suite is intentionally dropped — + # this run is just the causal experiment. - # Isolation variants on a tiny, all-trivial file. addopts is cleared with - # `-o addopts=` and rebuilt explicitly so each variant toggles exactly ONE - # factor vs the baseline. Compare per-test durations across 3.10 vs 3.11. - - name: "[probe] Subset: baseline (xdist + cov + socket-disabled)" + - name: "[probe] Microbench — Proactor (default loop)" if: always() shell: bash - run: | - echo "[probe $(date -u +%H:%M:%S)] baseline" - poetry run pytest tests/test_configuration.py -o addopts= \ - -n auto --disable-socket --allow-hosts=127.0.0.1,localhost --cov=galileo \ - -p no:cacheprovider --durations=0 -q + run: poetry run python .ci/probe_microbench.py - - name: "[probe] Subset: no-xdist (serial)" + - name: "[probe] Microbench — Selector loop" if: always() shell: bash - run: | - echo "[probe $(date -u +%H:%M:%S)] no-xdist" - poetry run pytest tests/test_configuration.py -o addopts= \ - -p no:xdist --disable-socket --allow-hosts=127.0.0.1,localhost --cov=galileo \ - -p no:cacheprovider --durations=0 -q + env: + PROBE_EVENT_LOOP: selector + run: poetry run python .ci/probe_microbench.py - - name: "[probe] Subset: no-cov" + # Same trivial 41-test file, serial, only the event loop policy differs. + # Compare the "setup total/avg" line in each run's TIMING SUMMARY. + - name: "[probe] Subset A/B — Proactor (default loop)" if: always() shell: bash run: | - echo "[probe $(date -u +%H:%M:%S)] no-cov" + echo "[probe $(date -u +%H:%M:%S)] subset serial — Proactor (default)" poetry run pytest tests/test_configuration.py -o addopts= \ - -n auto --disable-socket --allow-hosts=127.0.0.1,localhost \ + -p no:xdist --disable-socket --allow-hosts=127.0.0.1,localhost \ -p no:cacheprovider --durations=0 -q - - name: "[probe] Subset: no-socket-plugin (sockets enabled) — decisive for getaddrinfo theory" + - name: "[probe] Subset A/B — Selector loop" if: always() shell: bash + env: + PROBE_EVENT_LOOP: selector run: | - echo "[probe $(date -u +%H:%M:%S)] no-socket-plugin" + echo "[probe $(date -u +%H:%M:%S)] subset serial — Selector" poetry run pytest tests/test_configuration.py -o addopts= \ - -n auto --cov=galileo \ + -p no:xdist --disable-socket --allow-hosts=127.0.0.1,localhost \ -p no:cacheprovider --durations=0 -q - - - name: "[probe] Subset: serial + DEBUG logs + ResourceWarnings (cProfile)" - if: always() - shell: bash - run: | - echo "[probe $(date -u +%H:%M:%S)] cProfile serial run" - poetry run python -m cProfile -o profile.out -m pytest tests/test_configuration.py -o addopts= \ - --disable-socket --allow-hosts=127.0.0.1,localhost \ - -p no:cacheprovider -o log_cli=true --log-cli-level=DEBUG -W default -q 2>&1 | head -300 - echo "::group::cumulative profile (top 45 by cumtime)" - poetry run python -c "import pstats; p=pstats.Stats('profile.out'); p.sort_stats('cumulative'); p.print_stats(45)" - echo "::endgroup::" - echo "::group::profile (top 30 by total/internal time)" - poetry run python -c "import pstats; p=pstats.Stats('profile.out'); p.sort_stats('tottime'); p.print_stats(30)" - echo "::endgroup::" - - # Realistic full run last. The appended conftest timing plugin prints live - # [PHASE ...] lines and an end-of-run setup/call/teardown aggregate. - - name: "[probe] Full suite (all durations + phase aggregate)" - if: always() - shell: bash - run: | - echo "[probe $(date -u +%H:%M:%S)] full suite start" - # Keeps ini addopts (-n auto, --disable-socket, --timeout=120, ...) and - # adds --cov to mirror the real (slow) CI path; --durations=0 wins over - # the ini --durations=10. - poetry run pytest tests --cov=galileo --durations=0 -ra -q - echo "[probe $(date -u +%H:%M:%S)] full suite end" diff --git a/tests/conftest.py b/tests/conftest.py index e4bd00c6..b6978d07 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,6 +27,15 @@ _os.environ["GALILEO_PROJECT"] = "test-project" _os.environ["GALILEO_LOG_STREAM"] = "test-log-stream" _os.environ["OPENAI_API_KEY"] = "sk-test" +# PROBE (investigation branch): optionally force the selector event loop so we can +# A/B it against the Windows default ProactorEventLoop. Must run before any asyncio +# object (incl. galileo_core's EventLoopThreadPool) is created. +if _os.environ.get("PROBE_EVENT_LOOP") == "selector": + import asyncio as _asyncio + import sys as _sys + + if _sys.platform == "win32" and hasattr(_asyncio, "WindowsSelectorEventLoopPolicy"): + _asyncio.set_event_loop_policy(_asyncio.WindowsSelectorEventLoopPolicy()) del _os # Clean up temporary import # fmt: on From f6fa9f6d556c306ab4dcd14797d26d8125e7a81f Mon Sep 17 00:00:00 2001 From: Fernando Correia Date: Fri, 12 Jun 2026 15:13:02 -0700 Subject: [PATCH 11/12] ci: [probe] add yappi thread-aware profile of mocked config.get cProfile only profiles the calling thread; the validation requests run on a background EventLoopThread. yappi (wall clock, all threads, builtins) attributes the ~12x-more timer-quantized waits on 3.11 Windows to a concrete function and call count. Reproduces the respx-mocked path (the real ~685ms test conditions). Co-Authored-By: Claude Opus 4.8 (1M context) --- .ci/probe_yappi.py | 101 ++++++++++++++++++++++++++++++++ .github/workflows/ci-tests.yaml | 10 ++++ 2 files changed, 111 insertions(+) create mode 100644 .ci/probe_yappi.py diff --git a/.ci/probe_yappi.py b/.ci/probe_yappi.py new file mode 100644 index 00000000..0bc7f2d7 --- /dev/null +++ b/.ci/probe_yappi.py @@ -0,0 +1,101 @@ +"""Thread-aware profile of the per-test config validation (investigation branch). + +cProfile only sees the calling thread, but galileo_core runs the 3 validation +requests on a background EventLoopThread — so we use yappi (wall-clock, all +threads, builtins) to attribute where the ~12x-more timer-quantized waits on +Python 3.11 Windows actually accrue. + +Reproduces the *mocked* path (respx), i.e. the real test conditions (~685 ms on +3.11), NOT the unmocked DNS path. Profiles N config.get()+reset() cycles. +""" + +import contextlib +import datetime +import os +import sys +from unittest.mock import patch +from uuid import uuid4 + +_src = os.path.join(os.getcwd(), "src") +if os.path.isdir(_src): + sys.path.insert(0, _src) + +os.environ.setdefault("GALILEO_CONSOLE_URL", "http://localtest:8088") +os.environ.setdefault("GALILEO_API_KEY", "api-1234567890") + + +def _ts() -> str: + return datetime.datetime.now().strftime("%H:%M:%S.%f")[:-3] + + +def log(msg: str) -> None: + sys.stdout.write(f"[YAPPI {_ts()}] {msg}\n") + sys.stdout.flush() + + +import respx # noqa: E402 +import yappi # noqa: E402 + +from galileo.config import GalileoPythonConfig # noqa: E402 + +_USER = {"id": str(uuid4()), "email": "user@example.com", "role": "user"} +_N = 10 +_ok = 0 +_last_exc = None + + +def _one_cycle() -> None: + global _ok, _last_exc + try: + cfg = GalileoPythonConfig.get(console_url="http://localtest:8088", api_key="api-1234567890") + with contextlib.suppress(Exception): + cfg.reset() + _ok += 1 + except Exception as e: + _last_exc = e + + +log(f"python {sys.version.split()[0]} platform {sys.platform}") + +with ( + patch("galileo_core.schemas.base_config.jwt_decode", return_value={"exp": float("inf")}), + respx.mock(assert_all_called=False) as router, +): + router.get(url__regex=r".*/healthcheck.*").respond(200, json={"status": "ok"}) + router.post(url__regex=r".*/login/api_key.*").respond(200, json={"access_token": "secret_jwt_token"}) + router.get(url__regex=r".*/current_user.*").respond(200, json=_USER) + + _one_cycle() # warmup: also spins up the (one-time) EventLoopThreadPool + log(f"warmup ok={_ok} exc={type(_last_exc).__name__ if _last_exc else None}") + + yappi.set_clock_type("wall") + yappi.start(builtins=True) + for _ in range(_N): + _one_cycle() + yappi.stop() + +log(f"profiled {_N} cycles, ok={_ok}/{_N + 1}, last_exc={type(_last_exc).__name__ if _last_exc else None}") + +# Per-thread wall time (which thread holds the cost). +log("================ THREAD STATS ================") +yappi.get_thread_stats().print_all() + +# Top functions by total wall time across ALL threads. ncall reveals how many +# times each is hit per run — the 3.10 vs 3.11 delta should show as ncall. +log("================ TOP 50 FUNCTIONS BY ttot (all threads, builtins) ================") +stats = yappi.get_func_stats() +stats.sort("ttot", "desc") +for i, s in enumerate(stats): + if i >= 50: + break + sys.stdout.write(f" ttot={s.ttot * 1000:9.1f}ms tsub={s.tsub * 1000:9.1f}ms ncall={s.ncall:>8} {s.full_name}\n") +sys.stdout.flush() + +# Explicitly surface the usual Windows-wait suspects regardless of rank. +log("================ WAIT/SLEEP/POLL SUSPECTS ================") +_needles = ("sleep", "select", "GetQueuedCompletionStatus", "_run_once", "getaddrinfo", "poll", "wait", "Overlapped") +for s in stats: + if any(n.lower() in s.full_name.lower() for n in _needles): + sys.stdout.write(f" ttot={s.ttot * 1000:9.1f}ms ncall={s.ncall:>8} avg={s.tavg * 1000:7.3f}ms {s.full_name}\n") +sys.stdout.flush() +log("done") diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index 36b9ad5b..804a320f 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -123,3 +123,13 @@ jobs: poetry run pytest tests/test_configuration.py -o addopts= \ -p no:xdist --disable-socket --allow-hosts=127.0.0.1,localhost \ -p no:cacheprovider --durations=0 -q + + # Thread-aware profile of the mocked config.get path: attributes the + # ~12x-more timer waits on 3.11 to a concrete function/call-count (cProfile + # can't — the work runs on a background EventLoopThread). + - name: "[probe] yappi thread-aware profile (mocked config.get)" + if: always() + shell: bash + run: | + poetry run pip install --quiet yappi + poetry run python .ci/probe_yappi.py From c0d121611dd4f66f437adaec0e574d270fac05ee Mon Sep 17 00:00:00 2001 From: Fernando Correia Date: Fri, 12 Jun 2026 15:36:47 -0700 Subject: [PATCH 12/12] =?UTF-8?q?ci:=20[probe]=20verify=20fix=20=E2=80=94?= =?UTF-8?q?=20bypass=20slow=20async=20config=20validation=20+=20fake.test?= =?UTF-8?q?=20rename?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the timing probe branch, apply the proposed fix and the URI rename together, keeping all instrumentation, to confirm via CI whether the fix removes the Windows Py3.11+ slowness. - conftest set_validated_config: wrap the per-test GalileoPythonConfig.get build in _fast_config_validation(), which stubs ApiClient.make_request/request with canned, await-free results. Skips the 3 async validation round-trips (healthcheck/login/current_user) whose Windows IOCP poll is ~11x slower on 3.11+. Scoped to the build only; per-test reset and test-body validation are unchanged. (2015 passed / 5 skipped locally — no regressions.) - Rename test host localtest -> fake.test (RFC 6761; dotted). .ci probe scripts keep localtest intentionally. - Workflow: replace selector A/B + yappi with subset + full-suite timing runs. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci-tests.yaml | 49 ++++++++++----------------------- galileo-a2a/pyproject.toml | 2 +- galileo-a2a/tests/conftest.py | 2 +- galileo-adk/pyproject.toml | 2 +- galileo-adk/tests/conftest.py | 2 +- pyproject.toml | 2 +- tests/conftest.py | 46 +++++++++++++++++++++++++++++-- tests/test_experiments.py | 2 +- tests/test_prompts_global.py | 20 +++++++------- 9 files changed, 73 insertions(+), 54 deletions(-) diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index 804a320f..42b710f0 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -84,52 +84,31 @@ jobs: assert actual == expected, f'Expected Python {expected}, got {actual}' " - # CPython verification: A/B the Windows default Proactor loop against the - # Selector loop, both at the raw-primitive level (microbench: cross-thread - # dispatch latency) and at the suite level (per-test setup avg via the - # conftest timing plugin). The full 24-min suite is intentionally dropped — - # this run is just the causal experiment. - - - name: "[probe] Microbench — Proactor (default loop)" - if: always() - shell: bash - run: poetry run python .ci/probe_microbench.py - - - name: "[probe] Microbench — Selector loop" + # FIX VERIFICATION: the autouse set_validated_config fixture now bypasses + # the slow async validation round-trips (HYBIM-790). The conftest timing + # plugin prints the per-test "setup avg"; compare against the recorded + # pre-fix baselines (subset serial: 685ms on 3.11 / 55ms on 3.10; full + # suite parallel: 1335ms on 3.11). If the fix works, 3.11 setup collapses + # toward 3.10 and the full suite drops from ~23min to a few minutes. + + - name: "[probe] Microbench (getaddrinfo / asyncio / dispatch)" if: always() shell: bash - env: - PROBE_EVENT_LOOP: selector run: poetry run python .ci/probe_microbench.py - # Same trivial 41-test file, serial, only the event loop policy differs. - # Compare the "setup total/avg" line in each run's TIMING SUMMARY. - - name: "[probe] Subset A/B — Proactor (default loop)" - if: always() - shell: bash - run: | - echo "[probe $(date -u +%H:%M:%S)] subset serial — Proactor (default)" - poetry run pytest tests/test_configuration.py -o addopts= \ - -p no:xdist --disable-socket --allow-hosts=127.0.0.1,localhost \ - -p no:cacheprovider --durations=0 -q - - - name: "[probe] Subset A/B — Selector loop" + - name: "[probe] Subset serial (setup avg WITH fix)" if: always() shell: bash - env: - PROBE_EVENT_LOOP: selector run: | - echo "[probe $(date -u +%H:%M:%S)] subset serial — Selector" + echo "[probe $(date -u +%H:%M:%S)] subset serial — with fix" poetry run pytest tests/test_configuration.py -o addopts= \ -p no:xdist --disable-socket --allow-hosts=127.0.0.1,localhost \ -p no:cacheprovider --durations=0 -q - # Thread-aware profile of the mocked config.get path: attributes the - # ~12x-more timer waits on 3.11 to a concrete function/call-count (cProfile - # can't — the work runs on a background EventLoopThread). - - name: "[probe] yappi thread-aware profile (mocked config.get)" + - name: "[probe] Full suite (setup avg + total WITH fix)" if: always() shell: bash run: | - poetry run pip install --quiet yappi - poetry run python .ci/probe_yappi.py + echo "[probe $(date -u +%H:%M:%S)] full suite start — with fix" + poetry run pytest tests --durations=0 -ra -q + echo "[probe $(date -u +%H:%M:%S)] full suite end — with fix" diff --git a/galileo-a2a/pyproject.toml b/galileo-a2a/pyproject.toml index da6538e1..f3a9a925 100644 --- a/galileo-a2a/pyproject.toml +++ b/galileo-a2a/pyproject.toml @@ -69,7 +69,7 @@ python_files = ["test_*.py"] python_classes = ["Test*"] python_functions = ["test_*"] env = [ - "GALILEO_CONSOLE_URL=http://localtest:8088", + "GALILEO_CONSOLE_URL=http://fake.test:8088", "GALILEO_API_KEY=api-1234567890", "GALILEO_PROJECT=test-project", "GALILEO_LOG_STREAM=test-log-stream", diff --git a/galileo-a2a/tests/conftest.py b/galileo-a2a/tests/conftest.py index 217e599d..22e5d807 100644 --- a/galileo-a2a/tests/conftest.py +++ b/galileo-a2a/tests/conftest.py @@ -9,7 +9,7 @@ # 3. Security - prevents real API keys from leaking into test logs import os -os.environ["GALILEO_CONSOLE_URL"] = "http://localtest:8088" +os.environ["GALILEO_CONSOLE_URL"] = "http://fake.test:8088" os.environ["GALILEO_API_KEY"] = "api-1234567890" os.environ["GALILEO_PROJECT"] = "test-project" os.environ["GALILEO_LOG_STREAM"] = "test-log-stream" diff --git a/galileo-adk/pyproject.toml b/galileo-adk/pyproject.toml index f8be13b3..57ad7c88 100644 --- a/galileo-adk/pyproject.toml +++ b/galileo-adk/pyproject.toml @@ -41,7 +41,7 @@ python_files = ["test_*.py"] python_classes = ["Test*"] python_functions = ["test_*"] env = [ - "GALILEO_CONSOLE_URL=http://localtest:8088", + "GALILEO_CONSOLE_URL=http://fake.test:8088", "GALILEO_API_KEY=api-1234567890", "GALILEO_PROJECT=test-project", "GALILEO_LOG_STREAM=test-log-stream", diff --git a/galileo-adk/tests/conftest.py b/galileo-adk/tests/conftest.py index 843976a5..bb30b0a5 100644 --- a/galileo-adk/tests/conftest.py +++ b/galileo-adk/tests/conftest.py @@ -168,7 +168,7 @@ def set_validated_config( # Reset any cached loggers from previous tests GalileoLoggerSingleton().reset_all() - config = GalileoPythonConfig.get(console_url="http://localtest:8088", api_key="api-1234567890") + config = GalileoPythonConfig.get(console_url="http://fake.test:8088", api_key="api-1234567890") yield # Clean up after test GalileoLoggerSingleton().reset_all() diff --git a/pyproject.toml b/pyproject.toml index 779fceff..634c7341 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,7 @@ pythonpath = ["./src/"] # Note: Some env vars are also set in conftest.py for pytest-xdist compatibility # on Python 3.14+. This section remains for documentation and older Python support. env = [ - "GALILEO_CONSOLE_URL=http://localtest:8088", + "GALILEO_CONSOLE_URL=http://fake.test:8088", "GALILEO_API_KEY=api-1234567890", "GALILEO_PROJECT=test-project", "GALILEO_LOG_STREAM=test-log-stream", diff --git a/tests/conftest.py b/tests/conftest.py index b6978d07..8fc176f1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,7 +22,7 @@ ) from openai.types.responses.response_usage import InputTokensDetails, OutputTokensDetails -_os.environ["GALILEO_CONSOLE_URL"] = "http://localtest:8088" +_os.environ["GALILEO_CONSOLE_URL"] = "http://fake.test:8088" _os.environ["GALILEO_API_KEY"] = "api-1234567890" _os.environ["GALILEO_PROJECT"] = "test-project" _os.environ["GALILEO_LOG_STREAM"] = "test-log-stream" @@ -52,8 +52,10 @@ import logging # noqa: E402 import sys # noqa: E402 from collections.abc import Callable, Generator # noqa: E402 +from contextlib import contextmanager # noqa: E402 from io import StringIO # noqa: E402 from pathlib import Path # noqa: E402 +from typing import Any # noqa: E402 from unittest.mock import AsyncMock, MagicMock, patch # noqa: E402 from uuid import uuid4 # noqa: E402 @@ -67,6 +69,7 @@ from galileo.resources.models.messages_list_item import MessagesListItem # noqa: E402 from galileo_core.constants.request_method import RequestMethod # noqa: E402 from galileo_core.constants.routes import Routes as CoreRoutes # noqa: E402 +from galileo_core.helpers.api_client import ApiClient # noqa: E402 from galileo_core.schemas.core.user import User # noqa: E402 from galileo_core.schemas.core.user_role import UserRole # noqa: E402 from galileo_core.schemas.protect.rule import Rule, RuleOperator # noqa: E402 @@ -124,6 +127,41 @@ def reset_agent_control_bridge_state() -> Generator[None, None, None]: bridge_module._PREVIOUS_TRACE_CONTEXT_PROVIDER = None +def _fast_validation_payload(endpoint: Any) -> dict: + """Canned response for the 3 config-validation endpoints.""" + ep = str(endpoint) + if "login" in ep or "token" in ep: + return {"access_token": "secret_jwt_token"} + if "current_user" in ep: + return User.model_validate({"id": uuid4(), "email": "user@example.com", "role": UserRole.user}).model_dump( + mode="json" + ) + return {"status": "ok"} + + +@contextmanager +def _fast_config_validation() -> Generator[None, None, None]: + """HYBIM-790: building GalileoPythonConfig runs 3 async validation requests + (healthcheck/login/current_user) through galileo_core's async_run / + EventLoopThreadPool, whose Windows IOCP poll is ~11x slower on Python 3.11+ + (see .local/HYBIM-790-investigation.md). They are already mocked, so they add + no coverage — only event-loop cost. Replace them with canned, await-free + results so the dispatch is trivial. Scoped to the per-test config build only; + test bodies still exercise the real validation/connect code.""" + + async def _stub_make_request(request_method: Any, base_url: str, endpoint: Any, **kwargs: Any) -> dict: + return _fast_validation_payload(endpoint) + + def _stub_request(self: Any, request_method: Any, path: Any = None, **kwargs: Any) -> dict: + return _fast_validation_payload(path) + + with ( + patch.object(ApiClient, "make_request", staticmethod(_stub_make_request)), + patch.object(ApiClient, "request", _stub_request), + ): + yield + + @pytest.fixture(autouse=True) def set_validated_config( mock_healthcheck: None, mock_login_api_key: None, mock_get_current_user: None, mock_decode_jwt: MagicMock @@ -134,8 +172,10 @@ def set_validated_config( if GalileoPythonConfig._instance is not None: GalileoPythonConfig._instance.reset() # Initialize config with EXPLICIT values to avoid env var timing issues with pytest-xdist - # This ensures correct config even if env vars weren't set before module imports - config = GalileoPythonConfig.get(console_url="http://localtest:8088", api_key="api-1234567890") + # This ensures correct config even if env vars weren't set before module imports. + # HYBIM-790: bypass the slow async validation round-trips for the build only. + with _fast_config_validation(): + config = GalileoPythonConfig.get(console_url="http://fake.test:8088", api_key="api-1234567890") yield config.reset() diff --git a/tests/test_experiments.py b/tests/test_experiments.py index bb4af895..7129cf19 100644 --- a/tests/test_experiments.py +++ b/tests/test_experiments.py @@ -674,7 +674,7 @@ def test_run_experiment_without_metrics( prompt_settings=ANY, ) - @pytest.mark.parametrize("console_url", ["http://localtest:8088", "http://localtest:8088/"]) + @pytest.mark.parametrize("console_url", ["http://fake.test:8088", "http://fake.test:8088/"]) @travel(datetime(2012, 1, 1), tick=False) @patch.object(galileo.datasets.Datasets, "get") @patch.object(galileo.jobs.Jobs, "create") diff --git a/tests/test_prompts_global.py b/tests/test_prompts_global.py index af0089f5..3ca5716f 100644 --- a/tests/test_prompts_global.py +++ b/tests/test_prompts_global.py @@ -62,12 +62,12 @@ class TestGlobalPromptTemplates: def test_create_global_prompt(self, respx_mock: MockRouter, prompt_template_response): """Test creating a global prompt template.""" # Mock the query API (for uniqueness check) - query_route = respx_mock.post("http://localtest:8088/templates/query").mock( + query_route = respx_mock.post("http://fake.test:8088/templates/query").mock( return_value=httpx.Response(200, json={"templates": []}) ) # Mock the create API - create_route = respx_mock.post("http://localtest:8088/templates").mock( + create_route = respx_mock.post("http://fake.test:8088/templates").mock( return_value=httpx.Response(200, json=prompt_template_response) ) @@ -80,7 +80,7 @@ def test_create_global_prompt(self, respx_mock: MockRouter, prompt_template_resp def test_get_global_prompt_by_id(self, respx_mock: MockRouter, prompt_template_response): """Test retrieving a global prompt template by ID.""" - get_route = respx_mock.get(f"http://localtest:8088/templates/{prompt_template_response['id']}").mock( + get_route = respx_mock.get(f"http://fake.test:8088/templates/{prompt_template_response['id']}").mock( return_value=httpx.Response(200, json=prompt_template_response) ) @@ -92,7 +92,7 @@ def test_get_global_prompt_by_id(self, respx_mock: MockRouter, prompt_template_r def test_get_global_prompt_by_name(self, respx_mock: MockRouter, prompt_template_response): """Test retrieving a global prompt template by name.""" - query_route = respx_mock.post("http://localtest:8088/templates/query").mock( + query_route = respx_mock.post("http://fake.test:8088/templates/query").mock( return_value=httpx.Response( 200, json={"templates": [prompt_template_response], "next_starting_token": None} ) @@ -106,7 +106,7 @@ def test_get_global_prompt_by_name(self, respx_mock: MockRouter, prompt_template def test_list_global_prompts(self, respx_mock: MockRouter, prompt_template_response): """Test listing global prompt templates.""" - query_route = respx_mock.post("http://localtest:8088/templates/query").mock( + query_route = respx_mock.post("http://fake.test:8088/templates/query").mock( return_value=httpx.Response( 200, json={"templates": [prompt_template_response], "next_starting_token": None} ) @@ -120,7 +120,7 @@ def test_list_global_prompts(self, respx_mock: MockRouter, prompt_template_respo def test_delete_global_prompt_by_id(self, respx_mock: MockRouter): """Test deleting a global prompt template by ID.""" - delete_route = respx_mock.delete("http://localtest:8088/templates/template-id-123").mock( + delete_route = respx_mock.delete("http://fake.test:8088/templates/template-id-123").mock( return_value=httpx.Response(200, json={"message": "Template deleted successfully"}) ) @@ -131,14 +131,14 @@ def test_delete_global_prompt_by_id(self, respx_mock: MockRouter): def test_delete_global_prompt_by_name(self, respx_mock: MockRouter, prompt_template_response): """Test deleting a global prompt template by name.""" # Mock query to find template by name - query_route = respx_mock.post("http://localtest:8088/templates/query").mock( + query_route = respx_mock.post("http://fake.test:8088/templates/query").mock( return_value=httpx.Response( 200, json={"templates": [prompt_template_response], "next_starting_token": None} ) ) # Mock delete - delete_route = respx_mock.delete(f"http://localtest:8088/templates/{prompt_template_response['id']}").mock( + delete_route = respx_mock.delete(f"http://fake.test:8088/templates/{prompt_template_response['id']}").mock( return_value=httpx.Response(200, json={"message": "Template deleted successfully"}) ) @@ -151,13 +151,13 @@ def test_create_prompt_with_unique_name(self, respx_mock: MockRouter, prompt_tem """Test that duplicate names get auto-incremented.""" # Mock query to find existing template existing_template = {**prompt_template_response, "name": "test-template"} - query_route = respx_mock.post("http://localtest:8088/templates/query").mock( + query_route = respx_mock.post("http://fake.test:8088/templates/query").mock( return_value=httpx.Response(200, json={"templates": [existing_template], "next_starting_token": None}) ) # Mock create with new unique name new_template = {**prompt_template_response, "name": "test-template (1)"} - create_route = respx_mock.post("http://localtest:8088/templates").mock( + create_route = respx_mock.post("http://fake.test:8088/templates").mock( return_value=httpx.Response(200, json=new_template) )