diff --git a/Dockerfile b/Dockerfile index 925e5ad..d0bc498 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,6 @@ ARG PIXI_VERSION=0.67.0 ARG UBUNTU_VERSION=22.04 -ARG PYTANTAN_VERSION=0.1.3 # --------------------------------------------------------------------------- # Stage 1: build — resolve + install the pixi environment from pixi.lock @@ -13,118 +12,35 @@ FROM --platform=linux/amd64 ghcr.io/prefix-dev/pixi:${PIXI_VERSION} AS build WORKDIR /app +# Build tools needed by uv when pixi compiles pytantan from the git source +# (see pixi.toml). Installed before `pixi install` so the source build can run. +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git build-essential cmake zlib1g-dev binutils ca-certificates && \ + rm -rf /var/lib/apt/lists/* + # Copy only what's needed for the pixi install + the local funannotate2 source # referenced by `funannotate2 = { path = "." }` in pixi.toml. COPY pixi.toml pixi.lock pyproject.toml setup.py README.md LICENSE ./ COPY funannotate2 ./funannotate2 -# Install from the lockfile; no-op if the lockfile already matches. -RUN pixi install --locked +# Force pytantan's generic-only build (no SSE4/AVX2/NEON). The fork's +# CMakeLists.txt reads PYTANTAN_DISABLE_SIMD and skips SIMD detection, so the +# resulting wheel contains only platform/generic.so. This keeps the image +# safe on Rosetta 2 (Apple Silicon) and pre-Haswell x86_64 servers. +ENV CMAKE_ARGS="-DPYTANTAN_DISABLE_SIMD=ON" -# Rebuild pytantan from source with all SIMD backends disabled. -# -# Background: pytantan's PyPI wheels ship with AVX2 SIMD enabled, which SIGILLs -# on x86_64 hosts that don't translate every AVX2 opcode (Rosetta 2 on Apple -# Silicon has known gaps, and some pre-Haswell servers lack AVX2 outright). -# -# We pin to v0.1.3 and rebuild from a patched checkout. Two independent gates -# guarantee the resulting .so files are free of AVX2/AVX512: -# (1) CMakeLists.txt is patched to set HAVE_SSE4/HAVE_AVX2/HAVE_NEON to OFF -# and drop the include() lines for the corresponding Find*.cmake modules, -# so SIMD detection never runs and add_compile_options(-mavx2) cannot -# fire even if the env-var plumbing is wrong. -# (2) The build is invoked with scikit-build-core's real config-settings -# interface (cmake.define.*) and CMAKE_ARGS, both of which are honored -# (the previously-used SKBUILD_CMAKE_ARGS env var is NOT a real knob). -ARG PYTANTAN_VERSION -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - git build-essential cmake zlib1g-dev binutils ca-certificates && \ - rm -rf /var/lib/apt/lists/* +RUN pixi install --locked -# CACHEBUST forces the pytantan rebuild RUN below to re-execute when bumped, -# bypassing BuildKit's GHA layer cache. Bump on any pytantan-related change. -ARG PYTANTAN_CACHEBUST=4 - -# Stage-local ENVs so the values are available inside the quoted heredoc below -# without subjecting the Python source to Dockerfile-level ${...} expansion. -ENV _PT_VERSION=${PYTANTAN_VERSION} \ - _PT_CACHEBUST=${PYTANTAN_CACHEBUST} - -RUN --mount=type=tmpfs,target=/tmp/build <<'BASH' -set -eux -echo "pytantan rebuild (cachebust=${_PT_CACHEBUST}, version=${_PT_VERSION})" -git clone --depth 1 --branch "v${_PT_VERSION}" \ - --recurse-submodules --shallow-submodules \ - https://github.com/althonos/pytantan.git /tmp/build/pytantan -cd /tmp/build/pytantan - -/app/.pixi/envs/default/bin/python <<'PY' -import pathlib, re - -def strip_if_block(text, var, replacement): - """Remove a top-level if() ... endif() block, handling nested if().""" - lines = text.splitlines(keepends=True) - out = [] - i = 0 - open_re = re.compile(r'^\s*if\s*\(') - close_re = re.compile(r'^\s*endif\s*\(') - target_re = re.compile(rf'^\s*if\s*\(\s*{re.escape(var)}\s*\)') - while i < len(lines): - if target_re.match(lines[i]): - depth = 1 - i += 1 - while i < len(lines) and depth > 0: - if open_re.match(lines[i]): - depth += 1 - elif close_re.match(lines[i]): - depth -= 1 - i += 1 - if replacement: - out.append(replacement) - else: - out.append(lines[i]) - i += 1 - return "".join(out) - -for path, repl_factory in [ - ("CMakeLists.txt", lambda v: f"set({v} OFF)\n"), - ("src/pytantan/platform/CMakeLists.txt", lambda v: ""), -]: - p = pathlib.Path(path) - src = p.read_text() - if path == "CMakeLists.txt": - src = re.sub( - r'include\("src/scripts/cmake/Find(SSE4|AVX2|NEON)\.cmake"\)\n', - "", src, - ) - for var in ("HAVE_SSE4", "HAVE_AVX2", "HAVE_NEON"): - src = strip_if_block(src, var, repl_factory(var)) - p.write_text(src) - print(f"=== patched {path} ===") - print(src) -PY - -CMAKE_ARGS="-DHAVE_SSE4:BOOL=OFF -DHAVE_AVX2:BOOL=OFF -DHAVE_NEON:BOOL=OFF" \ -/app/.pixi/envs/default/bin/pip install -v \ - --no-deps --no-cache-dir --force-reinstall \ - --config-settings=cmake.define.HAVE_SSE4=OFF \ - --config-settings=cmake.define.HAVE_AVX2=OFF \ - --config-settings=cmake.define.HAVE_NEON=OFF \ - /tmp/build/pytantan -rm -rf /root/.cache/pip -BASH - -# Verify the rebuild was effective. Two independent checks: +# Verify pytantan was built without SIMD backends. Two independent checks: # (a) No avx*/sse4*/neon* platform module .so files should exist. -# (b) No .so under the pixi env may contain AVX2/AVX512 instructions -# (ymm/zmm register refs or VEX-encoded vector mnemonics). Anything -# SSE2 (xmm only, non-VEX) is safe — that's the x86_64 baseline. -# Fail the build loudly otherwise so we can never ship an image that SIGILLs -# at import time. +# (b) No .so under pytantan may contain AVX2/AVX512 instructions (ymm/zmm +# register refs or VEX-encoded vector mnemonics). Plain SSE2 (xmm only, +# non-VEX) is the x86_64 baseline and is fine. +# Fail the build loudly otherwise so we never ship an image that SIGILLs at +# import time. RUN set -eux; \ PY=/app/.pixi/envs/default/bin/python; \ - SITE=$("$PY" -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])'); \ PT_DIR=$("$PY" -c 'import pytantan, os; print(os.path.dirname(pytantan.__file__))'); \ echo "pytantan installed at: $PT_DIR"; \ "$PY" -c "import pytantan; print('pytantan version:', pytantan.__version__)"; \ @@ -132,7 +48,7 @@ RUN set -eux; \ if [ -d "$PT_DIR/platform" ]; then \ SIMD_MODS=$(ls "$PT_DIR/platform/" | grep -Ei '^(avx|sse|neon)' || true); \ if [ -n "$SIMD_MODS" ]; then \ - echo "ERROR: SIMD platform modules present despite HAVE_*=OFF:"; \ + echo "ERROR: SIMD platform modules present despite PYTANTAN_DISABLE_SIMD=ON:"; \ echo "$SIMD_MODS"; \ exit 1; \ fi; \ diff --git a/pixi.lock b/pixi.lock index 2f973f3..6b4b642 100644 --- a/pixi.lock +++ b/pixi.lock @@ -243,7 +243,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9f/a0/b1266b0d852d10ac8339540d09fb90e38c198b6a2bcdeb3a881338d4a8fc/keras-layer-normalization-0.16.0.tar.gz - pypi: https://files.pythonhosted.org/packages/54/4b/195ac84cc8f6077b4f0f421e8daee21b7f1bd88cb7716414234379fe68ec/numcodecs-0.16.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/01/83/15167305eeb4ffa8aca949030f6a9ef5c9ad579f783ab407425c3905f70f/pytantan-0.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: git+https://github.com/nextgenusfs/pytantan.git?rev=90818ff695c844cefd5f42eb3baa297263565f21#90818ff695c844cefd5f42eb3baa297263565f21 - pypi: https://files.pythonhosted.org/packages/36/4d/4a67f30778a45d542bbea5db2dbfa1e9e100bf9ba64aefe34215ba9f11f6/scikit_learn-1.8.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/09/7d/af933f0f6e0767995b4e2d705a0665e454d1c19402aa7e895de3951ebb04/scipy-1.17.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/1d/9c/9111e1e0fa82a8f48dd9d59853d7706cc3e04b01948fa6687e676fcf012d/scoring_matrices-0.3.4-cp311-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl @@ -504,7 +504,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9f/a0/b1266b0d852d10ac8339540d09fb90e38c198b6a2bcdeb3a881338d4a8fc/keras-layer-normalization-0.16.0.tar.gz - pypi: https://files.pythonhosted.org/packages/0e/cc/0d97ef55dda48cb0f93d7b92d761208e7a99bd2eea6b0e859426e6a99a21/numcodecs-0.16.5-cp311-cp311-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/82/0a/07990c8a23ec94b8dc2e88c6afd12da5131d0c8171e4246f3a8de5eafd44/pytantan-0.1.3-cp311-cp311-macosx_11_0_arm64.whl + - pypi: git+https://github.com/nextgenusfs/pytantan.git?rev=90818ff695c844cefd5f42eb3baa297263565f21#90818ff695c844cefd5f42eb3baa297263565f21 - pypi: https://files.pythonhosted.org/packages/01/18/d154dc1638803adf987910cdd07097d9c526663a55666a97c124d09fb96a/scikit_learn-1.8.0-cp311-cp311-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/f7/58/bccc2861b305abdd1b8663d6130c0b3d7cc22e8d86663edbc8401bfd40d4/scipy-1.17.1-cp311-cp311-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/3a/3e/998a2072e7fdc8b80f37ad6d78157f6e68894936e6b8b957c788494a5c79/scoring_matrices-0.3.4-cp311-abi3-macosx_11_0_arm64.whl @@ -4885,23 +4885,12 @@ packages: - pkg:pypi/pysocks?source=hash-mapping size: 21085 timestamp: 1733217331982 -- pypi: https://files.pythonhosted.org/packages/01/83/15167305eeb4ffa8aca949030f6a9ef5c9ad579f783ab407425c3905f70f/pytantan-0.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +- pypi: git+https://github.com/nextgenusfs/pytantan.git?rev=90818ff695c844cefd5f42eb3baa297263565f21#90818ff695c844cefd5f42eb3baa297263565f21 name: pytantan - version: 0.1.3 - sha256: 6f0c370767d955c9b4fbb252cac0e419e0a17e761157546118255ba8e47d1fe5 + version: 0.1.4 requires_dist: - archspec~=0.2 ; os_name != 'nt' - scoring-matrices~=0.3.0 - - importlib-resources ; python_full_version < '3.9' and extra == 'test' - requires_python: '>=3.7' -- pypi: https://files.pythonhosted.org/packages/82/0a/07990c8a23ec94b8dc2e88c6afd12da5131d0c8171e4246f3a8de5eafd44/pytantan-0.1.3-cp311-cp311-macosx_11_0_arm64.whl - name: pytantan - version: 0.1.3 - sha256: 9255e58da1233e796639b17d10cb7bac7df0c43a6ef8273a42a938b549f0b98c - requires_dist: - - archspec~=0.2 ; os_name != 'nt' - - scoring-matrices~=0.3.0 - - importlib-resources ; python_full_version < '3.9' and extra == 'test' requires_python: '>=3.7' - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-9.0.3-pyhc364b38_1.conda sha256: 960f59442173eee0731906a9077bd5ccf60f4b4226f05a22d1728ab9a21a879c diff --git a/pixi.toml b/pixi.toml index f2ab81b..075f56d 100644 --- a/pixi.toml +++ b/pixi.toml @@ -52,10 +52,13 @@ funannotate2 = { path = ".", editable = false } "funannotate2-addons" = "*" helixerlite = "*" annorefine = ">=2026.2.9" -# pytantan: pinned to 0.1.3 from PyPI; the 0.1.4 wheel/SIMD dispatch has been -# unreliable on x86_64 hosts without AVX2 (notably Rosetta 2 on Apple Silicon). -# The Docker image rebuilds this from source with AVX2 disabled. -pytantan = "==0.1.3" +# pytantan: built from the nextgenusfs/pytantan fork, which adds a +# `PYTANTAN_DISABLE_SIMD` CMake option and a `PYTANTAN_SIMD` env-var override. +# The PyPI wheels ship with AVX2 enabled and SIGILL under Rosetta 2 on Apple +# Silicon (and on pre-Haswell x86_64); building from source with +# `CMAKE_ARGS=-DPYTANTAN_DISABLE_SIMD=ON` (set in the Dockerfile) produces a +# generic-only build that runs everywhere. +pytantan = { git = "https://github.com/nextgenusfs/pytantan.git", rev = "90818ff695c844cefd5f42eb3baa297263565f21" } [tasks]