diff --git a/CITATION.cff b/CITATION.cff index 4a692c5..cde0668 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -18,4 +18,4 @@ keywords: - consensus gene models license: BSD-2-Clause version: version = "26.5.22" -date-released: '2026-05-23' +date-released: '2026-05-24' diff --git a/Dockerfile b/Dockerfile index c003f61..0e9548f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ARG PIXI_VERSION=0.67.0 ARG UBUNTU_VERSION=22.04 -ARG PYTANTAN_VERSION=0.1.4 +ARG PYTANTAN_VERSION=0.1.3 # --------------------------------------------------------------------------- # Stage 1: build — resolve + install the pixi environment from pixi.lock @@ -21,53 +21,44 @@ COPY funannotate2 ./funannotate2 # Install from the lockfile; no-op if the lockfile already matches. RUN pixi install --locked -# Rebuild pytantan from source with all SIMD backends disabled. +# Rebuild pytantan from source with AVX2 disabled. # -# Background: pytantan's wheels ship with AVX2 enabled, and prior to v0.1.4 -# the generic path was also polluted with AVX2 flags via `add_compile_options` -# in the project-level CMakeLists.txt. Either way, on x86_64 CPUs without AVX2 -# (notably Rosetta 2 on Apple Silicon) any code path that executes an AVX2 -# instruction raises SIGILL. The runtime-dispatch added in v0.1.4 does not -# help us here because we want a build that is portable to *any* x86_64. +# Background: pytantan's PyPI wheels ship with AVX2 SIMD enabled, which SIGILLs +# on x86_64 CPUs that lack AVX2 (notably Rosetta 2 on Apple Silicon, and some +# pre-Haswell servers). We pin to v0.1.3 — this is the last release with a +# straightforward CMake gate, where setting HAVE_AVX2=OFF and clearing the +# AVX2_C_FLAGS cache variable is sufficient to produce a portable build. # -# pytantan's CMakeLists.txt uses FindAVX2/FindSSE4/FindNEON which auto-detect -# on the build host (GitHub Actions runners have AVX2), so we have to force -# the HAVE_* flags OFF. Pre-defining them in FindAVX2.cmake's `if((DEFINED ...))` -# guard short-circuits detection entirely. -# -# We pass these via pip's `--config-settings=cmake.define.*` rather than -# CMAKE_ARGS/SKBUILD_CMAKE_ARGS env vars, since that route goes directly into -# scikit-build-core's parser and is not subject to env-var filtering. +# scikit-build-core (pytantan's build backend) reads SKBUILD_CMAKE_ARGS as a +# semicolon-separated CMake list and passes it directly to the inner cmake +# invocation, bypassing the env-var filtering that affects CMAKE_ARGS. ARG PYTANTAN_VERSION RUN apt-get update && \ apt-get install -y --no-install-recommends \ git build-essential cmake zlib1g-dev binutils ca-certificates && \ rm -rf /var/lib/apt/lists/* -RUN /app/.pixi/envs/default/bin/pip install \ - --no-deps --no-cache-dir --force-reinstall -v \ - --config-settings=cmake.define.HAVE_AVX2=OFF \ - --config-settings=cmake.define.HAVE_SSE4=OFF \ - --config-settings=cmake.define.HAVE_NEON=OFF \ - --config-settings=cmake.define.AVX2_C_FLAGS= \ - --config-settings=cmake.define.SSE4_C_FLAGS= \ - --config-settings=cmake.define.NEON_C_FLAGS= \ +RUN SKBUILD_CMAKE_ARGS="-DHAVE_AVX2:BOOL=OFF;-DAVX2_C_FLAGS:STRING=" \ + /app/.pixi/envs/default/bin/pip install \ + --no-deps --no-cache-dir --force-reinstall \ "pytantan @ git+https://github.com/althonos/pytantan.git@v${PYTANTAN_VERSION}" && \ rm -rf /root/.cache/pip -# Verify the rebuild was effective: only `generic` should be present in the -# platform/ directory and neither lib.*.so nor generic.*.so should contain -# any AVX/AVX2 instructions (no `ymm`/`zmm` register references, no `vex`- -# encoded vector ops). Fail the build loudly otherwise so we never ship an -# image that SIGILLs at import time. +# Verify the rebuild was effective: no avx2.*.so module should be present in +# the platform/ directory, and no shipped .so should contain AVX/AVX2/AVX512 +# vector instructions (ymm/zmm register references). SSE4 on x86_64 is fine +# under Rosetta 2 and is left enabled. Fail the build loudly otherwise so we +# never ship an image that SIGILLs at import time. RUN set -eux; \ PY=/app/.pixi/envs/default/bin/python; \ PT_DIR=$("$PY" -c 'import pytantan, os; print(os.path.dirname(pytantan.__file__))'); \ echo "pytantan installed at: $PT_DIR"; \ - ls -la "$PT_DIR/platform/"; \ - SIMD_MODS=$(ls "$PT_DIR/platform/" | grep -E '^(avx2|sse4|neon)\.' || true); \ - if [ -n "$SIMD_MODS" ]; then \ - echo "ERROR: SIMD platform modules were built despite HAVE_*=OFF: $SIMD_MODS"; \ - exit 1; \ + if [ -d "$PT_DIR/platform" ]; then ls -la "$PT_DIR/platform/"; fi; \ + if [ -d "$PT_DIR/platform" ]; then \ + AVX_MODS=$(ls "$PT_DIR/platform/" | grep -E '^avx[0-9]*\.' || true); \ + if [ -n "$AVX_MODS" ]; then \ + echo "ERROR: AVX platform modules were built despite HAVE_AVX2=OFF: $AVX_MODS"; \ + exit 1; \ + fi; \ fi; \ BAD=""; \ for so in $(find "$PT_DIR" -maxdepth 2 -name '*.so'); do \ @@ -78,7 +69,7 @@ RUN set -eux; \ fi; \ done; \ if [ -n "$BAD" ]; then exit 1; fi; \ - "$PY" -c "import pytantan; from pytantan import Alphabet, RepeatFinder, default_scoring_matrix; print('pytantan smoke test OK', pytantan.__version__)" + "$PY" -c "import pytantan; from pytantan.lib import RepeatFinder, default_scoring_matrix; print('pytantan smoke test OK', pytantan.__version__)" # Pre-generate an activation script so the final image doesn't need pixi. RUN mkdir -p /app/bin && \ diff --git a/funannotate2/fastx.py b/funannotate2/fastx.py index f18b8bb..24ace75 100755 --- a/funannotate2/fastx.py +++ b/funannotate2/fastx.py @@ -4,7 +4,6 @@ import pyfastx from natsort import natsorted -from pytantan.lib import RepeatFinder, default_scoring_matrix def softmask_fasta( @@ -39,6 +38,20 @@ def softmask_fasta( Returns: None """ + # pytantan is imported lazily: its compiled extension uses CPU-specific + # SIMD that can SIGILL on hosts without AVX2 (notably Rosetta 2 on Apple + # Silicon). Deferring the import keeps unrelated funannotate2 subcommands + # usable when pytantan is broken or absent. + try: + from pytantan.lib import RepeatFinder, default_scoring_matrix + except ImportError as e: + raise RuntimeError( + "pytantan is required for softmasking but could not be imported " + "(it may be missing or built with incompatible SIMD flags). " + "Install a working pytantan build, or pre-softmask the assembly " + "with an external repeat masker before running funannotate2. " + f"Underlying error: {e}" + ) from e # set default scoring matrix matrix = default_scoring_matrix(protein, match_score, mismatch_cost) repeat_finder = RepeatFinder( diff --git a/funannotate2/predict.py b/funannotate2/predict.py index fbafd33..96c0845 100755 --- a/funannotate2/predict.py +++ b/funannotate2/predict.py @@ -241,7 +241,16 @@ def predict(args): "Genome assembly is not softmasked, running pytantan to quickly softmask repeats" ) os.rename(GenomeFasta, GenomeFasta + ".original") - softmask_fasta(GenomeFasta + ".original", GenomeFasta) + try: + softmask_fasta(GenomeFasta + ".original", GenomeFasta) + except (RuntimeError, ImportError) as e: + os.rename(GenomeFasta + ".original", GenomeFasta) + logger.critical( + f"Softmasking with pytantan failed: {e}\n" + "Pre-softmask the assembly with an external repeat masker " + "(e.g. tantan, RepeatMasker, or red) and rerun." + ) + raise SystemExit(1) stats, bad_names, nuc_errors, contigs = analyzeAssembly( GenomeFasta, maskedRegions, diff --git a/pixi.lock b/pixi.lock index 28d52d6..2f973f3 100644 --- a/pixi.lock +++ b/pixi.lock @@ -243,7 +243,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9f/a0/b1266b0d852d10ac8339540d09fb90e38c198b6a2bcdeb3a881338d4a8fc/keras-layer-normalization-0.16.0.tar.gz - pypi: https://files.pythonhosted.org/packages/54/4b/195ac84cc8f6077b4f0f421e8daee21b7f1bd88cb7716414234379fe68ec/numcodecs-0.16.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/df/3e/6211c83f41a8d68e7002c615e8fa1b1da4d7a7a2c1fa0779f826871aa577/pytantan-0.1.4-cp311-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/01/83/15167305eeb4ffa8aca949030f6a9ef5c9ad579f783ab407425c3905f70f/pytantan-0.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/36/4d/4a67f30778a45d542bbea5db2dbfa1e9e100bf9ba64aefe34215ba9f11f6/scikit_learn-1.8.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/09/7d/af933f0f6e0767995b4e2d705a0665e454d1c19402aa7e895de3951ebb04/scipy-1.17.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/1d/9c/9111e1e0fa82a8f48dd9d59853d7706cc3e04b01948fa6687e676fcf012d/scoring_matrices-0.3.4-cp311-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl @@ -504,7 +504,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9f/a0/b1266b0d852d10ac8339540d09fb90e38c198b6a2bcdeb3a881338d4a8fc/keras-layer-normalization-0.16.0.tar.gz - pypi: https://files.pythonhosted.org/packages/0e/cc/0d97ef55dda48cb0f93d7b92d761208e7a99bd2eea6b0e859426e6a99a21/numcodecs-0.16.5-cp311-cp311-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/24/61/50579c1441cc76b6e377e996c14c77b8f45d12195a980cdfa1af8e29bd81/pytantan-0.1.4-cp311-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/82/0a/07990c8a23ec94b8dc2e88c6afd12da5131d0c8171e4246f3a8de5eafd44/pytantan-0.1.3-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/01/18/d154dc1638803adf987910cdd07097d9c526663a55666a97c124d09fb96a/scikit_learn-1.8.0-cp311-cp311-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/f7/58/bccc2861b305abdd1b8663d6130c0b3d7cc22e8d86663edbc8401bfd40d4/scipy-1.17.1-cp311-cp311-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/3a/3e/998a2072e7fdc8b80f37ad6d78157f6e68894936e6b8b957c788494a5c79/scoring_matrices-0.3.4-cp311-abi3-macosx_11_0_arm64.whl @@ -1472,7 +1472,7 @@ packages: - pypi: ./ name: funannotate2 version: 26.5.22 - sha256: 1339b472e59ff2ed43e26f450e21f70bac1be9e4fee106a667d6f16422658d74 + sha256: d892e17617b6b7a4d7aefb41a7bdcc90fb9bca2f8a14abcaa844ea08f52cb831 requires_dist: - natsort - numpy @@ -1485,7 +1485,7 @@ packages: - requests - gb-io>=0.3.2 - json-repair - - pytantan>=0.1.4 + - pytantan>=0.1.3 - psutil - annorefine>=2026.2.9 requires_python: '>=3.10.0,<3.14' @@ -4885,21 +4885,23 @@ packages: - pkg:pypi/pysocks?source=hash-mapping size: 21085 timestamp: 1733217331982 -- pypi: https://files.pythonhosted.org/packages/24/61/50579c1441cc76b6e377e996c14c77b8f45d12195a980cdfa1af8e29bd81/pytantan-0.1.4-cp311-abi3-macosx_11_0_arm64.whl +- pypi: https://files.pythonhosted.org/packages/01/83/15167305eeb4ffa8aca949030f6a9ef5c9ad579f783ab407425c3905f70f/pytantan-0.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: pytantan - version: 0.1.4 - sha256: 299c5927a753d9a6528df1bc795cbcfa2b96022e9a50ca010575ae496b599639 + version: 0.1.3 + sha256: 6f0c370767d955c9b4fbb252cac0e419e0a17e761157546118255ba8e47d1fe5 requires_dist: - archspec~=0.2 ; os_name != 'nt' - scoring-matrices~=0.3.0 + - importlib-resources ; python_full_version < '3.9' and extra == 'test' requires_python: '>=3.7' -- pypi: https://files.pythonhosted.org/packages/df/3e/6211c83f41a8d68e7002c615e8fa1b1da4d7a7a2c1fa0779f826871aa577/pytantan-0.1.4-cp311-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl +- pypi: https://files.pythonhosted.org/packages/82/0a/07990c8a23ec94b8dc2e88c6afd12da5131d0c8171e4246f3a8de5eafd44/pytantan-0.1.3-cp311-cp311-macosx_11_0_arm64.whl name: pytantan - version: 0.1.4 - sha256: 6273c806b2a3e106967136af616d55254263ffab225fd3e1147a3696e413a428 + version: 0.1.3 + sha256: 9255e58da1233e796639b17d10cb7bac7df0c43a6ef8273a42a938b549f0b98c requires_dist: - archspec~=0.2 ; os_name != 'nt' - scoring-matrices~=0.3.0 + - importlib-resources ; python_full_version < '3.9' and extra == 'test' requires_python: '>=3.7' - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-9.0.3-pyhc364b38_1.conda sha256: 960f59442173eee0731906a9077bd5ccf60f4b4226f05a22d1728ab9a21a879c diff --git a/pixi.toml b/pixi.toml index 9303601..f2ab81b 100644 --- a/pixi.toml +++ b/pixi.toml @@ -52,8 +52,10 @@ funannotate2 = { path = ".", editable = false } "funannotate2-addons" = "*" helixerlite = "*" annorefine = ">=2026.2.9" -# pytantan: bioconda lags at 0.1.3; pull 0.1.4+ from PyPI -pytantan = ">=0.1.4" +# pytantan: pinned to 0.1.3 from PyPI; the 0.1.4 wheel/SIMD dispatch has been +# unreliable on x86_64 hosts without AVX2 (notably Rosetta 2 on Apple Silicon). +# The Docker image rebuilds this from source with AVX2 disabled. +pytantan = "==0.1.3" [tasks] diff --git a/pyproject.toml b/pyproject.toml index 400e2d1..1a16fa4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "requests", "gb-io>=0.3.2", "json-repair", - "pytantan>=0.1.4", + "pytantan>=0.1.3", "psutil", "annorefine>=2026.2.9" ]