fix: update pyFF to pybergshamra

- Require pybergshamra 0.9.2 and refresh the lockfile.
- Remove the temporary sibling checkout override.
- Serialize PKCS#11 signing per token and release sessions on errors.
- Add regression tests for session cleanup and concurrent signing.
- Update dependency documentation and record server benchmarks.

Author: kushaldas

README.rst

@@ -41,11 +41,13 @@ Features
 Dependencies
 ============
 
-* pyXMLSecurity
-* PyKCS11 (optional)
+* pybergshamra >= 0.9.2 (XML signatures and PKCS#11 support)
+* pyuppsala >= 0.11.0 (XML processing)
 * pygments
 * gunicorn (for the standalone pyffd server)
-* ... cf requirements.txt
+* ... cf pyproject.toml
+
+``uv sync`` installs pybergshamra from PyPI; no sibling checkout is required.
 
 More information
 ================

docs/index.rst

@@ -13,7 +13,7 @@ SAML metadata.
 size including edugain.org.
 
 **pyFF** supports producing and validating digital signatures on SAML metadata using
-the pyXMLSecurity package which in turn supports PKCS#11 and other mechanisms for
+the pybergshamra package (0.9.2 or newer), which supports PKCS#11 and other mechanisms for
 talking to HSMs and other cryptographic hardware.
 
 **pyFF** is also a complete implementation of the SAML metadata query protocol as 

docs/usage/examples.rst

@@ -242,11 +242,17 @@ a slot selector with ``token_label`` or ``token_serial``. The token label select
 the HSM token; the final URI component (``sctest2`` above) selects the key object
 within that token.
 
+pybergshamra 0.9.2 rejects a new login when the token is already authenticated.
+pyFF serializes signing requests per token within each worker process and
+releases the session and signing objects after each operation, including on
+errors. Providers stay cached to keep the vendor module loaded. Other
+authenticated sessions for the same token in that process must be closed
+before pyFF can log in.
+
 Running this example requires some preparation. Run the 'p11setup.sh' script in the examples directory.
 This results in a SoftHSM token being setup with the PIN 'secret1' and SO_PIN 'secret2'. Now run pyFF (assuming
 you are using a unix-like environment).
 
 .. code-block:: bash
 
     # env PYKCS11PIN=secret1 SOFTHSM_CONF=softhsm.conf pyff --loglevel=DEBUG p11.fd
-

pyproject.toml

@@ -28,7 +28,7 @@ dependencies = [
     "pyramid>=2.0.2",
     "pytz>=2025.2",
     "pyuppsala>=0.11.0",
-    "pybergshamra>=0.9.1",
+    "pybergshamra>=0.9.2",
     "pyyaml>=3.10",
     "redis>=5.3.0",
     "redis-collections>=0.13.0",

src/pyff/builtins.py

@@ -74,6 +74,7 @@
 # remain short-lived and are opened separately for each signing operation.
 _pkcs11_provider_cache = {}
 _pkcs11_provider_cache_lock = threading.Lock()
+_pkcs11_signing_locks = {}
 
 
 @pipe
@@ -1226,6 +1227,30 @@ def _select_pkcs11_provider(
         return provider
 
 
+def _sign_pkcs11(provider, module_path, pin, label, xml, *, reference_id=None, cert_pem=None):
+    """Serialize token logins and release authenticated objects even on errors."""
+    # Different selectors can resolve to the same token. Lock its actual slot,
+    # rather than the configured label/serial/slot selector or key label.
+    token = (os.path.realpath(module_path), provider.slot_id)
+    with _pkcs11_provider_cache_lock:
+        lock = _pkcs11_signing_locks.setdefault(token, threading.Lock())
+
+    with lock:
+        session = signer = ctx = None
+        try:
+            session = provider.open_session(pin)
+            signer = pybergshamra.Pkcs11Signer(session, label, pybergshamra.Algorithm.RSA_SHA256)
+            ctx = pybergshamra.DsigContext(pybergshamra.KeysManager())
+            ctx.set_hsm_signer(signer)
+            return pybergshamra.sign_enveloped(ctx, xml, reference_id=reference_id, cert_pem=cert_pem)
+        finally:
+            # Exception tracebacks retain Python frames. Clear every local
+            # owner before unlocking so a failed request cannot retain login.
+            ctx = None
+            signer = None
+            session = None
+
+
 @pipe
 def sign(req: Plumbing.Request, *_opts):
     """
@@ -1255,7 +1280,7 @@ def sign(req: Plumbing.Request, *_opts):
 
     The pin parameter can be used to point to an environment variable containing the pin: "env:<ENV variable>".
     By default pin is "env:PYKCS11PIN" which tells sign to use the pin found in the PYKCS11PIN environment
-    variable. This is also the default for PyKCS11 which is used to communicate with the PKCS#11 module.
+    variable. Authentication and signing are serialized per token within the worker process.
 
     **Examples**
 
@@ -1310,9 +1335,6 @@ def sign(req: Plumbing.Request, *_opts):
         with open(cert_file) as fd:
             cert_pem = fd.read()
 
-    mgr = pybergshamra.KeysManager()
-    ctx = pybergshamra.DsigContext(mgr)
-
     is_pkcs11_key = key_file.startswith("pkcs11:")
     if is_pkcs11_key:
         # HSM signing via PKCS#11. The URI form is:
@@ -1334,27 +1356,22 @@ def sign(req: Plumbing.Request, *_opts):
             token_label=req.args.get('token_label'),
             token_serial=req.args.get('token_serial'),
         )
-        session = provider.open_session(pin)
-        signer = pybergshamra.Pkcs11Signer(session, label, pybergshamra.Algorithm.RSA_SHA256)
-        ctx.set_hsm_signer(signer)
+        # Keep HSM signing on an owned XML string. Authentication and all
+        # operation objects must be released before the next token login.
+        xml = etree.tostring(relt, encoding='unicode')
+        signed = _sign_pkcs11(provider, module_path, pin, label, xml, reference_id=idattr, cert_pem=cert_pem)
+        req.t = root(etree.fromstring(signed))
+        return req.t
     else:
         # On-disk private key path.
+        mgr = pybergshamra.KeysManager()
+        ctx = pybergshamra.DsigContext(mgr)
         key = pybergshamra.load_key_file(key_file)
         mgr.add_key(key)
 
-    if not is_pkcs11_key and is_document_root(relt):
-        # Sign the working tree in place through the shared native DOM
-        # (pyuppsala document handle). This avoids serializing the whole
-        # aggregate, the signer's internal re-parses, and the reparse of the
-        # signed result -- downstream pipes keep the very same tree, with the
-        # <ds:Signature> inserted as the root's first child.
-        #
-        # PKCS#11 signing deliberately uses the owned-string path below. In
-        # production, mutating a live pyuppsala aggregate through the native
-        # document capsule while a vendor PKCS#11 module signs has caused
-        # worker SIGSEGVs and partially serialized aggregates. Keeping the HSM
-        # operation on an owned document avoids that unsafe integration
-        # boundary while preserving the native fast path for software keys.
+    if is_document_root(relt):
+        # The document API uses an owned XML handoff and imports the signed
+        # result into the existing document, preserving downstream tree identity.
         #
         # Only valid when ``relt`` is the document root. After ``first`` (an
         # MDQ single-entity request) the working document is an
@@ -1366,7 +1383,7 @@ def sign(req: Plumbing.Request, *_opts):
         pybergshamra.sign_enveloped_document(ctx, doc, reference_id=idattr, cert_pem=cert_pem)
         req.t = relt
     else:
-        # PKCS11 signing, or a subtree of a larger document (small, so the
+        # A subtree of a larger document (small, so the
         # string round trip is cheap): serialize the working document, build
         # the enveloped signature, sign, and reparse so downstream pipes see a
         # standalone pyuppsala element.

src/pyff/test/test_pipeline.py

@@ -2,7 +2,9 @@
 import os
 import shutil
 import tempfile
+import threading
 import time
+import weakref
 from concurrent.futures import ThreadPoolExecutor
 from io import StringIO
 from unittest.mock import MagicMock, patch
@@ -377,9 +379,102 @@ class TestPkcs11SigningConfig:
 
     @pytest.fixture(autouse=True)
     def _isolated_provider_cache(self):
-        with patch.object(builtins, '_pkcs11_provider_cache', {}):
+        with (
+            patch.object(builtins, '_pkcs11_provider_cache', {}),
+            patch.object(builtins, '_pkcs11_signing_locks', {}),
+        ):
             yield
 
+    @pytest.mark.parametrize('failure_stage', ['signer', 'sign'])
+    def test_signing_failure_releases_session_with_retained_traceback(self, failure_stage):
+        """A retained error must not keep a token logged in on the next request."""
+        sessions = []
+
+        class Session:
+            pass
+
+        class Provider:
+            slot_id = 7
+
+            def open_session(self, _pin):
+                assert all(reference() is None for reference in sessions)
+                session = Session()
+                sessions.append(weakref.ref(session))
+                return session
+
+        class Signer:
+            def __init__(self, session, _label, _algorithm):
+                self.session = session
+
+        class Context:
+            def __init__(self, _manager):
+                pass
+
+            def set_hsm_signer(self, signer):
+                self.signer = signer
+
+        errors = []
+        # Native call argument errors model a PyO3 exception without adding a
+        # Python mock traceback frame that independently retains its arguments.
+        signer_type = object if failure_stage == 'signer' else Signer
+        with (
+            patch.object(builtins.pybergshamra, 'Pkcs11Signer', signer_type),
+            patch.object(builtins.pybergshamra, 'DsigContext', Context),
+        ):
+            for _ in range(2):
+                try:
+                    builtins._sign_pkcs11(Provider(), '/module.so', 'pin', 'key', '<root/>')
+                except TypeError as error:
+                    errors.append(error)
+                else:
+                    pytest.fail('Expected a native argument error')
+                assert all(reference() is None for reference in sessions)
+        assert len(errors) == len(sessions) == 2
+
+    def test_serializes_login_across_providers_for_same_token(self):
+        """Aliases for one token cannot overlap authenticated signing lifetimes."""
+        sessions = []
+        start = threading.Barrier(2)
+
+        class Session:
+            pass
+
+        class Provider:
+            slot_id = 7
+
+            def open_session(self, _pin):
+                assert all(reference() is None for reference in sessions), 'token already logged in'
+                session = Session()
+                sessions.append(weakref.ref(session))
+                time.sleep(0.05)
+                return session
+
+        class Signer:
+            def __init__(self, session, _label, _algorithm):
+                self.session = session
+
+        class Context:
+            def __init__(self, _manager):
+                pass
+
+            def set_hsm_signer(self, signer):
+                self.signer = signer
+
+        def sign(provider):
+            start.wait(timeout=5)
+            return builtins._sign_pkcs11(provider, '/module.so', 'pin', 'key', '<root/>')
+
+        with (
+            patch.object(builtins.pybergshamra, 'Pkcs11Signer', Signer),
+            patch.object(builtins.pybergshamra, 'DsigContext', Context),
+            patch.object(builtins.pybergshamra, 'sign_enveloped', lambda _ctx, xml, **_kwargs: xml),
+            ThreadPoolExecutor(max_workers=2) as executor,
+        ):
+            results = list(executor.map(sign, [Provider(), Provider()]))
+        assert results == ['<root/>', '<root/>']
+        assert len(sessions) == 2
+        assert all(reference() is None for reference in sessions)
+
     def test_parse_absolute_module_path(self):
         """Preserve an absolute module path and read the PIN environment selector."""
         module, label, pin, slot_id = builtins._parse_pkcs11_key_uri(

src/pyff/utils.py

@@ -317,9 +317,8 @@ def check_signature(t: ElementTree, key: Optional[str], only_one_signature: bool
 
     relt = root(t)
     if is_document_root(relt):
-        # Verify directly against the shared native DOM (pyuppsala document
-        # handle): no serialization of the working tree and no re-parse
-        # inside the verifier. Only valid when ``relt`` *is* the document --
+        # The document API verifies through an owned XML handoff.
+        # Only valid when ``relt`` *is* the document --
         # for a subtree the native document would be the enclosing aggregate.
         result = pybergshamra.verify_document(ctx, etree.native_document(relt))
     else: