diff --git a/docs/playbook_optimizer_assistant_backends.md b/docs/playbook_optimizer_assistant_backends.md index 89508cd36..95b5f8762 100644 --- a/docs/playbook_optimizer_assistant_backends.md +++ b/docs/playbook_optimizer_assistant_backends.md @@ -55,7 +55,9 @@ The "assistant backend" is the thing that turns *(messages, playbooks)* into the │ ) │ │ │ │ │ │ 7. if best passes commit thresholds: │ │ -│ _commit_if_allowed → archive incumbent, save successor │ │ +│ agent target → _commit_if_allowed │ │ +│ user target → stage proof + projection, then atomically │ │ +│ replace incumbent with one successor │ │ └───────────────────────────────────────────────────────────────────────┼──────────────┘ │ │ for each @@ -262,7 +264,15 @@ Useful as a smoke test — confirms the optimizer end-to-end without needing an ## 5. How a backend is selected at runtime -Selection happens once per `optimize()` call inside `PlaybookOptimizer._create_assistant`: +Before selecting a backend, `optimize()` applies the feature, target-kind, and +adoption kill switches. For user-playbook targets it then attempts to recover +any expired staged publication. Recovery uses the already durable candidate, +evaluation digests, decision proof, and search projection; it does not rerun +GEPA or require an assistant backend. A disabled kill switch prevents both a +new run and publication recovery. + +For a new optimization run, backend selection happens once per `optimize()` +call inside `PlaybookOptimizer._create_assistant`: ```python def _create_assistant(self, config) -> AssistantCallable | None: @@ -280,7 +290,11 @@ def _create_assistant(self, config) -> AssistantCallable | None: | Both set | **Rejected at config load time** by a Pydantic validator (`ValueError: Configure only one playbook optimizer assistant backend...`) | | Neither set | Optimizer logs `Skipping playbook optimization: no assistant backend configured` and returns without creating a job | -The backend check happens *before* loading the incumbent or resolving scenario windows, so an unconfigured optimizer short-circuits cheaply. After source-window resolution, the optimizer also skips before job creation when the validation holdout is smaller than `min_commit_windows` or the target kind has no enabled auto-update path. +The backend check happens *before* loading the incumbent or resolving scenario +windows, so an unconfigured optimizer short-circuits cheaply when there is no +recoverable publication. After source-window resolution, the optimizer also +skips before job creation when the validation holdout is smaller than +`min_commit_windows`. --- @@ -362,13 +376,13 @@ playbook_optimizer_config: ## 7. End-to-end workflow -What happens, in order, when a new agent playbook is generated and the optimizer is enabled: +What happens, in order, when a playbook is scheduled and the optimizer is enabled: | Step | Component | What it does | |---|---|---| | 1 | `PlaybookAggregator._enqueue_playbook_optimization` | After saving new agent playbooks, enqueues each PENDING playbook with the scheduler. | | 2 | `PlaybookOptimizationScheduler` | Debounces by `(org_id, kind, target_id)`, fires after a small jitter, spawns a daemon thread. | -| 3 | `PlaybookOptimizer.optimize(target)` | Loads config, calls `_create_assistant` → backend instance (or returns early). | +| 3 | `PlaybookOptimizer.optimize(target)` | Applies feature, target, and adoption gates. For user targets, resumes an expired staged publication before starting new search work. | | 4 | `ScenarioResolver` | Builds `ScenarioWindow`s from snapshotted agent source windows or a user playbook's `source_interaction_ids`. | | 5 | `gepa.api.optimize(...)` | Runs the candidate-search loop. | | 6 | `ReflexioPlaybookGEPAAdapter.evaluate` | For each `(candidate, window)`: | @@ -378,9 +392,12 @@ What happens, in order, when a new agent playbook is generated and the optimizer | | | • Persists `PlaybookOptimizationEvaluation` row | | 7 | Failure handling | Any `AssistantFailedError` → `verdict="aborted"`, `score=0.0`, GEPA continues. | | 8 | `PlaybookOptimizer._passes_commit_thresholds` | Checks score / likert / per-window verdict counts. | -| 9 | `PlaybookOptimizer._commit_if_allowed` | If gates pass → archive incumbent, save successor playbook. | +| 9a | Agent-playbook publication | `_commit_if_allowed` archives the pending incumbent and saves its successor. | +| 9b | User-playbook publication | Builds a fixed decision proof and search projection, stages them durably, then atomically replaces the current user playbook with exactly one successor. The compact proof binds each full durable evaluation through a canonical digest rather than duplicating rollout documents. | +| 10 | User-playbook recovery | A later enabled run may reclaim an expired publication lease and finish from the staged bytes. Incumbent changes produce `incumbent_changed`; retries do not rerun GEPA or regenerate the projection. | -The same flow runs for user playbooks, gated by `optimize_user_playbooks`. +User-playbook recovery and new runs both require `enabled`, +`optimize_user_playbooks`, and `auto_update_user_playbooks`. --- diff --git a/reflexio/cli/commands/status_cmd.py b/reflexio/cli/commands/status_cmd.py index da9d6f479..e005a8c2c 100644 --- a/reflexio/cli/commands/status_cmd.py +++ b/reflexio/cli/commands/status_cmd.py @@ -50,15 +50,18 @@ def check( render_error(err, json_mode=json_mode) raise SystemExit(1) from exc except requests.HTTPError as exc: + response = exc.response if json_mode: envelope = { "ok": False, "error": {"type": "unhealthy", "message": str(exc), "url": url}, } print(json.dumps(envelope, indent=2), file=sys.stderr) + elif response is None: + print(f"Error: Server at {url} returned an HTTP error", file=sys.stderr) else: print( - f"Error: Server at {url} returned {exc.response.status_code}", + f"Error: Server at {url} returned {response.status_code}", file=sys.stderr, ) raise SystemExit(1) from exc diff --git a/reflexio/cli/errors.py b/reflexio/cli/errors.py index ce0487e50..29c75b5d2 100644 --- a/reflexio/cli/errors.py +++ b/reflexio/cli/errors.py @@ -63,8 +63,9 @@ def _classify_http_error(exc: requests.HTTPError) -> CliError: ) if status == 422: detail = "" - with contextlib.suppress(ValueError, AttributeError): - detail = exc.response.json().get("detail", "") + if exc.response is not None: + with contextlib.suppress(ValueError, AttributeError): + detail = exc.response.json().get("detail", "") return CliError( error_type="validation", message=f"Invalid request: {detail or exc}", diff --git a/reflexio/cli/utils.py b/reflexio/cli/utils.py index 0c0ad99a0..27d32ca04 100644 --- a/reflexio/cli/utils.py +++ b/reflexio/cli/utils.py @@ -71,21 +71,23 @@ def _pids_from_lsof(port: int) -> list[int]: check=False, ) except FileNotFoundError: - return [] - if result.returncode != 0 or not result.stdout.strip(): - return [] + result = None pids: set[int] = set() - current_pid: int | None = None - suffix = f":{port}" - for line in result.stdout.splitlines(): - if line.startswith("p") and line[1:].isdigit(): - current_pid = int(line[1:]) - elif line.startswith("n") and current_pid is not None: - name = line[1:] - if "->" not in name and name.endswith(suffix): - pids.add(current_pid) - return sorted(pids) + if result is not None and result.returncode == 0: + current_pid: int | None = None + suffix = f":{port}" + for line in result.stdout.splitlines(): + if line.startswith("p") and line[1:].isdigit(): + current_pid = int(line[1:]) + elif line.startswith("n") and current_pid is not None: + name = line[1:] + if "->" not in name and name.endswith(suffix): + pids.add(current_pid) + if pids: + return sorted(pids) + + return [] def _pids_from_ss(port: int) -> list[int]: diff --git a/reflexio/client/__init__.py b/reflexio/client/__init__.py index facbfa857..9d3db3ace 100644 --- a/reflexio/client/__init__.py +++ b/reflexio/client/__init__.py @@ -1,3 +1,3 @@ -from .client import ReflexioClient +from .client import ConfigResponse, OfflineTunerConfigResponse, ReflexioClient -__all__ = ["ReflexioClient"] +__all__ = ["ConfigResponse", "OfflineTunerConfigResponse", "ReflexioClient"] diff --git a/reflexio/client/client.py b/reflexio/client/client.py index a8c8bc3bf..fb8331844 100644 --- a/reflexio/client/client.py +++ b/reflexio/client/client.py @@ -12,7 +12,7 @@ import aiohttp import requests -from pydantic import ConfigDict +from pydantic import BaseModel, ConfigDict from reflexio.defaults import DEFAULT_AGENT_VERSION from reflexio.models.api_schema.eval_overview_schema import ( @@ -135,7 +135,12 @@ from .cache import InMemoryCache -class _ClientConfigPayload(Config): +class OfflineTunerConfigResponse(BaseModel): + enabled: bool + + +class ConfigResponse(Config): + offline_tuner_config: OfflineTunerConfigResponse | None = None model_config = ConfigDict(extra="allow") @@ -1467,10 +1472,16 @@ def set_config(self, config: Config | dict) -> dict: Returns: dict: Response containing success status and message """ - config = self._convert_to_model( # type: ignore[reportAssignmentType] - config, - _ClientConfigPayload, - ) + if isinstance(config, Config): + config = Config.model_validate( + config.model_dump(mode="python"), + extra="ignore", + ) + else: + config = self._convert_to_model( # type: ignore[reportAssignmentType] + config, + Config, + ) return self._make_request( "POST", "/api/set_config", @@ -1506,17 +1517,17 @@ def update_config(self, partial: dict) -> dict: ) return self._make_request("POST", "/api/update_config", json=partial) - def get_config(self) -> Config: + def get_config(self) -> ConfigResponse: """Get configuration for the organization. Returns: - Config: The current configuration + ConfigResponse: The current configuration, including response overlays. """ response = self._make_request( "GET", "/api/get_config", ) - return _ClientConfigPayload(**response) + return ConfigResponse(**response) def invalidate_cache(self, org_id: str | None = None) -> dict: """Explicitly evict the server-side per-org Reflexio cache entry. diff --git a/reflexio/integrations/openclaw/plugin/tests/test_publish.py b/reflexio/integrations/openclaw/plugin/tests/test_publish.py index 9dc5a7978..21697c0bf 100644 --- a/reflexio/integrations/openclaw/plugin/tests/test_publish.py +++ b/reflexio/integrations/openclaw/plugin/tests/test_publish.py @@ -3,9 +3,12 @@ from __future__ import annotations import json +from collections.abc import Sequence +from typing import Any import pytest from openclaw_smart import publish, state +from openclaw_smart.reflexio_adapter import Adapter @pytest.fixture(autouse=True) @@ -15,11 +18,21 @@ def isolate_state_dir(monkeypatch, tmp_path): return sessions -class _Adapter: +class _Adapter(Adapter): def __init__(self) -> None: + super().__init__() self.calls = 0 - def publish(self, **_kwargs) -> bool: # noqa: ANN003 + def publish( + self, + *, + session_id: str, + project_id: str, + interactions: Sequence[dict[str, Any]], + force_extraction: bool = False, + skip_aggregation: bool = False, + ) -> bool: + del session_id, project_id, interactions, force_extraction, skip_aggregation self.calls += 1 return True diff --git a/reflexio/lib/_base.py b/reflexio/lib/_base.py index 2561d51d8..7af1d3a7f 100644 --- a/reflexio/lib/_base.py +++ b/reflexio/lib/_base.py @@ -14,10 +14,7 @@ from reflexio.models.config_schema import SearchMode from reflexio.server.api_endpoints.request_context import RequestContext from reflexio.server.llm.litellm_client import LiteLLMClient, LiteLLMConfig -from reflexio.server.llm.model_defaults import ( - ModelRole, - resolve_model_name, -) +from reflexio.server.llm.model_defaults import ModelRole, resolve_model_name from reflexio.server.services.configurator.base_configurator import BaseConfigurator from reflexio.server.services.storage.storage_base import BaseStorage from reflexio.server.site_var.site_var_manager import SiteVarManager @@ -31,6 +28,31 @@ ) +def create_generation_litellm_client( + request_context: RequestContext, +) -> LiteLLMClient: + """Build the tenant client used by normal generation and aggregation.""" + model_setting = SiteVarManager().get_site_var("llm_model_setting") + site_var = model_setting if isinstance(model_setting, dict) else {} + config = request_context.configurator.get_config() + api_key_config = config.api_key_config if config else None + config_llm_config = config.llm_config if config else None + generation_model_name = resolve_model_name( + ModelRole.GENERATION, + site_var_value=site_var.get("default_generation_model_name"), + config_override=( + config_llm_config.generation_model_name if config_llm_config else None + ), + api_key_config=api_key_config, + ) + return LiteLLMClient( + LiteLLMConfig( + model=generation_model_name, + api_key_config=api_key_config, + ) + ) + + def _require_storage[T: BaseModel]( response_type: type[T], *, msg_field: str = "message" ) -> Callable[..., Callable[..., T]]: @@ -88,29 +110,7 @@ def __init__( org_id=org_id, storage_base_dir=storage_base_dir, configurator=configurator ) - # Create single LLM client for all services - model_setting = SiteVarManager().get_site_var("llm_model_setting") - site_var = model_setting if isinstance(model_setting, dict) else {} - - # Get API key config and LLM config from configuration if available - config = self.request_context.configurator.get_config() - api_key_config = config.api_key_config if config else None - config_llm_config = config.llm_config if config else None - - generation_model_name = resolve_model_name( - ModelRole.GENERATION, - site_var_value=site_var.get("default_generation_model_name"), - config_override=config_llm_config.generation_model_name - if config_llm_config - else None, - api_key_config=api_key_config, - ) - - llm_config = LiteLLMConfig( - model=generation_model_name, - api_key_config=api_key_config, - ) - self.llm_client = LiteLLMClient(llm_config) + self.llm_client = create_generation_litellm_client(self.request_context) def _is_storage_configured(self) -> bool: """Check if storage is configured and available. diff --git a/reflexio/lib/_config.py b/reflexio/lib/_config.py index c0cf0bcdd..27a87c977 100644 --- a/reflexio/lib/_config.py +++ b/reflexio/lib/_config.py @@ -3,6 +3,9 @@ from reflexio.lib._base import ReflexioBase from reflexio.models.api_schema.retriever_schema import SetConfigResponse from reflexio.models.config_schema import Config, StorageConfigManagedSupabase +from reflexio.server.services.configurator.config_storage import ( + ConfigWriteConflictError, +) class ConfigMixin(ReflexioBase): @@ -18,8 +21,12 @@ def set_config(self, config: Config | dict) -> SetConfigResponse: try: configurator = self.request_context.configurator if isinstance(config, dict): - config = configurator.normalize_config_payload(config) - config = Config(**config) + normalized = configurator.normalize_config_payload(config) + config = ( + normalized + if isinstance(normalized, Config) + else Config(**normalized) + ) # Validate storage connection before setting config. # If no storage_config provided, or the caller round-tripped the @@ -65,6 +72,8 @@ def set_config(self, config: Config | dict) -> SetConfigResponse: configurator.set_config(config) return SetConfigResponse(success=True, msg="Configuration set successfully") + except ConfigWriteConflictError: + raise except Exception as e: return SetConfigResponse( success=False, msg=f"Failed to set configuration: {str(e)}" diff --git a/reflexio/lib/generation_client.py b/reflexio/lib/generation_client.py new file mode 100644 index 000000000..8639698bb --- /dev/null +++ b/reflexio/lib/generation_client.py @@ -0,0 +1,5 @@ +"""Public factory for the tenant generation client.""" + +from reflexio.lib._base import create_generation_litellm_client + +__all__ = ["create_generation_litellm_client"] diff --git a/reflexio/models/api_schema/domain/entities.py b/reflexio/models/api_schema/domain/entities.py index ba05999be..1e736e0a3 100644 --- a/reflexio/models/api_schema/domain/entities.py +++ b/reflexio/models/api_schema/domain/entities.py @@ -2,6 +2,7 @@ import json from datetime import UTC, datetime +from hashlib import sha256 from typing import Any, Final, Literal, Self from pydantic import ( @@ -107,7 +108,13 @@ "AgentPlaybookUpdateEntry", "PlaybookAggregationChangeLog", "PlaybookAggregationChangeLogResponse", + "OptimizerKind", + "OptimizationJobStage", + "OptimizationTerminalOutcome", + "OptimizationArtifactKind", + "OptimizationJobClaim", "PlaybookOptimizationJob", + "PlaybookOptimizationArtifact", "PlaybookOptimizationCandidate", "PlaybookOptimizationEvaluation", "PlaybookOptimizationEvent", @@ -145,6 +152,25 @@ "LearningStatusResponse", ] + +def canonicalize_artifact_json(content_json: str) -> str: + """Validate and serialize durable artifact content using the proof contract.""" + try: + value = json.loads( + content_json, + parse_constant=lambda constant: (_ for _ in ()).throw( + ValueError(f"invalid JSON constant: {constant}") + ), + ) + # Imported lazily because publication's contracts reference OptimizerKind + # from this module while defining the shared RFC 8785 encoder. + from reflexio.server.services.playbook.publication import canonical_json_bytes + + return canonical_json_bytes(value).decode("utf-8") + except (TypeError, ValueError, json.JSONDecodeError, UnicodeError) as exc: + raise ValueError("artifact content_json must be valid JSON") from exc + + # =============================== # Data Models # =============================== @@ -348,6 +374,63 @@ class AgentPlaybook(BaseModel): superseded_by: int | None = None +OptimizerKind = Literal[ + "gepa", + "offline_tuner_replay", + "offline_tuner_legacy", + "optimizer_legacy_unknown", +] + +OptimizationJobStage = Literal[ + "evidence_frozen", + "candidate_generated", + "replay_running", + "replay_evaluated", + "publishing", + "applied", + "abstained", + "failed", +] + +OptimizationTerminalOutcome = Literal[ + "applied", + "insufficient_negative_evidence", + "insufficient_positive_evidence", + "insufficient_coverage", + "replay_unsupported", + "deployment_unsupported", + "incomplete_replay_scope", + "insufficient_replay_cases", + "replay_inconclusive", + "candidate_regressed", + "candidate_did_not_improve", + "incumbent_changed", + "generation_failed", + "replay_failed", + "publication_failed", + "governance_erased", +] + +OptimizationArtifactKind = Literal[ + "expected_population_manifest", + "generation_selection", + "replay_manifest", + "candidate", + "candidate_search_projection", +] + +Sha256Digest = str + + +class OptimizationJobClaim(BaseModel): + """One renewable optimizer lease identified by a monotonic fence.""" + + job_id: int + owner: str + fence: int = Field(ge=1) + expires_at: int + + class PlaybookOptimizationJob(BaseModel): """One end-to-end optimizer run for a single playbook target. @@ -357,6 +440,7 @@ class PlaybookOptimizationJob(BaseModel): """ job_id: int = 0 + optimizer_kind: OptimizerKind = "optimizer_legacy_unknown" target_kind: Literal["agent_playbook", "user_playbook"] target_id: int status: Literal["pending", "running", "completed", "skipped", "failed"] = "pending" @@ -364,9 +448,69 @@ class PlaybookOptimizationJob(BaseModel): successor_target_id: int | None = None decision_reason: str = "" metadata_json: str = "{}" + discovery_key: str | None = None + attempt_key: str | None = None + lease_owner: str | None = None + lease_fence: int = Field(default=0, ge=0) + lease_expires_at: int | None = None + stage: OptimizationJobStage | None = None + terminal_outcome: OptimizationTerminalOutcome | None = None + expected_population_manifest_digest: Sha256Digest | None = None + generation_selection_manifest_digest: Sha256Digest | None = None + replay_manifest_digest: Sha256Digest | None = None + candidate_content_digest: Sha256Digest | None = None + search_projection_digest: Sha256Digest | None = None + publication_scope_digest: Sha256Digest | None = None created_at: int = Field(default_factory=lambda: int(datetime.now(UTC).timestamp())) updated_at: int = Field(default_factory=lambda: int(datetime.now(UTC).timestamp())) + @field_validator( + "expected_population_manifest_digest", + "generation_selection_manifest_digest", + "replay_manifest_digest", + "candidate_content_digest", + "search_projection_digest", + "publication_scope_digest", + ) + @classmethod + def validate_sha256_digest(cls, value: str | None) -> str | None: + if value is not None and ( + len(value) != 64 or any(char not in "0123456789abcdef" for char in value) + ): + raise ValueError("optimizer proof digests must be lowercase SHA-256 hex") + return value + + +class PlaybookOptimizationArtifact(BaseModel): + """One typed, content-bearing singleton artifact owned by an optimizer job.""" + + artifact_id: int = 0 + job_id: int + artifact_kind: OptimizationArtifactKind + content_json: str + content_digest: Sha256Digest + created_at: int = Field(default_factory=lambda: int(datetime.now(UTC).timestamp())) + updated_at: int = Field(default_factory=lambda: int(datetime.now(UTC).timestamp())) + + @field_validator("content_json") + @classmethod + def canonicalize_content_json(cls, value: str) -> str: + return canonicalize_artifact_json(value) + + @field_validator("content_digest") + @classmethod + def validate_content_digest(cls, value: str) -> str: + if len(value) != 64 or any(char not in "0123456789abcdef" for char in value): + raise ValueError("artifact digest must be lowercase SHA-256 hex") + return value + + @model_validator(mode="after") + def validate_content_digest_matches_content(self) -> Self: + expected = sha256(self.content_json.encode()).hexdigest() + if self.content_digest != expected: + raise ValueError("artifact digest must match canonical content_json") + return self + class PlaybookOptimizationCandidate(BaseModel): """A playbook content variant proposed by GEPA during a job. diff --git a/reflexio/server/llm/_litellm_text_generation.py b/reflexio/server/llm/_litellm_text_generation.py index f76f442a1..24d18c568 100644 --- a/reflexio/server/llm/_litellm_text_generation.py +++ b/reflexio/server/llm/_litellm_text_generation.py @@ -176,6 +176,16 @@ def _rung_reason(error: Exception | None) -> str: return "transport_error" +class ProviderRequestGuardError(RuntimeError): + """Raised when a caller rejects built params at the provider boundary.""" + + +ProviderRequestGuard = Callable[ + [dict[str, Any], float, tuple[str, ...], bool], + None, +] + + class TextGenerationMixin: """Chat/response generation, completion-param build, hard-timeout, cost, multimodal. @@ -309,6 +319,7 @@ def generate_chat_response( max_retries: int | None = None, fallback_models: list[str] | None = None, structured_output_validator: StructuredOutputValidator | None = None, + provider_request_guard: ProviderRequestGuard | None = None, **kwargs: Any, ) -> str | BaseModel | ToolCallingChatResponse: """ @@ -333,6 +344,8 @@ def generate_chat_response( structured_output_validator: Optional semantic validator for parsed structured output. Passing one opts the call into the corrective repair ladder for parse, blank, and semantic failures. + provider_request_guard: Optional internal guard invoked on the exact + built params immediately before each provider attempt. **kwargs: Additional parameters including: - response_format: Pydantic BaseModel class for structured output - parse_structured_output: Whether to parse structured output (default True) @@ -411,6 +424,8 @@ def generate_chat_response_with_provenance( kwargs["fallback_models"] = fallback_models if structured_output_validator is not None: kwargs["structured_output_validator"] = structured_output_validator + if provider_request_guard is not None: + kwargs["provider_request_guard"] = provider_request_guard return self._make_request(final_messages, **kwargs) @@ -1038,6 +1053,9 @@ def _make_request( # noqa: C901 structured_output_validator: StructuredOutputValidator | None = kwargs.pop( "structured_output_validator", None ) + provider_request_guard: ProviderRequestGuard | None = kwargs.pop( + "provider_request_guard", None + ) original_kwargs = dict(kwargs) def _finish( @@ -1107,6 +1125,13 @@ def _call_and_parse( ) try: with provider_slot(turn_params["model"]): + if provider_request_guard is not None: + provider_request_guard( + turn_params, + turn_hard_timeout, + tuple(ladder), + turn_parse_structured_output, + ) response = self._completion_with_hard_timeout( turn_params, turn_hard_timeout ) @@ -1191,6 +1216,8 @@ def _call_and_parse( model=str(turn_params.get("model")), provenance=provenance, ) + except ProviderRequestGuardError: + raise except ( StructuredOutputParseError, StructuredOutputRepairError, diff --git a/reflexio/server/prompt/prompt_manager.py b/reflexio/server/prompt/prompt_manager.py index 1b201ab11..8ec970ed4 100644 --- a/reflexio/server/prompt/prompt_manager.py +++ b/reflexio/server/prompt/prompt_manager.py @@ -4,7 +4,8 @@ import logging import re -from collections.abc import Sequence +from collections.abc import Mapping, Sequence +from hashlib import sha256 from pathlib import Path from typing import Any @@ -133,6 +134,37 @@ def render_prompt(self, prompt_id: str, variables: dict[str, Any]) -> str: if not prompt: raise ValueError(f"Prompt {prompt_id} not found") + return self._render_prompt(prompt_id, prompt, variables) + + def render_prompt_from_identity( + self, + prompt_id: str, + variables: dict[str, Any], + identity: Mapping[str, str], + ) -> str: + """Render the exact prompt template bound by a frozen identity.""" + if identity.get("prompt_id") != prompt_id: + raise ValueError(f"Frozen prompt identity does not match {prompt_id}") + version = identity.get("active_version") + expected_digest = identity.get("template_content_digest") + if not version or not expected_digest: + raise ValueError(f"Frozen prompt identity for {prompt_id} is incomplete") + prompt = self._load_prompt(prompt_id, version) + if prompt is None: + raise ValueError(f"Prompt {prompt_id} v{version} not found") + actual_digest = sha256(prompt.content.encode("utf-8")).hexdigest() + if actual_digest != expected_digest: + raise ValueError(f"Frozen prompt template drifted for {prompt_id}") + return self._render_prompt(prompt_id, prompt, variables) + + @staticmethod + def _render_prompt( + prompt_id: str, + prompt: Prompt, + variables: dict[str, Any], + ) -> str: + """Render an already resolved prompt template.""" + missing_vars = set(prompt.variables) - set(variables.keys()) if missing_vars: raise ValueError( @@ -180,6 +212,27 @@ def get_active_version(self, prompt_id: str) -> str | None: return self.version_override[prompt_id] return self._find_active_version(prompt_id) + def get_prompt_template_identity(self, prompt_id: str) -> dict[str, str]: + """Return identity for the active unrendered prompt template. + + This deliberately hashes ``Prompt.content`` before any variable + rendering so callers can bind prompt behavior without persisting + user/session inputs. + """ + active_version = self.get_active_version(prompt_id) + if active_version is None: + raise ValueError(f"Prompt {prompt_id} has no active version") + prompt = self._get_prompt(prompt_id, active_version) + if prompt is None: + raise ValueError(f"Prompt {prompt_id} v{active_version} not found") + return { + "active_version": active_version, + "prompt_id": prompt_id, + "template_content_digest": sha256( + prompt.content.encode("utf-8") + ).hexdigest(), + } + def get_all_prompt_ids(self) -> list[str]: """ Get list of all available prompt IDs. diff --git a/reflexio/server/routes/config.py b/reflexio/server/routes/config.py index d02362792..8164bd8ea 100644 --- a/reflexio/server/routes/config.py +++ b/reflexio/server/routes/config.py @@ -31,6 +31,9 @@ ) from reflexio.server.cache import reflexio_cache from reflexio.server.rate_limit import limiter +from reflexio.server.services.configurator.config_storage import ( + ConfigWriteConflictError, +) logger = logging.getLogger(__name__) router = APIRouter() @@ -94,8 +97,6 @@ def set_config( configurator = reflexio.request_context.configurator try: normalized_config = configurator.normalize_config_payload(config) - if not isinstance(normalized_config, dict): - normalized_config = config Config.model_validate(normalized_config) except ValidationError as exc: raise HTTPException( @@ -104,7 +105,16 @@ def set_config( ) from exc # Set the config using Reflexio's set_config method - response = reflexio.set_config(normalized_config) + try: + response = reflexio.set_config(normalized_config) + except ConfigWriteConflictError as exc: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "error": "config_write_conflict", + "message": str(exc), + }, + ) from exc # Invalidate cache on successful config change to ensure fresh instance next request if response.success: @@ -123,8 +133,10 @@ def update_config( """Apply a partial update to the org's config (PATCH semantics). Performs a **top-level shallow merge** of *partial* over the existing - config and round-trips through ``Config(**merged)`` so Pydantic - validates the result and rejects bogus top-level fields. + config and delegates normalization to the active configurator before the + shared ``Config`` validation. This lets deployment-specific configurators + consume their overlay fields while the default configurator still rejects + unknown fields. .. warning:: Nested objects (e.g. ``storage_config``, ``profile_extractor_config``, @@ -156,17 +168,15 @@ def update_config( from pydantic import ValidationError reflexio = reflexio_cache.get_reflexio(org_id=org_id) - existing_config = reflexio.request_context.configurator.get_config() + configurator = reflexio.request_context.configurator + existing_config = configurator.get_config() existing = existing_config.model_dump(mode="python") - merged = {**existing, **partial} - # Pydantic validates the merged shape and rejects unknown / malformed - # fields here, before storage validation in reflexio.set_config. # Convert ValidationError into 422 so callers passing a partial that # would replace a nested extractor object with an incomplete dict (e.g. # {"user_playbook_extractor_config": {"aggregation_config": {...}}}) # get a clean client-error response instead of a 500. try: - merged_config = Config(**merged) + merged_config = configurator.prepare_config_patch(partial) except ValidationError as exc: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, @@ -182,11 +192,27 @@ def update_config( "validation_errors": exc.errors(), }, ) from exc - if merged_config.model_dump(mode="python") == existing: + partial_uses_only_shared_fields = ( + partial.keys() <= type(existing_config).model_fields.keys() + ) + if ( + partial_uses_only_shared_fields + and merged_config.model_dump(mode="python") == existing + and getattr(configurator, "requires_durable_config_patch", False) is not True + ): logger.info("Skipping no-op config update for org %s", org_id) return SetConfigResponse(success=True, msg="Configuration unchanged") - response = reflexio.set_config(merged_config) + try: + response = reflexio.set_config(merged_config) + except ConfigWriteConflictError as exc: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "error": "config_write_conflict", + "message": str(exc), + }, + ) from exc if response.success: reflexio_cache.invalidate_reflexio_cache(org_id=org_id) return response diff --git a/reflexio/server/services/configurator/base_configurator.py b/reflexio/server/services/configurator/base_configurator.py index bddfd94d6..1f7d6c52f 100644 --- a/reflexio/server/services/configurator/base_configurator.py +++ b/reflexio/server/services/configurator/base_configurator.py @@ -78,10 +78,26 @@ def get_config_for_response(self) -> dict[str, Any]: """Return config serialized for API responses.""" return self.config.model_dump(mode="json") - def normalize_config_payload(self, config: dict[str, Any]) -> dict[str, Any]: + def normalize_config_payload( + self, config: dict[str, Any] + ) -> Config | dict[str, Any]: """Normalize raw API config payloads before Pydantic validation.""" return config + def prepare_config_patch(self, partial: dict[str, Any]) -> Config: + """Prepare a top-level PATCH payload for persistence. + + The shared behavior is intentionally shallow: nested config objects are + replaced wholesale by the caller's partial. + """ + existing = self.get_config().model_dump(mode="python") + normalized = self.normalize_config_payload({**existing, **partial}) + return ( + normalized + if isinstance(normalized, Config) + else Config.model_validate(normalized) + ) + def get_prompt_bank_paths(self) -> list[Path]: """Return additional prompt banks this configurator contributes.""" return [] diff --git a/reflexio/server/services/configurator/config_storage.py b/reflexio/server/services/configurator/config_storage.py index 40a27cda8..97a3ea063 100644 --- a/reflexio/server/services/configurator/config_storage.py +++ b/reflexio/server/services/configurator/config_storage.py @@ -4,6 +4,10 @@ from reflexio.models.config_schema import Config +class ConfigWriteConflictError(RuntimeError): + """Raised when a conditional config write loses a freshness race.""" + + class ConfigStorage(ABC): """ Abstract base class for configuration storage operations. diff --git a/reflexio/server/services/extractor_interaction_utils.py b/reflexio/server/services/extractor_interaction_utils.py index 815e31a79..ac5c29677 100644 --- a/reflexio/server/services/extractor_interaction_utils.py +++ b/reflexio/server/services/extractor_interaction_utils.py @@ -14,8 +14,8 @@ logger = logging.getLogger(__name__) -def get_extractor_window_params[TExtractorConfig]( - extractor_config: TExtractorConfig, +def get_extractor_window_params( + extractor_config: object, global_window_size: int | None, global_stride_size: int | None, ) -> tuple[int, int]: @@ -58,8 +58,8 @@ def get_extractor_window_params[TExtractorConfig]( return window_size, stride_size -def get_effective_source_filter[TExtractorConfig]( - extractor_config: TExtractorConfig, +def get_effective_source_filter( + extractor_config: object, triggering_source: str | None, ) -> tuple[bool, list[str] | None]: """ diff --git a/reflexio/server/services/generation_service.py b/reflexio/server/services/generation_service.py index 6bac7d79d..ef945bc41 100644 --- a/reflexio/server/services/generation_service.py +++ b/reflexio/server/services/generation_service.py @@ -1080,6 +1080,22 @@ def _schedule_group_evaluation_if_needed( session_id=session_id, evaluation_only=new_request.evaluation_only, ) + if self.storage is not None: + try: + self.storage.record_retrieved_learning_sampling_decision( + user_id=user_id, + session_id=session_id, + request_id=new_request.request_id, + sampled=run_retrieved_learning, + ) + except Exception: + logger.exception( + "Failed to persist retrieved-learning sampling decision for " + "request=%s session=%s user=%s", + sanitise_for_log(new_request.request_id), + session_id, + user_id, + ) if not (run_agent_success or run_retrieved_learning): logger.info( "Skipping group evaluation scheduling for unsampled session=%s user=%s", diff --git a/reflexio/server/services/governance/service.py b/reflexio/server/services/governance/service.py index 5b21faf53..fca64cdbf 100644 --- a/reflexio/server/services/governance/service.py +++ b/reflexio/server/services/governance/service.py @@ -1,7 +1,7 @@ from __future__ import annotations from contextlib import suppress -from typing import Any, Literal, TypedDict +from typing import Any, Literal, Protocol, TypedDict from reflexio.models.api_schema.domain.governance import ( AuditEvent, @@ -25,11 +25,16 @@ "agent_success_evaluation_result": "agent_success_evaluation_results", "retrieved_learning_evaluation_result": "retrieved_learning_evaluation_results", "evaluation_operation_state": "evaluation_operation_states", + "offline_tuner_reward_label": "offline_tuner_reward_labels", + "offline_tuner_reward_label_target_by_target_owner": ( + "offline_tuner_reward_label_targets_by_target_owner" + ), "profile_purge": "purged_profiles", "user_playbook_purge": "purged_user_playbooks", } _REQUIRED_DELETE_TARGET_NAMES = tuple(_DELETE_TARGET_NAME_TO_RESULT_KEY) _USER_PLAYBOOK_PAGE_SIZE = 1000 +_LIFECYCLE_COMPLETION_STATUS = "complete" class GovernanceActorContext(TypedDict): @@ -37,11 +42,31 @@ class GovernanceActorContext(TypedDict): actor_ref: str | None +class SubjectErasureLifecycle(Protocol): + """Deployment-specific erasure work that must precede barrier completion.""" + + def erase_subject( + self, + *, + storage: Any, + subject_ref: str, + purge_id: str, + ) -> None: ... + + class GovernanceService: - def __init__(self, *, storage: Any, org_id: str, ref_secret: str) -> None: + def __init__( + self, + *, + storage: Any, + org_id: str, + ref_secret: str, + subject_erasure_lifecycle: SubjectErasureLifecycle | None = None, + ) -> None: self.storage = storage self.org_id = org_id self.ref_secret = ref_secret + self.subject_erasure_lifecycle = subject_erasure_lifecycle def export_user( self, @@ -139,6 +164,16 @@ def erase_user( if not self._delete_targets_complete(purge_id): self.storage.apply_governance_user_data_delete(purge_id, user_id) + if ( + self.subject_erasure_lifecycle is not None + and not self._subject_erasure_lifecycle_complete(purge_id) + ): + self.subject_erasure_lifecycle.erase_subject( + storage=self.storage, + subject_ref=subref, + purge_id=purge_id, + ) + self._record_subject_erasure_lifecycle_complete(purge_id) deleted_counts = self._deleted_counts_from_targets(purge_id) rebuilt_agent_playbook_ids: list[int] = [] @@ -268,6 +303,45 @@ def _deleted_counts_from_targets(self, purge_id: str) -> dict[str, int]: counts[result_key] = int(target.deleted_count) return counts + def _subject_erasure_lifecycle_complete(self, purge_id: str) -> bool: + snapshot = self._prepared_target_snapshot(purge_id) + return bool( + snapshot is not None + and (snapshot.detail or {}).get("status") == _LIFECYCLE_COMPLETION_STATUS + ) + + def _record_subject_erasure_lifecycle_complete(self, purge_id: str) -> None: + snapshot = self._prepared_target_snapshot(purge_id) + if snapshot is None or snapshot.status != "complete": + raise ValueError( + "Subject erasure lifecycle requires a prepared target snapshot" + ) + detail = dict(snapshot.detail or {}) + detail["status"] = _LIFECYCLE_COMPLETION_STATUS + self.storage.record_purge_target( + purge_id=purge_id, + target_name="target_snapshot", + target_ref="all", + phase="prepare_targets", + status="complete", + detail=detail, + deleted_count=snapshot.deleted_count, + error_detail=snapshot.error_detail, + ) + + def _prepared_target_snapshot(self, purge_id: str) -> PurgeOperationTarget | None: + return next( + ( + target + for target in self.storage.list_purge_targets( + purge_id, phase="prepare_targets" + ) + if target.target_name == "target_snapshot" + and target.target_ref == "all" + ), + None, + ) + def _rebuilt_agent_playbook_ids_from_targets(self, purge_id: str) -> list[int]: return [ int(target.target_ref) diff --git a/reflexio/server/services/playbook/components/aggregator.py b/reflexio/server/services/playbook/components/aggregator.py index 8d7603f76..483797cb7 100644 --- a/reflexio/server/services/playbook/components/aggregator.py +++ b/reflexio/server/services/playbook/components/aggregator.py @@ -5,7 +5,8 @@ import time import uuid from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any +from contextlib import AbstractContextManager +from typing import TYPE_CHECKING, Any, Protocol if TYPE_CHECKING: import numpy as np @@ -56,6 +57,26 @@ logger = logging.getLogger(__name__) +class AggregationEffectCoordinator(Protocol): + """Optional managed boundary for one atomic aggregation effect.""" + + def prepare(self, playbooks: list[AgentPlaybook]) -> None: ... + + def apply_scope(self) -> AbstractContextManager[None]: ... + + def save_agent_playbook( + self, + playbook: AgentPlaybook, + *, + source_ids: list[str], + request_id: str, + run_mode: str, + provenance: ModelProvenance | None, + ) -> AgentPlaybook: ... + + def complete(self, result: dict[str, Any]) -> None: ... + + class PlaybookAggregator: def __init__( self, @@ -63,6 +84,7 @@ def __init__( request_context: RequestContext, agent_version: str, aggregation_prompt_processor: AggregationPromptProcessor | None = None, + effect_coordinator: AggregationEffectCoordinator | None = None, ) -> None: self.client = llm_client self.storage = request_context.storage @@ -70,6 +92,7 @@ def __init__( self.request_context = request_context self.agent_version = agent_version self.aggregation_prompt_processor = aggregation_prompt_processor + self.effect_coordinator = effect_coordinator # Cohesive pre/post-processing component (the enterprise redaction # Protocol seam). Constructed from the SAME injected instance stored # above — do NOT re-resolve the AGGREGATION_PROMPT_PROCESSOR ServiceKey. @@ -266,13 +289,32 @@ def run(self, playbook_aggregator_request: PlaybookAggregatorRequest) -> dict: """ aggregation_start = time.perf_counter() # Stable id for this aggregation run — groups all lineage events produced below. - _run_id = str(uuid.uuid4()) + _run_id = playbook_aggregator_request.operation_key or str(uuid.uuid4()) _empty_stats = { "clusters_found": 0, "user_playbooks_processed": 0, "playbooks_generated": 0, } + if ( + self.effect_coordinator is None + and playbook_aggregator_request.operation_key + and any( + event.op == "aggregate" + for event in self.storage.get_lineage_events( # type: ignore[reportOptionalMemberAccess] + request_id=playbook_aggregator_request.operation_key + ) + ) + ): + logger.info( + "Skipping aggregation operation %s because its effects already exist", + playbook_aggregator_request.operation_key, + ) + return { + **_empty_stats, + "skipped": "operation already applied", + } + # Singleton aggregation: one playbook kind per org. The name is a fixed # constant used only for bookmark/archive scoping and telemetry — it is # never a selection filter on the read queries below. @@ -436,8 +478,17 @@ def run(self, playbook_aggregator_request: PlaybookAggregatorRequest) -> dict: "No cluster changes detected for '%s', skipping LLM calls", playbook_name, ) - # Still update bookmark - self._update_operation_state(playbook_name, user_playbooks) + result = { + **_empty_stats, + "skipped": "no cluster changes detected", + } + if self.effect_coordinator is None: + self._update_operation_state(playbook_name, user_playbooks) + else: + self.effect_coordinator.prepare([]) + with self.effect_coordinator.apply_scope(): + self._update_operation_state(playbook_name, user_playbooks) + self.effect_coordinator.complete(result) record_usage_event( org_id=self.request_context.org_id, event_name="aggregation_succeeded", @@ -452,7 +503,7 @@ def run(self, playbook_aggregator_request: PlaybookAggregatorRequest) -> dict: ), metadata={"skipped": "no cluster changes detected"}, ) - return {**_empty_stats, "skipped": "no cluster changes detected"} + return result logger.info( "Detected %d changed clusters, %d playbooks to archive", @@ -460,6 +511,7 @@ def run(self, playbook_aggregator_request: PlaybookAggregatorRequest) -> dict: len(archived_playbook_ids), ) + effect_scope: AbstractContextManager[None] | None = None try: # Emit the started event inside the protected block so any failure # from here on is paired with an aggregation_failed event. @@ -480,6 +532,11 @@ def run(self, playbook_aggregator_request: PlaybookAggregatorRequest) -> dict: direction_overlap_threshold=playbook_aggregator_config.direction_overlap_threshold, ) new_playbooks = [playbook for playbook, _, _ in generated_pairs] + if self.effect_coordinator is not None: + self.effect_coordinator.prepare(new_playbooks) + pending_scope = self.effect_coordinator.apply_scope() + pending_scope.__enter__() + effect_scope = pending_scope previous_fingerprints_for_changed_clusters = {} changed_fps_by_previous_fp = {} @@ -568,20 +625,31 @@ def run(self, playbook_aggregator_request: PlaybookAggregatorRequest) -> dict: for fb in cluster_playbooks if fb.user_playbook_id ] - saved_fb = self.storage.save_agent_playbooks( # type: ignore[reportOptionalMemberAccess] - [playbook], - lineage_contexts=[ - LineageContext( - op_kind="aggregate", - actor="aggregator", - request_id=_run_id, - source_ids=member_ids, - reason=f"{AGGREGATE_REASON_PREFIX}{run_mode}", - model_name=provenance.model_name if provenance else None, - provider=provenance.provider if provenance else None, - ) - ], - )[0] + if self.effect_coordinator is None: + saved_fb = self.storage.save_agent_playbooks( # type: ignore[reportOptionalMemberAccess] + [playbook], + lineage_contexts=[ + LineageContext( + op_kind="aggregate", + actor="aggregator", + request_id=_run_id, + source_ids=member_ids, + reason=f"{AGGREGATE_REASON_PREFIX}{run_mode}", + model_name=( + provenance.model_name if provenance else None + ), + provider=provenance.provider if provenance else None, + ) + ], + )[0] + else: + saved_fb = self.effect_coordinator.save_agent_playbook( + playbook, + source_ids=member_ids, + request_id=_run_id, + run_mode=run_mode, + provenance=provenance, + ) saved_playbook_list.append(saved_fb) if saved_fb and saved_fb.agent_playbook_id: fp_key = self._compute_cluster_fingerprint(cluster_playbooks) @@ -719,14 +787,23 @@ def run(self, playbook_aggregator_request: PlaybookAggregatorRequest) -> dict: org_id=self.request_context.org_id, request_id=_run_id, ) - - self._enqueue_playbook_optimization(saved_playbook_list) + if self.effect_coordinator is not None: + raise stats = { "clusters_found": len(clusters), "user_playbooks_processed": len(user_playbooks), "playbooks_generated": len(saved_playbook_list), } + if self.effect_coordinator is not None: + self.effect_coordinator.complete(stats) + if effect_scope is None: + raise RuntimeError("aggregation effect scope was not entered") + completed_scope = effect_scope + effect_scope = None + completed_scope.__exit__(None, None, None) + + self._enqueue_playbook_optimization(saved_playbook_list) record_usage_event( org_id=self.request_context.org_id, event_name="aggregation_succeeded", @@ -753,6 +830,10 @@ def run(self, playbook_aggregator_request: PlaybookAggregatorRequest) -> dict: return stats except Exception as e: + if effect_scope is not None: + failed_scope = effect_scope + effect_scope = None + failed_scope.__exit__(type(e), e, e.__traceback__) record_usage_event( org_id=self.request_context.org_id, event_name="aggregation_failed", @@ -764,20 +845,26 @@ def run(self, playbook_aggregator_request: PlaybookAggregatorRequest) -> dict: duration_ms=int((time.perf_counter() - aggregation_start) * 1000), error_kind=type(e).__name__, ) - # Restore archived playbooks if any error occurs during aggregation - logger.error( - "Error during playbook aggregation for '%s': %s. Restoring archived playbooks.", - playbook_name, - str(e), - ) - if full_archive: - for name in full_archive_playbook_names: - self.storage.restore_archived_agent_playbooks_by_playbook_name( # type: ignore[reportOptionalMemberAccess] - name, agent_version=self.agent_version + if self.effect_coordinator is None: + logger.error( + "Error during playbook aggregation for '%s': %s. Restoring archived playbooks.", + playbook_name, + str(e), + ) + if full_archive: + for name in full_archive_playbook_names: + self.storage.restore_archived_agent_playbooks_by_playbook_name( # type: ignore[reportOptionalMemberAccess] + name, agent_version=self.agent_version + ) + elif archived_playbook_ids: + self.storage.restore_archived_agent_playbooks_by_ids( # type: ignore[reportOptionalMemberAccess] + archived_playbook_ids ) - elif archived_playbook_ids: - self.storage.restore_archived_agent_playbooks_by_ids( # type: ignore[reportOptionalMemberAccess] - archived_playbook_ids + else: + logger.error( + "Error during managed playbook aggregation for '%s': %s. The effect transaction was rolled back.", + playbook_name, + str(e), ) # Re-raise the exception after restoring raise diff --git a/reflexio/server/services/playbook/playbook_edit_apply.py b/reflexio/server/services/playbook/playbook_edit_apply.py deleted file mode 100644 index 7dbc784b0..000000000 --- a/reflexio/server/services/playbook/playbook_edit_apply.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Shared atomic supersede primitive for applying a playbook edit. - -Online and background playbook repair paths share one lifecycle -(insert-then-supersede, no orphan). -""" - -from typing import TYPE_CHECKING - -from reflexio.models.api_schema.domain.entities import LineageContext, UserPlaybook - -if TYPE_CHECKING: - from reflexio.server.services.storage.storage_base import BaseStorage - - -class _LostSupersedeRaceError(Exception): - """Internal signal used to roll back a provisional successor.""" - - -def apply_playbook_edit( - storage: "BaseStorage", - *, - incumbent_id: int, - new_playbook: UserPlaybook, - source: str, - request_id: str, - skip_embedding: bool = False, - revise_context: LineageContext | None = None, -) -> int: - """Insert a replacement playbook then atomically supersede the incumbent. - - Uses ``storage.supersede_record`` (atomic conditional CAS) so a lost race - never leaves an orphan CURRENT row: - - - Insert the new playbook as CURRENT. - - Call ``supersede_record(incumbent_id → new_id)``, which only succeeds when - the incumbent is still CURRENT (``status IS NULL``). - - If ``supersede_record`` returns ``False`` (incumbent already gone), roll - back the transaction and return ``-1``. - - Args: - storage: A BaseStorage instance providing ``save_user_playbooks``, - ``supersede_record``, and ``commit_scope``. - incumbent_id: ``user_playbook_id`` of the playbook being replaced. - new_playbook: The replacement playbook (inserted as CURRENT, i.e. - ``status=None``). - source: Provenance label stored on the new playbook row and in the - lineage event actor field. - request_id: Operation-run correlation id for the lineage event. Must be - non-empty; use an operation-scoped id. Raises ``ValueError`` - immediately (before any storage write) when empty, preventing - orphaned successor rows. - skip_embedding: Forwarded to ``save_user_playbooks``. Defaults to - ``False`` (precompute the embedding before opening the transaction). - - Returns: - The ``user_playbook_id`` of the newly inserted playbook, or ``-1`` if - the incumbent was not CURRENT (no mutation; no orphan left behind). - - Raises: - ValueError: If ``request_id`` is empty or None. - """ - if not request_id: - raise ValueError( - "apply_playbook_edit: request_id must be non-empty (operation-run correlation id)" - ) - new_playbook.source = source - if not skip_embedding: - storage.precompute_user_playbook_embeddings([new_playbook]) - - ctx = revise_context or LineageContext( - op_kind="revise", actor=source, request_id=request_id - ) - try: - with storage.commit_scope(): - storage.save_user_playbooks([new_playbook], skip_embedding=True) - new_id = new_playbook.user_playbook_id - if not storage.supersede_record( - entity_type="user_playbook", - incumbent_id=str(incumbent_id), - successor_id=str(new_id), - context=ctx, - ): - raise _LostSupersedeRaceError - except _LostSupersedeRaceError: - new_playbook.user_playbook_id = 0 - return -1 - return new_id diff --git a/reflexio/server/services/playbook/playbook_service_utils.py b/reflexio/server/services/playbook/playbook_service_utils.py index 56cc45bd6..90b9b7337 100644 --- a/reflexio/server/services/playbook/playbook_service_utils.py +++ b/reflexio/server/services/playbook/playbook_service_utils.py @@ -820,6 +820,7 @@ class PlaybookGenerationRequest(BaseModel): class PlaybookAggregatorRequest(BaseModel): agent_version: str rerun: bool = False + operation_key: str | None = Field(default=None, min_length=1) def construct_playbook_extraction_messages_from_sessions( diff --git a/reflexio/server/services/playbook/publication.py b/reflexio/server/services/playbook/publication.py new file mode 100644 index 000000000..5032b4a7c --- /dev/null +++ b/reflexio/server/services/playbook/publication.py @@ -0,0 +1,430 @@ +"""Strict shared contracts for atomic user-playbook publication.""" + +from __future__ import annotations + +import json +import re +from collections.abc import Mapping +from dataclasses import dataclass +from hashlib import sha256 +from typing import Literal, Protocol + +from reflexio.models.api_schema.domain.entities import OptimizerKind + +PublicationOutcome = Literal["applied", "incumbent_changed"] +PublishableOptimizerKind = Literal["gepa", "offline_tuner_replay"] +PublicationSource = Literal["gepa", "offline_optimizer"] + +_PUBLISHABLE_OPTIMIZERS = frozenset({"gepa", "offline_tuner_replay"}) +_PROJECTION_SCHEMA_VERSION = "offline-tuner-candidate-search-projection-v1" +_CANONICAL_DECIMAL = re.compile(r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]*[1-9])?\Z") +PUBLICATION_SUBJECT_EPOCHS_METADATA_KEY = "publication_subject_epochs" +PUBLICATION_PROOF_JSON_METADATA_KEY = "publication_proof_json" +PUBLICATION_PROJECTION_JSON_METADATA_KEY = "publication_projection_json" +PUBLICATION_INCUMBENT_CONTENT_DIGEST_METADATA_KEY = ( + "publication_incumbent_content_digest" +) +PUBLICATION_INCUMBENT_TRIGGER_METADATA_KEY = "publication_incumbent_trigger" +PUBLICATION_INCUMBENT_SEMANTIC_DIGEST_METADATA_KEY = ( + "publication_incumbent_semantic_digest" +) + + +def _require_text(name: str, value: object) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{name} must be non-empty") + return value + + +def _require_digest(name: str, value: object) -> str: + if ( + not isinstance(value, str) + or len(value) != 64 + or any(char not in "0123456789abcdef" for char in value) + ): + raise ValueError(f"{name} must be lowercase SHA-256 hex") + return value + + +def canonical_json_bytes(payload: object) -> bytes: + """Encode integer/string-only JSON values using RFC 8785 ordering.""" + return _canonical_json(payload).encode("utf-8") + + +def _canonical_json(value: object) -> str: + if value is None: + return "null" + if value is True: + return "true" + if value is False: + return "false" + if isinstance(value, str): + _reject_surrogates(value) + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + if isinstance(value, int): + if not -(2**53) < value < 2**53: + raise ValueError( + "RFC 8785 integers must be exactly representable by IEEE 754" + ) + return str(value) + if isinstance(value, tuple | list): + return "[" + ",".join(_canonical_json(item) for item in value) + "]" + if isinstance(value, Mapping): + if not all(isinstance(key, str) for key in value): + raise TypeError("RFC 8785 object keys must be strings") + for key in value: + _reject_surrogates(key) + keys = sorted(value, key=lambda key: key.encode("utf-16be")) + return ( + "{" + + ",".join( + f"{_canonical_json(key)}:{_canonical_json(value[key])}" for key in keys + ) + + "}" + ) + raise TypeError(f"Unsupported RFC 8785 value: {type(value).__name__}") + + +def _reject_surrogates(value: str) -> None: + if any(0xD800 <= ord(char) <= 0xDFFF for char in value): + raise ValueError("RFC 8785 strings cannot contain surrogate code points") + + +def _canonical_payload(name: str, value: str) -> object: + try: + payload = json.loads( + value, + parse_constant=lambda constant: (_ for _ in ()).throw( + ValueError(f"invalid JSON constant: {constant}") + ), + ) + canonical = canonical_json_bytes(payload).decode("utf-8") + except (TypeError, ValueError, json.JSONDecodeError, UnicodeError) as exc: + raise ValueError(f"{name} must be canonical JSON") from exc + if canonical != value: + raise ValueError(f"{name} must use canonical JSON bytes") + return payload + + +def _validate_optimizer(value: object) -> None: + if value not in _PUBLISHABLE_OPTIMIZERS: + raise ValueError("optimizer_kind is not publishable") + + +def publication_source_for_optimizer( + optimizer_kind: OptimizerKind, +) -> PublicationSource: + _validate_optimizer(optimizer_kind) + return "offline_optimizer" if optimizer_kind == "offline_tuner_replay" else "gepa" + + +def incumbent_user_playbook_semantic_digest( + *, content_digest: str, trigger: str | None +) -> str: + """Bind the behaviorally mutable incumbent fields used by publication.""" + _require_digest("incumbent content digest", content_digest) + if trigger is not None and not isinstance(trigger, str): + raise TypeError("incumbent trigger must be a string or null") + payload = { + "content_digest": content_digest, + "schema_version": "user-playbook-incumbent-semantic-v1", + "trigger": trigger, + } + return sha256(canonical_json_bytes(payload)).hexdigest() + + +@dataclass(frozen=True) +class PublicationClaim: + job_id: int + owner: str + fence: int + + def __post_init__(self) -> None: + if type(self.job_id) is not int or self.job_id <= 0: + raise ValueError("publication claim job_id must be positive") + _require_text("publication claim owner", self.owner) + if type(self.fence) is not int or self.fence <= 0: + raise ValueError("publication claim fence must be positive") + + +@dataclass(frozen=True) +class DecisionProofEnvelope: + optimizer_kind: OptimizerKind + schema_version: str + canonical_json: str + digest: str + decision: Literal["apply"] + + def __post_init__(self) -> None: + _validate_optimizer(self.optimizer_kind) + _require_text("decision proof schema_version", self.schema_version) + _require_digest("decision proof digest", self.digest) + if self.decision != "apply": + raise ValueError("publication decision must be apply") + payload = _canonical_payload( + "decision proof canonical_json", self.canonical_json + ) + if sha256(self.canonical_json.encode("utf-8")).hexdigest() != self.digest: + raise ValueError("decision proof digest does not match canonical JSON") + if not isinstance(payload, dict): + raise ValueError("decision proof canonical JSON must be an object") + if payload.get("optimizer_kind") != self.optimizer_kind: + raise ValueError("decision proof optimizer_kind does not match envelope") + if payload.get("schema_version") != self.schema_version: + raise ValueError("decision proof schema_version does not match envelope") + if payload.get("decision") != self.decision: + raise ValueError("decision proof decision does not match envelope") + + +@dataclass(frozen=True) +class PublicationSearchProjection: + schema_version: str + canonical_json: str + digest: str + projector_id: str + projector_version: str + projector_code_digest: str + candidate_content_digest: str + preserved_trigger: str | None + embedding_model_id: str + embedding: tuple[str, ...] + expanded_terms: tuple[str, ...] + lexical_document: str + + def __post_init__(self) -> None: + if self.schema_version != _PROJECTION_SCHEMA_VERSION: + raise ValueError("search projection schema is unsupported") + _require_digest("search projection digest", self.digest) + _require_text("search projection projector_id", self.projector_id) + _require_text("search projection projector_version", self.projector_version) + _require_digest( + "search projection projector_code_digest", self.projector_code_digest + ) + _require_digest( + "search projection candidate_content_digest", + self.candidate_content_digest, + ) + _require_text("search projection embedding_model_id", self.embedding_model_id) + _require_text("search projection lexical_document", self.lexical_document) + if self.preserved_trigger is not None: + _require_text("search projection preserved_trigger", self.preserved_trigger) + if not isinstance(self.embedding, tuple) or not self.embedding: + raise ValueError("search projection embedding must be a non-empty tuple") + if not isinstance(self.expanded_terms, tuple) or any( + not isinstance(term, str) or not term.strip() + for term in self.expanded_terms + ): + raise ValueError( + "search projection expanded_terms must contain non-empty text" + ) + for coordinate in self.embedding: + if ( + not isinstance(coordinate, str) + or coordinate == "-0" + or _CANONICAL_DECIMAL.fullmatch(coordinate) is None + ): + raise ValueError( + "search projection embedding must contain canonical decimals" + ) + payload = _canonical_payload( + "search projection canonical_json", self.canonical_json + ) + if sha256(self.canonical_json.encode("utf-8")).hexdigest() != self.digest: + raise ValueError("search projection digest does not match canonical JSON") + expected = { + "candidate_content_digest": self.candidate_content_digest, + "embedding": list(self.embedding), + "embedding_model_id": self.embedding_model_id, + "expanded_terms": list(self.expanded_terms), + "lexical_document": self.lexical_document, + "preserved_trigger": self.preserved_trigger, + "projector_code_digest": self.projector_code_digest, + "projector_id": self.projector_id, + "projector_version": self.projector_version, + "schema_version": self.schema_version, + } + if payload != expected: + raise ValueError("search projection fields do not match canonical JSON") + + +@dataclass(frozen=True) +class PublicationRequest: + optimizer_kind: OptimizerKind + job_id: int + attempt_key: str + publication_claim: PublicationClaim + worker_fence: int + incumbent_user_playbook_id: int + incumbent_content_digest: str + incumbent_trigger: str | None + incumbent_semantic_digest: str + revised_content: str + projection: PublicationSearchProjection + decision_proof: DecisionProofEnvelope + subject_epochs_json: str + request_id: str + + def __post_init__(self) -> None: + _validate_optimizer(self.optimizer_kind) + if type(self.job_id) is not int or self.job_id <= 0: + raise ValueError("publication job_id must be positive") + _require_text("publication attempt_key", self.attempt_key) + if self.publication_claim.job_id != self.job_id: + raise ValueError("publication claim job_id must match request job_id") + if type(self.worker_fence) is not int or self.worker_fence <= 0: + raise ValueError("worker_fence must be positive") + if ( + type(self.incumbent_user_playbook_id) is not int + or self.incumbent_user_playbook_id <= 0 + ): + raise ValueError("incumbent_user_playbook_id must be positive") + _require_digest("incumbent content digest", self.incumbent_content_digest) + _require_digest("incumbent semantic digest", self.incumbent_semantic_digest) + expected_semantic_digest = incumbent_user_playbook_semantic_digest( + content_digest=self.incumbent_content_digest, + trigger=self.incumbent_trigger, + ) + if self.incumbent_semantic_digest != expected_semantic_digest: + raise ValueError("incumbent semantic digest does not match frozen fields") + if self.projection.preserved_trigger != self.incumbent_trigger: + raise ValueError("search projection must preserve incumbent trigger") + _require_text("revised_content", self.revised_content) + _require_text("publication request_id", self.request_id) + if self.decision_proof.optimizer_kind != self.optimizer_kind: + raise ValueError("decision proof optimizer_kind must match request") + if sha256(self.revised_content.encode("utf-8")).hexdigest() != ( + self.projection.candidate_content_digest + ): + raise ValueError("revised content digest must match search projection") + epochs = _canonical_payload("subject_epochs_json", self.subject_epochs_json) + if ( + not isinstance(epochs, dict) + or set(epochs) != {"subjects"} + or not isinstance(epochs.get("subjects"), list) + or not epochs["subjects"] + ): + raise ValueError("subject epochs must contain a non-empty subjects list") + subject_refs: set[str] = set() + for item in epochs["subjects"]: + if not isinstance(item, dict): + raise ValueError("subject epochs must contain objects") + if set(item) != {"ref", "epoch"}: + raise ValueError("subject epochs must use ref and epoch fields") + subject_ref = item["ref"] + epoch = item["epoch"] + if ( + not isinstance(subject_ref, str) + or not subject_ref + or type(epoch) is not int + or epoch < 0 + ): + raise ValueError("subject epochs contain an invalid identity or epoch") + if subject_ref in subject_refs: + raise ValueError("subject epochs must contain unique subject refs") + subject_refs.add(subject_ref) + + +@dataclass(frozen=True) +class PublicationResult: + job_id: int + outcome: PublicationOutcome + successor_user_playbook_id: int | None + + def __post_init__(self) -> None: + if type(self.job_id) is not int or self.job_id <= 0: + raise ValueError("publication result job_id must be positive") + if self.outcome not in {"applied", "incumbent_changed"}: + raise ValueError("publication result outcome is invalid") + if self.outcome == "applied": + if ( + type(self.successor_user_playbook_id) is not int + or self.successor_user_playbook_id <= 0 + ): + raise ValueError("applied publication requires a successor id") + elif self.successor_user_playbook_id is not None: + raise ValueError("incumbent_changed publication cannot have a successor id") + + +class PublicationDecisionVerifier(Protocol): + """Optimizer-specific proof verification performed before any storage write.""" + + def verify(self, request: PublicationRequest) -> None: ... + + +class UserPlaybookPublicationStore(Protocol): + """Backend-neutral durable publication operations.""" + + def claim_user_playbook_publication( + self, *, job_id: int, owner: str, worker_fence: int + ) -> PublicationClaim: ... + + def stage_user_playbook_publication(self, request: PublicationRequest) -> None: ... + + def commit_user_playbook_publication( + self, request: PublicationRequest + ) -> PublicationResult: ... + + def load_user_playbook_publication_result( + self, job_id: int + ) -> PublicationResult | None: ... + + +class UserPlaybookPublicationService: + """Coordinates proof verification with durable staging and atomic commit.""" + + def __init__( + self, + storage: UserPlaybookPublicationStore, + verifier: PublicationDecisionVerifier, + ) -> None: + if not callable(getattr(verifier, "verify", None)): + raise TypeError("verifier must implement PublicationDecisionVerifier") + self._storage = storage + self._verifier = verifier + + def claim(self, *, job_id: int, owner: str, worker_fence: int) -> PublicationClaim: + return self._storage.claim_user_playbook_publication( + job_id=job_id, + owner=owner, + worker_fence=worker_fence, + ) + + def stage(self, request: PublicationRequest) -> None: + self._verifier.verify(request) + self._storage.stage_user_playbook_publication(request) + + def publish(self, request: PublicationRequest) -> PublicationResult: + self._verifier.verify(request) + self._storage.stage_user_playbook_publication(request) + return self._storage.commit_user_playbook_publication(request) + + def load_committed(self, job_id: int) -> PublicationResult | None: + return self._storage.load_user_playbook_publication_result(job_id) + + +def publish_user_playbook_successor( + service: UserPlaybookPublicationService, + request: PublicationRequest, +) -> PublicationResult: + """Publish one revised user playbook through the shared service.""" + return service.publish(request) + + +__all__ = [ + "DecisionProofEnvelope", + "PublicationClaim", + "PublicationDecisionVerifier", + "PublicationOutcome", + "PublicationRequest", + "PublicationResult", + "PublicationSearchProjection", + "PUBLICATION_INCUMBENT_CONTENT_DIGEST_METADATA_KEY", + "PUBLICATION_INCUMBENT_SEMANTIC_DIGEST_METADATA_KEY", + "PUBLICATION_INCUMBENT_TRIGGER_METADATA_KEY", + "PUBLICATION_SUBJECT_EPOCHS_METADATA_KEY", + "UserPlaybookPublicationService", + "UserPlaybookPublicationStore", + "canonical_json_bytes", + "incumbent_user_playbook_semantic_digest", + "publish_user_playbook_successor", +] diff --git a/reflexio/server/services/playbook_optimizer/gepa_publication.py b/reflexio/server/services/playbook_optimizer/gepa_publication.py new file mode 100644 index 000000000..1749d75b3 --- /dev/null +++ b/reflexio/server/services/playbook_optimizer/gepa_publication.py @@ -0,0 +1,480 @@ +"""GEPA-specific proof and search projection construction for publication.""" + +from __future__ import annotations + +import json +from decimal import Decimal +from hashlib import sha256 +from typing import Any + +from reflexio.models.api_schema.domain import ( + PlaybookOptimizationCandidate, + PlaybookOptimizationEvaluation, + PlaybookOptimizationJob, + UserPlaybook, +) +from reflexio.models.config_schema import PlaybookOptimizerConfig +from reflexio.server.services.playbook.publication import ( + PUBLICATION_INCUMBENT_CONTENT_DIGEST_METADATA_KEY, + PUBLICATION_INCUMBENT_SEMANTIC_DIGEST_METADATA_KEY, + PUBLICATION_INCUMBENT_TRIGGER_METADATA_KEY, + PUBLICATION_PROJECTION_JSON_METADATA_KEY, + PUBLICATION_PROOF_JSON_METADATA_KEY, + PUBLICATION_SUBJECT_EPOCHS_METADATA_KEY, + DecisionProofEnvelope, + PublicationRequest, + PublicationSearchProjection, + canonical_json_bytes, +) + +from .models import ScenarioWindow + +GEPA_PROJECTOR_ID = "reflexio-gepa-user-playbook-search-projector" +GEPA_PROJECTOR_VERSION = "2" +GEPA_PROJECTOR_CODE_DIGEST = ( + "c383e3dd0df8618d3b082ccbaceed7bd676f47af0b25d442bf8ff2c628fafb42" +) +_PROOF_SCHEMA_VERSION = "gepa-user-playbook-decision-v2" +_PROJECTION_SCHEMA_VERSION = "offline-tuner-candidate-search-projection-v1" +GEPA_PUBLICATION_AUTHORITY_METADATA_KEY = "gepa_publication_authority" + + +def _decimal(value: float) -> str: + decimal = Decimal(str(value)) + if not decimal.is_finite(): + raise ValueError("GEPA publication decimals must be finite") + return "0" if decimal == 0 else format(decimal.normalize(), "f") + + +def gepa_adoption_authority_from_config( + *, + config: PlaybookOptimizerConfig, + validation_windows: list[ScenarioWindow], + adoption_enabled: bool | None = None, +) -> dict[str, Any]: + return { + "adoption_policy": { + "auto_update_user_playbooks": ( + config.auto_update_user_playbooks + if adoption_enabled is None + else adoption_enabled + ), + "min_commit_likert": config.min_commit_likert, + "min_commit_score": repr(float(config.min_commit_score)), + "min_commit_windows": config.min_commit_windows, + }, + "validation_manifest": { + "windows": [ + { + "scenario_user_playbook_id": window.user_playbook_id, + "source_interaction_ids": list(window.source_interaction_ids), + "min_commit_likert": config.min_commit_likert, + "min_commit_score": repr(float(config.min_commit_score)), + } + for window in validation_windows + ], + }, + } + + +def build_gepa_search_projection( + storage: Any, + incumbent: UserPlaybook, + revised_content: str, +) -> PublicationSearchProjection: + successor = incumbent.model_copy( + update={ + "user_playbook_id": 0, + "content": revised_content, + "status": None, + "embedding": [], + "expanded_terms": None, + } + ) + storage.precompute_user_playbook_embeddings([successor]) + embedding = tuple(_decimal(value) for value in successor.embedding) + expanded_terms = ( + (successor.expanded_terms.strip(),) + if successor.expanded_terms and successor.expanded_terms.strip() + else () + ) + lexical_document = " ".join( + value + for value in (incumbent.trigger, revised_content, *expanded_terms) + if value + ) + content_digest = sha256(revised_content.encode()).hexdigest() + payload = { + "candidate_content_digest": content_digest, + "embedding": list(embedding), + "embedding_model_id": storage.embedding_model_name, + "expanded_terms": list(expanded_terms), + "lexical_document": lexical_document, + "preserved_trigger": incumbent.trigger, + "projector_code_digest": GEPA_PROJECTOR_CODE_DIGEST, + "projector_id": GEPA_PROJECTOR_ID, + "projector_version": GEPA_PROJECTOR_VERSION, + "schema_version": _PROJECTION_SCHEMA_VERSION, + } + canonical_json = canonical_json_bytes(payload).decode() + return PublicationSearchProjection( + schema_version=_PROJECTION_SCHEMA_VERSION, + canonical_json=canonical_json, + digest=sha256(canonical_json.encode()).hexdigest(), + projector_id=GEPA_PROJECTOR_ID, + projector_version=GEPA_PROJECTOR_VERSION, + projector_code_digest=GEPA_PROJECTOR_CODE_DIGEST, + candidate_content_digest=content_digest, + preserved_trigger=incumbent.trigger, + embedding_model_id=storage.embedding_model_name, + embedding=embedding, + expanded_terms=expanded_terms, + lexical_document=lexical_document, + ) + + +def _evaluation_evidence(item: PlaybookOptimizationEvaluation) -> dict[str, Any]: + return { + "candidate_id": item.candidate_id, + "candidate_rollout_json": item.candidate_rollout_json, + "created_at": item.created_at, + "evaluation_id": item.evaluation_id, + "incumbent_rollout_json": item.incumbent_rollout_json, + "likert": item.likert, + "rationale": item.rationale, + "scenario_user_playbook_id": item.scenario_user_playbook_id, + "score": _decimal(item.score), + "source_interaction_ids": item.source_interaction_ids, + "target_id": item.target_id, + "target_kind": item.target_kind, + "asi_json": item.asi_json, + "verdict": item.verdict, + } + + +def _proof_payload( + *, + job: PlaybookOptimizationJob, + winner: PlaybookOptimizationCandidate, + evaluations: list[PlaybookOptimizationEvaluation], + metadata: dict[str, Any], + subject_epochs_json: str, + projection_digest: str, +) -> dict[str, Any]: + authority = metadata.get(GEPA_PUBLICATION_AUTHORITY_METADATA_KEY) + if not isinstance(authority, dict): + raise ValueError("GEPA publication authority snapshot is missing") + all_evaluations = sorted(evaluations, key=lambda item: item.evaluation_id) + return { + "adoption_authority": authority, + "candidate": { + "aggregate_score": _decimal(winner.aggregate_score or 0.0), + "candidate_id": winner.candidate_id, + "candidate_index": winner.candidate_index, + "content_digest": sha256(winner.content.encode()).hexdigest(), + "metadata_json": winner.metadata_json, + "parent_candidate_ids": list(winner.parent_candidate_ids), + }, + "decision": "apply", + "evaluations": [ + { + "evaluation_digest": sha256( + canonical_json_bytes(_evaluation_evidence(item)) + ).hexdigest(), + "evaluation_id": item.evaluation_id, + } + for item in all_evaluations + ], + "job": { + "attempt_key": job.attempt_key, + "job_id": job.job_id, + "target_id": job.target_id, + "target_kind": job.target_kind, + }, + "optimizer_kind": "gepa", + "projection_digest": projection_digest, + "schema_version": _PROOF_SCHEMA_VERSION, + "subject_epochs": json.loads(subject_epochs_json), + "winning_adoption": _gepa_adoption_result_from_snapshot( + winner=winner, + evaluations=evaluations, + authority=authority, + ), + } + + +def build_gepa_decision_proof( + *, + job: PlaybookOptimizationJob, + winner: PlaybookOptimizationCandidate, + evaluations: list[PlaybookOptimizationEvaluation], + metadata: dict[str, Any], + subject_epochs_json: str, + projection_digest: str, +) -> DecisionProofEnvelope: + canonical_json = canonical_json_bytes( + _proof_payload( + job=job, + winner=winner, + evaluations=evaluations, + metadata=metadata, + subject_epochs_json=subject_epochs_json, + projection_digest=projection_digest, + ) + ).decode() + return DecisionProofEnvelope( + optimizer_kind="gepa", + schema_version=_PROOF_SCHEMA_VERSION, + canonical_json=canonical_json, + digest=sha256(canonical_json.encode()).hexdigest(), + decision="apply", + ) + + +def _gepa_adoption_result_from_snapshot( + *, + winner: PlaybookOptimizationCandidate, + evaluations: list[PlaybookOptimizationEvaluation], + authority: dict[str, Any], +) -> dict[str, Any]: + return gepa_winner_adoption_result( + winner_candidate_id=winner.candidate_id, + aggregate_score=winner.aggregate_score, + evaluations=evaluations, + authority=authority, + ) + + +def gepa_winner_adoption_result( + *, + winner_candidate_id: int, + aggregate_score: float | None, + evaluations: list[PlaybookOptimizationEvaluation], + authority: dict[str, Any], +) -> dict[str, Any]: + policy = authority.get("adoption_policy") + validation_manifest = authority.get("validation_manifest") + if not isinstance(policy, dict) or not isinstance(validation_manifest, dict): + raise ValueError("GEPA adoption authority snapshot is invalid") + windows = validation_manifest.get("windows") + if not isinstance(windows, list) or not windows: + raise ValueError("GEPA validation manifest is missing") + min_windows = int(policy["min_commit_windows"]) + window_thresholds = { + _window_key( + item.get("scenario_user_playbook_id"), + item.get("source_interaction_ids"), + ): { + "min_commit_likert": int(item["min_commit_likert"]), + "min_commit_score": str(item["min_commit_score"]), + } + for item in windows + } + passed_window_keys: set[tuple[int | None, tuple[int, ...]]] = set() + winning_windows = [] + for evaluation in evaluations: + if evaluation.candidate_id != winner_candidate_id: + continue + key = _window_key( + evaluation.scenario_user_playbook_id, + evaluation.source_interaction_ids, + ) + thresholds = window_thresholds.get(key) + if thresholds is None: + continue + min_score = float(thresholds["min_commit_score"]) + min_likert = int(thresholds["min_commit_likert"]) + passed = ( + evaluation.verdict == "candidate" + and evaluation.score >= min_score + and evaluation.likert >= min_likert + ) + if passed: + passed_window_keys.add(key) + winning_windows.append( + { + "evaluation_id": evaluation.evaluation_id, + "likert": evaluation.likert, + "passed": passed, + "score": _decimal(evaluation.score), + "thresholds": thresholds, + "window": { + "scenario_user_playbook_id": evaluation.scenario_user_playbook_id, + "source_interaction_ids": list(evaluation.source_interaction_ids), + }, + } + ) + policy_min_score = float(policy["min_commit_score"]) + passes = ( + bool(policy.get("auto_update_user_playbooks")) + and aggregate_score is not None + and aggregate_score >= policy_min_score + and len(passed_window_keys) >= min_windows + ) + return { + "aggregate_score": _decimal(aggregate_score or 0.0), + "min_commit_windows": min_windows, + "passes": passes, + "passed_window_count": len(passed_window_keys), + "winning_windows": winning_windows, + } + + +def _window_key( + scenario_user_playbook_id: object, + source_interaction_ids: object, +) -> tuple[int | None, tuple[int, ...]]: + if scenario_user_playbook_id is not None and not isinstance( + scenario_user_playbook_id, int + ): + raise ValueError("GEPA validation window playbook id is invalid") + if not isinstance(source_interaction_ids, list): + raise ValueError("GEPA validation window source ids are invalid") + return scenario_user_playbook_id, tuple( + int(item) for item in source_interaction_ids + ) + + +def parse_gepa_search_projection(canonical_json: str) -> PublicationSearchProjection: + payload = json.loads(canonical_json) + if not isinstance(payload, dict): + raise ValueError("GEPA search projection payload is invalid") + return PublicationSearchProjection( + schema_version=payload["schema_version"], + canonical_json=canonical_json, + digest=sha256(canonical_json.encode()).hexdigest(), + projector_id=payload["projector_id"], + projector_version=payload["projector_version"], + projector_code_digest=payload["projector_code_digest"], + candidate_content_digest=payload["candidate_content_digest"], + preserved_trigger=payload["preserved_trigger"], + embedding_model_id=payload["embedding_model_id"], + embedding=tuple(payload["embedding"]), + expanded_terms=tuple(payload["expanded_terms"]), + lexical_document=payload["lexical_document"], + ) + + +def parse_gepa_decision_proof(canonical_json: str) -> DecisionProofEnvelope: + payload = json.loads(canonical_json) + if not isinstance(payload, dict): + raise ValueError("GEPA decision proof payload is invalid") + return DecisionProofEnvelope( + optimizer_kind=payload["optimizer_kind"], + schema_version=payload["schema_version"], + canonical_json=canonical_json, + digest=sha256(canonical_json.encode()).hexdigest(), + decision=payload["decision"], + ) + + +class GEPAUserPlaybookDecisionVerifier: + """Rebuild GEPA adoption authority exclusively from durable records.""" + + def __init__( + self, + storage: Any, + ) -> None: + self._storage = storage + + def verify(self, request: PublicationRequest) -> None: + job = self._storage.get_playbook_optimization_job(request.job_id) + if ( + job is None + or (job.optimizer_kind, job.target_kind, job.target_id) + != ("gepa", "user_playbook", request.incumbent_user_playbook_id) + or job.attempt_key != request.attempt_key + or job.stage != "publishing" + or job.best_candidate_id is None + ): + raise ValueError("GEPA durable job is not publishable") + winners = [ + item + for item in self._storage.list_playbook_optimization_candidates(job.job_id) + if item.is_winner + ] + if len(winners) != 1 or winners[0].candidate_id != job.best_candidate_id: + raise ValueError("GEPA durable winner changed") + winner = winners[0] + evaluations = self._storage.list_playbook_optimization_evaluations(job.job_id) + if any(item.verdict == "aborted" for item in evaluations): + raise ValueError("GEPA evaluation aborted") + metadata = json.loads(job.metadata_json) + authority = metadata.get(GEPA_PUBLICATION_AUTHORITY_METADATA_KEY) + if not isinstance(authority, dict): + raise ValueError("GEPA durable authority snapshot is missing") + adoption = _gepa_adoption_result_from_snapshot( + winner=winner, + evaluations=evaluations, + authority=authority, + ) + if not adoption["passes"]: + raise ValueError("GEPA durable winner fails adoption rules") + if any( + item.target_kind != "user_playbook" or item.target_id != job.target_id + for item in evaluations + if item.candidate_id == winner.candidate_id + ): + raise ValueError("GEPA evaluation target changed") + subject_epochs = self._storage.get_user_playbook_publication_subject_epochs( + job.target_id + ) + incumbent = self._storage.get_user_playbook_by_id(job.target_id) + if ( + subject_epochs != request.subject_epochs_json + or metadata.get(PUBLICATION_SUBJECT_EPOCHS_METADATA_KEY) + != json.loads(subject_epochs) + or incumbent is None + or incumbent.trigger != request.projection.preserved_trigger + or sha256(incumbent.content.encode("utf-8")).hexdigest() + != request.incumbent_content_digest + or incumbent.trigger != request.incumbent_trigger + or metadata.get(PUBLICATION_INCUMBENT_CONTENT_DIGEST_METADATA_KEY) + != request.incumbent_content_digest + or metadata.get(PUBLICATION_INCUMBENT_TRIGGER_METADATA_KEY) + != request.incumbent_trigger + or metadata.get(PUBLICATION_INCUMBENT_SEMANTIC_DIGEST_METADATA_KEY) + != request.incumbent_semantic_digest + or winner.content != request.revised_content + or job.candidate_content_digest + != request.projection.candidate_content_digest + or job.search_projection_digest != request.projection.digest + or metadata.get(PUBLICATION_PROJECTION_JSON_METADATA_KEY) + != request.projection.canonical_json + ): + raise ValueError("GEPA publication binding changed") + stored_proof_json = metadata.get(PUBLICATION_PROOF_JSON_METADATA_KEY) + if not isinstance(stored_proof_json, str): + raise ValueError("GEPA durable decision proof is missing") + if parse_gepa_decision_proof(stored_proof_json) != request.decision_proof: + raise ValueError("GEPA durable decision proof changed") + proof_metadata = { + key: value + for key, value in metadata.items() + if key + not in { + "publication_proof_digest", + PUBLICATION_PROOF_JSON_METADATA_KEY, + PUBLICATION_PROJECTION_JSON_METADATA_KEY, + PUBLICATION_SUBJECT_EPOCHS_METADATA_KEY, + PUBLICATION_INCUMBENT_CONTENT_DIGEST_METADATA_KEY, + PUBLICATION_INCUMBENT_TRIGGER_METADATA_KEY, + PUBLICATION_INCUMBENT_SEMANTIC_DIGEST_METADATA_KEY, + } + } + try: + expected = build_gepa_decision_proof( + job=job, + winner=winner, + evaluations=evaluations, + metadata=proof_metadata, + subject_epochs_json=subject_epochs, + projection_digest=request.projection.digest, + ) + except (TypeError, ValueError) as exc: + raise ValueError("GEPA durable decision proof changed") from exc + if ( + expected != request.decision_proof + or metadata.get("publication_proof_digest") != expected.digest + ): + raise ValueError("GEPA durable decision proof changed") diff --git a/reflexio/server/services/playbook_optimizer/judge.py b/reflexio/server/services/playbook_optimizer/judge.py index a0e9de070..39cc0912d 100644 --- a/reflexio/server/services/playbook_optimizer/judge.py +++ b/reflexio/server/services/playbook_optimizer/judge.py @@ -1,11 +1,20 @@ from __future__ import annotations +import inspect import json import logging +from collections.abc import Mapping +from hashlib import sha256 +from typing import Any, cast + +import litellm from reflexio.models.api_schema.domain import AgentPlaybook from reflexio.server.api_endpoints.request_context import RequestContext +from reflexio.server.llm import _litellm_structured_output as structured_output_module +from reflexio.server.llm import _litellm_text_generation as text_generation_module from reflexio.server.llm.litellm_client import LiteLLMClient +from reflexio.server.prompt.prompt_manager import PromptManager from reflexio.server.services.service_utils import log_model_response from .models import JudgeOutput, RolloutTrace, ScenarioWindow @@ -13,6 +22,175 @@ logger = logging.getLogger(__name__) PLAYBOOK_OPTIMIZER_JUDGE_PROMPT_ID = "playbook_optimizer_judge" +PAIRWISE_JUDGE_TIMEOUT_SECONDS = 120 +PAIRWISE_JUDGE_MAX_RETRIES = 1 + +_LLM_IMPLEMENTATION_METHODS = ( + "generate_chat_response", + "_make_request", + "_resolve_ladder", + "_resolve_primary_model", + "_build_completion_params", + "_is_temperature_restricted_model", + "_resolve_api_key", + "_provider_for_model", + "_structured_output_strategy", + "_apply_structured_output_transport", + "_provider_response_format", + "_completion_with_hard_timeout", + "_coerce_timeout_seconds", + "_hard_timeout_grace_seconds", + "_should_process_isolate_completion", + "_maybe_parse_structured_output", + "_apply_prompt_caching", +) + +_PROMPT_IMPLEMENTATION_METHODS = ( + "get_prompt_template_identity", + "render_prompt_from_identity", + "get_active_version", + "_find_active_version", + "_get_prompt", + "_load_prompt", + "_render_prompt", +) + +_PROVIDER_PARAM_KEYS = frozenset( + { + "allowed_openai_params", + "api_base", + "api_key", + "api_version", + "drop_params", + "max_tokens", + "messages", + "metadata", + "model", + "num_retries", + "response_format", + "seed", + "temperature", + "timeout", + "top_p", + } +) + + +class FrozenEvaluatorPlanDriftError(text_generation_module.ProviderRequestGuardError): + """Raised before evaluation when live judge inputs differ from the frozen plan.""" + + +def canonicalize_pairwise_judge_request_plan(plan: Mapping[str, Any]) -> str: + """Return immutable canonical JSON bytes-as-text for a sanitized plan.""" + return json.dumps(plan, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def sanitize_pairwise_judge_provider_params( + params: Mapping[str, Any], +) -> dict[str, Any]: + """Select every non-secret, non-content parameter sent to the provider.""" + unexpected = set(params) - _PROVIDER_PARAM_KEYS + if unexpected: + raise ValueError( + f"Unsupported PairwiseJudge provider params: {sorted(unexpected)}" + ) + + sanitized: dict[str, Any] = {} + for key, value in sorted(params.items()): + if key in {"api_key", "messages", "metadata"}: + continue + if key == "api_base": + sanitized["api_base_digest"] = _text_digest(str(value)) if value else None + elif key == "response_format": + sanitized[key] = _response_format_identity(value) + else: + sanitized[key] = _sanitize_provider_value(value) + return sanitized + + +def build_pairwise_judge_request_plan( + *, + prompt_manager: PromptManager, + llm_client: LiteLLMClient, + model_name: str | None, +) -> dict[str, Any]: + """Build the sanitized plan persisted and verified by ``PairwiseJudge``.""" + client = cast(Any, llm_client) + requested_model = model_name or llm_client.config.model + ladder = client._resolve_ladder(model=requested_model) + grace_seconds = client._hard_timeout_grace_seconds() + callables = _pairwise_judge_implementation_identities(prompt_manager, llm_client) + return { + "judge_class": _type_identity(PairwiseJudge), + "judge_code_digest": _code_digest(PairwiseJudge), + "judge_model_id": requested_model, + "judge_prompt_id": PLAYBOOK_OPTIMIZER_JUDGE_PROMPT_ID, + "judge_prompt_identity": prompt_manager.get_prompt_template_identity( + PLAYBOOK_OPTIMIZER_JUDGE_PROMPT_ID + ), + "llm_client_class": _type_identity(type(llm_client)), + "llm_client_code_digest": _code_digest(type(llm_client)), + "judge_output_schema_class": _type_identity(JudgeOutput), + "judge_output_schema_code_digest": _code_digest(JudgeOutput), + "judge_output_schema_digest": _json_digest(JudgeOutput.model_json_schema()), + "implementation_callables": callables, + "judge_generation_settings": { + "requested_model": requested_model, + "resolved_primary_model": ladder[0], + "fallback_model_order": ladder[1:], + "resolved_model_ladder": ladder, + "pairwise_judge_max_retries": PAIRWISE_JUDGE_MAX_RETRIES, + "rungs": [ + _pairwise_judge_rung(llm_client, model, grace_seconds) + for model in ladder + ], + }, + } + + +def _pairwise_judge_rung( + llm_client: LiteLLMClient, + model: str, + grace_seconds: float, +) -> dict[str, Any]: + client = cast(Any, llm_client) + params, _response_format, parse_structured_output, _max_retries, _fallbacks = ( + client._build_completion_params( + [{"role": "user", "content": ""}], + model=model, + response_format=JudgeOutput, + timeout=PAIRWISE_JUDGE_TIMEOUT_SECONDS, + max_retries=PAIRWISE_JUDGE_MAX_RETRIES, + fallback_models=[], + ) + ) + params["num_retries"] = 0 + params.pop("fallbacks", None) + resolved_model = str(params["model"]) + timeout_seconds = client._coerce_timeout_seconds(params) + provider_params = sanitize_pairwise_judge_provider_params(params) + return { + "model": resolved_model, + "temperature": _decimal_string(provider_params["temperature"]), + "top_p": _decimal_string(provider_params.get("top_p", 1.0)), + "max_tokens": provider_params.get("max_tokens"), + "seed": provider_params["seed"], + "timeout_seconds": _decimal_string(timeout_seconds), + "hard_timeout_grace_seconds": _decimal_string(grace_seconds), + "hard_timeout_seconds": _decimal_string(timeout_seconds + grace_seconds), + "process_isolation": client._should_process_isolate_completion( + timeout_seconds, grace_seconds + ), + "provider_kind": client._provider_for_model(resolved_model) or "unconfigured", + "api_base_digest": provider_params.get("api_base_digest"), + "api_version": provider_params.get("api_version"), + "structured_output_strategy": client._structured_output_strategy( + model=resolved_model, + strict_response_format=True, + ), + "parse_structured_output": parse_structured_output, + "provider_params": provider_params, + } class PairwiseJudge: @@ -33,10 +211,22 @@ def __init__( request_context: RequestContext, llm_client: LiteLLMClient, model_name: str | None, + *, + frozen_request_plan: Mapping[str, Any] | None = None, ) -> None: self.request_context = request_context self.llm_client = llm_client self.model_name = model_name or llm_client.config.model + self._frozen_request_plan_json = ( + canonicalize_pairwise_judge_request_plan(frozen_request_plan) + if frozen_request_plan is not None + else None + ) + self._provider_request_guard = ( + self._enforce_frozen_request_plan_at_provider + if frozen_request_plan is not None + else None + ) def judge( self, @@ -54,24 +244,36 @@ def judge( likert=3, rationale="Candidate content is identical to incumbent content.", ) - prompt = self.request_context.prompt_manager.render_prompt( - PLAYBOOK_OPTIMIZER_JUDGE_PROMPT_ID, - { - "source_window_json": _json( - [interaction.model_dump() for interaction in window.interactions] - ), - "incumbent_playbook_json": _json(_playbook_payload(incumbent)), - "candidate_playbook_json": _json(_playbook_payload(candidate)), - "incumbent_rollout_json": incumbent_rollout.model_dump_json(), - "candidate_rollout_json": candidate_rollout.model_dump_json(), - }, - ) + frozen_plan = self._frozen_request_plan() + if frozen_plan is not None: + self._assert_frozen_implementation_identities(frozen_plan) + variables = { + "source_window_json": _json( + [interaction.model_dump() for interaction in window.interactions] + ), + "incumbent_playbook_json": _json(_playbook_payload(incumbent)), + "candidate_playbook_json": _json(_playbook_payload(candidate)), + "incumbent_rollout_json": incumbent_rollout.model_dump_json(), + "candidate_rollout_json": candidate_rollout.model_dump_json(), + } + if frozen_plan is None: + prompt = self.request_context.prompt_manager.render_prompt( + PLAYBOOK_OPTIMIZER_JUDGE_PROMPT_ID, variables + ) + else: + prompt = self.request_context.prompt_manager.render_prompt_from_identity( + PLAYBOOK_OPTIMIZER_JUDGE_PROMPT_ID, + variables, + frozen_plan["judge_prompt_identity"], + ) + self._assert_frozen_implementation_identities(frozen_plan) response = self.llm_client.generate_chat_response( messages=[{"role": "user", "content": prompt}], model=self.model_name, response_format=JudgeOutput, - timeout=120, - max_retries=1, + timeout=PAIRWISE_JUDGE_TIMEOUT_SECONDS, + max_retries=PAIRWISE_JUDGE_MAX_RETRIES, + provider_request_guard=self._provider_request_guard, ) log_model_response(logger, "Playbook optimizer judge response", response) if isinstance(response, JudgeOutput): @@ -83,6 +285,240 @@ def judge( rationale=f"Judge response was not parsed: {type(response).__name__}", ) + def _frozen_request_plan(self) -> dict[str, Any] | None: + if self._frozen_request_plan_json is None: + return None + return json.loads(self._frozen_request_plan_json) + + def _assert_frozen_implementation_identities( + self, frozen_plan: Mapping[str, Any] + ) -> None: + current = _pairwise_judge_implementation_identities( + self.request_context.prompt_manager, + self.llm_client, + ) + if current != frozen_plan["implementation_callables"]: + raise FrozenEvaluatorPlanDriftError( + "PairwiseJudge evaluator implementation drifted after job creation" + ) + + def _enforce_frozen_request_plan_at_provider( + self, + params: dict[str, Any], + hard_timeout: float, + ladder: tuple[str, ...], + parse_structured_output: bool, + ) -> None: + frozen = self._frozen_request_plan() + if frozen is None: # pragma: no cover - guard is installed only when frozen + return + self._assert_frozen_implementation_identities(frozen) + if ( + self.request_context.prompt_manager.get_prompt_template_identity( + PLAYBOOK_OPTIMIZER_JUDGE_PROMPT_ID + ) + != frozen["judge_prompt_identity"] + ): + raise FrozenEvaluatorPlanDriftError( + "PairwiseJudge prompt drifted after job creation" + ) + + generation = frozen["judge_generation_settings"] + if list(ladder) != generation["resolved_model_ladder"]: + raise FrozenEvaluatorPlanDriftError( + "PairwiseJudge model ladder drifted after job creation" + ) + model = str(params["model"]) + expected_rung = next( + (rung for rung in generation["rungs"] if rung["model"] == model), + None, + ) + if expected_rung is None: + raise FrozenEvaluatorPlanDriftError( + "PairwiseJudge provider rung drifted after job creation" + ) + provider_params = sanitize_pairwise_judge_provider_params(params) + timeout_seconds = float(provider_params["timeout"]) + grace_seconds = max(0.0, hard_timeout - timeout_seconds) + provider_kind = ( + cast(Any, self.llm_client)._provider_for_model(model) or "unconfigured" + ) + if ( + provider_params != expected_rung["provider_params"] + or _decimal_string(hard_timeout) != expected_rung["hard_timeout_seconds"] + or parse_structured_output != expected_rung["parse_structured_output"] + or provider_kind != expected_rung["provider_kind"] + or cast(Any, self.llm_client)._should_process_isolate_completion( + timeout_seconds, grace_seconds + ) + != expected_rung["process_isolation"] + ): + raise FrozenEvaluatorPlanDriftError( + "PairwiseJudge provider request drifted after job creation" + ) + + +def _bound_callable_identity(owner: Any, name: str) -> dict[str, Any]: + return { + **_callable_identity(getattr(owner, name)), + "instance_override": name in vars(owner), + } + + +def _pairwise_judge_implementation_identities( + prompt_manager: PromptManager, + llm_client: LiteLLMClient, +) -> dict[str, dict[str, Any]]: + client = cast(Any, llm_client) + callables = { + f"llm_client.{name}": _bound_callable_identity(client, name) + for name in _LLM_IMPLEMENTATION_METHODS + } + callables.update( + { + f"prompt_manager.{name}": _bound_callable_identity(prompt_manager, name) + for name in _PROMPT_IMPLEMENTATION_METHODS + } + ) + callables.update( + { + "judge.PairwiseJudge.judge": _callable_identity(PairwiseJudge.judge), + "judge.PairwiseJudge._assert_frozen_implementation_identities": ( + _callable_identity( + PairwiseJudge._assert_frozen_implementation_identities + ) + ), + "judge.PairwiseJudge._enforce_frozen_request_plan_at_provider": ( + _callable_identity( + PairwiseJudge._enforce_frozen_request_plan_at_provider + ) + ), + **_judge_helper_identities(), + **_implementation_helper_identities(), + } + ) + return callables + + +def _implementation_helper_identities() -> dict[str, dict[str, str]]: + helpers = { + "litellm.completion": litellm.completion, + "litellm.get_llm_provider": litellm.get_llm_provider, + "litellm.supports_response_schema": litellm.supports_response_schema, + "structured_output._extract_json_from_string": ( + structured_output_module._extract_json_from_string + ), + "structured_output._looks_truncated_json": ( + structured_output_module._looks_truncated_json + ), + "structured_output._sanitize_json_string": ( + structured_output_module._sanitize_json_string + ), + "structured_output._validate_structured_payload": ( + structured_output_module._validate_structured_payload + ), + "structured_output.assert_provider_safe_schema": ( + structured_output_module.assert_provider_safe_schema + ), + "structured_output.prompt_schema_instruction": ( + structured_output_module.prompt_schema_instruction + ), + "structured_output.strict_response_format_for_model": ( + structured_output_module.strict_response_format_for_model + ), + "text_generation._litellm_completion_worker": ( + text_generation_module._litellm_completion_worker + ), + "text_generation.default_max_tokens_for_model": ( + text_generation_module.default_max_tokens_for_model + ), + "text_generation.resolve_model_name": text_generation_module.resolve_model_name, + } + return {name: _callable_identity(value) for name, value in helpers.items()} + + +def _judge_helper_identities() -> dict[str, dict[str, str]]: + helpers = { + "judge.build_pairwise_judge_request_plan": build_pairwise_judge_request_plan, + "judge.canonicalize_pairwise_judge_request_plan": ( + canonicalize_pairwise_judge_request_plan + ), + "judge.sanitize_pairwise_judge_provider_params": ( + sanitize_pairwise_judge_provider_params + ), + "judge._pairwise_judge_implementation_identities": ( + _pairwise_judge_implementation_identities + ), + "judge._pairwise_judge_rung": _pairwise_judge_rung, + "judge._response_format_identity": _response_format_identity, + "judge._sanitize_provider_value": _sanitize_provider_value, + } + return {name: _callable_identity(value) for name, value in helpers.items()} + + +def _callable_identity(value: Any) -> dict[str, str]: + target = getattr(value, "__func__", value) + target_type = type(target) + identity = ( + f"{getattr(target, '__module__', target_type.__module__)}." + f"{getattr(target, '__qualname__', target_type.__qualname__)}" + ) + return {"identity": identity, "code_digest": _code_digest(target)} + + +def _type_identity(value: type[Any]) -> str: + return f"{value.__module__}.{value.__qualname__}" + + +def _code_digest(value: Any) -> str: + try: + source = inspect.getsource(value) + except (OSError, TypeError): + source = _callable_fallback_identity(value) + return _text_digest(source) + + +def _callable_fallback_identity(value: Any) -> str: + value_type = type(value) + return ( + f"{getattr(value, '__module__', value_type.__module__)}." + f"{getattr(value, '__qualname__', value_type.__qualname__)}" + ) + + +def _response_format_identity(value: Any) -> dict[str, str]: + if inspect.isclass(value): + return {"kind": "class", "identity": _type_identity(value)} + try: + digest = _json_digest(value) + except (TypeError, ValueError): + return {"kind": "type", "identity": _type_identity(type(value))} + return {"kind": "json", "digest": digest} + + +def _sanitize_provider_value(value: Any) -> Any: + if isinstance(value, float): + return repr(value) + if isinstance(value, list): + return [_sanitize_provider_value(item) for item in value] + if isinstance(value, dict): + return {str(key): _sanitize_provider_value(item) for key, item in value.items()} + return value + + +def _json_digest(value: Any) -> str: + return _text_digest( + json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + ) + + +def _text_digest(value: str) -> str: + return sha256(value.encode("utf-8")).hexdigest() + + +def _decimal_string(value: Any) -> str: + return repr(float(value)) + def _playbook_payload(playbook: AgentPlaybook) -> dict[str, object]: return { diff --git a/reflexio/server/services/playbook_optimizer/optimizer.py b/reflexio/server/services/playbook_optimizer/optimizer.py index 4c3425262..cf0e9b5e3 100644 --- a/reflexio/server/services/playbook_optimizer/optimizer.py +++ b/reflexio/server/services/playbook_optimizer/optimizer.py @@ -1,8 +1,12 @@ from __future__ import annotations +import importlib.metadata +import inspect import json import logging import os +import uuid +from hashlib import sha256 from pathlib import Path from typing import Any, Literal, cast @@ -23,12 +27,34 @@ from reflexio.server.services.playbook.aggregation_trigger import ( maybe_trigger_user_playbook_aggregation, ) -from reflexio.server.services.playbook.playbook_edit_apply import apply_playbook_edit +from reflexio.server.services.playbook.publication import ( + PUBLICATION_INCUMBENT_CONTENT_DIGEST_METADATA_KEY, + PUBLICATION_INCUMBENT_SEMANTIC_DIGEST_METADATA_KEY, + PUBLICATION_INCUMBENT_TRIGGER_METADATA_KEY, + PUBLICATION_PROJECTION_JSON_METADATA_KEY, + PUBLICATION_PROOF_JSON_METADATA_KEY, + PUBLICATION_SUBJECT_EPOCHS_METADATA_KEY, + PublicationRequest, + PublicationResult, + UserPlaybookPublicationService, + incumbent_user_playbook_semantic_digest, +) +from reflexio.server.services.storage.error import OptimizationJobLeaseLiveError from reflexio.server.tracing import sentry_tags from .assistant_webhook import AssistantCallable, LocalScriptAssistant, WebhookAssistant from .gepa_adapter import PLAYBOOK_CONTENT_COMPONENT, ReflexioPlaybookGEPAAdapter -from .judge import PairwiseJudge +from .gepa_publication import ( + GEPA_PUBLICATION_AUTHORITY_METADATA_KEY, + GEPAUserPlaybookDecisionVerifier, + build_gepa_decision_proof, + build_gepa_search_projection, + gepa_adoption_authority_from_config, + gepa_winner_adoption_result, + parse_gepa_decision_proof, + parse_gepa_search_projection, +) +from .judge import PairwiseJudge, build_pairwise_judge_request_plan from .models import ScenarioWindow from .rollout import MultiTurnRollout from .scenario_resolver import ScenarioResolver @@ -39,6 +65,8 @@ # Shared with tests so the format is defined in one place. _OPTIMIZER_RUN_ID_PREFIX = "optjob_" +_GEPA_PUBLICATION_LEASE_SECONDS = 60 + def optimizer_run_request_id(job_id: int) -> str: """Return the lineage request_id for a playbook optimizer run. @@ -100,10 +128,16 @@ def optimize( the incumbent in favour of a successor playbook. """ config = self._config() - if not self._enabled_for_target(config, target): + if not self._enabled_for_target(config, target) or not _can_adopt_winner( + target, config + ): return "skipped" - # Backend selection happens before any storage work so an unconfigured - # optimizer short-circuits cheaply — useful in tests and dev setups. + if target.kind == "user_playbook": + recovered = self._recover_gepa_user_playbook_publication(target) + if recovered is not None: + return recovered + # Backend selection happens before any new-run storage work so an + # unconfigured optimizer short-circuits cheaply. assistant = self._create_assistant(config) if assistant is None: logger.info( @@ -131,22 +165,40 @@ def optimize( config.min_commit_windows, ) return "skipped" - if not _can_adopt_winner(target, config): - logger.info( - "Skipping playbook optimization: no configured adoption path " - "target_kind=%s target_id=%d", - target.kind, - target.target_id, - ) - return "skipped" - split_metadata = _split_metadata(windows, train_windows, validation_windows) + split_metadata = _split_metadata( + windows, + train_windows, + validation_windows, + config=config, + assistant=assistant, + llm_client=self.llm_client, + prompt_manager=( + getattr(self.request_context, "prompt_manager", None) + if target.kind == "user_playbook" + else None + ), + include_publication_authority=target.kind == "user_playbook", + ) + frozen_judge_plan = ( + split_metadata[GEPA_PUBLICATION_AUTHORITY_METADATA_KEY][ + "evaluator_identity" + ]["pairwise_judge_request_plan"] + if target.kind == "user_playbook" + else None + ) job = self.storage.create_playbook_optimization_job( PlaybookOptimizationJob( + optimizer_kind="gepa", target_kind=target.kind, target_id=target.target_id, status="running", metadata_json=json.dumps(split_metadata, ensure_ascii=False), + attempt_key=( + f"gepa-user-{uuid.uuid4().hex}" + if target.kind == "user_playbook" + else None + ), ) ) run_request_id = optimizer_run_request_id(job.job_id) @@ -174,6 +226,7 @@ def optimize( self.request_context, self.llm_client, config.reflection_model, + frozen_request_plan=frozen_judge_plan, ), max_turns=config.max_turns, ) @@ -272,9 +325,20 @@ def optimize( ) return "completed" - successor_id = self._commit_if_allowed( - target, incumbent, best_content, config, run_request_id - ) + result_metadata = _result_metadata(result, split_metadata) + if target.kind == "user_playbook": + publication_result = self._publish_user_playbook_winner( + job=job, + incumbent_id=target.target_id, + winner_candidate_id=winner_candidate.candidate_id, + result_metadata=result_metadata, + run_request_id=run_request_id, + ) + successor_id = publication_result.successor_user_playbook_id + else: + successor_id = self._commit_if_allowed( + target, incumbent, best_content, config, run_request_id + ) logger.info( "event=playbook_optimization_committed job_id=%d candidate_id=%d " "successor_target_id=%s best_score=%.3f", @@ -283,18 +347,21 @@ def optimize( successor_id if successor_id is not None else "none", best_score, ) - self.storage.update_playbook_optimization_job( - job.job_id, - status="completed", - best_candidate_id=winner_candidate.candidate_id, - successor_target_id=successor_id, - decision_reason="committed" if successor_id else "winner persisted only", - metadata_json=json.dumps( - _result_metadata(result, split_metadata), - ensure_ascii=False, - default=str, - ), - ) + if target.kind == "agent_playbook": + self.storage.update_playbook_optimization_job( + job.job_id, + status="completed", + best_candidate_id=winner_candidate.candidate_id, + successor_target_id=successor_id, + decision_reason=( + "committed" if successor_id else "winner persisted only" + ), + metadata_json=json.dumps( + result_metadata, + ensure_ascii=False, + default=str, + ), + ) return "completed" def _run_gepa( @@ -419,29 +486,257 @@ def _passes_commit_thresholds( if best_score < config.min_commit_score: return False evaluations = self.storage.list_playbook_optimization_evaluations(job_id) - validation_keys = {_window_eval_key(window) for window in validation_windows} - winning_windows = { - _evaluation_key( - evaluation.scenario_user_playbook_id, - evaluation.source_interaction_ids, - ) - for evaluation in evaluations - if evaluation.candidate_id == candidate_id - and _evaluation_key( - evaluation.scenario_user_playbook_id, - evaluation.source_interaction_ids, - ) - in validation_keys - and evaluation.verdict == "candidate" - and evaluation.score >= config.min_commit_score - and evaluation.likert >= config.min_commit_likert - } - return len(winning_windows) >= config.min_commit_windows + authority = gepa_adoption_authority_from_config( + config=config, + validation_windows=validation_windows, + adoption_enabled=True, + ) + adoption = gepa_winner_adoption_result( + winner_candidate_id=candidate_id, + aggregate_score=best_score, + evaluations=evaluations, + authority=authority, + ) + return bool(adoption["passes"]) def _has_aborted_evaluations(self, job_id: int) -> bool: evaluations = self.storage.list_playbook_optimization_evaluations(job_id) return any(evaluation.verdict == "aborted" for evaluation in evaluations) + def _publish_user_playbook_winner( + self, + *, + job: PlaybookOptimizationJob, + incumbent_id: int, + winner_candidate_id: int, + result_metadata: dict[str, Any], + run_request_id: str, + ) -> PublicationResult: + incumbent = self.storage.get_user_playbook_by_id(incumbent_id) + if incumbent is None: + raise ValueError("GEPA publication incumbent no longer exists") + winner = next( + ( + candidate + for candidate in self.storage.list_playbook_optimization_candidates( + job.job_id + ) + if candidate.candidate_id == winner_candidate_id + ), + None, + ) + if winner is None: + raise ValueError("GEPA publication winner no longer exists") + projection = build_gepa_search_projection( + self.storage, incumbent, winner.content + ) + incumbent_content_digest = sha256(incumbent.content.encode("utf-8")).hexdigest() + incumbent_semantic_digest = incumbent_user_playbook_semantic_digest( + content_digest=incumbent_content_digest, + trigger=incumbent.trigger, + ) + subject_epochs_json = self.storage.get_user_playbook_publication_subject_epochs( + incumbent_id + ) + evaluations = self.storage.list_playbook_optimization_evaluations(job.job_id) + durable_metadata = json.loads(job.metadata_json) + proof = build_gepa_decision_proof( + job=job, + winner=winner, + evaluations=evaluations, + metadata=durable_metadata, + subject_epochs_json=subject_epochs_json, + projection_digest=projection.digest, + ) + owner = f"gepa-publication-{job.job_id}" + durable_job = self.storage.prepare_gepa_user_playbook_publication( + job_id=job.job_id, + owner=owner, + lease_seconds=_GEPA_PUBLICATION_LEASE_SECONDS, + winner_candidate_id=winner.candidate_id, + candidate_content_digest=projection.candidate_content_digest, + search_projection_digest=projection.digest, + publication_proof_digest=proof.digest, + projection_json=projection.canonical_json, + decision_proof_json=proof.canonical_json, + subject_epochs_json=subject_epochs_json, + metadata_json=json.dumps( + { + **result_metadata, + PUBLICATION_INCUMBENT_CONTENT_DIGEST_METADATA_KEY: ( + incumbent_content_digest + ), + PUBLICATION_INCUMBENT_TRIGGER_METADATA_KEY: incumbent.trigger, + PUBLICATION_INCUMBENT_SEMANTIC_DIGEST_METADATA_KEY: ( + incumbent_semantic_digest + ), + }, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + default=str, + ), + ) + if durable_job.attempt_key is None: + raise ValueError("GEPA publication attempt identity is missing") + if durable_job.lease_owner is None or durable_job.lease_fence <= 0: + raise ValueError("GEPA publication worker claim is missing") + service = UserPlaybookPublicationService( + self.storage, + GEPAUserPlaybookDecisionVerifier(self.storage), + ) + publication_claim = service.claim( + job_id=job.job_id, + owner=durable_job.lease_owner, + worker_fence=durable_job.lease_fence, + ) + request = PublicationRequest( + optimizer_kind="gepa", + job_id=job.job_id, + attempt_key=durable_job.attempt_key, + publication_claim=publication_claim, + worker_fence=durable_job.lease_fence, + incumbent_user_playbook_id=incumbent_id, + incumbent_content_digest=incumbent_content_digest, + incumbent_trigger=incumbent.trigger, + incumbent_semantic_digest=incumbent_semantic_digest, + revised_content=winner.content, + projection=projection, + decision_proof=proof, + subject_epochs_json=subject_epochs_json, + request_id=run_request_id, + ) + return self._publish_prepared_user_playbook_request(service, request) + + def _recover_gepa_user_playbook_publication( + self, target: PlaybookOptimizationTarget + ) -> PlaybookOptimizationRunStatus | None: + owner = f"gepa-publication-recovery-{target.target_id}" + try: + durable_job = self.storage.reclaim_gepa_user_playbook_publishing_job( + target.target_id, + owner, + _GEPA_PUBLICATION_LEASE_SECONDS, + ) + except OptimizationJobLeaseLiveError: + job = self.storage.get_unconsumed_gepa_user_playbook_publishing_job( + target.target_id + ) + logger.info( + "Skipping playbook optimization: GEPA publication lease is live " + "job_id=%s target_id=%d", + job.job_id if job is not None else "unknown", + target.target_id, + ) + return "skipped" + if durable_job is None: + return None + if durable_job.attempt_key is None: + raise ValueError("GEPA publication job disappeared during recovery") + if durable_job.lease_owner is None or durable_job.lease_fence <= 0: + raise ValueError("GEPA publication recovery claim is missing") + metadata = json.loads(durable_job.metadata_json) + projection_json = metadata.get(PUBLICATION_PROJECTION_JSON_METADATA_KEY) + proof_json = metadata.get(PUBLICATION_PROOF_JSON_METADATA_KEY) + subject_epochs = metadata.get(PUBLICATION_SUBJECT_EPOCHS_METADATA_KEY) + incumbent_content_digest = metadata.get( + PUBLICATION_INCUMBENT_CONTENT_DIGEST_METADATA_KEY + ) + incumbent_trigger = metadata.get(PUBLICATION_INCUMBENT_TRIGGER_METADATA_KEY) + incumbent_semantic_digest = metadata.get( + PUBLICATION_INCUMBENT_SEMANTIC_DIGEST_METADATA_KEY + ) + if ( + not isinstance(projection_json, str) + or not isinstance(proof_json, str) + or not isinstance(subject_epochs, dict) + or not isinstance(incumbent_content_digest, str) + or ( + incumbent_trigger is not None and not isinstance(incumbent_trigger, str) + ) + or not isinstance(incumbent_semantic_digest, str) + ): + raise ValueError("GEPA publication durable bytes are missing") + winner = next( + ( + candidate + for candidate in self.storage.list_playbook_optimization_candidates( + durable_job.job_id + ) + if candidate.candidate_id == durable_job.best_candidate_id + ), + None, + ) + if winner is None: + raise ValueError("GEPA publication winner is missing during recovery") + projection = parse_gepa_search_projection(projection_json) + proof = parse_gepa_decision_proof(proof_json) + service = UserPlaybookPublicationService( + self.storage, + GEPAUserPlaybookDecisionVerifier(self.storage), + ) + publication_claim = service.claim( + job_id=durable_job.job_id, + owner=durable_job.lease_owner, + worker_fence=durable_job.lease_fence, + ) + request = PublicationRequest( + optimizer_kind="gepa", + job_id=durable_job.job_id, + attempt_key=durable_job.attempt_key, + publication_claim=publication_claim, + worker_fence=durable_job.lease_fence, + incumbent_user_playbook_id=target.target_id, + incumbent_content_digest=incumbent_content_digest, + incumbent_trigger=incumbent_trigger, + incumbent_semantic_digest=incumbent_semantic_digest, + revised_content=winner.content, + projection=projection, + decision_proof=proof, + subject_epochs_json=json.dumps( + subject_epochs, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ), + request_id=optimizer_run_request_id(durable_job.job_id), + ) + self._publish_prepared_user_playbook_request(service, request) + return "completed" + + def _publish_prepared_user_playbook_request( + self, + service: UserPlaybookPublicationService, + request: PublicationRequest, + ) -> PublicationResult: + try: + publication_result = service.publish(request) + except Exception: + committed = service.load_committed(request.job_id) + if committed is None: + raise + publication_result = committed + + if publication_result.outcome == "applied": + successor_id = publication_result.successor_user_playbook_id + if successor_id is None: + raise RuntimeError("applied GEPA publication has no successor") + try: + successor = self.storage.get_user_playbook_by_id(successor_id) + if successor is not None and successor.agent_version: + maybe_trigger_user_playbook_aggregation( + request_context=self.request_context, + llm_client=self.llm_client, + agent_version=successor.agent_version, + reason="playbook_optimizer", + ) + except Exception: # noqa: BLE001 + logger.exception( + "playbook_optimizer aggregation trigger failed after successor commit", + extra={"successor_id": successor_id}, + ) + return publication_result + def _commit_if_allowed( self, target: PlaybookOptimizationTarget, @@ -482,39 +777,7 @@ def _commit_if_allowed( successor_id, source_windows ) return successor_id - current_user = self.storage.get_user_playbook_by_id(target.target_id) - if current_user is None or current_user.status is not None: - return None - if current_user.user_id is None: - return None - # The optimizer can legitimately flip framing (positive guidance -> - # negative anti-pattern or vice versa). Orientation lives entirely in - # the rule wording, so writing ``best_content`` is sufficient — there - # is no derived polarity label or separate polarity field to keep in - # sync. - successor_id = _supersede_user_playbook( - self.storage, - current_user, - best_content, - "playbook_optimizer", - request_id=run_request_id, - ) - if successor_id is not None: - try: - successor = self.storage.get_user_playbook_by_id(successor_id) - if successor is not None and successor.agent_version: - maybe_trigger_user_playbook_aggregation( - request_context=self.request_context, - llm_client=self.llm_client, - agent_version=successor.agent_version, - reason="playbook_optimizer", - ) - except Exception: # noqa: BLE001 - logger.exception( - "playbook_optimizer aggregation trigger failed after successor commit", - extra={"successor_id": successor_id}, - ) - return successor_id + raise ValueError("GEPA user playbooks require durable atomic publication") def _agent_like_playbook(playbook: UserPlaybook) -> AgentPlaybook: @@ -546,57 +809,6 @@ def _append_optimizer_metadata(existing: str, predecessor_id: int) -> str: return f"{existing}; {suffix}" -def _supersede_user_playbook( - storage: Any, - incumbent: UserPlaybook, - best_content: str, - source: str, - *, - request_id: str, -) -> int | None: - """Insert a user-playbook successor then atomically supersede the incumbent. - - Returns the new ``user_playbook_id`` on success, or ``None`` when the - incumbent is no longer CURRENT (lost race / already superseded). - - Args: - storage: A storage instance implementing the canonical atomic edit path. - incumbent: The current user playbook to replace. - best_content: Content for the successor playbook. - source: Provenance label written to the lineage event actor field. - request_id: Run-scoped correlation id for the lineage event. Must be - non-empty; use the job-derived id from the calling optimizer run. - - Returns: - int | None: ``user_playbook_id`` of the successor, or ``None`` if the - incumbent was not CURRENT. - - Raises: - ValueError: If ``request_id`` is empty or None. - """ - if not request_id: - raise ValueError( - "_supersede_user_playbook: request_id must be non-empty (run-correlation id)" - ) - successor = incumbent.model_copy( - update={"user_playbook_id": 0, "content": best_content, "status": None} - ) - ctx = LineageContext( - op_kind="revise", - actor=source, - request_id=request_id, - ) - successor_id = apply_playbook_edit( - storage, - incumbent_id=incumbent.user_playbook_id, - new_playbook=successor, - source=source, - request_id=request_id, - revise_context=ctx, - ) - return None if successor_id == -1 else successor_id - - class _LostAgentSupersedeRaceError(Exception): """Internal rollback signal for an agent-playbook successor race.""" @@ -779,17 +991,212 @@ def _split_metadata( windows: list[ScenarioWindow], train_windows: list[ScenarioWindow], validation_windows: list[ScenarioWindow], + *, + config: PlaybookOptimizerConfig, + assistant: AssistantCallable, + llm_client: LiteLLMClient, + prompt_manager: Any, + include_publication_authority: bool, ) -> dict[str, Any]: - return { + metadata = { "source_window_count": len(windows), "train_window_count": len(train_windows), + "train_windows": [ + { + "scenario_user_playbook_id": window.user_playbook_id, + "source_interaction_ids": list(window.source_interaction_ids), + } + for window in train_windows + ], "validation_window_count": len(validation_windows), "validation_scenario_user_playbook_ids": [ window.user_playbook_id for window in validation_windows ], + "validation_windows": [ + { + "scenario_user_playbook_id": window.user_playbook_id, + "source_interaction_ids": list(window.source_interaction_ids), + } + for window in validation_windows + ], + } + if include_publication_authority: + metadata[GEPA_PUBLICATION_AUTHORITY_METADATA_KEY] = _gepa_publication_authority( + config=config, + train_windows=train_windows, + validation_windows=validation_windows, + assistant=assistant, + llm_client=llm_client, + prompt_manager=prompt_manager, + ) + return metadata + + +def _gepa_publication_authority( + *, + config: PlaybookOptimizerConfig, + train_windows: list[ScenarioWindow], + validation_windows: list[ScenarioWindow], + assistant: AssistantCallable, + llm_client: LiteLLMClient, + prompt_manager: Any, +) -> dict[str, Any]: + judge_model_id = config.reflection_model or llm_client.config.model + adoption_authority = gepa_adoption_authority_from_config( + config=config, + validation_windows=validation_windows, + ) + adoption_authority["validation_manifest"]["digest"] = _window_manifest_digest( + validation_windows + ) + return { + **adoption_authority, + "backend_identity": _assistant_backend_identity(config, assistant), + "budget_settings": { + "max_metric_calls": config.max_metric_calls, + "max_turns": config.max_turns, + "reflection_minibatch_size": config.reflection_minibatch_size, + }, + "evaluator_identity": { + "pairwise_judge_request_plan": build_pairwise_judge_request_plan( + prompt_manager=prompt_manager, + llm_client=llm_client, + model_name=judge_model_id, + ), + }, + "gepa_algorithm": { + "batch_sampler": "epoch_shuffled", + "cache_evaluation": True, + "candidate_selection_strategy": "pareto", + "display_progress_bar": False, + "frontier_type": "instance", + "raise_on_exception": False, + }, + "gepa_engine_identity": _gepa_engine_identity(), + "merge_settings": { + "max_merge_invocations": config.max_merge_invocations, + "use_merge": config.use_merge, + }, + "model_identity": { + "default_lm": llm_client.config.model, + "reflection_lm": judge_model_id, + }, + "optimizer_identity": { + "adapter_class": _code_identity(ReflexioPlaybookGEPAAdapter), + "adapter_code_digest": _code_digest(ReflexioPlaybookGEPAAdapter), + "rollout_class": _code_identity(MultiTurnRollout), + "rollout_code_digest": _code_digest(MultiTurnRollout), + }, + "split_settings": { + "max_validation_windows": config.max_validation_windows, + }, + "stop_settings": { + "early_stop_score": _decimal_string(config.early_stop_score), + "stopper_class": "gepa.utils.stop_condition.ScoreThresholdStopper", + }, + "train_manifest": { + "digest": _window_manifest_digest(train_windows), + "windows": [_window_manifest_item(window) for window in train_windows], + }, + } + + +def _gepa_engine_identity() -> dict[str, Any]: + try: + package_version = importlib.metadata.version("gepa") + except importlib.metadata.PackageNotFoundError: + package_version = "unknown" + try: + from gepa.api import optimize as gepa_optimize + + optimize_code_digest = _code_digest(gepa_optimize) + optimize_identity = _code_identity(gepa_optimize) + except Exception: # noqa: BLE001 + optimize_code_digest = _text_digest("gepa.api.optimize") + optimize_identity = "gepa.api.optimize" + return { + "optimize_callable": optimize_identity, + "optimize_code_digest": optimize_code_digest, + "package_name": "gepa", + "package_version": package_version, } +def _assistant_backend_identity( + config: PlaybookOptimizerConfig, + assistant: AssistantCallable, +) -> dict[str, Any]: + identity: dict[str, Any] = { + "backoff_base_seconds": _decimal_string(config.webhook_backoff_base_seconds), + "backend_class": _code_identity(type(assistant)), + "backend_class_code_digest": _code_digest(type(assistant)), + "max_retries": config.webhook_max_retries, + "timeout_seconds": config.webhook_timeout_seconds, + } + if config.webhook_url: + identity["backend_kind"] = "webhook" + identity["webhook_auth_configured"] = bool(config.webhook_auth_header) + identity["webhook_url_digest"] = _text_digest(config.webhook_url) + identity["webhook_auth_scheme"] = _webhook_auth_scheme( + config.webhook_auth_header + ) + elif config.assistant_script_path: + identity["backend_kind"] = "local_script" + identity["script_args_digest"] = _json_digest(config.assistant_script_args) + identity["script_content_digest"] = sha256( + Path(config.assistant_script_path).read_bytes() + ).hexdigest() + else: + identity["backend_kind"] = "none" + return identity + + +def _webhook_auth_scheme(value: str | None) -> str | None: + if not value: + return None + scheme, separator, _credential = value.strip().partition(" ") + return scheme if separator and scheme else None + + +def _window_manifest_item(window: ScenarioWindow) -> dict[str, Any]: + return { + "scenario_user_playbook_id": window.user_playbook_id, + "source_interaction_ids": list(window.source_interaction_ids), + } + + +def _window_manifest_digest(windows: list[ScenarioWindow]) -> str: + return _json_digest([_window_manifest_item(window) for window in windows]) + + +def _json_digest(value: Any) -> str: + return sha256( + json.dumps( + value, ensure_ascii=False, separators=(",", ":"), sort_keys=True + ).encode("utf-8") + ).hexdigest() + + +def _text_digest(value: str) -> str: + return sha256(value.encode("utf-8")).hexdigest() + + +def _code_identity(value: Any) -> str: + return f"{value.__module__}.{value.__qualname__}" + + +def _code_digest(value: Any) -> str: + try: + source = inspect.getsource(value) + except (OSError, TypeError): + source = _code_identity(value) + return _text_digest(source) + + +def _decimal_string(value: float) -> str: + return repr(float(value)) + + def _is_executable_file(path: str) -> bool: candidate = Path(path) return candidate.is_file() and os.access(candidate, os.X_OK) @@ -805,9 +1212,3 @@ def _result_metadata(result: Any, split_metadata: dict[str, Any]) -> dict[str, A def _window_eval_key(window: ScenarioWindow) -> tuple[int | None, tuple[int, ...]]: return window.user_playbook_id, tuple(window.source_interaction_ids) - - -def _evaluation_key( - scenario_user_playbook_id: int | None, source_interaction_ids: list[int] -) -> tuple[int | None, tuple[int, ...]]: - return scenario_user_playbook_id, tuple(source_interaction_ids) diff --git a/reflexio/server/services/storage/error.py b/reflexio/server/services/storage/error.py index 859d73473..cf696ed79 100644 --- a/reflexio/server/services/storage/error.py +++ b/reflexio/server/services/storage/error.py @@ -18,6 +18,10 @@ class SubjectWriteBarrierError(StorageError): """Raised when a write targets a subject with an active erasure barrier.""" +class OptimizationJobLeaseLiveError(StorageError): + """Raised when an optimizer recovery attempt finds a non-expired lease.""" + + def require_non_empty_session_id(value: Any) -> str: """Return a stripped, non-empty request ``session_id`` or raise ``StorageError``. diff --git a/reflexio/server/services/storage/retention.py b/reflexio/server/services/storage/retention.py index 48fa6fef9..d48035689 100644 --- a/reflexio/server/services/storage/retention.py +++ b/reflexio/server/services/storage/retention.py @@ -22,6 +22,22 @@ class RetentionTarget: priority_statuses: tuple[str, ...] = () +@dataclass(frozen=True, slots=True) +class OptimizationRetentionClass: + """Fixed owner for one optimization artifact lifetime.""" + + artifact_class: str + owner: str + + +OPTIMIZATION_RETENTION_CLASSES: tuple[OptimizationRetentionClass, ...] = ( + OptimizationRetentionClass("event", "governance_audit"), + OptimizationRetentionClass("source_reference", "existing_source_retention"), + OptimizationRetentionClass("staging", "lease_stale_claim"), + OptimizationRetentionClass("terminal", "lineage_tombstone_grace"), +) + + RETENTION_TARGETS: tuple[RetentionTarget, ...] = ( RetentionTarget( "profiles", diff --git a/reflexio/server/services/storage/retention_mixin.py b/reflexio/server/services/storage/retention_mixin.py index fd7be0caf..969cab1e2 100644 --- a/reflexio/server/services/storage/retention_mixin.py +++ b/reflexio/server/services/storage/retention_mixin.py @@ -104,6 +104,38 @@ def delete_oldest_retention_target_rows(self, target_name: str, count: int) -> i self._retention_perform_delete(target, keys) return len(keys) + def gc_retired_optimization_jobs( + self, + *, + older_than_epoch: int, + stale_before_epoch: int, + limit: int = 1000, + ) -> int: + """Apply fixed optimization terminal and staging retention ownership. + + Enterprise SQL backends override the protected hook. OSS backends have + no provider/publication staging tables and therefore return zero. + """ + if older_than_epoch < 0 or stale_before_epoch < 0: + raise ValueError("optimization retention cutoffs must be non-negative") + if limit <= 0: + raise ValueError("optimization retention limit must be positive") + return self._retention_gc_retired_optimization_jobs( + older_than_epoch=older_than_epoch, + stale_before_epoch=stale_before_epoch, + limit=limit, + ) + + def _retention_gc_retired_optimization_jobs( + self, + *, + older_than_epoch: int, + stale_before_epoch: int, + limit: int, + ) -> int: + del older_than_epoch, stale_before_epoch, limit + return 0 + def _retention_select_keys( self, target: RetentionTarget, count: int ) -> list[tuple[Any, ...]]: diff --git a/reflexio/server/services/storage/sqlite_storage/_base.py b/reflexio/server/services/storage/sqlite_storage/_base.py index c4fe1c60b..adc799ec1 100644 --- a/reflexio/server/services/storage/sqlite_storage/_base.py +++ b/reflexio/server/services/storage/sqlite_storage/_base.py @@ -759,6 +759,7 @@ def migrate(self) -> bool: # absent), so running it before _DDL is safe on fresh databases too. self._migrate_eval_result_user_id() self._migrate_retrieved_learning_interaction_identity() + self._migrate_playbook_optimization_job_columns() with self._lock: cur = self.conn.cursor() cur.executescript(_DDL) @@ -785,6 +786,9 @@ def migrate(self) -> bool: self._migrate_retired_at() self._migrate_lineage_event_table() self._migrate_playbook_optimization_candidate_metadata() + self._migrate_user_playbook_publication_staging_columns() + self._classify_legacy_playbook_optimization_jobs() + self._enforce_playbook_optimization_job_constraints() self._migrate_retire_profile_change_logs() self._migrate_retire_playbook_aggregation_change_logs() init_stall_state_table(self.conn) @@ -1268,6 +1272,367 @@ def _migrate_playbook_optimization_candidate_metadata(self) -> None: ) self.conn.commit() + def _migrate_playbook_optimization_job_columns(self) -> None: + """Add durable optimizer identity, lease, stage, and digest columns.""" + existing_cols = { + row["name"] + for row in self.conn.execute( + "PRAGMA table_info(playbook_optimization_jobs)" + ).fetchall() + } + if not existing_cols: + return + columns = { + "optimizer_kind": "TEXT", + "discovery_key": "TEXT", + "attempt_key": "TEXT", + "lease_owner": "TEXT", + "lease_fence": "INTEGER NOT NULL DEFAULT 0", + "lease_expires_at": "INTEGER", + "stage": "TEXT", + "terminal_outcome": "TEXT", + "expected_population_manifest_digest": "TEXT", + "generation_selection_manifest_digest": "TEXT", + "replay_manifest_digest": "TEXT", + "candidate_content_digest": "TEXT", + "search_projection_digest": "TEXT", + "publication_scope_digest": "TEXT", + } + for column, definition in columns.items(): + if column not in existing_cols: + self.conn.execute( + f"ALTER TABLE playbook_optimization_jobs " + f"ADD COLUMN {column} {definition}" # noqa: S608 + ) + self.conn.commit() + + def _migrate_user_playbook_publication_staging_columns(self) -> None: + """Add frozen incumbent CAS fields to legacy publication staging.""" + existing_cols = { + row["name"] + for row in self.conn.execute( + "PRAGMA table_info(user_playbook_publication_staging)" + ).fetchall() + } + if not existing_cols: + return + columns = { + "incumbent_content_digest": "TEXT", + "incumbent_trigger": "TEXT", + "incumbent_semantic_digest": "TEXT", + } + for column, definition in columns.items(): + if column not in existing_cols: + self.conn.execute( + "ALTER TABLE user_playbook_publication_staging " + f"ADD COLUMN {column} {definition}" # noqa: S608 + ) + self.conn.commit() + + def _classify_legacy_playbook_optimization_jobs(self) -> None: + """Classify legacy optimizer history without choosing ambiguous rule order.""" + columns = { + row["name"] + for row in self.conn.execute( + "PRAGMA table_info(playbook_optimization_jobs)" + ).fetchall() + } + if "optimizer_kind" not in columns: + return + for index_name in ( + "uq_poj_active_discovery", + "uq_poj_active_attempt", + "uq_poj_active_target", + ): + self.conn.execute(f"DROP INDEX IF EXISTS {index_name}") # noqa: S608 + self.conn.execute( + """ + WITH signatures AS ( + SELECT + jobs.job_id, + ( + EXISTS ( + SELECT 1 + FROM playbook_optimization_events AS events + WHERE events.job_id = jobs.job_id + AND events.event_type LIKE 'offline_tuner_%' + ) + OR ( + json_valid(jobs.metadata_json) + AND json_type(jobs.metadata_json, '$.offline_tuner') + IS NOT NULL + ) + OR EXISTS ( + SELECT 1 + FROM playbook_optimization_candidates AS candidates + WHERE candidates.job_id = jobs.job_id + AND json_valid(candidates.metadata_json) + AND ( + json_type( + candidates.metadata_json, + '$.offline_tuner_metrics' + ) IS NOT NULL + OR json_type( + candidates.metadata_json, + '$.rollback_baseline' + ) IS NOT NULL + OR json_type( + candidates.metadata_json, + '$.frozen_selection_set' + ) IS NOT NULL + OR json_type( + candidates.metadata_json, + '$.proposed_edit' + ) IS NOT NULL + ) + ) + ) AS tuner_signature, + ( + json_valid(jobs.metadata_json) + AND json_type(jobs.metadata_json, '$.source_window_count') + IS NOT NULL + AND json_type(jobs.metadata_json, '$.train_window_count') + IS NOT NULL + AND json_type( + jobs.metadata_json, + '$.validation_window_count' + ) IS NOT NULL + ) AS gepa_signature + FROM playbook_optimization_jobs AS jobs + WHERE jobs.optimizer_kind IS NULL + ) + UPDATE playbook_optimization_jobs + SET optimizer_kind = CASE + WHEN tuner_signature AND NOT gepa_signature + THEN 'offline_tuner_legacy' + WHEN gepa_signature AND NOT tuner_signature THEN 'gepa' + ELSE 'optimizer_legacy_unknown' + END + FROM signatures + WHERE playbook_optimization_jobs.job_id = signatures.job_id + """ + ) + self.conn.execute( + """ + WITH ranked AS ( + SELECT + job_id, + row_number() OVER ( + PARTITION BY optimizer_kind, target_kind, target_id + ORDER BY created_at, job_id + ) AS target_rank, + CASE WHEN discovery_key IS NOT NULL THEN row_number() OVER ( + PARTITION BY optimizer_kind, discovery_key + ORDER BY created_at, job_id + ) END AS discovery_rank, + CASE WHEN attempt_key IS NOT NULL THEN row_number() OVER ( + PARTITION BY optimizer_kind, attempt_key + ORDER BY created_at, job_id + ) END AS attempt_rank + FROM playbook_optimization_jobs + WHERE status IN ('pending', 'running') + ) + UPDATE playbook_optimization_jobs + SET status = 'skipped', + decision_reason = 'retired_duplicate_legacy_active_job', + lease_owner = NULL, + lease_expires_at = NULL, + updated_at = CAST(strftime('%s', 'now') AS INTEGER) + WHERE job_id IN ( + SELECT job_id + FROM ranked + WHERE target_rank > 1 + OR discovery_rank > 1 + OR attempt_rank > 1 + ) + """ + ) + self.conn.execute( + """ + UPDATE playbook_optimization_jobs + SET status = 'skipped', + decision_reason = 'retired_by_replay_redesign', + lease_owner = NULL, + lease_expires_at = NULL, + updated_at = CAST(strftime('%s', 'now') AS INTEGER) + WHERE optimizer_kind IN ( + 'offline_tuner_legacy', + 'optimizer_legacy_unknown' + ) + AND status IN ('pending', 'running') + """ + ) + self.conn.execute( + """CREATE UNIQUE INDEX uq_poj_active_discovery + ON playbook_optimization_jobs(optimizer_kind, discovery_key) + WHERE status IN ('pending', 'running') + AND discovery_key IS NOT NULL""" + ) + self.conn.execute( + """CREATE UNIQUE INDEX uq_poj_active_attempt + ON playbook_optimization_jobs(optimizer_kind, attempt_key) + WHERE status IN ('pending', 'running') + AND attempt_key IS NOT NULL""" + ) + self.conn.execute( + """CREATE UNIQUE INDEX uq_poj_active_target + ON playbook_optimization_jobs(optimizer_kind, target_kind, target_id) + WHERE status IN ('pending', 'running')""" + ) + self.conn.commit() + + def _enforce_playbook_optimization_job_constraints(self) -> None: + """Rebuild upgraded optimizer tables so legacy values receive fresh checks.""" + table_sql_row = self.conn.execute( + """SELECT sql FROM sqlite_master + WHERE type = 'table' AND name = 'playbook_optimization_jobs'""" + ).fetchone() + if table_sql_row is None: + return + table_sql = table_sql_row["sql"] + required_checks = ( + "CHECK (optimizer_kind IN", + "CHECK (stage IS NULL OR stage IN", + "CHECK (terminal_outcome IS NULL OR terminal_outcome IN", + "'governance_erased'", + ) + if all(check in table_sql for check in required_checks): + return + foreign_keys_enabled = bool( + self.conn.execute("PRAGMA foreign_keys").fetchone()[0] + ) + self.conn.commit() + if foreign_keys_enabled: + self.conn.execute("PRAGMA foreign_keys=OFF") + try: + self.conn.execute("BEGIN IMMEDIATE") + self.conn.execute("DROP TABLE IF EXISTS playbook_optimization_jobs_new") + self.conn.execute( + """ + CREATE TABLE playbook_optimization_jobs_new ( + job_id INTEGER PRIMARY KEY AUTOINCREMENT, + optimizer_kind TEXT NOT NULL DEFAULT 'optimizer_legacy_unknown' + CHECK (optimizer_kind IN ( + 'gepa', + 'offline_tuner_replay', + 'offline_tuner_legacy', + 'optimizer_legacy_unknown' + )), + target_kind TEXT NOT NULL, + target_id INTEGER NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + best_candidate_id INTEGER, + successor_target_id INTEGER, + decision_reason TEXT NOT NULL DEFAULT '', + metadata_json TEXT NOT NULL DEFAULT '{}', + discovery_key TEXT, + attempt_key TEXT, + lease_owner TEXT, + lease_fence INTEGER NOT NULL DEFAULT 0 CHECK (lease_fence >= 0), + lease_expires_at INTEGER, + stage TEXT CHECK (stage IS NULL OR stage IN ( + 'evidence_frozen', + 'candidate_generated', + 'replay_running', + 'replay_evaluated', + 'publishing', + 'applied', + 'abstained', + 'failed' + )), + terminal_outcome TEXT CHECK (terminal_outcome IS NULL OR terminal_outcome IN ( + 'applied', + 'insufficient_negative_evidence', + 'insufficient_positive_evidence', + 'insufficient_coverage', + 'replay_unsupported', + 'deployment_unsupported', + 'incomplete_replay_scope', + 'insufficient_replay_cases', + 'replay_inconclusive', + 'candidate_regressed', + 'candidate_did_not_improve', + 'incumbent_changed', + 'generation_failed', + 'replay_failed', + 'publication_failed', + 'governance_erased' + )), + expected_population_manifest_digest TEXT, + generation_selection_manifest_digest TEXT, + replay_manifest_digest TEXT, + candidate_content_digest TEXT, + search_projection_digest TEXT, + publication_scope_digest TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) + """ + ) + self.conn.execute( + """ + INSERT INTO playbook_optimization_jobs_new ( + job_id, optimizer_kind, target_kind, target_id, status, + best_candidate_id, successor_target_id, decision_reason, + metadata_json, discovery_key, attempt_key, lease_owner, + lease_fence, lease_expires_at, stage, terminal_outcome, + expected_population_manifest_digest, + generation_selection_manifest_digest, replay_manifest_digest, + candidate_content_digest, search_projection_digest, + publication_scope_digest, created_at, updated_at + ) SELECT + job_id, optimizer_kind, target_kind, target_id, status, + best_candidate_id, successor_target_id, decision_reason, + metadata_json, discovery_key, attempt_key, lease_owner, + lease_fence, lease_expires_at, stage, terminal_outcome, + expected_population_manifest_digest, + generation_selection_manifest_digest, replay_manifest_digest, + candidate_content_digest, search_projection_digest, + publication_scope_digest, created_at, updated_at + FROM playbook_optimization_jobs; + """ + ) + self.conn.execute("DROP TABLE playbook_optimization_jobs") + self.conn.execute( + "ALTER TABLE playbook_optimization_jobs_new " + "RENAME TO playbook_optimization_jobs" + ) + self.conn.execute( + "CREATE INDEX idx_poj_target " + "ON playbook_optimization_jobs(target_kind, target_id)" + ) + self.conn.execute( + "CREATE INDEX idx_poj_status ON playbook_optimization_jobs(status)" + ) + self.conn.execute( + """CREATE UNIQUE INDEX uq_poj_active_discovery + ON playbook_optimization_jobs(optimizer_kind, discovery_key) + WHERE status IN ('pending', 'running') AND discovery_key IS NOT NULL""" + ) + self.conn.execute( + """CREATE UNIQUE INDEX uq_poj_active_attempt + ON playbook_optimization_jobs(optimizer_kind, attempt_key) + WHERE status IN ('pending', 'running') AND attempt_key IS NOT NULL""" + ) + self.conn.execute( + """CREATE UNIQUE INDEX uq_poj_active_target + ON playbook_optimization_jobs(optimizer_kind, target_kind, target_id) + WHERE status IN ('pending', 'running')""" + ) + violations = self.conn.execute("PRAGMA foreign_key_check").fetchall() + if violations: + raise sqlite3.IntegrityError( + f"foreign key check failed after optimizer migration: {violations}" + ) + self.conn.commit() + except Exception: + self.conn.rollback() + raise + finally: + self.conn.execute( + f"PRAGMA foreign_keys={'ON' if foreign_keys_enabled else 'OFF'}" + ) + def _migrate_retire_profile_change_logs(self) -> None: """Retire the frozen ``profile_change_logs`` table via a reversible RENAME. @@ -2130,6 +2495,13 @@ def clear_user_data(self, user_id: str) -> dict[str, int]: CREATE TABLE IF NOT EXISTS playbook_optimization_jobs ( job_id INTEGER PRIMARY KEY AUTOINCREMENT, + optimizer_kind TEXT NOT NULL DEFAULT 'optimizer_legacy_unknown' + CHECK (optimizer_kind IN ( + 'gepa', + 'offline_tuner_replay', + 'offline_tuner_legacy', + 'optimizer_legacy_unknown' + )), target_kind TEXT NOT NULL, target_id INTEGER NOT NULL, status TEXT NOT NULL DEFAULT 'pending', @@ -2137,11 +2509,108 @@ def clear_user_data(self, user_id: str) -> dict[str, int]: successor_target_id INTEGER, decision_reason TEXT NOT NULL DEFAULT '', metadata_json TEXT NOT NULL DEFAULT '{}', + discovery_key TEXT, + attempt_key TEXT, + lease_owner TEXT, + lease_fence INTEGER NOT NULL DEFAULT 0 CHECK (lease_fence >= 0), + lease_expires_at INTEGER, + stage TEXT CHECK (stage IS NULL OR stage IN ( + 'evidence_frozen', + 'candidate_generated', + 'replay_running', + 'replay_evaluated', + 'publishing', + 'applied', + 'abstained', + 'failed' + )), + terminal_outcome TEXT CHECK (terminal_outcome IS NULL OR terminal_outcome IN ( + 'applied', + 'insufficient_negative_evidence', + 'insufficient_positive_evidence', + 'insufficient_coverage', + 'replay_unsupported', + 'deployment_unsupported', + 'incomplete_replay_scope', + 'insufficient_replay_cases', + 'replay_inconclusive', + 'candidate_regressed', + 'candidate_did_not_improve', + 'incumbent_changed', + 'generation_failed', + 'replay_failed', + 'publication_failed', + 'governance_erased' + )), + expected_population_manifest_digest TEXT, + generation_selection_manifest_digest TEXT, + replay_manifest_digest TEXT, + candidate_content_digest TEXT, + search_projection_digest TEXT, + publication_scope_digest TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL ); CREATE INDEX IF NOT EXISTS idx_poj_target ON playbook_optimization_jobs(target_kind, target_id); CREATE INDEX IF NOT EXISTS idx_poj_status ON playbook_optimization_jobs(status); +CREATE UNIQUE INDEX IF NOT EXISTS uq_poj_active_discovery + ON playbook_optimization_jobs(optimizer_kind, discovery_key) + WHERE status IN ('pending', 'running') AND discovery_key IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS uq_poj_active_attempt + ON playbook_optimization_jobs(optimizer_kind, attempt_key) + WHERE status IN ('pending', 'running') AND attempt_key IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS uq_poj_active_target + ON playbook_optimization_jobs(optimizer_kind, target_kind, target_id) + WHERE status IN ('pending', 'running'); + +CREATE TABLE IF NOT EXISTS case_first_exposure ( + equivalence_group_id TEXT PRIMARY KEY, + first_role TEXT NOT NULL CHECK (first_role IN ( + 'generation', 'improvement', 'preservation', 'safety' + )), + owner_discovery_key TEXT NOT NULL, + content_digest TEXT NOT NULL, + created_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS case_exposure_uses ( + job_id INTEGER NOT NULL, + equivalence_group_id TEXT NOT NULL, + role TEXT NOT NULL CHECK (role IN ( + 'generation', 'improvement', 'preservation', 'safety' + )), + ownership TEXT NOT NULL CHECK (ownership IN ( + 'self_generation', 'self_improvement', 'self_preservation', + 'self_safety', 'foreign' + )), + content_digest TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (job_id, equivalence_group_id, role), + FOREIGN KEY (job_id) REFERENCES playbook_optimization_jobs(job_id) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idx_case_exposure_uses_group + ON case_exposure_uses(equivalence_group_id); + +CREATE TABLE IF NOT EXISTS playbook_optimization_artifacts ( + artifact_id INTEGER PRIMARY KEY AUTOINCREMENT, + job_id INTEGER NOT NULL, + artifact_kind TEXT NOT NULL CHECK (artifact_kind IN ( + 'expected_population_manifest', + 'generation_selection', + 'replay_manifest', + 'candidate', + 'candidate_search_projection' + )), + content_json TEXT NOT NULL, + content_digest TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE (job_id, artifact_kind), + FOREIGN KEY (job_id) REFERENCES playbook_optimization_jobs(job_id) + ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idx_poa_job + ON playbook_optimization_artifacts(job_id); CREATE TABLE IF NOT EXISTS playbook_optimization_candidates ( candidate_id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -2185,6 +2654,57 @@ def clear_user_data(self, user_id: str) -> dict[str, int]: ); CREATE INDEX IF NOT EXISTS idx_poev_job ON playbook_optimization_events(job_id); +CREATE TABLE IF NOT EXISTS user_playbook_publication_claims ( + optimizer_kind TEXT NOT NULL, + job_id INTEGER NOT NULL, + owner TEXT NOT NULL, + publication_fence INTEGER NOT NULL CHECK (publication_fence >= 1), + worker_fence INTEGER NOT NULL CHECK (worker_fence >= 1), + consumed INTEGER NOT NULL DEFAULT 0 CHECK (consumed IN (0, 1)), + updated_at INTEGER NOT NULL, + PRIMARY KEY (optimizer_kind, job_id), + FOREIGN KEY (job_id) REFERENCES playbook_optimization_jobs(job_id) + ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS user_playbook_publication_staging ( + optimizer_kind TEXT NOT NULL, + job_id INTEGER NOT NULL, + attempt_key TEXT NOT NULL, + claim_owner TEXT NOT NULL, + publication_fence INTEGER NOT NULL, + worker_fence INTEGER NOT NULL, + incumbent_user_playbook_id INTEGER NOT NULL, + incumbent_content_digest TEXT NOT NULL, + incumbent_trigger TEXT, + incumbent_semantic_digest TEXT NOT NULL, + revised_content TEXT NOT NULL, + content_digest TEXT NOT NULL, + projection_json TEXT NOT NULL, + projection_digest TEXT NOT NULL, + proof_json TEXT NOT NULL, + proof_digest TEXT NOT NULL, + subject_epochs_json TEXT NOT NULL, + request_id TEXT NOT NULL, + staging_digest TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (optimizer_kind, job_id), + FOREIGN KEY (job_id) REFERENCES playbook_optimization_jobs(job_id) + ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS user_playbook_publication_results ( + optimizer_kind TEXT NOT NULL, + job_id INTEGER NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('applied', 'incumbent_changed')), + successor_user_playbook_id INTEGER, + staging_digest TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (optimizer_kind, job_id), + FOREIGN KEY (job_id) REFERENCES playbook_optimization_jobs(job_id) + ON DELETE CASCADE +); + CREATE TABLE IF NOT EXISTS _operation_state ( service_name TEXT PRIMARY KEY, operation_state TEXT NOT NULL DEFAULT '{}', diff --git a/reflexio/server/services/storage/sqlite_storage/playbook/_optimization.py b/reflexio/server/services/storage/sqlite_storage/playbook/_optimization.py index 40d43b4b9..d97fd6f24 100644 --- a/reflexio/server/services/storage/sqlite_storage/playbook/_optimization.py +++ b/reflexio/server/services/storage/sqlite_storage/playbook/_optimization.py @@ -1,14 +1,201 @@ """Playbook optimization job store methods for SQLite storage.""" +import json import sqlite3 +import time +from hashlib import sha256 from typing import Any +from reflexio.models.api_schema.domain.entities import canonicalize_artifact_json from reflexio.models.api_schema.service_schemas import ( + OptimizationArtifactKind, + OptimizationJobClaim, + OptimizationJobStage, + OptimizationTerminalOutcome, + PlaybookOptimizationArtifact, PlaybookOptimizationCandidate, PlaybookOptimizationEvaluation, PlaybookOptimizationEvent, PlaybookOptimizationJob, ) +from reflexio.server.services.playbook.publication import ( + PUBLICATION_PROJECTION_JSON_METADATA_KEY, + PUBLICATION_PROOF_JSON_METADATA_KEY, + PUBLICATION_SUBJECT_EPOCHS_METADATA_KEY, + canonical_json_bytes, +) +from reflexio.server.services.storage.error import ( + OptimizationJobLeaseLiveError, + StorageError, +) + +_FAILURE_OUTCOMES = {"generation_failed", "replay_failed", "publication_failed"} +_ABSTENTION_OUTCOMES = { + "insufficient_negative_evidence", + "insufficient_positive_evidence", + "insufficient_coverage", + "replay_unsupported", + "deployment_unsupported", + "incomplete_replay_scope", + "insufficient_replay_cases", + "replay_inconclusive", + "candidate_regressed", + "candidate_did_not_improve", + "incumbent_changed", +} +_STAGE_PREDECESSORS: dict[str, str] = { + "candidate_generated": "evidence_frozen", + "replay_running": "candidate_generated", + "replay_evaluated": "replay_running", + "publishing": "replay_evaluated", + "applied": "publishing", +} + + +def _row_to_playbook_optimization_job(row: sqlite3.Row) -> PlaybookOptimizationJob: + return PlaybookOptimizationJob( + job_id=row["job_id"], + optimizer_kind=row["optimizer_kind"], + target_kind=row["target_kind"], + target_id=row["target_id"], + status=row["status"], + best_candidate_id=row["best_candidate_id"], + successor_target_id=row["successor_target_id"], + decision_reason=row["decision_reason"], + metadata_json=row["metadata_json"] or "{}", + discovery_key=row["discovery_key"], + attempt_key=row["attempt_key"], + lease_owner=row["lease_owner"], + lease_fence=row["lease_fence"], + lease_expires_at=row["lease_expires_at"], + stage=row["stage"], + terminal_outcome=row["terminal_outcome"], + expected_population_manifest_digest=row["expected_population_manifest_digest"], + generation_selection_manifest_digest=row[ + "generation_selection_manifest_digest" + ], + replay_manifest_digest=row["replay_manifest_digest"], + candidate_content_digest=row["candidate_content_digest"], + search_projection_digest=row["search_projection_digest"], + publication_scope_digest=row["publication_scope_digest"], + created_at=row["created_at"], + updated_at=row["updated_at"], + ) + + +def _row_to_playbook_optimization_artifact( + row: sqlite3.Row, +) -> PlaybookOptimizationArtifact: + return PlaybookOptimizationArtifact( + artifact_id=row["artifact_id"], + job_id=row["job_id"], + artifact_kind=row["artifact_kind"], + content_json=row["content_json"], + content_digest=row["content_digest"], + created_at=row["created_at"], + updated_at=row["updated_at"], + ) + + +def _job_insert_values(job: PlaybookOptimizationJob) -> tuple[Any, ...]: + return ( + job.optimizer_kind, + job.target_kind, + job.target_id, + job.status, + job.best_candidate_id, + job.successor_target_id, + job.decision_reason, + job.metadata_json, + job.discovery_key, + job.attempt_key, + job.lease_owner, + job.lease_fence, + job.lease_expires_at, + job.stage, + job.terminal_outcome, + job.expected_population_manifest_digest, + job.generation_selection_manifest_digest, + job.replay_manifest_digest, + job.candidate_content_digest, + job.search_projection_digest, + job.publication_scope_digest, + job.created_at, + job.updated_at, + ) + + +_JOB_INSERT_SQL = """INSERT INTO playbook_optimization_jobs + (optimizer_kind, target_kind, target_id, status, best_candidate_id, + successor_target_id, decision_reason, metadata_json, discovery_key, + attempt_key, lease_owner, lease_fence, lease_expires_at, stage, + terminal_outcome, expected_population_manifest_digest, + generation_selection_manifest_digest, replay_manifest_digest, + candidate_content_digest, search_projection_digest, + publication_scope_digest, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""" + + +def _gepa_publication_authority_from_metadata(metadata_json: str | None) -> dict: + try: + metadata = json.loads(metadata_json or "{}") + except json.JSONDecodeError as exc: + raise StorageError("GEPA durable publication authority is invalid") from exc + authority = metadata.get("gepa_publication_authority") + if not isinstance(authority, dict): + raise StorageError("GEPA durable publication authority is missing") + return authority + + +def _validate_gepa_publication_prepare_metadata( + *, + lease_seconds: int, + candidate_content_digest: str, + search_projection_digest: str, + publication_proof_digest: str, + projection_json: str, + decision_proof_json: str, + subject_epochs_json: str, + metadata_json: str, +) -> dict[str, Any]: + if lease_seconds <= 0: + raise ValueError("lease_seconds must be positive") + digests = ( + candidate_content_digest, + search_projection_digest, + publication_proof_digest, + ) + if any( + len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest) + for digest in digests + ): + raise ValueError("GEPA publication digests must be lowercase SHA-256") + try: + metadata = json.loads(metadata_json) + json.loads(subject_epochs_json) + except json.JSONDecodeError as exc: + raise ValueError("GEPA publication metadata must be valid JSON") from exc + if not isinstance(metadata, dict): + raise ValueError("GEPA publication metadata must be an object") + if sha256(projection_json.encode("utf-8")).hexdigest() != search_projection_digest: + raise ValueError("GEPA projection digest does not match canonical JSON") + if sha256(decision_proof_json.encode("utf-8")).hexdigest() != ( + publication_proof_digest + ): + raise ValueError("GEPA proof digest does not match canonical JSON") + try: + if canonical_json_bytes(json.loads(projection_json)).decode() != ( + projection_json + ): + raise ValueError("projection JSON is not canonical") + if canonical_json_bytes(json.loads(decision_proof_json)).decode() != ( + decision_proof_json + ): + raise ValueError("proof JSON is not canonical") + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise ValueError("GEPA publication bytes must be canonical JSON") from exc + return metadata + from .._base import ( SQLiteStorageBase, @@ -62,7 +249,10 @@ class OptimizationJobStoreMixin: _lock: Any conn: sqlite3.Connection _execute: Any + _fetchone: Any _fetchall: Any + _own_transaction: Any + get_user_playbook_publication_subject_epochs: Any # ------------------------------------------------------------------ # Playbook optimizer methods @@ -73,28 +263,616 @@ def create_playbook_optimization_job( self, job: PlaybookOptimizationJob ) -> PlaybookOptimizationJob: with self._lock: - cur = self.conn.execute( - """INSERT INTO playbook_optimization_jobs - (target_kind, target_id, status, best_candidate_id, - successor_target_id, decision_reason, metadata_json, - created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", - ( - job.target_kind, - job.target_id, - job.status, - job.best_candidate_id, - job.successor_target_id, - job.decision_reason, - job.metadata_json, - job.created_at, - job.updated_at, - ), - ) + cur = self.conn.execute(_JOB_INSERT_SQL, _job_insert_values(job)) job.job_id = cur.lastrowid or 0 self.conn.commit() return job + @SQLiteStorageBase.handle_exceptions + def get_playbook_optimization_job( + self, job_id: int + ) -> PlaybookOptimizationJob | None: + row = self._fetchone( + "SELECT * FROM playbook_optimization_jobs WHERE job_id = ?", + (job_id,), + ) + return None if row is None else _row_to_playbook_optimization_job(row) + + @SQLiteStorageBase.handle_exceptions + def prepare_gepa_user_playbook_publication( + self, + *, + job_id: int, + owner: str, + lease_seconds: int, + winner_candidate_id: int, + candidate_content_digest: str, + search_projection_digest: str, + publication_proof_digest: str, + projection_json: str, + decision_proof_json: str, + subject_epochs_json: str, + metadata_json: str, + ) -> PlaybookOptimizationJob: + metadata = _validate_gepa_publication_prepare_metadata( + lease_seconds=lease_seconds, + candidate_content_digest=candidate_content_digest, + search_projection_digest=search_projection_digest, + publication_proof_digest=publication_proof_digest, + projection_json=projection_json, + decision_proof_json=decision_proof_json, + subject_epochs_json=subject_epochs_json, + metadata_json=metadata_json, + ) + subject_epochs = json.loads(subject_epochs_json) + + prepared_at = self._lease_now(None) + with self._lock: + self.conn.execute("BEGIN IMMEDIATE") + try: + job = self.conn.execute( + "SELECT * FROM playbook_optimization_jobs WHERE job_id = ?", + (job_id,), + ).fetchone() + if job is None: + raise StorageError("GEPA publication job does not exist") + if ( + job["optimizer_kind"] != "gepa" + or job["target_kind"] != "user_playbook" + ): + raise StorageError("GEPA publication job identity changed") + if ( + job["status"] != "running" + or job["stage"] is not None + or job["lease_owner"] is not None + or job["lease_expires_at"] is not None + ): + raise StorageError("GEPA publication job is not prepareable") + if not job["attempt_key"]: + raise StorageError("GEPA publication attempt identity is missing") + durable_authority = _gepa_publication_authority_from_metadata( + job["metadata_json"] + ) + incoming_authority = metadata.get("gepa_publication_authority") + if incoming_authority != durable_authority: + raise StorageError("GEPA publication authority changed") + candidate = self.conn.execute( + """SELECT * FROM playbook_optimization_candidates + WHERE candidate_id = ? AND job_id = ?""", + (winner_candidate_id, job_id), + ).fetchone() + if candidate is None or not candidate["is_winner"]: + raise StorageError("GEPA publication winner changed") + if sha256(candidate["content"].encode("utf-8")).hexdigest() != ( + candidate_content_digest + ): + raise StorageError("GEPA publication winner content changed") + expected_subject_epochs = ( + self.get_user_playbook_publication_subject_epochs(job["target_id"]) + ) + if expected_subject_epochs != subject_epochs_json: + raise StorageError("GEPA publication subject vector changed") + + metadata["gepa_publication_authority"] = durable_authority + metadata["publication_proof_digest"] = publication_proof_digest + metadata[PUBLICATION_PROOF_JSON_METADATA_KEY] = decision_proof_json + metadata[PUBLICATION_PROJECTION_JSON_METADATA_KEY] = projection_json + metadata[PUBLICATION_SUBJECT_EPOCHS_METADATA_KEY] = subject_epochs + durable_metadata_json = json.dumps( + metadata, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + updated = self.conn.execute( + """UPDATE playbook_optimization_jobs + SET best_candidate_id = ?, stage = 'publishing', + candidate_content_digest = ?, search_projection_digest = ?, + metadata_json = ?, decision_reason = 'publishing', + lease_owner = ?, lease_fence = lease_fence + 1, + lease_expires_at = ?, + updated_at = ? + WHERE job_id = ? AND optimizer_kind = 'gepa' + AND target_kind = 'user_playbook' AND status = 'running' + AND stage IS NULL AND lease_owner IS NULL + AND lease_expires_at IS NULL""", + ( + winner_candidate_id, + candidate_content_digest, + search_projection_digest, + durable_metadata_json, + owner, + prepared_at + lease_seconds, + prepared_at, + job_id, + ), + ) + if updated.rowcount != 1: + raise StorageError("GEPA publication preparation lost its fence") + row = self.conn.execute( + "SELECT * FROM playbook_optimization_jobs WHERE job_id = ?", + (job_id,), + ).fetchone() + if row is None: + raise StorageError("GEPA publication job disappeared") + self.conn.commit() + return _row_to_playbook_optimization_job(row) + except Exception: + self.conn.rollback() + raise + + @SQLiteStorageBase.handle_exceptions + def get_unconsumed_gepa_user_playbook_publishing_job( + self, target_id: int + ) -> PlaybookOptimizationJob | None: + row = self._fetchone( + """SELECT job.* + FROM playbook_optimization_jobs AS job + LEFT JOIN user_playbook_publication_results AS result + ON result.optimizer_kind = job.optimizer_kind + AND result.job_id = job.job_id + WHERE job.optimizer_kind = 'gepa' + AND job.target_kind = 'user_playbook' + AND job.target_id = ? + AND job.status = 'running' + AND job.stage = 'publishing' + AND result.job_id IS NULL + ORDER BY job.job_id ASC + LIMIT 1""", + (target_id,), + ) + return None if row is None else _row_to_playbook_optimization_job(row) + + @SQLiteStorageBase.handle_exceptions + def reclaim_gepa_user_playbook_publishing_job( + self, + target_id: int, + owner: str, + lease_seconds: int, + *, + now: int | None = None, + ) -> PlaybookOptimizationJob | None: + if lease_seconds <= 0: + raise ValueError("lease_seconds must be positive") + if not owner.strip(): + raise ValueError("owner must be non-empty") + reclaimed_at = self._lease_now(now) + with self._lock: + self.conn.execute("BEGIN IMMEDIATE") + try: + existing = self.conn.execute( + """SELECT job.* + FROM playbook_optimization_jobs AS job + LEFT JOIN user_playbook_publication_results AS result + ON result.optimizer_kind = job.optimizer_kind + AND result.job_id = job.job_id + WHERE job.optimizer_kind = 'gepa' + AND job.target_kind = 'user_playbook' + AND job.target_id = ? + AND job.status = 'running' + AND job.stage = 'publishing' + AND result.job_id IS NULL + ORDER BY job.job_id ASC + LIMIT 1""", + (target_id,), + ).fetchone() + if existing is None: + self.conn.commit() + return None + if ( + existing["lease_expires_at"] is None + or existing["lease_expires_at"] > reclaimed_at + ): + raise OptimizationJobLeaseLiveError( + "GEPA publication optimizer job lease is live" + ) + row = self.conn.execute( + """UPDATE playbook_optimization_jobs + SET lease_owner = ?, + lease_fence = lease_fence + 1, + lease_expires_at = ?, + updated_at = ? + WHERE job_id = ? + AND optimizer_kind = 'gepa' + AND target_kind = 'user_playbook' + AND status = 'running' + AND stage = 'publishing' + AND lease_expires_at <= ? + RETURNING *""", + ( + owner, + reclaimed_at + lease_seconds, + reclaimed_at, + existing["job_id"], + reclaimed_at, + ), + ).fetchone() + if row is None: + raise OptimizationJobLeaseLiveError( + "GEPA publication optimizer job lease is live" + ) + self.conn.commit() + return _row_to_playbook_optimization_job(row) + except Exception: + self.conn.rollback() + raise + + @SQLiteStorageBase.handle_exceptions + def create_or_get_playbook_optimization_job( + self, job: PlaybookOptimizationJob + ) -> PlaybookOptimizationJob: + if job.discovery_key is None and job.attempt_key is None: + raise ValueError( + "durable optimizer jobs require a discovery or attempt key" + ) + with self._lock: + owns_transaction = self._own_transaction() + if owns_transaction: + self.conn.execute("BEGIN IMMEDIATE") + try: + by_discovery = None + if job.discovery_key is not None: + by_discovery = self.conn.execute( + """SELECT * FROM playbook_optimization_jobs + WHERE optimizer_kind = ? AND discovery_key = ? + AND status IN ('pending', 'running')""", + (job.optimizer_kind, job.discovery_key), + ).fetchone() + by_attempt = None + if job.attempt_key is not None: + by_attempt = self.conn.execute( + """SELECT * FROM playbook_optimization_jobs + WHERE optimizer_kind = ? AND attempt_key = ? + AND status IN ('pending', 'running')""", + (job.optimizer_kind, job.attempt_key), + ).fetchone() + if ( + by_discovery is not None + and by_attempt is not None + and by_discovery["job_id"] != by_attempt["job_id"] + ): + raise ValueError("conflicting immutable optimizer job identity") + existing = by_discovery or by_attempt + if existing is not None: + if ( + existing["target_kind"] != job.target_kind + or existing["target_id"] != job.target_id + or ( + by_discovery is not None + and existing["attempt_key"] != job.attempt_key + ) + ): + raise ValueError("conflicting immutable optimizer job identity") + result = _row_to_playbook_optimization_job(existing) + else: + cur = self.conn.execute(_JOB_INSERT_SQL, _job_insert_values(job)) + row = self.conn.execute( + "SELECT * FROM playbook_optimization_jobs WHERE job_id = ?", + (cur.lastrowid,), + ).fetchone() + if row is None: + raise RuntimeError("optimizer job insert returned no row") + result = _row_to_playbook_optimization_job(row) + if owns_transaction: + self.conn.commit() + return result + except Exception: + if owns_transaction: + self.conn.rollback() + raise + + @staticmethod + def _lease_now(now: int | None) -> int: + return int(time.time()) if now is None else now + + @SQLiteStorageBase.handle_exceptions + def claim_playbook_optimization_job( + self, + job_id: int, + owner: str, + lease_seconds: int, + *, + now: int | None = None, + ) -> OptimizationJobClaim: + if lease_seconds <= 0: + raise ValueError("lease_seconds must be positive") + claimed_at = self._lease_now(now) + with self._lock: + row = self.conn.execute( + """UPDATE playbook_optimization_jobs + SET lease_owner = ?, + lease_fence = lease_fence + 1, + lease_expires_at = ?, + status = 'running', + updated_at = ? + WHERE job_id = ? + AND status IN ('pending', 'running') + AND lease_owner IS NULL + RETURNING job_id, lease_owner, lease_fence, lease_expires_at""", + (owner, claimed_at + lease_seconds, claimed_at, job_id), + ).fetchone() + if row is None: + raise RuntimeError("optimizer job is not available to claim") + if self._own_transaction(): + self.conn.commit() + return OptimizationJobClaim( + job_id=row["job_id"], + owner=row["lease_owner"], + fence=row["lease_fence"], + expires_at=row["lease_expires_at"], + ) + + @SQLiteStorageBase.handle_exceptions + def reclaim_playbook_optimization_job( + self, + job_id: int, + owner: str, + lease_seconds: int = 60, + *, + now: int | None = None, + ) -> OptimizationJobClaim: + if lease_seconds <= 0: + raise ValueError("lease_seconds must be positive") + reclaimed_at = self._lease_now(now) + with self._lock: + row = self.conn.execute( + """UPDATE playbook_optimization_jobs + SET lease_owner = ?, + lease_fence = lease_fence + 1, + lease_expires_at = ?, + status = 'running', + updated_at = ? + WHERE job_id = ? + AND status IN ('pending', 'running') + AND lease_owner IS NOT NULL + AND lease_expires_at IS NOT NULL + AND lease_expires_at <= ? + RETURNING job_id, lease_owner, lease_fence, lease_expires_at""", + ( + owner, + reclaimed_at + lease_seconds, + reclaimed_at, + job_id, + reclaimed_at, + ), + ).fetchone() + if row is None: + raise RuntimeError("optimizer job lease is not expired") + if self._own_transaction(): + self.conn.commit() + return OptimizationJobClaim( + job_id=row["job_id"], + owner=row["lease_owner"], + fence=row["lease_fence"], + expires_at=row["lease_expires_at"], + ) + + @SQLiteStorageBase.handle_exceptions + def renew_playbook_optimization_job_lease( + self, + job_id: int, + owner: str, + fence: int, + lease_seconds: int, + *, + now: int | None = None, + ) -> OptimizationJobClaim: + if lease_seconds <= 0: + raise ValueError("lease_seconds must be positive") + renewed_at = self._lease_now(now) + with self._lock: + row = self.conn.execute( + """UPDATE playbook_optimization_jobs + SET lease_expires_at = ?, updated_at = ? + WHERE job_id = ? + AND status IN ('pending', 'running') + AND lease_owner = ? + AND lease_fence = ? + AND lease_expires_at > ? + RETURNING job_id, lease_owner, lease_fence, lease_expires_at""", + ( + renewed_at + lease_seconds, + renewed_at, + job_id, + owner, + fence, + renewed_at, + ), + ).fetchone() + if row is None: + raise RuntimeError("optimizer job lease is no longer current") + if self._own_transaction(): + self.conn.commit() + return OptimizationJobClaim( + job_id=row["job_id"], + owner=row["lease_owner"], + fence=row["lease_fence"], + expires_at=row["lease_expires_at"], + ) + + @SQLiteStorageBase.handle_exceptions + def advance_playbook_optimization_stage( + self, + job_id: int, + fence: int, + stage: OptimizationJobStage, + *, + terminal_outcome: OptimizationTerminalOutcome | None = None, + now: int | None = None, + ) -> bool: + advanced_at = self._lease_now(now) + predecessor = _STAGE_PREDECESSORS.get(stage) + terminal_status: str | None = None + if stage == "applied": + if terminal_outcome not in (None, "applied"): + return False + terminal_outcome = "applied" + terminal_status = "completed" + elif stage == "failed": + if terminal_outcome not in _FAILURE_OUTCOMES: + return False + terminal_status = "failed" + elif stage == "abstained": + if terminal_outcome not in _ABSTENTION_OUTCOMES: + return False + terminal_status = "skipped" + elif predecessor is None or terminal_outcome is not None: + return False + with self._lock: + if terminal_status is None: + cur = self.conn.execute( + """UPDATE playbook_optimization_jobs + SET stage = ?, updated_at = ? + WHERE job_id = ? + AND status IN ('pending', 'running') + AND lease_fence = ? + AND lease_expires_at > ? + AND stage = ?""", + (stage, advanced_at, job_id, fence, advanced_at, predecessor), + ) + elif stage == "applied": + cur = self.conn.execute( + """UPDATE playbook_optimization_jobs + SET stage = ?, + terminal_outcome = ?, + status = ?, + lease_owner = NULL, + lease_expires_at = NULL, + updated_at = ? + WHERE job_id = ? + AND status IN ('pending', 'running') + AND lease_fence = ? + AND lease_expires_at > ? + AND stage = 'publishing'""", + ( + stage, + terminal_outcome, + terminal_status, + advanced_at, + job_id, + fence, + advanced_at, + ), + ) + else: + cur = self.conn.execute( + """UPDATE playbook_optimization_jobs + SET stage = ?, + terminal_outcome = ?, + status = ?, + lease_owner = NULL, + lease_expires_at = NULL, + updated_at = ? + WHERE job_id = ? + AND status IN ('pending', 'running') + AND lease_fence = ? + AND lease_expires_at > ? + AND stage IN ( + 'evidence_frozen', + 'candidate_generated', + 'replay_running', + 'replay_evaluated', + 'publishing' + )""", + ( + stage, + terminal_outcome, + terminal_status, + advanced_at, + job_id, + fence, + advanced_at, + ), + ) + if self._own_transaction(): + self.conn.commit() + return cur.rowcount == 1 + + @SQLiteStorageBase.handle_exceptions + def upsert_playbook_optimization_artifact( + self, + artifact: PlaybookOptimizationArtifact, + fence: int, + *, + now: int | None = None, + ) -> PlaybookOptimizationArtifact: + artifact_content_json = canonicalize_artifact_json(artifact.content_json) + if ( + sha256(artifact_content_json.encode()).hexdigest() + != artifact.content_digest + ): + raise ValueError("optimizer artifact digest does not match content") + written_at = self._lease_now(now) + with self._lock: + owns_transaction = self._own_transaction() + if owns_transaction: + self.conn.execute("BEGIN IMMEDIATE") + try: + lease = self.conn.execute( + """SELECT job_id FROM playbook_optimization_jobs + WHERE job_id = ? + AND status IN ('pending', 'running') + AND lease_fence = ? + AND lease_expires_at > ?""", + (artifact.job_id, fence, written_at), + ).fetchone() + if lease is None: + raise ValueError("optimizer job lease is no longer current") + existing = self.conn.execute( + """SELECT * FROM playbook_optimization_artifacts + WHERE job_id = ? AND artifact_kind = ?""", + (artifact.job_id, artifact.artifact_kind), + ).fetchone() + if existing is not None: + if existing["content_digest"] != artifact.content_digest: + raise ValueError("optimizer artifact digest conflict") + if existing["content_json"] != artifact_content_json: + raise ValueError("optimizer artifact content conflict") + result = _row_to_playbook_optimization_artifact(existing) + else: + cur = self.conn.execute( + """INSERT INTO playbook_optimization_artifacts + (job_id, artifact_kind, content_json, content_digest, + created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?)""", + ( + artifact.job_id, + artifact.artifact_kind, + artifact_content_json, + artifact.content_digest, + artifact.created_at, + artifact.updated_at, + ), + ) + row = self.conn.execute( + """SELECT * FROM playbook_optimization_artifacts + WHERE artifact_id = ?""", + (cur.lastrowid,), + ).fetchone() + if row is None: + raise RuntimeError("optimizer artifact insert returned no row") + result = _row_to_playbook_optimization_artifact(row) + if owns_transaction: + self.conn.commit() + return result + except Exception: + if owns_transaction: + self.conn.rollback() + raise + + @SQLiteStorageBase.handle_exceptions + def get_playbook_optimization_artifact( + self, + job_id: int, + artifact_kind: OptimizationArtifactKind, + ) -> PlaybookOptimizationArtifact | None: + row = self._fetchone( + """SELECT * FROM playbook_optimization_artifacts + WHERE job_id = ? AND artifact_kind = ?""", + (job_id, artifact_kind), + ) + return None if row is None else _row_to_playbook_optimization_artifact(row) + @SQLiteStorageBase.handle_exceptions def update_playbook_optimization_job( self, diff --git a/reflexio/server/services/storage/sqlite_storage/playbook/_user.py b/reflexio/server/services/storage/sqlite_storage/playbook/_user.py index dd10e65f7..252ab6019 100644 --- a/reflexio/server/services/storage/sqlite_storage/playbook/_user.py +++ b/reflexio/server/services/storage/sqlite_storage/playbook/_user.py @@ -4,6 +4,7 @@ import sqlite3 import uuid from concurrent.futures import ThreadPoolExecutor +from hashlib import sha256 from typing import Any from reflexio.models.api_schema.common import BlockingIssue @@ -12,6 +13,16 @@ from reflexio.models.api_schema.service_schemas import Status, UserPlaybook from reflexio.models.config_schema import SearchMode, SearchOptions from reflexio.server.services.embedding_text import resolve_retrieval_threshold +from reflexio.server.services.playbook.publication import ( + PUBLICATION_SUBJECT_EPOCHS_METADATA_KEY, + PublicationClaim, + PublicationRequest, + PublicationResult, + canonical_json_bytes, + incumbent_user_playbook_semantic_digest, + publication_source_for_optimizer, +) +from reflexio.server.services.storage.error import StorageError from reflexio.server.services.storage.lifecycle_filters import ( validate_include_inactive, ) @@ -59,6 +70,91 @@ def _emit_supersede_user_playbook( ) +def _publication_staging_payload(request: PublicationRequest) -> dict[str, object]: + return { + "attempt_key": request.attempt_key, + "claim_owner": request.publication_claim.owner, + "content_digest": request.projection.candidate_content_digest, + "incumbent_user_playbook_id": request.incumbent_user_playbook_id, + "incumbent_content_digest": request.incumbent_content_digest, + "incumbent_trigger": request.incumbent_trigger, + "incumbent_semantic_digest": request.incumbent_semantic_digest, + "job_id": request.job_id, + "optimizer_kind": request.optimizer_kind, + "projection_digest": request.projection.digest, + "projection_json": request.projection.canonical_json, + "proof_digest": request.decision_proof.digest, + "proof_json": request.decision_proof.canonical_json, + "publication_fence": request.publication_claim.fence, + "request_id": request.request_id, + "revised_content": request.revised_content, + "subject_epochs_json": request.subject_epochs_json, + "worker_fence": request.worker_fence, + } + + +def _publication_staging_digest(request: PublicationRequest) -> str: + payload = _publication_staging_payload(request) + for mutable_field in ("claim_owner", "worker_fence", "publication_fence"): + del payload[mutable_field] + canonical = json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + return sha256(canonical.encode("utf-8")).hexdigest() + + +_STAGING_CONFLICT_FIELDS = ( + ("optimizer_kind", "optimizer kind"), + ("job_id", "job id"), + ("attempt_key", "attempt key"), + ("incumbent_user_playbook_id", "incumbent"), + ("incumbent_content_digest", "incumbent content digest"), + ("incumbent_trigger", "incumbent trigger"), + ("incumbent_semantic_digest", "incumbent semantic digest"), + ("revised_content", "revised content"), + ("content_digest", "content digest"), + ("projection_digest", "projection digest"), + ("projection_json", "projection bytes"), + ("proof_digest", "proof digest"), + ("proof_json", "proof bytes"), + ("subject_epochs_json", "subject epochs"), + ("request_id", "request identity"), +) + +_STAGING_BINDING_FIELDS = ( + ("claim_owner", "publication owner"), + ("worker_fence", "worker fence"), + ("publication_fence", "publication fence"), +) + + +def _assert_staging_matches(row: sqlite3.Row, request: PublicationRequest) -> None: + expected = _publication_staging_payload(request) + for field, label in _STAGING_CONFLICT_FIELDS: + if row[field] != expected[field]: + raise StorageError(f"staged publication conflicts on {label}") + if row["staging_digest"] != _publication_staging_digest(request): + raise StorageError("staged publication conflicts on staging digest") + + +def _assert_staging_binding_matches( + row: sqlite3.Row, request: PublicationRequest +) -> None: + expected = _publication_staging_payload(request) + for field, label in _STAGING_BINDING_FIELDS: + if row[field] != expected[field]: + raise StorageError(f"staged publication conflicts on {label}") + + +def _assert_publication_lease_live(row: sqlite3.Row, *, now: int) -> None: + lease_expires_at = row["lease_expires_at"] + if lease_expires_at is None or lease_expires_at <= now: + raise StorageError("publication optimizer job lease expired") + + class UserPlaybookStoreMixin: """Mixin providing user playbook CRUD + search for SQLite storage.""" @@ -79,6 +175,565 @@ class UserPlaybookStoreMixin: _assert_subject_writable_locked: Any _own_transaction: Any commit_scope: Any + _has_sqlite_vec: bool + + @SQLiteStorageBase.handle_exceptions + def get_user_playbook_publication_subject_epochs( + self, user_playbook_id: int + ) -> str: + with self._lock: + row = self.conn.execute( + "SELECT * FROM user_playbooks WHERE user_playbook_id = ?", + (user_playbook_id,), + ).fetchone() + if row is None: + raise StorageError("publication incumbent does not exist") + subject_ref = self._subject_ref_from_user_playbook_row(row) + return canonical_json_bytes( + {"subjects": [{"epoch": 0, "ref": subject_ref}]} + ).decode("utf-8") + + def _publication_job_locked( + self, + request: PublicationRequest, + *, + now: int, + ) -> sqlite3.Row: + row = self.conn.execute( + "SELECT * FROM playbook_optimization_jobs WHERE job_id = ?", + (request.job_id,), + ).fetchone() + if row is None: + raise StorageError("publication optimizer job does not exist") + if row["optimizer_kind"] != request.optimizer_kind: + raise StorageError("publication optimizer kind changed") + if row["target_kind"] != "user_playbook": + raise StorageError("publication target is not a user playbook") + if row["target_id"] != request.incumbent_user_playbook_id: + raise StorageError("publication incumbent changed in optimizer job") + if row["attempt_key"] != request.attempt_key: + raise StorageError("publication attempt identity changed") + if row["status"] not in {"pending", "running"}: + raise StorageError("publication optimizer job is terminal") + if row["stage"] != "publishing": + raise StorageError("publication optimizer job is not at publishing stage") + if row["lease_owner"] != request.publication_claim.owner: + raise StorageError("publication worker owner changed") + if row["lease_fence"] != request.worker_fence: + raise StorageError("publication worker fence changed") + _assert_publication_lease_live(row, now=now) + if ( + row["candidate_content_digest"] + != request.projection.candidate_content_digest + ): + raise StorageError("publication content digest changed") + if row["search_projection_digest"] != request.projection.digest: + raise StorageError("publication projection digest changed") + try: + metadata = json.loads(row["metadata_json"] or "{}") + except json.JSONDecodeError as exc: + raise StorageError("publication optimizer metadata is invalid") from exc + if metadata.get("publication_proof_digest") != request.decision_proof.digest: + raise StorageError("publication proof digest changed") + try: + request_subject_epochs = json.loads(request.subject_epochs_json) + except json.JSONDecodeError as exc: + raise StorageError("publication subject epoch vector is invalid") from exc + if ( + metadata.get(PUBLICATION_SUBJECT_EPOCHS_METADATA_KEY) + != request_subject_epochs + ): + raise StorageError("publication subject epochs vector changed") + return row + + def _publication_incumbent_and_subjects_locked( + self, + request: PublicationRequest, + ) -> sqlite3.Row: + incumbent = self.conn.execute( + "SELECT * FROM user_playbooks WHERE user_playbook_id = ?", + (request.incumbent_user_playbook_id,), + ).fetchone() + if incumbent is None: + raise StorageError("publication incumbent does not exist") + incumbent_subject_ref = self._subject_ref_from_user_playbook_row(incumbent) + subjects = json.loads(request.subject_epochs_json)["subjects"] + subject_refs = tuple(str(item["ref"]) for item in subjects) + if incumbent_subject_ref not in subject_refs: + raise StorageError( + "publication incumbent governance subject is absent from frozen vector" + ) + for subject_ref in subject_refs: + self._assert_subject_writable_locked(subject_ref) + return incumbent + + def _publication_claim_locked( + self, + request: PublicationRequest, + ) -> sqlite3.Row: + row = self.conn.execute( + """SELECT * FROM user_playbook_publication_claims + WHERE optimizer_kind = ? AND job_id = ?""", + (request.optimizer_kind, request.job_id), + ).fetchone() + if row is None: + raise StorageError("publication claim does not exist") + if row["owner"] != request.publication_claim.owner: + raise StorageError("publication claim owner changed") + if row["publication_fence"] != request.publication_claim.fence: + raise StorageError("publication fence changed") + if row["worker_fence"] != request.worker_fence: + raise StorageError("publication worker fence changed") + if row["consumed"]: + raise StorageError("publication claim was already consumed") + return row + + @SQLiteStorageBase.handle_exceptions + def claim_user_playbook_publication( + self, *, job_id: int, owner: str, worker_fence: int + ) -> PublicationClaim: + if job_id <= 0 or worker_fence <= 0 or not owner.strip(): + raise ValueError("publication claim identity is invalid") + with self._lock: + self.conn.execute("BEGIN IMMEDIATE") + try: + now = _epoch_now() + job = self.conn.execute( + "SELECT * FROM playbook_optimization_jobs WHERE job_id = ?", + (job_id,), + ).fetchone() + if job is None: + raise StorageError("publication optimizer job does not exist") + if job["optimizer_kind"] not in {"gepa", "offline_tuner_replay"}: + raise StorageError("publication optimizer kind is not publishable") + if job["target_kind"] != "user_playbook": + raise StorageError("publication target is not a user playbook") + if job["status"] not in {"pending", "running"}: + raise StorageError("publication optimizer job is terminal") + if job["stage"] != "publishing": + raise StorageError( + "publication optimizer job is not at publishing stage" + ) + if job["lease_owner"] != owner: + raise StorageError("publication worker owner changed") + if job["lease_fence"] != worker_fence: + raise StorageError("publication worker fence changed") + _assert_publication_lease_live(job, now=now) + existing = self.conn.execute( + """SELECT * FROM user_playbook_publication_claims + WHERE optimizer_kind = ? AND job_id = ?""", + (job["optimizer_kind"], job_id), + ).fetchone() + if existing is not None: + if existing["consumed"]: + raise StorageError("publication claim was already consumed") + if ( + existing["owner"] == owner + and existing["worker_fence"] == worker_fence + ): + fence = int(existing["publication_fence"]) + else: + fence = int(existing["publication_fence"]) + 1 + self.conn.execute( + """UPDATE user_playbook_publication_claims + SET owner = ?, publication_fence = ?, worker_fence = ?, + updated_at = ? + WHERE optimizer_kind = ? AND job_id = ?""", + ( + owner, + fence, + worker_fence, + now, + job["optimizer_kind"], + job_id, + ), + ) + else: + fence = 1 + self.conn.execute( + """INSERT INTO user_playbook_publication_claims + (optimizer_kind, job_id, owner, publication_fence, + worker_fence, consumed, updated_at) + VALUES (?, ?, ?, ?, ?, 0, ?)""", + ( + job["optimizer_kind"], + job_id, + owner, + fence, + worker_fence, + now, + ), + ) + self.conn.commit() + except Exception: + self.conn.rollback() + raise + return PublicationClaim(job_id=job_id, owner=owner, fence=fence) + + @SQLiteStorageBase.handle_exceptions + def stage_user_playbook_publication(self, request: PublicationRequest) -> None: + request.__post_init__() + with self._lock: + self.conn.execute("BEGIN IMMEDIATE") + try: + now = _epoch_now() + existing = self.conn.execute( + """SELECT * FROM user_playbook_publication_staging + WHERE optimizer_kind = ? AND job_id = ?""", + (request.optimizer_kind, request.job_id), + ).fetchone() + terminal = self.conn.execute( + """SELECT staging_digest FROM user_playbook_publication_results + WHERE optimizer_kind = ? AND job_id = ?""", + (request.optimizer_kind, request.job_id), + ).fetchone() + if terminal is not None: + if existing is None: + raise StorageError( + "committed publication lost its staging record" + ) + _assert_staging_matches(existing, request) + if terminal["staging_digest"] != existing["staging_digest"]: + raise StorageError( + "committed publication staging digest changed" + ) + self.conn.commit() + return + if existing is not None: + _assert_staging_matches(existing, request) + self._publication_job_locked(request, now=now) + self._publication_claim_locked(request) + self._publication_incumbent_and_subjects_locked(request) + self.conn.execute( + """UPDATE user_playbook_publication_staging + SET claim_owner = ?, publication_fence = ?, worker_fence = ? + WHERE optimizer_kind = ? AND job_id = ?""", + ( + request.publication_claim.owner, + request.publication_claim.fence, + request.worker_fence, + request.optimizer_kind, + request.job_id, + ), + ) + self.conn.commit() + return + self._publication_job_locked(request, now=now) + self._publication_claim_locked(request) + self._publication_incumbent_and_subjects_locked(request) + self.conn.execute( + """INSERT INTO user_playbook_publication_staging + (optimizer_kind, job_id, attempt_key, claim_owner, + publication_fence, worker_fence, incumbent_user_playbook_id, + incumbent_content_digest, incumbent_trigger, + incumbent_semantic_digest, + revised_content, content_digest, projection_json, + projection_digest, proof_json, proof_digest, + subject_epochs_json, request_id, staging_digest, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + request.optimizer_kind, + request.job_id, + request.attempt_key, + request.publication_claim.owner, + request.publication_claim.fence, + request.worker_fence, + request.incumbent_user_playbook_id, + request.incumbent_content_digest, + request.incumbent_trigger, + request.incumbent_semantic_digest, + request.revised_content, + request.projection.candidate_content_digest, + request.projection.canonical_json, + request.projection.digest, + request.decision_proof.canonical_json, + request.decision_proof.digest, + request.subject_epochs_json, + request.request_id, + _publication_staging_digest(request), + now, + ), + ) + self.conn.commit() + except Exception: + self.conn.rollback() + raise + + def _finish_publication_locked( + self, + request: PublicationRequest, + *, + outcome: str, + successor_id: int | None, + staging_digest: str, + now: int, + ) -> None: + self.conn.execute( + """INSERT INTO user_playbook_publication_results + (optimizer_kind, job_id, outcome, successor_user_playbook_id, + staging_digest, created_at) + VALUES (?, ?, ?, ?, ?, ?)""", + ( + request.optimizer_kind, + request.job_id, + outcome, + successor_id, + staging_digest, + now, + ), + ) + consumed = self.conn.execute( + """UPDATE user_playbook_publication_claims + SET consumed = 1, updated_at = ? + WHERE optimizer_kind = ? AND job_id = ? AND owner = ? + AND publication_fence = ? AND worker_fence = ? AND consumed = 0""", + ( + now, + request.optimizer_kind, + request.job_id, + request.publication_claim.owner, + request.publication_claim.fence, + request.worker_fence, + ), + ) + if consumed.rowcount != 1: + raise StorageError("publication claim consumption lost its fence") + job_stage = "applied" if outcome == "applied" else "abstained" + job_status = "completed" if outcome == "applied" else "skipped" + updated = self.conn.execute( + """UPDATE playbook_optimization_jobs + SET stage = ?, terminal_outcome = ?, status = ?, + successor_target_id = ?, lease_owner = NULL, + lease_expires_at = NULL, updated_at = ? + WHERE job_id = ? AND optimizer_kind = ? AND attempt_key = ? + AND target_kind = 'user_playbook' AND target_id = ? + AND status IN ('pending', 'running') AND stage = 'publishing' + AND lease_owner = ? AND lease_fence = ? + AND lease_expires_at > ?""", + ( + job_stage, + outcome, + job_status, + successor_id, + now, + request.job_id, + request.optimizer_kind, + request.attempt_key, + request.incumbent_user_playbook_id, + request.publication_claim.owner, + request.worker_fence, + now, + ), + ) + if updated.rowcount != 1: + raise StorageError("publication optimizer job transition lost its fence") + + @SQLiteStorageBase.handle_exceptions + def commit_user_playbook_publication( + self, request: PublicationRequest + ) -> PublicationResult: + request.__post_init__() + with self._lock: + self.conn.execute("BEGIN IMMEDIATE") + try: + now = _epoch_now() + terminal = self.conn.execute( + """SELECT * FROM user_playbook_publication_results + WHERE optimizer_kind = ? AND job_id = ?""", + (request.optimizer_kind, request.job_id), + ).fetchone() + staged = self.conn.execute( + """SELECT * FROM user_playbook_publication_staging + WHERE optimizer_kind = ? AND job_id = ?""", + (request.optimizer_kind, request.job_id), + ).fetchone() + if staged is None: + raise StorageError("publication successor is not staged") + _assert_staging_matches(staged, request) + if terminal is not None: + if terminal["staging_digest"] != staged["staging_digest"]: + raise StorageError( + "committed publication staging digest changed" + ) + result = PublicationResult( + job_id=request.job_id, + outcome=terminal["outcome"], + successor_user_playbook_id=terminal[ + "successor_user_playbook_id" + ], + ) + self.conn.commit() + return result + _assert_staging_binding_matches(staged, request) + self._publication_job_locked(request, now=now) + self._publication_claim_locked(request) + incumbent = self._publication_incumbent_and_subjects_locked(request) + subject_ref = self._subject_ref_from_user_playbook_row(incumbent) + live_content_digest = sha256( + incumbent["content"].encode("utf-8") + ).hexdigest() + live_semantic_digest = incumbent_user_playbook_semantic_digest( + content_digest=live_content_digest, + trigger=incumbent["trigger"], + ) + if ( + incumbent["status"] is not None + or live_content_digest != request.incumbent_content_digest + or live_semantic_digest != request.incumbent_semantic_digest + ): + self._finish_publication_locked( + request, + outcome="incumbent_changed", + successor_id=None, + staging_digest=staged["staging_digest"], + now=now, + ) + self.conn.execute( + """INSERT INTO playbook_optimization_events + (job_id, event_type, payload_json, created_at) + VALUES (?, 'publication_incumbent_changed', ?, ?)""", + ( + request.job_id, + json.dumps( + { + "outcome": "incumbent_changed", + "request_id": request.request_id, + }, + separators=(",", ":"), + sort_keys=True, + ), + now, + ), + ) + self.conn.commit() + return PublicationResult( + job_id=request.job_id, + outcome="incumbent_changed", + successor_user_playbook_id=None, + ) + embedding = [float(value) for value in request.projection.embedding] + created_at = _epoch_to_iso(now) + inserted = self.conn.execute( + """INSERT INTO user_playbooks + (user_id, playbook_name, created_at, request_id, agent_version, + content, trigger, rationale, blocking_issue, + source_interaction_ids, status, source, embedding, + expanded_terms, source_span, notes, reader_angle, tags, + merged_into, superseded_by, governance_subject_ref) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, + NULL, NULL, ?)""", + ( + incumbent["user_id"], + incumbent["playbook_name"], + created_at, + request.request_id, + incumbent["agent_version"], + request.revised_content, + request.projection.preserved_trigger, + incumbent["rationale"], + incumbent["blocking_issue"], + incumbent["source_interaction_ids"], + publication_source_for_optimizer(request.optimizer_kind), + json.dumps(embedding, separators=(",", ":")), + " ".join(request.projection.expanded_terms), + incumbent["source_span"], + incumbent["notes"], + incumbent["reader_angle"], + incumbent["tags"], + subject_ref, + ), + ) + successor_id = inserted.lastrowid + if successor_id is None: + raise StorageError("publication successor insert returned no id") + self.conn.execute( + "INSERT INTO user_playbooks_fts(rowid, search_text) VALUES (?, ?)", + (successor_id, request.projection.lexical_document), + ) + if self._has_sqlite_vec: + self.conn.execute( + "INSERT INTO user_playbooks_vec(rowid, embedding) VALUES (?, ?)", + (successor_id, json.dumps(embedding)), + ) + superseded = self.conn.execute( + """UPDATE user_playbooks + SET status = ?, superseded_by = ?, retired_at = ? + WHERE user_playbook_id = ? AND status IS NULL + AND content = ? AND trigger IS ?""", + ( + Status.SUPERSEDED.value, + successor_id, + now, + request.incumbent_user_playbook_id, + incumbent["content"], + request.incumbent_trigger, + ), + ) + if superseded.rowcount != 1: + raise StorageError("publication incumbent changed during commit") + _append_event_stmt( + self.conn, + org_id=self.org_id, + entity_type="user_playbook", + entity_id=str(successor_id), + op="revise", + prov="wasRevisionOf", + source_ids=[str(request.incumbent_user_playbook_id)], + actor=request.optimizer_kind, + request_id=request.request_id, + reason="atomic optimizer publication", + created_at=now, + ) + event_payload = json.dumps( + { + "outcome": "applied", + "proof_digest": request.decision_proof.digest, + "projection_digest": request.projection.digest, + "request_id": request.request_id, + "successor_user_playbook_id": successor_id, + }, + separators=(",", ":"), + sort_keys=True, + ) + self.conn.execute( + """INSERT INTO playbook_optimization_events + (job_id, event_type, payload_json, created_at) + VALUES (?, 'publication_applied', ?, ?)""", + (request.job_id, event_payload, now), + ) + self._finish_publication_locked( + request, + outcome="applied", + successor_id=successor_id, + staging_digest=staged["staging_digest"], + now=now, + ) + self.conn.commit() + return PublicationResult( + job_id=request.job_id, + outcome="applied", + successor_user_playbook_id=successor_id, + ) + except Exception: + self.conn.rollback() + raise + + @SQLiteStorageBase.handle_exceptions + def load_user_playbook_publication_result( + self, job_id: int + ) -> PublicationResult | None: + row = self._fetchone( + """SELECT job_id, outcome, successor_user_playbook_id + FROM user_playbook_publication_results WHERE job_id = ?""", + (job_id,), + ) + if row is None: + return None + return PublicationResult( + job_id=row["job_id"], + outcome=row["outcome"], + successor_user_playbook_id=row["successor_user_playbook_id"], + ) def _subject_ref_from_user_playbook_row(self, row: sqlite3.Row) -> str: subject_ref = row["governance_subject_ref"] diff --git a/reflexio/server/services/storage/storage_base/playbook/_optimization.py b/reflexio/server/services/storage/storage_base/playbook/_optimization.py index 4cc79d3a5..d3ae42e3b 100644 --- a/reflexio/server/services/storage/storage_base/playbook/_optimization.py +++ b/reflexio/server/services/storage/storage_base/playbook/_optimization.py @@ -3,6 +3,11 @@ from abc import abstractmethod from reflexio.models.api_schema.domain import ( + OptimizationArtifactKind, + OptimizationJobClaim, + OptimizationJobStage, + OptimizationTerminalOutcome, + PlaybookOptimizationArtifact, PlaybookOptimizationCandidate, PlaybookOptimizationEvaluation, PlaybookOptimizationEvent, @@ -24,6 +29,146 @@ def create_playbook_optimization_job( """Persist a playbook optimization job and return it with id populated.""" raise NotImplementedError + def get_playbook_optimization_job( + self, job_id: int + ) -> PlaybookOptimizationJob | None: + """Load one optimizer job by its durable identity.""" + raise NotImplementedError( + "Storage backend does not support replay-gated playbook optimization" + ) + + def prepare_gepa_user_playbook_publication( + self, + *, + job_id: int, + owner: str, + lease_seconds: int, + winner_candidate_id: int, + candidate_content_digest: str, + search_projection_digest: str, + publication_proof_digest: str, + projection_json: str, + decision_proof_json: str, + subject_epochs_json: str, + metadata_json: str, + ) -> PlaybookOptimizationJob: + """Fence and persist GEPA publication authority before staging.""" + raise NotImplementedError( + "Storage backend does not support replay-gated playbook optimization" + ) + + def get_unconsumed_gepa_user_playbook_publishing_job( + self, target_id: int + ) -> PlaybookOptimizationJob | None: + """Load an active GEPA user publication job with no terminal result.""" + raise NotImplementedError( + "Storage backend does not support replay-gated playbook optimization" + ) + + def reclaim_gepa_user_playbook_publishing_job( + self, + target_id: int, + owner: str, + lease_seconds: int, + *, + now: int | None = None, + ) -> PlaybookOptimizationJob | None: + """Recover one unconsumed GEPA user publication job. + + Outcomes are part of the storage contract: + return the reclaimed job when an expired publishing lease is fenced to + the new owner, return ``None`` when no unconsumed publishing job exists, + and raise ``OptimizationJobLeaseLiveError`` when a matching job exists + but its optimizer lease is still live. + """ + raise NotImplementedError( + "Storage backend does not support replay-gated playbook optimization" + ) + + def create_or_get_playbook_optimization_job( + self, job: PlaybookOptimizationJob + ) -> PlaybookOptimizationJob: + """Atomically insert or return the active job with the same replay identity.""" + raise NotImplementedError( + "Storage backend does not support replay-gated playbook optimization" + ) + + def claim_playbook_optimization_job( + self, + job_id: int, + owner: str, + lease_seconds: int, + *, + now: int | None = None, + ) -> OptimizationJobClaim: + """Claim an unleased active optimizer job and issue a new fence.""" + raise NotImplementedError( + "Storage backend does not support replay-gated playbook optimization" + ) + + def reclaim_playbook_optimization_job( + self, + job_id: int, + owner: str, + lease_seconds: int = 60, + *, + now: int | None = None, + ) -> OptimizationJobClaim: + """Reclaim an expired active optimizer job and issue a newer fence.""" + raise NotImplementedError( + "Storage backend does not support replay-gated playbook optimization" + ) + + def renew_playbook_optimization_job_lease( + self, + job_id: int, + owner: str, + fence: int, + lease_seconds: int, + *, + now: int | None = None, + ) -> OptimizationJobClaim: + """Extend a current, unexpired optimizer lease without changing its fence.""" + raise NotImplementedError( + "Storage backend does not support replay-gated playbook optimization" + ) + + def advance_playbook_optimization_stage( + self, + job_id: int, + fence: int, + stage: OptimizationJobStage, + *, + terminal_outcome: OptimizationTerminalOutcome | None = None, + now: int | None = None, + ) -> bool: + """Advance the linear replay stage only for the current unexpired fence.""" + raise NotImplementedError( + "Storage backend does not support replay-gated playbook optimization" + ) + + def upsert_playbook_optimization_artifact( + self, + artifact: PlaybookOptimizationArtifact, + fence: int, + *, + now: int | None = None, + ) -> PlaybookOptimizationArtifact: + """Write a singleton artifact only under the current lease fence.""" + raise NotImplementedError( + "Storage backend does not support replay-gated playbook optimization" + ) + + def get_playbook_optimization_artifact( + self, + job_id: int, + artifact_kind: OptimizationArtifactKind, + ) -> PlaybookOptimizationArtifact | None: + """Return one typed singleton artifact when present.""" + raise NotImplementedError( + "Storage backend does not support replay-gated playbook optimization" + ) + @abstractmethod def update_playbook_optimization_job( self, diff --git a/reflexio/server/services/storage/storage_base/playbook/_user.py b/reflexio/server/services/storage/storage_base/playbook/_user.py index a0899def3..0d30983d7 100644 --- a/reflexio/server/services/storage/storage_base/playbook/_user.py +++ b/reflexio/server/services/storage/storage_base/playbook/_user.py @@ -1,6 +1,7 @@ """Abstract user playbook CRUD + search declarations.""" from abc import abstractmethod +from typing import TYPE_CHECKING from reflexio.models.api_schema.common import BlockingIssue from reflexio.models.api_schema.domain import Status, UserPlaybook @@ -8,10 +9,55 @@ from reflexio.models.api_schema.retriever_schema import SearchUserPlaybookRequest from reflexio.models.config_schema import SearchOptions +if TYPE_CHECKING: + from reflexio.server.services.playbook.publication import ( + PublicationClaim, + PublicationRequest, + PublicationResult, + ) + class UserPlaybookStoreMixin: """Abstract user playbook CRUD + search methods.""" + def get_user_playbook_publication_subject_epochs( + self, user_playbook_id: int + ) -> str: + """Return the canonical durable subject vector for one incumbent.""" + raise NotImplementedError( + "Storage backend does not support atomic user-playbook publication" + ) + + def claim_user_playbook_publication( + self, *, job_id: int, owner: str, worker_fence: int + ) -> "PublicationClaim": + """Claim a publication attempt under the current optimizer worker fence.""" + raise NotImplementedError( + "Storage backend does not support atomic user-playbook publication" + ) + + def stage_user_playbook_publication(self, request: "PublicationRequest") -> None: + """Persist one immutable successor payload outside visible playbook tables.""" + raise NotImplementedError( + "Storage backend does not support atomic user-playbook publication" + ) + + def commit_user_playbook_publication( + self, request: "PublicationRequest" + ) -> "PublicationResult": + """Atomically commit a staged successor or an incumbent-changed result.""" + raise NotImplementedError( + "Storage backend does not support atomic user-playbook publication" + ) + + def load_user_playbook_publication_result( + self, job_id: int + ) -> "PublicationResult | None": + """Load the immutable terminal publication result for a job.""" + raise NotImplementedError( + "Storage backend does not support atomic user-playbook publication" + ) + @abstractmethod def save_user_playbooks( self, diff --git a/reflexio/server/services/storage/storage_base/profiles/_interaction_store.py b/reflexio/server/services/storage/storage_base/profiles/_interaction_store.py index bf94ffdb3..dbd21fa64 100644 --- a/reflexio/server/services/storage/storage_base/profiles/_interaction_store.py +++ b/reflexio/server/services/storage/storage_base/profiles/_interaction_store.py @@ -46,6 +46,22 @@ def add_user_interactions_bulk( """ raise NotImplementedError + def record_retrieved_learning_sampling_decision( + self, + *, + user_id: str, # noqa: ARG002 + session_id: str, # noqa: ARG002 + request_id: str, # noqa: ARG002 + sampled: bool, # noqa: ARG002 + ) -> None: + """Best-effort hook for the retrieved-learning scheduler decision. + + Backends without managed-platform impact-evidence storage intentionally + do nothing. Managed storage overrides this to persist the decision for + units created by the just-published request. + """ + return + def prepare_interaction_embeddings(self, interactions: list[Interaction]) -> None: # noqa: ARG002 """Pre-populate interaction.embedding for each interaction without writing to storage. diff --git a/reflexio/server/tracing.py b/reflexio/server/tracing.py index 084136f1d..e6cdb3e86 100644 --- a/reflexio/server/tracing.py +++ b/reflexio/server/tracing.py @@ -10,7 +10,10 @@ import logging from collections.abc import Iterator, Mapping from contextlib import AbstractContextManager, contextmanager -from typing import Any, Protocol +from typing import TYPE_CHECKING, Any, Protocol + +if TYPE_CHECKING: + from sentry_sdk._types import LogLevelStr logger = logging.getLogger(__name__) @@ -158,7 +161,12 @@ def sentry_tags(**tags: Any) -> Iterator[None]: logger.warning("Failed to close Sentry scope: %s", exc) -def capture_anomaly(message: str, *, level: str = "warning", **tags: Any) -> None: +def capture_anomaly( + message: str, + *, + level: LogLevelStr = "warning", + **tags: Any, +) -> None: """Report a non-fatal anomaly to Sentry, if the SDK is installed. For silent-but-noteworthy conditions that return rather than raise — e.g. diff --git a/tests/cli/test_commands.py b/tests/cli/test_commands.py index 32d3a60a7..e60dac10e 100644 --- a/tests/cli/test_commands.py +++ b/tests/cli/test_commands.py @@ -132,6 +132,35 @@ def test_list_profiles(self, runner, app, mock_client) -> None: assert "Likes Python" in result.output +class TestSearchShortcut: + def test_json_search_preserves_exact_user_playbook_id( + self, runner, app, mock_client + ) -> None: + """The top-level search shortcut must not lose the published successor ID.""" + successor = MagicMock() + successor.model_dump.return_value = {"user_playbook_id": 77} + response = MagicMock() + response.model_dump.return_value = { + "success": True, + "profiles": [], + "agent_playbooks": [], + "user_playbooks": [{"user_playbook_id": 77}], + } + response.profiles = [] + response.agent_playbooks = [] + response.user_playbooks = [successor] + mock_client.search.return_value = response + + result = runner.invoke( + app, ["--json", "search", "improvement", "--user-id", "u1"] + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output)["data"]["user_playbooks"] == [ + {"user_playbook_id": 77} + ] + + class TestProfilesAdd: """Tests for the new 'user-profiles add' command.""" diff --git a/tests/cli/test_utils.py b/tests/cli/test_utils.py index 769e11c4e..10414c673 100644 --- a/tests/cli/test_utils.py +++ b/tests/cli/test_utils.py @@ -122,7 +122,37 @@ def fake_run(cmd: list[str], **_kwargs) -> CompletedProcess[str]: assert utils.find_pids_on_port(8090) == [] -@pytest.mark.skipif(shutil.which("lsof") is None, reason="lsof not available") +def test_find_pids_on_port_falls_back_to_ss(monkeypatch) -> None: + calls: list[list[str]] = [] + + def fake_run(cmd: list[str], **_kwargs) -> CompletedProcess[str]: + calls.append(cmd) + if cmd[0] == "lsof": + return CompletedProcess(cmd, 1, stdout="") + return CompletedProcess( + cmd, + 0, + stdout=( + "CLOSE 0 0 127.0.0.1:8090 0.0.0.0:* " + 'users:(("python",pid=456,fd=7),("python",pid=456,fd=8))\n' + "ESTAB 0 0 127.0.0.1:8090 127.0.0.1:55006 " + 'users:(("client",pid=789,fd=9))\n' + ), + ) + + monkeypatch.setattr(utils.subprocess, "run", fake_run) + + assert utils.find_pids_on_port(8090) == [456] + assert calls == [ + ["lsof", "-nP", "-Fpn", "-iTCP:8090"], + ["ss", "-tanpH", "sport", "=", ":8090"], + ] + + +@pytest.mark.skipif( + shutil.which("lsof") is None and shutil.which("ss") is None, + reason="neither lsof nor ss is available", +) def test_find_pids_on_port_detects_bound_socket_without_listen() -> None: # Regression: an orphaned process can hold a port bound without listening # (e.g. a leaked uvicorn --reload worker); it must still be detected. diff --git a/tests/client/test_config_client.py b/tests/client/test_config_client.py index f4f5a9016..ba1c422ad 100644 --- a/tests/client/test_config_client.py +++ b/tests/client/test_config_client.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock, patch -from reflexio.client import ReflexioClient +from reflexio.client import ConfigResponse, OfflineTunerConfigResponse, ReflexioClient from reflexio.models.config_schema import Config, StorageConfigSQLite @@ -32,13 +32,30 @@ def test_client_get_config_accepts_unknown_overlay(mock_session_class) -> None: client = ReflexioClient(api_key="test_key", url_endpoint="http://localhost:8000") result = client.get_config() - assert isinstance(result, Config) + assert isinstance(result, ConfigResponse) assert result.storage_config == StorageConfigSQLite(db_path="/tmp/test.db") assert result.model_dump()["x_extension_config"] == payload["x_extension_config"] @patch("reflexio.client.client.requests.Session") -def test_client_set_config_preserves_unknown_overlay(mock_session_class) -> None: +def test_client_get_config_types_offline_tuner_overlay(mock_session_class) -> None: + mock_session = MagicMock() + mock_session_class.return_value = mock_session + payload = Config( + storage_config=StorageConfigSQLite(db_path="/tmp/test.db") + ).model_dump() + payload["offline_tuner_config"] = {"enabled": False} + mock_session.request.return_value = _json_response(payload) + + client = ReflexioClient(api_key="test_key", url_endpoint="http://localhost:8000") + result = client.get_config() + + assert isinstance(result.offline_tuner_config, OfflineTunerConfigResponse) + assert result.offline_tuner_config.enabled is False + + +@patch("reflexio.client.client.requests.Session") +def test_client_set_config_does_not_send_response_overlay(mock_session_class) -> None: mock_session = MagicMock() mock_session_class.return_value = mock_session get_payload = Config( @@ -48,6 +65,7 @@ def test_client_set_config_preserves_unknown_overlay(mock_session_class) -> None "enabled": True, "version": "extension-v1", } + get_payload["offline_tuner_config"] = {"enabled": False} mock_session.request.side_effect = [ _json_response(get_payload), _json_response({"success": True, "msg": "Configuration set successfully"}), @@ -62,4 +80,5 @@ def test_client_set_config_preserves_unknown_overlay(mock_session_class) -> None args, kwargs = mock_session.request.call_args assert args[0] == "POST" assert args[1].endswith("/api/set_config") - assert kwargs["json"]["x_extension_config"] == get_payload["x_extension_config"] + assert "x_extension_config" not in kwargs["json"] + assert "offline_tuner_config" not in kwargs["json"] diff --git a/tests/deploy_guard/test_module_layout_contracts.py b/tests/deploy_guard/test_module_layout_contracts.py index 2ed457bfb..eff708c94 100644 --- a/tests/deploy_guard/test_module_layout_contracts.py +++ b/tests/deploy_guard/test_module_layout_contracts.py @@ -194,7 +194,6 @@ def test_canonical_service_symbols_are_importable(module_name, public_names): "service.py", "playbook_service_utils.py", "playbook_service_constants.py", - "playbook_edit_apply.py", "components/__init__.py", "components/extractor.py", "components/consolidator.py", diff --git a/tests/e2e_tests/conftest.py b/tests/e2e_tests/conftest.py index d974598bf..de86b5a75 100644 --- a/tests/e2e_tests/conftest.py +++ b/tests/e2e_tests/conftest.py @@ -2,6 +2,7 @@ import csv import json +from collections.abc import Iterator from pathlib import Path from typing import Any from unittest.mock import patch @@ -25,6 +26,7 @@ ) from reflexio.server.services.configurator.configurator import DefaultConfigurator from reflexio.server.services.tagging.tagging_scheduler import drain_tagging +from reflexio.test_support.llm_mock import patched_litellm _TEST_DATA_DIR = Path(__file__).resolve().parent.parent / "test_data" _SCENARIO_DIR = _TEST_DATA_DIR / "scenarios" / "e2e" @@ -37,7 +39,20 @@ @pytest.fixture(autouse=True) -def _zero_group_evaluation_delay(): +def mock_llm( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch +) -> Iterator[None]: + """Keep the standard E2E tier deterministic and credential-free.""" + if request.node.get_closest_marker("requires_credentials"): + monkeypatch.delenv("MOCK_LLM_RESPONSE", raising=False) + yield + return + with patched_litellm(): + yield + + +@pytest.fixture(autouse=True) +def _zero_group_evaluation_delay() -> Iterator[None]: """Remove the 600s completion-delay gate in group evaluation for e2e tests. `run_group_evaluation` skips a session if its latest request is newer than @@ -54,7 +69,19 @@ def _zero_group_evaluation_delay(): @pytest.fixture(autouse=True) -def _drain_background_tagging_callbacks(): +def _zero_tagging_delay() -> Iterator[None]: + """Fire deferred tagging promptly so E2E teardown can drain it deterministically.""" + with patch( + "reflexio.server.services.tagging.tagging_scheduler._EFFECTIVE_DELAY_SECONDS", + 0, + ): + yield + + +@pytest.fixture(autouse=True) +def _drain_background_tagging_callbacks( + _zero_tagging_delay: None, mock_llm: None +) -> Iterator[None]: yield assert drain_tagging(timeout_seconds=10.0), ( "background tagging callbacks did not drain before test teardown" @@ -333,6 +360,7 @@ def _get_playbook_names(instance: Reflexio) -> list[str]: def _cleanup_storage(instance: Reflexio): """Helper function to cleanup storage for an Reflexio instance.""" + tagging_drained = drain_tagging(timeout_seconds=10.0) try: storage = instance.request_context.storage assert storage is not None @@ -347,6 +375,9 @@ def _cleanup_storage(instance: Reflexio): storage.delete_all_operation_states() except Exception as e: print(f"Error during cleanup: {str(e)}") + assert tagging_drained, ( + "background tagging callbacks did not drain before storage cleanup" + ) @pytest.fixture diff --git a/tests/e2e_tests/test_cleanup_storage.py b/tests/e2e_tests/test_cleanup_storage.py new file mode 100644 index 000000000..b6572672b --- /dev/null +++ b/tests/e2e_tests/test_cleanup_storage.py @@ -0,0 +1,35 @@ +from unittest.mock import MagicMock + +import pytest + +from tests.e2e_tests import conftest as e2e_conftest + + +def test_cleanup_deletes_storage_before_reporting_tagging_drain_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + storage = MagicMock() + instance = MagicMock() + instance.request_context.storage = storage + monkeypatch.setattr(e2e_conftest, "drain_tagging", lambda **_kwargs: False) + monkeypatch.setattr( + e2e_conftest, "_get_playbook_names", lambda _instance: ["playbook"] + ) + + with pytest.raises( + AssertionError, + match="background tagging callbacks did not drain before storage cleanup", + ): + e2e_conftest._cleanup_storage(instance) + + storage.delete_all_user_playbooks_by_playbook_name.assert_called_once_with( + "playbook" + ) + storage.delete_all_agent_playbooks_by_playbook_name.assert_called_once_with( + "playbook" + ) + storage.delete_all_interactions.assert_called_once_with() + storage.delete_all_profiles.assert_called_once_with() + storage.delete_all_agent_success_evaluation_results.assert_called_once_with() + storage.delete_all_requests.assert_called_once_with() + storage.delete_all_operation_states.assert_called_once_with() diff --git a/tests/e2e_tests/test_complete_workflows.py b/tests/e2e_tests/test_complete_workflows.py index b7351452e..1a81670fe 100644 --- a/tests/e2e_tests/test_complete_workflows.py +++ b/tests/e2e_tests/test_complete_workflows.py @@ -91,7 +91,9 @@ def test_complete_workflow_end_to_end( # Step 2: Search interactions search_interaction_response = reflexio_instance.search_interactions( - SearchInteractionRequest(user_id=user_id, query="Priya", top_k=5) + SearchInteractionRequest( + user_id=user_id, query="order dress return exchange", top_k=5 + ) ) assert search_interaction_response.success is True assert len(search_interaction_response.interactions) > 0 @@ -744,7 +746,9 @@ def test_full_workflow_with_all_features( # Search interactions search_interaction_response = reflexio_instance.search_interactions( - SearchInteractionRequest(user_id=user_id, query="Priya", top_k=5) + SearchInteractionRequest( + user_id=user_id, query="order dress return exchange", top_k=5 + ) ) assert search_interaction_response.success is True assert len(search_interaction_response.interactions) > 0 diff --git a/tests/e2e_tests/test_knowledge_gap_real_llm.py b/tests/e2e_tests/test_knowledge_gap_real_llm.py index 7faa3c001..f21b3ba4b 100644 --- a/tests/e2e_tests/test_knowledge_gap_real_llm.py +++ b/tests/e2e_tests/test_knowledge_gap_real_llm.py @@ -20,6 +20,7 @@ from reflexio.lib.reflexio_lib import Reflexio from reflexio.models.api_schema.retriever_schema import GetUserPlaybooksRequest from reflexio.models.api_schema.service_schemas import InteractionData, UserPlaybook +from reflexio.models.config_schema import SINGLETON_USER_PLAYBOOK_NAME from tests.server.test_utils import skip_low_priority pytestmark = [pytest.mark.e2e, pytest.mark.requires_credentials] @@ -35,6 +36,10 @@ def test_knowledge_gap_extraction_real_llm( Uses the default agent_context_prompt and user_playbook_extractor_config. Verify the extracted playbook captures the knowledge gap honestly. """ + assert os.environ.get("MOCK_LLM_RESPONSE", "").strip().lower() != "true", ( + "requires_credentials E2E tests must run without MOCK_LLM_RESPONSE=true" + ) + interactions = [ InteractionData( role="User", @@ -62,82 +67,75 @@ def test_knowledge_gap_extraction_real_llm( ), ] - # Publish with real LLM (no MOCK_LLM_RESPONSE) - original_mock = os.environ.pop("MOCK_LLM_RESPONSE", None) - try: - response = reflexio_instance_playbook_only.publish_interaction( - { - "user_id": "knowledge_gap_user", - "session_id": "e2e_test_session", - "interaction_data_list": interactions, - "source": "test_knowledge_gap", - "agent_version": "v1.0", - "force_extraction": True, - } - ) - assert response.success is True + response = reflexio_instance_playbook_only.publish_interaction( + { + "user_id": "knowledge_gap_user", + "session_id": "e2e_test_session", + "interaction_data_list": interactions, + "source": "test_knowledge_gap", + "agent_version": "v1.0", + "force_extraction": True, + } + ) + assert response.success is True - # Retrieve extracted playbooks - playbooks_response = reflexio_instance_playbook_only.get_user_playbooks( - GetUserPlaybooksRequest( - playbook_name="test_playbook", - status_filter=[None], - ) - ) - assert playbooks_response.success is True - user_playbooks = playbooks_response.user_playbooks - assert user_playbooks, ( - "Expected at least one playbook from knowledge-gap interaction" + # Retrieve extracted playbooks + playbooks_response = reflexio_instance_playbook_only.get_user_playbooks( + GetUserPlaybooksRequest( + playbook_name=SINGLETON_USER_PLAYBOOK_NAME, + status_filter=[None], ) + ) + assert playbooks_response.success is True + user_playbooks = playbooks_response.user_playbooks + assert user_playbooks, ( + "Expected at least one playbook from knowledge-gap interaction" + ) - # Print extracted playbooks for inspection - print("\n" + "=" * 70) - print("EXTRACTED PLAYBOOKS FROM KNOWLEDGE-GAP INTERACTION") - print("=" * 70) - for i, pb in enumerate(user_playbooks, 1): - print(f"\n--- Playbook {i} ---") - print(f" Trigger: {pb.trigger}") - print(f" Content: {pb.content}") - if pb.rationale: - print(f" Rationale: {pb.rationale}") - if pb.blocking_issue: - print( - f" Blocking: [{pb.blocking_issue.kind}] {pb.blocking_issue.details}" - ) - print("\n" + "=" * 70) - - # Verify schema: flat fields, no instruction/pitfall - for pb in user_playbooks: - assert pb.content and pb.content.strip() - assert pb.trigger and pb.trigger.strip() - assert "instruction" not in UserPlaybook.model_fields - assert "pitfall" not in UserPlaybook.model_fields + # Print extracted playbooks for inspection + print("\n" + "=" * 70) + print("EXTRACTED PLAYBOOKS FROM KNOWLEDGE-GAP INTERACTION") + print("=" * 70) + for i, pb in enumerate(user_playbooks, 1): + print(f"\n--- Playbook {i} ---") + print(f" Trigger: {pb.trigger}") + print(f" Content: {pb.content}") + if pb.rationale: + print(f" Rationale: {pb.rationale}") + if pb.blocking_issue: + print( + f" Blocking: [{pb.blocking_issue.kind}] {pb.blocking_issue.details}" + ) + print("\n" + "=" * 70) - # Verify content quality: should mention the gap, not hallucinate a fix - all_content = " ".join(pb.content for pb in user_playbooks).lower() - # The playbook should reference the core issue: guessing / fabricating / no access - gap_keywords = [ - "access", - "check", - "look up", - "guess", - "fabricat", - "admit", - "transparent", - "don't have", - "cannot", - "unable", - "honest", - "limitation", - "assume", - "verify", - "make up", - "invented", - ] - assert any(kw in all_content for kw in gap_keywords), ( - f"Playbook content should reference the knowledge gap. Got: {all_content}" - ) + # Verify schema: flat fields, no instruction/pitfall + for pb in user_playbooks: + assert pb.content and pb.content.strip() + assert pb.trigger and pb.trigger.strip() + assert "instruction" not in UserPlaybook.model_fields + assert "pitfall" not in UserPlaybook.model_fields - finally: - if original_mock is not None: - os.environ["MOCK_LLM_RESPONSE"] = original_mock + # Verify content quality: should mention the gap, not hallucinate a fix + all_content = " ".join(pb.content for pb in user_playbooks).lower() + # The playbook should reference the core issue: guessing / fabricating / no access + gap_keywords = [ + "access", + "check", + "look up", + "guess", + "fabricat", + "admit", + "transparent", + "don't have", + "cannot", + "unable", + "honest", + "limitation", + "assume", + "verify", + "make up", + "invented", + ] + assert any(kw in all_content for kw in gap_keywords), ( + f"Playbook content should reference the knowledge gap. Got: {all_content}" + ) diff --git a/tests/e2e_tests/test_resumable_extraction_e2e.py b/tests/e2e_tests/test_resumable_extraction_e2e.py index b12b6c223..1079f1b1a 100644 --- a/tests/e2e_tests/test_resumable_extraction_e2e.py +++ b/tests/e2e_tests/test_resumable_extraction_e2e.py @@ -17,6 +17,7 @@ from reflexio import InteractionData, ReflexioClient from reflexio.models.api_schema.service_schemas import Interaction, Request from reflexio.models.config_schema import ( + SINGLETON_USER_PLAYBOOK_NAME, Config, PendingToolCallConfig, ProfileExtractorConfig, @@ -225,9 +226,13 @@ def _poll_until( deadline = time.monotonic() + timeout_seconds last_value: Any = None while time.monotonic() < deadline: - last_value = predicate() - if last_value: - return last_value + try: + last_value = predicate() + except requests.RequestException as exc: + last_value = exc + else: + if last_value: + return last_value time.sleep(_LIVE_E2E_POLL_SECONDS) raise AssertionError(f"Timed out waiting for {label}; last value: {last_value!r}") @@ -253,46 +258,45 @@ def _apply_live_config_patch( headers: dict[str, str], patch_payload: dict[str, Any], *, - expected_extractor_name: str | None = None, + expected_playbook_extractor_name: str | None = None, ) -> dict[str, Any]: _api_request("POST", base_url, "/api/update_config", headers, json=patch_payload) _api_request("POST", base_url, "/api/admin/cache/invalidate", headers, json={}) config = _api_request("GET", base_url, "/api/get_config", headers) - if expected_extractor_name is not None: - profile_config = config.get("profile_extractor_config") or {} - assert profile_config.get("extractor_name") == expected_extractor_name + if expected_playbook_extractor_name is not None: + playbook_config = config.get("user_playbook_extractor_config") or {} + assert playbook_config.get("extractor_name") == expected_playbook_extractor_name return config def _live_resumable_config(marker: str) -> tuple[dict[str, Any], str, str]: - extractor_name = "resumable_human_question_live_e2e" question_text = ( f"For live resumable test {marker}, what deployment target should be " "treated as canonical?" ) - profile_prefix = f"Live resumable test {marker} canonical deployment target:" + playbook_prefix = f"Live resumable test {marker} canonical deployment target:" return ( { - "profile_extractor_config": { - "extractor_name": extractor_name, + "profile_extractor_config": None, + "user_playbook_extractor_config": { + "extractor_name": SINGLETON_USER_PLAYBOOK_NAME, "extraction_definition_prompt": ( f"This is an end-to-end test for marker {marker}. " "If the session mentions this marker but does not explicitly " "provide the canonical deployment target, call ask_human before " - "extracting the deployment-target profile. Use this exact " + "extracting the deployment-target playbook. Use this exact " f"question text: {question_text!r}. Use answer_format " f"'short text' and include the tag {marker!r}. After a human " "answer is available, you must respond with the structured " - "result containing exactly one profile. Never respond with " - "profiles null after a resolved answer is present. The profile " - f"content must be exactly {profile_prefix!r} followed by one " - "space and the human-provided answer. Use time_to_live " - "'one_year'. Your first action for this marker must be the " + "result containing exactly one playbook. Never respond with " + "an empty playbooks list after a resolved answer is present. " + "The playbook content must be exactly " + f"{playbook_prefix!r} followed by one space and the " + "human-provided answer. Your first action for this marker must be the " "ask_human tool call. Do not infer or invent the target." ), "context_prompt": "Live E2E test for resumable extraction.", }, - "user_playbook_extractor_config": None, "pending_tool_call_config": { "enabled": True, "pending_ttl_seconds": 3600, @@ -312,7 +316,7 @@ def _live_resumable_config(marker: str) -> tuple[dict[str, Any], str, str]: "stride_size": 4, }, question_text, - profile_prefix, + playbook_prefix, ) @@ -339,15 +343,19 @@ def _find_pending_question( return None -def _profile_content_with( +def _playbook_content_with( client: ReflexioClient, *, user_id: str, required_parts: tuple[str, ...], ) -> str | None: - profiles = client.get_profiles(user_id=user_id, force_refresh=True).user_profiles - for profile in profiles: - content = profile.content + playbooks = client.get_user_playbooks( + user_id=user_id, + playbook_name=SINGLETON_USER_PLAYBOOK_NAME, + ).user_playbooks + for playbook in playbooks: + assert playbook.playbook_name == SINGLETON_USER_PLAYBOOK_NAME + content = playbook.content if all(part in content for part in required_parts): return content return None @@ -407,6 +415,33 @@ def test_resumable_extraction_resumes_after_human_answer( ] +def test_poll_until_retries_request_exception(monkeypatch): + outcomes = iter( + [requests.RequestException("transient connection failure"), "ready"] + ) + monkeypatch.setattr(time, "sleep", lambda _seconds: None) + + def predicate(): + outcome = next(outcomes) + if isinstance(outcome, Exception): + raise outcome + return outcome + + assert _poll_until("transient request", predicate, timeout_seconds=1) == "ready" + + +def test_live_resumable_config_uses_playbook_extractor_for_ask_human(): + marker = "RESUMABLE_LIVE_E2E_CONFIG_TEST" + + config_patch, question_text, _ = _live_resumable_config(marker) + + assert config_patch["profile_extractor_config"] is None + playbook_config = config_patch["user_playbook_extractor_config"] + assert playbook_config["extractor_name"] == SINGLETON_USER_PLAYBOOK_NAME + assert "call ask_human" in playbook_config["extraction_definition_prompt"] + assert question_text in playbook_config["extraction_definition_prompt"] + + @pytest.mark.requires_credentials @pytest.mark.timeout(360) def test_live_resumable_question_resolve_and_edit_roundtrip(): @@ -426,7 +461,7 @@ def test_live_resumable_question_resolve_and_edit_roundtrip(): session_id = f"resumable-live-e2e-{uuid.uuid4().hex[:8]}" first_answer = "AWS ECS" edited_answer = "Google Cloud Run" - config_patch, question_text, profile_prefix = _live_resumable_config(marker) + config_patch, question_text, playbook_prefix = _live_resumable_config(marker) original_config = _api_request("GET", base_url, "/api/get_config", headers) restore_patch = _config_restore_patch(original_config) @@ -436,7 +471,7 @@ def test_live_resumable_question_resolve_and_edit_roundtrip(): base_url, headers, config_patch, - expected_extractor_name="resumable_human_question_live_e2e", + expected_playbook_extractor_name=SINGLETON_USER_PLAYBOOK_NAME, ) client.publish_interaction( @@ -453,16 +488,16 @@ def test_live_resumable_question_resolve_and_edit_roundtrip(): "The canonical deployment target is unknown and not " "available anywhere in this transcript. The only valid " "next step is to ask the configured human follow-up " - "question before extracting any profile. Once a human " - "answer exists, the durable profile should contain exactly " - f"this prefix: {profile_prefix}" + "question before extracting any playbook. Once a human " + "answer exists, the durable playbook should contain exactly " + f"this prefix: {playbook_prefix}" ), ), InteractionData( role="assistant", content=( "I cannot infer the deployment target. I need a human " - "answer before storing any deployment-target profile." + "answer before storing any deployment-target playbook." ), ), ], @@ -493,15 +528,15 @@ def test_live_resumable_question_resolve_and_edit_roundtrip(): assert resolved_call["status"] == "resolved" assert resolved_call["result"]["answer"] == first_answer - first_profile = _poll_until( - "live profile generated from resolved human answer", - lambda: _profile_content_with( + first_playbook = _poll_until( + "live playbook generated from resolved human answer", + lambda: _playbook_content_with( client, user_id=user_id, - required_parts=(profile_prefix, first_answer), + required_parts=(playbook_prefix, first_answer), ), ) - assert marker in first_profile + assert marker in first_playbook edited_call = _api_request( "PATCH", @@ -514,15 +549,15 @@ def test_live_resumable_question_resolve_and_edit_roundtrip(): assert edited_call["result"]["answer"] == edited_answer assert edited_call["result"].get("not_applicable") is not True - edited_profile = _poll_until( - "live profile regenerated from edited human answer", - lambda: _profile_content_with( + edited_playbook = _poll_until( + "live playbook regenerated from edited human answer", + lambda: _playbook_content_with( client, user_id=user_id, - required_parts=(profile_prefix, edited_answer), + required_parts=(playbook_prefix, edited_answer), ), ) - assert marker in edited_profile + assert marker in edited_playbook latest_resolved_call = _find_pending_question( base_url, diff --git a/tests/lib/test_config_unit.py b/tests/lib/test_config_unit.py index ca1a6089e..5de7424fa 100644 --- a/tests/lib/test_config_unit.py +++ b/tests/lib/test_config_unit.py @@ -7,10 +7,15 @@ from typing import Any, cast from unittest.mock import MagicMock +import pytest + from reflexio.lib._config import ConfigMixin from reflexio.lib._dashboard import DashboardMixin from reflexio.models.api_schema.retriever_schema import GetDashboardStatsRequest from reflexio.models.config_schema import Config, StorageConfigSQLite +from reflexio.server.services.configurator.config_storage import ( + ConfigWriteConflictError, +) # --------------------------------------------------------------------------- # ConfigMixin helpers @@ -241,6 +246,26 @@ def test_set_config_dict_input(self): assert response.success is True + def test_set_config_uses_exact_config_returned_by_normalizer(self): + """A typed normalization result is persisted without reconstruction.""" + mixin = _make_config_mixin() + payload = {"storage_config": {"db_path": "/var/data/test.db"}} + normalized = Config( + storage_config=StorageConfigSQLite(db_path="/var/data/test.db") + ) + configurator = _get_configurator(mixin) + configurator.normalize_config_payload.return_value = normalized + configurator.get_current_storage_configuration.return_value = ( + normalized.storage_config + ) + + response = mixin.set_config(payload) + + assert response.success is True + configurator.normalize_config_payload.assert_called_once_with(payload) + configurator.set_config.assert_called_once_with(normalized) + assert configurator.set_config.call_args.args[0] is normalized + def test_set_config_exception(self): """Returns failure on unexpected exception.""" mixin = _make_config_mixin() @@ -256,6 +281,20 @@ def test_set_config_exception(self): assert response.success is False assert "unexpected" in (response.msg or "") + def test_set_config_reraises_known_write_conflict(self): + """CAS conflicts must reach routes so callers receive HTTP 409.""" + mixin = _make_config_mixin() + storage_config = StorageConfigSQLite(db_path="/var/data/current.db") + config = Config(storage_config=storage_config, window_size=25) + configurator = _get_configurator(mixin) + configurator.get_current_storage_configuration.return_value = storage_config + configurator.set_config.side_effect = ConfigWriteConflictError( + "Configuration changed while writing" + ) + + with pytest.raises(ConfigWriteConflictError): + mixin.set_config(config) + # --------------------------------------------------------------------------- # DashboardMixin helpers diff --git a/tests/server/api_endpoints/test_api_routes.py b/tests/server/api_endpoints/test_api_routes.py index a53656ad2..0d405f376 100644 --- a/tests/server/api_endpoints/test_api_routes.py +++ b/tests/server/api_endpoints/test_api_routes.py @@ -10,6 +10,9 @@ from pathlib import Path from unittest.mock import MagicMock, patch +import pytest +from fastapi import HTTPException + from reflexio.models.api_schema.retriever_schema import ( GetProfilesViewResponse, SearchInteractionResponse, @@ -23,6 +26,10 @@ UserProfile, ) from reflexio.models.config_schema import Config, StorageConfigSQLite +from reflexio.server.rate_limit import limiter +from reflexio.server.services.configurator.config_storage import ( + ConfigWriteConflictError, +) class TestHealthEndpoints: @@ -417,9 +424,34 @@ def test_missing_required_fields_returns_422(self, client): class TestSetConfigRoute: """Tests for POST /api/set_config (full replacement semantics).""" + def test_passes_exact_config_returned_by_normalizer( + self, client, patched_reflexio, mock_reflexio + ): + payload = { + "storage_config": { + "db_path": str(Path(tempfile.gettempdir()) / "set-config.db") + } + } + normalized = Config.model_validate(payload) + configurator = mock_reflexio.request_context.configurator + configurator.normalize_config_payload.return_value = normalized + mock_reflexio.set_config.return_value = SetConfigResponse( + success=True, msg="Configuration set successfully" + ) + + response = client.post("/api/set_config", json=payload) + + assert response.status_code == 200, response.text + configurator.normalize_config_payload.assert_called_once_with(payload) + mock_reflexio.set_config.assert_called_once_with(normalized) + assert mock_reflexio.set_config.call_args.args[0] is normalized + def test_unknown_field_returns_422_before_set_config( self, client, patched_reflexio, mock_reflexio ): + configurator = mock_reflexio.request_context.configurator + configurator.normalize_config_payload.side_effect = lambda payload: payload + response = client.post( "/api/set_config", json={ @@ -438,12 +470,14 @@ class TestUpdateConfigRoute: """Tests for POST /api/update_config (PATCH-style partial update). The endpoint fetches the existing config, shallow-merges the partial - payload over it, and round-trips through ``Config(**merged)`` so - Pydantic rejects unknown fields. Storage validation lives in - ``reflexio.set_config``; we mock it out and assert the merged dict - that arrives there. + payload over it, and delegates to the active configurator before shared + Pydantic validation. Storage validation lives in ``reflexio.set_config``. """ + @pytest.fixture(autouse=True) + def _reset_rate_limit(self) -> None: + limiter.reset() + @staticmethod def _existing_config() -> Config: # Platform-aware temp path — Ruff S108 flags hardcoded ``/tmp``. @@ -456,6 +490,14 @@ def _existing_config() -> Config: def _wire_mock(self, mock_reflexio: MagicMock, existing: Config) -> None: configurator = MagicMock() configurator.get_config.return_value = existing + configurator.normalize_config_payload.side_effect = lambda payload: payload + configurator.prepare_config_patch.side_effect = lambda partial: ( + Config.model_validate( + configurator.normalize_config_payload( + {**existing.model_dump(mode="python"), **partial} + ) + ) + ) mock_reflexio.request_context.configurator = configurator mock_reflexio.set_config.return_value = SetConfigResponse( success=True, msg="Configuration set successfully" @@ -484,10 +526,28 @@ def test_partial_dict_succeeds(self, client, patched_reflexio, mock_reflexio): assert isinstance(merged, Config) assert merged.window_size == 25 assert merged.storage_config == existing.storage_config + configurator = mock_reflexio.request_context.configurator + configurator.prepare_config_patch.assert_called_once_with({"window_size": 25}) # Cache invalidated on success. mock_invalidate.assert_called_once_with(org_id="test-org") + def test_patch_uses_prepared_config_instance( + self, client, patched_reflexio, mock_reflexio + ): + existing = self._existing_config() + self._wire_mock(mock_reflexio, existing) + configurator = mock_reflexio.request_context.configurator + prepared = existing.model_copy(update={"window_size": 31}) + object.__setattr__(prepared, "_patch_marker", "prepared") + configurator.prepare_config_patch.return_value = prepared + configurator.prepare_config_patch.side_effect = None + + response = client.post("/api/update_config", json={"window_size": 31}) + + assert response.status_code == 200, response.text + assert mock_reflexio.set_config.call_args.args[0] is prepared + def test_no_op_patch_skips_set_config_and_cache_invalidation( self, client, patched_reflexio, mock_reflexio ): @@ -510,6 +570,30 @@ def test_no_op_patch_skips_set_config_and_cache_invalidation( mock_reflexio.set_config.assert_not_called() mock_invalidate.assert_not_called() + def test_managed_patch_matching_cache_is_persisted( + self, client, patched_reflexio, mock_reflexio + ): + """A durable managed snapshot may have changed after the cached read.""" + existing = self._existing_config() + self._wire_mock(mock_reflexio, existing) + configurator = mock_reflexio.request_context.configurator + configurator.requires_durable_config_patch = True + + with patch( + "reflexio.server.cache.reflexio_cache.invalidate_reflexio_cache" + ) as mock_invalidate: + response = client.post( + "/api/update_config", + json={"window_size": existing.window_size}, + ) + + assert response.status_code == 200, response.text + assert mock_reflexio.set_config.call_count == 1 + assert mock_reflexio.set_config.call_args.args[0].window_size == ( + existing.window_size + ) + mock_invalidate.assert_called_once_with(org_id="test-org") + def test_unknown_field_returns_422_before_set_config( self, client, patched_reflexio, mock_reflexio ): @@ -525,6 +609,64 @@ def test_unknown_field_returns_422_before_set_config( assert response.status_code == 422, response.text mock_reflexio.set_config.assert_not_called() + def test_overlay_field_is_validated_by_configurator_before_shared_config( + self, client, patched_reflexio, mock_reflexio + ): + existing = self._existing_config() + self._wire_mock(mock_reflexio, existing) + configurator = mock_reflexio.request_context.configurator + normalized = existing.model_copy(update={"window_size": 25}) + object.__setattr__(normalized, "_overlay_marker", "normalized") + configurator.normalize_config_payload.return_value = normalized + configurator.normalize_config_payload.side_effect = None + + response = client.post( + "/api/update_config", + json={"offline_tuner_config": {"enabled": True}, "window_size": 25}, + ) + + assert response.status_code == 200, response.text + patch_payload = configurator.prepare_config_patch.call_args.args[0] + assert patch_payload["offline_tuner_config"] == {"enabled": True} + assert mock_reflexio.set_config.call_args.args[0] is normalized + + def test_configurator_capability_error_propagates( + self, client, patched_reflexio, mock_reflexio + ): + existing = self._existing_config() + self._wire_mock(mock_reflexio, existing) + configurator = mock_reflexio.request_context.configurator + configurator.normalize_config_payload.side_effect = HTTPException( + status_code=409, + detail={"error": "replay_unsupported"}, + ) + + response = client.post( + "/api/update_config", + json={"offline_tuner_config": {"enabled": True}}, + ) + + assert response.status_code == 409, response.text + assert response.json()["detail"] == {"error": "replay_unsupported"} + mock_reflexio.set_config.assert_not_called() + + def test_config_write_conflict_returns_409( + self, client, patched_reflexio, mock_reflexio + ): + existing = self._existing_config() + self._wire_mock(mock_reflexio, existing) + mock_reflexio.set_config.side_effect = ConfigWriteConflictError( + "Configuration changed while writing" + ) + + response = client.post("/api/update_config", json={"window_size": 25}) + + assert response.status_code == 409, response.text + assert response.json()["detail"] == { + "error": "config_write_conflict", + "message": "Configuration changed while writing", + } + def test_replaces_nested_object_wholesale( self, client, patched_reflexio, mock_reflexio ): @@ -547,9 +689,7 @@ def test_does_not_invalidate_on_failure( ): """When reflexio.set_config returns success=False, cache stays warm.""" existing = self._existing_config() - configurator = MagicMock() - configurator.get_config.return_value = existing - mock_reflexio.request_context.configurator = configurator + self._wire_mock(mock_reflexio, existing) mock_reflexio.set_config.return_value = SetConfigResponse( success=False, msg="storage validation failed" ) diff --git a/tests/server/services/governance/test_governance_local_e2e.py b/tests/server/services/governance/test_governance_local_e2e.py index 571e53005..d8ee06b06 100644 --- a/tests/server/services/governance/test_governance_local_e2e.py +++ b/tests/server/services/governance/test_governance_local_e2e.py @@ -707,6 +707,84 @@ def _raise_prepare(*args, **kwargs) -> None: assert failed_barriers[0]["error_detail"] == "RuntimeError" +def test_subject_erasure_lifecycle_retry_is_idempotent_and_counted( + storage: SQLiteStorage, + monkeypatch: pytest.MonkeyPatch, +) -> None: + class RetrySafeLifecycle: + calls = 0 + deleted = 0 + + def erase_subject( + self, + *, + storage: SQLiteStorage, + subject_ref: str, + purge_id: str, + ) -> None: + del subject_ref + self.calls += 1 + target = next( + target + for target in storage.list_purge_targets(purge_id, phase="delete") + if target.target_name == "offline_tuner_reward_label" + ) + if target.deleted_count == 2: + return + self.deleted += 2 + storage.record_purge_target( + purge_id=purge_id, + target_name="offline_tuner_reward_label", + target_ref="all", + phase="delete", + status="complete", + detail={"count": 2}, + deleted_count=2, + ) + + lifecycle = RetrySafeLifecycle() + service = GovernanceService( + storage=storage, + org_id=storage.org_id, + ref_secret="test-governance-secret", + subject_erasure_lifecycle=lifecycle, + ) + complete = storage.complete_subject_erasure_barrier_after_empty_check + completion_attempts = 0 + + def fail_after_first_lifecycle(*args, **kwargs): + nonlocal completion_attempts + completion_attempts += 1 + if completion_attempts == 1: + raise RuntimeError("forced post-lifecycle failure") + return complete(*args, **kwargs) + + monkeypatch.setattr( + storage, + "complete_subject_erasure_barrier_after_empty_check", + fail_after_first_lifecycle, + ) + + with pytest.raises(RuntimeError, match="forced post-lifecycle failure"): + service.erase_user(user_id="alice", request_id="erase-lifecycle-retry") + + retried = service.erase_user(user_id="alice", request_id="erase-lifecycle-retry") + + assert retried.status == "complete" + assert retried.deleted_counts["offline_tuner_reward_labels"] == 2 + assert lifecycle.calls == 1 + assert lifecycle.deleted == 2 + snapshot = next( + target + for target in storage.list_purge_targets( + retried.purge_id, phase="prepare_targets" + ) + if target.target_name == "target_snapshot" + ) + assert snapshot.detail is not None + assert snapshot.detail["status"] == "complete" + + def test_session_export_paginates_by_returned_rows_when_requests_are_missing() -> None: class _Storage: def __init__(self) -> None: diff --git a/tests/server/services/playbook/components/test_aggregator_effect_scope.py b/tests/server/services/playbook/components/test_aggregator_effect_scope.py new file mode 100644 index 000000000..5bf93cf95 --- /dev/null +++ b/tests/server/services/playbook/components/test_aggregator_effect_scope.py @@ -0,0 +1,82 @@ +"""Regression tests for managed aggregation effect scopes.""" + +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from reflexio.models.api_schema.service_schemas import UserPlaybook +from reflexio.models.config_schema import PlaybookAggregatorConfig +from reflexio.server.services.playbook.components import aggregator as aggregator_module +from reflexio.server.services.playbook.components.aggregator import PlaybookAggregator +from reflexio.server.services.playbook.playbook_service_utils import ( + PlaybookAggregatorRequest, +) + + +class _OriginalEntryError(RuntimeError): + pass + + +class _EntryFailureScope: + def __init__(self) -> None: + self.exit_called = False + + def __enter__(self) -> None: + raise _OriginalEntryError("effect transaction entry failed") + + def __exit__(self, *_args: object) -> None: + self.exit_called = True + raise RuntimeError("unentered scope exit masked original error") + + +def test_effect_scope_entry_failure_preserves_original_error(monkeypatch) -> None: + storage = MagicMock() + storage.count_user_playbooks.return_value = 2 + storage.get_agent_playbooks.return_value = [] + user_playbooks = [ + UserPlaybook( + user_playbook_id=index, + request_id=f"request-{index}", + agent_version="v1", + playbook_name="feedback", + content=f"content-{index}", + ) + for index in (1, 2) + ] + storage.get_user_playbooks.return_value = user_playbooks + config = SimpleNamespace( + user_playbook_extractor_config=SimpleNamespace( + aggregation_config=PlaybookAggregatorConfig( + min_cluster_size=2, + reaggregation_trigger_count=2, + ) + ) + ) + context = SimpleNamespace( + org_id="test-org", + storage=storage, + configurator=SimpleNamespace(get_config=lambda: config), + ) + coordinator = MagicMock() + scope = _EntryFailureScope() + coordinator.apply_scope.return_value = scope + aggregator = PlaybookAggregator( + llm_client=MagicMock(), + request_context=context, # type: ignore[arg-type] + agent_version="v1", + effect_coordinator=coordinator, + ) + aggregator.get_clusters = MagicMock(return_value={0: user_playbooks}) # type: ignore[method-assign] + state = MagicMock() + state.get_cluster_fingerprints.return_value = {} + aggregator._create_state_manager = MagicMock(return_value=state) # type: ignore[method-assign] + aggregator._generate_playbooks_with_source_clusters = MagicMock( # type: ignore[method-assign] + return_value=[] + ) + monkeypatch.setattr(aggregator_module, "record_usage_event", lambda **_kw: None) + + with pytest.raises(_OriginalEntryError, match="effect transaction entry failed"): + aggregator.run(PlaybookAggregatorRequest(agent_version="v1", rerun=True)) + + assert scope.exit_called is False diff --git a/tests/server/services/playbook/test_aggregation_lineage_integration.py b/tests/server/services/playbook/test_aggregation_lineage_integration.py index 90b3cef2e..3dffc2312 100644 --- a/tests/server/services/playbook/test_aggregation_lineage_integration.py +++ b/tests/server/services/playbook/test_aggregation_lineage_integration.py @@ -21,11 +21,15 @@ import os import tempfile +from collections.abc import Generator +from contextlib import contextmanager +from typing import Any from unittest.mock import MagicMock, patch import pytest from reflexio.lib._agent_playbook import reconstruct_playbook_aggregation_change_log +from reflexio.models.api_schema.domain.entities import LineageContext from reflexio.models.api_schema.service_schemas import ( AgentPlaybook, PlaybookStatus, @@ -33,11 +37,13 @@ ) from reflexio.models.config_schema import PlaybookAggregatorConfig, PlaybookConfig from reflexio.server.api_endpoints.request_context import RequestContext +from reflexio.server.llm._litellm_types import ModelProvenance from reflexio.server.services.playbook.components.aggregator import PlaybookAggregator from reflexio.server.services.playbook.playbook_service_utils import ( PlaybookAggregatorRequest, ) from reflexio.server.services.storage.sqlite_storage import SQLiteStorage +from reflexio.server.services.storage.storage_base import AGGREGATE_REASON_PREFIX pytestmark = pytest.mark.integration @@ -195,6 +201,159 @@ def test_aggregation_emits_aggregate_lineage_event( assert evt.request_id != "", "request_id must be non-empty" +def test_non_managed_aggregation_preserves_lineage_operation_key_deduplication( + sqlite_storage: SQLiteStorage, + request_context: RequestContext, + aggregator: PlaybookAggregator, +) -> None: + up_a = _seed_user_playbook(sqlite_storage, uid=101, org_id=request_context.org_id) + up_b = _seed_user_playbook(sqlite_storage, uid=102, org_id=request_context.org_id) + cluster_playbooks = [up_a, up_b] + unsaved = AgentPlaybook( + agent_playbook_id=0, + playbook_name="default", + agent_version="v0", + content="Durably keyed aggregation.", + playbook_status=PlaybookStatus.PENDING, + ) + + with ( + patch.object( + PlaybookAggregator, + "get_clusters", + return_value={0: cluster_playbooks}, + ), + patch.object( + PlaybookAggregator, + "_generate_playbooks_with_source_clusters", + return_value=[(unsaved, cluster_playbooks, None)], + ), + ): + aggregator.run( + PlaybookAggregatorRequest( + agent_version="v0", + rerun=True, + operation_key="42", + ) + ) + retry = aggregator.run( + PlaybookAggregatorRequest( + agent_version="v0", + rerun=True, + operation_key="42", + ) + ) + + saved = sqlite_storage.get_agent_playbooks() + assert len(saved) == 1 + events = sqlite_storage.get_lineage_events( + entity_type="agent_playbook", + entity_id=str(saved[0].agent_playbook_id), + ) + assert [event.request_id for event in events if event.op == "aggregate"] == ["42"] + + assert retry["skipped"] == "operation already applied" + assert len(sqlite_storage.get_agent_playbooks()) == 1 + + +def test_managed_aggregation_prepares_then_atomically_completes_all_effects( + sqlite_storage: SQLiteStorage, + request_context: RequestContext, +) -> None: + up_a = _seed_user_playbook(sqlite_storage, uid=103, org_id=request_context.org_id) + up_b = _seed_user_playbook(sqlite_storage, uid=104, org_id=request_context.org_id) + cluster_playbooks = [up_a, up_b] + unsaved = AgentPlaybook( + agent_playbook_id=0, + playbook_name="default", + agent_version="v0", + content="Retry-safe aggregation.", + playbook_status=PlaybookStatus.PENDING, + ) + + class _Coordinator: + prepared = False + active = False + completed_result: dict[str, Any] | None = None + + def prepare(self, playbooks: list[AgentPlaybook]) -> None: + assert not self.active + assert playbooks == [unsaved] + self.prepared = True + + @contextmanager + def apply_scope(self) -> Generator[None, None, None]: + assert self.prepared + self.active = True + try: + yield + finally: + self.active = False + + def save_agent_playbook( + self, + playbook: AgentPlaybook, + *, + source_ids: list[str], + request_id: str, + run_mode: str, + provenance: ModelProvenance | None, + ) -> AgentPlaybook: + assert self.active + return sqlite_storage.save_agent_playbooks( + [playbook], + lineage_contexts=[ + LineageContext( + op_kind="aggregate", + actor="aggregator", + request_id=request_id, + source_ids=source_ids, + reason=f"{AGGREGATE_REASON_PREFIX}{run_mode}", + model_name=provenance.model_name if provenance else None, + provider=provenance.provider if provenance else None, + ) + ], + )[0] + + def complete(self, result: dict[str, Any]) -> None: + assert self.active + self.completed_result = result + + coordinator = _Coordinator() + aggregator = PlaybookAggregator( + llm_client=MagicMock(), + request_context=request_context, + agent_version="v0", + effect_coordinator=coordinator, + ) + + with ( + patch.object( + PlaybookAggregator, + "get_clusters", + return_value={0: cluster_playbooks}, + ), + patch.object( + PlaybookAggregator, + "_generate_playbooks_with_source_clusters", + return_value=[(unsaved, cluster_playbooks, None)], + ), + ): + result = aggregator.run( + PlaybookAggregatorRequest( + agent_version="v0", + rerun=True, + operation_key="43", + ) + ) + + assert result["playbooks_generated"] == 1 + assert coordinator.completed_result == result + assert coordinator.active is False + assert len(sqlite_storage.get_agent_playbooks()) == 1 + assert len(sqlite_storage.get_lineage_events(request_id="43")) == 1 + + def test_aggregate_save_failure_aborts_and_restores( sqlite_storage: SQLiteStorage, request_context: RequestContext, diff --git a/tests/server/services/playbook/test_apply_playbook_edit_integration.py b/tests/server/services/playbook/test_apply_playbook_edit_integration.py deleted file mode 100644 index d98025f5f..000000000 --- a/tests/server/services/playbook/test_apply_playbook_edit_integration.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Integration tests for apply_playbook_edit() — atomic supersede path.""" - -import pytest - -from reflexio.models.api_schema.domain.entities import UserPlaybook -from reflexio.models.api_schema.domain.enums import Status -from reflexio.server.services.playbook.playbook_edit_apply import apply_playbook_edit -from reflexio.server.services.storage.sqlite_storage import SQLiteStorage - -pytestmark = pytest.mark.integration - - -def test_apply_supersedes_incumbent_and_links(tmp_path): - s = SQLiteStorage(org_id="test_org", db_path=str(tmp_path / "t.db")) - s.migrate() - inc = UserPlaybook(user_id="u", agent_version="v", request_id="r", content="v1") - s.save_user_playbooks([inc]) - new = UserPlaybook(user_id="u", agent_version="v", request_id="r", content="v2") - new_id = apply_playbook_edit( - s, - incumbent_id=inc.user_playbook_id, - new_playbook=new, - source="offline_optimizer", - request_id="run-test-1", - ) - assert new_id > 0 - tomb = s.get_user_playbook_by_id(inc.user_playbook_id, include_tombstones=True) - assert tomb.status is Status.SUPERSEDED and tomb.superseded_by == new_id - - -def test_apply_no_orphan_when_incumbent_already_gone(tmp_path): - s = SQLiteStorage(org_id="test_org", db_path=str(tmp_path / "t.db")) - s.migrate() - inc = UserPlaybook( - user_id="u", - agent_version="v", - request_id="r", - content="v1", - status=Status.ARCHIVED, - ) - s.save_user_playbooks([inc]) - new = UserPlaybook(user_id="u", agent_version="v", request_id="r", content="v2") - rc = apply_playbook_edit( - s, - incumbent_id=inc.user_playbook_id, - new_playbook=new, - source="offline_optimizer", - request_id="run-test-2", - ) - assert rc == -1 - # no orphan CURRENT row left behind - currents = list(s.get_user_playbooks(user_id="u")) - assert all(p.content != "v2" for p in currents) - - -def test_apply_lost_cas_rolls_back_successor_and_leaves_no_orphan(tmp_path): - s = SQLiteStorage(org_id="test_org", db_path=str(tmp_path / "t.db")) - s.migrate() - inc = UserPlaybook(user_id="u", agent_version="v", request_id="r", content="v1") - s.save_user_playbooks([inc]) - - winner_id = apply_playbook_edit( - s, - incumbent_id=inc.user_playbook_id, - new_playbook=UserPlaybook( - user_id="u", agent_version="v", request_id="r", content="winner" - ), - source="offline_optimizer", - request_id="run-test-3a", - ) - assert winner_id > 0 - - rc = apply_playbook_edit( - s, - incumbent_id=inc.user_playbook_id, - new_playbook=UserPlaybook( - user_id="u", agent_version="v", request_id="r", content="loser" - ), - source="offline_optimizer", - request_id="run-test-3b", - ) - - assert rc == -1 - rows = s.conn.execute( - "SELECT content, status FROM user_playbooks ORDER BY user_playbook_id" - ).fetchall() - assert len(rows) == 2 - assert {row["content"] for row in rows} == {"v1", "winner"} - currents = list(s.get_user_playbooks(user_id="u")) - assert len(currents) == 1 - assert currents[0].content == "winner" - events = s.get_lineage_events(entity_type="user_playbook") - assert [e.op for e in events] == ["create", "create", "revise"] diff --git a/tests/server/services/playbook/test_playbook_aggregator.py b/tests/server/services/playbook/test_playbook_aggregator.py index 5af287b0c..2ccc6c543 100644 --- a/tests/server/services/playbook/test_playbook_aggregator.py +++ b/tests/server/services/playbook/test_playbook_aggregator.py @@ -1001,6 +1001,36 @@ def test_rerun_supersedes_archived_playbooks_after_success( ) agg.storage.delete_archived_agent_playbooks_by_playbook_name.assert_not_called() + @patch.object(PlaybookAggregator, "get_clusters") + @patch.object(PlaybookAggregator, "_generate_playbooks_with_source_clusters") + def test_managed_run_does_not_complete_when_supersession_fails( + self, mock_gen, mock_clust + ): + agg = self._make_runnable_aggregator() + raws = [_raw(rid=1), _raw(rid=2)] + generated = _agent_playbook(fid=100) + mock_clust.return_value = {0: raws} + mock_gen.return_value = [(generated, raws, None)] + + coordinator = MagicMock() + + @contextmanager + def _apply_scope(): + yield + + coordinator.apply_scope.side_effect = _apply_scope + coordinator.save_agent_playbook.return_value = generated + agg.effect_coordinator = coordinator + agg.storage.supersede_agent_playbooks_by_playbook_name.side_effect = ( + RuntimeError("supersession failed") + ) + + with pytest.raises(RuntimeError, match="supersession failed"): + agg.run(PlaybookAggregatorRequest(agent_version="v1", rerun=True)) + + coordinator.complete.assert_not_called() + agg.storage.restore_archived_agent_playbooks_by_playbook_name.assert_not_called() + @patch.object(PlaybookAggregator, "get_clusters") @patch.object(PlaybookAggregator, "_generate_playbooks_with_source_clusters") def test_first_run_no_prev_fingerprints_full_archive(self, mock_gen, mock_clust): diff --git a/tests/server/services/playbook/test_playbook_edit_apply.py b/tests/server/services/playbook/test_playbook_edit_apply.py deleted file mode 100644 index cee4198e8..000000000 --- a/tests/server/services/playbook/test_playbook_edit_apply.py +++ /dev/null @@ -1,263 +0,0 @@ -"""Tests for apply_playbook_edit() — the shared archive+insert primitive.""" - -import tempfile -from unittest.mock import patch - -import pytest - -from reflexio.models.api_schema.domain.entities import UserPlaybook -from reflexio.server.services.storage.sqlite_storage import SQLiteStorage -from reflexio.server.services.storage.sqlite_storage._lineage import ( - _EMPTY_REQUEST_ID_MSG, -) - - -def _storage(tmp: str) -> SQLiteStorage: - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - return SQLiteStorage(org_id="org_apply_1", db_path=f"{tmp}/t.db") - - -def _playbook(user_id: str = "u1", content: str = "old") -> UserPlaybook: - return UserPlaybook( - user_id=user_id, - agent_version="v1", - request_id="req_test", - playbook_name="refund", - content=content, - trigger="refund", - ) - - -def test_apply_inserts_new_and_archives_incumbent(): - from reflexio.server.services.playbook.playbook_edit_apply import ( - apply_playbook_edit, - ) - - with tempfile.TemporaryDirectory() as tmp: - s = _storage(tmp) - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - old = _playbook(content="old") - s.save_user_playbooks([old]) - old_id = old.user_playbook_id - assert old_id > 0 - - new = _playbook(content="new") - new_id = apply_playbook_edit( - s, - incumbent_id=old_id, - new_playbook=new, - source="offline_optimizer", - request_id="run-abc", - ) - assert new_id > 0 - - # Only new_id should be CURRENT (status=None); old_id should be archived - all_pbs = s.get_user_playbooks(user_id="u1") - current_ids = {p.user_playbook_id for p in all_pbs if p.status is None} - assert new_id in current_ids - assert old_id not in current_ids - - -def test_apply_skips_archive_when_incumbent_not_current(): - from reflexio.server.services.playbook.playbook_edit_apply import ( - apply_playbook_edit, - ) - - with tempfile.TemporaryDirectory() as tmp: - s = _storage(tmp) - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - old = _playbook(content="old") - s.save_user_playbooks([old]) - old_id = old.user_playbook_id - assert old_id > 0 - - # Someone else archived it first - s.archive_user_playbook_by_id(user_id="u1", user_playbook_id=old_id) - - new = _playbook(content="new") - new_id = apply_playbook_edit( - s, - incumbent_id=old_id, - new_playbook=new, - source="offline_optimizer", - request_id="run-abc", - ) - # Optimistic-concurrency: incumbent was already archived → skip and return -1 - assert new_id == -1 - - -def test_apply_expect_current_false_archives(): - """With expect_current=False, new playbook is inserted and incumbent is archived.""" - from reflexio.server.services.playbook.playbook_edit_apply import ( - apply_playbook_edit, - ) - - with tempfile.TemporaryDirectory() as tmp: - s = _storage(tmp) - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - old = _playbook(content="old") - s.save_user_playbooks([old]) - old_id = old.user_playbook_id - assert old_id > 0 - - new = _playbook(content="new") - new_id = apply_playbook_edit( - s, - incumbent_id=old_id, - new_playbook=new, - source="offline_optimizer", - request_id="run-abc", - ) - assert new_id > 0 - - # Only new_id should be CURRENT (status=None); old_id should be archived - all_pbs = s.get_user_playbooks(user_id="u1") - current_ids = {p.user_playbook_id for p in all_pbs if p.status is None} - assert new_id in current_ids - assert old_id not in current_ids - - -def test_apply_expect_current_false_returns_minus1_and_no_orphan(): - """When incumbent is already archived, supersede_record returns False. - - The transaction rolls back the provisional successor and its create event, - so the -1 return value indicates the lost race, not an orphan. - """ - from reflexio.server.services.playbook.playbook_edit_apply import ( - apply_playbook_edit, - ) - - with tempfile.TemporaryDirectory() as tmp: - s = _storage(tmp) - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - old = _playbook(content="old") - s.save_user_playbooks([old]) - old_id = old.user_playbook_id - assert old_id > 0 - - # Archive first so supersede_record will return False - s.archive_user_playbook_by_id(user_id="u1", user_playbook_id=old_id) - event_ids_before = { - event.event_id for event in s.get_lineage_events(org_id="org_apply_1") - } - - new = _playbook(content="new") - new_id = apply_playbook_edit( - s, - incumbent_id=old_id, - new_playbook=new, - source="offline_optimizer", - request_id="run-abc", - ) - # supersede_record returned False → -1, transaction rolled back. - assert new_id == -1 - - # No orphan: the inserted successor was deleted - all_pbs = s.get_user_playbooks(user_id="u1") - current_ids = {p.user_playbook_id for p in all_pbs if p.status is None} - assert len(current_ids) == 0 - assert { - event.event_id for event in s.get_lineage_events(org_id="org_apply_1") - } == event_ids_before - - -def test_apply_raises_on_empty_request_id_before_write(): - """apply_playbook_edit raises ValueError on empty request_id before any storage write. - - The I2 (orphan) guard: an empty request_id is rejected immediately so no - successor row is ever inserted when the caller forgets to supply a run id. - """ - from reflexio.server.services.playbook.playbook_edit_apply import ( - apply_playbook_edit, - ) - - with tempfile.TemporaryDirectory() as tmp: - s = _storage(tmp) - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - old = _playbook(content="old") - s.save_user_playbooks([old]) - old_id = old.user_playbook_id - - # Patch save_user_playbooks to confirm it is never reached on empty request_id. - with patch.object(s, "save_user_playbooks") as mock_save: - with pytest.raises(ValueError, match=_EMPTY_REQUEST_ID_MSG): - apply_playbook_edit( - s, - incumbent_id=old_id, - new_playbook=_playbook(content="new"), - source="offline_optimizer", - request_id="", - ) - mock_save.assert_not_called() - - # No orphan: no successor row was inserted (incumbent still CURRENT, count==1). - count = s.conn.execute( - "SELECT COUNT(*) FROM user_playbooks WHERE status IS NULL" - ).fetchone()[0] - assert count == 1, ( - "no orphan successor row should be inserted on empty request_id" - ) - - -@pytest.mark.parametrize("bad_request_id", ["", None]) -def test_apply_raises_on_empty_or_none_request_id(bad_request_id): - """apply_playbook_edit raises ValueError for both empty string and None request_id.""" - from reflexio.server.services.playbook.playbook_edit_apply import ( - apply_playbook_edit, - ) - - with tempfile.TemporaryDirectory() as tmp: - s = _storage(tmp) - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - old = _playbook(content="old") - s.save_user_playbooks([old]) - old_id = old.user_playbook_id - - with pytest.raises((ValueError, TypeError)): - apply_playbook_edit( - s, - incumbent_id=old_id, - new_playbook=_playbook(content="new"), - source="offline_optimizer", - request_id=bad_request_id, # type: ignore[arg-type] - ) - - -def test_apply_lineage_event_carries_operation_run_id(): - """apply_playbook_edit records the operation-run request_id on the revise event. - - The lineage event must carry the operation request_id, - NOT the incumbent's birth request_id. This enables correct run-correlation. - """ - from reflexio.server.services.playbook.playbook_edit_apply import ( - apply_playbook_edit, - ) - - with tempfile.TemporaryDirectory() as tmp: - s = _storage(tmp) - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - old = _playbook(content="old") - s.save_user_playbooks([old]) - old_id = old.user_playbook_id - assert old_id > 0 - - operation_run_id = "optimizer_run_xyz" - new = _playbook(content="new") - new_id = apply_playbook_edit( - s, - incumbent_id=old_id, - new_playbook=new, - source="offline_optimizer", - request_id=operation_run_id, - ) - assert new_id > 0 - - events = s.get_lineage_events( - entity_type="user_playbook", entity_id=str(new_id) - ) - assert [event.op for event in events] == ["create", "revise"] - revise_event = events[1] - assert revise_event.request_id == operation_run_id, ( - f"lineage event must carry the operation run id {operation_run_id!r}, " - f"not the incumbent's birth request_id {old.request_id!r}" - ) diff --git a/tests/server/services/playbook/test_publication_models.py b/tests/server/services/playbook/test_publication_models.py new file mode 100644 index 000000000..4eb5d150d --- /dev/null +++ b/tests/server/services/playbook/test_publication_models.py @@ -0,0 +1,364 @@ +from __future__ import annotations + +import json +from hashlib import sha256 + +import pytest + +from reflexio.server.services.playbook.publication import ( + DecisionProofEnvelope, + PublicationClaim, + PublicationRequest, + PublicationSearchProjection, + UserPlaybookPublicationService, + canonical_json_bytes, + incumbent_user_playbook_semantic_digest, +) + + +def _canonical(payload: dict[str, object]) -> str: + return json.dumps( + payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True + ) + + +def _digest(value: str) -> str: + return sha256(value.encode()).hexdigest() + + +def _projection() -> PublicationSearchProjection: + canonical = _canonical( + { + "candidate_content_digest": _digest("new content"), + "embedding": ["0.125", "0.5"], + "embedding_model_id": "test-embedding-v1", + "expanded_terms": ["refund", "escalation"], + "lexical_document": "refund escalation exact projection", + "preserved_trigger": "refund", + "projector_code_digest": "a" * 64, + "projector_id": "reflexio.search.user-playbook", + "projector_version": "1", + "schema_version": "offline-tuner-candidate-search-projection-v1", + } + ) + return PublicationSearchProjection( + schema_version="offline-tuner-candidate-search-projection-v1", + canonical_json=canonical, + digest=_digest(canonical), + projector_id="reflexio.search.user-playbook", + projector_version="1", + projector_code_digest="a" * 64, + candidate_content_digest=_digest("new content"), + preserved_trigger="refund", + embedding_model_id="test-embedding-v1", + embedding=("0.125", "0.5"), + expanded_terms=("refund", "escalation"), + lexical_document="refund escalation exact projection", + ) + + +def _proof() -> DecisionProofEnvelope: + canonical = _canonical( + { + "decision": "apply", + "optimizer_kind": "gepa", + "schema_version": "gepa-publication-proof-v1", + "source": "playbook_optimizer", + } + ) + return DecisionProofEnvelope( + optimizer_kind="gepa", + schema_version="gepa-publication-proof-v1", + canonical_json=canonical, + digest=_digest(canonical), + decision="apply", + ) + + +def test_publication_models_accept_strict_canonical_payloads() -> None: + request = PublicationRequest( + optimizer_kind="gepa", + job_id=7, + attempt_key="attempt-7", + publication_claim=PublicationClaim(job_id=7, owner="worker-a", fence=3), + worker_fence=11, + incumbent_user_playbook_id=101, + incumbent_content_digest=_digest("old content"), + incumbent_trigger="refund", + incumbent_semantic_digest=incumbent_user_playbook_semantic_digest( + content_digest=_digest("old content"), trigger="refund" + ), + revised_content="new content", + projection=_projection(), + decision_proof=_proof(), + subject_epochs_json=_canonical({"subjects": [{"ref": "user:u1", "epoch": 0}]}), + request_id="request-7", + ) + + assert request.projection.digest == _digest(request.projection.canonical_json) + assert request.decision_proof.digest == _digest( + request.decision_proof.canonical_json + ) + + +@pytest.mark.parametrize( + "bad_digest", + [ + "A" * 64, + "0" * 63, + "g" * 64, + ], +) +def test_publication_digest_validation_rejects_non_lowercase_sha256( + bad_digest: str, +) -> None: + canonical = _canonical({"decision": "apply"}) + + with pytest.raises(ValueError, match="lowercase SHA-256"): + DecisionProofEnvelope( + optimizer_kind="gepa", + schema_version="gepa-publication-proof-v1", + canonical_json=canonical, + digest=bad_digest, + decision="apply", + ) + + +def test_publication_canonical_json_must_match_digest_and_bytes() -> None: + noncanonical = json.dumps({"b": 1, "a": 2}) + + with pytest.raises(ValueError, match="canonical JSON"): + DecisionProofEnvelope( + optimizer_kind="gepa", + schema_version="gepa-publication-proof-v1", + canonical_json=noncanonical, + digest=_digest(noncanonical), + decision="apply", + ) + + +def test_publication_envelopes_bind_their_declared_fields() -> None: + proof = _proof() + with pytest.raises(ValueError, match="optimizer_kind"): + DecisionProofEnvelope( + optimizer_kind="offline_tuner_replay", + schema_version=proof.schema_version, + canonical_json=proof.canonical_json, + digest=proof.digest, + decision="apply", + ) + + projection = _projection() + with pytest.raises(ValueError, match="projection fields"): + PublicationSearchProjection( + schema_version=projection.schema_version, + canonical_json=projection.canonical_json, + digest=projection.digest, + projector_id=projection.projector_id, + projector_version=projection.projector_version, + projector_code_digest=projection.projector_code_digest, + candidate_content_digest=projection.candidate_content_digest, + preserved_trigger=projection.preserved_trigger, + embedding_model_id=projection.embedding_model_id, + embedding=projection.embedding, + expanded_terms=("changed",), + lexical_document=projection.lexical_document, + ) + + +def test_publication_request_binds_content_optimizer_and_canonical_epochs() -> None: + claim = PublicationClaim(job_id=7, owner="worker-a", fence=3) + common = { + "optimizer_kind": "gepa", + "job_id": 7, + "attempt_key": "attempt-7", + "publication_claim": claim, + "worker_fence": 11, + "incumbent_user_playbook_id": 101, + "incumbent_content_digest": _digest("old content"), + "incumbent_trigger": "refund", + "incumbent_semantic_digest": incumbent_user_playbook_semantic_digest( + content_digest=_digest("old content"), trigger="refund" + ), + "projection": _projection(), + "decision_proof": _proof(), + "request_id": "request-7", + } + + with pytest.raises(ValueError, match="content digest"): + PublicationRequest( + **common, + revised_content="different content", + subject_epochs_json=_canonical({"subjects": []}), + ) + with pytest.raises(ValueError, match="canonical JSON"): + PublicationRequest( + **common, + revised_content="new content", + subject_epochs_json=json.dumps({"subjects": []}), + ) + with pytest.raises(ValueError, match="optimizer_kind"): + PublicationRequest( + **{**common, "optimizer_kind": "offline_tuner_replay"}, + revised_content="new content", + subject_epochs_json=_canonical({"subjects": []}), + ) + + canonical = _canonical({"a": 2, "b": 1}) + with pytest.raises(ValueError, match="digest"): + DecisionProofEnvelope( + optimizer_kind="gepa", + schema_version="gepa-publication-proof-v1", + canonical_json=canonical, + digest="0" * 64, + decision="apply", + ) + + +def test_publication_request_rejects_wrong_claim_kind_and_non_apply_decision() -> None: + canonical = _canonical( + { + "decision": "abstain", + "optimizer_kind": "gepa", + "schema_version": "gepa-publication-proof-v1", + } + ) + with pytest.raises(ValueError, match="decision"): + DecisionProofEnvelope( + optimizer_kind="gepa", + schema_version="gepa-publication-proof-v1", + canonical_json=canonical, + digest=_digest(canonical), + decision="abstain", # type: ignore[arg-type] + ) + + with pytest.raises(ValueError, match="job_id"): + PublicationRequest( + optimizer_kind="gepa", + job_id=7, + attempt_key="attempt-7", + publication_claim=PublicationClaim(job_id=8, owner="worker-a", fence=1), + worker_fence=11, + incumbent_user_playbook_id=101, + incumbent_content_digest=_digest("old content"), + incumbent_trigger="refund", + incumbent_semantic_digest=incumbent_user_playbook_semantic_digest( + content_digest=_digest("old content"), trigger="refund" + ), + revised_content="new content", + projection=_projection(), + decision_proof=_proof(), + subject_epochs_json=_canonical({"subjects": []}), + request_id="request-7", + ) + + +@pytest.mark.parametrize( + "subjects", + [ + [], + ["not-an-object"], + [{"epoch": -1, "ref": "subject:a"}], + [{"epoch": 0, "ref": ""}], + [{"epoch": 0, "ref": "subject:a"}, {"epoch": 1, "ref": "subject:a"}], + [{"epoch": 0, "subject_ref": "subject:a"}], + [{"epoch": 0, "ref": "subject:a", "unexpected": True}], + ], +) +def test_publication_request_rejects_invalid_subject_epoch_vectors( + subjects: list[object], +) -> None: + with pytest.raises(ValueError, match="subject epochs"): + PublicationRequest( + optimizer_kind="gepa", + job_id=7, + attempt_key="attempt-7", + publication_claim=PublicationClaim(job_id=7, owner="worker-a", fence=3), + worker_fence=11, + incumbent_user_playbook_id=101, + incumbent_content_digest=_digest("old content"), + incumbent_trigger="refund", + incumbent_semantic_digest=incumbent_user_playbook_semantic_digest( + content_digest=_digest("old content"), trigger="refund" + ), + revised_content="new content", + projection=_projection(), + decision_proof=_proof(), + subject_epochs_json=_canonical({"subjects": subjects}), + request_id="request-7", + ) + + +def test_publication_projection_accepts_exact_task6_bytes_and_digest() -> None: + canonical = ( + '{"candidate_content_digest":"fe32608c9ef5b6cf7e3f946480253ff76f24f4ec0678f3d0f07f9844cbff9601",' + '"embedding":["0.25","-1","0"],"embedding_model_id":"test-embedding-v1",' + '"expanded_terms":["refund","escalation"],' + '"lexical_document":"refund escalation exact projection",' + '"preserved_trigger":"refund",' + '"projector_code_digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",' + '"projector_id":"reflexio.search.user-playbook","projector_version":"1",' + '"schema_version":"offline-tuner-candidate-search-projection-v1"}' + ) + + projection = PublicationSearchProjection( + schema_version="offline-tuner-candidate-search-projection-v1", + canonical_json=canonical, + digest="f092c63fec1376c1e20089086427092613c4f63f16dab317f44eb8f71622b338", + projector_id="reflexio.search.user-playbook", + projector_version="1", + projector_code_digest="a" * 64, + candidate_content_digest=_digest("new content"), + preserved_trigger="refund", + embedding_model_id="test-embedding-v1", + embedding=("0.25", "-1", "0"), + expanded_terms=("refund", "escalation"), + lexical_document="refund escalation exact projection", + ) + + assert projection.canonical_json.encode() == canonical.encode() + assert ( + projection.digest + == "f092c63fec1376c1e20089086427092613c4f63f16dab317f44eb8f71622b338" + ) + + +@pytest.mark.parametrize("value", [-0.0, 0.0, 1.5]) +def test_rfc8785_encoder_rejects_all_floats(value: float) -> None: + with pytest.raises(TypeError, match="Unsupported RFC 8785 value: float"): + canonical_json_bytes({"value": value}) + + +@pytest.mark.parametrize("value", [-(2**53), 2**53]) +def test_rfc8785_encoder_rejects_inexact_integer_bounds(value: int) -> None: + with pytest.raises(ValueError, match="exactly representable"): + canonical_json_bytes({"value": value}) + + +@pytest.mark.parametrize("value", [-(2**53) + 1, 2**53 - 1]) +def test_rfc8785_encoder_accepts_exact_integer_bounds(value: int) -> None: + assert canonical_json_bytes({"value": value}) + + +def test_rfc8785_encoder_rejects_surrogates_and_uses_utf16_key_order() -> None: + with pytest.raises(ValueError, match="surrogate"): + canonical_json_bytes({"value": "\ud800"}) + + assert canonical_json_bytes({"\ue000": "bmp", "\U00010000": "astral"}) == ( + '{"\U00010000":"astral","\ue000":"bmp"}'.encode() + ) + + +def test_publication_service_requires_explicit_verifier() -> None: + with pytest.raises(TypeError): + UserPlaybookPublicationService(object()) # type: ignore[arg-type] + with pytest.raises(TypeError, match="PublicationDecisionVerifier"): + UserPlaybookPublicationService( + object(), # type: ignore[arg-type] + verifier=None, # type: ignore[arg-type] + ) + with pytest.raises(TypeError, match="PublicationDecisionVerifier"): + UserPlaybookPublicationService( + object(), # type: ignore[arg-type] + verifier=object(), # type: ignore[arg-type] + ) diff --git a/tests/server/services/playbook_optimizer/test_gepa_user_playbook_publication.py b/tests/server/services/playbook_optimizer/test_gepa_user_playbook_publication.py new file mode 100644 index 000000000..fadb29163 --- /dev/null +++ b/tests/server/services/playbook_optimizer/test_gepa_user_playbook_publication.py @@ -0,0 +1,1866 @@ +from __future__ import annotations + +import inspect +import json +from hashlib import sha256 +from pathlib import Path +from types import SimpleNamespace +from typing import Any, cast +from unittest.mock import Mock, patch + +import pytest + +from reflexio.models.api_schema.domain import ( + Interaction, + PlaybookOptimizationCandidate, + PlaybookOptimizationEvaluation, + UserPlaybook, +) +from reflexio.models.api_schema.domain.enums import Status +from reflexio.models.config_schema import ( + APIKeyConfig, + AzureOpenAIConfig, + Config, + OpenAIConfig, + PlaybookOptimizerConfig, + StorageConfigSQLite, +) +from reflexio.server.llm.litellm_client import LiteLLMClient, LiteLLMConfig +from reflexio.server.prompt.prompt_manager import PromptManager +from reflexio.server.services.playbook_optimizer.gepa_adapter import ( + PLAYBOOK_CONTENT_COMPONENT, +) +from reflexio.server.services.playbook_optimizer.gepa_publication import ( + GEPA_PROJECTOR_CODE_DIGEST, + GEPA_PROJECTOR_ID, + GEPA_PROJECTOR_VERSION, + GEPA_PUBLICATION_AUTHORITY_METADATA_KEY, + _gepa_adoption_result_from_snapshot, + build_gepa_search_projection, +) +from reflexio.server.services.playbook_optimizer.judge import ( + PLAYBOOK_OPTIMIZER_JUDGE_PROMPT_ID, +) +from reflexio.server.services.playbook_optimizer.models import ScenarioWindow +from reflexio.server.services.playbook_optimizer.optimizer import ( + PlaybookOptimizationTarget, + PlaybookOptimizer, +) +from reflexio.server.services.storage.error import StorageError +from reflexio.server.services.storage.sqlite_storage import SQLiteStorage + + +def _storage(tmp_path) -> SQLiteStorage: + with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): + storage = SQLiteStorage( + org_id="gepa-publication-test", + db_path=str(tmp_path / "reflexio.db"), + ) + embedding = [0.25, -0.5, *([0.0] * 510)] + storage._get_embedding = Mock(return_value=embedding) # noqa: SLF001 + storage.llm_client.get_embeddings = Mock(return_value=[embedding]) + return storage + + +def _optimizer(storage: SQLiteStorage, tmp_path) -> PlaybookOptimizer: + config = _optimizer_config(tmp_path) + return _optimizer_with_config(storage, config) + + +def _optimizer_config(tmp_path) -> Config: + return Config( + storage_config=StorageConfigSQLite(db_path=str(tmp_path / "reflexio.db")), + playbook_optimizer_config=PlaybookOptimizerConfig( + enabled=True, + optimize_user_playbooks=True, + auto_update_user_playbooks=True, + webhook_url="https://assistant.example.test/rollout", + min_commit_windows=1, + min_commit_score=0.5, + min_commit_likert=4, + ), + ) + + +def _optimizer_with_config( + storage: SQLiteStorage, + config: Config, + *, + llm_client: LiteLLMClient | None = None, + prompt_manager: PromptManager | None = None, +) -> PlaybookOptimizer: + context = SimpleNamespace( + org_id=storage.org_id, + storage=storage, + configurator=SimpleNamespace(get_config=lambda: config), + prompt_manager=prompt_manager or PromptManager(), + ) + llm_client = llm_client or LiteLLMClient(LiteLLMConfig(model="fake-model")) + return PlaybookOptimizer(cast(Any, context), cast(Any, llm_client)) + + +def _window(incumbent_id: int) -> ScenarioWindow: + return ScenarioWindow( + user_playbook_id=incumbent_id, + source_interaction_ids=[101], + interactions=[ + Interaction( + interaction_id=101, + user_id="u1", + request_id="request-1", + role="User", + content="Help with a refund", + ) + ], + ) + + +def _install_winning_gepa( + optimizer: PlaybookOptimizer, + storage: SQLiteStorage, + window: ScenarioWindow, +) -> None: + optimizer._resolve_windows = Mock(return_value=[window]) # type: ignore[method-assign] + + def fake_run_gepa(config, seed, train, validation, adapter): # noqa: ARG001 + assert window.user_playbook_id is not None + candidate = adapter._ensure_candidate("new guidance") # noqa: SLF001 + storage.insert_playbook_optimization_evaluation( + PlaybookOptimizationEvaluation( + job_id=adapter.job_id, + candidate_id=candidate.candidate_id, + target_kind="user_playbook", + target_id=window.user_playbook_id, + scenario_user_playbook_id=window.user_playbook_id, + source_interaction_ids=window.source_interaction_ids, + score=0.9, + verdict="candidate", + likert=5, + rationale="candidate handled the refund policy more clearly", + asi_json='{"score":0.9,"rubric":"refund"}', + incumbent_rollout_json='[{"role":"Assistant","content":"old"}]', + candidate_rollout_json='[{"role":"Assistant","content":"new"}]', + ) + ) + return SimpleNamespace( + best_candidate={PLAYBOOK_CONTENT_COMPONENT: "new guidance"}, + val_aggregate_scores=[0.9], + best_idx=0, + to_dict=lambda: {"best_idx": 0}, + ) + + optimizer._run_gepa = fake_run_gepa # type: ignore[method-assign] + + +def _candidate( + *, + aggregate_score: float = 0.9, +) -> PlaybookOptimizationCandidate: + return PlaybookOptimizationCandidate( + candidate_id=11, + job_id=7, + content="new guidance", + aggregate_score=aggregate_score, + is_winner=True, + ) + + +def _evaluation( + *, + evaluation_id: int, + candidate_id: int = 11, + scenario_user_playbook_id: int = 101, + source_interaction_ids: list[int] | None = None, + score: float = 0.9, + likert: int = 5, +) -> PlaybookOptimizationEvaluation: + return PlaybookOptimizationEvaluation( + evaluation_id=evaluation_id, + job_id=7, + candidate_id=candidate_id, + target_kind="user_playbook", + target_id=101, + scenario_user_playbook_id=scenario_user_playbook_id, + source_interaction_ids=source_interaction_ids or [1001], + score=score, + verdict="candidate", + likert=likert, + ) + + +def _authority( + *, + min_commit_windows: int, + min_commit_score: str = "0.5", + min_commit_likert: int = 4, + windows: list[dict[str, object]] | None = None, +) -> dict[str, object]: + return { + "adoption_policy": { + "auto_update_user_playbooks": True, + "min_commit_likert": min_commit_likert, + "min_commit_score": min_commit_score, + "min_commit_windows": min_commit_windows, + }, + "validation_manifest": { + "digest": "a" * 64, + "windows": windows + or [ + { + "scenario_user_playbook_id": 101, + "source_interaction_ids": [1001], + "min_commit_likert": min_commit_likert, + "min_commit_score": min_commit_score, + } + ], + }, + } + + +def _incumbent(storage: SQLiteStorage) -> UserPlaybook: + incumbent = UserPlaybook( + user_id="u1", + request_id="request-1", + agent_version="v1", + playbook_name="refunds", + content="old guidance", + trigger="refund request", + ) + storage.save_user_playbooks([incumbent]) + return incumbent + + +def _completed_user_authority( + tmp_path, + *, + config: Config | None = None, + llm_client: LiteLLMClient | None = None, + prompt_manager: PromptManager | None = None, +) -> dict[str, Any]: + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + optimizer = _optimizer_with_config( + storage, + config or _optimizer_config(tmp_path), + llm_client=llm_client, + prompt_manager=prompt_manager, + ) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + assert ( + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + == "completed" + ) + row = storage.conn.execute( + "SELECT metadata_json FROM playbook_optimization_jobs" + ).fetchone() + return json.loads(row["metadata_json"])[GEPA_PUBLICATION_AUTHORITY_METADATA_KEY] + + +def _canonical_authority(authority: dict[str, Any]) -> str: + return json.dumps(authority, sort_keys=True, separators=(",", ":")) + + +def _azure_client( + *, + endpoint: str = "https://azure-one.example.test/", + api_version: str = "2024-02-15-preview", + api_key: str = "azure-secret-one", + model: str = "azure/judge-deployment", + temperature: float = 0.2, + top_p: float = 0.7, + max_tokens: int | None = 300, + fallback_models: list[str] | None = None, +) -> LiteLLMClient: + return LiteLLMClient( + LiteLLMConfig( + model=model, + temperature=temperature, + top_p=top_p, + max_tokens=max_tokens, + fallback_models=fallback_models or ["azure/fallback-a"], + api_key_config=APIKeyConfig( + openai=OpenAIConfig( + azure_config=AzureOpenAIConfig( + api_key=api_key, + endpoint=cast(Any, endpoint), + api_version=api_version, + ) + ) + ), + ) + ) + + +def _base_judge_config(tmp_path) -> Config: + config = _optimizer_config(tmp_path) + config.playbook_optimizer_config.reflection_model = "azure/judge-deployment" + return config + + +def _judge_authority(tmp_path, **kwargs: Any) -> dict[str, Any]: + config = kwargs.pop("config", _base_judge_config(tmp_path)) + client = kwargs.pop("llm_client", _azure_client()) + return _completed_user_authority( + tmp_path, config=config, llm_client=client, **kwargs + ) + + +def test_gepa_adoption_counts_duplicate_validation_window_once(): + adoption = _gepa_adoption_result_from_snapshot( + winner=_candidate(), + evaluations=[ + _evaluation(evaluation_id=1), + _evaluation(evaluation_id=2), + ], + authority=_authority(min_commit_windows=2), + ) + + assert adoption["passes"] is False + + +def test_gepa_adoption_counts_distinct_validation_windows(): + adoption = _gepa_adoption_result_from_snapshot( + winner=_candidate(), + evaluations=[ + _evaluation(evaluation_id=1, scenario_user_playbook_id=101), + _evaluation( + evaluation_id=2, + scenario_user_playbook_id=102, + source_interaction_ids=[1002], + ), + ], + authority=_authority( + min_commit_windows=2, + windows=[ + { + "scenario_user_playbook_id": 101, + "source_interaction_ids": [1001], + "min_commit_likert": 4, + "min_commit_score": "0.5", + }, + { + "scenario_user_playbook_id": 102, + "source_interaction_ids": [1002], + "min_commit_likert": 4, + "min_commit_score": "0.5", + }, + ], + ), + ) + + assert adoption["passes"] is True + + +def test_gepa_adoption_uses_per_window_frozen_thresholds(): + adoption = _gepa_adoption_result_from_snapshot( + winner=_candidate(), + evaluations=[ + _evaluation(evaluation_id=1, scenario_user_playbook_id=101, score=0.6), + _evaluation( + evaluation_id=2, + scenario_user_playbook_id=102, + source_interaction_ids=[1002], + score=0.6, + ), + ], + authority=_authority( + min_commit_windows=2, + windows=[ + { + "scenario_user_playbook_id": 101, + "source_interaction_ids": [1001], + "min_commit_likert": 4, + "min_commit_score": "0.5", + }, + { + "scenario_user_playbook_id": 102, + "source_interaction_ids": [1002], + "min_commit_likert": 4, + "min_commit_score": "0.9", + }, + ], + ), + ) + + assert adoption["passes"] is False + + +def test_gepa_authority_uses_round_trippable_float_thresholds(tmp_path): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + config = _optimizer_config(tmp_path) + threshold = 0.1 + 0.2 + config.playbook_optimizer_config.min_commit_score = threshold + optimizer = _optimizer_with_config(storage, config) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + + status = optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + assert status == "completed" + job = storage.conn.execute( + "SELECT metadata_json FROM playbook_optimization_jobs" + ).fetchone() + authority = json.loads(job["metadata_json"])[ + GEPA_PUBLICATION_AUTHORITY_METADATA_KEY + ] + assert authority["adoption_policy"]["min_commit_score"] == repr(threshold) + assert authority["validation_manifest"]["windows"][0]["min_commit_score"] == repr( + threshold + ) + + +def test_gepa_user_job_creation_freezes_complete_sanitized_authority(tmp_path): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + config = _optimizer_config(tmp_path) + config.playbook_optimizer_config.max_metric_calls = 37 + config.playbook_optimizer_config.max_turns = 6 + config.playbook_optimizer_config.early_stop_score = 0.1 + 0.2 + config.playbook_optimizer_config.reflection_minibatch_size = 3 + config.playbook_optimizer_config.max_validation_windows = 1 + config.playbook_optimizer_config.use_merge = False + config.playbook_optimizer_config.max_merge_invocations = 0 + config.playbook_optimizer_config.reflection_model = "judge-model" + config.playbook_optimizer_config.webhook_auth_header = "Bearer secret" + optimizer = _optimizer_with_config(storage, config) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + + assert ( + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + == "completed" + ) + + row = storage.conn.execute( + "SELECT metadata_json FROM playbook_optimization_jobs" + ).fetchone() + authority = json.loads(row["metadata_json"])[ + GEPA_PUBLICATION_AUTHORITY_METADATA_KEY + ] + assert authority["budget_settings"] == { + "max_metric_calls": 37, + "max_turns": 6, + "reflection_minibatch_size": 3, + } + assert authority["split_settings"] == {"max_validation_windows": 1} + assert authority["merge_settings"] == { + "max_merge_invocations": 0, + "use_merge": False, + } + assert authority["stop_settings"] == { + "early_stop_score": repr(0.1 + 0.2), + "stopper_class": "gepa.utils.stop_condition.ScoreThresholdStopper", + } + assert authority["gepa_algorithm"] == { + "batch_sampler": "epoch_shuffled", + "cache_evaluation": True, + "candidate_selection_strategy": "pareto", + "display_progress_bar": False, + "frontier_type": "instance", + "raise_on_exception": False, + } + assert authority["model_identity"] == { + "default_lm": "fake-model", + "reflection_lm": "judge-model", + } + assert authority["gepa_engine_identity"]["package_name"] == "gepa" + assert isinstance(authority["gepa_engine_identity"]["package_version"], str) + assert len(authority["gepa_engine_identity"]["optimize_code_digest"]) == 64 + assert authority["backend_identity"]["backend_kind"] == "webhook" + assert authority["backend_identity"]["webhook_auth_configured"] is True + assert authority["backend_identity"]["webhook_auth_scheme"] == "Bearer" + assert ( + authority["backend_identity"]["webhook_url_digest"] + == sha256(b"https://assistant.example.test/rollout").hexdigest() + ) + serialized = json.dumps(authority, sort_keys=True) + assert "https://assistant.example.test/rollout" not in serialized + assert "Bearer secret" not in serialized + + +# --------------------------------------------------------------------------- +# Evaluator authority matrix +# --------------------------------------------------------------------------- + + +def test_gepa_user_authority_freezes_prompt_and_schema_identity(tmp_path): + authority = _completed_user_authority(tmp_path) + evaluator = authority["evaluator_identity"]["pairwise_judge_request_plan"] + prompt = evaluator["judge_prompt_identity"] + + assert prompt["prompt_id"] == PLAYBOOK_OPTIMIZER_JUDGE_PROMPT_ID + assert prompt["active_version"] == "1.2.0" + assert len(prompt["template_content_digest"]) == 64 + assert evaluator["llm_client_class"].endswith(".LiteLLMClient") + assert len(evaluator["llm_client_code_digest"]) == 64 + assert evaluator["judge_output_schema_class"].endswith(".JudgeOutput") + assert len(evaluator["judge_output_schema_code_digest"]) == 64 + assert len(evaluator["judge_output_schema_digest"]) == 64 + assert "Help with a refund" not in _canonical_authority(authority) + + +def test_gepa_user_authority_freezes_generation_ladder_and_retry_contract(tmp_path): + config = _optimizer_config(tmp_path) + config.playbook_optimizer_config.reflection_model = "gpt-5-mini" + authority = _completed_user_authority( + tmp_path, + config=config, + llm_client=LiteLLMClient( + LiteLLMConfig( + model="base-model", + temperature=0.23, + top_p=0.74, + max_tokens=None, + fallback_models=[ + "minimax/MiniMax-M3", + "local/embedding-only", + "fallback-b", + "minimax/MiniMax-M3", + ], + ) + ), + ) + evaluator = authority["evaluator_identity"]["pairwise_judge_request_plan"] + generation = evaluator["judge_generation_settings"] + + assert generation["resolved_primary_model"] == "gpt-5-mini" + assert generation["fallback_model_order"] == [ + "minimax/MiniMax-M3", + "fallback-b", + ] + assert generation["resolved_model_ladder"] == [ + "gpt-5-mini", + "minimax/MiniMax-M3", + "fallback-b", + ] + assert generation["pairwise_judge_max_retries"] == 1 + assert ( + len( + evaluator["implementation_callables"]["llm_client._make_request"][ + "code_digest" + ] + ) + == 64 + ) + assert [rung["model"] for rung in generation["rungs"]] == generation[ + "resolved_model_ladder" + ] + assert all(rung["timeout_seconds"] == "120.0" for rung in generation["rungs"]) + assert all(rung["hard_timeout_seconds"] == "125.0" for rung in generation["rungs"]) + assert generation["rungs"][0]["temperature"] == "1.0" + assert generation["rungs"][1]["max_tokens"] == 8192 + assert generation["rungs"][0]["structured_output_strategy"] + assert generation["rungs"][0]["parse_structured_output"] is True + + +@pytest.mark.parametrize( + ("name", "config_updates", "client"), + [ + ( + "model", + {"reflection_model": "judge-model-v2"}, + LiteLLMConfig( + model="judge-model", + temperature=0.2, + top_p=0.7, + max_tokens=300, + fallback_models=["fallback-a", "fallback-b"], + ), + ), + ( + "temperature", + {}, + LiteLLMConfig( + model="judge-model", + temperature=0.4, + top_p=0.7, + max_tokens=300, + fallback_models=["fallback-a", "fallback-b"], + ), + ), + ( + "top_p", + {}, + LiteLLMConfig( + model="judge-model", + temperature=0.2, + top_p=0.9, + max_tokens=300, + fallback_models=["fallback-a", "fallback-b"], + ), + ), + ( + "max_tokens", + {}, + LiteLLMConfig( + model="judge-model", + temperature=0.2, + top_p=0.7, + max_tokens=500, + fallback_models=["fallback-a", "fallback-b"], + ), + ), + ( + "fallback_order", + {}, + LiteLLMConfig( + model="judge-model", + temperature=0.2, + top_p=0.7, + max_tokens=300, + fallback_models=["fallback-b", "fallback-a"], + ), + ), + ], +) +def test_gepa_user_authority_changes_for_generation_settings( + tmp_path, name, config_updates, client +): + base_config = _optimizer_config(tmp_path / "base") + base_config.playbook_optimizer_config.reflection_model = "judge-model" + base = _canonical_authority( + _completed_user_authority( + tmp_path / "base", + config=base_config, + llm_client=LiteLLMClient( + LiteLLMConfig( + model="judge-model", + temperature=0.2, + top_p=0.7, + max_tokens=300, + fallback_models=["fallback-a", "fallback-b"], + ) + ), + ) + ) + variant_config = _optimizer_config(tmp_path / name) + variant_config.playbook_optimizer_config.reflection_model = "judge-model" + for key, value in config_updates.items(): + setattr(variant_config.playbook_optimizer_config, key, value) + changed = _canonical_authority( + _completed_user_authority( + tmp_path / name, + config=variant_config, + llm_client=LiteLLMClient(client), + ) + ) + assert changed != base + + +@pytest.mark.parametrize( + ("name", "client"), + [ + ("provider", LiteLLMClient(LiteLLMConfig(model="gpt-4o-mini"))), + ("endpoint", _azure_client(endpoint="https://azure-two.example.test/")), + ("api_version", _azure_client(api_version="2025-01-01-preview")), + ], +) +def test_gepa_user_authority_changes_for_independent_provider_identity( + tmp_path, name, client +): + base = _canonical_authority( + _judge_authority(tmp_path / "base", llm_client=_azure_client()) + ) + changed = _canonical_authority(_judge_authority(tmp_path / name, llm_client=client)) + assert changed != base + + +def test_gepa_user_authority_freezes_non_secret_provider_identity(tmp_path): + endpoint = "https://azure-one.example.test/" + authority = _judge_authority(tmp_path, llm_client=_azure_client(endpoint=endpoint)) + plan = authority["evaluator_identity"]["pairwise_judge_request_plan"] + rung = plan["judge_generation_settings"]["rungs"][0] + assert rung["provider_kind"] == "azure" + assert rung["api_base_digest"] == sha256(endpoint.encode()).hexdigest() + assert rung["api_version"] == "2024-02-15-preview" + assert endpoint not in _canonical_authority(authority) + + +def test_gepa_user_authority_changes_with_effective_request_seed(tmp_path, monkeypatch): + monkeypatch.setenv("REFLEXIO_LLM_SEED", "42") + first = _canonical_authority(_judge_authority(tmp_path / "first")) + monkeypatch.setenv("REFLEXIO_LLM_SEED", "43") + second = _canonical_authority(_judge_authority(tmp_path / "second")) + + assert first != second + + +def test_gepa_user_authority_excludes_credentials_but_binds_webhook_scheme( + tmp_path, +): + config_a = _base_judge_config(tmp_path / "a") + config_a.playbook_optimizer_config.webhook_auth_header = "Bearer alpha-secret" + config_b = _base_judge_config(tmp_path / "b") + config_b.playbook_optimizer_config.webhook_auth_header = "Bearer beta-secret" + a = _judge_authority( + tmp_path / "a", + config=config_a, + llm_client=_azure_client(api_key="azure-secret-one"), + ) + b = _judge_authority( + tmp_path / "b", + config=config_b, + llm_client=_azure_client(api_key="azure-secret-two"), + ) + + assert _canonical_authority(a) == _canonical_authority(b) + assert a["backend_identity"]["webhook_auth_configured"] is True + assert a["backend_identity"]["webhook_auth_scheme"] == "Bearer" + serialized = _canonical_authority(a) + for forbidden in ( + "alpha-secret", + "beta-secret", + "azure-secret-one", + "azure-secret-two", + "https://assistant.example.test/rollout", + "credential", + "fingerprint", + "verifier", + ): + assert forbidden not in serialized + + +@pytest.mark.parametrize("scheme", ["Signature", None]) +def test_gepa_user_authority_binds_webhook_auth_scheme(tmp_path, scheme): + config = _base_judge_config(tmp_path) + config.playbook_optimizer_config.webhook_auth_header = ( + f"{scheme} secret" if scheme else None + ) + authority = _judge_authority(tmp_path, config=config) + backend = authority["backend_identity"] + assert backend["webhook_auth_configured"] is bool(scheme) + assert backend["webhook_auth_scheme"] == scheme + + +def test_gepa_user_winner_publishes_exact_projection_then_triggers_aggregation( + tmp_path, +): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + optimizer = _optimizer(storage, tmp_path) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + aggregation_saw_visible_successor: list[bool] = [] + + def record_aggregation(**kwargs): # noqa: ARG001 + current = storage.get_user_playbooks(status_filter=[None]) + aggregation_saw_visible_successor.append( + any(playbook.content == "new guidance" for playbook in current) + ) + + with patch( + "reflexio.server.services.playbook_optimizer.optimizer." + "maybe_trigger_user_playbook_aggregation", + side_effect=record_aggregation, + ) as trigger: + status = optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + assert status == "completed" + trigger.assert_called_once() + assert aggregation_saw_visible_successor == [True] + job = storage.conn.execute("SELECT * FROM playbook_optimization_jobs").fetchone() + assert job["stage"] == "applied" + assert job["terminal_outcome"] == "applied" + metadata = json.loads(job["metadata_json"]) + assert metadata["publication_subject_epochs"]["subjects"] + assert metadata["publication_proof_digest"] + + staged = storage.conn.execute( + "SELECT * FROM user_playbook_publication_staging WHERE job_id = ?", + (job["job_id"],), + ).fetchone() + projection = json.loads(staged["projection_json"]) + assert projection == { + "candidate_content_digest": staged["content_digest"], + "embedding": ["0.25", "-0.5", *(["0"] * 510)], + "embedding_model_id": storage.embedding_model_name, + "expanded_terms": [], + "lexical_document": "refund request new guidance", + "preserved_trigger": "refund request", + "projector_code_digest": ( + "c383e3dd0df8618d3b082ccbaceed7bd676f47af0b25d442bf8ff2c628fafb42" + ), + "projector_id": "reflexio-gepa-user-playbook-search-projector", + "projector_version": "2", + "schema_version": "offline-tuner-candidate-search-projection-v1", + } + successor_id = storage.conn.execute( + "SELECT successor_user_playbook_id FROM user_playbook_publication_results" + ).fetchone()[0] + successor = storage.get_user_playbook_by_id(successor_id) + assert successor is not None + assert successor.content == "new guidance" + assert successor.trigger == "refund request" + stored_embedding = storage.conn.execute( + "SELECT embedding FROM user_playbooks WHERE user_playbook_id = ?", + (successor_id,), + ).fetchone()[0] + assert json.loads(stored_embedding) == [0.25, -0.5, *([0.0] * 510)] + fts = storage.conn.execute( + "SELECT search_text FROM user_playbooks_fts WHERE rowid = ?", + (successor_id,), + ).fetchone() + assert fts["search_text"] == "refund request new guidance" + + +def test_gepa_user_incumbent_cas_loss_publishes_no_successor_or_aggregation(tmp_path): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + optimizer = _optimizer(storage, tmp_path) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + original_commit = storage.commit_user_playbook_publication + + def lose_incumbent(request): + storage.conn.execute( + "UPDATE user_playbooks SET status = ? WHERE user_playbook_id = ?", + (Status.ARCHIVED.value, incumbent.user_playbook_id), + ) + storage.conn.commit() + return original_commit(request) + + with ( + patch.object( + storage, + "commit_user_playbook_publication", + side_effect=lose_incumbent, + ), + patch( + "reflexio.server.services.playbook_optimizer.optimizer." + "maybe_trigger_user_playbook_aggregation" + ) as trigger, + ): + status = optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + assert status == "completed" + trigger.assert_not_called() + terminal = storage.conn.execute( + "SELECT outcome, successor_user_playbook_id FROM user_playbook_publication_results" + ).fetchone() + assert tuple(terminal) == ("incumbent_changed", None) + assert ( + storage.conn.execute("SELECT COUNT(*) FROM user_playbooks").fetchone()[0] == 1 + ) + + +def test_gepa_verifier_rechecks_adoption_and_writes_no_staging_on_rejection(tmp_path): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + optimizer = _optimizer(storage, tmp_path) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + original_prepare = storage.prepare_gepa_user_playbook_publication + + def prepare_then_reject(**kwargs): + job = original_prepare(**kwargs) + storage.conn.execute( + "UPDATE playbook_optimization_evaluations SET verdict = 'incumbent'" + ) + storage.conn.commit() + return job + + with ( + patch.object( + storage, + "prepare_gepa_user_playbook_publication", + side_effect=prepare_then_reject, + ), + pytest.raises(ValueError, match="fails adoption rules"), + ): + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + assert ( + storage.conn.execute( + "SELECT COUNT(*) FROM user_playbook_publication_staging" + ).fetchone()[0] + == 0 + ) + assert ( + storage.conn.execute( + "SELECT COUNT(*) FROM user_playbook_publication_results" + ).fetchone()[0] + == 0 + ) + assert ( + storage.conn.execute("SELECT COUNT(*) FROM user_playbooks").fetchone()[0] == 1 + ) + + +def test_gepa_verifier_rejects_durable_winner_tampering_before_staging(tmp_path): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + optimizer = _optimizer(storage, tmp_path) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + original_prepare = storage.prepare_gepa_user_playbook_publication + + def prepare_then_tamper(**kwargs): + job = original_prepare(**kwargs) + storage.conn.execute( + "UPDATE playbook_optimization_candidates SET content = 'tampered' WHERE is_winner = 1" + ) + storage.conn.commit() + return job + + with ( + patch.object( + storage, + "prepare_gepa_user_playbook_publication", + side_effect=prepare_then_tamper, + ), + pytest.raises(ValueError, match="publication binding changed"), + ): + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + assert ( + storage.conn.execute( + "SELECT COUNT(*) FROM user_playbook_publication_staging" + ).fetchone()[0] + == 0 + ) + assert ( + storage.conn.execute("SELECT COUNT(*) FROM user_playbooks").fetchone()[0] == 1 + ) + + +def test_gepa_stale_worker_fence_rejects_before_staging(tmp_path): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + optimizer = _optimizer(storage, tmp_path) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + original_claim = storage.claim_user_playbook_publication + + def claim_then_stale(**kwargs): + claim = original_claim(**kwargs) + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET lease_fence = lease_fence + 1" + ) + storage.conn.commit() + return claim + + with ( + patch.object( + storage, + "claim_user_playbook_publication", + side_effect=claim_then_stale, + ), + pytest.raises(Exception, match="worker fence"), + ): + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + assert ( + storage.conn.execute( + "SELECT COUNT(*) FROM user_playbook_publication_staging" + ).fetchone()[0] + == 0 + ) + assert ( + storage.conn.execute("SELECT COUNT(*) FROM user_playbooks").fetchone()[0] == 1 + ) + + +def test_gepa_committed_response_loss_recovers_once_and_aggregates_once(tmp_path): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + optimizer = _optimizer(storage, tmp_path) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + original_commit = storage.commit_user_playbook_publication + + def commit_then_lose_response(request): + original_commit(request) + raise RuntimeError("response lost after commit") + + with ( + patch.object( + storage, + "commit_user_playbook_publication", + side_effect=commit_then_lose_response, + ) as commit, + patch( + "reflexio.server.services.playbook_optimizer.optimizer." + "maybe_trigger_user_playbook_aggregation" + ) as trigger, + ): + status = optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + assert status == "completed" + commit.assert_called_once() + trigger.assert_called_once() + assert ( + storage.conn.execute( + "SELECT COUNT(*) FROM user_playbook_publication_results" + ).fetchone()[0] + == 1 + ) + assert ( + storage.conn.execute( + "SELECT COUNT(*) FROM user_playbooks WHERE status IS NULL" + ).fetchone()[0] + == 1 + ) + assert ( + storage.conn.execute( + "SELECT COUNT(*) FROM playbook_optimization_events WHERE event_type = 'publication_applied'" + ).fetchone()[0] + == 1 + ) + + +def test_gepa_post_commit_successor_reload_failure_preserves_publication(tmp_path): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + optimizer = _optimizer(storage, tmp_path) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + original_reload = storage.get_user_playbook_by_id + + def fail_only_after_commit(user_playbook_id): + committed = storage.conn.execute( + "SELECT COUNT(*) FROM user_playbook_publication_results" + ).fetchone()[0] + if committed: + raise RuntimeError("successor reload failed") + return original_reload(user_playbook_id) + + with ( + patch.object( + storage, + "get_user_playbook_by_id", + side_effect=fail_only_after_commit, + ), + patch( + "reflexio.server.services.playbook_optimizer.optimizer." + "maybe_trigger_user_playbook_aggregation" + ) as trigger, + ): + status = optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + assert status == "completed" + trigger.assert_not_called() + assert ( + storage.conn.execute( + "SELECT outcome FROM user_playbook_publication_results" + ).fetchone()[0] + == "applied" + ) + assert ( + storage.conn.execute( + "SELECT COUNT(*) FROM user_playbooks WHERE status IS NULL" + ).fetchone()[0] + == 1 + ) + + +def test_gepa_post_commit_aggregation_failure_preserves_publication(tmp_path): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + optimizer = _optimizer(storage, tmp_path) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + + with patch( + "reflexio.server.services.playbook_optimizer.optimizer." + "maybe_trigger_user_playbook_aggregation", + side_effect=RuntimeError("aggregation unavailable"), + ) as trigger: + status = optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + assert status == "completed" + trigger.assert_called_once() + assert ( + storage.conn.execute( + "SELECT outcome FROM user_playbook_publication_results" + ).fetchone()[0] + == "applied" + ) + assert ( + storage.conn.execute( + "SELECT COUNT(*) FROM user_playbooks WHERE status IS NULL" + ).fetchone()[0] + == 1 + ) + + +def test_gepa_publication_uses_frozen_adoption_policy_when_live_config_drifts( + tmp_path, +): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + config = _optimizer_config(tmp_path) + optimizer = _optimizer_with_config(storage, config) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + original_prepare = storage.prepare_gepa_user_playbook_publication + + def prepare_then_drift_live_config(**kwargs): + prepared = original_prepare(**kwargs) + config.playbook_optimizer_config.min_commit_score = 1.0 + config.playbook_optimizer_config.min_commit_likert = 5 + config.playbook_optimizer_config.min_commit_windows = 2 + config.playbook_optimizer_config.auto_update_user_playbooks = False + return prepared + + with patch.object( + storage, + "prepare_gepa_user_playbook_publication", + side_effect=prepare_then_drift_live_config, + ): + status = optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + assert status == "completed" + assert ( + storage.conn.execute( + "SELECT outcome FROM user_playbook_publication_results" + ).fetchone()[0] + == "applied" + ) + + +@pytest.mark.parametrize( + ("category", "tamper_sql", "params"), + [ + ( + "train manifest", + """UPDATE playbook_optimization_jobs + SET metadata_json = json_set( + metadata_json, + '$.gepa_publication_authority.train_manifest.windows[0].source_interaction_ids[0]', + 999 + )""", + (), + ), + ( + "evaluation rationale", + "UPDATE playbook_optimization_evaluations SET rationale = 'tampered'", + (), + ), + ( + "evaluation ASI", + "UPDATE playbook_optimization_evaluations SET asi_json = '{\"tampered\":true}'", + (), + ), + ( + "incumbent rollout", + "UPDATE playbook_optimization_evaluations SET incumbent_rollout_json = '[]'", + (), + ), + ( + "candidate rollout", + "UPDATE playbook_optimization_evaluations SET candidate_rollout_json = '[]'", + (), + ), + ( + "candidate identity", + 'UPDATE playbook_optimization_candidates SET metadata_json = \'{"candidate_identity":"tampered"}\'', + (), + ), + ( + "evaluator identity", + """UPDATE playbook_optimization_jobs + SET metadata_json = json_set( + metadata_json, + '$.gepa_publication_authority.evaluator_identity.pairwise_judge_request_plan.judge_model_id', + 'tampered-model' + )""", + (), + ), + ( + "backend identity", + """UPDATE playbook_optimization_jobs + SET metadata_json = json_set( + metadata_json, + '$.gepa_publication_authority.backend_identity.backend_kind', + 'tampered-backend' + )""", + (), + ), + ( + "per-window threshold", + """UPDATE playbook_optimization_jobs + SET metadata_json = json_set( + metadata_json, + '$.gepa_publication_authority.validation_manifest.windows[0].min_commit_score', + 0.1 + )""", + (), + ), + ], +) +def test_gepa_verifier_rejects_tampered_complete_authority_categories( + tmp_path, + category, + tamper_sql, + params, +): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + optimizer = _optimizer(storage, tmp_path) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + original_prepare = storage.prepare_gepa_user_playbook_publication + + def prepare_then_tamper(**kwargs): + prepared = original_prepare(**kwargs) + storage.conn.execute(tamper_sql, params) + storage.conn.commit() + return prepared + + with ( + patch.object( + storage, + "prepare_gepa_user_playbook_publication", + side_effect=prepare_then_tamper, + ), + pytest.raises(ValueError, match="GEPA durable decision proof changed"), + ): + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + assert ( + storage.conn.execute( + "SELECT COUNT(*) FROM user_playbook_publication_results" + ).fetchone()[0] + == 0 + ), category + + +def test_gepa_recovery_resumes_crash_after_prepare_before_staging(tmp_path): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + optimizer = _optimizer(storage, tmp_path) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + original_prepare = storage.prepare_gepa_user_playbook_publication + calls = {"run_gepa": 0, "prepare": 0} + original_run_gepa = optimizer._run_gepa + + def counted_run_gepa(*args, **kwargs): + calls["run_gepa"] += 1 + if calls["run_gepa"] > 1: + raise AssertionError("GEPA search reran instead of resuming") + return original_run_gepa(*args, **kwargs) + + def prepare_then_crash_once(**kwargs): + calls["prepare"] += 1 + prepared = original_prepare(**kwargs) + if calls["prepare"] == 1: + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET lease_expires_at = 0 WHERE job_id = ?", + (prepared.job_id,), + ) + storage.conn.commit() + raise RuntimeError("crash after durable prepare before staging") + return prepared + + optimizer._run_gepa = counted_run_gepa # type: ignore[method-assign] + with patch.object( + storage, + "prepare_gepa_user_playbook_publication", + side_effect=prepare_then_crash_once, + ): + with pytest.raises(RuntimeError, match="crash after durable prepare"): + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + status = optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + assert status == "completed" + assert calls["run_gepa"] == 1 + assert ( + storage.conn.execute( + "SELECT COUNT(*) FROM playbook_optimization_jobs" + ).fetchone()[0] + == 1 + ) + assert ( + storage.conn.execute( + "SELECT outcome FROM user_playbook_publication_results" + ).fetchone()[0] + == "applied" + ) + + +def test_gepa_recovery_resumes_crash_after_staging(tmp_path): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + optimizer = _optimizer(storage, tmp_path) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + original_stage = storage.stage_user_playbook_publication + calls = {"run_gepa": 0, "stage": 0} + original_run_gepa = optimizer._run_gepa + + def counted_run_gepa(*args, **kwargs): + calls["run_gepa"] += 1 + if calls["run_gepa"] > 1: + raise AssertionError("GEPA search reran instead of resuming") + return original_run_gepa(*args, **kwargs) + + def stage_then_crash_once(request): + calls["stage"] += 1 + original_stage(request) + if calls["stage"] == 1: + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET lease_expires_at = 0 WHERE job_id = ?", + (request.job_id,), + ) + storage.conn.commit() + raise RuntimeError("crash after staging") + + optimizer._run_gepa = counted_run_gepa # type: ignore[method-assign] + with patch.object( + storage, + "stage_user_playbook_publication", + side_effect=stage_then_crash_once, + ): + with pytest.raises(RuntimeError, match="crash after staging"): + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + status = optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + assert status == "completed" + assert calls["run_gepa"] == 1 + assert ( + storage.conn.execute( + "SELECT COUNT(*) FROM user_playbook_publication_staging" + ).fetchone()[0] + == 1 + ) + assert ( + storage.conn.execute( + "SELECT outcome FROM user_playbook_publication_results" + ).fetchone()[0] + == "applied" + ) + + +def test_gepa_expired_publishing_job_is_reclaimed_before_new_search(tmp_path): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + optimizer = _optimizer(storage, tmp_path) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + original_prepare = storage.prepare_gepa_user_playbook_publication + prepared_job_ids: list[int] = [] + + def prepare_then_crash(**kwargs): + prepared = original_prepare(**kwargs) + prepared_job_ids.append(prepared.job_id) + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET lease_expires_at = 0 WHERE job_id = ?", + (prepared.job_id,), + ) + storage.conn.commit() + raise RuntimeError("crash after durable prepare") + + with ( + patch.object( + storage, + "prepare_gepa_user_playbook_publication", + side_effect=prepare_then_crash, + ), + pytest.raises(RuntimeError), + ): + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + optimizer._run_gepa = Mock( # type: ignore[method-assign] + side_effect=AssertionError("expired publication must resume without GEPA") + ) + status = optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + assert status == "completed" + assert prepared_job_ids == [ + storage.conn.execute( + "SELECT job_id FROM playbook_optimization_jobs" + ).fetchone()[0] + ] + + +def test_gepa_live_publishing_lease_excludes_duplicate_worker(tmp_path): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + optimizer = _optimizer(storage, tmp_path) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + original_prepare = storage.prepare_gepa_user_playbook_publication + + def prepare_then_crash_with_live_lease(**kwargs): + original_prepare(**kwargs) + raise RuntimeError("crash with live lease") + + with ( + patch.object( + storage, + "prepare_gepa_user_playbook_publication", + side_effect=prepare_then_crash_with_live_lease, + ), + pytest.raises(RuntimeError), + ): + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + optimizer._run_gepa = Mock( # type: ignore[method-assign] + side_effect=AssertionError("live publication lease must block duplicate GEPA") + ) + assert ( + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + == "skipped" + ) + assert ( + storage.conn.execute( + "SELECT COUNT(*) FROM playbook_optimization_jobs" + ).fetchone()[0] + == 1 + ) + + +@pytest.mark.parametrize( + "kill_switch", + ["enabled", "optimize_user_playbooks", "auto_update_user_playbooks"], +) +def test_gepa_recovery_respects_live_kill_switches(tmp_path, kill_switch): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + config = _optimizer_config(tmp_path) + optimizer = _optimizer_with_config(storage, config) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + original_prepare = storage.prepare_gepa_user_playbook_publication + + def prepare_then_crash(**kwargs): + prepared = original_prepare(**kwargs) + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET lease_expires_at = 0 WHERE job_id = ?", + (prepared.job_id,), + ) + storage.conn.commit() + raise RuntimeError("crash after durable prepare") + + with ( + patch.object( + storage, + "prepare_gepa_user_playbook_publication", + side_effect=prepare_then_crash, + ), + pytest.raises(RuntimeError), + ): + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + setattr(config.playbook_optimizer_config, kill_switch, False) + optimizer._run_gepa = Mock( # type: ignore[method-assign] + side_effect=AssertionError("recovery must not rerun GEPA") + ) + claim_before = storage.conn.execute( + """SELECT lease_owner, lease_fence, lease_expires_at + FROM playbook_optimization_jobs""" + ).fetchone() + + assert ( + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + == "skipped" + ) + assert ( + storage.conn.execute( + "SELECT outcome FROM user_playbook_publication_results" + ).fetchone() + is None + ) + claim_after = storage.conn.execute( + """SELECT lease_owner, lease_fence, lease_expires_at + FROM playbook_optimization_jobs""" + ).fetchone() + assert tuple(claim_after) == tuple(claim_before) + + +def test_gepa_projector_does_not_require_runtime_source(tmp_path, monkeypatch): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + monkeypatch.setattr( + inspect, + "getsource", + Mock(side_effect=OSError("source unavailable")), + ) + + projection = build_gepa_search_projection(storage, incumbent, "new guidance") + + assert json.loads(projection.canonical_json) == { + "candidate_content_digest": sha256(b"new guidance").hexdigest(), + "embedding": ["0.25", "-0.5", *(["0"] * 510)], + "embedding_model_id": storage.embedding_model_name, + "expanded_terms": [], + "lexical_document": "refund request new guidance", + "preserved_trigger": "refund request", + "projector_code_digest": GEPA_PROJECTOR_CODE_DIGEST, + "projector_id": GEPA_PROJECTOR_ID, + "projector_version": GEPA_PROJECTOR_VERSION, + "schema_version": "offline-tuner-candidate-search-projection-v1", + } + + +def test_gepa_decision_proof_uses_compact_evaluation_digests(tmp_path): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + optimizer = _optimizer(storage, tmp_path) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + + assert ( + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + == "completed" + ) + + row = storage.conn.execute( + "SELECT metadata_json FROM playbook_optimization_jobs" + ).fetchone() + proof = json.loads(json.loads(row["metadata_json"])["publication_proof_json"]) + assert proof["schema_version"] == "gepa-user-playbook-decision-v2" + assert proof["evaluations"] == [ + { + "evaluation_digest": proof["evaluations"][0]["evaluation_digest"], + "evaluation_id": proof["evaluations"][0]["evaluation_id"], + } + ] + assert len(proof["evaluations"][0]["evaluation_digest"]) == 64 + serialized = json.dumps(proof) + assert "candidate_rollout_json" not in serialized + assert "incumbent_rollout_json" not in serialized + + +def test_gepa_recovery_propagates_non_live_lease_reclaim_storage_error(tmp_path): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + optimizer = _optimizer(storage, tmp_path) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + original_prepare = storage.prepare_gepa_user_playbook_publication + + def prepare_then_crash(**kwargs): + prepared = original_prepare(**kwargs) + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET lease_expires_at = 0 WHERE job_id = ?", + (prepared.job_id,), + ) + storage.conn.commit() + raise RuntimeError("crash after durable prepare") + + with ( + patch.object( + storage, + "prepare_gepa_user_playbook_publication", + side_effect=prepare_then_crash, + ), + pytest.raises(RuntimeError), + ): + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + with ( + patch.object( + storage, + "reclaim_gepa_user_playbook_publishing_job", + side_effect=StorageError( + "optimizer job lease is not expired; sqlite unavailable" + ), + ), + pytest.raises(StorageError, match="sqlite unavailable"), + ): + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + +def test_gepa_recovery_reuses_canonical_projection_without_regeneration(tmp_path): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + optimizer = _optimizer(storage, tmp_path) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + original_prepare = storage.prepare_gepa_user_playbook_publication + + def prepare_then_crash(**kwargs): + prepared = original_prepare(**kwargs) + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET lease_expires_at = 0 WHERE job_id = ?", + (prepared.job_id,), + ) + storage.conn.commit() + raise RuntimeError("crash after durable prepare") + + with ( + patch.object( + storage, + "prepare_gepa_user_playbook_publication", + side_effect=prepare_then_crash, + ), + pytest.raises(RuntimeError), + ): + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + storage._get_embedding = Mock( # noqa: SLF001 + side_effect=AssertionError("projection was regenerated") + ) + optimizer._run_gepa = Mock( # type: ignore[method-assign] + side_effect=AssertionError("GEPA search reran") + ) + status = optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + + assert status == "completed" + staged = storage.conn.execute( + "SELECT projection_json FROM user_playbook_publication_staging" + ).fetchone() + assert '"embedding":["0.25","-0.5"' in staged["projection_json"] + + +def test_gepa_local_script_identity_binds_script_content_and_code_digests(tmp_path): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + script = tmp_path / "assistant.py" + script.write_text( + "#!/usr/bin/env python\n" + "import json, sys\n" + "json.load(sys.stdin)\n" + "print(json.dumps({'content': 'ok'}))\n", + encoding="utf-8", + ) + script.chmod(0o755) + config = _optimizer_config(tmp_path) + config.playbook_optimizer_config.webhook_url = None + config.playbook_optimizer_config.assistant_script_path = str(script) + optimizer = _optimizer_with_config(storage, config) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + + assert ( + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + == "completed" + ) + + job = storage.conn.execute( + "SELECT metadata_json FROM playbook_optimization_jobs" + ).fetchone() + authority = json.loads(job["metadata_json"])["gepa_publication_authority"] + backend_identity = authority["backend_identity"] + assert backend_identity["backend_kind"] == "local_script" + assert "backend_class_code_digest" in backend_identity + assert ( + backend_identity["script_content_digest"] + == sha256(script.read_bytes()).hexdigest() + ) + assert "script_path_digest" not in backend_identity + assert "adapter_code_digest" in authority["optimizer_identity"] + assert "rollout_code_digest" in authority["optimizer_identity"] + assert ( + "judge_code_digest" + in authority["evaluator_identity"]["pairwise_judge_request_plan"] + ) + + +def test_sqlite_storage_uses_shared_publication_metadata_key_constants(): + source = Path(__file__).parents[4] / ( + "reflexio/server/services/storage/sqlite_storage/playbook/_optimization.py" + ) + text = source.read_text(encoding="utf-8") + + assert "PUBLICATION_PROOF_JSON_METADATA_KEY" in text + assert "PUBLICATION_PROJECTION_JSON_METADATA_KEY" in text + assert ( + '_GEPA_PUBLICATION_PROOF_JSON_METADATA_KEY = "publication_proof_json"' + not in text + ) + assert ( + '_GEPA_PUBLICATION_PROJECTION_JSON_METADATA_KEY = "publication_projection_json"' + not in text + ) + + +def test_gepa_recovery_and_retry_create_one_successor_event_and_aggregation(tmp_path): + storage = _storage(tmp_path) + incumbent = _incumbent(storage) + optimizer = _optimizer(storage, tmp_path) + _install_winning_gepa(optimizer, storage, _window(incumbent.user_playbook_id)) + original_stage = storage.stage_user_playbook_publication + original_run_gepa = optimizer._run_gepa + calls = {"stage": 0, "aggregation": 0, "run_gepa": 0} + + def counted_run_gepa(*args, **kwargs): + calls["run_gepa"] += 1 + if calls["run_gepa"] > 1: + raise AssertionError("GEPA search reran instead of resuming") + return original_run_gepa(*args, **kwargs) + + def stage_then_crash_once(request): + calls["stage"] += 1 + original_stage(request) + if calls["stage"] == 1: + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET lease_expires_at = 0 WHERE job_id = ?", + (request.job_id,), + ) + storage.conn.commit() + raise RuntimeError("crash after staging") + + def aggregate_once(**kwargs): # noqa: ARG001 + calls["aggregation"] += 1 + + optimizer._run_gepa = counted_run_gepa # type: ignore[method-assign] + with ( + patch.object( + storage, + "stage_user_playbook_publication", + side_effect=stage_then_crash_once, + ), + patch( + "reflexio.server.services.playbook_optimizer.optimizer." + "maybe_trigger_user_playbook_aggregation", + side_effect=aggregate_once, + ), + ): + with pytest.raises(RuntimeError): + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + assert ( + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + == "completed" + ) + assert ( + optimizer.optimize( + PlaybookOptimizationTarget( + kind="user_playbook", target_id=incumbent.user_playbook_id + ) + ) + == "skipped" + ) + + assert calls["aggregation"] == 1 + assert calls["run_gepa"] == 1 + assert ( + storage.conn.execute( + "SELECT COUNT(*) FROM playbook_optimization_jobs" + ).fetchone()[0] + == 1 + ) + assert ( + storage.conn.execute( + "SELECT COUNT(*) FROM user_playbooks WHERE status IS NULL" + ).fetchone()[0] + == 1 + ) + assert ( + storage.conn.execute( + "SELECT COUNT(*) FROM playbook_optimization_events WHERE event_type = 'publication_applied'" + ).fetchone()[0] + == 1 + ) + assert ( + storage.conn.execute( + "SELECT COUNT(*) FROM user_playbook_publication_results" + ).fetchone()[0] + == 1 + ) + + +def test_optimizer_has_no_direct_user_playbook_supersede_route(): + source = Path(__file__).parents[4] / ( + "reflexio/server/services/playbook_optimizer/optimizer.py" + ) + text = source.read_text(encoding="utf-8") + + assert "_supersede_user_playbook" not in text + assert "save_user_playbooks" not in text diff --git a/tests/server/services/playbook_optimizer/test_judge_frozen_plan.py b/tests/server/services/playbook_optimizer/test_judge_frozen_plan.py new file mode 100644 index 000000000..883bfb5fa --- /dev/null +++ b/tests/server/services/playbook_optimizer/test_judge_frozen_plan.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +import json +from types import SimpleNamespace +from typing import Any, cast +from unittest.mock import Mock + +import pytest + +from reflexio.models.api_schema.domain import AgentPlaybook, Interaction +from reflexio.models.config_schema import ( + APIKeyConfig, + AzureOpenAIConfig, + OpenAIConfig, +) +from reflexio.server.llm.litellm_client import LiteLLMClient, LiteLLMConfig +from reflexio.server.prompt.prompt_manager import PromptManager +from reflexio.server.services.playbook_optimizer import judge as judge_module +from reflexio.server.services.playbook_optimizer.judge import ( + PLAYBOOK_OPTIMIZER_JUDGE_PROMPT_ID, + PairwiseJudge, +) +from reflexio.server.services.playbook_optimizer.models import ( + RolloutTrace, + ScenarioWindow, +) + + +def _inputs() -> dict[str, Any]: + return { + "window": ScenarioWindow( + user_playbook_id=1, + source_interaction_ids=[101], + interactions=[ + Interaction( + interaction_id=101, + user_id="u1", + request_id="request-1", + role="User", + content="Help with a refund", + ) + ], + ), + "incumbent": AgentPlaybook( + agent_version="v1", + content="old guidance", + trigger="refund request", + ), + "candidate": AgentPlaybook( + agent_version="v1", + content="new guidance", + trigger="refund request", + ), + "incumbent_rollout": RolloutTrace(), + "candidate_rollout": RolloutTrace(), + } + + +def _freeze( + prompt_manager: PromptManager, + client: LiteLLMClient, +) -> dict[str, Any]: + return judge_module.build_pairwise_judge_request_plan( + prompt_manager=prompt_manager, + llm_client=client, + model_name="azure/judge-deployment", + ) + + +@pytest.mark.parametrize( + "drift", + ["instance_method", "helper", "config", "seed", "grace", "prompt_version"], +) +def test_pairwise_judge_rejects_frozen_plan_drift_before_provider_execution( + monkeypatch, drift +): + monkeypatch.setenv("REFLEXIO_LLM_SEED", "42") + monkeypatch.setenv("REFLEXIO_LLM_HARD_TIMEOUT_GRACE_SECONDS", "5") + prompt_manager = PromptManager() + client = LiteLLMClient(LiteLLMConfig(model="azure/judge-deployment")) + provider = Mock() + cast(Any, client)._completion_with_hard_timeout = provider + frozen_plan = _freeze(prompt_manager, client) + judge = PairwiseJudge( + cast(Any, SimpleNamespace(prompt_manager=prompt_manager)), + client, + "azure/judge-deployment", + frozen_request_plan=frozen_plan, + ) + + if drift == "instance_method": + cast(Any, client)._make_request = Mock() + elif drift == "helper": + monkeypatch.setattr( + judge_module, + "build_pairwise_judge_request_plan", + Mock(return_value=frozen_plan), + ) + elif drift == "config": + client.config.top_p = 0.25 + elif drift == "seed": + monkeypatch.setenv("REFLEXIO_LLM_SEED", "43") + elif drift == "grace": + monkeypatch.setenv("REFLEXIO_LLM_HARD_TIMEOUT_GRACE_SECONDS", "6") + else: + prompt_manager.version_override = {PLAYBOOK_OPTIMIZER_JUDGE_PROMPT_ID: "1.1.0"} + + with pytest.raises(judge_module.FrozenEvaluatorPlanDriftError): + judge.judge(**_inputs()) + provider.assert_not_called() + + +def test_pairwise_judge_provider_params_match_frozen_sanitized_plan(monkeypatch): + monkeypatch.setenv("REFLEXIO_LLM_SEED", "47") + monkeypatch.setenv("REFLEXIO_LLM_HARD_TIMEOUT_GRACE_SECONDS", "7") + endpoint = "https://azure-one.example.test/" + secret = "azure-secret" + client = LiteLLMClient( + LiteLLMConfig( + model="azure/judge-deployment", + temperature=0.2, + top_p=0.7, + max_tokens=300, + api_key_config=APIKeyConfig( + openai=OpenAIConfig( + azure_config=AzureOpenAIConfig( + api_key=secret, + endpoint=cast(Any, endpoint), + api_version="2024-02-15-preview", + ) + ) + ), + ) + ) + captured: dict[str, Any] = {} + compared: dict[str, Any] = {} + sanitize = judge_module.sanitize_pairwise_judge_provider_params + + def recording_sanitize(params: Any) -> dict[str, Any]: + compared["params"] = params + return sanitize(params) + + monkeypatch.setattr( + judge_module, + "sanitize_pairwise_judge_provider_params", + recording_sanitize, + ) + + def completion(params: dict[str, Any], hard_timeout: float) -> Any: + captured["params"] = params + captured["hard_timeout"] = hard_timeout + message = SimpleNamespace( + content=json.dumps( + { + "verdict": "candidate", + "score": 0.9, + "likert": 5, + "rationale": "candidate is clearer", + } + ), + refusal=None, + ) + return SimpleNamespace( + choices=[ + SimpleNamespace( + message=message, + finish_reason="stop", + stop_reason=None, + ) + ], + usage=None, + stop_reason=None, + ) + + cast(Any, client)._completion_with_hard_timeout = completion + prompt_manager = PromptManager() + frozen_plan = _freeze(prompt_manager, client) + judge = PairwiseJudge( + cast(Any, SimpleNamespace(prompt_manager=prompt_manager)), + client, + "azure/judge-deployment", + frozen_request_plan=frozen_plan, + ) + + result = judge.judge(**_inputs()) + + rung = frozen_plan["judge_generation_settings"]["rungs"][0] + assert result.verdict == "candidate" + assert compared["params"] is captured["params"] + assert sanitize(captured["params"]) == rung["provider_params"] + assert captured["hard_timeout"] == float(rung["hard_timeout_seconds"]) + serialized = json.dumps(frozen_plan, sort_keys=True) + assert rung["provider_params"]["seed"] == 47 + assert secret not in serialized + assert endpoint not in serialized + + +def test_pairwise_judge_rejects_drift_during_prompt_render_before_provider( + monkeypatch, +): + monkeypatch.setenv("REFLEXIO_LLM_SEED", "42") + prompt_manager = PromptManager() + client = LiteLLMClient(LiteLLMConfig(model="azure/judge-deployment")) + provider = Mock() + cast(Any, client)._completion_with_hard_timeout = provider + render = prompt_manager.render_prompt_from_identity + + def mutating_render(*args: Any, **kwargs: Any) -> str: + rendered = render(*args, **kwargs) + client.config.top_p = 0.25 + return rendered + + cast(Any, prompt_manager).render_prompt_from_identity = mutating_render + frozen_plan = _freeze(prompt_manager, client) + judge = PairwiseJudge( + cast(Any, SimpleNamespace(prompt_manager=prompt_manager)), + client, + "azure/judge-deployment", + frozen_request_plan=frozen_plan, + ) + + with pytest.raises(judge_module.FrozenEvaluatorPlanDriftError): + judge.judge(**_inputs()) + provider.assert_not_called() diff --git a/tests/server/services/playbook_optimizer/test_optimizer_supersede_integration.py b/tests/server/services/playbook_optimizer/test_optimizer_supersede_integration.py index f0b26bc59..2eec4c509 100644 --- a/tests/server/services/playbook_optimizer/test_optimizer_supersede_integration.py +++ b/tests/server/services/playbook_optimizer/test_optimizer_supersede_integration.py @@ -1,9 +1,7 @@ -"""Integration tests for the optimizer's atomic supersede helpers. +"""Integration tests for the optimizer's agent supersede helper. -Tests the ``_supersede_user_playbook`` and ``_supersede_agent_playbook`` helpers -that were extracted from ``PlaybookOptimizer._commit_if_allowed`` as part of the -lineage Phase A work. These helpers are unit-tested directly against a real -SQLite storage so no full PlaybookOptimizer construction is needed. +User playbooks now use the durable publication service. Agent playbooks retain +their separate PENDING/approval supersede behavior and are covered here. B3 request_id contract: each supersede call must stamp a non-empty, job-derived request_id on its revise lineage event, enabling correct run-correlation (tying @@ -20,12 +18,10 @@ from reflexio.models.api_schema.domain import ( AgentPlaybook, PlaybookStatus, - UserPlaybook, ) from reflexio.models.api_schema.domain.enums import Status from reflexio.server.services.playbook_optimizer.optimizer import ( _supersede_agent_playbook, - _supersede_user_playbook, optimizer_run_request_id, ) from reflexio.server.services.storage.sqlite_storage import SQLiteStorage @@ -46,99 +42,6 @@ def _storage(tmp_path): return storage -# --------------------------------------------------------------------------- -# User-playbook supersede helper -# --------------------------------------------------------------------------- - - -def test_supersede_user_playbook_sets_superseded_by_and_revise_event(tmp_path): - """Happy path: incumbent becomes SUPERSEDED with superseded_by set; a revise lineage event is recorded.""" - storage = _storage(tmp_path) - incumbent = UserPlaybook( - user_id="u1", - agent_version="v1", - request_id="req-1", - playbook_name="support", - content="old content", - ) - storage.save_user_playbooks([incumbent]) - incumbent_id = incumbent.user_playbook_id - - result = _supersede_user_playbook( - storage, - incumbent, - "new content", - "playbook_optimizer", - request_id=optimizer_run_request_id(1), - ) - - assert result is not None, "helper should return the successor id on success" - - # Incumbent must now be SUPERSEDED - row = storage.conn.execute( - "SELECT status, superseded_by FROM user_playbooks WHERE user_playbook_id=?", - (incumbent_id,), - ).fetchone() - assert row["status"] == Status.SUPERSEDED.value - assert int(row["superseded_by"]) == result - - # Successor must be CURRENT (status IS NULL) - successor_row = storage.conn.execute( - "SELECT status, content FROM user_playbooks WHERE user_playbook_id=?", - (result,), - ).fetchone() - assert successor_row["status"] is None - assert successor_row["content"] == "new content" - - # A revise lineage event must exist for the successor - events = storage.get_lineage_events( - entity_type="user_playbook", entity_id=str(result) - ) - assert [event.op for event in events] == ["create", "revise"] - assert events[1].actor == "playbook_optimizer" - assert str(incumbent_id) in events[1].source_ids - - -def test_supersede_user_playbook_returns_none_for_non_current_incumbent(tmp_path): - """If the incumbent is already SUPERSEDED (not CURRENT), the helper returns None and leaves no orphan.""" - storage = _storage(tmp_path) - # Create an already-archived/superseded incumbent by inserting and immediately archiving - incumbent = UserPlaybook( - user_id="u1", - agent_version="v1", - request_id="req-2", - playbook_name="support", - content="stale content", - status=Status.ARCHIVED, # not CURRENT - ) - storage.save_user_playbooks([incumbent]) - events_before = storage.get_lineage_events(entity_type="user_playbook") - - playbooks_before = storage.conn.execute( - "SELECT COUNT(*) as cnt FROM user_playbooks" - ).fetchone()["cnt"] - - result = _supersede_user_playbook( - storage, - incumbent, - "new content", - "playbook_optimizer", - request_id=optimizer_run_request_id(2), - ) - - assert result is None, "helper should return None when incumbent is not CURRENT" - - # No orphan successor should have been left behind - playbooks_after = storage.conn.execute( - "SELECT COUNT(*) as cnt FROM user_playbooks" - ).fetchone()["cnt"] - assert playbooks_after == playbooks_before, "no orphan row should remain" - - # The failed successor contributes no row or event; the incumbent origin remains. - events = storage.get_lineage_events(entity_type="user_playbook") - assert events == events_before - - # --------------------------------------------------------------------------- # Agent-playbook supersede helper # --------------------------------------------------------------------------- @@ -241,44 +144,6 @@ def test_supersede_agent_playbook_returns_none_for_non_current_incumbent(tmp_pat # --------------------------------------------------------------------------- -def test_supersede_user_playbook_revise_event_carries_job_request_id(tmp_path): - """_supersede_user_playbook stamps the passed request_id on the revise event. - - The value of the B3 request_id change is correct run-correlation: tying each - optimizer/edit event to its originating job id. The revise event's request_id - must be non-empty and equal the run id passed in — not empty, not the incumbent's - birth request_id. - """ - storage = _storage(tmp_path) - incumbent = UserPlaybook( - user_id="u1", - agent_version="v1", - request_id="birth-req-original", - playbook_name="support", - content="old content", - ) - storage.save_user_playbooks([incumbent]) - - run_id = optimizer_run_request_id(42) - result = _supersede_user_playbook( - storage, - incumbent, - "new content", - "playbook_optimizer", - request_id=run_id, - ) - - assert result is not None - events = storage.get_lineage_events( - entity_type="user_playbook", entity_id=str(result) - ) - assert [event.op for event in events] == ["create", "revise"] - assert events[1].request_id == run_id, ( - f"revise event must carry the job-derived run id {run_id!r}, " - f"got {events[1].request_id!r}" - ) - - def test_supersede_agent_playbook_revise_event_carries_job_request_id(tmp_path): """_supersede_agent_playbook stamps the passed request_id on the revise event. @@ -317,28 +182,6 @@ def test_supersede_agent_playbook_revise_event_carries_job_request_id(tmp_path): ) -def test_supersede_user_playbook_raises_on_empty_request_id(tmp_path): - """_supersede_user_playbook raises ValueError on empty request_id before any write.""" - storage = _storage(tmp_path) - incumbent = UserPlaybook( - user_id="u1", - agent_version="v1", - request_id="birth-req", - playbook_name="support", - content="old content", - ) - storage.save_user_playbooks([incumbent]) - - with pytest.raises(ValueError, match=_EMPTY_REQUEST_ID_MSG): - _supersede_user_playbook( - storage, incumbent, "new content", "playbook_optimizer", request_id="" - ) - - # No orphan successor should have been inserted - count = storage.conn.execute("SELECT COUNT(*) FROM user_playbooks").fetchone()[0] - assert count == 1, "no orphan row should be inserted when request_id is empty" - - def test_supersede_agent_playbook_raises_on_empty_request_id(tmp_path): """_supersede_agent_playbook raises ValueError on empty request_id before any write.""" storage = _storage(tmp_path) diff --git a/tests/server/services/playbook_optimizer/test_playbook_optimizer.py b/tests/server/services/playbook_optimizer/test_playbook_optimizer.py index 897c600d0..d43fc2a6e 100644 --- a/tests/server/services/playbook_optimizer/test_playbook_optimizer.py +++ b/tests/server/services/playbook_optimizer/test_playbook_optimizer.py @@ -4,6 +4,7 @@ import sys import threading import time +from pathlib import Path from types import SimpleNamespace from typing import Any, cast from unittest.mock import Mock, patch @@ -38,6 +39,9 @@ PLAYBOOK_CONTENT_COMPONENT, ReflexioPlaybookGEPAAdapter, ) +from reflexio.server.services.playbook_optimizer.gepa_publication import ( + GEPA_PUBLICATION_AUTHORITY_METADATA_KEY, +) from reflexio.server.services.playbook_optimizer.judge import JudgeOutput from reflexio.server.services.playbook_optimizer.models import ( CandidateEvaluationOutput, @@ -48,7 +52,6 @@ from reflexio.server.services.playbook_optimizer.optimizer import ( PlaybookOptimizationRunStatus, PlaybookOptimizer, - _agent_like_playbook, _split_train_validation_windows, ) from reflexio.server.services.playbook_optimizer.rollout import MultiTurnRollout @@ -210,7 +213,9 @@ def test_local_script_assistant_times_out(tmp_path): def test_adapter_calls_assistant_with_same_seed_and_different_content(tmp_path): storage = _sqlite_storage(tmp_path) job = storage.create_playbook_optimization_job( - PlaybookOptimizationJob(target_kind="agent_playbook", target_id=1) + PlaybookOptimizationJob( + optimizer_kind="gepa", target_kind="agent_playbook", target_id=1 + ) ) incumbent = AgentPlaybook( agent_playbook_id=1, @@ -272,7 +277,9 @@ def test_adapter_calls_assistant_with_same_seed_and_different_content(tmp_path): def test_adapter_cache_distinguishes_windows_with_same_playbook_id(tmp_path): storage = _sqlite_storage(tmp_path) job = storage.create_playbook_optimization_job( - PlaybookOptimizationJob(target_kind="agent_playbook", target_id=1) + PlaybookOptimizationJob( + optimizer_kind="gepa", target_kind="agent_playbook", target_id=1 + ) ) incumbent = AgentPlaybook( agent_playbook_id=1, @@ -359,7 +366,9 @@ def test_adapter_cache_distinguishes_windows_with_same_playbook_id(tmp_path): def test_adapter_does_not_cache_aborted_evaluations(tmp_path): storage = _sqlite_storage(tmp_path) job = storage.create_playbook_optimization_job( - PlaybookOptimizationJob(target_kind="agent_playbook", target_id=1) + PlaybookOptimizationJob( + optimizer_kind="gepa", target_kind="agent_playbook", target_id=1 + ) ) incumbent = AgentPlaybook( agent_playbook_id=1, @@ -680,64 +689,6 @@ def test_optimizer_skips_when_winner_cannot_be_adopted( assert jobs == [] -def test_user_playbook_optimizer_successor_triggers_aggregation_with_successor_version( - tmp_path, -): - storage = _sqlite_storage(tmp_path) - config = Config( - storage_config=StorageConfigSQLite(db_path=str(tmp_path / "reflexio.db")), - playbook_optimizer_config=PlaybookOptimizerConfig( - auto_update_user_playbooks=True - ), - ) - optimizer = _optimizer_for_test(storage, config) - user_playbook = UserPlaybook( - user_playbook_id=0, - user_id="u1", - agent_version="v-incumbent", - request_id="req-1", - content="old", - trigger="when old", - ) - storage.save_user_playbooks([user_playbook]) - target = PlaybookOptimizationTarget( - kind="user_playbook", target_id=user_playbook.user_playbook_id - ) - original_supersede_record = storage.supersede_record - - def supersede_and_update_successor(*args, **kwargs): - ok = original_supersede_record(*args, **kwargs) - if ok: - storage.conn.execute( - "UPDATE user_playbooks SET agent_version = ? WHERE user_playbook_id = ?", - ("v-successor", int(kwargs["successor_id"])), - ) - storage.conn.commit() - return ok - - with ( - patch.object( - storage, "supersede_record", side_effect=supersede_and_update_successor - ), - patch( - "reflexio.server.services.playbook_optimizer.optimizer." - "maybe_trigger_user_playbook_aggregation", - ) as trigger, - ): - successor_id = optimizer._commit_if_allowed( # noqa: SLF001 - target, - _agent_like_playbook(user_playbook), - "new", - config.playbook_optimizer_config, - "run-1", - ) - - assert successor_id is not None - trigger.assert_called_once() - assert trigger.call_args.kwargs["agent_version"] == "v-successor" - assert trigger.call_args.kwargs["reason"] == "playbook_optimizer" - - def test_agent_playbook_optimizer_successor_does_not_trigger_aggregation(tmp_path): storage = _sqlite_storage(tmp_path) config = Config( @@ -778,51 +729,78 @@ def test_agent_playbook_optimizer_successor_does_not_trigger_aggregation(tmp_pat trigger.assert_not_called() -def test_user_playbook_optimizer_successor_reload_failure_does_not_fail_commit( +def test_agent_optimizer_local_backend_does_not_build_user_publication_authority( tmp_path, ): storage = _sqlite_storage(tmp_path) config = Config( storage_config=StorageConfigSQLite(db_path=str(tmp_path / "reflexio.db")), playbook_optimizer_config=PlaybookOptimizerConfig( - auto_update_user_playbooks=True + enabled=True, + optimize_agent_playbooks=True, + auto_update_pending_agent_playbooks=True, + assistant_script_path=sys.executable, + min_commit_windows=1, + min_commit_score=0.1, + min_commit_likert=1, ), ) optimizer = _optimizer_for_test(storage, config) - user_playbook = UserPlaybook( - user_playbook_id=0, - user_id="u1", - agent_version="v1", - request_id="req-1", - content="old", - trigger="when old", - ) - storage.save_user_playbooks([user_playbook]) - target = PlaybookOptimizationTarget( - kind="user_playbook", target_id=user_playbook.user_playbook_id + [agent_playbook] = storage.save_agent_playbooks( + [ + AgentPlaybook( + playbook_name="support", + agent_version="v1", + content="old", + trigger="when old", + playbook_status=PlaybookStatus.PENDING, + ) + ] ) + window = _scenario_window(agent_playbook.agent_playbook_id) + optimizer._resolve_windows = Mock(return_value=[window]) # type: ignore[method-assign] - with ( - patch.object( - storage, - "get_user_playbook_by_id", - side_effect=[user_playbook, RuntimeError("reload failed")], - ), - patch( - "reflexio.server.services.playbook_optimizer.optimizer." - "maybe_trigger_user_playbook_aggregation", - ) as trigger, + def fake_run_gepa(config, seed_content, train_windows, validation_windows, adapter): # noqa: ARG001 + candidate = adapter._ensure_candidate("candidate content") # noqa: SLF001 + storage.insert_playbook_optimization_evaluation( + PlaybookOptimizationEvaluation( + job_id=adapter.job_id, + candidate_id=candidate.candidate_id, + target_kind="agent_playbook", + target_id=agent_playbook.agent_playbook_id, + scenario_user_playbook_id=window.user_playbook_id, + source_interaction_ids=window.source_interaction_ids, + score=0.9, + verdict="candidate", + likert=5, + ) + ) + return SimpleNamespace( + best_candidate={PLAYBOOK_CONTENT_COMPONENT: "candidate content"}, + val_aggregate_scores=[0.9], + best_idx=0, + to_dict=lambda: {"best_idx": 0}, + ) + + optimizer._run_gepa = fake_run_gepa # type: ignore[method-assign] + with patch.object( + Path, + "read_bytes", + side_effect=PermissionError("publication hashing must not read agent backend"), ): - successor_id = optimizer._commit_if_allowed( # noqa: SLF001 - target, - _agent_like_playbook(user_playbook), - "new", - config.playbook_optimizer_config, - "run-1", + result = optimizer.optimize( + PlaybookOptimizationTarget( + kind="agent_playbook", target_id=agent_playbook.agent_playbook_id + ) ) - assert successor_id is not None - trigger.assert_not_called() + assert result == "completed" + job = storage.conn.execute( + "SELECT metadata_json FROM playbook_optimization_jobs" + ).fetchone() + assert GEPA_PUBLICATION_AUTHORITY_METADATA_KEY not in json.loads( + job["metadata_json"] + ) def test_optimizer_runs_single_window_when_commit_threshold_is_one(tmp_path): @@ -1173,7 +1151,9 @@ def test_sqlite_persists_source_mapping_and_winner_candidate(tmp_path): assert storage.get_source_user_playbook_ids_for_agent_playbook(10) == [id1, id2] job = storage.create_playbook_optimization_job( - PlaybookOptimizationJob(target_kind="agent_playbook", target_id=10) + PlaybookOptimizationJob( + optimizer_kind="gepa", target_kind="agent_playbook", target_id=10 + ) ) candidate = storage.insert_playbook_optimization_candidate( PlaybookOptimizationCandidate( @@ -1196,7 +1176,9 @@ def test_sqlite_persists_source_mapping_and_winner_candidate(tmp_path): def test_sqlite_persists_candidate_metadata_json(tmp_path): storage = _sqlite_storage(tmp_path) job = storage.create_playbook_optimization_job( - PlaybookOptimizationJob(target_kind="user_playbook", target_id=10) + PlaybookOptimizationJob( + optimizer_kind="gepa", target_kind="user_playbook", target_id=10 + ) ) metadata_json = json.dumps( { @@ -1380,7 +1362,9 @@ def fake_run_gepa(config, seed_content, train_windows, validation_windows, adapt def test_commit_thresholds_only_count_winner_candidate(tmp_path): storage = _sqlite_storage(tmp_path) job = storage.create_playbook_optimization_job( - PlaybookOptimizationJob(target_kind="agent_playbook", target_id=10) + PlaybookOptimizationJob( + optimizer_kind="gepa", target_kind="agent_playbook", target_id=10 + ) ) losing_candidate = storage.insert_playbook_optimization_candidate( PlaybookOptimizationCandidate(job_id=job.job_id, content="losing") diff --git a/tests/server/services/storage/sqlite_storage/test_playbook_optimization_candidate_metadata_migration.py b/tests/server/services/storage/sqlite_storage/test_playbook_optimization_candidate_metadata_migration.py index ca9195d8b..ff6ec08ff 100644 --- a/tests/server/services/storage/sqlite_storage/test_playbook_optimization_candidate_metadata_migration.py +++ b/tests/server/services/storage/sqlite_storage/test_playbook_optimization_candidate_metadata_migration.py @@ -55,7 +55,9 @@ def test_migration_adds_candidate_metadata_column_and_persists_metadata( assert "metadata_json" in _candidate_columns(db_path) job = storage.create_playbook_optimization_job( - PlaybookOptimizationJob(target_kind="user_playbook", target_id=10) + PlaybookOptimizationJob( + optimizer_kind="gepa", target_kind="user_playbook", target_id=10 + ) ) metadata_json = json.dumps( { diff --git a/tests/server/services/storage/sqlite_storage/test_playbook_remaining_methods_integration.py b/tests/server/services/storage/sqlite_storage/test_playbook_remaining_methods_integration.py index 6ee8db1d7..0bd973c26 100644 --- a/tests/server/services/storage/sqlite_storage/test_playbook_remaining_methods_integration.py +++ b/tests/server/services/storage/sqlite_storage/test_playbook_remaining_methods_integration.py @@ -88,7 +88,9 @@ def _make_eval_result( def _make_job(*, target_id: int = 1) -> PlaybookOptimizationJob: - return PlaybookOptimizationJob(target_kind="agent_playbook", target_id=target_id) + return PlaybookOptimizationJob( + optimizer_kind="gepa", target_kind="agent_playbook", target_id=target_id + ) def _make_candidate( diff --git a/tests/server/services/storage/test_playbook_optimization_replay_contract_integration.py b/tests/server/services/storage/test_playbook_optimization_replay_contract_integration.py new file mode 100644 index 000000000..d6439ec30 --- /dev/null +++ b/tests/server/services/storage/test_playbook_optimization_replay_contract_integration.py @@ -0,0 +1,888 @@ +"""SQLite contracts for durable replay optimizer jobs and artifacts.""" + +from __future__ import annotations + +import json +import sqlite3 +from collections.abc import Generator +from hashlib import sha256 +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from reflexio.models.api_schema import service_schemas as schemas +from reflexio.server.services.playbook.publication import canonical_json_bytes +from reflexio.server.services.storage.error import ( + OptimizationJobLeaseLiveError, + StorageError, +) +from reflexio.server.services.storage.sqlite_storage import SQLiteStorage +from reflexio.server.services.storage.storage_base import BaseStorage + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def storage(tmp_path: Path) -> Generator[BaseStorage]: + store = SQLiteStorage( + org_id="optimization-replay-contract", + db_path=str(tmp_path / "reflexio.db"), + ) + try: + yield store + finally: + store.conn.close() + + +def _replay_job( + discovery_key: str, + attempt_key: str, +) -> schemas.PlaybookOptimizationJob: + return schemas.PlaybookOptimizationJob( + optimizer_kind="offline_tuner_replay", + target_kind="user_playbook", + target_id=41, + discovery_key=discovery_key, + attempt_key=attempt_key, + stage="evidence_frozen", + expected_population_manifest_digest="a" * 64, + generation_selection_manifest_digest="b" * 64, + replay_manifest_digest="c" * 64, + candidate_content_digest="d" * 64, + search_projection_digest="e" * 64, + publication_scope_digest="f" * 64, + ) + + +def _gepa_user_publication_job(target_id: int = 41) -> schemas.PlaybookOptimizationJob: + return schemas.PlaybookOptimizationJob( + optimizer_kind="gepa", + target_kind="user_playbook", + target_id=target_id, + status="running", + stage="publishing", + best_candidate_id=17, + metadata_json="{}", + attempt_key="gepa-user-attempt", + lease_owner="worker-a", + lease_fence=1, + lease_expires_at=2_000, + ) + + +def _artifact( + *, + job_id: int, + digest: str | None = None, + content_json: str = '{"eligible_ids":[1,2]}', +): + canonical = canonical_json_bytes(json.loads(content_json)).decode() + return schemas.PlaybookOptimizationArtifact( + job_id=job_id, + artifact_kind="expected_population_manifest", + content_json=content_json, + content_digest=digest or sha256(canonical.encode()).hexdigest(), + ) + + +def test_replay_job_model_exposes_typed_durable_fields() -> None: + job = _replay_job("d1", "a1") + + assert job.optimizer_kind == "offline_tuner_replay" + assert job.stage == "evidence_frozen" + assert job.lease_fence == 0 + assert job.terminal_outcome is None + assert job.expected_population_manifest_digest == "a" * 64 + + +def test_same_discovery_key_returns_one_active_job(storage: BaseStorage) -> None: + first = storage.create_or_get_playbook_optimization_job(_replay_job("d1", "a1")) + second = storage.create_or_get_playbook_optimization_job(_replay_job("d1", "a1")) + + assert second.job_id == first.job_id + + +def test_same_attempt_key_returns_one_active_job(storage: BaseStorage) -> None: + first = storage.create_or_get_playbook_optimization_job(_replay_job("d1", "a1")) + second = storage.create_or_get_playbook_optimization_job(_replay_job("d2", "a1")) + + assert second.job_id == first.job_id + + +def test_conflicting_active_identity_is_rejected(storage: BaseStorage) -> None: + storage.create_or_get_playbook_optimization_job(_replay_job("d1", "a1")) + + with pytest.raises(StorageError, match="immutable optimizer job identity"): + storage.create_or_get_playbook_optimization_job(_replay_job("d1", "a2")) + + +def test_gepa_publication_reclaim_contract_has_none_live_and_reclaimed_outcomes( + storage: BaseStorage, +) -> None: + assert ( + storage.reclaim_gepa_user_playbook_publishing_job( + 41, "worker-b", lease_seconds=60, now=2_001 + ) + is None + ) + + job = storage.create_playbook_optimization_job(_gepa_user_publication_job()) + with pytest.raises(OptimizationJobLeaseLiveError): + storage.reclaim_gepa_user_playbook_publishing_job( + 41, "worker-b", lease_seconds=60, now=1_999 + ) + + reclaimed = storage.reclaim_gepa_user_playbook_publishing_job( + 41, "worker-b", lease_seconds=60, now=2_001 + ) + assert reclaimed is not None + assert reclaimed.job_id == job.job_id + assert reclaimed.lease_owner == "worker-b" + assert reclaimed.lease_fence == 2 + assert reclaimed.lease_expires_at == 2_061 + + +def test_stale_lease_fence_cannot_advance_stage(storage: BaseStorage) -> None: + job = storage.create_or_get_playbook_optimization_job(_replay_job("d1", "a1")) + claim = storage.claim_playbook_optimization_job( + job_id=job.job_id, + owner="worker-a", + lease_seconds=60, + now=1_000, + ) + reclaimed = storage.reclaim_playbook_optimization_job( + job_id=job.job_id, + owner="worker-b", + now=claim.expires_at + 1, + lease_seconds=60, + ) + + assert reclaimed.fence > claim.fence + assert ( + storage.advance_playbook_optimization_stage( + job_id=job.job_id, + fence=claim.fence, + stage="candidate_generated", + now=claim.expires_at + 1, + ) + is False + ) + assert ( + storage.advance_playbook_optimization_stage( + job_id=job.job_id, + fence=reclaimed.fence, + stage="candidate_generated", + now=claim.expires_at + 1, + ) + is True + ) + + +def test_current_owner_can_renew_but_stale_owner_cannot(storage: BaseStorage) -> None: + job = storage.create_or_get_playbook_optimization_job(_replay_job("d1", "a1")) + claim = storage.claim_playbook_optimization_job( + job_id=job.job_id, + owner="worker-a", + lease_seconds=60, + now=2_000, + ) + renewed = storage.renew_playbook_optimization_job_lease( + job_id=job.job_id, + owner="worker-a", + fence=claim.fence, + lease_seconds=120, + now=2_010, + ) + + assert renewed.expires_at == 2_130 + with pytest.raises(StorageError, match="lease is no longer current"): + storage.renew_playbook_optimization_job_lease( + job_id=job.job_id, + owner="worker-b", + fence=claim.fence, + lease_seconds=120, + now=2_020, + ) + + +def test_stage_advancement_is_linear(storage: BaseStorage) -> None: + job = storage.create_or_get_playbook_optimization_job(_replay_job("d1", "a1")) + claim = storage.claim_playbook_optimization_job( + job_id=job.job_id, + owner="worker-a", + lease_seconds=60, + now=3_000, + ) + + assert ( + storage.advance_playbook_optimization_stage( + job_id=job.job_id, + fence=claim.fence, + stage="replay_running", + now=3_001, + ) + is False + ) + assert ( + storage.advance_playbook_optimization_stage( + job_id=job.job_id, + fence=claim.fence, + stage="candidate_generated", + now=3_001, + ) + is True + ) + + +@pytest.mark.parametrize( + ("stage", "outcome", "expected_status"), + [ + ("abstained", "candidate_did_not_improve", "skipped"), + ("failed", "generation_failed", "failed"), + ], +) +def test_terminal_stage_records_outcome_and_releases_lease( + storage: BaseStorage, + stage: schemas.OptimizationJobStage, + outcome: schemas.OptimizationTerminalOutcome, + expected_status: str, +) -> None: + job = storage.create_or_get_playbook_optimization_job(_replay_job("d1", "a1")) + claim = storage.claim_playbook_optimization_job( + job_id=job.job_id, + owner="worker-a", + lease_seconds=60, + now=3_000, + ) + + assert storage.advance_playbook_optimization_stage( + job_id=job.job_id, + fence=claim.fence, + stage=stage, + terminal_outcome=outcome, + now=3_001, + ) + assert isinstance(storage, SQLiteStorage) + row = storage.conn.execute( + "SELECT * FROM playbook_optimization_jobs WHERE job_id = ?", (job.job_id,) + ).fetchone() + assert row["stage"] == stage + assert row["terminal_outcome"] == outcome + assert row["status"] == expected_status + assert row["lease_owner"] is None + assert row["lease_expires_at"] is None + + +def test_stale_lease_fence_cannot_write_singleton_artifact( + storage: BaseStorage, +) -> None: + job = storage.create_or_get_playbook_optimization_job(_replay_job("d1", "a1")) + claim = storage.claim_playbook_optimization_job( + job_id=job.job_id, + owner="worker-a", + lease_seconds=60, + now=4_000, + ) + reclaimed = storage.reclaim_playbook_optimization_job( + job_id=job.job_id, + owner="worker-b", + lease_seconds=60, + now=claim.expires_at + 1, + ) + + with pytest.raises(StorageError, match="lease is no longer current"): + storage.upsert_playbook_optimization_artifact( + _artifact(job_id=job.job_id), + fence=claim.fence, + now=claim.expires_at + 1, + ) + + saved = storage.upsert_playbook_optimization_artifact( + _artifact(job_id=job.job_id), + fence=reclaimed.fence, + now=claim.expires_at + 1, + ) + assert saved.job_id == job.job_id + + +def test_artifact_upsert_canonicalizes_equivalent_json_and_requires_digest_and_content( + storage: BaseStorage, +) -> None: + job = storage.create_or_get_playbook_optimization_job(_replay_job("d1", "a1")) + claim = storage.claim_playbook_optimization_job( + job_id=job.job_id, + owner="worker-a", + lease_seconds=60, + now=5_000, + ) + first = storage.upsert_playbook_optimization_artifact( + _artifact( + job_id=job.job_id, + content_json='{"eligible_ids":[1,2],"meta":{"b":2,"a":1}}', + ), + fence=claim.fence, + now=5_001, + ) + second = storage.upsert_playbook_optimization_artifact( + _artifact( + job_id=job.job_id, + content_json='{ "meta" : { "a" : 1, "b" : 2 }, "eligible_ids" : [1,2] }', + ), + fence=claim.fence, + now=5_001, + ) + + assert second.artifact_id == first.artifact_id + assert second.content_json == '{"eligible_ids":[1,2],"meta":{"a":1,"b":2}}' + + with pytest.raises(StorageError, match="digest does not match content"): + storage.upsert_playbook_optimization_artifact( + first.model_copy(update={"content_digest": "b" * 64}), + fence=claim.fence, + now=5_001, + ) + with pytest.raises(StorageError, match="digest does not match content"): + storage.upsert_playbook_optimization_artifact( + first.model_copy(update={"content_json": '{"eligible_ids":[3]}'}), + fence=claim.fence, + now=5_001, + ) + with pytest.raises(StorageError, match="artifact digest conflict"): + storage.upsert_playbook_optimization_artifact( + _artifact( + job_id=job.job_id, + content_json='{"eligible_ids":[3]}', + ), + fence=claim.fence, + now=5_001, + ) + + +def test_artifact_model_rejects_malformed_json() -> None: + with pytest.raises( + ValidationError, match="artifact content_json must be valid JSON" + ): + schemas.PlaybookOptimizationArtifact( + job_id=1, + artifact_kind="expected_population_manifest", + content_json="{", + content_digest="a" * 64, + ) + + +def test_artifact_model_rejects_digest_for_different_canonical_content() -> None: + with pytest.raises(ValidationError, match="artifact digest must match"): + schemas.PlaybookOptimizationArtifact( + job_id=1, + artifact_kind="expected_population_manifest", + content_json='{"eligible_ids":[1,2]}', + content_digest=sha256(b'{"eligible_ids":[2,1]}').hexdigest(), + ) + + +def test_artifact_model_binds_digest_to_canonical_equivalent_json() -> None: + digest = sha256(b'{"eligible_ids":[1,2],"meta":{"a":1,"b":2}}').hexdigest() + + artifact = schemas.PlaybookOptimizationArtifact( + job_id=1, + artifact_kind="expected_population_manifest", + content_json='{ "meta": {"b": 2, "a": 1}, "eligible_ids": [1, 2] }', + content_digest=digest, + ) + + assert artifact.content_json == '{"eligible_ids":[1,2],"meta":{"a":1,"b":2}}' + assert artifact.content_digest == digest + + +def test_artifact_model_uses_rfc8785_utf16_key_order() -> None: + canonical = '{"\U00010000":"astral","\ue000":"bmp"}' + + artifact = schemas.PlaybookOptimizationArtifact( + job_id=1, + artifact_kind="expected_population_manifest", + content_json='{"\ue000":"bmp","\U00010000":"astral"}', + content_digest=sha256(canonical.encode()).hexdigest(), + ) + + assert artifact.content_json == canonical + + +def test_artifact_model_uses_publication_numeric_contract() -> None: + canonical = '{"maximum":9007199254740991,"minimum":-9007199254740991}' + artifact = schemas.PlaybookOptimizationArtifact( + job_id=1, + artifact_kind="expected_population_manifest", + content_json='{ "minimum": -9007199254740991, "maximum": 9007199254740991 }', + content_digest=sha256(canonical.encode()).hexdigest(), + ) + + assert artifact.content_json == canonical + with pytest.raises( + ValidationError, match="artifact content_json must be valid JSON" + ): + schemas.PlaybookOptimizationArtifact( + job_id=1, + artifact_kind="expected_population_manifest", + content_json='{"value":1.0}', + content_digest="a" * 64, + ) + + +@pytest.mark.parametrize( + "current_stage", + [ + "evidence_frozen", + "candidate_generated", + "replay_running", + "replay_evaluated", + ], +) +def test_applied_terminal_stage_requires_publishing( + storage: BaseStorage, + current_stage: schemas.OptimizationJobStage, +) -> None: + job = storage.create_or_get_playbook_optimization_job(_replay_job("d1", "a1")) + claim = storage.claim_playbook_optimization_job( + job_id=job.job_id, + owner="worker-a", + lease_seconds=60, + now=6_000, + ) + assert isinstance(storage, SQLiteStorage) + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET stage = ? WHERE job_id = ?", + (current_stage, job.job_id), + ) + storage.conn.commit() + + assert ( + storage.advance_playbook_optimization_stage( + job_id=job.job_id, + fence=claim.fence, + stage="applied", + terminal_outcome="applied", + now=6_001, + ) + is False + ) + persisted = storage.get_playbook_optimization_job(job.job_id) + assert persisted is not None + assert persisted.stage == current_stage + assert persisted.status == "running" + assert persisted.terminal_outcome is None + + +def test_applied_terminal_stage_advances_from_publishing(storage: BaseStorage) -> None: + job = storage.create_or_get_playbook_optimization_job(_replay_job("d1", "a1")) + claim = storage.claim_playbook_optimization_job( + job_id=job.job_id, + owner="worker-a", + lease_seconds=60, + now=6_000, + ) + assert isinstance(storage, SQLiteStorage) + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET stage = 'publishing' WHERE job_id = ?", + (job.job_id,), + ) + storage.conn.commit() + + assert storage.advance_playbook_optimization_stage( + job_id=job.job_id, + fence=claim.fence, + stage="applied", + terminal_outcome="applied", + now=6_001, + ) + persisted = storage.get_playbook_optimization_job(job.job_id) + assert persisted is not None + assert persisted.stage == "applied" + assert persisted.status == "completed" + assert persisted.terminal_outcome == "applied" + + +def test_governance_erased_terminal_outcome_round_trips( + storage: BaseStorage, +) -> None: + job = storage.create_or_get_playbook_optimization_job(_replay_job("d1", "a1")) + assert isinstance(storage, SQLiteStorage) + storage.conn.execute( + """UPDATE playbook_optimization_jobs + SET status = 'skipped', stage = 'failed', + terminal_outcome = 'governance_erased' + WHERE job_id = ?""", + (job.job_id,), + ) + storage.conn.commit() + + persisted = storage.get_playbook_optimization_job(job.job_id) + + assert persisted is not None + assert persisted.terminal_outcome == "governance_erased" + + +def test_ordinary_stage_advance_rejects_governance_erased( + storage: BaseStorage, +) -> None: + job = storage.create_or_get_playbook_optimization_job(_replay_job("d1", "a1")) + claim = storage.claim_playbook_optimization_job( + job_id=job.job_id, + owner="worker-a", + lease_seconds=60, + now=6_000, + ) + + assert ( + storage.advance_playbook_optimization_stage( + job_id=job.job_id, + fence=claim.fence, + stage="abstained", + terminal_outcome="governance_erased", + now=6_001, + ) + is False + ) + persisted = storage.get_playbook_optimization_job(job.job_id) + assert persisted is not None + assert persisted.status == "running" + assert persisted.stage == "evidence_frozen" + assert persisted.terminal_outcome is None + + +def _create_legacy_optimizer_schema(db_path: Path) -> None: + conn = sqlite3.connect(db_path) + conn.executescript( + """ + CREATE TABLE playbook_optimization_jobs ( + job_id INTEGER PRIMARY KEY AUTOINCREMENT, + target_kind TEXT NOT NULL, + target_id INTEGER NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + best_candidate_id INTEGER, + successor_target_id INTEGER, + decision_reason TEXT NOT NULL DEFAULT '', + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE TABLE playbook_optimization_candidates ( + candidate_id INTEGER PRIMARY KEY AUTOINCREMENT, + job_id INTEGER NOT NULL, + candidate_index INTEGER NOT NULL DEFAULT 0, + content TEXT NOT NULL, + parent_candidate_ids TEXT NOT NULL DEFAULT '[]', + aggregate_score REAL, + is_winner INTEGER NOT NULL DEFAULT 0, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at INTEGER NOT NULL + ); + CREATE TABLE playbook_optimization_events ( + event_id INTEGER PRIMARY KEY AUTOINCREMENT, + job_id INTEGER NOT NULL, + event_type TEXT NOT NULL, + payload_json TEXT NOT NULL DEFAULT '{}', + created_at INTEGER NOT NULL + ); + """ + ) + jobs = [ + (1, "running", {"offline_tuner": {}}), + ( + 2, + "pending", + { + "source_window_count": 3, + "train_window_count": 2, + "validation_window_count": 1, + }, + ), + ( + 3, + "running", + { + "offline_tuner": {}, + "source_window_count": 3, + "train_window_count": 2, + "validation_window_count": 1, + }, + ), + (4, "pending", {}), + (5, "completed", {}), + (6, "pending", {}), + ] + conn.executemany( + """INSERT INTO playbook_optimization_jobs + (job_id, target_kind, target_id, status, metadata_json, created_at, updated_at) + VALUES (?, 'user_playbook', ?, ?, ?, 1, 1)""", + [ + (job_id, job_id, status, json.dumps(metadata)) + for job_id, status, metadata in jobs + ], + ) + conn.execute( + """INSERT INTO playbook_optimization_events + (job_id, event_type, created_at) VALUES (5, 'offline_tuner_selected', 1)""" + ) + conn.execute( + """INSERT INTO playbook_optimization_candidates + (job_id, content, metadata_json, created_at) + VALUES (6, 'candidate', '{"proposed_edit": {}}', 1)""" + ) + conn.commit() + conn.close() + + +def test_legacy_optimizer_rows_are_classified_mutually_exclusively( + tmp_path: Path, +) -> None: + db_path = tmp_path / "legacy.db" + _create_legacy_optimizer_schema(db_path) + + store = SQLiteStorage(org_id="legacy-classification", db_path=str(db_path)) + try: + rows = store.conn.execute( + """SELECT job_id, optimizer_kind, status, decision_reason + FROM playbook_optimization_jobs ORDER BY job_id""" + ).fetchall() + finally: + store.conn.close() + + assert [row["optimizer_kind"] for row in rows] == [ + "offline_tuner_legacy", + "gepa", + "optimizer_legacy_unknown", + "optimizer_legacy_unknown", + "offline_tuner_legacy", + "offline_tuner_legacy", + ] + assert (rows[0]["status"], rows[0]["decision_reason"]) == ( + "skipped", + "retired_by_replay_redesign", + ) + assert rows[1]["status"] == "pending" + assert rows[2]["status"] == "skipped" + assert rows[3]["status"] == "skipped" + assert rows[4]["status"] == "completed" + assert rows[5]["status"] == "skipped" + + +def test_legacy_duplicate_active_gepa_jobs_are_deterministically_retired( + tmp_path: Path, +) -> None: + db_path = tmp_path / "legacy-duplicates.db" + _create_legacy_optimizer_schema(db_path) + conn = sqlite3.connect(db_path) + conn.execute( + """INSERT INTO playbook_optimization_jobs + (job_id, target_kind, target_id, status, metadata_json, created_at, updated_at) + VALUES (7, 'user_playbook', 2, 'running', ?, 2, 2)""", + ( + json.dumps( + { + "source_window_count": 3, + "train_window_count": 2, + "validation_window_count": 1, + } + ), + ), + ) + conn.commit() + conn.close() + + store = SQLiteStorage(org_id="legacy-duplicates", db_path=str(db_path)) + try: + rows = store.conn.execute( + """SELECT job_id, optimizer_kind, status, decision_reason + FROM playbook_optimization_jobs + WHERE target_kind = 'user_playbook' AND target_id = 2 + ORDER BY job_id""" + ).fetchall() + finally: + store.conn.close() + + assert [(row["job_id"], row["status"]) for row in rows] == [ + (2, "pending"), + (7, "skipped"), + ] + assert rows[1]["optimizer_kind"] == "gepa" + assert rows[1]["decision_reason"] == "retired_duplicate_legacy_active_job" + + +@pytest.mark.parametrize("key_column", ["discovery_key", "attempt_key"]) +def test_legacy_duplicate_active_gepa_job_keys_are_deterministically_retired( + tmp_path: Path, + key_column: str, +) -> None: + db_path = tmp_path / f"legacy-duplicate-{key_column}.db" + _create_legacy_optimizer_schema(db_path) + conn = sqlite3.connect(db_path) + conn.execute("ALTER TABLE playbook_optimization_jobs ADD COLUMN discovery_key TEXT") + conn.execute("ALTER TABLE playbook_optimization_jobs ADD COLUMN attempt_key TEXT") + metadata_json = json.dumps( + { + "source_window_count": 3, + "train_window_count": 2, + "validation_window_count": 1, + } + ) + conn.execute( + """INSERT INTO playbook_optimization_jobs + (job_id, target_kind, target_id, status, metadata_json, + discovery_key, attempt_key, created_at, updated_at) + VALUES (7, 'user_playbook', 7, 'running', ?, 'discovery-7', + 'attempt-7', 2, 2)""", + (metadata_json,), + ) + conn.execute( + f"UPDATE playbook_optimization_jobs SET {key_column} = ? " # noqa: S608 + "WHERE job_id IN (2, 7)", + (f"duplicate-{key_column}",), + ) + conn.commit() + conn.close() + + store = SQLiteStorage(org_id=f"legacy-duplicate-{key_column}", db_path=str(db_path)) + try: + rows = store.conn.execute( + """SELECT job_id, status, decision_reason + FROM playbook_optimization_jobs + WHERE job_id IN (2, 7) ORDER BY job_id""" + ).fetchall() + finally: + store.conn.close() + + assert [(row["job_id"], row["status"]) for row in rows] == [ + (2, "pending"), + (7, "skipped"), + ] + assert rows[1]["decision_reason"] == "retired_duplicate_legacy_active_job" + + +def _create_legacy_optimizer_schema_with_child( + db_path: Path, + *, + orphan_child: bool = False, + stale_rebuild_table: bool = False, +) -> None: + _create_legacy_optimizer_schema(db_path) + conn = sqlite3.connect(db_path) + conn.execute("PRAGMA foreign_keys=ON") + conn.execute( + """CREATE TABLE optimizer_job_child ( + child_id INTEGER PRIMARY KEY, + job_id INTEGER NOT NULL, + FOREIGN KEY (job_id) REFERENCES playbook_optimization_jobs(job_id) + ON DELETE CASCADE + )""" + ) + conn.execute("INSERT INTO optimizer_job_child VALUES (1, 2)") + conn.commit() + if orphan_child: + conn.execute("PRAGMA foreign_keys=OFF") + conn.execute("INSERT INTO optimizer_job_child VALUES (2, 999)") + if stale_rebuild_table: + conn.execute( + "CREATE TABLE playbook_optimization_jobs_new (job_id INTEGER PRIMARY KEY)" + ) + conn.commit() + conn.close() + + +def test_legacy_optimizer_rebuild_preserves_fk_children_and_restores_enforcement( + tmp_path: Path, +) -> None: + db_path = tmp_path / "legacy-child.db" + _create_legacy_optimizer_schema_with_child(db_path) + + store = SQLiteStorage(org_id="legacy-child", db_path=str(db_path)) + try: + child = store.conn.execute("SELECT * FROM optimizer_job_child").fetchone() + foreign_keys = store.conn.execute("PRAGMA foreign_keys").fetchone()[0] + violations = store.conn.execute("PRAGMA foreign_key_check").fetchall() + finally: + store.conn.close() + + assert child is not None + assert child["job_id"] == 2 + assert foreign_keys == 1 + assert violations == [] + + +def test_legacy_optimizer_rebuild_recovers_stale_temporary_table( + tmp_path: Path, +) -> None: + db_path = tmp_path / "legacy-stale-rebuild.db" + _create_legacy_optimizer_schema_with_child(db_path, stale_rebuild_table=True) + + store = SQLiteStorage(org_id="legacy-stale-rebuild", db_path=str(db_path)) + try: + stale_table = store.conn.execute( + """SELECT 1 FROM sqlite_master + WHERE type = 'table' AND name = 'playbook_optimization_jobs_new'""" + ).fetchone() + child_count = store.conn.execute( + "SELECT COUNT(*) FROM optimizer_job_child" + ).fetchone()[0] + finally: + store.conn.close() + + assert stale_table is None + assert child_count == 1 + + +def test_legacy_optimizer_rebuild_rolls_back_on_foreign_key_violation( + tmp_path: Path, +) -> None: + db_path = tmp_path / "legacy-invalid-child.db" + _create_legacy_optimizer_schema_with_child(db_path, orphan_child=True) + + with pytest.raises(sqlite3.IntegrityError, match="foreign key check"): + SQLiteStorage(org_id="legacy-invalid-child", db_path=str(db_path)) + + conn = sqlite3.connect(db_path) + try: + parent_table = conn.execute( + """SELECT 1 FROM sqlite_master + WHERE type = 'table' AND name = 'playbook_optimization_jobs'""" + ).fetchone() + stale_table = conn.execute( + """SELECT 1 FROM sqlite_master + WHERE type = 'table' AND name = 'playbook_optimization_jobs_new'""" + ).fetchone() + conn.execute("DELETE FROM optimizer_job_child WHERE job_id = 999") + conn.commit() + finally: + conn.close() + + assert parent_table is not None + assert stale_table is None + + store = SQLiteStorage(org_id="legacy-invalid-child-retry", db_path=str(db_path)) + store.conn.close() + + +@pytest.mark.parametrize( + ("column", "invalid_value"), + [ + ("optimizer_kind", "not-an-optimizer"), + ("stage", "not-a-stage"), + ("terminal_outcome", "not-an-outcome"), + ], +) +def test_upgraded_legacy_optimizer_schema_rejects_invalid_durable_values( + tmp_path: Path, + column: str, + invalid_value: str, +) -> None: + db_path = tmp_path / "legacy.db" + _create_legacy_optimizer_schema(db_path) + store = SQLiteStorage(org_id="legacy-constraints", db_path=str(db_path)) + try: + with pytest.raises(sqlite3.IntegrityError): + store.conn.execute( + f"UPDATE playbook_optimization_jobs SET {column} = ? WHERE job_id = 2", # noqa: S608 + (invalid_value,), + ) + finally: + store.conn.close() diff --git a/tests/server/services/storage/test_sqlite_surface.py b/tests/server/services/storage/test_sqlite_surface.py index 696a3abd0..b314193cd 100644 --- a/tests/server/services/storage/test_sqlite_surface.py +++ b/tests/server/services/storage/test_sqlite_surface.py @@ -10,11 +10,36 @@ from reflexio.server.services.storage.sqlite_storage import SQLiteStorage from reflexio.server.services.storage.storage_base import BaseStorage +_OPTIONAL_OPTIMIZER_METHODS = frozenset( + { + "advance_playbook_optimization_stage", + "claim_playbook_optimization_job", + "commit_user_playbook_publication", + "create_or_get_playbook_optimization_job", + "get_playbook_optimization_artifact", + "get_playbook_optimization_job", + "get_unconsumed_gepa_user_playbook_publishing_job", + "get_user_playbook_publication_subject_epochs", + "load_user_playbook_publication_result", + "prepare_gepa_user_playbook_publication", + "reclaim_gepa_user_playbook_publishing_job", + "reclaim_playbook_optimization_job", + "renew_playbook_optimization_job_lease", + "stage_user_playbook_publication", + "upsert_playbook_optimization_artifact", + "claim_user_playbook_publication", + } +) + # RetentionMixin helpers are present on SQLiteStorage but not declared in the # BaseStorage ABC — intentional; subclasses opt into retention without the ABC # requiring it. _RETENTION_MIXIN_METHODS: frozenset[str] = frozenset( - {"count_retention_target_rows", "delete_oldest_retention_target_rows"} + { + "count_retention_target_rows", + "delete_oldest_retention_target_rows", + "gc_retired_optimization_jobs", + } ) _HELPER_METHODS: frozenset[str] = frozenset({"close", "handle_exceptions"}) @@ -34,7 +59,7 @@ def test_sqlite_surface_matches_base_abc() -> None: """SQLiteStorage public surface == BaseStorage ABC + RetentionMixin allowlist. Fails if any method is dropped from or added to SQLiteStorage without a - matching update to BaseStorage. The allowlist captures the two retention + matching update to BaseStorage. The allowlist captures the retention helpers that are intentionally on SQLiteStorage but not on the ABC. """ sqlite_methods = _public_methods(SQLiteStorage) @@ -54,3 +79,29 @@ def test_sqlite_surface_matches_base_abc() -> None: f"{sqlite_methods - base_methods - _RETENTION_MIXIN_METHODS}\n" f" extra base (missing in sqlite): {base_methods - sqlite_methods}" ) + + +def test_optimizer_capability_is_optional_for_legacy_storage_backends() -> None: + """A backend implementing the pre-replay ABC remains instantiable.""" + + def legacy_implementation(*args: object, **kwargs: object) -> None: + del args, kwargs + + legacy_methods = dict.fromkeys( + BaseStorage.__abstractmethods__ - _OPTIONAL_OPTIMIZER_METHODS, + legacy_implementation, + ) + legacy_storage_type = type("LegacyStorage", (BaseStorage,), legacy_methods) + + storage = legacy_storage_type(org_id="legacy-org") + + for operation in ( + lambda: storage.get_playbook_optimization_job(1), + lambda: storage.get_user_playbook_publication_subject_epochs(1), + ): + try: + operation() + except NotImplementedError as exc: + assert "does not support" in str(exc) + else: + raise AssertionError("optional optimizer operation unexpectedly succeeded") diff --git a/tests/server/services/storage/test_user_playbook_publication_sqlite.py b/tests/server/services/storage/test_user_playbook_publication_sqlite.py new file mode 100644 index 000000000..27b0e22f8 --- /dev/null +++ b/tests/server/services/storage/test_user_playbook_publication_sqlite.py @@ -0,0 +1,1198 @@ +from __future__ import annotations + +import json +from concurrent.futures import ThreadPoolExecutor +from hashlib import sha256 +from pathlib import Path +from threading import Barrier +from unittest.mock import Mock, patch + +import pytest + +from reflexio.models.api_schema.domain import ( + PlaybookOptimizationCandidate, + PlaybookOptimizationJob, + UserPlaybook, +) +from reflexio.models.api_schema.domain.enums import Status +from reflexio.models.api_schema.retriever_schema import SearchUserPlaybookRequest +from reflexio.server.services.governance.config import ( + get_governance_ref_secret, + governance_subject_ref, +) +from reflexio.server.services.playbook.publication import ( + DecisionProofEnvelope, + PublicationClaim, + PublicationRequest, + PublicationResult, + PublicationSearchProjection, + PublishableOptimizerKind, + UserPlaybookPublicationService, + incumbent_user_playbook_semantic_digest, + publication_source_for_optimizer, +) +from reflexio.server.services.playbook_optimizer.gepa_publication import ( + GEPA_PUBLICATION_AUTHORITY_METADATA_KEY, +) +from reflexio.server.services.storage.error import StorageError +from reflexio.server.services.storage.sqlite_storage import SQLiteStorage + +pytestmark = pytest.mark.integration + +_ORG_ID = "publication-sqlite" +_LIVE_LEASE_EXPIRY = 4_000_000_000 +_EPOCH_NOW_PATCH = ( + "reflexio.server.services.storage.sqlite_storage.playbook._user._epoch_now" +) + + +@pytest.mark.parametrize( + ("optimizer_kind", "expected_source"), + [("gepa", "gepa"), ("offline_tuner_replay", "offline_optimizer")], +) +def test_publication_source_is_explicitly_mapped( + optimizer_kind: str, expected_source: str +) -> None: + assert publication_source_for_optimizer(optimizer_kind) == expected_source # type: ignore[arg-type] + + +def _canonical(payload: dict[str, object]) -> str: + return json.dumps( + payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True + ) + + +def _digest(value: str) -> str: + return sha256(value.encode()).hexdigest() + + +def _store(tmp_path: Path) -> SQLiteStorage: + with patch.object(SQLiteStorage, "_get_embedding", return_value=[9.0] * 512): + store = SQLiteStorage(org_id=_ORG_ID, db_path=str(tmp_path / "r.db")) + store._get_embedding = Mock(return_value=[9.0] * 512) # noqa: SLF001 + store.llm_client.get_embeddings = Mock(return_value=[[9.0] * 512]) + return store + + +def _incumbent() -> UserPlaybook: + return UserPlaybook( + user_id="u1", + agent_version="agent-v1", + request_id="seed-request", + playbook_name="refund", + content="old content", + trigger="refund trigger", + rationale="keep customer calm", + tags=["billing"], + source_interaction_ids=[1, 2], + source="seed", + ) + + +def _job( + *, + target_id: int, + attempt_key: str = "attempt-1", + worker_fence: int = 5, + projection_digest: str, + content_digest: str, + proof_digest: str, + subject_epochs_json: str, + stage: str | None = "publishing", + optimizer_kind: PublishableOptimizerKind = "gepa", +) -> PlaybookOptimizationJob: + return PlaybookOptimizationJob( + optimizer_kind=optimizer_kind, + target_kind="user_playbook", + target_id=target_id, + status="running", + metadata_json=_canonical( + { + "publication_proof_digest": proof_digest, + "publication_subject_epochs": json.loads(subject_epochs_json), + } + ), + attempt_key=attempt_key, + lease_owner="worker-a", + lease_fence=worker_fence, + lease_expires_at=_LIVE_LEASE_EXPIRY, + stage=stage, # type: ignore[arg-type] + candidate_content_digest=content_digest, + search_projection_digest=projection_digest, + ) + + +def _projection( + content: str = "new content", *, trigger: str | None = "refund trigger" +) -> PublicationSearchProjection: + embedding = ["0.25"] * 512 + payload = { + "candidate_content_digest": _digest(content), + "embedding": embedding, + "embedding_model_id": "test-embedding-v1", + "expanded_terms": ["exact-expanded", "projection-token"], + "lexical_document": "exact lexical projection-token", + "preserved_trigger": trigger, + "projector_code_digest": "a" * 64, + "projector_id": "reflexio.search.user-playbook", + "projector_version": "1", + "schema_version": "offline-tuner-candidate-search-projection-v1", + } + canonical = _canonical(payload) + return PublicationSearchProjection( + schema_version="offline-tuner-candidate-search-projection-v1", + canonical_json=canonical, + digest=_digest(canonical), + projector_id="reflexio.search.user-playbook", + projector_version="1", + projector_code_digest="a" * 64, + candidate_content_digest=_digest(content), + preserved_trigger=trigger, + embedding_model_id="test-embedding-v1", + embedding=tuple(embedding), + expanded_terms=("exact-expanded", "projection-token"), + lexical_document="exact lexical projection-token", + ) + + +def _proof( + source: str = "playbook_optimizer", + *, + optimizer_kind: PublishableOptimizerKind = "gepa", +) -> DecisionProofEnvelope: + payload = { + "adoption": {"min_commit_windows": 1, "score": "0.91"}, + "decision": "apply", + "optimizer_kind": optimizer_kind, + "schema_version": "gepa-publication-proof-v1", + "source": source, + } + canonical = _canonical(payload) + return DecisionProofEnvelope( + optimizer_kind=optimizer_kind, + schema_version="gepa-publication-proof-v1", + canonical_json=canonical, + digest=_digest(canonical), + decision="apply", + ) + + +def _request( + *, + job_id: int, + incumbent_id: int, + claim: PublicationClaim, + worker_fence: int = 5, + content: str = "new content", + projection: PublicationSearchProjection | None = None, + proof: DecisionProofEnvelope | None = None, + request_id: str = "publish-request-1", + subject_epochs_json: str | None = None, + optimizer_kind: PublishableOptimizerKind = "gepa", +) -> PublicationRequest: + return PublicationRequest( + optimizer_kind=optimizer_kind, + job_id=job_id, + attempt_key="attempt-1", + publication_claim=claim, + worker_fence=worker_fence, + incumbent_user_playbook_id=incumbent_id, + incumbent_content_digest=_digest("old content"), + incumbent_trigger="refund trigger", + incumbent_semantic_digest=incumbent_user_playbook_semantic_digest( + content_digest=_digest("old content"), trigger="refund trigger" + ), + revised_content=content, + projection=projection or _projection(content), + decision_proof=proof or _proof(optimizer_kind=optimizer_kind), + subject_epochs_json=subject_epochs_json or _subject_epochs_json(), + request_id=request_id, + ) + + +def _seed( + storage: SQLiteStorage, + *, + optimizer_kind: PublishableOptimizerKind = "gepa", +) -> tuple[UserPlaybook, PlaybookOptimizationJob]: + incumbent = _incumbent() + storage.save_user_playbooks([incumbent]) + projection = _projection() + proof = _proof(optimizer_kind=optimizer_kind) + job = storage.create_playbook_optimization_job( + _job( + target_id=incumbent.user_playbook_id, + projection_digest=projection.digest, + content_digest=projection.candidate_content_digest, + proof_digest=proof.digest, + subject_epochs_json=_subject_epochs_json(), + optimizer_kind=optimizer_kind, + ) + ) + return incumbent, job + + +def test_prepare_gepa_publication_rejects_authority_substitution_before_metadata_overwrite( + tmp_path: Path, +) -> None: + storage = _store(tmp_path) + incumbent = _incumbent() + storage.save_user_playbooks([incumbent]) + durable_authority = { + "adoption_policy": {"min_commit_score": "0.75"}, + "validation_manifest": {"windows": []}, + } + substituted_authority = { + "adoption_policy": {"min_commit_score": "0.25"}, + "validation_manifest": {"windows": []}, + } + job = storage.create_playbook_optimization_job( + PlaybookOptimizationJob( + optimizer_kind="gepa", + target_kind="user_playbook", + target_id=incumbent.user_playbook_id, + status="running", + metadata_json=_canonical( + {GEPA_PUBLICATION_AUTHORITY_METADATA_KEY: durable_authority} + ), + attempt_key="attempt-prepare", + ) + ) + candidate = storage.insert_playbook_optimization_candidate( + PlaybookOptimizationCandidate( + job_id=job.job_id, + content="new content", + aggregate_score=0.9, + is_winner=True, + ) + ) + projection = _projection("new content") + proof = _proof("substituted-proof") + incoming_metadata = _canonical( + { + GEPA_PUBLICATION_AUTHORITY_METADATA_KEY: substituted_authority, + "best_idx": 0, + } + ) + + with pytest.raises(StorageError, match="authority"): + storage.prepare_gepa_user_playbook_publication( + job_id=job.job_id, + owner="worker-a", + lease_seconds=60, + winner_candidate_id=candidate.candidate_id, + candidate_content_digest=projection.candidate_content_digest, + search_projection_digest=projection.digest, + publication_proof_digest=proof.digest, + projection_json=projection.canonical_json, + decision_proof_json=proof.canonical_json, + subject_epochs_json=storage.get_user_playbook_publication_subject_epochs( + incumbent.user_playbook_id + ), + metadata_json=incoming_metadata, + ) + + row = storage.conn.execute( + "SELECT metadata_json, stage FROM playbook_optimization_jobs WHERE job_id = ?", + (job.job_id,), + ).fetchone() + assert json.loads(row["metadata_json"])[ + GEPA_PUBLICATION_AUTHORITY_METADATA_KEY + ] == (durable_authority) + assert row["stage"] is None + + +def _subject_ref() -> str: + return governance_subject_ref(_ORG_ID, "u1", get_governance_ref_secret()) + + +def _subject_epochs_json(*, epoch: int = 0, subject_ref: str | None = None) -> str: + return _canonical( + {"subjects": [{"epoch": epoch, "ref": subject_ref or _subject_ref()}]} + ) + + +class _AcceptingVerifier: + def verify(self, request: PublicationRequest) -> None: + assert request.optimizer_kind in {"gepa", "offline_tuner_replay"} + + +def _service(storage: SQLiteStorage) -> UserPlaybookPublicationService: + return UserPlaybookPublicationService(storage, verifier=_AcceptingVerifier()) + + +def test_stage_is_hidden_from_user_playbook_reads_and_search(tmp_path: Path) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + service = _service(storage) + claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + request = _request( + job_id=job.job_id, incumbent_id=incumbent.user_playbook_id, claim=claim + ) + + service.stage(request) + staged = storage.conn.execute( + "SELECT * FROM user_playbook_publication_staging WHERE job_id = ?", + (job.job_id,), + ).fetchone() + assert staged["projection_json"] == request.projection.canonical_json + assert staged["projection_digest"] == request.projection.digest + assert staged["content_digest"] == request.projection.candidate_content_digest + + assert storage.get_user_playbooks(query="new content") == [] + assert ( + storage.search_user_playbooks( + SearchUserPlaybookRequest(user_id="u1", query="projection-token") + ) + == [] + ) + assert ( + storage.conn.execute( + "SELECT COUNT(*) AS count FROM user_playbooks WHERE content = 'new content'" + ).fetchone()["count"] + == 0 + ) + + +def test_publish_commits_exact_staged_projection_and_terminal_result( + tmp_path: Path, +) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + service = _service(storage) + claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + request = _request( + job_id=job.job_id, incumbent_id=incumbent.user_playbook_id, claim=claim + ) + + service.stage(request) + result = service.publish(request) + + assert result.outcome == "applied" + assert result.successor_user_playbook_id is not None + successors = storage.get_user_playbooks( + user_playbook_id=result.successor_user_playbook_id, + include_embedding=True, + ) + assert len(successors) == 1 + successor = successors[0] + assert successor.content == "new content" + assert successor.source == "gepa" + assert successor.embedding == [0.25] * 512 + assert successor.expanded_terms == "exact-expanded projection-token" + tombstone = storage.get_user_playbook_by_id( + incumbent.user_playbook_id, include_tombstones=True + ) + assert tombstone is not None + assert tombstone.status is Status.SUPERSEDED + assert tombstone.superseded_by == result.successor_user_playbook_id + + fts = storage.conn.execute( + "SELECT search_text FROM user_playbooks_fts WHERE rowid = ?", + (result.successor_user_playbook_id,), + ).fetchone() + assert fts["search_text"] == "exact lexical projection-token" + terminal = service.load_committed(job.job_id) + assert terminal == result + + +def test_offline_tuner_publication_persists_public_source(tmp_path: Path) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage, optimizer_kind="offline_tuner_replay") + service = _service(storage) + claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + request = _request( + job_id=job.job_id, + incumbent_id=incumbent.user_playbook_id, + claim=claim, + optimizer_kind="offline_tuner_replay", + ) + + result = service.publish(request) + + assert result.successor_user_playbook_id is not None + successor = storage.get_user_playbook_by_id(result.successor_user_playbook_id) + assert successor is not None + assert successor.source == "offline_optimizer" + staging = storage.conn.execute( + "SELECT optimizer_kind FROM user_playbook_publication_staging WHERE job_id = ?", + (job.job_id,), + ).fetchone() + assert staging["optimizer_kind"] == "offline_tuner_replay" + + +def test_publish_lost_incumbent_cas_returns_incumbent_changed_without_orphan( + tmp_path: Path, +) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + storage.archive_user_playbook_by_id("u1", incumbent.user_playbook_id) + service = _service(storage) + claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + request = _request( + job_id=job.job_id, incumbent_id=incumbent.user_playbook_id, claim=claim + ) + + result = service.publish(request) + + assert result.outcome == "incumbent_changed" + assert result.successor_user_playbook_id is None + assert ( + storage.conn.execute( + "SELECT COUNT(*) AS count FROM user_playbooks WHERE content = 'new content'" + ).fetchone()["count"] + == 0 + ) + + +@pytest.mark.parametrize( + ("update"), + [ + {"content": "human-edited content"}, + {"trigger": "human-edited trigger"}, + ], + ids=["content", "trigger"], +) +def test_publish_rejects_in_place_incumbent_semantic_change_after_staging( + tmp_path: Path, update: dict[str, str] +) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + service = _service(storage) + claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + request = _request( + job_id=job.job_id, incumbent_id=incumbent.user_playbook_id, claim=claim + ) + service.stage(request) + + storage.update_user_playbook(incumbent.user_playbook_id, **update) + result = service.publish(request) + + assert result.outcome == "incumbent_changed" + assert result.successor_user_playbook_id is None + assert ( + storage.conn.execute( + "SELECT COUNT(*) AS count FROM user_playbooks WHERE content = 'new content'" + ).fetchone()["count"] + == 0 + ) + + +def test_publication_request_rejects_projector_trigger_mutation( + tmp_path: Path, +) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + service = _service(storage) + claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + + with pytest.raises(ValueError, match="preserve incumbent trigger"): + _request( + job_id=job.job_id, + incumbent_id=incumbent.user_playbook_id, + claim=claim, + projection=_projection(trigger="malicious trigger"), + ) + + +@pytest.mark.parametrize( + ("field", "message"), + [ + ("worker_fence", "worker fence"), + ("publication_fence", "publication fence"), + ("epochs", "subject epochs"), + ("proof", "proof digest"), + ("projection", "projection digest"), + ], +) +def test_publish_rejects_changed_identity_fences_and_digests( + tmp_path: Path, field: str, message: str +) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + service = _service(storage) + claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + request = _request( + job_id=job.job_id, incumbent_id=incumbent.user_playbook_id, claim=claim + ) + service.stage(request) + + if field == "worker_fence": + bad = _request( + job_id=job.job_id, + incumbent_id=incumbent.user_playbook_id, + claim=claim, + worker_fence=4, + ) + elif field == "publication_fence": + bad = _request( + job_id=job.job_id, + incumbent_id=incumbent.user_playbook_id, + claim=PublicationClaim( + job_id=job.job_id, owner="worker-a", fence=claim.fence + 1 + ), + ) + elif field == "epochs": + bad = _request( + job_id=job.job_id, + incumbent_id=incumbent.user_playbook_id, + claim=claim, + subject_epochs_json=_canonical( + {"subjects": [{"epoch": 1, "ref": "user:u1"}]} + ), + ) + elif field == "proof": + proof = _proof(source="changed-source") + bad = _request( + job_id=job.job_id, + incumbent_id=incumbent.user_playbook_id, + claim=claim, + proof=proof, + ) + else: + embedding = ["0.5"] * 512 + payload = { + "candidate_content_digest": _digest("new content"), + "embedding": embedding, + "embedding_model_id": "test-embedding-v1", + "expanded_terms": ["changed-expanded"], + "lexical_document": "changed lexical document", + "preserved_trigger": "refund trigger", + "projector_code_digest": "a" * 64, + "projector_id": "reflexio.search.user-playbook", + "projector_version": "1", + "schema_version": "offline-tuner-candidate-search-projection-v1", + } + canonical = _canonical(payload) + projection = PublicationSearchProjection( + schema_version="offline-tuner-candidate-search-projection-v1", + canonical_json=canonical, + digest=_digest(canonical), + projector_id="reflexio.search.user-playbook", + projector_version="1", + projector_code_digest="a" * 64, + candidate_content_digest=_digest("new content"), + preserved_trigger="refund trigger", + embedding_model_id="test-embedding-v1", + embedding=tuple(embedding), + expanded_terms=("changed-expanded",), + lexical_document="changed lexical document", + ) + bad = _request( + job_id=job.job_id, + incumbent_id=incumbent.user_playbook_id, + claim=claim, + projection=projection, + ) + + with pytest.raises(StorageError, match=message): + service.publish(bad) + assert ( + storage.conn.execute( + "SELECT COUNT(*) AS count FROM user_playbooks WHERE content = 'new content'" + ).fetchone()["count"] + == 0 + ) + + +def test_stage_idempotent_for_identical_request_and_rejects_conflict( + tmp_path: Path, +) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + service = _service(storage) + claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + request = _request( + job_id=job.job_id, incumbent_id=incumbent.user_playbook_id, claim=claim + ) + + service.stage(request) + service.stage(request) + with pytest.raises(StorageError, match="staged publication conflicts"): + service.stage( + _request( + job_id=job.job_id, + incumbent_id=incumbent.user_playbook_id, + claim=claim, + content="different content", + ) + ) + + +def test_publication_claim_rejects_lease_expired_before_claim_without_mutation( + tmp_path: Path, +) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET lease_expires_at = ? WHERE job_id = ?", + (100, job.job_id), + ) + storage.conn.commit() + changes_before = storage.conn.total_changes + + with ( + patch(_EPOCH_NOW_PATCH, return_value=101), + pytest.raises(StorageError, match="lease expired"), + ): + _service(storage).claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + + assert storage.conn.total_changes == changes_before + assert ( + storage.conn.execute( + "SELECT COUNT(*) AS count FROM user_playbook_publication_claims" + ).fetchone()["count"] + == 0 + ) + persisted_incumbent = storage.get_user_playbook_by_id( + incumbent.user_playbook_id, include_tombstones=True + ) + assert persisted_incumbent is not None + assert persisted_incumbent.status is None + + +def test_publication_stage_rejects_lease_expired_after_claim_without_mutation( + tmp_path: Path, +) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET lease_expires_at = ? WHERE job_id = ?", + (101, job.job_id), + ) + storage.conn.commit() + service = _service(storage) + with patch(_EPOCH_NOW_PATCH, return_value=100): + claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + request = _request( + job_id=job.job_id, incumbent_id=incumbent.user_playbook_id, claim=claim + ) + changes_before = storage.conn.total_changes + + with ( + patch(_EPOCH_NOW_PATCH, return_value=102), + pytest.raises(StorageError, match="lease expired"), + ): + service.stage(request) + + assert storage.conn.total_changes == changes_before + assert ( + storage.conn.execute( + "SELECT COUNT(*) AS count FROM user_playbook_publication_staging" + ).fetchone()["count"] + == 0 + ) + + +def test_publication_commit_rejects_lease_expired_after_stage_without_mutation( + tmp_path: Path, +) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET lease_expires_at = ? WHERE job_id = ?", + (101, job.job_id), + ) + storage.conn.commit() + service = _service(storage) + with patch(_EPOCH_NOW_PATCH, return_value=100): + claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + request = _request( + job_id=job.job_id, + incumbent_id=incumbent.user_playbook_id, + claim=claim, + ) + service.stage(request) + changes_before = storage.conn.total_changes + + with ( + patch(_EPOCH_NOW_PATCH, return_value=102), + pytest.raises(StorageError, match="lease expired"), + ): + storage.commit_user_playbook_publication(request) + + assert storage.conn.total_changes == changes_before + assert service.load_committed(job.job_id) is None + assert ( + storage.conn.execute( + "SELECT COUNT(*) AS count FROM user_playbooks WHERE content = 'new content'" + ).fetchone()["count"] + == 0 + ) + claim_row = storage.conn.execute( + "SELECT consumed FROM user_playbook_publication_claims WHERE job_id = ?", + (job.job_id,), + ).fetchone() + assert claim_row["consumed"] == 0 + + +def test_publication_lease_is_expired_at_exact_epoch_boundary( + tmp_path: Path, +) -> None: + storage = _store(tmp_path) + _, job = _seed(storage) + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET lease_expires_at = ? WHERE job_id = ?", + (100, job.job_id), + ) + storage.conn.commit() + changes_before = storage.conn.total_changes + + with ( + patch(_EPOCH_NOW_PATCH, return_value=100), + pytest.raises(StorageError, match="lease expired"), + ): + _service(storage).claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + + assert storage.conn.total_changes == changes_before + + +def test_committed_response_loss_retry_returns_same_successor_after_lease_expiry( + tmp_path: Path, +) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + service = _service(storage) + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET lease_expires_at = ? WHERE job_id = ?", + (101, job.job_id), + ) + storage.conn.commit() + + with patch(_EPOCH_NOW_PATCH, return_value=100): + claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + request = _request( + job_id=job.job_id, + incumbent_id=incumbent.user_playbook_id, + claim=claim, + ) + first = service.publish(request) + changes_before = storage.conn.total_changes + + with patch(_EPOCH_NOW_PATCH, return_value=102): + retry = service.publish(request) + + assert retry == first + assert storage.conn.total_changes == changes_before + assert ( + storage.conn.execute( + "SELECT COUNT(*) AS count FROM user_playbooks WHERE content = 'new content'" + ).fetchone()["count"] + == 1 + ) + events = storage.get_lineage_events(entity_type="user_playbook") + assert [event.op for event in events] == ["create", "revise"] + assert ( + storage.conn.execute( + "SELECT COUNT(*) AS count FROM playbook_optimization_events WHERE job_id = ?", + (job.job_id,), + ).fetchone()["count"] + == 1 + ) + + +def test_two_concurrent_publishers_have_one_successor(tmp_path: Path) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + first_service = _service(storage) + second_storage = _store(tmp_path) + second_service = _service(second_storage) + first_claim = first_service.claim( + job_id=job.job_id, owner="worker-a", worker_fence=5 + ) + second_claim = second_service.claim( + job_id=job.job_id, owner="worker-a", worker_fence=5 + ) + first_request = _request( + job_id=job.job_id, incumbent_id=incumbent.user_playbook_id, claim=first_claim + ) + second_request = _request( + job_id=job.job_id, incumbent_id=incumbent.user_playbook_id, claim=second_claim + ) + first_service.stage(first_request) + second_service.stage(second_request) + commit_barrier = Barrier(2) + + def commit( + item: tuple[SQLiteStorage, PublicationRequest], + ) -> PublicationResult: + item_storage, request = item + commit_barrier.wait(timeout=5) + return item_storage.commit_user_playbook_publication(request) + + with ThreadPoolExecutor(max_workers=2) as executor: + results = list( + executor.map( + commit, + [(storage, first_request), (second_storage, second_request)], + ) + ) + + assert all(result.outcome == "applied" for result in results) + assert results[0] == results[1] + assert ( + storage.conn.execute( + "SELECT COUNT(*) AS count FROM user_playbooks WHERE content = 'new content'" + ).fetchone()["count"] + == 1 + ) + + +def test_erasure_barrier_added_after_staging_rejects_publication( + tmp_path: Path, +) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + service = _service(storage) + claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + request = _request( + job_id=job.job_id, incumbent_id=incumbent.user_playbook_id, claim=claim + ) + service.stage(request) + subject_ref = storage.conn.execute( + "SELECT governance_subject_ref FROM user_playbooks WHERE user_playbook_id = ?", + (incumbent.user_playbook_id,), + ).fetchone()["governance_subject_ref"] + storage.conn.execute( + """INSERT INTO subject_write_barriers + (org_id, subject_ref, purge_id, status, created_at, updated_at) + VALUES (?, ?, 'purge-publication', 'erased', 1, 1)""", + (storage.org_id, subject_ref), + ) + storage.conn.commit() + + with pytest.raises(StorageError, match="blocked by erasure barrier"): + service.publish(request) + + assert service.load_committed(job.job_id) is None + assert ( + storage.conn.execute( + "SELECT COUNT(*) AS count FROM user_playbooks WHERE content = 'new content'" + ).fetchone()["count"] + == 0 + ) + + +@pytest.mark.parametrize( + ("job_change", "message"), + [ + ({"stage": "replay_evaluated"}, "publishing"), + ({"attempt_key": "changed-attempt"}, "attempt"), + ({"optimizer_kind": "offline_tuner_replay"}, "optimizer"), + ({"target_id": 999}, "incumbent"), + ], +) +def test_publish_rejects_changed_durable_job_identity( + tmp_path: Path, job_change: dict[str, object], message: str +) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + service = _service(storage) + claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + request = _request( + job_id=job.job_id, incumbent_id=incumbent.user_playbook_id, claim=claim + ) + service.stage(request) + column, value = next(iter(job_change.items())) + storage.conn.execute( + f"UPDATE playbook_optimization_jobs SET {column} = ? WHERE job_id = ?", # noqa: S608 + (value, job.job_id), + ) + storage.conn.commit() + + with pytest.raises(StorageError, match=message): + service.publish(request) + assert ( + storage.conn.execute( + "SELECT COUNT(*) AS count FROM user_playbooks WHERE content = 'new content'" + ).fetchone()["count"] + == 0 + ) + + +def test_failure_inside_atomic_commit_rolls_back_every_visible_effect( + tmp_path: Path, +) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + service = _service(storage) + claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + request = _request( + job_id=job.job_id, incumbent_id=incumbent.user_playbook_id, claim=claim + ) + service.stage(request) + + with ( + patch( + "reflexio.server.services.storage.sqlite_storage.playbook._user._append_event_stmt", + side_effect=RuntimeError("injected publication crash"), + ), + pytest.raises(StorageError, match="injected publication crash"), + ): + service.publish(request) + + current = storage.get_user_playbook_by_id( + incumbent.user_playbook_id, include_tombstones=True + ) + assert current is not None and current.status is None + assert ( + storage.conn.execute( + "SELECT COUNT(*) AS count FROM user_playbooks WHERE content = 'new content'" + ).fetchone()["count"] + == 0 + ) + assert ( + storage.conn.execute( + "SELECT COUNT(*) AS count FROM user_playbooks_fts WHERE search_text = ?", + (request.projection.lexical_document,), + ).fetchone()["count"] + == 0 + ) + assert ( + storage.conn.execute( + "SELECT COUNT(*) AS count FROM playbook_optimization_events WHERE job_id = ?", + (job.job_id,), + ).fetchone()["count"] + == 0 + ) + assert service.load_committed(job.job_id) is None + + +def test_verifier_rejection_happens_before_hidden_staging(tmp_path: Path) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + verifier = Mock() + verifier.verify.side_effect = ValueError("proof rejected") + service = UserPlaybookPublicationService(storage, verifier=verifier) + claim = PublicationClaim(job_id=job.job_id, owner="worker-a", fence=1) + request = _request( + job_id=job.job_id, incumbent_id=incumbent.user_playbook_id, claim=claim + ) + + with pytest.raises(ValueError, match="proof rejected"): + service.publish(request) + + assert ( + storage.conn.execute( + "SELECT COUNT(*) AS count FROM user_playbook_publication_staging" + ).fetchone()["count"] + == 0 + ) + assert ( + storage.conn.execute( + "SELECT COUNT(*) AS count FROM user_playbook_publication_claims" + ).fetchone()["count"] + == 0 + ) + assert ( + storage.conn.execute( + "SELECT COUNT(*) AS count FROM user_playbook_publication_results" + ).fetchone()["count"] + == 0 + ) + + +def test_reclaimed_worker_refreshes_stage_binding_and_old_worker_is_rejected( + tmp_path: Path, +) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + service = _service(storage) + old_claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + old_request = _request( + job_id=job.job_id, + incumbent_id=incumbent.user_playbook_id, + claim=old_claim, + ) + service.stage(old_request) + + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET lease_owner = ?, lease_fence = ? WHERE job_id = ?", + ("worker-b", 6, job.job_id), + ) + storage.conn.commit() + new_claim = service.claim(job_id=job.job_id, owner="worker-b", worker_fence=6) + new_request = _request( + job_id=job.job_id, + incumbent_id=incumbent.user_playbook_id, + claim=new_claim, + worker_fence=6, + ) + + service.stage(new_request) + staged = storage.conn.execute( + "SELECT * FROM user_playbook_publication_staging WHERE job_id = ?", + (job.job_id,), + ).fetchone() + assert staged["claim_owner"] == "worker-b" + assert staged["worker_fence"] == 6 + assert staged["publication_fence"] == new_claim.fence + + with pytest.raises(StorageError, match="publication worker owner changed"): + service.publish(old_request) + + result = service.publish(new_request) + assert result.outcome == "applied" + + +def test_reclaimed_worker_cannot_change_immutable_staging_identity( + tmp_path: Path, +) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + service = _service(storage) + old_claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + service.stage( + _request( + job_id=job.job_id, + incumbent_id=incumbent.user_playbook_id, + claim=old_claim, + ) + ) + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET lease_owner = ?, lease_fence = ? WHERE job_id = ?", + ("worker-b", 6, job.job_id), + ) + storage.conn.commit() + new_claim = service.claim(job_id=job.job_id, owner="worker-b", worker_fence=6) + + with pytest.raises(StorageError, match="request identity"): + service.stage( + _request( + job_id=job.job_id, + incumbent_id=incumbent.user_playbook_id, + claim=new_claim, + worker_fence=6, + request_id="changed-request", + ) + ) + + +def test_publication_requires_incumbent_in_frozen_subject_vector( + tmp_path: Path, +) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + frozen = _subject_epochs_json(subject_ref="subject:other") + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET metadata_json = ? WHERE job_id = ?", + ( + _canonical( + { + "publication_proof_digest": _proof().digest, + "publication_subject_epochs": json.loads(frozen), + } + ), + job.job_id, + ), + ) + storage.conn.commit() + service = _service(storage) + claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + + with pytest.raises(StorageError, match="incumbent governance subject"): + service.stage( + _request( + job_id=job.job_id, + incumbent_id=incumbent.user_playbook_id, + claim=claim, + subject_epochs_json=frozen, + ) + ) + + +def test_publication_rejects_request_subject_vector_different_from_job( + tmp_path: Path, +) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + service = _service(storage) + claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + + with pytest.raises(StorageError, match="subject epochs vector"): + service.stage( + _request( + job_id=job.job_id, + incumbent_id=incumbent.user_playbook_id, + claim=claim, + subject_epochs_json=_subject_epochs_json(epoch=1), + ) + ) + + +def test_publication_rechecks_frozen_subject_vector_at_commit(tmp_path: Path) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + service = _service(storage) + claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + request = _request( + job_id=job.job_id, + incumbent_id=incumbent.user_playbook_id, + claim=claim, + ) + service.stage(request) + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET metadata_json = ? WHERE job_id = ?", + ( + _canonical( + { + "publication_proof_digest": request.decision_proof.digest, + "publication_subject_epochs": json.loads( + _subject_epochs_json(epoch=1) + ), + } + ), + job.job_id, + ), + ) + storage.conn.commit() + + with pytest.raises(StorageError, match="subject epochs vector"): + storage.commit_user_playbook_publication(request) + + +def test_publication_checks_erasure_barrier_for_every_frozen_subject( + tmp_path: Path, +) -> None: + storage = _store(tmp_path) + incumbent, job = _seed(storage) + frozen = _canonical( + { + "subjects": [ + {"epoch": 0, "ref": _subject_ref()}, + {"epoch": 2, "ref": "subject:auxiliary"}, + ] + } + ) + storage.conn.execute( + "UPDATE playbook_optimization_jobs SET metadata_json = ? WHERE job_id = ?", + ( + _canonical( + { + "publication_proof_digest": _proof().digest, + "publication_subject_epochs": json.loads(frozen), + } + ), + job.job_id, + ), + ) + storage.conn.commit() + service = _service(storage) + claim = service.claim(job_id=job.job_id, owner="worker-a", worker_fence=5) + request = _request( + job_id=job.job_id, + incumbent_id=incumbent.user_playbook_id, + claim=claim, + subject_epochs_json=frozen, + ) + service.stage(request) + storage.conn.execute( + """INSERT INTO subject_write_barriers + (org_id, subject_ref, purge_id, status, created_at, updated_at) + VALUES (?, 'subject:auxiliary', 'purge-aux', 'erased', 1, 1)""", + (storage.org_id,), + ) + storage.conn.commit() + + with pytest.raises(StorageError, match="blocked by erasure barrier"): + service.publish(request) diff --git a/tests/server/services/test_generation_service_scheduling.py b/tests/server/services/test_generation_service_scheduling.py index 5afaae589..e5bc57772 100644 --- a/tests/server/services/test_generation_service_scheduling.py +++ b/tests/server/services/test_generation_service_scheduling.py @@ -401,6 +401,76 @@ def test_neither_family_sampled_does_not_schedule_at_all( runner.assert_not_called() +@pytest.mark.parametrize( + ("retrieved_rate", "expected_sampled", "expected_schedule_count"), + [(1.0, True, 1), (0.0, False, 0)], +) +def test_publish_persists_the_retrieved_learning_sampling_decision( + service: GenerationService, + monkeypatch: pytest.MonkeyPatch, + retrieved_rate: float, + expected_sampled: bool, + expected_schedule_count: int, +) -> None: + _set_rates(service, success=0.0, retrieved=retrieved_rate) + storage = cast(MagicMock, service.storage) + scheduler = MagicMock() + monkeypatch.setattr( + "reflexio.server.services.generation_service.GroupEvaluationScheduler.get_instance", + lambda: scheduler, + ) + + service._schedule_post_publish_evaluations( + new_request=Request( + request_id="request-sampling-decision", + user_id="user_test", + session_id="session-sampling-decision", + ), + interactions=[], + user_id="user_test", + agent_version="v_test", + source=None, + ) + + storage.record_retrieved_learning_sampling_decision.assert_called_once_with( + user_id="user_test", + session_id="session-sampling-decision", + request_id="request-sampling-decision", + sampled=expected_sampled, + ) + assert scheduler.schedule.call_count == expected_schedule_count + + +def test_sampling_decision_persistence_failure_does_not_break_publish_scheduling( + service: GenerationService, monkeypatch: pytest.MonkeyPatch +) -> None: + _set_rates(service, success=0.0, retrieved=1.0) + storage = cast(MagicMock, service.storage) + storage.record_retrieved_learning_sampling_decision.side_effect = RuntimeError( + "sampling persistence unavailable" + ) + scheduler = MagicMock() + monkeypatch.setattr( + "reflexio.server.services.generation_service.GroupEvaluationScheduler.get_instance", + lambda: scheduler, + ) + + service._schedule_post_publish_evaluations( + new_request=Request( + request_id="request-sampling-failure", + user_id="user_test", + session_id="session-sampling-failure", + ), + interactions=[], + user_id="user_test", + agent_version="v_test", + source=None, + ) + + storage.record_retrieved_learning_sampling_decision.assert_called_once() + scheduler.schedule.assert_called_once() + + def test_unset_retrieved_rate_inherits_success_rate( service: GenerationService, monkeypatch: pytest.MonkeyPatch ) -> None: