From fe84bae65b7d1f724da18f593708eefcaa7ef442 Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 11:28:25 -0700 Subject: [PATCH 01/19] PR1: ecosystem abstraction (EVM + Solana front-half) Runtime Ecosystem/Language seam; the pipeline driver generalized over FeatureUnit/Main; EVM reproduces today's behavior exactly; Solana added as a second ecosystem (analysis + property extraction) proven against a null (no-verifier) backend. Built on origin/master (which already carries the command sandbox, #73). Stacked-PR 1 of 3 (eric/ecosystem -> eric/rust -> eric/crucible-app); see docs/pr-split-plan.md. Squashed from eric/crucible's final file state for this layer. Co-Authored-By: Claude Opus 4.8 (1M context) --- ARCHITECTURE.md | 284 ++++++++++ CLAUDE.md | 29 + composer/foundry/pipeline.py | 8 +- composer/pipeline/cli.py | 9 +- composer/pipeline/core.py | 171 +++--- composer/pipeline/ecosystem.py | 316 +++++++++++ composer/pipeline/ptypes.py | 16 +- composer/spec/prop_inference.py | 28 +- composer/spec/solana/__init__.py | 9 + composer/spec/solana/model.py | 274 ++++++++++ composer/spec/solana/null_backend.py | 175 ++++++ composer/spec/source/pipeline.py | 10 +- composer/spec/system_analysis.py | 18 +- composer/spec/system_model.py | 68 ++- composer/templates/solana/_failure_modes.j2 | 30 + composer/templates/solana/analysis_prompt.j2 | 74 +++ composer/templates/solana/analysis_system.j2 | 28 + .../templates/solana/instruction_context.j2 | 52 ++ composer/templates/solana/program_context.j2 | 21 + composer/templates/solana/property_prompt.j2 | 114 ++++ composer/templates/solana/property_system.j2 | 65 +++ docs/application-abstraction.md | 515 ++++++++++++++++++ docs/ecosystem-abstraction.md | 468 ++++++++++++++++ 23 files changed, 2685 insertions(+), 97 deletions(-) create mode 100644 ARCHITECTURE.md create mode 100644 CLAUDE.md create mode 100644 composer/pipeline/ecosystem.py create mode 100644 composer/spec/solana/__init__.py create mode 100644 composer/spec/solana/model.py create mode 100644 composer/spec/solana/null_backend.py create mode 100644 composer/templates/solana/_failure_modes.j2 create mode 100644 composer/templates/solana/analysis_prompt.j2 create mode 100644 composer/templates/solana/analysis_system.j2 create mode 100644 composer/templates/solana/instruction_context.j2 create mode 100644 composer/templates/solana/program_context.j2 create mode 100644 composer/templates/solana/property_prompt.j2 create mode 100644 composer/templates/solana/property_system.j2 create mode 100644 docs/application-abstraction.md create mode 100644 docs/ecosystem-abstraction.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 00000000..3627b824 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,284 @@ +# AutoProver — Architecture & High-Level Design + +> A companion to the [README](README.md). The README tells you how to *run* AutoProver; +> this document explains how it is *built* — the major subsystems, the abstractions that +> hold them together, and the data flow through a run. + +## 1. What it is + +AutoProver (internally "AI Composer") is a **multi-agent pipeline that turns a Solidity +codebase + a design document into verified formal specifications.** Given a project root, a +main contract, and a system description, it drives a fleet of LLM agents to analyze the +system, formulate properties, author CVL (Certora Verification Language) specs, and run the +Certora Prover in a feedback loop until the specs verify (or the agent gives up with a +reason). + +The same generic pipeline also powers two sibling workflows — **Foundry test generation** +and **NatSpec greenfield code generation** — which reuse the shared analysis/extraction/ +reporting machinery behind a backend protocol. + +## 2. Design philosophy + +A handful of decisions shape the whole codebase: + +- **Generic driver, pluggable backends.** One backend-agnostic driver + ([composer/pipeline/core.py](composer/pipeline/core.py)) owns the steps that are the same + for everyone (system analysis, property extraction, caching, report assembly). Anything + backend-specific (how a spec is authored and verified) is contributed through a small + protocol. Adding the Foundry backend required *no* changes to the shared steps. +- **Phase-chain immutability.** Each phase produces an immutable object that is the + constructor input to the next: `Backend → PreparedSystem → Formalizer`. The *existence* of + a `Formalizer` proves that system analysis and preparation already succeeded — ordering is + a type-level dependency, not a call-order convention, so there is no half-initialized state. +- **Agents are graphs; everything is checkpointed.** Every agent is a LangGraph state graph + built from a reusable framework ([graphcore/](graphcore/)). State, conversations, and + intermediate results are persisted to Postgres so any run can be resumed, time-traveled, + inspected, or replayed. +- **Tight LLM↔tool loops with hard validation gates.** Agents don't emit free text that is + hoped to be correct — every CVL write passes the Certora type-checker, every spec is run + through the actual prover, and a separate "judge" agent adjudicates whether a property was + legitimately handled. Invalid output is rejected at the tool boundary with actionable + feedback. +- **Deterministic, LLM-free replay for tests.** A "tape" system records real LLM responses + per task and replays them, letting the entire pipeline run end-to-end in CI with no API + calls. + +## 3. The big picture + +``` + ┌──────────────────────────────────────────────┐ + CLI / TUI │ GENERIC PIPELINE DRIVER │ + (console_autoprove, │ composer/pipeline/core.py │ + tui_autoprove) │ │ + │ │ 1. System Analysis ───────► SourceApplication│ + │ builds context │ 2. backend.prepare_system ─► PreparedSystem │ + ▼ │ 3. prepare_formalization ∥ property extraction│ + AIComposerContext │ 4. per-component formalize (parallel) │ + (LLM, RAG, prover opts, │ 5. backend-agnostic Report │ + VFS, handler factory) └───────────────┬────────────────────────────────┘ + │ │ PipelineBackend protocol + │ ┌─────────────────┼─────────────────┐ + ▼ ▼ ▼ ▼ + ┌─────────┐ Prover backend Foundry backend NatSpec workflow + │ graphcore│ (CVL specs + (.t.sol tests + (greenfield stubs + │ agent fw │ Certora Prover) forge test) + CVL) + └────┬─────┘ + │ each agent = LangGraph state graph + tools + ▼ + ┌──────────────────────────────────────────────────────────────────┐ + │ Tools: filesystem/VFS · CVL author+typecheck · prover · RAG │ + │ search · knowledge base · human-in-loop · memory │ + └──────────────────────────────────────────────────────────────────┘ + │ + ▼ + ┌──────────────────────────────────────────────────────────────────┐ + │ Persistence (Postgres): rag_db · langgraph_store_db · │ + │ langgraph_checkpoint_db · memory_tool_db · audit_db │ + └──────────────────────────────────────────────────────────────────┘ +``` + +## 4. The generic pipeline driver + +[composer/pipeline/core.py](composer/pipeline/core.py) is the spine. `run_pipeline()` executes +five steps and never inspects anything backend-specific: + +1. **System analysis** (shared). Runs `run_component_analysis` to produce a + `SourceApplication` — the contracts, components, external actors, and their interactions. + Always yields the same type regardless of backend. +2. **`backend.prepare_system(analyzed)`** — the backend's transform. The prover backend lifts + the source app into a *harnessed* application (generating harness contracts for external + dependencies); the Foundry backend is an identity transform. +3. **`prepared.prepare_formalization()` runs concurrently with property extraction.** Neither + depends on the other, so the prover's expensive AutoSetup/summary/invariant work overlaps + with per-component property inference. Property extraction fans out one agent per component, + bounded by a semaphore (`--max-concurrent`). +4. **Per-component formalization** (parallel). For each component's properties, the backend's + `Formalizer.formalize()` is invoked. Results are cached by the backend's result type. +5. **Report assembly** (shared, best-effort). The driver collects per-component verdicts via a + backend-supplied `fetch_verdicts` callback, an LLM groups properties into semantic clusters, + and a coverage check validates the result. A failure here never fails the run. + +### The backend contract + +A backend implements `PipelineBackend[P, FormT, H, A]` plus three phase objects: + +| Object | Responsibility | Prover impl | Foundry impl | +|---|---|---|---| +| `PipelineBackend` | Phase-enum map, analysis prompt, artifact store, `prepare_system` | [spec/source/pipeline.py](composer/spec/source/pipeline.py) | [foundry/pipeline.py](composer/foundry/pipeline.py) | +| `PreparedSystem` | Holds the located main contract; builds the `Formalizer` | harness-lifted app + AutoSetup/invariants | identity | +| `Formalizer` | `formalize()` one component; `fetch_verdicts`; `finalize` | `batch_cvl_generation` + prover | `batch_foundry_test_generation` + `forge test` | + +The phase enum (`CorePhases`) lets each backend label its own phases while the driver tags the +three universal ones (analysis / extraction / formalization) for the UI and the replay tapes. + +## 5. The agent framework (graphcore + workflow) + +Every LLM agent in the system is a LangGraph state graph, constructed through a reusable, +type-safe framework. + +- **[graphcore/](graphcore/)** is a standalone library (its own package, used by other Certora + products too). It provides a fluent `Builder[State, Context, Input]` that binds a model, + state type, tools, system/initial prompt templates, and an optional summarization config, + then compiles a checkpointed `StateGraph`. The agent loop is the standard + *LLM → tool calls → tool results → LLM …* cycle, terminating when the model produces a + final structured output instead of more tool calls. +- **Tools** are Pydantic models with `run()` implementations and mixins for injecting graph + state / tool-call id. Tools return either a string, a `Command` that merges into state, or an + `interrupt()` that pauses the graph for human input. +- **[composer/workflow/](composer/workflow/)** wires graphcore to this application: model + capability parsing ([llm.py](composer/workflow/llm.py) — thinking budgets, interleaved + thinking, memory-tool beta, cache control), Postgres checkpointer + store services + ([services.py](composer/workflow/services.py)), and summarization tuned for long spec runs + ([summarization.py](composer/workflow/summarization.py)). +- **[composer/io/](composer/io/)** is the execution and observability layer. `run_task` + ([multi_job.py](composer/io/multi_job.py)) wraps each schedulable unit with a handler factory, + an optional concurrency semaphore, and lifecycle hooks. Graph execution emits an immutable + event stream (Start / StateUpdate / Checkpoint / CustomUpdate / End) onto a lock-free queue; + a background drainer dispatches events to the active IO handler (console or TUI). Nested + sub-agent graphs are tracked transparently so handlers can reconstruct the full call path. + +### State, context, and editing + +- `AIComposerState` ([core/state.py](composer/core/state.py)) extends LangGraph's message state + with a virtual filesystem (VFS), validation results, and the working spec being edited. +- `AIComposerContext` ([core/context.py](composer/core/context.py)) is the run-scoped dependency + container: the bound LLM, RAG connection, prover options, and VFS materializer. +- Spec edits go through `replace_unique` ([core/edit.py](composer/core/edit.py)) — a surgical, + match-exactly-once text replacement that fails with actionable guidance on ambiguity, keeping + the model honest about what it's changing and saving tokens. + +## 6. The prover (default) backend — phase by phase + +Implemented under [composer/spec/source/](composer/spec/source/). The phases map onto the +README's Phase 0–5: + +- **System analysis** ([spec/system_analysis.py](composer/spec/system_analysis.py), + [system_model.py](composer/spec/system_model.py)) — an agent reads the design doc and source + to produce a `SourceApplication` of contracts → components, each with entry points, state + variables, interactions, and requirements. +- **Harness setup** ([spec/source/harness.py](composer/spec/source/harness.py)) — a classifier + agent categorizes external contracts (singleton / multiple / dynamic, ERC20s, interfaces) and + generates harness contracts, lifting the system to a `HarnessedApplication`. +- **AutoSetup** ([spec/source/autosetup.py](composer/spec/source/autosetup.py)) — shells out to + the [certora_autosetup/](certora_autosetup/) package to compile the project and produce a + prover `compilation_config.conf` plus summaries for known externals. +- **Custom summaries** ([spec/source/summarizer.py](composer/spec/source/summarizer.py)) — + generates CVL summaries for ERC20s and external interfaces. +- **Structural invariants** ([spec/source/struct_invariant.py](composer/spec/source/struct_invariant.py)) + — a two-agent loop: one proposes invariants, a judge accepts/rejects each (not structural / + not inductive / unlikely to hold / …). Survivors become `certora/specs/invariants.spec`, + importable by later phases. +- **Per-component property extraction** ([spec/prop_inference.py](composer/spec/prop_inference.py)) + — multi-round agent producing `PropertyFormulation`s (attack vectors, safety properties, + invariants), optionally refined interactively or against a threat model. +- **CVL generation** ([spec/cvl_generation.py](composer/spec/cvl_generation.py), + [spec/source/author.py](composer/spec/source/author.py)) — the core feedback loop. The agent + authors CVL with `put_cvl`/`edit_cvl` (type-checked on every write), runs the prover via a + `verify_spec` tool, analyzes any counterexamples, and revises. A property-feedback judge + validates coverage and adjudicates the agent's objections (e.g. "this property is vacuous + because…"). Output is a `GeneratedCVL` carrying the spec, skipped properties with reasons, + the property→rule mapping, and the final prover run link. + +### Outputs and artifacts + +`ArtifactStore` ([spec/artifacts.py](composer/spec/artifacts.py), prover subclass in +[spec/source/](composer/spec/source/)) owns the on-disk layout under the project's `certora/`: +specs in `certora/specs/`, configs in `certora/confs/`, per-spec metadata (properties, +property→rules, commentary) in `certora/properties/`, and a final `certora/ap_report/report.json`. + +## 7. Caching & resumption + +Two complementary mechanisms: + +- **Phase cache** (`--cache-ns`). `WorkflowContext` / `CacheKey` + ([spec/context.py](composer/spec/context.py)) form a hierarchical, type-parameterized cache + in the LangGraph store: `None → SourceApplication → Properties → ComponentGroup → CVLGeneration`. + Each key incorporates a hash of its inputs, so changing the project, doc, or contract + invalidates exactly the affected subtree. Repeated runs skip completed phases. +- **Checkpointing** (`--thread-id` / `--checkpoint-id`). LangGraph persists graph state after + every node to `langgraph_checkpoint_db`, enabling crash recovery and "time travel" — resuming + from any prior checkpoint, even a non-latest one. + +## 8. Prover integration + +[composer/prover/](composer/prover/) abstracts running the Certora Prover. `run_prover` +([core.py](composer/prover/core.py)) spawns `certoraRun`, streams stdout, and resolves results +through either a **local** results path or a **cloud** path ([cloud.py](composer/prover/cloud.py), +polled via job URL) depending on `ProverOptions.cloud`. Violated rules are fed to a +counterexample analyzer ([analysis.py](composer/prover/analysis.py)) whose findings go back to +the authoring agent. A callback protocol streams per-rule outcomes to the UI and the audit DB. + +## 9. Knowledge: RAG + curated KB + +- **RAG** ([composer/rag/](composer/rag/)) — the CVL manual, chunked and embedded + (`nomic-embed-text-v1.5`) into `rag_db` (pgvector, with a ChromaDB fallback). Agents query it + for CVL syntax/semantics during authoring. Read-only at runtime; rebuilt offline from the docs. +- **Curated KB** ([composer/kb/](composer/kb/)) — ~30 hand-written articles on CVL pitfalls + (vacuity traps, ghost semantics, summary misapplication…) stored in the LangGraph store and + searched semantically by symptom. Agents can also contribute new articles (`KBPut`). + +## 10. Alternate backends & workflows + +- **Foundry backend** ([composer/foundry/](composer/foundry/)) — same driver, but formalizes + properties as Solidity `.t.sol` tests verified by `forge test` instead of CVL + prover. + Verdicts come from test pass/expected-failure status. Entry points: + [cli/console_foundry.py](composer/cli/console_foundry.py), `tui_foundry.py`. +- **NatSpec** ([composer/spec/natspec/](composer/spec/natspec/)) — a *greenfield* workflow + (its own asyncio orchestrator, not the generic driver) that goes from a design doc to Solidity + interfaces, stub implementations, and CVL. A semaphore-serialized "semantic registry" + ([natspec/registry.py](composer/spec/natspec/registry.py)) lets parallel agents share and + reuse generated state fields without conflicting. + +## 11. CLI, TUI & observability + +- **Entry points** ([composer/cli/](composer/cli/)) — `console-autoprove` (headless, prints to + stdout; best for `print`/log debugging) and `tui-autoprove` (Textual UI with per-phase panels + and live prover-output logs). Both build the context and call the same pipeline; CLI args are + parsed by introspecting `Annotated` protocol definitions in [composer/input/](composer/input/). +- **TUI** ([composer/ui/](composer/ui/)) — observes the pipeline purely through the IO event + stream, rendering per-task lanes, tool calls, and streamed prover output. +- **Diagnostics** ([composer/diagnostics/](composer/diagnostics/) + [scripts/](scripts/)) — + per-phase timing/token aggregation; `snapshot_viewer.py` replays a single agent's conversation + by mnemonic; `traceDump.py` renders a full run to HTML; `autoprove_cache_explorer.py` inspects + (and edits) cached phase results and agent memories. + +## 12. Testing — deterministic replay tapes + +[composer/testing/](composer/testing/) solves end-to-end testing without an LLM. A +`TapeRecorder` captures real `AIMessage` responses **per task, in call order** (including +out-of-graph calls like CEX analysis and interleaved sub-agents). `HarnessFakeLLM` replays +them by routing on the active task id (a `ContextVar` set by `run_task`). Tapes are +human-editable Python modules with embedded JSON; smoke scenarios live in +[test_scenarios/](test_scenarios/). Recording is curated into a clean tape before use (see the +`generate-tape` and `inspect-run` skills). + +## 13. Persistence map + +All state lives in Postgres (a single `pgvector/pgvector:pg16` container provisions all five): + +| Database | Holds | +|---|---| +| `rag_db` | CVL-manual embeddings for RAG search | +| `langgraph_store_db` | LangGraph document/index store — phase cache, KB articles | +| `langgraph_checkpoint_db` | Per-node workflow checkpoints (resume / time-travel) | +| `memory_tool_db` | Hierarchical LLM context memory (per agent) | +| `audit_db` | Run history, VFS snapshots, prover results, summaries (resumption + `traceDump`) | + +## 14. Top-level packages at a glance + +| Package | Role | +|---|---| +| [composer/](composer/) | The application — pipeline, agents, backends, tools, UI | +| [graphcore/](graphcore/) | Reusable LangGraph agent-building framework (separate package) | +| [certora_autosetup/](certora_autosetup/) | Solidity project analysis, compilation, harness/conf generation (Phase 1) | +| [analyzer/](analyzer/) | Standalone counterexample analyzer (also used inline by the prover backend) | +| [sanity_analyzer/](sanity_analyzer/) | Diagnoses unsatisfiable / sanity-failed prover runs | +| [scripts/](scripts/) | Docker entry, DB/RAG setup, trace & snapshot debugging tools | +| [tests/](tests/), [test_scenarios/](test_scenarios/) | Unit tests and tape-based smoke scenarios | + +--- + +*This document is a high-level map. For runtime/setup details see [README.md](README.md) and +[AICOMPOSER_INFRA.md](AICOMPOSER_INFRA.md); for the canonical contract between the driver and a +backend, read the module docstring and protocols in +[composer/pipeline/core.py](composer/pipeline/core.py).* diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..e662afc0 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,29 @@ +# Repository conventions for coding agents + +Guidance for AI coding agents (and humans) working in this repo. Keep changes consistent +with these unless a maintainer says otherwise. + +## Python + +### Do NOT use `from __future__ import annotations` + +Never add `from __future__ import annotations` to a module. If you touch a file that has it +and it's reasonable to do so, remove it. + +Why: +- It's a dead-end feature. PEP 563 (the string-annotations behaviour this import enables) was + never made the default and has effectively been superseded by PEP 649 / PEP 749 (lazy + evaluation) from Python 3.14 on. Relying on the `__future__` behaviour is betting on a path + the language is moving away from — it will change/break under you in future versions. +- It doesn't actually solve the problem it's reached for. Stringizing *all* annotations breaks + anything that introspects them at runtime — pydantic, dataclasses, `typing.get_type_hints`, + and our own annotation-driven graph wiring (see `composer/rustapp/_llm_agent.py`, which had + to stay eager precisely because stringized `NotRequired[T]` broke pydantic unwrapping). It + trades one set of problems for a subtler set. + +What to do instead (we target Python 3.12+): +- Modern syntax works at runtime without the future import: `X | None`, `list[str]`, + `dict[str, int]`, `tuple[int, ...]`, PEP 695 generics (`class Foo[T]:` / `def f[T]()`). +- For a genuine forward reference (a name not yet defined where the annotation is evaluated — + e.g. a dataclass field typed as a class defined later in the file), quote just that one + annotation: `backend: "RustBackend"`. Quote the specific ref; don't stringize the whole module. diff --git a/composer/foundry/pipeline.py b/composer/foundry/pipeline.py index ef7c323e..2ff056c4 100644 --- a/composer/foundry/pipeline.py +++ b/composer/foundry/pipeline.py @@ -106,7 +106,7 @@ class FoundryPhase(enum.Enum): TEST_GENERATION = "test_generation" REPORT = "report" -class FoundryFormalizer(Formalizer[GeneratedFoundryTest]): +class FoundryFormalizer(Formalizer[GeneratedFoundryTest, ContractComponentInstance]): def __init__(self, conf: _ForgeRunConfig): super().__init__(GeneratedFoundryTest, "foundry") self.conf = conf @@ -138,11 +138,11 @@ async def fetch_verdicts(self, inp: ReportComponentInput[GeneratedFoundryTest]) return await _foundry_verdicts(inp) @dataclass -class FoundrySystem(PreparedSystem[GeneratedFoundryTest]): +class FoundrySystem(PreparedSystem[GeneratedFoundryTest, ContractComponentInstance]): form: FoundryFormalizer @override - async def prepare_formalization(self, run: PipelineRun) -> Formalizer[GeneratedFoundryTest]: + async def prepare_formalization(self, run: PipelineRun) -> Formalizer[GeneratedFoundryTest, ContractComponentInstance]: return self.form @dataclass @@ -166,7 +166,7 @@ async def prepare_system( self, analyzed: SourceApplication, run: PipelineRun[FoundryPhase, None] - ) -> PreparedSystem[GeneratedFoundryTest]: + ) -> PreparedSystem[GeneratedFoundryTest, ContractComponentInstance]: return FoundrySystem( main_instance( analyzed, run.source diff --git a/composer/pipeline/cli.py b/composer/pipeline/cli.py index ccdd16c5..60ac151e 100644 --- a/composer/pipeline/cli.py +++ b/composer/pipeline/cli.py @@ -20,6 +20,7 @@ CorePipelineResult ) from composer.spec.artifacts import ArtifactIdentifier +from composer.spec.system_model import FeatureUnit from composer.spec.service_host import ModelProvider from composer.spec.system_analysis import SolidityIdentifier from .core import PipelineBackend, run_pipeline @@ -113,10 +114,10 @@ class StagedPipeline: root_key: str class Continuation[P: enum.Enum, H](Protocol): - async def __call__[FormT: BackendResult, A: ArtifactIdentifier]( + async def __call__[FormT: BackendResult, A: ArtifactIdentifier, U: FeatureUnit]( self, env: ServiceHost, - backend: PipelineBackend[P, FormT, H, A] + backend: PipelineBackend[P, FormT, H, A, U] ) -> CorePipelineResult[FormT]: ... @@ -249,9 +250,9 @@ async def cli_pipeline[P: enum.Enum, H]( relative_path=init_source.relative_path ) - async def cont[FormT: BackendResult, A: ArtifactIdentifier]( + async def cont[FormT: BackendResult, A: ArtifactIdentifier, U: FeatureUnit]( env: ServiceHost, - backend: PipelineBackend[P, FormT, H, A] + backend: PipelineBackend[P, FormT, H, A, U] ) -> CorePipelineResult[FormT]: full_ctx = WorkflowContext.create( services=conns.memory, diff --git a/composer/pipeline/core.py b/composer/pipeline/core.py index 3c92a585..31afb878 100644 --- a/composer/pipeline/core.py +++ b/composer/pipeline/core.py @@ -17,10 +17,9 @@ import enum import logging from dataclasses import dataclass -from typing import Protocol +from typing import Protocol, Any, cast from abc import ABC, abstractmethod -from pydantic import BaseModel from composer.io.multi_job import TaskInfo from composer.spec.artifacts import ArtifactStore @@ -28,7 +27,7 @@ WorkflowContext, CacheKey, Properties, ComponentGroup, SourceCode ) from composer.spec.system_model import ( - SourceApplication, ContractInstance, ContractComponentInstance, AnyApplication + SourceApplication, FeatureUnit ) from composer.spec.types import PropertyFormulation, ArtifactIdentifier from composer.spec.system_analysis import run_component_analysis @@ -40,6 +39,10 @@ from composer.spec.source.report.schema import RuleName, ReportBackend from composer.spec.source.report import build as report_build from composer.spec.source.task_ids import SYSTEM_ANALYSIS_TASK_ID, REPORT_TASK_ID +# The ecosystem seam supplies the domain-specific front half (analyzed model type, prompts, +# analysis validation, unit enumeration). ``main_instance`` moved here too and is re-exported +# so existing EVM backends keep doing ``from composer.pipeline.core import main_instance``. +from composer.pipeline.ecosystem import Ecosystem, EVM, main_instance from .ptypes import ( BackendJob, BackendResult, ComponentOutcome, CorePhases, CorePipelineResult, Delivered, GaveUp, PipelineRun, SystemAnalysisSpec ) @@ -49,18 +52,22 @@ _log = logging.getLogger(__name__) @dataclass -class Formalizer[FormT: BackendResult](ABC): +class Formalizer[FormT: BackendResult, U: FeatureUnit](ABC): """Immutable, fully constructed by prepare_formalization. Carries the prover's config/resources/prover_tool/invariant-results (or nothing, for foundry) as constructor - state — never set post-hoc. `FormT: ReportableResult` is what makes the report a core step.""" + state — never set post-hoc. `FormT: ReportableResult` is what makes the report a core step. + + Generic over ``U``, the *formalized unit* type it consumes (EVM's ``ContractComponentInstance``, + a Rust backend's ``FeatureUnit``): the backend works with its concrete unit — reading its + members without casts — while the driver stays unit-agnostic.""" formalized_type: type[FormT] backend_tag: ReportBackend - + @abstractmethod async def formalize( self, label: str, - feat: ContractComponentInstance, + feat: U, props: list[PropertyFormulation], ctx: WorkflowContext[FormT], run: PipelineRun @@ -77,20 +84,22 @@ async def fetch_verdicts(self, inp: ReportComponentInput[FormT]) -> dict[RuleNam off-thread. Foundry: read straight off inp.formalized.result.""" ... - async def finalize(self, outcomes: list[ComponentOutcome[FormT]], run: PipelineRun) -> None: + async def finalize(self, outcomes: list[ComponentOutcome[FormT, U]], run: PipelineRun) -> None: """Emit any backend-specific run-level artifacts from the full outcome set (prover: components_to_prover_runs.json). Default: none.""" return None @dataclass -class PreparedSystem[FormT: BackendResult](ABC): - main: ContractInstance +class PreparedSystem[FormT: BackendResult, U: FeatureUnit](ABC): + # The located "main" unit is ecosystem-specific (EVM ContractInstance, Solana program, …); + # only the ecosystem's own ``units()`` consumes it, so the base keeps it untyped. + main: Any @abstractmethod - async def prepare_formalization(self, run: PipelineRun) -> Formalizer[FormT]: ... + async def prepare_formalization(self, run: PipelineRun) -> Formalizer[FormT, U]: ... -class PipelineBackend[P: enum.Enum, FormT: BackendResult, H, A: ArtifactIdentifier](Protocol): +class PipelineBackend[P: enum.Enum, FormT: BackendResult, H, A: ArtifactIdentifier, U: FeatureUnit](Protocol): @property def backend_guidance(self) -> str: ... @@ -106,9 +115,9 @@ def artifact_store(self) -> ArtifactStore[A, FormT]: ... async def prepare_system( self, analyzed: SourceApplication, run: PipelineRun[P, H] - ) -> PreparedSystem[FormT]: ... + ) -> PreparedSystem[FormT, U]: ... - def to_artifact_id(self, c: ContractComponentInstance) -> A: ... + def to_artifact_id(self, c: U) -> A: ... # ---- shared helpers (the de-duplicated cache keys + batch) ------------------- @@ -116,26 +125,17 @@ def PROPERTIES_KEY(nm: str): return CacheKey[None, Properties](nm) -def main_instance(app: AnyApplication, source: SourceCode) -> ContractInstance: - """Locate the application's main contract — the one whose solidity identifier matches - ``source.contract_name`` — and return a ``ContractInstance`` pointing at it. Backends call this - from ``prepare_system`` to seed the per-component loop; component analysis should already have - guaranteed the contract is present (via ``expected_main_id``).""" - for i, c in enumerate(app.contract_components): - if c.solidity_identifier == source.contract_name: - return ContractInstance(i, app) - raise ValueError(f"main contract {source.contract_name!r} not found in analyzed application") - - @dataclass -class _Batch(BackendJob): +class _Batch[U: FeatureUnit](BackendJob[U]): feat_ctx: WorkflowContext[ComponentGroup] -def _component_cache_key(c: ContractComponentInstance) -> CacheKey[Properties, ComponentGroup]: - return CacheKey(string_hash("|".join([c.app.model_dump_json(), str(c.ind), str(c._contract.ind)]))) +def _component_cache_key(c: FeatureUnit) -> CacheKey[Properties, ComponentGroup]: + # ``cache_material`` is the ecosystem-agnostic view of what identifies a unit; EVM's + # implementation reproduces the previous inline key (app JSON | ind | contract ind) exactly. + return CacheKey(string_hash(c.cache_material())) -def _batch_cache_key[FormT: BaseModel](props: list[PropertyFormulation]) -> CacheKey[ComponentGroup, FormT]: +def _batch_cache_key(props: list[PropertyFormulation]) -> CacheKey[ComponentGroup, Any]: return CacheKey(string_hash("|".join(p.model_dump_json() for p in props))) @@ -147,28 +147,33 @@ def formalize_task_id(idx: int) -> str: return f"formalize-{idx}" # ---- the driver -------------------------------------------------------------- -async def run_pipeline[P: enum.Enum, FormT: BackendResult, H, A: ArtifactIdentifier]( - backend: PipelineBackend[P, FormT, H, A], +async def run_pipeline[P: enum.Enum, FormT: BackendResult, H, A: ArtifactIdentifier, U: FeatureUnit]( + backend: PipelineBackend[P, FormT, H, A, U], run: PipelineRun[P, H], *, interactive: bool = False, threat_model: Document | None = None, max_bug_rounds: int = 3, + ecosystem: Ecosystem[Any, Any, Any] = EVM, ) -> CorePipelineResult[FormT]: + # ``ecosystem`` supplies the domain-specific front half; it defaults to ``EVM``, which + # reproduces the previous hardcoded Solidity behavior exactly, so EVM callers (and cli.py) + # need pass nothing. Non-EVM backends (e.g. the Rust/Crucible backend) pass ``ecosystem=SOLANA``. spec, phases = backend.analysis_spec, backend.core_phases source = run.source - # 1. System analysis (shared primitive, backend-parameterized; always yields SourceApplication). + # 1. System analysis (shared primitive; the ecosystem supplies the analyzed model type, + # prompts, validation, and front-matter — EVM reproduces prior behavior exactly). analyzed = await run.runner( TaskInfo(SYSTEM_ANALYSIS_TASK_ID, "System Analysis", phases["analysis"]), lambda: run_component_analysis( - ty=SourceApplication, child_ctxt=run.ctx.child(CacheKey(spec.analysis_key)), - input=source, env=run.env, extra_input=[ - f"The main entry point of this application has been explicitly identified as {source.contract_name} at relative path {source.relative_path}. " - "Your output MUST contain an explicit contract instance with this solidity identifier.", - *spec.extra_input - ], + ty=ecosystem.system_model, child_ctxt=run.ctx.child(CacheKey(spec.analysis_key)), + input=source, env=run.env, + extra_input=[*ecosystem.analysis_extra_input(source), *spec.extra_input], expected_main_id=source.contract_name, + system_template=ecosystem.analysis_prompts.system, + initial_template=ecosystem.analysis_prompts.initial, + validate=ecosystem.validate_analysis, ), ) if analyzed is None: @@ -181,17 +186,22 @@ async def run_pipeline[P: enum.Enum, FormT: BackendResult, H, A: ArtifactIdentif # this preserves the prover's autosetup ∥ bug-analysis overlap, generically. formalizer_task = asyncio.create_task(prepared.prepare_formalization(run)) - batches = await _extract_all( - backend.analysis_spec.properties_key, - prepared.main, backend.backend_guidance, run, - phases["extraction"], interactive, threat_model, max_bug_rounds + # Extraction yields ``FeatureUnit`` batches (the ecosystem is invariant in its unit type and + # callers pass a concrete chain, so it can't be tied to the backend's ``U`` at the signature). + # The paired backend guarantees these units are its ``U``; widen once, here, honestly. + batches: list[_Batch[U]] = cast( + "list[_Batch[U]]", + await _extract_all( + backend.analysis_spec.properties_key, + prepared.main, backend.backend_guidance, run, + phases["extraction"], interactive, threat_model, max_bug_rounds, ecosystem), ) formalizer = await formalizer_task if not batches: raise ValueError("No properties extracted from any component.") # 4. Per-component formalization. Caching is core-owned, keyed by the backend's result type. - async def _run(batch: _Batch) -> ComponentOutcome[FormT]: + async def _run(batch: _Batch[U]) -> ComponentOutcome[FormT, U]: result_key = backend.to_artifact_id(batch.feat) backend.artifact_store.write_properties(result_key, batch.props) child : WorkflowContext[FormT] = await batch.feat_ctx.child( @@ -200,11 +210,11 @@ async def _run(batch: _Batch) -> ComponentOutcome[FormT]: cached_result: FormT | None = await child.cache_get(formalizer.formalized_type) result : FormT | GaveUp if cached_result is None: - label = f"{batch.feat.component.name} ({len(batch.props)} properties)" + label = f"{batch.feat.display_name} ({len(batch.props)} properties)" result : FormT | GaveUp = await run.runner( TaskInfo( - formalize_task_id(batch.feat.ind), - f"{batch.feat.component.name} ({len(batch.props)} properties)", + formalize_task_id(batch.feat.unit_index), + label, phases["formalization"] ), lambda: formalizer.formalize(label, batch.feat, batch.props, child, run), @@ -232,7 +242,7 @@ async def _run(batch: _Batch) -> ComponentOutcome[FormT]: # never fails the run. inputs = [ ReportComponentInput( - name=o.feat.component.name, + name=o.feat.display_name, props=o.props, formalized=o.result if isinstance(o.result, Delivered) else None, ) @@ -256,32 +266,67 @@ async def _run(batch: _Batch) -> ComponentOutcome[FormT]: async def _extract_all[P: enum.Enum, H]( prop_key: str, - main: ContractInstance, backend_guidance: str, run: PipelineRun[P, H], + main: Any, backend_guidance: str, run: PipelineRun[P, H], phase: P, interactive: bool, threat_model: Document | None, max_rounds: int, -) -> list[_Batch]: + ecosystem: Ecosystem[Any, Any, Any], +) -> list[_Batch[FeatureUnit]]: prop_ctx = run.ctx.child(PROPERTIES_KEY(prop_key)) - async def _one(idx: int) -> _Batch | None: - feat = ContractComponentInstance(_contract=main, ind=idx) - feat_ctx = await prop_ctx.child(_component_cache_key(feat), - {"component": feat.component.model_dump()}) + async def _unit_ctx(unit: FeatureUnit) -> WorkflowContext[ComponentGroup]: + return await prop_ctx.child(_component_cache_key(unit), unit.context_tag()) + + async def _extract(unit: FeatureUnit) -> tuple[WorkflowContext[ComponentGroup], list[PropertyFormulation]]: + ctx = await _unit_ctx(unit) props = await run.runner( - TaskInfo(extract_task_id(idx), feat.component.name, phase), + TaskInfo(extract_task_id(unit.unit_index), unit.display_name, phase), lambda conv: run_property_inference( - feat_ctx, run.env, feat, refinement=conv if interactive else None, - threat_model=threat_model, max_rounds=max_rounds, backend_guidance=backend_guidance), + ctx, run.env, unit, refinement=conv if interactive else None, + threat_model=threat_model, max_rounds=max_rounds, backend_guidance=backend_guidance, + system_template=ecosystem.property_prompts.system, + initial_template=ecosystem.property_prompts.initial), ) - return _Batch(feat, props, feat_ctx) if props else None + return ctx, props + + # Both strategies reduce to a list of (unit, properties, unit-context) triples; the ecosystem + # differs only in the *fan direction*. Global extraction infers once over the whole program + # and fans each property into its own unit (one prop per batch); per-component infers per unit + # (concurrently) and keeps that unit's properties grouped. The ``_Batch`` build below is shared. + # (Extraction is unit-agnostic — over the ``FeatureUnit`` protocol; run_pipeline's single cast + # reunites the batches with the paired backend's concrete unit type ``U``.) + triples: list[tuple[FeatureUnit, list[PropertyFormulation], WorkflowContext[ComponentGroup]]] + if ecosystem.global_extraction: + assert ecosystem.extraction_unit is not None + ext_unit = ecosystem.extraction_unit(main) + ext_ctx, props = await _extract(ext_unit) + if ecosystem.collapse_units: + # One whole-program batch: every invariant is formalized into a single harness + run + # (docs/crucible-unit-granularity.md §3). Empty props → no batch (nothing to formalize). + triples = [(ext_unit, props, ext_ctx)] if props else [] + else: + assert ecosystem.property_unit is not None + triples = [] + for i, prop in enumerate(props): + unit = ecosystem.property_unit(main, prop, i) + triples.append((unit, [prop], await _unit_ctx(unit))) + else: + units = ecosystem.units(main) + extracted = await asyncio.gather(*[_extract(u) for u in units]) + triples = [(u, props, ctx) for u, (ctx, props) in zip(units, extracted) if props] - got = await asyncio.gather(*[_one(i) for i in range(len(main.contract.components))]) - return [b for b in got if b is not None] + return [_Batch(unit, props, ctx) for unit, props, ctx in triples] -def _tally[FormT: BackendResult](outcomes: list[ComponentOutcome[FormT]]) -> CorePipelineResult[FormT]: +def _tally[FormT: BackendResult, U: FeatureUnit]( + outcomes: list[ComponentOutcome[FormT, U]] +) -> CorePipelineResult[FormT]: failures: list[str] = [] for o in outcomes: if isinstance(o.result, BaseException): - failures.append(f"{o.feat.component.name}: {o.result}") + failures.append(f"{o.feat.display_name}: {o.result}") elif isinstance(o.result, GaveUp): - failures.append(f"{o.feat.component.name}: GAVE_UP: {o.result.reason}") - return CorePipelineResult(len(outcomes), sum(len(o.props) for o in outcomes), outcomes, failures) + failures.append(f"{o.feat.display_name}: GAVE_UP: {o.result.reason}") + # The rollup is unit-agnostic; widen the concrete-unit outcomes to the protocol for storage. + return CorePipelineResult( + len(outcomes), sum(len(o.props) for o in outcomes), + cast(list[ComponentOutcome[FormT, FeatureUnit]], outcomes), failures, + ) diff --git a/composer/pipeline/ecosystem.py b/composer/pipeline/ecosystem.py new file mode 100644 index 00000000..2feedfef --- /dev/null +++ b/composer/pipeline/ecosystem.py @@ -0,0 +1,316 @@ +"""The ecosystem seam — the *front half* of the pipeline made parametric over the +blockchain/source domain (see ``docs/ecosystem-abstraction.md``). + +An **ecosystem** bundles everything the shared analysis + property-extraction steps need +that is domain-specific: the system-model type they produce, the analysis/property prompt +templates, connectivity validation, the main-unit locator, and the per-unit enumeration. +It factors into a **language** facet (the conventions for reading the *analyzed* program's +source — Solidity, Rust — shared across chains that use the same language) and the **chain** +facet (the platform model + prompts). The language here is that of the *code under analysis*, +not the language the AutoProver backend is implemented in (see :class:`Language`). + +Phase 1 introduces the seam and captures today's behavior as ``EVM = SOLIDITY ⊕ evm`` — +a *move*, not a rewrite: the existing `SourceApplication`, prompt templates, +`_validate_connectivity`, `main_instance`, and unit enumeration become the EVM ecosystem's +members. The driver defaults to ``EVM``, so existing applications are unchanged. Solana / +Soroban chains and the prompt-fragment split land in later phases. +""" + +from dataclasses import dataclass +from typing import Any, Callable, Literal + +from composer.spec.context import SourceCode +from composer.spec.code_explorer import CODE_EXPLORER_SYS_PROMPT +from composer.spec.system_analysis import _validate_connectivity +from composer.spec.system_model import ( + AnyApplication, + BaseApplication, + ContractComponentInstance, + ContractInstance, + FeatureUnit, + SolidityIdentifier, + SourceApplication, +) +from composer.spec.solana.model import ( + SolanaApplication, + SolanaInstructionInstance, + SolanaInvariantUnit, + SolanaProgramInstance, +) +from composer.spec.types import PropertyFormulation +from composer.spec.util import FS_FORBIDDEN_READ + +LanguageTag = Literal["solidity", "rust"] +ChainTag = Literal["evm", "solana", "soroban"] + + +@dataclass(frozen=True) +class PromptPair: + """A (system prompt, initial prompt) template-name pair for one agent.""" + + system: str + initial: str + + +@dataclass(frozen=True) +class Language: + """The language of the **code being analyzed** — a facet of the ecosystem, shared by every + chain whose programs are written in it (e.g. the ``rust`` facet is shared by Solana and + Soroban). It drives how the shared front half *reads* the target's source (fs-exclusion + pattern, code-explorer prompt, failure modes). + + This is emphatically **not** the language the AutoProver backend is *implemented* in: a + backend implemented as a Rust wheel (:mod:`composer.rustapp`) may analyze Solidity + (``echoprover`` → EVM) or Rust (Crucible → Solana). The implementation language is not + associated with the ecosystem; only the analyzed-source language is. + + Its members are captured here for the seam; consumers (the entry point's ``forbidden_read``, + the ``code_explorer`` prompt) are rewired to read from it in a later phase, when a + non-Solidity analyzed language first needs them.""" + + name: LanguageTag + default_forbidden_read: str + code_explorer_prompt: str + # The j2 partial with this language's failure modes (overflow, panics, …). Reserved for + # the prompt-fragment split; unused while prompts are still monolithic. + failure_modes_partial: str | None = None + + +@dataclass(frozen=True) +class Ecosystem[App: BaseApplication, Main, Unit: FeatureUnit]: + """A resolved ecosystem = a chain that carries its language. The driver consumes it to + drive the shared front half without hardcoding any one domain. + + Generic over ``App`` (the analyzed system-model type), ``Main`` (the located main-unit + wrapper), and ``Unit`` (the per-unit item the extraction phase iterates). A backend is + paired with an ecosystem by these types: ``run_pipeline`` ties + ``PipelineBackend[..., App, Main, Unit]`` to ``Ecosystem[App, Main, Unit]``, so the analyzed + model, the main-unit, and the per-unit values flow through without casts. EVM binds + ``(SourceApplication, ContractInstance, ContractComponentInstance)``; Solana binds its own.""" + + name: ChainTag + language: Language + #: The pydantic model the analysis phase produces. + system_model: type[App] + #: Prompts for the system-analysis agent. + analysis_prompts: PromptPair + #: Prompts for the per-component property-inference agent. + property_prompts: PromptPair + #: Connectivity/shape validation of the analyzed model (retry feedback on failure). + #: Typed over ``BaseApplication`` (not ``App``): the validator receives the produced model + #: and narrows internally (as ``_validate_connectivity`` does), and this keeps it assignable + #: to ``run_component_analysis``'s ``validate`` parameter without a contravariance clash. + validate_analysis: Callable[[BaseApplication, SolidityIdentifier | None], str | None] + #: Locate the target unit (the "main contract"/program) in the analyzed model. + locate_main: Callable[[App, SourceCode], Main] + #: Enumerate the per-unit items the extraction phase infers properties for. + units: Callable[[Main], list[Unit]] + #: Domain-specific front-matter appended to the analysis input (was hardcoded in the driver). + analysis_extra_input: Callable[[SourceCode], list[str | dict]] + + # -- Extraction strategy (docs/crucible-unit-granularity.md) ------------------------- + #: When True, the driver runs ONE whole-program property extraction (context = + #: ``extraction_unit(main)``) and fans each resulting property out into its own unit + #: via ``property_unit`` — one harness + verdict per property. When False (the EVM + #: default) it extracts per ``units(main)`` (one batch per component). Solana uses + #: global extraction so the fuzzer gets whole-program invariants, one run per invariant. + global_extraction: bool = False + #: The whole-program context the global extraction reads (only used when global_extraction). + extraction_unit: Callable[[Main], FeatureUnit] | None = None + #: Build the per-property unit from (main, property, index) (only used when global_extraction + #: and NOT collapse_units). + property_unit: Callable[[Main, PropertyFormulation, int], FeatureUnit] | None = None + #: When True (with global_extraction), keep all extracted properties in a SINGLE batch on the + #: whole-program ``extraction_unit`` — one formalization, one harness, one run covering every + #: invariant (docs/crucible-unit-granularity.md §3; prototype of the "single whole-program + #: unit" collapse). When False, fan each property into its own ``property_unit``. + collapse_units: bool = False + + +# --------------------------------------------------------------------------- +# main-unit location (moved out of pipeline.core; re-exported there for callers) +# --------------------------------------------------------------------------- + + +def main_instance(app: AnyApplication, source: SourceCode) -> ContractInstance: + """Locate the application's main contract — the one whose solidity identifier matches + ``source.contract_name`` — and return a ``ContractInstance`` pointing at it. Backends call + this from ``prepare_system`` to seed the per-component loop; component analysis should + already have guaranteed the contract is present (via ``expected_main_id``).""" + for i, c in enumerate(app.contract_components): + if c.solidity_identifier == source.contract_name: + return ContractInstance(i, app) + raise ValueError(f"main contract {source.contract_name!r} not found in analyzed application") + + +# --------------------------------------------------------------------------- +# The EVM ecosystem (= today's behavior) +# --------------------------------------------------------------------------- + + +def _evm_units(main: ContractInstance) -> list[ContractComponentInstance]: + return [ + ContractComponentInstance(_contract=main, ind=i) + for i in range(len(main.contract.components)) + ] + + +def _evm_analysis_extra_input(source: SourceCode) -> list[str | dict]: + return [ + f"The main entry point of this application has been explicitly identified as " + f"{source.contract_name} at relative path {source.relative_path}. " + "Your output MUST contain an explicit contract instance with this solidity identifier." + ] + + +SOLIDITY = Language( + name="solidity", + default_forbidden_read=FS_FORBIDDEN_READ, + code_explorer_prompt=CODE_EXPLORER_SYS_PROMPT, +) + +EVM: Ecosystem[SourceApplication, ContractInstance, ContractComponentInstance] = Ecosystem( + name="evm", + language=SOLIDITY, + system_model=SourceApplication, + analysis_prompts=PromptPair( + "application_analysis_system.j2", "application_analysis_prompt.j2" + ), + property_prompts=PromptPair( + "property_analysis_system_prompt.j2", "property_analysis_prompt.j2" + ), + validate_analysis=_validate_connectivity, + locate_main=main_instance, + units=_evm_units, + analysis_extra_input=_evm_analysis_extra_input, +) + + +# --------------------------------------------------------------------------- +# The RUST language facet (shared by Solana and, later, Soroban) +# --------------------------------------------------------------------------- + +#: Cargo/Anchor project layout: hide build output, VCS, lockfiles, and the JS side; keep the +#: crate sources and `tests/`. (Contrast the Foundry-shaped ``FS_FORBIDDEN_READ``.) +RUST_FORBIDDEN_READ = r"(^target/.*)|(^\.git.*)|(^node_modules/.*)|(.*\.lock$)" +# NOTE: the confined-build scratch dirs (``.sandbox_cargo`` / ``.sandbox_rustup`` / +# ``.sandbox_tmp`` and nested ``target/``) are also excluded, but that extension lives with the +# rust-framework layer that introduces confined Rust builds — no build runs in this front-half, so +# those dirs never exist here. + +RUST_CODE_EXPLORER_PROMPT = """\ +You are a code-exploration assistant analyzing Rust source for on-chain programs (e.g. Solana +/ Anchor). You have file tools (list_files, get_file, grep_files) to explore the project. +Answer the question concretely, citing the relevant items: instruction handlers, account +validation structs (e.g. Anchor `#[derive(Accounts)]`), account/state types, PDA seed +derivations, signer/owner checks, and cross-program invocations. Quote the exact Rust snippets +that establish or omit a check; do not speculate about code you have not read. +""" + +RUST = Language( + name="rust", + default_forbidden_read=RUST_FORBIDDEN_READ, + code_explorer_prompt=RUST_CODE_EXPLORER_PROMPT, + failure_modes_partial="rust/_failure_modes.j2", +) + + +# --------------------------------------------------------------------------- +# The Solana chain (RUST ⊕ solana) +# --------------------------------------------------------------------------- + + +def _solana_validate(app: BaseApplication, expected_main: SolidityIdentifier | None) -> str | None: + """Connectivity/shape validation for a ``SolanaApplication`` (retry feedback on failure). + Mirrors the EVM ``_validate_connectivity`` structure: unique program identifiers, unique + instruction slugs within a program, the expected main program present, CPI targets known.""" + if not isinstance(app, SolanaApplication): + return None + errors: list[str] = [] + known_programs: set[str] = set() + for prog in app.programs: + if prog.program_identifier in known_programs: + errors.append(f"Duplicate program identifier: {prog.program_identifier}") + known_programs.add(prog.program_identifier) + slug_origin: dict[str, str] = {} + from composer.spec.util import slugify_filename + + for ins in prog.instructions: + slug = slugify_filename(ins.name) + if slug in slug_origin: + errors.append( + f"Instructions {slug_origin[slug]!r} and {ins.name!r} in {prog.name} " + f"reduce to the same filename slug {slug!r}; give them more-distinct names." + ) + slug_origin[slug] = ins.name + # CPI targets may be well-known external programs (SPL Token, System, …) + # that are not declared in the model; we do not flag those. A future + # policy can require known_programs | known_authorities | an allowlist. + if expected_main is not None and expected_main not in known_programs: + errors.append( + f"Expected a program with identifier {expected_main!r}; declared programs: " + f"{sorted(known_programs) or '(none)'}." + ) + if not errors: + return None + if len(errors) == 1: + return errors[0] + return "Multiple validation errors; fix all before resubmitting:\n" + "\n".join(f"- {e}" for e in errors) + + +def _solana_locate_main(app: SolanaApplication, source: SourceCode) -> SolanaProgramInstance: + for i, prog in enumerate(app.programs): + if prog.program_identifier == source.contract_name: + return SolanaProgramInstance(i, app) + raise ValueError(f"main program {source.contract_name!r} not found in analyzed application") + + +def _solana_units(main: SolanaProgramInstance) -> list[SolanaInstructionInstance]: + # Per-instruction units — unused under global extraction (kept for a per-instruction + # fallback and to satisfy the ecosystem's Unit type). + return [SolanaInstructionInstance(i, main) for i in range(len(main.program.instructions))] + + +def _solana_extraction_unit(main: SolanaProgramInstance) -> SolanaProgramInstance: + # The whole program is the extraction context; SolanaProgramInstance is itself a + # FeatureUnit, so the driver reads it directly to propose whole-program invariants. + return main + + +def _solana_property_unit( + main: SolanaProgramInstance, prop: PropertyFormulation, ind: int +) -> SolanaInvariantUnit: + return SolanaInvariantUnit(ind, main, prop) + + +def _solana_analysis_extra_input(source: SourceCode) -> list[str | dict]: + return [ + f"The main program of this application has been explicitly identified as " + f"{source.contract_name} at relative path {source.relative_path}. " + "Your output MUST contain a program whose program_identifier is this exact identifier." + ] + + +SOLANA: Ecosystem[SolanaApplication, SolanaProgramInstance, SolanaInstructionInstance] = Ecosystem( + name="solana", + language=RUST, + system_model=SolanaApplication, + analysis_prompts=PromptPair("solana/analysis_system.j2", "solana/analysis_prompt.j2"), + property_prompts=PromptPair("solana/property_system.j2", "solana/property_prompt.j2"), + validate_analysis=_solana_validate, + locate_main=_solana_locate_main, + units=_solana_units, + analysis_extra_input=_solana_analysis_extra_input, + # Fuzzing wants whole-program invariants — docs/crucible-unit-granularity.md. Prototype: + # collapse all invariants into ONE whole-program harness + run (§3), instead of one per + # invariant, now that finding-level attribution recovers which property a counterexample hits. + global_extraction=True, + extraction_unit=_solana_extraction_unit, + property_unit=_solana_property_unit, + collapse_units=True, +) + + +#: Registry of available ecosystems, keyed by chain tag. Heterogeneous in ``App``/``Main``/``Unit`` +#: (each chain has its own model), hence ``Ecosystem[Any, Any, Any]``. +ECOSYSTEMS: dict[ChainTag, Ecosystem[Any, Any, Any]] = {"evm": EVM, "solana": SOLANA} diff --git a/composer/pipeline/ptypes.py b/composer/pipeline/ptypes.py index e665c790..43f65fda 100644 --- a/composer/pipeline/ptypes.py +++ b/composer/pipeline/ptypes.py @@ -13,7 +13,7 @@ ) from composer.spec.service_host import ServiceHost from composer.spec.system_model import ( - ContractComponentInstance + FeatureUnit, ) from composer.spec.types import PropertyFormulation, FormalResult from composer.spec.source.report.collect import ReportableResult @@ -71,8 +71,12 @@ class SystemAnalysisSpec: @dataclass -class BackendJob: - feat: ContractComponentInstance +class BackendJob[U: FeatureUnit]: + # ``U`` is the *formalized unit* type the paired backend consumes (EVM's + # ``ContractComponentInstance``, Solana's invariant unit, …) — see ``FeatureUnit``. The shared + # driver is generic over it; a backend narrows it to its concrete unit and reads its members + # without casts. + feat: U props: list[PropertyFormulation] @dataclass(frozen=True) @@ -94,14 +98,16 @@ def run_link(self) -> str | None: return self.result.output_link @dataclass -class ComponentOutcome[FormT: BackendResult](BackendJob): +class ComponentOutcome[FormT: BackendResult, U: FeatureUnit](BackendJob[U]): result: Delivered[FormT] | GaveUp | BaseException @dataclass class CorePipelineResult[FormT: BackendResult]: + # The result rollup is unit-agnostic — it only reads ``feat.display_name`` (a ``FeatureUnit`` + # member) — so it stays mono in the unit type and widens the outcomes to the protocol. n_components: int n_properties: int - outcomes: list[ComponentOutcome[FormT]] + outcomes: list[ComponentOutcome[FormT, FeatureUnit]] failures: list[str] @property diff --git a/composer/spec/prop_inference.py b/composer/spec/prop_inference.py index 3edc92c0..635137ab 100644 --- a/composer/spec/prop_inference.py +++ b/composer/spec/prop_inference.py @@ -21,7 +21,7 @@ from composer.spec.context import WorkflowContext, CacheKey, ComponentGroup from composer.spec.graph_builder import bind_standard, run_to_completion from composer.spec.types import PropertyFormulation -from composer.spec.system_model import ContractComponentInstance +from composer.spec.system_model import FeatureUnit from composer.tools.thinking import RoughDraftState, get_rough_draft_tools from composer.spec.service_host import Sort, ServiceHost from composer.io.conversation import ConversationContextProvider @@ -101,13 +101,14 @@ class RefinementState(MessagesState): def _get_initial_prompt( - context: ContractComponentInstance, + context: FeatureUnit, sort: Sort, prev_results: list[_AgentRoundResult], backend_guidance: str, + template: str = "property_analysis_prompt.j2", ) -> str: return load_jinja_template( - "property_analysis_prompt.j2", + template, context=context, backend_guidance=backend_guidance, sort=sort, @@ -279,12 +280,14 @@ def validate(_state: Any, result: _AgentRoundResult) -> str | None: async def _run_bug_round( env: ServiceHost, - component: ContractComponentInstance, + component: FeatureUnit, front_matter_items: Sequence[str | dict], ctx: WorkflowContext[_AgentResult], round: int, prev: list[_AgentRoundResult], backend_guidance: str, + system_template: str = "property_analysis_system_prompt.j2", + initial_template: str = "property_analysis_prompt.j2", ) -> _AgentRoundWithHistory: round_ctx = ctx.child(agent_round_key(round)) if (cached := await round_ctx.cache_get(_AgentRoundWithHistory)) is not None: @@ -305,13 +308,13 @@ class ST(MessagesState, RoughDraftState): ).with_input( BugAnalysisInput ).with_initial_prompt( - _get_initial_prompt(component, env.sort, prev, backend_guidance) + _get_initial_prompt(component, env.sort, prev, backend_guidance, initial_template) ).with_tools( get_rough_draft_tools(ST) ).with_tools( env.analysis_tools ).with_sys_prompt_template( - "property_analysis_system_prompt.j2", sort=env.sort + system_template, sort=env.sort ).compile_async() flow_input: BugAnalysisInput = BugAnalysisInput( @@ -341,11 +344,13 @@ class ST(MessagesState, RoughDraftState): async def _run_bug_analysis_inner( agent_component_analysis: WorkflowContext[_AgentResult], env: ServiceHost, - component: ContractComponentInstance, + component: FeatureUnit, extra_input: Sequence[str | dict], threat_model: Document | None, max_rounds: int, backend_guidance: str, + system_template: str = "property_analysis_system_prompt.j2", + initial_template: str = "property_analysis_prompt.j2", ) -> _AgentResult: if (cached := await agent_component_analysis.cache_get(_AgentResult)) is not None: return cached @@ -367,7 +372,7 @@ async def _run_bug_analysis_inner( for i in range(0, max_rounds): next_result = await _run_bug_round( env, component, front_matter_items, agent_component_analysis, i, prev_rounds, - backend_guidance, + backend_guidance, system_template, initial_template, ) if len(next_result.items) == 0: assert last_round_convo is not None @@ -389,12 +394,15 @@ async def _run_bug_analysis_inner( async def run_property_inference( ctx: WorkflowContext[ComponentGroup], env: ServiceHost, - component: ContractComponentInstance, + component: FeatureUnit, extra_input : Sequence[str | dict] = tuple(), threat_model: Document | None = None, refinement: ConversationContextProvider | None = None, max_rounds: int = 3, backend_guidance: str = CERTORA_BACKEND_GUIDANCE, + *, + system_template: str = "property_analysis_system_prompt.j2", + initial_template: str = "property_analysis_prompt.j2", ) -> list[PropertyFormulation]: """ Extract security properties for a component. @@ -419,6 +427,8 @@ async def run_property_inference( threat_model, max_rounds=max_rounds, backend_guidance=backend_guidance, + system_template=system_template, + initial_template=initial_template, ) if refinement is None: to_ret = agent_attempt.items diff --git a/composer/spec/solana/__init__.py b/composer/spec/solana/__init__.py new file mode 100644 index 00000000..63f53c3d --- /dev/null +++ b/composer/spec/solana/__init__.py @@ -0,0 +1,9 @@ +"""Solana ecosystem: system model + (in the ecosystem module) prompts and wiring. + +The Solana chain of the ecosystem abstraction (see docs/ecosystem-abstraction.md). This +package holds the Solana-native system model the shared analysis phase produces +(``SolanaApplication``) and the index-wrapper instances the driver iterates +(``SolanaProgramInstance`` / ``SolanaInstructionInstance``, the latter satisfying the +``FeatureUnit`` protocol). The ecosystem object that binds these + the Rust language facet + +the Solana prompts lives in ``composer/pipeline/ecosystem.py``. +""" diff --git a/composer/spec/solana/model.py b/composer/spec/solana/model.py new file mode 100644 index 00000000..7e2a8d1b --- /dev/null +++ b/composer/spec/solana/model.py @@ -0,0 +1,274 @@ +"""The Solana system model — the standalone analog of the EVM ``SourceApplication``. + +Where the EVM model is contracts → components with storage variables and external functions, +Solana is **programs → instructions** that operate on **accounts passed in by the caller** +(there is no per-contract owned storage; state lives in accounts the instruction validates +and mutates). The model captures that shape natively — accounts + their signer/owner/PDA +constraints, cross-program invocations (CPIs), and the authorities involved — rather than +reusing the EVM field names. + +``SolanaApplication`` is what the shared analysis phase produces (it is a ``BaseApplication`` +so ``run_component_analysis`` accepts it). ``SolanaProgramInstance`` / ``SolanaInstructionInstance`` +are the driver's ``Main`` / ``Unit``: thin index wrappers over the model, with the instruction +instance satisfying the ecosystem-agnostic ``FeatureUnit`` protocol so the shared driver's +cache keys / task ids / labels work unchanged. +""" + +from dataclasses import dataclass +from functools import cached_property +from typing import Literal + +from pydantic import BaseModel, Field + +from composer.spec.system_model import BaseApplication +from composer.spec.types import PropertyFormulation +from composer.spec.util import slugify_filename + +#: How an account is expected to be supplied to an instruction. Drives the "missing signer / +#: owner check" and "account substitution" reasoning in the property prompt. +AccountRole = Literal["signer", "writable", "readonly", "pda", "program", "sysvar"] + + +class AccountConstraint(BaseModel): + """One account an instruction expects in its accounts context, plus the constraints the + program is responsible for enforcing on it.""" + + name: str = Field(description="The account's name in the instruction's accounts struct/context.") + account_type: str = Field( + description="The account's declared type (e.g. 'Signer', 'Account', 'Program', " + "'SystemAccount', 'UncheckedAccount', a PDA of some seeds)." + ) + roles: list[AccountRole] = Field( + default_factory=list, + description="Roles this account plays: signer / writable / readonly / pda / program / sysvar.", + ) + constraints: list[str] = Field( + default_factory=list, + description="Validations the program must enforce on this account — e.g. Anchor " + "constraints (has_one, seeds+bump, address, owner), an explicit owner/signer check, or " + "a documented invariant. Empty means the program performs no checks (often a finding).", + ) + + +class CpiCall(BaseModel): + """A cross-program invocation the instruction makes.""" + + target_program: str = Field(description="The program invoked (name or program id).") + description: str = Field(description="What the CPI does and any authority/PDA-signer it uses.") + + +class SolanaInstruction(BaseModel): + """A single instruction (entry point) of a program.""" + + name: str = Field(description="The instruction's snake_case name (its handler function).") + description: str = Field(description="What the instruction does, at the behavioral level (not how).") + accounts: list[AccountConstraint] = Field( + default_factory=list, description="The accounts the instruction takes and their constraints." + ) + signers: list[str] = Field( + default_factory=list, + description="Which accounts must sign (authorities/owners the instruction authenticates).", + ) + cpis: list[CpiCall] = Field( + default_factory=list, description="Cross-program invocations this instruction performs." + ) + args: list[str] = Field( + default_factory=list, description="The instruction's non-account arguments (name & type)." + ) + requirements: list[str] = Field( + description="Natural-language behavioral requirements — the instruction's specification." + ) + + +class SolanaProgram(BaseModel): + """A concrete on-chain program in the system.""" + + name: str = Field( + description="A short conceptual name for the program, used to refer to it across the system." + ) + program_identifier: str = Field( + pattern=r"^[a-zA-Z_][a-zA-Z0-9_]*$", + description="The program's Rust crate/module identifier as it appears in source. A valid " + "Rust identifier (snake_case).", + ) + program_id: str | None = Field( + default=None, description="The on-chain program id (base58), if declared (e.g. declare_id!)." + ) + description: str = Field(description="The program's role in the system.") + instructions: list[SolanaInstruction] = Field(description="The program's instructions.") + account_types: list[str] = Field( + default_factory=list, + description="The account/state types this program owns and derives (PDAs), name & purpose.", + ) + + +class SolanaAuthority(BaseModel): + """An external actor: a signer/authority, an off-chain keypair, or another program the + system interacts with but does not itself implement.""" + + name: str = Field(description="A short unique identifier for this authority/actor.") + description: str = Field(description="A short technical description.") + assumptions: list[str] = Field( + default_factory=list, description="Assumptions about this actor's behavior/trust." + ) + + +type SolanaComponent = SolanaProgram | SolanaAuthority + + +class SolanaApplication(BaseApplication[SolanaComponent]): + """A Solana application: a set of programs (+ external authorities).""" + + @cached_property + def programs(self) -> list[SolanaProgram]: + return [c for c in self.components if isinstance(c, SolanaProgram)] + + @cached_property + def authorities(self) -> list[SolanaAuthority]: + return [c for c in self.components if isinstance(c, SolanaAuthority)] + + +# --------------------------------------------------------------------------- +# Index wrappers — the driver's Main (program) and Unit (instruction). +# --------------------------------------------------------------------------- + + +@dataclass +class SolanaProgramInstance: + """The located target program — the ecosystem's ``Main``. + + Also serves as the **whole-program extraction unit** (satisfies + ``composer.spec.system_model.FeatureUnit``): under the global extraction strategy + (docs/crucible-unit-granularity.md) it is the context the property phase reads to + propose whole-program invariants, before those invariants fan out into + :class:`SolanaInvariantUnit`\\ s.""" + + ind: int + app: SolanaApplication + + @property + def program(self) -> SolanaProgram: + return self.app.programs[self.ind] + + # -- FeatureUnit protocol (whole-program extraction context) ------------------------ + @property + def display_name(self) -> str: + return self.program.name + + @property + def slug(self) -> str: + return slugify_filename(self.program.name) + + @property + def unit_index(self) -> int: + return self.ind + + def cache_material(self) -> str: + return "|".join([self.app.model_dump_json(), str(self.ind), "program"]) + + def context_tag(self) -> dict: + return {"program": self.program.model_dump()} + + def feature_json(self) -> dict: + return { + "program": self.program.name, + "instructions": [i.model_dump(mode="json") for i in self.program.instructions], + } + + +@dataclass +class SolanaInvariantUnit: + """One whole-program invariant — a ``Unit`` for the global extraction strategy. + + Produced by fanning the invariants out of a single whole-program extraction, so each + invariant gets its own harness fn + fuzz run + report row (satisfies + ``composer.spec.system_model.FeatureUnit``). The invariant travels in the formalize + batch's ``props``; ``feature_json`` carries the whole-program API so the test author + can drive any ``action_*`` in the sequence it asserts over.""" + + ind: int + _program: SolanaProgramInstance + invariant: PropertyFormulation + + @property + def app(self) -> SolanaApplication: + return self._program.app + + @property + def program(self) -> SolanaProgram: + return self._program.program + + # -- FeatureUnit protocol ----------------------------------------------------------- + @property + def display_name(self) -> str: + return self.invariant.title + + @property + def slug(self) -> str: + return slugify_filename(self.invariant.title) + + @property + def unit_index(self) -> int: + return self.ind + + def cache_material(self) -> str: + return "|".join( + [self.app.model_dump_json(), str(self._program.ind), str(self.ind), self.invariant.title] + ) + + def context_tag(self) -> dict: + return {"invariant": self.invariant.model_dump(mode="json"), "program": self.program.name} + + def feature_json(self) -> dict: + return { + "program": self.program.name, + "instructions": [i.model_dump(mode="json") for i in self.program.instructions], + } + + +@dataclass +class SolanaInstructionInstance: + """One instruction of the target program — the ecosystem's ``Unit`` (satisfies + ``composer.spec.system_model.FeatureUnit``).""" + + ind: int + _program: SolanaProgramInstance + + @property + def app(self) -> SolanaApplication: + return self._program.app + + @property + def program(self) -> SolanaProgram: + return self._program.program + + @property + def instruction(self) -> SolanaInstruction: + return self.program.instructions[self.ind] + + # -- FeatureUnit protocol ----------------------------------------------------------- + @property + def display_name(self) -> str: + return self.instruction.name + + @property + def slug(self) -> str: + return slugify_filename(self.instruction.name) + + @property + def unit_index(self) -> int: + return self.ind + + def cache_material(self) -> str: + return "|".join([self.app.model_dump_json(), str(self.ind), str(self._program.ind)]) + + def context_tag(self) -> dict: + return {"instruction": self.instruction.model_dump()} + + def feature_json(self) -> dict: + # The unit's semantic content for a backend marshalling it across a boundary: + # the instruction, tagged with its program (a Solana backend needs both). + return { + "program": self.program.name, + "instruction": self.instruction.model_dump(mode="json"), + } diff --git a/composer/spec/solana/null_backend.py b/composer/spec/solana/null_backend.py new file mode 100644 index 00000000..cd9e6cea --- /dev/null +++ b/composer/spec/solana/null_backend.py @@ -0,0 +1,175 @@ +"""A null Solana backend — records extracted properties without verifying them. + +It satisfies the full ``PipelineBackend`` contract over the Solana ecosystem's +``(SolanaApplication, SolanaProgramInstance, SolanaInstructionInstance)`` triple, but its +``formalize`` just echoes the extracted properties into a trivial result and its +``fetch_verdicts`` returns nothing. + +**Role:** a **test double** for the Solana front half (analysis + property extraction) +without a real verifier — see ``tests/test_solana_gate.py``. Production Solana +verification is the Crucible fuzzer backend. +""" + +import enum +import json +from dataclasses import dataclass +from pathlib import Path +from typing import cast, override + +from pydantic import BaseModel, Field + +from composer.pipeline.core import ( + CorePhases, + Formalizer, + GaveUp, + PipelineRun, + PreparedSystem, + SystemAnalysisSpec, +) +from composer.spec.artifacts import ArtifactStore +from composer.spec.context import WorkflowContext +from composer.spec.cvl_generation import SkippedProperty +from composer.spec.solana.model import ( + SolanaApplication, + SolanaInstructionInstance, + SolanaProgramInstance, +) +from composer.spec.source.report.collect import ReportComponentInput, Verdict +from composer.spec.source.report.schema import ReportBackend, RuleName +from composer.spec.system_model import FeatureUnit +from composer.spec.types import PropertyFormulation +from composer.spec.util import ensure_dir + +SOLANA_NULL_GUIDANCE: str = """\ +These properties are recorded by a null backend (no verification is performed). Extract +properties a Solana verification tool could plausibly check: account/state invariants, access +control (signer/owner/authority), PDA-derivation correctness, and arithmetic safety. Freely +state universally-quantified properties. +""" + + +class SolanaPhase(enum.Enum): + ANALYSIS = "analysis" + EXTRACTION = "extraction" + FORMALIZATION = "formalization" + REPORT = "report" + + +class NullResult(BaseModel): + """A trivial formalization result: it just carries the properties back out.""" + + commentary: str = "" + property_rules: list[tuple[str, list[str]]] = Field(default_factory=list) + skipped: list[SkippedProperty] = Field(default_factory=list) + + def property_units(self) -> list[tuple[str, list[str]]]: + return [(t, list(u)) for t, u in self.property_rules] + + @property + def artifact_text(self) -> str: + return json.dumps( + {"commentary": self.commentary, "properties": self.property_units()}, indent=2 + ) + + @property + def output_link(self) -> str | None: + return None + + +@dataclass(frozen=True) +class NullArtifact: + slug: str + + @property + def stem(self) -> str: + return f"null_{self.slug}" + + @property + def artifact_file(self) -> str: + return f"{self.stem}.json" + + +class NullSolanaArtifactStore(ArtifactStore[NullArtifact, NullResult]): + def __init__(self, project_root: str): + super().__init__( + project_root, + "property_units", + deliverable_dir="certora/solana_null", + internal_dir=".certora_internal/solana_null", + report_dir="certora/solana_null/reports", + ) + + @override + def _artifact_dir(self) -> Path: + return ensure_dir(Path(self._project_root) / "certora/solana_null/artifacts") + + +class NullSolanaFormalizer(Formalizer[NullResult, FeatureUnit]): + def __init__(self) -> None: + # Reuses the ``"crucible"`` report backend (the real Solana verifier this null backend + # models); its results are all-UNKNOWN, so the label choice is provenance only. + # Intermediate (PR1): report/schema.py's ReportBackend literal is still master's + # {prover, foundry}; cast the tag until PR3 closes the literal to include "crucible". + super().__init__(NullResult, cast(ReportBackend, "crucible")) + + @override + async def formalize( + self, + label: str, + feat: FeatureUnit, + props: list[PropertyFormulation], + ctx: WorkflowContext[NullResult], + run: PipelineRun, + ) -> NullResult | GaveUp: + return NullResult( + commentary=f"Null formalization of instruction {feat.display_name} " + f"({len(props)} properties recorded, unverified).", + property_rules=[(p.title, [p.title]) for p in props], + ) + + @override + async def fetch_verdicts( + self, inp: ReportComponentInput[NullResult] + ) -> dict[RuleName, Verdict]: + return {} + + +@dataclass +class NullSolanaPrepared(PreparedSystem[NullResult, FeatureUnit]): + form: NullSolanaFormalizer + + @override + async def prepare_formalization( + self, run: PipelineRun + ) -> Formalizer[NullResult, FeatureUnit]: + return self.form + + +@dataclass +class NullSolanaBackend: + """``PipelineBackend[SolanaPhase, NullResult, None, NullArtifact, SolanaApplication, + SolanaProgramInstance, SolanaInstructionInstance]`` — structural.""" + + artifact_store: NullSolanaArtifactStore + backend_guidance = SOLANA_NULL_GUIDANCE + analysis_spec = SystemAnalysisSpec("solana-analysis", "solana-properties") + core_phases = CorePhases( + { + "analysis": SolanaPhase.ANALYSIS, + "extraction": SolanaPhase.EXTRACTION, + "formalization": SolanaPhase.FORMALIZATION, + "report": SolanaPhase.REPORT, + } + ) + + async def prepare_system( + self, analyzed: SolanaApplication, run: PipelineRun[SolanaPhase, None] + ) -> PreparedSystem[NullResult, FeatureUnit]: + # Use the Solana ecosystem's locate_main so the backend and ecosystem agree on the + # target program (imported lazily to avoid an import cycle with pipeline.ecosystem). + from composer.pipeline.ecosystem import SOLANA + + return NullSolanaPrepared(SOLANA.locate_main(analyzed, run.source), NullSolanaFormalizer()) + + def to_artifact_id(self, c: FeatureUnit) -> NullArtifact: + return NullArtifact(c.slug) diff --git a/composer/spec/source/pipeline.py b/composer/spec/source/pipeline.py index c6ac5120..aea1603d 100644 --- a/composer/spec/source/pipeline.py +++ b/composer/spec/source/pipeline.py @@ -98,7 +98,7 @@ def _lift_harnessed( @dataclass -class ProverRunner(Formalizer[GeneratedCVL]): +class ProverRunner(Formalizer[GeneratedCVL, ContractComponentInstance]): """Immutable formalizer: per-batch CVL generation against a fixed prover config + resource set (already including ``invariants.spec`` when there are structural invariants), plus the in-memory invariant result for the report.""" @@ -149,7 +149,7 @@ async def fetch_verdicts( return await self._fetch(inp) @override - async def finalize(self, outcomes: list[ComponentOutcome[GeneratedCVL]], run: PipelineRun) -> None: + async def finalize(self, outcomes: list[ComponentOutcome[GeneratedCVL, ContractComponentInstance]], run: PipelineRun) -> None: # components_to_prover_runs.json: {run_key (slug): prover /output/ link}. runs: dict[str, str] = { ComponentSpec(o.feat.slugified_name).run_key: o.result.run_link @@ -164,7 +164,7 @@ async def finalize(self, outcomes: list[ComponentOutcome[GeneratedCVL]], run: Pi @dataclass -class ProverPrepared(PreparedSystem[GeneratedCVL]): +class ProverPrepared(PreparedSystem[GeneratedCVL, ContractComponentInstance]): """Post-harness system: holds the harnessed app + prover tool, and runs the prover-only pre-formalization fan-out in ``prepare_formalization``.""" _store: ProverArtifactStore @@ -175,7 +175,7 @@ class ProverPrepared(PreparedSystem[GeneratedCVL]): _analyzed: SourceApplication @override - async def prepare_formalization(self, run: PipelineRun) -> Formalizer[GeneratedCVL]: + async def prepare_formalization(self, run: PipelineRun) -> Formalizer[GeneratedCVL, ContractComponentInstance]: # AutoSetup (+ custom summaries) ∥ structural-invariant formulation; both # depend only on the harnessed app, so they run concurrently. (setup_config, resources), invariants = await asyncio.gather( @@ -287,7 +287,7 @@ class ProverBackend: async def prepare_system( self, analyzed: SourceApplication, run: PipelineRun[AutoProvePhase, None], - ) -> PreparedSystem[GeneratedCVL]: + ) -> PreparedSystem[GeneratedCVL, ContractComponentInstance]: sys_desc = await run.runner( TaskInfo(HARNESS_TASK_ID, "Harness Creation", AutoProvePhase.HARNESS), lambda: run_harness_creation(run.ctx, run.source, run.env, analyzed), diff --git a/composer/spec/system_analysis.py b/composer/spec/system_analysis.py index 57953825..a5661f3d 100644 --- a/composer/spec/system_analysis.py +++ b/composer/spec/system_analysis.py @@ -1,4 +1,4 @@ -from typing import NotRequired, Any +from typing import NotRequired, Any, Callable from graphcore.graph import MessagesState, FlowInput @@ -95,8 +95,16 @@ async def run_component_analysis[T: BaseApplication]( env: ServiceHost, extra_input: list[str | dict], expected_main_id: SolidityIdentifier | None = None, + *, + system_template: str = "application_analysis_system.j2", + initial_template: str = "application_analysis_prompt.j2", + validate: Callable[[BaseApplication, SolidityIdentifier | None], str | None] = _validate_connectivity, ) -> T | None: - """Analyze application components from a system doc and optionally source code.""" + """Analyze application components from a system doc and optionally source code. + + The prompt templates and connectivity ``validate`` are parameters so the ecosystem + (``composer.pipeline.ecosystem``) can supply domain-specific ones; the defaults are the + EVM/Solidity values, so existing callers are unaffected.""" if (cached := await child_ctxt.cache_get(ty)) is not None: return cached @@ -114,7 +122,7 @@ class AnalysisInput(RoughDraftState, FlowInput): def _validation_wrapper( _: Any, app: BaseApplication ) -> str | None: - return _validate_connectivity(app, expected_main_id) + return validate(app, expected_main_id) b = bind_standard( builder=env.builder_lite(), @@ -123,13 +131,13 @@ def _validation_wrapper( ).with_input( AnalysisInput ).with_sys_prompt_template( - "application_analysis_system.j2", + system_template, sort=env.sort, has_doc=input is not None ).with_tools( [memory, *get_rough_draft_tools(AnalysisState), *env.analysis_tools] ).with_initial_prompt_template( - "application_analysis_prompt.j2", + initial_template, sort=env.sort, has_doc=input is not None ) diff --git a/composer/spec/system_model.py b/composer/spec/system_model.py index 7bdb4059..68042b9c 100644 --- a/composer/spec/system_model.py +++ b/composer/spec/system_model.py @@ -1,10 +1,49 @@ from dataclasses import dataclass -from typing import Literal +from typing import Literal, Protocol, runtime_checkable from pydantic import BaseModel, Field from functools import cached_property from composer.spec.util import slugify_filename from .types import ComponentName, SolidityIdentifier, ContractName + +@runtime_checkable +class FeatureUnit(Protocol): + """The per-unit interface the shared pipeline driver needs, independent of ecosystem. + + Each ecosystem's unit wrapper (EVM's :class:`ContractComponentInstance`, Solana's + instruction instance, …) satisfies it; ecosystem-specific code — prompts, backends, + formalizers — works with the concrete unit type. This keeps the driver's per-unit cache + keys, task ids, labels, and context tags ecosystem-agnostic.""" + + @property + def display_name(self) -> str: + """Human label for tasks / report rows.""" + ... + + @property + def slug(self) -> str: + """Filesystem-safe slug used as an artifact-id base.""" + ... + + @property + def unit_index(self) -> int: + """Stable per-run index used in task ids.""" + ... + + def cache_material(self) -> str: + """Stable string identifying this unit, hashed into its per-unit cache key.""" + ... + + def context_tag(self) -> dict: + """The tag persisted alongside this unit's workflow context.""" + ... + + def feature_json(self) -> dict: + """The unit's semantic content as a JSON-able dict, for a backend that + marshals it across a boundary (e.g. a Rust wheel's ``FormalizeInput.component``). + The shape is ecosystem-specific; the paired backend knows how to read it.""" + ... + type ContractSort = Literal["dynamic", "singleton", "multiple"] @@ -92,7 +131,10 @@ class SourceExternalActor(ExternalActor): type SystemComponent = ExternalActor | ExplicitContract -class BaseApplication[T : SystemComponent](BaseModel): +# Bound is ``BaseModel`` (not ``SystemComponent``) so non-EVM ecosystems can parameterize it +# with their own component unions (e.g. Solana's programs/authorities); the EVM subclasses below +# still pin the concrete ``SystemComponent`` union. +class BaseApplication[T : BaseModel](BaseModel): application_type: str = Field(description="A concise, description of the type of application (AMM/Liquidity Provider/etc.)") description: str = Field(description="A description of the application's main functionality (2 - 3 sentences max)") components : list[T] = Field(description="The system components (explicit contract & external actors) that comprise this application") @@ -234,6 +276,28 @@ def slugified_name(self) -> str: ``_validate_connectivity``), so no disambiguation is needed here.""" return slugify_filename(self.component.name) + # -- FeatureUnit protocol (the ecosystem-agnostic view the driver consumes) --------- + @property + def display_name(self) -> str: + return self.component.name + + @property + def slug(self) -> str: + return self.slugified_name + + @property + def unit_index(self) -> int: + return self.ind + + def cache_material(self) -> str: + return "|".join([self.app.model_dump_json(), str(self.ind), str(self._contract.ind)]) + + def context_tag(self) -> dict: + return {"component": self.component.model_dump()} + + def feature_json(self) -> dict: + return self.component.model_dump(mode="json") + @staticmethod def from_app( app: AnyApplication, diff --git a/composer/templates/solana/_failure_modes.j2 b/composer/templates/solana/_failure_modes.j2 new file mode 100644 index 00000000..0da21b80 --- /dev/null +++ b/composer/templates/solana/_failure_modes.j2 @@ -0,0 +1,30 @@ +Solana platform failure modes (the account/PDA/CPI model — reason about these specifically): + +- **Missing signer check.** An instruction that mutates state or moves value on behalf of an + authority but never checks that authority `is_signer` (or omits the Anchor `Signer<'info>` / + `has_one` / `#[account(signer)]` constraint) lets anyone act as that authority. +- **Missing owner check.** An account deserialized as a program type without verifying its + `owner` is the expected program lets an attacker substitute a look-alike account they created + under a program they control (`UncheckedAccount` / `AccountInfo` without checks is the smell; + Anchor `Account<'info, T>` checks owner, raw does not). +- **Account substitution / confused deputy.** Two accounts that must be related (e.g. a vault and + its authority, a token account and its mint/owner) but whose relationship is not constrained + (`has_one`, `constraint = a.key() == b.x`, `address = ...`) let an attacker pass a mismatched + pair — draining a different user's vault, minting to the wrong mint, etc. +- **Unvalidated PDA seeds / bump.** A PDA account not re-derived and checked against its expected + `seeds` + canonical `bump` lets an attacker pass an arbitrary account in its place; a mutable + bump or non-canonical bump enables collisions/hijacking. +- **Arbitrary CPI / program substitution.** Invoking a program taken from an unchecked account + (not pinned to a known program id) lets an attacker redirect the CPI to malicious code; missing + checks on the callee for token/system operations. +- **Signer-seed / authority misuse in CPI.** A PDA that signs a CPI (`invoke_signed`) with seeds + an attacker can influence, or over-broad authority, lets value move without proper authorization. +- **Lamport / rent draining & account close bugs.** Manual lamport transfers that don't preserve + rent exemption, or `close` handling that doesn't zero data / send to the right account, enable + reinitialization attacks or fund theft. +- **Duplicate mutable accounts.** Passing the same account twice where the program assumes two + distinct accounts (e.g. `from`/`to`) can double-count or bypass a balance check. +- **Missing reinitialization / init guards.** An `init`-like path callable twice, or state that + can be re-initialized after close, resets ownership/config. +- **Sysvar / clock spoofing.** Reading a sysvar (clock, rent) from a passed account rather than the + canonical sysvar lets an attacker forge time/rent values. diff --git a/composer/templates/solana/analysis_prompt.j2 b/composer/templates/solana/analysis_prompt.j2 new file mode 100644 index 00000000..9dc5c16a --- /dev/null +++ b/composer/templates/solana/analysis_prompt.j2 @@ -0,0 +1,74 @@ +# Background + +{% if sort == "greenfield" %} +You have been provided a system/design document describing a Solana application. No +implementation exists yet, and the document may vary in technical rigor. +{% else %} +You have been provided a system/design document describing a Solana application, plus access to +the Rust source (native and/or Anchor) that implements it. +{% endif %} + +# Task + +Analyze this design document {% if sort != "greenfield" %}and implementation {% endif %}and extract a holistic, structured model of the +application. Your model describes the **programs** {% if sort == "greenfield" %}that need to be written{% else %}present in the implementation{% endif %}, the +**instructions** each program exposes, the **accounts** each instruction operates on (with the +checks the program must enforce), and any **external authorities/actors** involved. Produce +natural-language descriptions — do *NOT* write Rust or pseudo-code. + +## Application Description + +Provide an "Application Type" — a broad category (e.g. "AMM", "Staking", "NFT marketplace", +"Escrow") — and a short (2–3 sentence) description that refines it. Then a list of "System +Components": each is either a **Program** or an **External Authority**. + +### External Authorities + +Any actor not part of the core application the programs interact with: an admin/authority +keypair, an off-chain signer, or a third-party program (e.g. an oracle) the system does not +itself implement. The **SPL Token program, the System program, and other standard Solana +programs are external authorities** unless the application implements them. Give each a short +name, a description of its role, and the assumptions/trust placed in it. + +### Programs + +Each program the application implements. For each program provide: + +- `name`: a short conceptual label (e.g. "Vault", "Staking Pool"). +- `program_identifier`: the program's Rust crate/module identifier as it appears (or will + appear) in source — a valid Rust identifier (snake_case, matching `[a-zA-Z_][a-zA-Z0-9_]*`). +- `program_id`: the on-chain base58 program id if declared (e.g. via `declare_id!`), else null. +- `description`: the program's role in the application. +- `account_types`: the account/state types this program owns and derives (PDAs) — name & purpose. +- `instructions`: the program's instructions (entry points), described below. + +#### Instructions + +Each instruction (the program's entry points / handlers). For each: + +- `name`: the instruction's snake_case name. +- `description`: what it does behaviorally (not how). +- `args`: its non-account arguments (name & type). +- `accounts`: every account it takes in its accounts context. For each account give its `name`, + its `account_type` (e.g. `Signer`, `Account`, `Program`, `SystemAccount`, + `UncheckedAccount`, a PDA of specific seeds), its `roles` (signer / writable / readonly / pda / + program / sysvar), and the `constraints` the program is responsible for enforcing on it + (signer/owner checks, `has_one`, `seeds`+`bump`, `address`, or documented invariants). If an + account has no enforced constraints, record an empty list — that is itself important signal. +- `signers`: which accounts must sign (the authorities the instruction authenticates). +- `cpis`: cross-program invocations it performs (target program + what it does / signer used). +- `requirements`: the instruction's behavioral specification, stated as requirements ("The + instruction must ..."). + +## On Interactions + +Enumerate programs and authorities as comprehensively as the inputs allow. Halt transitive +exploration at external-authority boundaries — do not speculate about what those actors do +internally. When an instruction CPIs into another program the application implements, that +callee is a Program (a component), not an external authority. + +# Memory + +You have a memory tool that may include prior progress on this task. Assume the documentation / +source has NOT changed since the memories were written; you need not re-verify memorized facts +and may proceed with them as if you derived them yourself. diff --git a/composer/templates/solana/analysis_system.j2 b/composer/templates/solana/analysis_system.j2 new file mode 100644 index 00000000..b59bbf88 --- /dev/null +++ b/composer/templates/solana/analysis_system.j2 @@ -0,0 +1,28 @@ +You are an experienced system architect well-versed in Solana on-chain programs (native and +Anchor). You are analyzing a Solana application's design document {% if sort != "greenfield" %}and its Rust implementation {% endif %}to extract a +structured model of its behavior. This model guides the {% if sort != "greenfield" %}verification{% else %}implementation{% endif %} of the application. + +## Behavior + +- After any collection of data, use your adaptive thinking capabilities to consider all of your + information before proceeding. +- Ground any results you produce in verifiable information as much as possible. +- When resolving ambiguity during your work, favor self-consistency. + +## Tools + +- You have access to a "memory" tool, presented as a filesystem-like abstraction. Whenever you + synthesize new knowledge or reach a conclusion, record it (and your reasoning) with the tool. +- On a multi-step task, track progress in `/memories/progress.md` (current step + task data). +{% with draft_subject = "your response" %} +{% include "rough_draft_protocol.j2" %} +{% endwith %} +{% if sort != "greenfield" %} +{% with source_tools_has_explorer = sort == "existing" %} +{% include "source_tools_system_prompt.j2" %} +{% endwith %} +When reading Rust/Anchor source, pay attention to the instruction handlers, their +`#[derive(Accounts)]` account-validation structs (and the constraints on each account: +`signer`, `mut`, `has_one`, `seeds`/`bump`, `owner`, `address`), the account/state types the +program declares, and any cross-program invocations. +{% endif %} diff --git a/composer/templates/solana/instruction_context.j2 b/composer/templates/solana/instruction_context.j2 new file mode 100644 index 00000000..19e7fb36 --- /dev/null +++ b/composer/templates/solana/instruction_context.j2 @@ -0,0 +1,52 @@ +This task targets the security properties of the instruction `{{ context.instruction.name }}` +of the program `{{ context.program.name }}` (`{{ context.program.program_identifier }}`). + +This instruction is described as: + +{{ context.instruction.description }} + +The requirements for this instruction are: +{% for req in context.instruction.requirements %} +* {{ req }} +{% endfor %} +{% if context.instruction.args %} + +Non-account arguments: {% for a in context.instruction.args %}`{{ a }}`{% if not loop.last %}, {% endif %}{% endfor %}. +{% endif %} + +Accounts this instruction takes, and the checks the program is responsible for enforcing on each: +{% for acc in context.instruction.accounts %} +* `{{ acc.name }}` — {{ acc.account_type }}{% if acc.roles %} [{{ acc.roles | join(", ") }}]{% endif %}. + {% if acc.constraints %}Program-enforced constraints: {{ acc.constraints | join("; ") }}.{% else %}No constraints were recorded for this account — consider whether a signer/owner/relationship check is required here.{% endif %} +{% endfor %} +{% if context.instruction.signers %} + +Accounts that must sign: {{ context.instruction.signers | join(", ") }}. +{% endif %} +{% if context.instruction.cpis %} + +Cross-program invocations this instruction performs: +{% for cpi in context.instruction.cpis %} +* → `{{ cpi.target_program }}`: {{ cpi.description }} +{% endfor %} +{% endif %} + +The `{{ context.program.name }}` program is part of a `{{ context.app.application_type }}` application. +{% if context.program.account_types %} +It owns / derives these account (state) types: {{ context.program.account_types | join(", ") }}. +{% endif %} + +{% set siblings = [] %} +{% for p in context.app.programs %}{% if p.program_identifier != context.program.program_identifier %}{% set _ = siblings.append(p) %}{% endif %}{% endfor %} +{% if siblings %} +Other programs in the application: +{% for p in siblings %} +* `{{ p.name }}` (`{{ p.program_identifier }}`): {{ p.description }} +{% endfor %} +{% endif %} +{% if context.app.authorities %} +External authorities / actors the system interacts with: +{% for a in context.app.authorities %} +* {{ a.name }}: {{ a.description }} +{% endfor %} +{% endif %} diff --git a/composer/templates/solana/program_context.j2 b/composer/templates/solana/program_context.j2 new file mode 100644 index 00000000..03cb993d --- /dev/null +++ b/composer/templates/solana/program_context.j2 @@ -0,0 +1,21 @@ +This task targets whole-program security invariants of the program `{{ context.program.name }}` +(`{{ context.program.program_identifier }}`), part of a `{{ context.app.application_type }}` application. + +{% if context.program.account_types %} +The program owns / derives these account (state) types: {{ context.program.account_types | join(", ") }}. +{% endif %} + +The program exposes these instructions (the actions a fuzzer will drive in random sequences): +{% for ins in context.program.instructions %} +### `{{ ins.name }}` +{{ ins.description }} +{% if ins.requirements %}Requirements: +{% for req in ins.requirements %} - {{ req }} +{% endfor %}{% endif %} +{% if ins.args %}Non-account arguments: {% for a in ins.args %}`{{ a }}`{% if not loop.last %}, {% endif %}{% endfor %}. +{% endif %}Accounts (and the checks the program must enforce): +{% for acc in ins.accounts %} - `{{ acc.name }}` — {{ acc.account_type }}{% if acc.roles %} [{{ acc.roles | join(", ") }}]{% endif %}{% if acc.constraints %}; constraints: {{ acc.constraints | join("; ") }}{% endif %}. +{% endfor %}{% if ins.signers %}Must sign: {{ ins.signers | join(", ") }}. +{% endif %}{% if ins.cpis %}CPIs: {% for cpi in ins.cpis %}→ `{{ cpi.target_program }}` ({{ cpi.description }}){% if not loop.last %}; {% endif %}{% endfor %}. +{% endif %} +{% endfor %} diff --git a/composer/templates/solana/property_prompt.j2 b/composer/templates/solana/property_prompt.j2 new file mode 100644 index 00000000..3c6435f2 --- /dev/null +++ b/composer/templates/solana/property_prompt.j2 @@ -0,0 +1,114 @@ + +You are performing a security review of a Solana program as a whole, to produce **program-level +invariants** for a coverage-guided fuzzer. The fuzzer drives *random sequences* of the program's +instructions and checks each property after every action — so the properties you write must hold +across ANY reachable sequence of actions, not only within a single instruction. + + + +{% include "solana/program_context.j2" %} + + +{% if prior_properties %} + +You are running iteratively. {{ prior_properties | length }} prior round(s) have already analyzed +this program. Their findings and reasoning are below. The properties listed here MUST NOT be +re-proposed — including reframings of the same underlying claim as a different sort. Your task this +round is to surface security properties that prior rounds *missed*. + +{% for round_result in prior_properties %} +### Round {{ loop.index }} extracted properties: +{% for p in round_result.items %} +- [{{ p.sort }}] `{{ p.title }}`: {{ p.description }} +{% endfor %} + +### Round {{ loop.index }} reasoning: +{{ round_result.reasoning }} + +{% endfor %} + +Treat the prior rounds' reasoning as the authoritative record of what has been considered. If a +prior round is silent on an angle, that angle is fair game. If the meaningful surface is already +covered, returning an empty list is the right answer. + +{% endif %} + + +Input: {% if sort != "greenfield" %}the Rust/Anchor implementation of a Solana program{% else %}a design document for a Solana program{% endif %}, and a description of its instructions. +Output: a list of whole-program "security properties" a fuzzer can check after every action. + +Follow these steps exactly: + +## Step 1 + +Analyze the provided {% if sort != "greenfield" %}implementation{% else %}design{% endif %} and formulate a set of security +properties that are PLAUSIBLE for this program. Prefer **cross-instruction** properties — ones a +random sequence of actions could violate (conservation, monotonicity, access control that must hold +no matter what came before) — over ones local to a single call. Each description must be precise and +concise, and phrased as something checkable against on-chain state after any action. + +Security properties fall into three categories: + +1. **Invariants** — representational invariants that should always hold for the program's accounts/state, + across every reachable state. + Good: "The vault's token balance always equals the sum of all depositors' recorded shares." + Good: "A config PDA's `admin` field is only ever the key that signed the last `set_admin`." + Bad: "The accounts should be correct" (overly broad). + +2. **Safety properties** — concrete statements about what should not be possible in a correct program, + over any action sequence. + These SHOULD include intended-normal-behavior properties, not only immediately-security-critical ones. + Good: "Only the account stored in `vault.authority` (and it must sign) can ever reduce the vault balance." + Good: "`initialize` can succeed at most once for a given config PDA, no matter the call order." + Good: "No sequence of actions lets a user withdraw more than they have deposited." + Bad: "A user should not be able to hack the program" (overly broad). + +3. **Attack vectors** — plausible issues/edge cases exploitable to the protocol's detriment. You do + NOT need evidence they exist, only that they are PLAUSIBLE given the {% if sort != "greenfield" %}implementation{% else %}design{% endif %}. + Good: "If `vault` is an `UncheckedAccount` whose owner is not checked, an attacker can pass a + look-alike account they control and redirect the withdrawal." + Good: "If the `authority` account is not constrained to sign, anyone can invoke `withdraw` as + the authority (missing signer check)." + Good: "If the fee PDA's `bump` is taken from the instruction rather than re-derived, an attacker + can substitute a different PDA." + Bad: "The program could be exploited somehow" (overly broad). + +When reasoning about attack vectors and the checks each account requires, consider these failure modes: + +{% include "rust/_failure_modes.j2" %} + +{% include "solana/_failure_modes.j2" %} + +{{ backend_guidance }} + +NB: you do **NOT** need to formalize the properties yourself; limit your analysis to properties that +can reasonably be formalized by the downstream verification tool. + +## Step 2 +Write a rough-draft list of your properties using the `write_rough_draft` tool. + +## Step 3 +Review your rough draft. For each property, confirm it meets the Step-1 guidelines — a clear +violation consequence, non-trivial, plausibly verifiable as a whole-program invariant. Drop anything +you cannot defend. + +## Step 4 +Output the results using the result tool. + + + + {% if sort != "greenfield" %} + + Derive invariants and safety properties from first principles — what the program SHOULD + guarantee — relying as little as possible on what the code CURRENTLY does. + + {% endif %} + + You need not prove a safety property/invariant definitely holds — only that it, with extreme + likelihood, SHOULD hold for a correct implementation. + + + For attack vectors you need not find evidence the issue exists — only that it is plausible + given the program and its {% if sort != "greenfield" %}implementation{% else %}design{% endif %}. + + diff --git a/composer/templates/solana/property_system.j2 b/composer/templates/solana/property_system.j2 new file mode 100644 index 00000000..f3cb9c0c --- /dev/null +++ b/composer/templates/solana/property_system.j2 @@ -0,0 +1,65 @@ +You are an expert security auditor of Solana on-chain programs (native and Anchor). You know the +failure modes that recur on Solana — missing signer/owner checks, account substitution / confused +deputy, unvalidated PDAs, arbitrary CPIs, lamport/rent and account-close bugs, duplicate mutable +accounts, reinitialization — plus the Rust-level ones (overflow-panic DoS, truncating casts). You +recognize them in design documents and in Rust/Anchor code. + +Your job is to extract a list of whole-program *security properties* (invariants a coverage-guided +fuzzer checks after every action in a random instruction sequence). The work happens in a multi-round +loop: each round produces only the NEW properties not surfaced by prior rounds, plus a written +reasoning record that later rounds and a human reviewer will read. + +# Behavior + +## Quality over quantity — and "nothing new" is the right answer when it's true + +The point of multiple rounds is to *eventually exhaust* the meaningful property surface for this +program. An empty `items` list with a `reasoning` field explaining what you looked at and why +nothing new emerged is a good round's desired outcome, not a failure. + +DO NOT propose properties just to feel productive. DO NOT pad the list. Concretely avoid: + +- Restating a prior property in different words and calling it new. +- Trivial corollaries ("X holds" plus "X holds for account A"). +- Edge cases the design does not motivate, invented to have something to return. +- Properties about non-security-relevant or non-verifiable behavior. +- Properties whose violation has no meaningful consequence. + +For every property you DO include, your `reasoning` must defend why it matters and, when prior +rounds exist, why a prior round could legitimately have missed it. If you can't, drop it. If you +think "this is a stretch but I should include something" — return an empty list. + +## Reasoning is load-bearing + +Your `reasoning` field is read by future-round agents and by a human reviewer. Be specific: name +the instruction, the accounts, the seeds, the CPI target, the attack pattern. "I considered the +missing owner check on `vault` but ruled it out because the Anchor `Account` type already +enforces owner = this program" is the right granularity; "I thought about it and decided Y" is not. + +## Adaptive thinking and self-consistency + +After gathering information from the source / design doc, integrate it before drafting properties; +don't propose from a single read. Favor self-consistency when resolving ambiguity. + +# Tools + +## Rough draft + +{% with draft_subject = "your property list" %} +{% include "rough_draft_protocol.j2" %} +{% endwith %} +When reviewing your draft, verify each property against the task criteria — verifiability, +non-triviality, a clear violation consequence, and defensibility in your `reasoning`. + +{% if sort != "greenfield" %} +## Source exploration + +{% with source_tools_has_explorer = sort == "existing" %} +{% include "source_tools_system_prompt.j2" %} +{% endwith %} + +Use the tools to ground reasoning in the actual Rust/Anchor implementation when the design is +ambiguous or an attack hinges on a code-level detail — which account checks are (or are not) +present in the `#[derive(Accounts)]` struct, how a PDA is derived, where a CPI's target program +comes from. Don't speculate when you can read the code. +{% endif %} diff --git a/docs/application-abstraction.md b/docs/application-abstraction.md new file mode 100644 index 00000000..9c0e24ac --- /dev/null +++ b/docs/application-abstraction.md @@ -0,0 +1,515 @@ +# Design Doc — What Is an AutoProver "Application" + +> How the pieces — argument parsing, service setup, the pipeline, and the UI — are +> wired into a single, runnable *application* such as **autoprove** or **foundry**, +> and the conventions a new application is expected to follow. +> +> Companion to [ARCHITECTURE.md](../ARCHITECTURE.md) and +> [formalization-abstraction.md](./formalization-abstraction.md). Where the +> formalization doc zooms into the *backend* seam (how a property becomes a verified +> artifact), this doc zooms out to the *whole vertical*: everything from `argv` to a +> rendered TUI. The [MultiJobApp design](../composer/ui/MULTI_JOB_DESIGN.md) covers +> the generic UI base this leans on. + +--- + +## 1. What "application" means here + +An AutoProver **application** is a complete, runnable vertical slice that takes a +Solidity project + a design document and drives the shared property-extraction / +formalization pipeline to a set of on-disk deliverables, rendered live to the user. + +There are two today: + +| Application | Deliverable | Backend | Entry points | +|---|---|---|---| +| **autoprove** | CVL `.spec` + `.conf`, verified by the Certora Prover | `ProverBackend` | [tui_autoprove.py](../composer/cli/tui_autoprove.py) · [console_autoprove.py](../composer/cli/console_autoprove.py) | +| **foundry** | `.t.sol` tests, gated by `forge test` | `FoundryBackend` | [tui_foundry.py](../composer/cli/tui_foundry.py) · [console_foundry.py](../composer/cli/console_foundry.py) | + +Crucially, "application" is **not** a single class. It is a *convention*: a set of +five collaborating pieces, each an implementation of a shared abstraction, wired +together by a thin `main()`. The value of the convention is that the pieces are +mutually orthogonal — you can swap the frontend (TUI ↔ console) without touching the +pipeline, and swap the backend without touching either frontend. + +--- + +## 2. The five pieces of an application + +Every application is assembled from exactly these, each keyed off one shared +type parameter — the application's **phase enum** `P`: + +``` + ┌─────────────────────────────────────────────────────────────┐ + │ main() (composer/cli/*.py) │ + │ async with entry_point(summary) as run: ← the Executor │ + │ app = FrontendApp() ← the Frontend │ + │ await run(app.make_handler) ← the seam │ + └─────────────────────────────────────────────────────────────┘ + │ │ + ┌───────────▼───────────┐ ┌─────────────▼─────────────┐ + │ 2. Entry point / │ │ 4. Frontend │ + │ Executor │ │ MultiJobApp[P, H] │ + │ argv → services → │ │ OR console handler │ + │ a run(handler) closure│ │ supplies make_handler: │ + └───────────┬───────────┘ │ HandlerFactory[P, H] │ + │ calls └───────────────────────────┘ + ┌───────────▼───────────┐ + │ 3. Pipeline │ 1. Phase enum P: HasName + │ run_pipeline(backend) │ (the spine that threads all + │ + a PipelineBackend │ five together) + └───────────┬───────────┘ + │ contributes + ┌───────────▼───────────┐ + │ 5. Artifact store │ + │ on-disk layout │ + └───────────────────────┘ +``` + +1. **Phase enum** `P` — the task-grouping vocabulary. +2. **Entry point / Executor** — argv → configured services → a `run(handler)` closure. +3. **Pipeline + backend** — the work, expressed as a `PipelineBackend` fed to the + shared `run_pipeline` driver. +4. **Frontend** — a `MultiJobApp[P, H]` subclass (TUI) or console handler that + supplies a `HandlerFactory[P, H]`. +5. **Artifact store** — the on-disk deliverable layout. + +The rest of this doc walks each piece with the autoprove and foundry +implementations side by side. + +--- + +## 3. The seam that makes it compose: `HandlerFactory` + +Before the pieces, understand the seam between them. The pipeline and the frontend +never reference each other. They meet at one protocol, +[`HandlerFactory[P, H]`](../composer/io/multi_job.py): + +```python +# composer/io/multi_job.py +class HandlerFactory[P: HasName, H](Protocol): + def __call__(self, /, info: TaskInfo[P]) -> Awaitable[TaskHandle[H]]: ... +``` + +- The **pipeline** is a producer of *work*. Every task it launches is described by a + `TaskInfo(task_id, label, phase)` and run through `run_task`, which calls the factory + to obtain a `TaskHandle` (an `IOHandler` + `EventHandler` + lifecycle callbacks). +- The **frontend** is a producer of *handlers*. It implements the factory: given a + `TaskInfo`, it mounts a panel, builds a per-task renderer, and returns the + `TaskHandle`. + +So the entire application boils down to: + +```python +async with entry_point(summary) as run: # run: Executor (the pipeline, service-loaded) + app = FrontendApp() # the frontend + result = await run(app.make_handler) # hand the factory to the pipeline +``` + +`run` is typed as an **Executor**, and its whole signature *is* the seam: + +```python +# composer/spec/source/autoprove_common.py +type Executor = Callable[[HandlerFactory[AutoProvePhase, None]], Awaitable[CorePipelineResult[GeneratedCVL]]] +# composer/foundry/pipeline.py +type FoundryPipelineExecutor = Callable[[HandlerFactory[FoundryPhase, None]], Awaitable[FoundryPipelineResult]] +``` + +Because the pipeline only ever *calls* the factory and the frontend only ever +*implements* it, the two are swappable independently. That is why autoprove has both +a TUI ([`AutoProveApp`](../composer/ui/autoprove_app.py)) and a console +([`AutoProveConsoleHandler`](../composer/ui/autoprove_console.py)) frontend against +the same pipeline, selected purely by which `make_handler` `main()` passes in. + +`H` is the human-interaction schema. Both current applications are non-interactive at +the per-task level (`H = None`; their handlers raise from `format_hitl_prompt`), but +the seam carries the type so an interactive application (e.g. the NatSpec pipeline) +plugs in without changing the contract. + +--- + +## 4. Piece 1 — the phase enum `P` + +Every application defines a single enum whose members are its task-grouping phases. +This enum is the type parameter that threads through the frontend +(`MultiJobApp[P, ...]`), the seam (`HandlerFactory[P, H]`, `TaskInfo[P]`), and the +backend (`CorePhases[P]`). It only needs to satisfy `HasName` (an enum trivially does). + +```python +# composer/ui/autoprove_app.py +class AutoProvePhase(enum.Enum): + HARNESS = "harness" + AUTOSETUP = "autosetup" + INVARIANTS = "invariants" + SUMMARIES = "summaries" + COMPONENT_ANALYSIS = "component_analysis" + BUG_ANALYSIS = "bug_analysis" + CVL_GEN = "cvl_gen" + REPORT = "report" +``` + +```python +# composer/foundry/pipeline.py +class FoundryPhase(enum.Enum): + SYSTEM_ANALYSIS = "system_analysis" + PROPERTY_EXTRACTION = "property_extraction" + TEST_GENERATION = "test_generation" + REPORT = "report" +``` + +The phase serves two roles: + +- **Grouping in the UI.** The frontend maps each phase to a human label and an + ordering, and every task lands in the section for its phase: + + ```python + # composer/ui/foundry_app.py + FOUNDRY_PHASE_LABELS = { + FoundryPhase.SYSTEM_ANALYSIS: "System Analysis", + FoundryPhase.PROPERTY_EXTRACTION: "Property Extraction", + FoundryPhase.TEST_GENERATION: "Test Generation", + } + FOUNDRY_SECTION_ORDER = ["System Analysis", "Property Extraction", "Test Generation"] + ``` + +- **The driver ↔ backend contract.** The shared driver tags four *core* phases; the + backend maps its own enum onto them via `CorePhases[P]` (see §6). Note the two enums + above differ in granularity: foundry has three phases, autoprove has eight — the + prover contributes several extra prep phases (harness, autosetup, summaries, + invariants) that the driver never knows about. The enum is the application's own + vocabulary; only the four core slots are shared. + +--- + +## 5. Piece 2 — the entry point / Executor + +Each application has an `_entry_point` async context manager that owns **all** the +imperative setup and yields the Executor closure. Its shape is a strict convention +(the foundry file's docstring literally says "Mirrors autoprove_common's shape"): + +> parse args → set up DB / RAG / store / checkpointer / logging / thread logger → +> yield a closure the caller drives with a handler factory. + +```python +# composer/spec/source/autoprove_common.py (shape shared by composer/foundry/entry.py) +@asynccontextmanager +async def _entry_point(summary: RunSummary) -> AsyncIterator[Executor]: + parser = argparse.ArgumentParser(...) + add_protocol_args(parser, RAGDBOptions) + add_protocol_args(parser, ExtendedModelOptions) + parser.add_argument("project_root", ...) + parser.add_argument("main_contract", help="Main contract as path:ContractName") + parser.add_argument("system_doc", ...) + # ... application-specific flags ... + args = cast(AutoProveArgs, parser.parse_args()) + async with autoprove_executor(args, summary) as runner: + yield runner +``` + +The heavy lifting is in the inner context manager, which: + +1. Resolves `project_root` + `main_contract` (`path:ContractName`) + `system_doc`. +2. Computes a **root cache key** from the inputs (`_root_cache_key` hashes project + root + doc content + relative path + contract name) — identical helper in both. +3. Opens the shared connection stack — `standard_connections`, a RAG DB, the async + tool context, the thread logger — under a single `async with`. +4. Reads the design doc into a `SourceCode`, builds the environment (`ServiceHost`), + and creates the `WorkflowContext`. +5. Yields a `runner(handler)` closure that calls the application's pipeline function. + +The args are declared as a **Protocol** (`AutoProveArgs`, `FoundryArgs`), not a class, +so the parser and the typed access agree without a dataclass in between: + +```python +# composer/spec/source/autoprove_common.py +class AutoProveArgs(ExtendedModelOptions, RAGDBOptions, Protocol): + project_root: str + main_contract: str + system_doc: str + max_concurrent: int + cloud: bool # ← prover-only: run jobs in the cloud + ... +``` + +```python +# composer/foundry/entry.py +class FoundryArgs(TieredModelOptions, FoundryRAGDBOptions, Protocol): + project_root: str + main_contract: str + system_doc: str + forge_binary: str # ← foundry-only + forge_timeout_s: int # ← foundry-only + max_forge_runners: int # ← foundry-only + ... +``` + +The `runner` closure each yields is the Executor: + +```python +# autoprove +async def runner(handler: HandlerFactory[AutoProvePhase, None]) -> CorePipelineResult[GeneratedCVL]: + return await run_autoprove_pipeline( + ctx=ctx, source_input=system_doc, env=source_env, handler_factory=handler, + prover_opts=prover_opts, interactive=args.interactive, ...) + +# foundry +async def runner(handler: HandlerFactory[FoundryPhase, None]) -> FoundryPipelineResult: + return await run_foundry_pipeline( + source_input=source_input, ctx=ctx, handler_factory=handler, env=env, + forge_binary=args.forge_binary, forge_timeout_s=args.forge_timeout_s, ...) +``` + +Convention points worth naming: + +- **Foundry validates its precondition in the entry point** (`foundry.toml` must + exist) — application-specific input validation belongs here, before any service is + opened. +- **Each application owns its RAG DB choice.** Foundry overrides `--rag-db`'s default + to the cheatcodes DB via a Protocol (`FoundryRAGDBOptions`) rather than a new flag. +- **The `finally` block is where run-close artifacts land** (autoprove dumps + `token_usage.json` there) — guarded so a diagnostics failure can't mask the run's + own outcome. +- **The entry point never imports a frontend.** It yields the Executor; `main()` + chooses the frontend. That is what lets one entry point back both a TUI and a + console `main()`. + +--- + +## 6. Piece 3 — the pipeline and its backend + +The Executor's closure calls the application's `run_*_pipeline` function. For both +current applications, that function is a thin wrapper that constructs a +`PipelineBackend` + a `PipelineRun` and hands them to the shared driver +[`run_pipeline`](../composer/pipeline/core.py): + +```python +# composer/spec/source/pipeline.py +async def run_autoprove_pipeline(source_input, ctx, handler_factory, env, *, prover_opts, ...): + backend = ProverBackend(ProverArtifactStore(source_input.project_root, source_input.contract_name), prover_opts) + run = PipelineRun(ctx, env, source_input, handler_factory, asyncio.Semaphore(max_concurrent)) + return await run_pipeline(backend, run, interactive=interactive, ...) +``` + +```python +# composer/foundry/pipeline.py +async def run_foundry_pipeline(source_input, ctx, handler_factory, env, *, forge_binary, ...): + artifacts = FoundryArtifactStore(source_input.project_root) + backend = FoundryBackend(artifacts, _ForgeRunConfig(forge_binary, forge_timeout_s, foundry_sem)) + run = PipelineRun(ctx, env, source_input, handler_factory, asyncio.Semaphore(max_concurrent)) + return await run_pipeline(backend, run, ...) +``` + +Notice the `handler_factory` (the frontend seam) is bundled into the `PipelineRun` — +`run.runner(task_info, job)` is how every phase of the driver spins up a task through +whatever frontend was supplied. + +The backend itself is the four-slot contract the driver reads. The application maps +its phase enum onto the four **core phases** the driver tags: + +```python +# composer/spec/source/pipeline.py # composer/foundry/pipeline.py +core_phases = CorePhases({ core_phases = CorePhases({ + "analysis": AutoProvePhase.COMPONENT_ANALYSIS, "analysis": FoundryPhase.SYSTEM_ANALYSIS, + "extraction": AutoProvePhase.BUG_ANALYSIS, "extraction": FoundryPhase.PROPERTY_EXTRACTION, + "formalization": AutoProvePhase.CVL_GEN, "formalization": FoundryPhase.TEST_GENERATION, + "report": AutoProvePhase.REPORT, "report": FoundryPhase.REPORT, +}) }) +``` + +Everything below this — `prepare_system`, `prepare_formalization`, `formalize`, +`fetch_verdicts` — is the **formalization abstraction**, documented in full in +[formalization-abstraction.md](./formalization-abstraction.md). The one-line summary +of the contrast: + +| | autoprove (`ProverBackend`) | foundry (`FoundryBackend`) | +|---|---|---| +| `FormT` | `GeneratedCVL` | `GeneratedFoundryTest` | +| `prepare_system` | harness lift + build prover tool | identity (`main_instance` only) | +| `prepare_formalization` | AutoSetup ∥ summaries ∥ invariants fan-out | trivial (formalizer already built) | +| `formalize` | author CVL, run prover, revise on CEX | author `.t.sol`, run `forge test` | +| `backend_guidance` | `CERTORA_BACKEND_GUIDANCE` | `FOUNDRY_BACKEND_GUIDANCE` | + +`backend_guidance` deserves a note as an application-shaping convention: it is a prose +string injected into the property-extraction prompt telling the agent what the +verification surface can and can't express. Foundry's, for instance, explains that a +fuzzer can't *prove* universals but *refutations are valuable* — so the same shared +extraction step produces backend-appropriate properties without the driver knowing +anything about it. + +--- + +## 7. Piece 4 — the frontend + +The frontend implements the `HandlerFactory` seam. The TUI frontends are thin +subclasses of the generic [`MultiJobApp[P, T]`](../composer/ui/multi_job_app.py) +(see [its design doc](../composer/ui/MULTI_JOB_DESIGN.md)). A frontend supplies four +things and inherits everything else: + +1. **Phase labels + section order** (constructor args), covered in §4. +2. **A per-task handler** — `create_task_handler`, returning a + `MultiJobTaskHandler` subclass. +3. **A per-task event handler** — `create_event_handler`, for domain-specific + streaming events beyond LLM messages. +4. **`make_handler`** — inherited from `MultiJobApp`; this *is* the `HandlerFactory`. + It mounts the panel/summary-row and calls the two `create_*` hooks. + +The autoprove and foundry TUIs are nearly identical in shape; they differ only in +what streams into a task's log. Both make their task handler double as its own +`EventHandler` via the `NullEventHandler` mixin: + +```python +# composer/ui/foundry_app.py +class FoundryTaskHandler(MultiJobTaskHandler[None], NullEventHandler): + async def handle_event(self, payload, path, checkpoint_id) -> None: + evt = cast(ForgeTestRunEvent, payload) + if evt["type"] == "forge_test_run": # stream forge run summaries + log = await self._ensure_forge_log() + log.write(evt["summary"]) + +class FoundryApp(MultiJobApp[FoundryPhase, FoundryTaskHandler]): + def __init__(self): + super().__init__(phase_labels=FOUNDRY_PHASE_LABELS, + section_order=FOUNDRY_SECTION_ORDER, + header_text="Foundry Test Author | ...") + def create_task_handler(self, panel, info) -> FoundryTaskHandler: + return FoundryTaskHandler(info.task_id, info.label, panel, self, ToolDisplayConfig()) + def create_event_handler(self, handler, info) -> EventHandler: + return handler # handler is its own event handler +``` + +```python +# composer/ui/autoprove_app.py — same structure; the domain events differ +class AutoProveTaskHandler(MultiJobTaskHandler[None], NullEventHandler): + async def handle_event(self, payload, path, checkpoint_id) -> None: + evt = cast(AutoProveEvent, payload) + match evt["type"]: + case "prover_output": ... # stream Certora Prover output lines + case "cloud_polling": ... # stream cloud job status +``` + +The autoprove handler additionally implements `handle_progress_event` to stream the +AutoSetup agent's output — an example of an application surfacing a backend-specific +sub-agent in its own panel. Neither handler supports HITL, so both raise from +`format_hitl_prompt` — a deliberate, explicit opt-out of a base-class hook. + +**The console frontend is the proof the seam works.** `AutoProveConsoleHandler` is a +*different* implementation of the same `HandlerFactory[AutoProvePhase, None]` that +renders to stdout instead of a Textual app: + +```python +# composer/ui/autoprove_console.py +class AutoProveConsoleHandler(MultiJobConsoleHandler[AutoProvePhase]): + """IOHandler[Never] + HandlerFactory for the auto-prove pipeline.""" +``` + +The pipeline can't tell the difference — it only ever calls `make_handler`. + +--- + +## 8. Piece 5 — the artifact store + +Deliverables are written through an [`ArtifactStore[I, FormT]`](../composer/spec/artifacts.py) +subclass — one per application. The base owns everything identical across +applications (`properties.json`, `commentary.md`, the property→units map, +`token_usage.json`); the subclass fixes the on-disk layout and adds the +format-specific bundle. This is covered in detail in +[formalization-abstraction.md §6](./formalization-abstraction.md); the application-level +point is the *convention that both applications share a project root without +colliding*: + +``` +autoprove → certora/specs/… certora/confs/… certora/ap_report/… +foundry → /*.t.sol certora/foundry/… certora/foundry/reports/… +``` + +Foundry deliberately materializes its `.t.sol` into the foundry project's own `test/` +dir (so `forge` finds them) but keeps all metadata under `certora/foundry/`, so a +co-located autoprove run and foundry run share one project without clobbering each +other's outputs. + +--- + +## 9. Piece 0 — the wiring: `main()` + +The `main()` in each `composer/cli/*.py` is the whole application in ~20 lines. It is +the *only* place that names both an entry point and a frontend, and its job is to +glue them via the seam and translate the result into user-facing output. + +```python +# composer/cli/tui_foundry.py (tui_autoprove.py is identical in shape) +async def _main() -> int: + summary = RunSummary() + async with _entry_point(summary) as pipeline: # piece 2: Executor + app = FoundryApp() # piece 4: frontend + async def work(): + result = await pipeline(app.make_handler) # ← the seam + app.notify(f"Foundry tests complete: {result.n_components} components, ...") + app._pipeline_done = True + app.set_work(work) + await app.run_async() # TUI owns the event loop + print(summary.format()) + return 0 +``` + +Two `main()` shapes exist, differing only in who owns the event loop: + +- **TUI** — the pipeline runs as a background *worker* inside the Textual app + (`app.set_work(work); await app.run_async()`), so the UI stays responsive while + the pipeline streams into it. +- **Console** — the pipeline runs directly and results print on completion: + + ```python + # composer/cli/console_autoprove.py + async with _entry_point(summary) as run: + result = await run(AutoProveConsoleHandler().make_handler) + print(summary.format()) + print(f" Components: {result.n_components} Properties: {result.n_properties}") + ``` + +Both call `import composer.bind as _` first — the side-effecting binding module that +must load before anything touches the DI container. + +--- + +## 10. Extending: defining a new application + +Because each piece is an implementation of a shared abstraction, adding an +application is a fill-in-the-blanks exercise; nothing in the driver, the UI base, or +the seam changes. + +1. **Phase enum** `P(enum.Enum)` — your task-grouping vocabulary, with at least the + four core phases (analysis / extraction / formalization / report) representable. +2. **Backend** — implement `PipelineBackend[P, FormT, H, A]` and its three phase + objects (`prepare_system` → `PreparedSystem.prepare_formalization` → `Formalizer`), + plus `backend_guidance`, `core_phases`, `analysis_spec`, `artifact_store`, + `to_artifact_id`. (Full checklist in + [formalization-abstraction.md §9](./formalization-abstraction.md).) +3. **Artifact store** — subclass `ArtifactStore`; define an `ArtifactIdentifier`. +4. **Result type** `FormT` satisfying `FormalResult` + `ReportableResult`. +5. **Pipeline function** `run__pipeline(...)` — build backend + `PipelineRun`, + call `run_pipeline`. +6. **Entry point** — an `_entry_point` context manager following the + parse → services → `yield runner` convention; declare args as a `Protocol`. +7. **Frontend(s)** — a `MultiJobApp[P, T]` subclass (phase labels, section order, + `create_task_handler`, per-task event streaming) and/or a console handler. +8. **`main()`** — glue an entry point to a frontend via `run(app.make_handler)`. + +The dependency direction is the guardrail: `main` → (entry point, frontend); +entry point → pipeline; pipeline → backend + `PipelineRun(handler_factory)`. Frontend +and backend never reference each other, and neither references `main`. Keep those +edges and the pieces stay swappable. + +--- + +## 11. Key files + +| Piece | autoprove | foundry | shared abstraction | +|---|---|---|---| +| Phase enum | [autoprove_app.py](../composer/ui/autoprove_app.py) | [foundry/pipeline.py](../composer/foundry/pipeline.py) | `HasName` ([multi_job.py](../composer/io/multi_job.py)) | +| Entry point / Executor | [autoprove_common.py](../composer/spec/source/autoprove_common.py) | [foundry/entry.py](../composer/foundry/entry.py) | — | +| Pipeline + backend | [spec/source/pipeline.py](../composer/spec/source/pipeline.py) | [foundry/pipeline.py](../composer/foundry/pipeline.py) | [pipeline/core.py](../composer/pipeline/core.py) | +| Frontend (TUI) | [autoprove_app.py](../composer/ui/autoprove_app.py) | [foundry_app.py](../composer/ui/foundry_app.py) | [multi_job_app.py](../composer/ui/multi_job_app.py) | +| Frontend (console) | [autoprove_console.py](../composer/ui/autoprove_console.py) | — | — | +| Artifact store | [spec/source/artifacts.py](../composer/spec/source/artifacts.py) | [foundry/artifacts.py](../composer/foundry/artifacts.py) | [spec/artifacts.py](../composer/spec/artifacts.py) | +| The seam | — | — | `HandlerFactory` / `TaskInfo` / `TaskHandle` ([multi_job.py](../composer/io/multi_job.py)) | +| `main()` | [tui_autoprove.py](../composer/cli/tui_autoprove.py) · [console_autoprove.py](../composer/cli/console_autoprove.py) | [tui_foundry.py](../composer/cli/tui_foundry.py) · [console_foundry.py](../composer/cli/console_foundry.py) | — | diff --git a/docs/ecosystem-abstraction.md b/docs/ecosystem-abstraction.md new file mode 100644 index 00000000..ed6daf10 --- /dev/null +++ b/docs/ecosystem-abstraction.md @@ -0,0 +1,468 @@ +# Proposal — The Ecosystem Abstraction (EVM, Solana, Soroban) + +> A proposal to make AutoProver's shared pipeline parametric over an **ecosystem** — +> the blockchain/source domain being analyzed — selected by an application-level parameter +> that picks the right system model, prompts, source conventions, and validation. Today the +> "generic" pipeline is generic over the *backend* (how a property becomes a verified +> artifact) but silently hardwired to Solidity for everything else. This introduces a second, +> orthogonal axis — and factors it further into a **language** facet (Solidity, Rust) and a +> **chain** facet (EVM, Solana, Soroban), so the Rust-specific prompts and source conventions +> are written once and shared between Solana and Soroban while their blockchain-specific +> models and failure modes stay separate. +> +> Companion to [formalization-abstraction.md](./formalization-abstraction.md) (the backend +> seam), [application-abstraction.md](./application-abstraction.md) (the five pieces of an +> application), and [rust-applications.md](./rust-applications.md) (the Rust app framework +> the first Solana/Soroban backends will likely use). Status: **proposal / for review.** + +--- + +## 1. Problem & motivation + +We want to author properties for **Solana** programs (Rust/Anchor) and **Soroban** contracts +(Rust/soroban-sdk), not just EVM/Solidity. The pipeline advertises itself as backend-agnostic, +and it is — `run_pipeline` is generic over the result type `FormT` +([formalization-abstraction.md](./formalization-abstraction.md)). But "backend-agnostic" is +not "domain-agnostic." An audit of the shared spine (see §3) shows Solidity/EVM assumptions +baked into three shared places the driver owns: + +1. the **system model** — the pydantic types the analysis phase produces; +2. the **prompts** — the analysis and property-extraction templates; +3. a few **source conventions** — the fs-exclusion default, the "main contract" locator. + +None of this is tooling (no `solc`/`slither` in the shared steps — it's pure LLM + generic +file reading), so the work is types + prompts + a small driver generalization, not new +analysis engines. + +We propose making this a first-class **ecosystem** parameter, so `evm` reproduces today's +behavior exactly and `solana` / `soroban` slot in beside it without forking the pipeline — +and, because two of those three are Rust, factoring the ecosystem so the Rust-specific parts +are shared rather than copied (§2.1). + +--- + +## 2. Two orthogonal axes + +The pipeline has a front half and a back half joined by *properties*: + +``` + ┌─────────── ECOSYSTEM owns ───────────┐ ┌──────── BACKEND owns ────────┐ +source ─analyze─▶ SystemModel ─extract─▶ properties ─formalize─▶ artifact ─verdicts─▶ + (how we MODEL and REASON about (how a property becomes a + the domain: contracts vs programs, checkable, verified artifact: + storage vs accounts, reentrancy CVL+prover / foundry / a Rust + vs missing-signer) Solana verifier) + └──────── SHARED: report ────────┘ +``` + +- **Ecosystem** = the *front half*: the system-model types, the analysis + property-extraction + prompts, source conventions, and connectivity validation. +- **Backend** = the *back half*: `prepare_system` → `Formalizer` (`formalize` / `fetch_verdicts`), + documented in [formalization-abstraction.md](./formalization-abstraction.md). +- **Report** stays shared and neutral. + +The axes are conceptually independent — but they meet at the analyzed model: the backend's +`prepare_system(analyzed: App)` consumes the ecosystem's `App` type. So a **backend is written +against an ecosystem's model** (the CVL prover backend needs `SourceApplication`; a Solana +verifier needs `SolanaApplication`). An application picks a **(ecosystem, backend) pair that +agree on `App`.** That pairing is the one coupling to make explicit (§6). + +| | EVM | Solana | Soroban | +| --- | --- | --- | --- | +| Compatible backends | `prover` (CVL), `foundry` | a Rust verifier (via `rustapp`) | a Rust verifier (via `rustapp`) | +| System model | `SourceApplication` (contracts) | `SolanaApplication` (programs) — new | `SorobanApplication` (contracts) — new | +| Unit of extraction | contract component | instruction / account-validation group | contract function | + +### 2.1 Ecosystems factor into a *language* facet and a *chain* facet + +Solana and Soroban are both **Rust**; EVM is **Solidity**. Much of what the front half needs +is fixed by the *source language* — how to read/navigate it, the project layout to exclude +(Cargo vs Foundry), the language-level failure modes (Rust: integer overflow, `panic!` / +`unwrap` / `expect` aborts, ownership) — and is identical across every chain that uses that +language. The rest is fixed by the *chain/platform*: the system model, the storage and +authorization semantics, the platform failure modes. So an ecosystem is a **composition**: + +```text + ecosystem = language facet ⊕ chain facet + evm = solidity ⊕ evm + solana = rust ⊕ solana ┐ share the SAME rust language facet + soroban = rust ⊕ soroban ┘ (prompts, fs conventions, overflow/panic) +``` + +| Facet | Owns | `solidity` | `rust` (shared by `solana` + `soroban`) | +| --- | --- | --- | --- | +| **Language** | fs-exclusion default, `code_explorer` prompt, source-navigation framing, the language-level failure-mode prompt fragment | assembly / delegatecall, checked-arith caveat | integer overflow/underflow, `panic!`/`unwrap`/`expect` aborts, ownership/borrow | +| **Chain** | system-model type, connectivity validation, main-unit locator + units, the platform failure-mode prompt fragment, SDK conventions | contracts, storage, ERC standards | *differs per chain* — see §8 | + +The payoff (the Soroban question directly): **the Rust language facet is authored once and +shared by both Solana and Soroban** — the Rust source conventions, the `code_explorer` prompt, +and the Rust failure-mode prompt fragment. Only the chain facet differs between them. + +One honest caveat: the sharing is **not strictly hierarchical.** Solana and Soroban share the +Rust *language*, but Soroban's *model* — a contract that owns typed storage, with explicit +authorization and cross-contract calls — is closer to EVM's than to Solana's (programs +operating on externally-passed accounts). So facets are best expressed as **composable prompt +fragments keyed by concern** (§4.1), not a rigid two-level inheritance: `soroban` pulls the +`rust` language fragments *and* may reuse EVM-flavored "contract-owns-storage / authorization" +analysis fragments, while `solana` does not. + +--- + +## 3. What is Solidity-specific today (audit) + +Condensed from the audit; all in the *shared* pipeline, not the backends. + +| Concern | Where | Ecosystem-specific? | +|---|---|---| +| System-model types (`ExplicitContract`, `solidity_identifier` + regex, `ContractComponent.state_variables`/`external_entry_points`, `ContractSort`, EOA `ExternalActor`) | [system_model.py](../composer/spec/system_model.py) | **Yes** — EVM-shaped | +| Driver pins the model: `run_component_analysis(ty=SourceApplication)`, `prepare_system(analyzed: SourceApplication)`, `main_instance` matching `solidity_identifier`, `_extract_all` → `ContractComponentInstance`, the "explicit contract instance with this solidity identifier" `extra_input` | [core.py](../composer/pipeline/core.py) | **Yes** — hardcoded | +| Analysis prompts ("Smart Contracts", `solidity_identifier` block, `ContractSort` deploy semantics, ERC20/4626/721) | `application_analysis_system.j2` / `application_analysis_prompt.j2` | **Yes** — heavy rewrite | +| Property-extraction prompts (reentrancy, oracle manipulation, MEV, storage layout, checked arithmetic/`uint256`) | `property_analysis_system_prompt.j2` / `property_analysis_prompt.j2` | **Yes** — failure-mode vocabulary | +| Connectivity validation (contract/component/actor shape) | `_validate_connectivity` in [system_analysis.py](../composer/spec/system_analysis.py) | **Yes** — structure reusable, types EVM | +| fs-exclusion default (`lib/`, `test/`, `.sol` carve-out) | `FS_FORBIDDEN_READ` [util.py:59](../composer/spec/util.py) | **Yes** — but already a per-input param | +| `code_explorer` prompt ("smart contract source code") | [code_explorer.py](../composer/spec/code_explorer.py) | Cosmetic | +| Source tools (`fs_tools`, `code_explorer`, `code_document_ref`) | [source_env.py](../composer/spec/source/source_env.py) | **No** — language-neutral, read Rust fine | +| `backend_guidance` ("what's expressible downstream") | [prop_inference.py](../composer/spec/prop_inference.py) | **No** — already backend-supplied | +| Report (`Verdict`, `RuleName`, `unit_file`, `Outcome`) | [report/collect.py](../composer/spec/source/report/collect.py) | **No** — neutral | + +Two useful facts fell out: `run_component_analysis` is *already* generic (`[T: BaseApplication]`) +— only the driver pins `SourceApplication`; and `SolidityIdentifier`'s regex already accepts +Rust identifiers, so it's not a hard blocker (just misnamed). + +--- + +## 4. The seam: a `Language` facet and a `Chain` facet + +Two small protocols, composed. The **chain** carries its **language**; the driver consumes the +composed ecosystem (which is just a resolved chain). Sketch (illustrative, not final signatures): + +```python +# composer/pipeline/ecosystem.py +type LanguageTag = Literal["solidity", "rust"] +type ChainTag = Literal["evm", "solana", "soroban"] + +@dataclass(frozen=True) +class PromptPair: + system: str # j2 template name + initial: str + +class Language(Protocol): + """Shared by every chain that uses this source language.""" + name: LanguageTag + default_forbidden_read: str # Cargo layout vs Foundry layout + code_explorer_prompt: str # "Rust source" vs "Solidity source" + failure_modes_partial: str # j2 partial: language-level failure modes (overflow, panics …) + +class Chain[App: BaseApplication, Main, Unit](Protocol): + name: ChainTag + language: Language # <-- the shared facet (RUST for solana AND soroban) + + # --- domain model --- + system_model: type[App] # the analyzed pydantic type + def validate_analysis(self, app: App, expected_main: str | None) -> list[str]: ... + def locate_main(self, app: App, source: SourceCode) -> Main: ... + def units(self, main: Main) -> list[Unit]: ... + + # --- prompts (chain templates that compose in the language partials, §4.1) --- + analysis_prompts: PromptPair + property_prompts: PromptPair + +type Ecosystem = Chain # an ecosystem is a chain that carries its language +``` + +`Main`/`Unit` generalize today's `ContractInstance` / `ContractComponentInstance` — thin index +wrappers over `App` that the driver hands to the backend (`to_artifact_id`, `prepare_system`) and +to property inference. For EVM they *are* those types unchanged. + +A registry selects by chain tag; `RUST` and `SOLIDITY` are the shared language singletons: + +```python +RUST = _RustLanguage(...) # authored ONCE +SOLIDITY = _SolidityLanguage(...) +ECOSYSTEMS: dict[ChainTag, Ecosystem] = {"evm": EVM, "solana": SOLANA, "soroban": SOROBAN} +# where SOLANA.language is RUST and SOROBAN.language is RUST — the same object. +``` + +### 4.1 Prompt composition — how the Rust prompts are shared + +Prompts are **assembled from fragments with Jinja2 includes/inheritance**, not duplicated. A +chain's property template pulls the shared language fragment, then adds its own: + +```jinja +{# composer/templates/solana/property_prompt.j2 (soroban/ is identical but for the last include) #} +{% extends "property_prompt_base.j2" %} +{% block failure_modes %} + {% include "rust/_failure_modes.j2" %} {# SHARED: overflow, panics, unwrap — solana AND soroban #} + {% include "solana/_failure_modes.j2" %} {# chain-specific: signer/owner/PDA/CPI checks #} +{% endblock %} +``` + +So `rust/_failure_modes.j2`, the Rust `code_explorer` prompt, and the Cargo `forbidden_read` are +authored once as the `RUST` language and referenced by both `solana` and `soroban`; Soroban swaps +only the final `{% include "soroban/_failure_modes.j2" %}` and its chain facet. The base template +(`property_prompt_base.j2`) holds the ecosystem-neutral skeleton — the invariant / safety / +attack-vector framing, the output contract — so *that* is shared across all three. + +--- + +## 5. Selection: an application parameter + +The ecosystem is chosen per application, threaded to `run_pipeline` alongside the backend. + +- **Built-in apps.** `run_autoprove_pipeline` / `run_foundry_pipeline` pass `ecosystem=EVM` + explicitly (one line; both are EVM). +- **Rust apps.** The [`AppDescriptor`](./rust-applications.md) gains an `ecosystem` field + (default `"evm"`); the host resolves `ECOSYSTEMS[descriptor.ecosystem]` and passes it into + `run_application`. This adds one string field to `rust/autoprover-sdk` and one lookup in + `composer/rustapp/host.py`. + +```python +# composer/rustapp/descriptor.py +class AppDescriptor(BaseModel): + ... + ecosystem: ChainTag = "evm" # "evm" | "solana" | "soroban" +``` + +So a Solana application is `ecosystem="solana"` + a Solana backend wheel, and a Soroban +application is `ecosystem="soroban"` + a Soroban backend wheel. Nothing else in the app shell +changes — the generic entry point/frontend already synthesize from the descriptor, and both Rust +chains transparently pick up the shared `RUST` language facet. + +--- + +## 6. Driver generalization + +Localized changes; the phase chain and concurrency are untouched. + +**`run_pipeline`** ([core.py](../composer/pipeline/core.py)) takes `ecosystem: Ecosystem` and +stops hardcoding EVM: + +```python +async def run_pipeline[P, FormT, H, A, App]( + backend: PipelineBackend[P, FormT, H, A, App], + run: PipelineRun[P, H], + ecosystem: Ecosystem[App, ...], + *, ... +): + analyzed = await run.runner(..., lambda: run_component_analysis( + ty=ecosystem.system_model, # was: SourceApplication + prompts=ecosystem.analysis_prompts, # was: hardcoded templates + validate=ecosystem.validate_analysis, # was: _validate_connectivity + expected_main_id=source.contract_name, ...)) + prepared = await backend.prepare_system(analyzed, run) + ... + main = ecosystem.locate_main(analyzed, run.source) # was: main_instance(...) by solidity_identifier + batches = await _extract_all(ecosystem.units(main), ecosystem.property_prompts, ...) +``` + +**`run_component_analysis`** ([system_analysis.py](../composer/spec/system_analysis.py)) — +already generic over `T`; additionally accept the prompt pair + validation function instead of +importing `_validate_connectivity` and hardcoding template names. + +**`run_property_inference`** ([prop_inference.py](../composer/spec/prop_inference.py)) — accept +the ecosystem's property prompt pair and a generic `Unit` (it already takes `backend_guidance` +as a param, so the "expressible downstream" axis stays backend-owned; the "failure modes in this +domain" axis moves into the ecosystem's prompt). + +**`PipelineBackend` / `SystemAnalysisSpec`** — add the `App` type parameter so +`prepare_system(analyzed: App)` and `to_artifact_id(unit: Unit)` type-check against the paired +ecosystem. `SystemAnalysisSpec` keeps `analysis_key` + `extra_input` (backend/app-owned); the +analyzed *type* and templates move to the ecosystem. + +--- + +## 7. The EVM ecosystem (= today, zero behavior change) + +`EVM` = the `SOLIDITY` language facet ⊕ the `evm` chain facet, a faithful capture of current +behavior, so autoprove/foundry are byte-for-byte unchanged: + +```python +SOLIDITY = _Language( + name="solidity", + default_forbidden_read=FS_FORBIDDEN_READ, + code_explorer_prompt=CODE_EXPLORER_SYS_PROMPT, + failure_modes_partial="solidity/_failure_modes.j2", +) + +EVM = _Chain( + name="evm", + language=SOLIDITY, + system_model=SourceApplication, + analysis_prompts=PromptPair("application_analysis_system.j2", "application_analysis_prompt.j2"), + property_prompts=PromptPair("property_analysis_system_prompt.j2", "property_analysis_prompt.j2"), + validate_analysis=_validate_connectivity, # moved, not rewritten + locate_main=main_instance, # moved, not rewritten + units=lambda main: [ContractComponentInstance(_contract=main, ind=i) + for i in range(len(main.contract.components))], +) +``` + +The migration is a *move*, not a rewrite: existing types, prompts, and functions become the EVM +ecosystem's members (the current monolithic prompts stay as-is at first; splitting out a +`solidity/_failure_modes.j2` partial can wait until Soroban wants to reuse EVM fragments, §8). +This is the safety property of the proposal — the refactor is provably behavior-preserving for +EVM before any Rust chain adds anything. + +--- + +## 8. The Rust chains: Solana and Soroban (shared language facet) + +Both are `RUST` ⊕ their chain facet. The **`RUST` language facet is authored once** and both +reuse it verbatim: + +```python +RUST = _Language( + name="rust", + # Cargo layout: exclude target/ and .git; KEEP tests/ (unlike Foundry) and the crate sources. + default_forbidden_read=r"(^target/.*)|(^\.git.*)|(.*\.lock$)", + code_explorer_prompt=RUST_CODE_EXPLORER_PROMPT, # "Rust source … modules/traits/impls" + failure_modes_partial="rust/_failure_modes.j2", # overflow/underflow, panic!/unwrap/expect, ownership +) +``` + +`rust/_failure_modes.j2` is the concrete answer to "share Rust prompts between Solana and +Soroban": it is `{% include %}`d by both chains' property templates (§4.1). Each chain then +supplies only its own model + validation + platform fragment. + +### 8.1 Solana chain (`RUST ⊕ solana`) + +- **System model** (`composer/spec/solana/model.py`, new) — `SolanaApplication` with `Program` + (program id / `crate::module`), `Instruction` (entry points), `AccountGroup` / account + constraints (Solana accounts are **passed in**, not owned storage), and CPI targets / signers + in place of EOA `ExternalActor`. +- **Platform failure fragment** (`solana/_failure_modes.j2`) — missing signer/owner checks, + account substitution / confused-deputy, unvalidated PDA seeds, arbitrary CPI, lamport/rent + draining, missing Anchor constraints (`has_one`, `constraint`, `seeds`/`bump`). +- **`locate_main` / `units`** — main = the target program; units = its instructions (or + account-validation structs). + +### 8.2 Soroban chain (`RUST ⊕ soroban`) + +- **System model** (`composer/spec/soroban/model.py`, new) — `SorobanApplication` with `Contract` + (`#[contract]`), `ContractFunction` (`#[contractimpl]` entry points), and **typed contract + storage** (`instance` / `persistent` / `temporary`, each with TTL/archival) — i.e. the contract + **owns** its state, closer to EVM than to Solana. Authorization is explicit (`require_auth` / + `require_auth_for_args`); cross-contract calls go through generated clients; custom types are + `#[contracttype]`. +- **Platform failure fragment** (`soroban/_failure_modes.j2`) — missing/incorrectly-scoped + `require_auth`, storage-durability misuse (temporary vs persistent) and TTL/archival + (entry-expiration) bugs, unchecked cross-contract results / reentrancy, replay. The Rust + overflow/panic modes come from the shared `rust/_failure_modes.j2` — **not repeated here.** +- **`locate_main` / `units`** — main = the target contract; units = its contract functions. +- **Fragment reuse across the *chain* axis, too.** Because Soroban's model is + contract-owns-typed-storage with explicit authorization, its *analysis* prompt can + `{% include %}` EVM-flavored fragments about storage and authorization that Solana cannot — + the non-hierarchical sharing noted in §2.1. This is exactly why fragments beat rigid inheritance. + +The Solana/Soroban *backends* (formalization) are out of scope here — each plugs in via +[rust-applications.md](./rust-applications.md) and pairs with its chain's `App` model. + +--- + +## 9. What stays shared and unchanged + +- Source tools (`fs_tools`, `code_explorer`, `code_document_ref`) — already language-neutral; the + only ecosystem input is the `forbidden_read` default and the explorer prompt string. +- The report (`collect` / `Verdict` / schema) — neutral; `ReportBackend` already widened. +- Caching, the multi-round property loop, interactive refinement, `run_to_completion`, the whole + agent plumbing. +- The backend seam and the Rust app framework — a Solana or Soroban verifier is "just another + backend." + +--- + +## 10. Phased plan + +*Status: Phases 1–4 implemented (EVM preserved; Solana front half verified by a live gate). +Phase 5 (Soroban) and Phase 6 (verification backends) remain.* + +1. ✅ **Done — Extract `Language` + `Chain` + `EVM` (= `SOLIDITY ⊕ evm`), behavior-preserving.** + Added `composer/pipeline/ecosystem.py`; moved the `SourceApplication` reference, template + names, `_validate_connectivity`, `main_instance`, and unit-enumeration into `EVM`, and the + `forbidden_read`/`code_explorer` defaults into `SOLIDITY`. Threaded `ecosystem` through + `run_pipeline` / `run_component_analysis` / `run_property_inference`. Prompts stay monolithic + (no fragment split yet). **Gate met:** the autoprove end-to-end integration test passes + identically on this commit and its pre-refactor parent (real Postgres + live prover), and EVM + reproduces the prior template names / validator / analysis front-matter verbatim. + > **Env note (orthogonal to the refactor):** running that gate surfaced a pre-existing solc + > provisioning issue — the Counter scenario pins `pragma ^0.8.29` but the environment's default + > `solc` was 0.8.21, so the prover couldn't compile it (manifesting as an "exhausted tape"). The + > fix is environmental (point `solc` at ≥0.8.29; versioned `solc8.29` was already present), not a + > tape re-record. Worth pinning a matching `solc` in the gate's prover config so it's robust. +2. ✅ **Done — Add the `App` type parameter.** `PipelineBackend[P, FormT, H, A, App]` with + `prepare_system(analyzed: App)`; `Ecosystem` is generic over `App` (`EVM: Ecosystem[SourceApplication]`); + `run_pipeline` takes `ecosystem: Ecosystem[App]` explicitly and the Phase 1 + `cast(SourceApplication, analyzed)` is gone. `SystemAnalysisSpec` was intentionally **not** + parameterized — it carries only `analysis_key` + `extra_input`, no `App`-typed member. + **Gate met:** pyright reports 0 errors on the touched files (pairing type-checks, no cast). +3. ✅ **Done — Wire selection into `rustapp`.** `AppDescriptor.ecosystem: ChainTag` (Rust SDK, + default `"evm"`, + the Python mirror) and `resolve_ecosystem()` in `host.py` (registry lookup, + clear error for an unregistered chain), threaded through `build_application` / + `run_application` / `run_rust_pipeline` in place of the hardcoded `EVM`. **Gate met:** rustapp + tests + pyright green; only `evm` resolves for now (Solana/Soroban register in Phases 4–5). +4. ✅ **Done — Author the `RUST` language facet + the Solana chain.** This phase also performed + the driver's `Unit`/`Main` generalization deferred from Phase 2 (a `FeatureUnit` protocol in + `system_model`, threaded as type params through `Ecosystem`/`PreparedSystem`/`PipelineBackend`/ + `Formalizer`/`run_pipeline`; EVM stays bound to `ContractComponentInstance`/`ContractInstance` + with byte-identical cache keys). Added the standalone `SolanaApplication` model + (`composer/spec/solana/model.py`), the `RUST` language (Cargo `forbidden_read`, Rust + `code_explorer` prompt, `rust/_failure_modes.j2`) and the `SOLANA` chain (validate / + `locate_main` / `units`). The fragment-composition convention is realized with `{% include %}` + of the shared `rust/_failure_modes.j2` + `solana/_failure_modes.j2` into the Solana templates + (a neutral `property_prompt_base.j2` proved unnecessary — the Solana templates are + self-contained and just pull the fragments). A reusable `NullSolanaBackend` + an Anchor + `solana_vault` scenario back the gate. **Gate met:** a live-LLM run + (`tests/test_solana_gate.py`) analyzed the vault into 3 instructions and extracted 27 sane, + Solana-native properties (signer/owner checks, PDA + canonical-bump derivation, System-Program + substitution, reinit-once, arithmetic overflow) — no prover needed. +5. **Author the Soroban chain, reusing `RUST`.** `SorobanApplication` model + Soroban prompts; + the *only* new prompt content is `soroban/_failure_modes.j2` and the Soroban analysis template + — the Rust language fragments are inherited. **Gate:** the shared `rust/_failure_modes.j2` is + referenced by both chains and appears in neither chain's own fragment (no duplication). +6. **Solana / Soroban backends** — separate efforts, via the Rust framework. + +--- + +## 11. Open questions + +1. **Is `Unit` uniform enough across ecosystems?** EVM's unit is a contract component; Solana's + might be an instruction *or* an account-validation struct. If per-unit shape diverges too much, + `run_property_inference` may need the ecosystem to own unit rendering (a `render_unit(unit) -> dict` + hook) rather than a shared template variable. Decide when authoring the Solana property prompt. +2. **One backend, multiple ecosystems?** Could a single Rust backend serve both (unlikely given the + `App` pairing)? Keep the pairing explicit for now; revisit if a genuinely cross-domain backend + appears. +3. **Report labels by ecosystem.** Should the report say "program"/"instruction" vs + "contract"/"rule"? Today `backend_tag` drives labels; an `ecosystem` tag on the report may be + the cleaner source for domain nouns. Minor; defer. +4. **Prompt template packaging.** Templates live in a shared dir; the fragment convention needs + per-facet subdirs (`rust/`, `solidity/`, `solana/`, `soroban/`, plus shared `*_base.j2`) and a + Jinja loader that resolves includes across them. Confirm the loader supports `{% extends %}` / + `{% include %}` across subdirs (it should — it's stock Jinja). +5. **Fragment granularity.** Is a single `_failure_modes.j2` partial per facet the right grain, or + do we want finer per-concern fragments (storage / authorization / arithmetic) so Soroban can + pull the EVM *storage/auth* fragments (§8.2) without the EVM *contract-deployment* ones? Start + coarse (one partial per facet); split a fragment only when a second consumer wants part of it. +6. **`SolidityIdentifier` naming.** Rename the shared field/type to an ecosystem-neutral + `SourceIdentifier`, or leave it (regex already fits Rust)? Cosmetic; a rename touches many + annotations — sequence it after Phase 1. + +--- + +## 12. Key files + +| Concern | File | +|---|---| +| Driver to generalize | [composer/pipeline/core.py](../composer/pipeline/core.py) | +| Ecosystem seam (new) | `composer/pipeline/ecosystem.py` | +| System analysis (accept ecosystem) | [composer/spec/system_analysis.py](../composer/spec/system_analysis.py) | +| Property inference (accept ecosystem) | [composer/spec/prop_inference.py](../composer/spec/prop_inference.py) | +| EVM system model (→ EVM ecosystem) | [composer/spec/system_model.py](../composer/spec/system_model.py) | +| Analysis / property prompts (EVM) | `composer/templates/application_analysis_*.j2` · `property_analysis_*.j2` | +| fs-exclusion default | [composer/spec/util.py](../composer/spec/util.py) | +| Source tools (unchanged) | [composer/spec/source/source_env.py](../composer/spec/source/source_env.py) · [code_explorer.py](../composer/spec/code_explorer.py) | +| Language facets + shared prompt fragments (new) | `composer/templates/{rust,solidity}/_failure_modes.j2` · `*_base.j2` | +| Solana chain model / prompts (new) | `composer/spec/solana/…` · `composer/templates/solana/…` | +| Soroban chain model / prompts (new) | `composer/spec/soroban/…` · `composer/templates/soroban/…` | +| Ecosystem selection in Rust apps | [composer/rustapp/descriptor.py](../composer/rustapp/descriptor.py) · [host.py](../composer/rustapp/host.py) · [rust/autoprover-sdk/src/lib.rs](../rust/autoprover-sdk/src/lib.rs) | +| The backend seam (unchanged) | [docs/formalization-abstraction.md](./formalization-abstraction.md) | From 93320ca5ad3abbfcc37c746b28b82e8e152c5bf8 Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 13:59:44 -0700 Subject: [PATCH 02/19] pipeline: type PreparedSystem.main via a Main generic instead of Any MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The located "main" was `Any`. It is a distinct axis from `FeatureUnit` — the per-unit protocol the extraction phase iterates — and EVM's main (`ContractInstance`) is not a FeatureUnit at all, so it can't just be typed as one. It IS the `Main` type the ecosystem seam already carries (`Ecosystem[App, Main, Unit]`). Thread that `Main` through: `PreparedSystem[FormT, U, Main]` (main: Main), `PipelineBackend[..., U, Main]` (prepare_system -> PreparedSystem[FormT, U, Main]), and `run_pipeline`. Each backend now binds it concretely — EVM foundry/prover to `ContractInstance`, the Solana null backend to `SolanaProgramInstance`. The internal `_extract_all` keeps `main: Any` on purpose: it drives the type-erased `Ecosystem[Any, Any, Any]`, so there is nothing to tie it to there. Pyright (CI paths) clean; pipeline/foundry unit tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- composer/foundry/pipeline.py | 6 +++--- composer/pipeline/cli.py | 8 ++++---- composer/pipeline/core.py | 22 ++++++++++++++-------- composer/spec/solana/null_backend.py | 8 ++++---- composer/spec/source/pipeline.py | 9 +++++---- 5 files changed, 30 insertions(+), 23 deletions(-) diff --git a/composer/foundry/pipeline.py b/composer/foundry/pipeline.py index 2ff056c4..918de59b 100644 --- a/composer/foundry/pipeline.py +++ b/composer/foundry/pipeline.py @@ -78,7 +78,7 @@ system (a uint256 being non-negative, etc.) are also uninteresting. """ from composer.spec.system_model import ( - ContractComponentInstance, SourceApplication, + ContractComponentInstance, ContractInstance, SourceApplication, ) from composer.io.multi_job import HandlerFactory @@ -138,7 +138,7 @@ async def fetch_verdicts(self, inp: ReportComponentInput[GeneratedFoundryTest]) return await _foundry_verdicts(inp) @dataclass -class FoundrySystem(PreparedSystem[GeneratedFoundryTest, ContractComponentInstance]): +class FoundrySystem(PreparedSystem[GeneratedFoundryTest, ContractComponentInstance, ContractInstance]): form: FoundryFormalizer @override @@ -166,7 +166,7 @@ async def prepare_system( self, analyzed: SourceApplication, run: PipelineRun[FoundryPhase, None] - ) -> PreparedSystem[GeneratedFoundryTest, ContractComponentInstance]: + ) -> PreparedSystem[GeneratedFoundryTest, ContractComponentInstance, ContractInstance]: return FoundrySystem( main_instance( analyzed, run.source diff --git a/composer/pipeline/cli.py b/composer/pipeline/cli.py index 60ac151e..45cb65db 100644 --- a/composer/pipeline/cli.py +++ b/composer/pipeline/cli.py @@ -114,10 +114,10 @@ class StagedPipeline: root_key: str class Continuation[P: enum.Enum, H](Protocol): - async def __call__[FormT: BackendResult, A: ArtifactIdentifier, U: FeatureUnit]( + async def __call__[FormT: BackendResult, A: ArtifactIdentifier, U: FeatureUnit, Main]( self, env: ServiceHost, - backend: PipelineBackend[P, FormT, H, A, U] + backend: PipelineBackend[P, FormT, H, A, U, Main] ) -> CorePipelineResult[FormT]: ... @@ -250,9 +250,9 @@ async def cli_pipeline[P: enum.Enum, H]( relative_path=init_source.relative_path ) - async def cont[FormT: BackendResult, A: ArtifactIdentifier, U: FeatureUnit]( + async def cont[FormT: BackendResult, A: ArtifactIdentifier, U: FeatureUnit, Main]( env: ServiceHost, - backend: PipelineBackend[P, FormT, H, A, U] + backend: PipelineBackend[P, FormT, H, A, U, Main] ) -> CorePipelineResult[FormT]: full_ctx = WorkflowContext.create( services=conns.memory, diff --git a/composer/pipeline/core.py b/composer/pipeline/core.py index 31afb878..56611090 100644 --- a/composer/pipeline/core.py +++ b/composer/pipeline/core.py @@ -90,16 +90,20 @@ async def finalize(self, outcomes: list[ComponentOutcome[FormT, U]], run: Pipeli return None @dataclass -class PreparedSystem[FormT: BackendResult, U: FeatureUnit](ABC): - # The located "main" unit is ecosystem-specific (EVM ContractInstance, Solana program, …); - # only the ecosystem's own ``units()`` consumes it, so the base keeps it untyped. - main: Any +class PreparedSystem[FormT: BackendResult, U: FeatureUnit, Main](ABC): + #: The located "main" of the analyzed program — the ecosystem's ``Main`` type (EVM's + #: :class:`~composer.spec.system_model.ContractInstance`, Solana's + #: :class:`~composer.spec.solana.model.SolanaProgramInstance`). This is a *different* axis + #: from :class:`FeatureUnit` (the per-unit items ``units()`` iterates): EVM's main is not a + #: unit. The driver treats it opaquely — it only hands it to ``ecosystem.units(main)`` / + #: ``extraction_unit(main)`` — so each backend binds ``Main`` to its ecosystem's type. + main: Main @abstractmethod async def prepare_formalization(self, run: PipelineRun) -> Formalizer[FormT, U]: ... -class PipelineBackend[P: enum.Enum, FormT: BackendResult, H, A: ArtifactIdentifier, U: FeatureUnit](Protocol): +class PipelineBackend[P: enum.Enum, FormT: BackendResult, H, A: ArtifactIdentifier, U: FeatureUnit, Main](Protocol): @property def backend_guidance(self) -> str: ... @@ -115,7 +119,7 @@ def artifact_store(self) -> ArtifactStore[A, FormT]: ... async def prepare_system( self, analyzed: SourceApplication, run: PipelineRun[P, H] - ) -> PreparedSystem[FormT, U]: ... + ) -> PreparedSystem[FormT, U, Main]: ... def to_artifact_id(self, c: U) -> A: ... @@ -147,8 +151,8 @@ def formalize_task_id(idx: int) -> str: return f"formalize-{idx}" # ---- the driver -------------------------------------------------------------- -async def run_pipeline[P: enum.Enum, FormT: BackendResult, H, A: ArtifactIdentifier, U: FeatureUnit]( - backend: PipelineBackend[P, FormT, H, A, U], +async def run_pipeline[P: enum.Enum, FormT: BackendResult, H, A: ArtifactIdentifier, U: FeatureUnit, Main]( + backend: PipelineBackend[P, FormT, H, A, U, Main], run: PipelineRun[P, H], *, interactive: bool = False, @@ -266,6 +270,8 @@ async def _run(batch: _Batch[U]) -> ComponentOutcome[FormT, U]: async def _extract_all[P: enum.Enum, H]( prop_key: str, + # ``main`` stays untyped here: this internal helper drives the type-erased + # ``Ecosystem[Any, Any, Any]`` (see run_pipeline), so there is nothing to tie it to. main: Any, backend_guidance: str, run: PipelineRun[P, H], phase: P, interactive: bool, threat_model: Document | None, max_rounds: int, ecosystem: Ecosystem[Any, Any, Any], diff --git a/composer/spec/solana/null_backend.py b/composer/spec/solana/null_backend.py index cd9e6cea..5f7e3e1d 100644 --- a/composer/spec/solana/null_backend.py +++ b/composer/spec/solana/null_backend.py @@ -135,7 +135,7 @@ async def fetch_verdicts( @dataclass -class NullSolanaPrepared(PreparedSystem[NullResult, FeatureUnit]): +class NullSolanaPrepared(PreparedSystem[NullResult, FeatureUnit, SolanaProgramInstance]): form: NullSolanaFormalizer @override @@ -147,8 +147,8 @@ async def prepare_formalization( @dataclass class NullSolanaBackend: - """``PipelineBackend[SolanaPhase, NullResult, None, NullArtifact, SolanaApplication, - SolanaProgramInstance, SolanaInstructionInstance]`` — structural.""" + """``PipelineBackend[SolanaPhase, NullResult, None, NullArtifact, FeatureUnit, + SolanaProgramInstance]`` (P, FormT, H, A, Unit, Main) — structural.""" artifact_store: NullSolanaArtifactStore backend_guidance = SOLANA_NULL_GUIDANCE @@ -164,7 +164,7 @@ class NullSolanaBackend: async def prepare_system( self, analyzed: SolanaApplication, run: PipelineRun[SolanaPhase, None] - ) -> PreparedSystem[NullResult, FeatureUnit]: + ) -> PreparedSystem[NullResult, FeatureUnit, SolanaProgramInstance]: # Use the Solana ecosystem's locate_main so the backend and ecosystem agree on the # target program (imported lazily to avoid an import cycle with pipeline.ecosystem). from composer.pipeline.ecosystem import SOLANA diff --git a/composer/spec/source/pipeline.py b/composer/spec/source/pipeline.py index aea1603d..5218666c 100644 --- a/composer/spec/source/pipeline.py +++ b/composer/spec/source/pipeline.py @@ -30,7 +30,7 @@ from composer.spec.types import PropertyFormulation from composer.spec.gen_types import CVLResource, SPECS_DIR, certora_relative_to_project from composer.spec.system_model import ( - ContractComponentInstance, SourceApplication, HarnessedApplication, + ContractComponentInstance, ContractInstance, SourceApplication, HarnessedApplication, SourceExplicitContract, HarnessedExplicitContract, SourceExternalActor, HarnessDefinition, SolidityIdentifier, ) @@ -164,7 +164,7 @@ async def finalize(self, outcomes: list[ComponentOutcome[GeneratedCVL, ContractC @dataclass -class ProverPrepared(PreparedSystem[GeneratedCVL, ContractComponentInstance]): +class ProverPrepared(PreparedSystem[GeneratedCVL, ContractComponentInstance, ContractInstance]): """Post-harness system: holds the harnessed app + prover tool, and runs the prover-only pre-formalization fan-out in ``prepare_formalization``.""" _store: ProverArtifactStore @@ -272,7 +272,8 @@ async def _invariants(self, run: PipelineRun): @dataclass class ProverBackend: - """PipelineBackend[AutoProvePhase, GeneratedCVL, None, ComponentSpec].""" + """PipelineBackend[AutoProvePhase, GeneratedCVL, None, ComponentSpec, + ContractComponentInstance, ContractInstance] (P, FormT, H, A, Unit, Main).""" backend_guidance = CERTORA_BACKEND_GUIDANCE core_phases = CorePhases({ "analysis": AutoProvePhase.COMPONENT_ANALYSIS, @@ -287,7 +288,7 @@ class ProverBackend: async def prepare_system( self, analyzed: SourceApplication, run: PipelineRun[AutoProvePhase, None], - ) -> PreparedSystem[GeneratedCVL, ContractComponentInstance]: + ) -> PreparedSystem[GeneratedCVL, ContractComponentInstance, ContractInstance]: sys_desc = await run.runner( TaskInfo(HARNESS_TASK_ID, "Harness Creation", AutoProvePhase.HARNESS), lambda: run_harness_creation(run.ctx, run.source, run.env, analyzed), From 6815140ff502635e3e797c8101f9f37b1d176375 Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 14:15:33 -0700 Subject: [PATCH 03/19] pipeline: type _batch_cache_key via a result-type witness (drop Any) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _batch_cache_key returned CacheKey[ComponentGroup, Any] because its value type isn't inferable from `props`, CacheKey/WorkflowContext are invariant (so the BackendResult bound won't assign to the caller's WorkflowContext[FormT]), and a return-only TypeVar trips reportInvalidTypeVarUse. Thread the concrete result type as a witness argument (`result_type: type[FormT]`, passed as the formalizer's `formalized_type` already in scope at the sole core-owned call site). FormT is now inferred from an argument, so the batch cache key is typed to exactly the backend's result — no Any, no warning. Pure typing change; the key value (props hash) is unchanged. Pyright (CI paths) clean; pipeline/foundry unit tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- composer/pipeline/core.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/composer/pipeline/core.py b/composer/pipeline/core.py index 56611090..df3af52d 100644 --- a/composer/pipeline/core.py +++ b/composer/pipeline/core.py @@ -139,7 +139,15 @@ def _component_cache_key(c: FeatureUnit) -> CacheKey[Properties, ComponentGroup] return CacheKey(string_hash(c.cache_material())) -def _batch_cache_key(props: list[PropertyFormulation]) -> CacheKey[ComponentGroup, Any]: +def _batch_cache_key[FormT: BackendResult]( + props: list[PropertyFormulation], result_type: type[FormT] +) -> CacheKey[ComponentGroup, FormT]: + # The cache VALUE type can't be inferred from ``props`` (they don't carry the backend's + # result type), and ``CacheKey``/``WorkflowContext`` are invariant — so a bare bound + # (``CacheKey[ComponentGroup, BackendResult]``) won't assign to the caller's + # ``WorkflowContext[FormT]``, and a return-only TypeVar trips ``reportInvalidTypeVarUse``. + # Take the concrete ``result_type`` (the formalizer's ``formalized_type``) as a witness so + # ``FormT`` is inferred from an argument: the key is typed to exactly the caller's result. return CacheKey(string_hash("|".join(p.model_dump_json() for p in props))) @@ -209,7 +217,8 @@ async def _run(batch: _Batch[U]) -> ComponentOutcome[FormT, U]: result_key = backend.to_artifact_id(batch.feat) backend.artifact_store.write_properties(result_key, batch.props) child : WorkflowContext[FormT] = await batch.feat_ctx.child( - _batch_cache_key(batch.props), {"properties": [p.model_dump() for p in batch.props]}, + _batch_cache_key(batch.props, formalizer.formalized_type), + {"properties": [p.model_dump() for p in batch.props]}, ) cached_result: FormT | None = await child.cache_get(formalizer.formalized_type) result : FormT | GaveUp From 71a0773d9f93f94ce42ea37744b34cbb52bfa56d Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 14:35:15 -0700 Subject: [PATCH 04/19] pipeline: document why run_pipeline's ecosystem type is erased to Any MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `ecosystem: Ecosystem[Any, Any, Any]` param reads like an accidental weakening; it isn't. Ecosystem is invariant, the backend (and its Main) is a free var at this generic boundary so a concrete EVM/SOLANA argument can't unify with a tied param, prepare_system fixes analyzed: SourceApplication, and the backend's U isn't the ecosystem's Unit — so App/Main/Unit can't be tied without a protocol refactor, and the pairing is a runtime contract. Comment only. Co-Authored-By: Claude Opus 4.8 (1M context) --- composer/pipeline/core.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/composer/pipeline/core.py b/composer/pipeline/core.py index df3af52d..9250e89b 100644 --- a/composer/pipeline/core.py +++ b/composer/pipeline/core.py @@ -171,6 +171,18 @@ async def run_pipeline[P: enum.Enum, FormT: BackendResult, H, A: ArtifactIdentif # ``ecosystem`` supplies the domain-specific front half; it defaults to ``EVM``, which # reproduces the previous hardcoded Solidity behavior exactly, so EVM callers (and cli.py) # need pass nothing. Non-EVM backends (e.g. the Rust/Crucible backend) pass ``ecosystem=SOLANA``. + # + # Its ``[App, Main, Unit]`` params are deliberately erased to ``Any``, not tied to the + # backend's — ``Ecosystem`` is INVARIANT (it holds callables that both consume and produce + # those types), so ``Ecosystem[SolanaApplication, …]`` and ``Ecosystem[SourceApplication, …]`` + # are unrelated. At this generic boundary the backend (hence its ``Main``) is itself a free + # var, and — by invariance — a concretely-typed argument (the ``EVM`` default, or an explicit + # ``SOLANA``) can't unify with a tied ``Main``; the only type accepting both is ``Any``. The + # coupling is loose besides: ``prepare_system`` fixes ``analyzed: SourceApplication`` (so a + # non-EVM ``App`` can't be tied without making it generic), and the backend's ``U`` isn't the + # ecosystem's ``Unit`` (the null backend uses ``FeatureUnit``; ``collapse_units`` fans a + # program into invariant units) — which is why the unit list is ``cast`` below, not inferred. + # So the backend↔ecosystem pairing is a runtime contract; the caller is trusted to pair them. spec, phases = backend.analysis_spec, backend.core_phases source = run.source From 37302e4472b1d7311a626c2bc309a741ed4d52f2 Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 15:04:50 -0700 Subject: [PATCH 05/19] ecosystem: drop the dead per-property fan-out (property_unit) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The only global_extraction ecosystem (SOLANA) sets collapse_units=True, so the per-property fan-out branch — Ecosystem.property_unit, _solana_property_unit, and the `else` limb in _extract_all — was never reached. It was the prototype that finding-level attribution superseded. Remove it: global extraction now always collapses to one whole-program batch (asserted). Also drops the now-unused SolanaInvariantUnit / PropertyFormulation imports. Pyright (CI paths) clean; pipeline/foundry unit tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- composer/pipeline/core.py | 28 +++++++++++----------------- composer/pipeline/ecosystem.py | 34 +++++++++++----------------------- 2 files changed, 22 insertions(+), 40 deletions(-) diff --git a/composer/pipeline/core.py b/composer/pipeline/core.py index 9250e89b..40c44edc 100644 --- a/composer/pipeline/core.py +++ b/composer/pipeline/core.py @@ -314,27 +314,21 @@ async def _extract(unit: FeatureUnit) -> tuple[WorkflowContext[ComponentGroup], ) return ctx, props - # Both strategies reduce to a list of (unit, properties, unit-context) triples; the ecosystem - # differs only in the *fan direction*. Global extraction infers once over the whole program - # and fans each property into its own unit (one prop per batch); per-component infers per unit - # (concurrently) and keeps that unit's properties grouped. The ``_Batch`` build below is shared. - # (Extraction is unit-agnostic — over the ``FeatureUnit`` protocol; run_pipeline's single cast - # reunites the batches with the paired backend's concrete unit type ``U``.) + # Both strategies reduce to a list of (unit, properties, unit-context) triples. Global + # extraction infers once over the whole program and keeps ALL invariants in a single + # whole-program batch (collapse_units); per-component infers per unit (concurrently) and keeps + # that unit's properties grouped. The ``_Batch`` build below is shared. (Extraction is + # unit-agnostic — over the ``FeatureUnit`` protocol; run_pipeline's single cast reunites the + # batches with the paired backend's concrete unit type ``U``.) triples: list[tuple[FeatureUnit, list[PropertyFormulation], WorkflowContext[ComponentGroup]]] if ecosystem.global_extraction: - assert ecosystem.extraction_unit is not None + # Infer once over the whole program, then formalize every invariant in ONE whole-program + # harness + run (docs/crucible-unit-granularity.md §3). Empty props → no batch. Today + # global extraction always collapses (the per-property fan-out prototype was removed). + assert ecosystem.extraction_unit is not None and ecosystem.collapse_units ext_unit = ecosystem.extraction_unit(main) ext_ctx, props = await _extract(ext_unit) - if ecosystem.collapse_units: - # One whole-program batch: every invariant is formalized into a single harness + run - # (docs/crucible-unit-granularity.md §3). Empty props → no batch (nothing to formalize). - triples = [(ext_unit, props, ext_ctx)] if props else [] - else: - assert ecosystem.property_unit is not None - triples = [] - for i, prop in enumerate(props): - unit = ecosystem.property_unit(main, prop, i) - triples.append((unit, [prop], await _unit_ctx(unit))) + triples = [(ext_unit, props, ext_ctx)] if props else [] else: units = ecosystem.units(main) extracted = await asyncio.gather(*[_extract(u) for u in units]) diff --git a/composer/pipeline/ecosystem.py b/composer/pipeline/ecosystem.py index 2feedfef..79e33582 100644 --- a/composer/pipeline/ecosystem.py +++ b/composer/pipeline/ecosystem.py @@ -34,10 +34,8 @@ from composer.spec.solana.model import ( SolanaApplication, SolanaInstructionInstance, - SolanaInvariantUnit, SolanaProgramInstance, ) -from composer.spec.types import PropertyFormulation from composer.spec.util import FS_FORBIDDEN_READ LanguageTag = Literal["solidity", "rust"] @@ -110,20 +108,17 @@ class Ecosystem[App: BaseApplication, Main, Unit: FeatureUnit]: # -- Extraction strategy (docs/crucible-unit-granularity.md) ------------------------- #: When True, the driver runs ONE whole-program property extraction (context = - #: ``extraction_unit(main)``) and fans each resulting property out into its own unit - #: via ``property_unit`` — one harness + verdict per property. When False (the EVM - #: default) it extracts per ``units(main)`` (one batch per component). Solana uses - #: global extraction so the fuzzer gets whole-program invariants, one run per invariant. + #: ``extraction_unit(main)``) and formalizes every resulting invariant in a single + #: whole-program harness + run (see ``collapse_units``). When False (the EVM default) it + #: extracts per ``units(main)`` — one batch per component. Solana uses global extraction so + #: the fuzzer gets whole-program invariants. global_extraction: bool = False #: The whole-program context the global extraction reads (only used when global_extraction). extraction_unit: Callable[[Main], FeatureUnit] | None = None - #: Build the per-property unit from (main, property, index) (only used when global_extraction - #: and NOT collapse_units). - property_unit: Callable[[Main, PropertyFormulation, int], FeatureUnit] | None = None - #: When True (with global_extraction), keep all extracted properties in a SINGLE batch on the - #: whole-program ``extraction_unit`` — one formalization, one harness, one run covering every - #: invariant (docs/crucible-unit-granularity.md §3; prototype of the "single whole-program - #: unit" collapse). When False, fan each property into its own ``property_unit``. + #: With global_extraction, keep all extracted properties in a SINGLE batch on the whole-program + #: ``extraction_unit`` — one formalization, one harness, one run covering every invariant + #: (docs/crucible-unit-granularity.md §3), finding-level attribution recovering which property a + #: counterexample hit. This is the only global-extraction mode today. collapse_units: bool = False @@ -277,12 +272,6 @@ def _solana_extraction_unit(main: SolanaProgramInstance) -> SolanaProgramInstanc return main -def _solana_property_unit( - main: SolanaProgramInstance, prop: PropertyFormulation, ind: int -) -> SolanaInvariantUnit: - return SolanaInvariantUnit(ind, main, prop) - - def _solana_analysis_extra_input(source: SourceCode) -> list[str | dict]: return [ f"The main program of this application has been explicitly identified as " @@ -301,12 +290,11 @@ def _solana_analysis_extra_input(source: SourceCode) -> list[str | dict]: locate_main=_solana_locate_main, units=_solana_units, analysis_extra_input=_solana_analysis_extra_input, - # Fuzzing wants whole-program invariants — docs/crucible-unit-granularity.md. Prototype: - # collapse all invariants into ONE whole-program harness + run (§3), instead of one per - # invariant, now that finding-level attribution recovers which property a counterexample hits. + # Fuzzing wants whole-program invariants (docs/crucible-unit-granularity.md §3): infer once + # over the program, then collapse every invariant into ONE whole-program harness + run — + # finding-level attribution recovers which property a counterexample hit. global_extraction=True, extraction_unit=_solana_extraction_unit, - property_unit=_solana_property_unit, collapse_units=True, ) From 4e0a15406a173789a092a6065757a283993d356f Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 15:11:23 -0700 Subject: [PATCH 06/19] ecosystem: select extraction mode by callable, drop the redundant booleans After removing the per-property fan-out, global_extraction and collapse_units each carried the same single bit as `extraction_unit is not None`, and Solana's required `units` (_solana_units) was dead (never called in whole-program mode). Collapse the three signals into one invariant: an ecosystem sets exactly one of `units` (per-component) xor `extraction_unit` (whole-program), and _extract_all branches on which is present. Both callables are now Optional; drop global_extraction, collapse_units, and _solana_units; type SOLANA's Unit param as SolanaProgramInstance (the whole-program unit) and drop the now-unused SolanaInstructionInstance import. Pyright (CI paths) clean; pipeline/foundry unit tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- composer/pipeline/core.py | 21 ++++++++--------- composer/pipeline/ecosystem.py | 42 ++++++++++++---------------------- 2 files changed, 24 insertions(+), 39 deletions(-) diff --git a/composer/pipeline/core.py b/composer/pipeline/core.py index 40c44edc..4fc8e804 100644 --- a/composer/pipeline/core.py +++ b/composer/pipeline/core.py @@ -180,8 +180,8 @@ async def run_pipeline[P: enum.Enum, FormT: BackendResult, H, A: ArtifactIdentif # ``SOLANA``) can't unify with a tied ``Main``; the only type accepting both is ``Any``. The # coupling is loose besides: ``prepare_system`` fixes ``analyzed: SourceApplication`` (so a # non-EVM ``App`` can't be tied without making it generic), and the backend's ``U`` isn't the - # ecosystem's ``Unit`` (the null backend uses ``FeatureUnit``; ``collapse_units`` fans a - # program into invariant units) — which is why the unit list is ``cast`` below, not inferred. + # ecosystem's ``Unit`` (the null backend uses ``FeatureUnit``; whole-program extraction yields + # one program-level unit) — which is why the unit list is ``cast`` below, not inferred. # So the backend↔ecosystem pairing is a runtime contract; the caller is trusted to pair them. spec, phases = backend.analysis_spec, backend.core_phases source = run.source @@ -314,22 +314,21 @@ async def _extract(unit: FeatureUnit) -> tuple[WorkflowContext[ComponentGroup], ) return ctx, props - # Both strategies reduce to a list of (unit, properties, unit-context) triples. Global - # extraction infers once over the whole program and keeps ALL invariants in a single - # whole-program batch (collapse_units); per-component infers per unit (concurrently) and keeps - # that unit's properties grouped. The ``_Batch`` build below is shared. (Extraction is + # Both strategies reduce to a list of (unit, properties, unit-context) triples. The mode is + # selected by which callable the ecosystem set: ``extraction_unit`` (whole-program) xor + # ``units`` (per-component). The ``_Batch`` build below is shared. (Extraction is # unit-agnostic — over the ``FeatureUnit`` protocol; run_pipeline's single cast reunites the # batches with the paired backend's concrete unit type ``U``.) triples: list[tuple[FeatureUnit, list[PropertyFormulation], WorkflowContext[ComponentGroup]]] - if ecosystem.global_extraction: - # Infer once over the whole program, then formalize every invariant in ONE whole-program - # harness + run (docs/crucible-unit-granularity.md §3). Empty props → no batch. Today - # global extraction always collapses (the per-property fan-out prototype was removed). - assert ecosystem.extraction_unit is not None and ecosystem.collapse_units + if ecosystem.extraction_unit is not None: + # Whole-program: infer once over the whole program, then formalize every invariant in ONE + # harness + run (docs/crucible-unit-granularity.md §3). Empty props → no batch. ext_unit = ecosystem.extraction_unit(main) ext_ctx, props = await _extract(ext_unit) triples = [(ext_unit, props, ext_ctx)] if props else [] else: + # Per-component: one batch per unit. + assert ecosystem.units is not None, "ecosystem must set exactly one of units / extraction_unit" units = ecosystem.units(main) extracted = await asyncio.gather(*[_extract(u) for u in units]) triples = [(u, props, ctx) for u, (ctx, props) in zip(units, extracted) if props] diff --git a/composer/pipeline/ecosystem.py b/composer/pipeline/ecosystem.py index 79e33582..4326ae83 100644 --- a/composer/pipeline/ecosystem.py +++ b/composer/pipeline/ecosystem.py @@ -33,7 +33,6 @@ ) from composer.spec.solana.model import ( SolanaApplication, - SolanaInstructionInstance, SolanaProgramInstance, ) from composer.spec.util import FS_FORBIDDEN_READ @@ -101,25 +100,18 @@ class Ecosystem[App: BaseApplication, Main, Unit: FeatureUnit]: validate_analysis: Callable[[BaseApplication, SolidityIdentifier | None], str | None] #: Locate the target unit (the "main contract"/program) in the analyzed model. locate_main: Callable[[App, SourceCode], Main] - #: Enumerate the per-unit items the extraction phase infers properties for. - units: Callable[[Main], list[Unit]] #: Domain-specific front-matter appended to the analysis input (was hardcoded in the driver). analysis_extra_input: Callable[[SourceCode], list[str | dict]] - # -- Extraction strategy (docs/crucible-unit-granularity.md) ------------------------- - #: When True, the driver runs ONE whole-program property extraction (context = - #: ``extraction_unit(main)``) and formalizes every resulting invariant in a single - #: whole-program harness + run (see ``collapse_units``). When False (the EVM default) it - #: extracts per ``units(main)`` — one batch per component. Solana uses global extraction so - #: the fuzzer gets whole-program invariants. - global_extraction: bool = False - #: The whole-program context the global extraction reads (only used when global_extraction). + # -- Extraction strategy (docs/crucible-unit-granularity.md) -------------------------- + # An ecosystem supplies exactly ONE of the two below; which one is set selects the mode. + #: Per-component: enumerate the units the extraction phase infers properties for — one batch + #: per unit. Set by EVM (one component = one unit). + units: Callable[[Main], list[Unit]] | None = None + #: Whole-program: the single whole-program context extraction reads. All invariants are kept in + #: ONE harness + run (docs/crucible-unit-granularity.md §3), with finding-level attribution + #: recovering which property a counterexample hit. Set by Solana; its presence selects this mode. extraction_unit: Callable[[Main], FeatureUnit] | None = None - #: With global_extraction, keep all extracted properties in a SINGLE batch on the whole-program - #: ``extraction_unit`` — one formalization, one harness, one run covering every invariant - #: (docs/crucible-unit-granularity.md §3), finding-level attribution recovering which property a - #: counterexample hit. This is the only global-extraction mode today. - collapse_units: bool = False # --------------------------------------------------------------------------- @@ -260,12 +252,6 @@ def _solana_locate_main(app: SolanaApplication, source: SourceCode) -> SolanaPro raise ValueError(f"main program {source.contract_name!r} not found in analyzed application") -def _solana_units(main: SolanaProgramInstance) -> list[SolanaInstructionInstance]: - # Per-instruction units — unused under global extraction (kept for a per-instruction - # fallback and to satisfy the ecosystem's Unit type). - return [SolanaInstructionInstance(i, main) for i in range(len(main.program.instructions))] - - def _solana_extraction_unit(main: SolanaProgramInstance) -> SolanaProgramInstance: # The whole program is the extraction context; SolanaProgramInstance is itself a # FeatureUnit, so the driver reads it directly to propose whole-program invariants. @@ -280,7 +266,9 @@ def _solana_analysis_extra_input(source: SourceCode) -> list[str | dict]: ] -SOLANA: Ecosystem[SolanaApplication, SolanaProgramInstance, SolanaInstructionInstance] = Ecosystem( +# Whole-program mode: no per-component units — the extraction unit IS the whole program, so the +# ``Unit`` type param is ``SolanaProgramInstance`` (same as ``Main``). +SOLANA: Ecosystem[SolanaApplication, SolanaProgramInstance, SolanaProgramInstance] = Ecosystem( name="solana", language=RUST, system_model=SolanaApplication, @@ -288,14 +276,12 @@ def _solana_analysis_extra_input(source: SourceCode) -> list[str | dict]: property_prompts=PromptPair("solana/property_system.j2", "solana/property_prompt.j2"), validate_analysis=_solana_validate, locate_main=_solana_locate_main, - units=_solana_units, analysis_extra_input=_solana_analysis_extra_input, # Fuzzing wants whole-program invariants (docs/crucible-unit-granularity.md §3): infer once - # over the program, then collapse every invariant into ONE whole-program harness + run — - # finding-level attribution recovers which property a counterexample hit. - global_extraction=True, + # over the program, then keep every invariant in ONE whole-program harness + run — + # finding-level attribution recovers which property a counterexample hit. Setting + # ``extraction_unit`` (and no ``units``) selects this mode. extraction_unit=_solana_extraction_unit, - collapse_units=True, ) From 0f441b2e128139a81c5e3398a2e801a79ad2eb79 Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 15:35:40 -0700 Subject: [PATCH 07/19] ecosystem: unify extraction into one units() path (drop extraction_unit) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit extraction_unit was equivalent to units() returning a single unit: both feed the same per-unit _extract (same cache context, task, prompts, empty-props drop). It was only distinct as a vestige of the removed fan-out. Collapse it: units is required again and returns a list; Solana returns a singleton [main] (its whole program is the one unit), EVM one per component. _extract_all is now a single gather over ecosystem.units(main) — structurally master's _one loop again, minus the ecosystem-generic bits (units(main), feat.context_tag(), ecosystem prompts). Pyright (CI paths) clean; EVM pipeline/foundry tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- composer/pipeline/core.py | 41 ++++++++++------------------------ composer/pipeline/ecosystem.py | 37 +++++++++++++----------------- 2 files changed, 28 insertions(+), 50 deletions(-) diff --git a/composer/pipeline/core.py b/composer/pipeline/core.py index 4fc8e804..89710839 100644 --- a/composer/pipeline/core.py +++ b/composer/pipeline/core.py @@ -299,41 +299,24 @@ async def _extract_all[P: enum.Enum, H]( ) -> list[_Batch[FeatureUnit]]: prop_ctx = run.ctx.child(PROPERTIES_KEY(prop_key)) - async def _unit_ctx(unit: FeatureUnit) -> WorkflowContext[ComponentGroup]: - return await prop_ctx.child(_component_cache_key(unit), unit.context_tag()) - - async def _extract(unit: FeatureUnit) -> tuple[WorkflowContext[ComponentGroup], list[PropertyFormulation]]: - ctx = await _unit_ctx(unit) + # The ecosystem-generalized form of the per-component loop: the ecosystem enumerates the units + # to infer over (EVM: one per component; Solana: a singleton whole-program unit), and each is + # extracted concurrently into at most one batch. ``_Batch`` is over the ``FeatureUnit`` protocol; + # run_pipeline's single cast reunites the batches with the backend's concrete unit type ``U``. + async def _one(feat: FeatureUnit) -> _Batch[FeatureUnit] | None: + feat_ctx = await prop_ctx.child(_component_cache_key(feat), feat.context_tag()) props = await run.runner( - TaskInfo(extract_task_id(unit.unit_index), unit.display_name, phase), + TaskInfo(extract_task_id(feat.unit_index), feat.display_name, phase), lambda conv: run_property_inference( - ctx, run.env, unit, refinement=conv if interactive else None, + feat_ctx, run.env, feat, refinement=conv if interactive else None, threat_model=threat_model, max_rounds=max_rounds, backend_guidance=backend_guidance, system_template=ecosystem.property_prompts.system, initial_template=ecosystem.property_prompts.initial), ) - return ctx, props - - # Both strategies reduce to a list of (unit, properties, unit-context) triples. The mode is - # selected by which callable the ecosystem set: ``extraction_unit`` (whole-program) xor - # ``units`` (per-component). The ``_Batch`` build below is shared. (Extraction is - # unit-agnostic — over the ``FeatureUnit`` protocol; run_pipeline's single cast reunites the - # batches with the paired backend's concrete unit type ``U``.) - triples: list[tuple[FeatureUnit, list[PropertyFormulation], WorkflowContext[ComponentGroup]]] - if ecosystem.extraction_unit is not None: - # Whole-program: infer once over the whole program, then formalize every invariant in ONE - # harness + run (docs/crucible-unit-granularity.md §3). Empty props → no batch. - ext_unit = ecosystem.extraction_unit(main) - ext_ctx, props = await _extract(ext_unit) - triples = [(ext_unit, props, ext_ctx)] if props else [] - else: - # Per-component: one batch per unit. - assert ecosystem.units is not None, "ecosystem must set exactly one of units / extraction_unit" - units = ecosystem.units(main) - extracted = await asyncio.gather(*[_extract(u) for u in units]) - triples = [(u, props, ctx) for u, (ctx, props) in zip(units, extracted) if props] - - return [_Batch(unit, props, ctx) for unit, props, ctx in triples] + return _Batch(feat, props, feat_ctx) if props else None + + got = await asyncio.gather(*[_one(u) for u in ecosystem.units(main)]) + return [b for b in got if b is not None] def _tally[FormT: BackendResult, U: FeatureUnit]( diff --git a/composer/pipeline/ecosystem.py b/composer/pipeline/ecosystem.py index 4326ae83..9a16dc1b 100644 --- a/composer/pipeline/ecosystem.py +++ b/composer/pipeline/ecosystem.py @@ -100,19 +100,14 @@ class Ecosystem[App: BaseApplication, Main, Unit: FeatureUnit]: validate_analysis: Callable[[BaseApplication, SolidityIdentifier | None], str | None] #: Locate the target unit (the "main contract"/program) in the analyzed model. locate_main: Callable[[App, SourceCode], Main] + #: Enumerate the units the extraction phase infers properties for — one batch per unit. EVM + #: returns one per component; a whole-program ecosystem (Solana) returns a singleton ``[main]``, + #: so all its invariants are inferred + formalized in a single harness + run + #: (docs/crucible-unit-granularity.md §3). + units: Callable[[Main], list[Unit]] #: Domain-specific front-matter appended to the analysis input (was hardcoded in the driver). analysis_extra_input: Callable[[SourceCode], list[str | dict]] - # -- Extraction strategy (docs/crucible-unit-granularity.md) -------------------------- - # An ecosystem supplies exactly ONE of the two below; which one is set selects the mode. - #: Per-component: enumerate the units the extraction phase infers properties for — one batch - #: per unit. Set by EVM (one component = one unit). - units: Callable[[Main], list[Unit]] | None = None - #: Whole-program: the single whole-program context extraction reads. All invariants are kept in - #: ONE harness + run (docs/crucible-unit-granularity.md §3), with finding-level attribution - #: recovering which property a counterexample hit. Set by Solana; its presence selects this mode. - extraction_unit: Callable[[Main], FeatureUnit] | None = None - # --------------------------------------------------------------------------- # main-unit location (moved out of pipeline.core; re-exported there for callers) @@ -252,10 +247,11 @@ def _solana_locate_main(app: SolanaApplication, source: SourceCode) -> SolanaPro raise ValueError(f"main program {source.contract_name!r} not found in analyzed application") -def _solana_extraction_unit(main: SolanaProgramInstance) -> SolanaProgramInstance: - # The whole program is the extraction context; SolanaProgramInstance is itself a - # FeatureUnit, so the driver reads it directly to propose whole-program invariants. - return main +def _solana_units(main: SolanaProgramInstance) -> list[SolanaProgramInstance]: + # Whole-program mode: a single unit that IS the program — so all invariants are inferred and + # formalized in one harness + run (docs/crucible-unit-granularity.md §3). SolanaProgramInstance + # is itself a FeatureUnit, so the driver reads it directly to propose whole-program invariants. + return [main] def _solana_analysis_extra_input(source: SourceCode) -> list[str | dict]: @@ -266,8 +262,11 @@ def _solana_analysis_extra_input(source: SourceCode) -> list[str | dict]: ] -# Whole-program mode: no per-component units — the extraction unit IS the whole program, so the -# ``Unit`` type param is ``SolanaProgramInstance`` (same as ``Main``). +# Whole-program mode: ``units`` returns a singleton ``[main]`` (the ``Unit`` type param is +# ``SolanaProgramInstance``, same as ``Main``). Fuzzing wants whole-program invariants +# (docs/crucible-unit-granularity.md §3): all invariants are inferred over the program and +# formalized into ONE harness + run — finding-level attribution recovers which property a +# counterexample hit. SOLANA: Ecosystem[SolanaApplication, SolanaProgramInstance, SolanaProgramInstance] = Ecosystem( name="solana", language=RUST, @@ -276,12 +275,8 @@ def _solana_analysis_extra_input(source: SourceCode) -> list[str | dict]: property_prompts=PromptPair("solana/property_system.j2", "solana/property_prompt.j2"), validate_analysis=_solana_validate, locate_main=_solana_locate_main, + units=_solana_units, analysis_extra_input=_solana_analysis_extra_input, - # Fuzzing wants whole-program invariants (docs/crucible-unit-granularity.md §3): infer once - # over the program, then keep every invariant in ONE whole-program harness + run — - # finding-level attribution recovers which property a counterexample hit. Setting - # ``extraction_unit`` (and no ``units``) selects this mode. - extraction_unit=_solana_extraction_unit, ) From 5c61dbda0f6a01d26737e17256ea0c50c1598a6c Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 15:42:50 -0700 Subject: [PATCH 08/19] Remove unneeded comment --- composer/pipeline/core.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/composer/pipeline/core.py b/composer/pipeline/core.py index 89710839..53cee3f5 100644 --- a/composer/pipeline/core.py +++ b/composer/pipeline/core.py @@ -299,10 +299,6 @@ async def _extract_all[P: enum.Enum, H]( ) -> list[_Batch[FeatureUnit]]: prop_ctx = run.ctx.child(PROPERTIES_KEY(prop_key)) - # The ecosystem-generalized form of the per-component loop: the ecosystem enumerates the units - # to infer over (EVM: one per component; Solana: a singleton whole-program unit), and each is - # extracted concurrently into at most one batch. ``_Batch`` is over the ``FeatureUnit`` protocol; - # run_pipeline's single cast reunites the batches with the backend's concrete unit type ``U``. async def _one(feat: FeatureUnit) -> _Batch[FeatureUnit] | None: feat_ctx = await prop_ctx.child(_component_cache_key(feat), feat.context_tag()) props = await run.runner( From 7d36604bd4be892c667f5ac333c9bc470ca20687 Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 15:46:38 -0700 Subject: [PATCH 09/19] Comment tweaks --- composer/pipeline/ecosystem.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/composer/pipeline/ecosystem.py b/composer/pipeline/ecosystem.py index 9a16dc1b..8f1d9718 100644 --- a/composer/pipeline/ecosystem.py +++ b/composer/pipeline/ecosystem.py @@ -110,7 +110,7 @@ class Ecosystem[App: BaseApplication, Main, Unit: FeatureUnit]: # --------------------------------------------------------------------------- -# main-unit location (moved out of pipeline.core; re-exported there for callers) +# main-unit location # --------------------------------------------------------------------------- @@ -126,7 +126,7 @@ def main_instance(app: AnyApplication, source: SourceCode) -> ContractInstance: # --------------------------------------------------------------------------- -# The EVM ecosystem (= today's behavior) +# The EVM ecosystem # --------------------------------------------------------------------------- @@ -169,7 +169,7 @@ def _evm_analysis_extra_input(source: SourceCode) -> list[str | dict]: # --------------------------------------------------------------------------- -# The RUST language facet (shared by Solana and, later, Soroban) +# The RUST language facet (shared by Solana, Soroban) # --------------------------------------------------------------------------- #: Cargo/Anchor project layout: hide build output, VCS, lockfiles, and the JS side; keep the From 29c2a9f9acbede04dbea8bc8b9e58be9b6567f79 Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 15:50:25 -0700 Subject: [PATCH 10/19] Comment tweaks --- composer/pipeline/ecosystem.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/composer/pipeline/ecosystem.py b/composer/pipeline/ecosystem.py index 8f1d9718..8e2dec2c 100644 --- a/composer/pipeline/ecosystem.py +++ b/composer/pipeline/ecosystem.py @@ -263,10 +263,7 @@ def _solana_analysis_extra_input(source: SourceCode) -> list[str | dict]: # Whole-program mode: ``units`` returns a singleton ``[main]`` (the ``Unit`` type param is -# ``SolanaProgramInstance``, same as ``Main``). Fuzzing wants whole-program invariants -# (docs/crucible-unit-granularity.md §3): all invariants are inferred over the program and -# formalized into ONE harness + run — finding-level attribution recovers which property a -# counterexample hit. +# ``SolanaProgramInstance``, same as ``Main``). We may support finer-grained units in the future. SOLANA: Ecosystem[SolanaApplication, SolanaProgramInstance, SolanaProgramInstance] = Ecosystem( name="solana", language=RUST, From 530ed48ff4d96c9482ad8ba64cf1f0d7b2654b6e Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 15:59:56 -0700 Subject: [PATCH 11/19] tweaks --- composer/pipeline/ecosystem.py | 11 +++++++++++ composer/spec/system_model.py | 3 +-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/composer/pipeline/ecosystem.py b/composer/pipeline/ecosystem.py index 8e2dec2c..7349da15 100644 --- a/composer/pipeline/ecosystem.py +++ b/composer/pipeline/ecosystem.py @@ -145,6 +145,17 @@ def _evm_analysis_extra_input(source: SourceCode) -> list[str | dict]: ] +# Adding Vyper support (a second EVM source language) would, at a very high level: +# 1. Extend ``LanguageTag`` with ``"vyper"`` and add a ``VYPER`` ``Language`` facet here (its +# own ``forbidden_read``, code-explorer prompt, and — eventually — failure-modes partial). +# 2. Bind it to a Vyper-flavored EVM ``Ecosystem`` (its own analysis/property prompts) and +# route to it by detecting the target's source language at the entry point. +# 3. Loosen the analysis model's Solidity assumptions: contracts are keyed by +# ``SolidityIdentifier`` / ``solidity_identifier`` throughout (see ``system_model`` and +# ``main_instance``), which would need to become language-neutral. +# The CVL backend needs the least work — the Certora Prover already accepts Vyper (it verifies +# compiled bytecode) — while the Foundry backend is Solidity-only by construction (it authors +# and runs ``.t.sol`` tests), so it would need a separate Vyper story or be left EVM/Solidity-only. SOLIDITY = Language( name="solidity", default_forbidden_read=FS_FORBIDDEN_READ, diff --git a/composer/spec/system_model.py b/composer/spec/system_model.py index 68042b9c..d71e09c1 100644 --- a/composer/spec/system_model.py +++ b/composer/spec/system_model.py @@ -1,12 +1,11 @@ from dataclasses import dataclass -from typing import Literal, Protocol, runtime_checkable +from typing import Literal, Protocol from pydantic import BaseModel, Field from functools import cached_property from composer.spec.util import slugify_filename from .types import ComponentName, SolidityIdentifier, ContractName -@runtime_checkable class FeatureUnit(Protocol): """The per-unit interface the shared pipeline driver needs, independent of ecosystem. From a259e01960d48a7a715005889646fe276c90a319 Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 16:08:22 -0700 Subject: [PATCH 12/19] system_model: drop @runtime_checkable, strengthen unit tag/json dicts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FeatureUnit has no isinstance/issubclass call sites, so @runtime_checkable (and its import) bought nothing. Type context_tag()/feature_json() as dict[str, object] across the protocol and every impl (ContractComponentInstance, Solana program/invariant/ instruction units) — all return str-keyed, JSON-able dicts, and the sole consumer (WorkflowContext.child(tag)) accepts it. Co-Authored-By: Claude Opus 4.8 --- composer/spec/solana/model.py | 12 ++++++------ composer/spec/system_model.py | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/composer/spec/solana/model.py b/composer/spec/solana/model.py index 7e2a8d1b..13b70c91 100644 --- a/composer/spec/solana/model.py +++ b/composer/spec/solana/model.py @@ -166,10 +166,10 @@ def unit_index(self) -> int: def cache_material(self) -> str: return "|".join([self.app.model_dump_json(), str(self.ind), "program"]) - def context_tag(self) -> dict: + def context_tag(self) -> dict[str, object]: return {"program": self.program.model_dump()} - def feature_json(self) -> dict: + def feature_json(self) -> dict[str, object]: return { "program": self.program.name, "instructions": [i.model_dump(mode="json") for i in self.program.instructions], @@ -216,10 +216,10 @@ def cache_material(self) -> str: [self.app.model_dump_json(), str(self._program.ind), str(self.ind), self.invariant.title] ) - def context_tag(self) -> dict: + def context_tag(self) -> dict[str, object]: return {"invariant": self.invariant.model_dump(mode="json"), "program": self.program.name} - def feature_json(self) -> dict: + def feature_json(self) -> dict[str, object]: return { "program": self.program.name, "instructions": [i.model_dump(mode="json") for i in self.program.instructions], @@ -262,10 +262,10 @@ def unit_index(self) -> int: def cache_material(self) -> str: return "|".join([self.app.model_dump_json(), str(self.ind), str(self._program.ind)]) - def context_tag(self) -> dict: + def context_tag(self) -> dict[str, object]: return {"instruction": self.instruction.model_dump()} - def feature_json(self) -> dict: + def feature_json(self) -> dict[str, object]: # The unit's semantic content for a backend marshalling it across a boundary: # the instruction, tagged with its program (a Solana backend needs both). return { diff --git a/composer/spec/system_model.py b/composer/spec/system_model.py index d71e09c1..0b31a824 100644 --- a/composer/spec/system_model.py +++ b/composer/spec/system_model.py @@ -33,11 +33,11 @@ def cache_material(self) -> str: """Stable string identifying this unit, hashed into its per-unit cache key.""" ... - def context_tag(self) -> dict: + def context_tag(self) -> dict[str, object]: """The tag persisted alongside this unit's workflow context.""" ... - def feature_json(self) -> dict: + def feature_json(self) -> dict[str, object]: """The unit's semantic content as a JSON-able dict, for a backend that marshals it across a boundary (e.g. a Rust wheel's ``FormalizeInput.component``). The shape is ecosystem-specific; the paired backend knows how to read it.""" @@ -291,10 +291,10 @@ def unit_index(self) -> int: def cache_material(self) -> str: return "|".join([self.app.model_dump_json(), str(self.ind), str(self._contract.ind)]) - def context_tag(self) -> dict: + def context_tag(self) -> dict[str, object]: return {"component": self.component.model_dump()} - def feature_json(self) -> dict: + def feature_json(self) -> dict[str, object]: return self.component.model_dump(mode="json") @staticmethod From a2058eb837e7a62085350978d65245d88560efb2 Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 16:13:07 -0700 Subject: [PATCH 13/19] docs: bring ARCHITECTURE.md up to date with the ecosystem seam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The doc shipped in the ecosystem-abstraction commit but still described the pre-ecosystem, Solidity-only, per-component world. Update it to match: - §1/§2: reframe as smart-contract (not Solidity-only) and call out the ecosystem front-half as a second, backend-orthogonal axis of pluggability. - §3/§4: per-component -> per-unit (units() / FeatureUnit); correct the backend signature to PipelineBackend[P, FormT, H, A, U, Main] and PreparedSystem holding Main; add a new "ecosystem seam" subsection. - §4 step 1: analyzed model type is set by the ecosystem, not the backend. - §10: add the Solana front-half (model + prompts + whole-program units). Also drop the now-dangling extraction_unit reference in core.py's PreparedSystem.main comment (removed in the units() unification). Co-Authored-By: Claude Opus 4.8 --- ARCHITECTURE.md | 66 +++++++++++++++++++++++++++++---------- composer/pipeline/core.py | 4 +-- 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3627b824..6aa336e4 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -6,13 +6,17 @@ ## 1. What it is -AutoProver (internally "AI Composer") is a **multi-agent pipeline that turns a Solidity +AutoProver (internally "AI Composer") is a **multi-agent pipeline that turns a smart-contract codebase + a design document into verified formal specifications.** Given a project root, a -main contract, and a system description, it drives a fleet of LLM agents to analyze the +main contract/program, and a system description, it drives a fleet of LLM agents to analyze the system, formulate properties, author CVL (Certora Verification Language) specs, and run the Certora Prover in a feedback loop until the specs verify (or the agent gives up with a reason). +The default target is Solidity + the Certora Prover, but the front half is parameterized by an +**ecosystem** (an analyzed *language* × *chain* pairing — see §4), so the same driver also +analyzes Rust/Solana programs; the domain is not hardcoded into the shared steps. + The same generic pipeline also powers two sibling workflows — **Foundry test generation** and **NatSpec greenfield code generation** — which reuse the shared analysis/extraction/ reporting machinery behind a backend protocol. @@ -21,11 +25,14 @@ reporting machinery behind a backend protocol. A handful of decisions shape the whole codebase: -- **Generic driver, pluggable backends.** One backend-agnostic driver +- **Generic driver, two axes of pluggability.** One backend-agnostic driver ([composer/pipeline/core.py](composer/pipeline/core.py)) owns the steps that are the same - for everyone (system analysis, property extraction, caching, report assembly). Anything - backend-specific (how a spec is authored and verified) is contributed through a small - protocol. Adding the Foundry backend required *no* changes to the shared steps. + for everyone (system analysis, property extraction, caching, report assembly). Two things + plug into it independently: a **backend** contributes how a spec is authored and verified + (the *back* half), and an **ecosystem** supplies the domain-specific *front* half — the + analyzed-model type, prompts, source-reading conventions, and how the target is split into + units. Adding the Foundry backend required *no* changes to the shared steps; adding the + Solana ecosystem required *no* backend changes. - **Phase-chain immutability.** Each phase produces an immutable object that is the constructor input to the next: `Backend → PreparedSystem → Formalizer`. The *existence* of a `Formalizer` proves that system analysis and preparation already succeeded — ordering is @@ -53,7 +60,7 @@ A handful of decisions shape the whole codebase: │ │ 1. System Analysis ───────► SourceApplication│ │ builds context │ 2. backend.prepare_system ─► PreparedSystem │ ▼ │ 3. prepare_formalization ∥ property extraction│ - AIComposerContext │ 4. per-component formalize (parallel) │ + AIComposerContext │ 4. per-unit formalize (parallel) │ (LLM, RAG, prover opts, │ 5. backend-agnostic Report │ VFS, handler factory) └───────────────┬────────────────────────────────┘ │ │ PipelineBackend protocol @@ -82,31 +89,51 @@ A handful of decisions shape the whole codebase: [composer/pipeline/core.py](composer/pipeline/core.py) is the spine. `run_pipeline()` executes five steps and never inspects anything backend-specific: -1. **System analysis** (shared). Runs `run_component_analysis` to produce a - `SourceApplication` — the contracts, components, external actors, and their interactions. - Always yields the same type regardless of backend. +1. **System analysis** (shared). Runs `run_component_analysis` to produce the ecosystem's + analyzed model (`App`) — for EVM, a `SourceApplication` of contracts, components, external + actors, and their interactions. Determined by the ecosystem, not the backend: every backend + on a given ecosystem sees the same analyzed type. 2. **`backend.prepare_system(analyzed)`** — the backend's transform. The prover backend lifts the source app into a *harnessed* application (generating harness contracts for external dependencies); the Foundry backend is an identity transform. 3. **`prepared.prepare_formalization()` runs concurrently with property extraction.** Neither depends on the other, so the prover's expensive AutoSetup/summary/invariant work overlaps - with per-component property inference. Property extraction fans out one agent per component, - bounded by a semaphore (`--max-concurrent`). -4. **Per-component formalization** (parallel). For each component's properties, the backend's + with per-unit property inference. Property extraction fans out one agent per *unit* — + `ecosystem.units(main)` (EVM yields one per contract component; a whole-program ecosystem + like Solana yields a single unit that is the program) — bounded by a semaphore + (`--max-concurrent`). +4. **Per-unit formalization** (parallel). For each unit's properties, the backend's `Formalizer.formalize()` is invoked. Results are cached by the backend's result type. -5. **Report assembly** (shared, best-effort). The driver collects per-component verdicts via a +5. **Report assembly** (shared, best-effort). The driver collects per-unit verdicts via a backend-supplied `fetch_verdicts` callback, an LLM groups properties into semantic clusters, and a coverage check validates the result. A failure here never fails the run. +### The ecosystem seam + +`run_pipeline` also takes an `ecosystem: Ecosystem[App, Main, Unit]` +([composer/pipeline/ecosystem.py](composer/pipeline/ecosystem.py)) — the *front-half* plug point, +orthogonal to the backend. It pairs a **language** (how the target's source is read — fs-exclusion +pattern, code-explorer prompt) with a **chain** and supplies the domain-specific pieces the shared +steps need: the analyzed-model type (`App`), the analysis/property prompts, model validation, how +to locate the target `Main`, and `units(main) -> list[Unit]` (the per-unit split the extraction and +formalization phases iterate). `EVM` binds `(SourceApplication, ContractInstance, +ContractComponentInstance)`; `SOLANA` binds its own whole-program types. A unit is any +`FeatureUnit` ([spec/system_model.py](composer/spec/system_model.py)) — the ecosystem-agnostic +interface the driver uses for per-unit cache keys, task ids, and labels. The default is `EVM`, so +Solidity backends pass nothing. + ### The backend contract -A backend implements `PipelineBackend[P, FormT, H, A]` plus three phase objects: +A backend implements `PipelineBackend[P, FormT, H, A, U, Main]` plus three phase objects. `U` +(the `FeatureUnit` type) and `Main` are the ecosystem-tying parameters — `run_pipeline` binds a +`PipelineBackend[..., U, Main]` to an `Ecosystem[App, Main, U]` so the analyzed model, main-unit, +and per-unit values flow through without casts: | Object | Responsibility | Prover impl | Foundry impl | |---|---|---|---| | `PipelineBackend` | Phase-enum map, analysis prompt, artifact store, `prepare_system` | [spec/source/pipeline.py](composer/spec/source/pipeline.py) | [foundry/pipeline.py](composer/foundry/pipeline.py) | -| `PreparedSystem` | Holds the located main contract; builds the `Formalizer` | harness-lifted app + AutoSetup/invariants | identity | -| `Formalizer` | `formalize()` one component; `fetch_verdicts`; `finalize` | `batch_cvl_generation` + prover | `batch_foundry_test_generation` + `forge test` | +| `PreparedSystem` | Holds the located `Main`; builds the `Formalizer` | harness-lifted app + AutoSetup/invariants | identity | +| `Formalizer` | `formalize()` one unit; `fetch_verdicts`; `finalize` | `batch_cvl_generation` + prover | `batch_foundry_test_generation` + `forge test` | The phase enum (`CorePhases`) lets each backend label its own phases while the driver tags the three universal ones (analysis / extraction / formalization) for the UI and the replay tapes. @@ -223,6 +250,11 @@ the authoring agent. A callback protocol streams per-rule outcomes to the UI and properties as Solidity `.t.sol` tests verified by `forge test` instead of CVL + prover. Verdicts come from test pass/expected-failure status. Entry points: [cli/console_foundry.py](composer/cli/console_foundry.py), `tui_foundry.py`. +- **Solana ecosystem** ([composer/spec/solana/](composer/spec/solana/)) — the *front half* for + Rust/Solana targets: a `SolanaApplication` analysis model, Solana-specific analysis/property + prompts ([templates/solana/](composer/templates/solana/)), and whole-program `units()` (one + unit per program rather than per component). It plugs into the same driver via the `SOLANA` + ecosystem; the matching verification backend (Crucible) lands separately. - **NatSpec** ([composer/spec/natspec/](composer/spec/natspec/)) — a *greenfield* workflow (its own asyncio orchestrator, not the generic driver) that goes from a design doc to Solidity interfaces, stub implementations, and CVL. A semaphore-serialized "semantic registry" diff --git a/composer/pipeline/core.py b/composer/pipeline/core.py index 53cee3f5..5f42ae9d 100644 --- a/composer/pipeline/core.py +++ b/composer/pipeline/core.py @@ -95,8 +95,8 @@ class PreparedSystem[FormT: BackendResult, U: FeatureUnit, Main](ABC): #: :class:`~composer.spec.system_model.ContractInstance`, Solana's #: :class:`~composer.spec.solana.model.SolanaProgramInstance`). This is a *different* axis #: from :class:`FeatureUnit` (the per-unit items ``units()`` iterates): EVM's main is not a - #: unit. The driver treats it opaquely — it only hands it to ``ecosystem.units(main)`` / - #: ``extraction_unit(main)`` — so each backend binds ``Main`` to its ecosystem's type. + #: unit. The driver treats it opaquely — it only hands it to ``ecosystem.units(main)`` — + #: so each backend binds ``Main`` to its ecosystem's type. main: Main @abstractmethod From 6add7a1d55c26eb8dbb2907299aea0e22ab61327 Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 16:23:59 -0700 Subject: [PATCH 14/19] docs/ecosystem: rewrite as description-of-what-is; fix misplaced Rust fragment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ecosystem-abstraction.md was a proposal (title, "Status: proposal", phased plan, open questions, future tense) and had drifted from the code. Rewrite it as present-tense documentation of the implemented seam: - Describe the actual Language + Ecosystem dataclasses and the ECOSYSTEMS registry (not the earlier Language+Chain protocol sketch). - Cover only what exists: the seam, EVM (SOLIDITY ⊕ evm), and the Solana front half (RUST ⊕ solana, whole-program units, shared Rust fragment). - Drop the unbuilt/off-branch material (Soroban chain, verification backends, rustapp selection wiring) and the dead companion links. - Refresh the ecosystem.py module docstring to match. Also fix a real PR-split bug this surfaced: solana/property_prompt.j2 does `{% include "rust/_failure_modes.j2" %}` and RUST.failure_modes_partial points at it, but the file was only added on the PR2 (rust) branch — so rendering the Solana property prompt raised TemplateNotFound on this branch. Add the file here, alongside the Solana front half that consumes it. Co-Authored-By: Claude Opus 4.8 --- composer/pipeline/ecosystem.py | 9 +- composer/templates/rust/_failure_modes.j2 | 15 + docs/ecosystem-abstraction.md | 555 +++++++--------------- 3 files changed, 183 insertions(+), 396 deletions(-) create mode 100644 composer/templates/rust/_failure_modes.j2 diff --git a/composer/pipeline/ecosystem.py b/composer/pipeline/ecosystem.py index 7349da15..23a57d9d 100644 --- a/composer/pipeline/ecosystem.py +++ b/composer/pipeline/ecosystem.py @@ -9,11 +9,10 @@ facet (the platform model + prompts). The language here is that of the *code under analysis*, not the language the AutoProver backend is implemented in (see :class:`Language`). -Phase 1 introduces the seam and captures today's behavior as ``EVM = SOLIDITY ⊕ evm`` — -a *move*, not a rewrite: the existing `SourceApplication`, prompt templates, -`_validate_connectivity`, `main_instance`, and unit enumeration become the EVM ecosystem's -members. The driver defaults to ``EVM``, so existing applications are unchanged. Solana / -Soroban chains and the prompt-fragment split land in later phases. +``EVM = SOLIDITY ⊕ evm`` binds the EVM types, prompt templates, ``_validate_connectivity``, +``main_instance``, and unit enumeration into the seam; ``SOLANA = RUST ⊕ solana`` binds the +Solana model + prompts and reuses the shared ``RUST`` language facet. The driver defaults to +``EVM``, so Solidity applications pass no ecosystem. See ``docs/ecosystem-abstraction.md``. """ from dataclasses import dataclass diff --git a/composer/templates/rust/_failure_modes.j2 b/composer/templates/rust/_failure_modes.j2 new file mode 100644 index 00000000..99d71091 --- /dev/null +++ b/composer/templates/rust/_failure_modes.j2 @@ -0,0 +1,15 @@ +Rust language-level failure modes (shared by every Rust on-chain ecosystem — Solana, Soroban, …): + +- **Integer overflow / underflow.** On-chain Rust builds usually enable `overflow-checks`, so an + overflow *panics* (aborting the transaction) rather than wrapping — a denial-of-service / + griefing vector, not silent corruption. But code that uses `wrapping_*`, `unchecked_*`, `as` + casts (truncation), or arithmetic in a non-checked build can silently produce wrong values. + Look for arithmetic on balances, amounts, indices, and lengths. +- **Panics as aborts.** `unwrap()`, `expect()`, `panic!`, array indexing out of bounds, and + `unreachable!` all abort the transaction. An attacker-controllable input that forces one is a + DoS / stuck-funds vector (e.g. an operation that can be made to always panic). +- **Lossy conversions / precision.** `as` casts between integer widths truncate; integer + division rounds toward zero. Rounding direction that favors the caller over the protocol + (or vice versa) across two code paths is an arbitrage / value-leak vector. +- **Unchecked results.** An ignored `Result`/error (`let _ = ...`, `.ok()`, `?` swallowed) that + should have aborted lets execution continue in an inconsistent state. diff --git a/docs/ecosystem-abstraction.md b/docs/ecosystem-abstraction.md index ed6daf10..a7b06cc2 100644 --- a/docs/ecosystem-abstraction.md +++ b/docs/ecosystem-abstraction.md @@ -1,468 +1,241 @@ -# Proposal — The Ecosystem Abstraction (EVM, Solana, Soroban) - -> A proposal to make AutoProver's shared pipeline parametric over an **ecosystem** — -> the blockchain/source domain being analyzed — selected by an application-level parameter -> that picks the right system model, prompts, source conventions, and validation. Today the -> "generic" pipeline is generic over the *backend* (how a property becomes a verified -> artifact) but silently hardwired to Solidity for everything else. This introduces a second, -> orthogonal axis — and factors it further into a **language** facet (Solidity, Rust) and a -> **chain** facet (EVM, Solana, Soroban), so the Rust-specific prompts and source conventions -> are written once and shared between Solana and Soroban while their blockchain-specific -> models and failure modes stay separate. -> -> Companion to [formalization-abstraction.md](./formalization-abstraction.md) (the backend -> seam), [application-abstraction.md](./application-abstraction.md) (the five pieces of an -> application), and [rust-applications.md](./rust-applications.md) (the Rust app framework -> the first Solana/Soroban backends will likely use). Status: **proposal / for review.** +# The Ecosystem Abstraction ---- - -## 1. Problem & motivation - -We want to author properties for **Solana** programs (Rust/Anchor) and **Soroban** contracts -(Rust/soroban-sdk), not just EVM/Solidity. The pipeline advertises itself as backend-agnostic, -and it is — `run_pipeline` is generic over the result type `FormT` -([formalization-abstraction.md](./formalization-abstraction.md)). But "backend-agnostic" is -not "domain-agnostic." An audit of the shared spine (see §3) shows Solidity/EVM assumptions -baked into three shared places the driver owns: - -1. the **system model** — the pydantic types the analysis phase produces; -2. the **prompts** — the analysis and property-extraction templates; -3. a few **source conventions** — the fs-exclusion default, the "main contract" locator. +AutoProver's shared pipeline is parametric over an **ecosystem** — the blockchain/source +domain being analyzed. The ecosystem supplies the domain-specific *front half* of a run: the +system-model type the analysis phase produces, the analysis and property-extraction prompts, +the source-reading conventions, connectivity validation, how the target's "main" is located, +and how it is split into units. Everything downstream of properties (how a property becomes a +verified artifact) belongs to the **backend**, a separate axis. -None of this is tooling (no `solc`/`slither` in the shared steps — it's pure LLM + generic -file reading), so the work is types + prompts + a small driver generalization, not new -analysis engines. - -We propose making this a first-class **ecosystem** parameter, so `evm` reproduces today's -behavior exactly and `solana` / `soroban` slot in beside it without forking the pipeline — -and, because two of those three are Rust, factoring the ecosystem so the Rust-specific parts -are shared rather than copied (§2.1). +Today two ecosystems are implemented: `EVM` (Solidity, fully wired to the CVL/prover and +Foundry backends) and `SOLANA` (Rust, front half only — analysis + property extraction, gated +by a null backend). Companion: [application-abstraction.md](./application-abstraction.md) (the +pieces of an analyzed application). --- -## 2. Two orthogonal axes +## 1. Two orthogonal axes -The pipeline has a front half and a back half joined by *properties*: +The pipeline has a front half and a back half, joined by *properties*: ``` ┌─────────── ECOSYSTEM owns ───────────┐ ┌──────── BACKEND owns ────────┐ source ─analyze─▶ SystemModel ─extract─▶ properties ─formalize─▶ artifact ─verdicts─▶ (how we MODEL and REASON about (how a property becomes a the domain: contracts vs programs, checkable, verified artifact: - storage vs accounts, reentrancy CVL+prover / foundry / a Rust - vs missing-signer) Solana verifier) + storage vs accounts, reentrancy CVL + prover / foundry test / …) + vs missing-signer) └──────── SHARED: report ────────┘ ``` -- **Ecosystem** = the *front half*: the system-model types, the analysis + property-extraction - prompts, source conventions, and connectivity validation. -- **Backend** = the *back half*: `prepare_system` → `Formalizer` (`formalize` / `fetch_verdicts`), - documented in [formalization-abstraction.md](./formalization-abstraction.md). -- **Report** stays shared and neutral. - -The axes are conceptually independent — but they meet at the analyzed model: the backend's -`prepare_system(analyzed: App)` consumes the ecosystem's `App` type. So a **backend is written -against an ecosystem's model** (the CVL prover backend needs `SourceApplication`; a Solana -verifier needs `SolanaApplication`). An application picks a **(ecosystem, backend) pair that -agree on `App`.** That pairing is the one coupling to make explicit (§6). - -| | EVM | Solana | Soroban | -| --- | --- | --- | --- | -| Compatible backends | `prover` (CVL), `foundry` | a Rust verifier (via `rustapp`) | a Rust verifier (via `rustapp`) | -| System model | `SourceApplication` (contracts) | `SolanaApplication` (programs) — new | `SorobanApplication` (contracts) — new | -| Unit of extraction | contract component | instruction / account-validation group | contract function | - -### 2.1 Ecosystems factor into a *language* facet and a *chain* facet - -Solana and Soroban are both **Rust**; EVM is **Solidity**. Much of what the front half needs -is fixed by the *source language* — how to read/navigate it, the project layout to exclude -(Cargo vs Foundry), the language-level failure modes (Rust: integer overflow, `panic!` / -`unwrap` / `expect` aborts, ownership) — and is identical across every chain that uses that -language. The rest is fixed by the *chain/platform*: the system model, the storage and -authorization semantics, the platform failure modes. So an ecosystem is a **composition**: - -```text - ecosystem = language facet ⊕ chain facet - evm = solidity ⊕ evm - solana = rust ⊕ solana ┐ share the SAME rust language facet - soroban = rust ⊕ soroban ┘ (prompts, fs conventions, overflow/panic) -``` - -| Facet | Owns | `solidity` | `rust` (shared by `solana` + `soroban`) | -| --- | --- | --- | --- | -| **Language** | fs-exclusion default, `code_explorer` prompt, source-navigation framing, the language-level failure-mode prompt fragment | assembly / delegatecall, checked-arith caveat | integer overflow/underflow, `panic!`/`unwrap`/`expect` aborts, ownership/borrow | -| **Chain** | system-model type, connectivity validation, main-unit locator + units, the platform failure-mode prompt fragment, SDK conventions | contracts, storage, ERC standards | *differs per chain* — see §8 | - -The payoff (the Soroban question directly): **the Rust language facet is authored once and -shared by both Solana and Soroban** — the Rust source conventions, the `code_explorer` prompt, -and the Rust failure-mode prompt fragment. Only the chain facet differs between them. +- **Ecosystem** = the *front half*: the system-model type, the analysis + property-extraction + prompts, source conventions, connectivity validation, main-unit location, and unit split. +- **Backend** = the *back half*: `prepare_system` → `Formalizer` (`formalize` / `fetch_verdicts`). +- **Report** is shared and domain-neutral. -One honest caveat: the sharing is **not strictly hierarchical.** Solana and Soroban share the -Rust *language*, but Soroban's *model* — a contract that owns typed storage, with explicit -authorization and cross-contract calls — is closer to EVM's than to Solana's (programs -operating on externally-passed accounts). So facets are best expressed as **composable prompt -fragments keyed by concern** (§4.1), not a rigid two-level inheritance: `soroban` pulls the -`rust` language fragments *and* may reuse EVM-flavored "contract-owns-storage / authorization" -analysis fragments, while `solana` does not. +The axes meet at the analyzed model: the backend's `prepare_system(analyzed: App)` consumes the +ecosystem's `App` type, so a **backend is written against an ecosystem's model** — the CVL prover +backend needs `SourceApplication`; a Solana backend needs `SolanaApplication`. `run_pipeline` +ties a `PipelineBackend[..., U, Main]` to an `Ecosystem[App, Main, U]`, so the analyzed model, +main-unit, and per-unit values flow through without casts. --- -## 3. What is Solidity-specific today (audit) +## 2. The seam -Condensed from the audit; all in the *shared* pipeline, not the backends. - -| Concern | Where | Ecosystem-specific? | -|---|---|---| -| System-model types (`ExplicitContract`, `solidity_identifier` + regex, `ContractComponent.state_variables`/`external_entry_points`, `ContractSort`, EOA `ExternalActor`) | [system_model.py](../composer/spec/system_model.py) | **Yes** — EVM-shaped | -| Driver pins the model: `run_component_analysis(ty=SourceApplication)`, `prepare_system(analyzed: SourceApplication)`, `main_instance` matching `solidity_identifier`, `_extract_all` → `ContractComponentInstance`, the "explicit contract instance with this solidity identifier" `extra_input` | [core.py](../composer/pipeline/core.py) | **Yes** — hardcoded | -| Analysis prompts ("Smart Contracts", `solidity_identifier` block, `ContractSort` deploy semantics, ERC20/4626/721) | `application_analysis_system.j2` / `application_analysis_prompt.j2` | **Yes** — heavy rewrite | -| Property-extraction prompts (reentrancy, oracle manipulation, MEV, storage layout, checked arithmetic/`uint256`) | `property_analysis_system_prompt.j2` / `property_analysis_prompt.j2` | **Yes** — failure-mode vocabulary | -| Connectivity validation (contract/component/actor shape) | `_validate_connectivity` in [system_analysis.py](../composer/spec/system_analysis.py) | **Yes** — structure reusable, types EVM | -| fs-exclusion default (`lib/`, `test/`, `.sol` carve-out) | `FS_FORBIDDEN_READ` [util.py:59](../composer/spec/util.py) | **Yes** — but already a per-input param | -| `code_explorer` prompt ("smart contract source code") | [code_explorer.py](../composer/spec/code_explorer.py) | Cosmetic | -| Source tools (`fs_tools`, `code_explorer`, `code_document_ref`) | [source_env.py](../composer/spec/source/source_env.py) | **No** — language-neutral, read Rust fine | -| `backend_guidance` ("what's expressible downstream") | [prop_inference.py](../composer/spec/prop_inference.py) | **No** — already backend-supplied | -| Report (`Verdict`, `RuleName`, `unit_file`, `Outcome`) | [report/collect.py](../composer/spec/source/report/collect.py) | **No** — neutral | - -Two useful facts fell out: `run_component_analysis` is *already* generic (`[T: BaseApplication]`) -— only the driver pins `SourceApplication`; and `SolidityIdentifier`'s regex already accepts -Rust identifiers, so it's not a hard blocker (just misnamed). - ---- - -## 4. The seam: a `Language` facet and a `Chain` facet - -Two small protocols, composed. The **chain** carries its **language**; the driver consumes the -composed ecosystem (which is just a resolved chain). Sketch (illustrative, not final signatures): +The seam lives in [composer/pipeline/ecosystem.py](../composer/pipeline/ecosystem.py) as two +frozen dataclasses. An ecosystem factors into a **language** facet (conventions for reading the +analyzed program's *source* — shared by every chain written in that language) and the **chain** +itself (the platform model + prompts). The language here is that of the *code under analysis*, +not the language a backend is implemented in. ```python -# composer/pipeline/ecosystem.py -type LanguageTag = Literal["solidity", "rust"] -type ChainTag = Literal["evm", "solana", "soroban"] +LanguageTag = Literal["solidity", "rust"] +ChainTag = Literal["evm", "solana", "soroban"] # "soroban" is reserved; not yet wired @dataclass(frozen=True) -class PromptPair: - system: str # j2 template name - initial: str - -class Language(Protocol): - """Shared by every chain that uses this source language.""" +class Language: name: LanguageTag - default_forbidden_read: str # Cargo layout vs Foundry layout - code_explorer_prompt: str # "Rust source" vs "Solidity source" - failure_modes_partial: str # j2 partial: language-level failure modes (overflow, panics …) + default_forbidden_read: str # fs-exclusion regex (Cargo layout vs Foundry layout) + code_explorer_prompt: str # source-navigation framing ("Rust source" vs "Solidity") + failure_modes_partial: str | None = None # j2 partial of language-level failure modes -class Chain[App: BaseApplication, Main, Unit](Protocol): +@dataclass(frozen=True) +class Ecosystem[App: BaseApplication, Main, Unit: FeatureUnit]: name: ChainTag - language: Language # <-- the shared facet (RUST for solana AND soroban) - - # --- domain model --- - system_model: type[App] # the analyzed pydantic type - def validate_analysis(self, app: App, expected_main: str | None) -> list[str]: ... - def locate_main(self, app: App, source: SourceCode) -> Main: ... - def units(self, main: Main) -> list[Unit]: ... - - # --- prompts (chain templates that compose in the language partials, §4.1) --- - analysis_prompts: PromptPair + language: Language + system_model: type[App] # the pydantic type analysis produces + analysis_prompts: PromptPair # (system, initial) template names property_prompts: PromptPair - -type Ecosystem = Chain # an ecosystem is a chain that carries its language -``` - -`Main`/`Unit` generalize today's `ContractInstance` / `ContractComponentInstance` — thin index -wrappers over `App` that the driver hands to the backend (`to_artifact_id`, `prepare_system`) and -to property inference. For EVM they *are* those types unchanged. - -A registry selects by chain tag; `RUST` and `SOLIDITY` are the shared language singletons: - -```python -RUST = _RustLanguage(...) # authored ONCE -SOLIDITY = _SolidityLanguage(...) -ECOSYSTEMS: dict[ChainTag, Ecosystem] = {"evm": EVM, "solana": SOLANA, "soroban": SOROBAN} -# where SOLANA.language is RUST and SOROBAN.language is RUST — the same object. -``` - -### 4.1 Prompt composition — how the Rust prompts are shared - -Prompts are **assembled from fragments with Jinja2 includes/inheritance**, not duplicated. A -chain's property template pulls the shared language fragment, then adds its own: - -```jinja -{# composer/templates/solana/property_prompt.j2 (soroban/ is identical but for the last include) #} -{% extends "property_prompt_base.j2" %} -{% block failure_modes %} - {% include "rust/_failure_modes.j2" %} {# SHARED: overflow, panics, unwrap — solana AND soroban #} - {% include "solana/_failure_modes.j2" %} {# chain-specific: signer/owner/PDA/CPI checks #} -{% endblock %} -``` - -So `rust/_failure_modes.j2`, the Rust `code_explorer` prompt, and the Cargo `forbidden_read` are -authored once as the `RUST` language and referenced by both `solana` and `soroban`; Soroban swaps -only the final `{% include "soroban/_failure_modes.j2" %}` and its chain facet. The base template -(`property_prompt_base.j2`) holds the ecosystem-neutral skeleton — the invariant / safety / -attack-vector framing, the output contract — so *that* is shared across all three. - ---- - -## 5. Selection: an application parameter - -The ecosystem is chosen per application, threaded to `run_pipeline` alongside the backend. - -- **Built-in apps.** `run_autoprove_pipeline` / `run_foundry_pipeline` pass `ecosystem=EVM` - explicitly (one line; both are EVM). -- **Rust apps.** The [`AppDescriptor`](./rust-applications.md) gains an `ecosystem` field - (default `"evm"`); the host resolves `ECOSYSTEMS[descriptor.ecosystem]` and passes it into - `run_application`. This adds one string field to `rust/autoprover-sdk` and one lookup in - `composer/rustapp/host.py`. - -```python -# composer/rustapp/descriptor.py -class AppDescriptor(BaseModel): - ... - ecosystem: ChainTag = "evm" # "evm" | "solana" | "soroban" + validate_analysis: Callable[[BaseApplication, SolidityIdentifier | None], str | None] + locate_main: Callable[[App, SourceCode], Main] # find the "main" contract/program + units: Callable[[Main], list[Unit]] # split into per-unit extraction items + analysis_extra_input: Callable[[SourceCode], list[str | dict]] ``` -So a Solana application is `ecosystem="solana"` + a Solana backend wheel, and a Soroban -application is `ecosystem="soroban"` + a Soroban backend wheel. Nothing else in the app shell -changes — the generic entry point/frontend already synthesize from the descriptor, and both Rust -chains transparently pick up the shared `RUST` language facet. - ---- - -## 6. Driver generalization +`Main` and `Unit` generalize what were EVM's `ContractInstance` / `ContractComponentInstance` — +thin index wrappers over `App` that the driver hands to the backend and to property inference. +`Unit` is any [`FeatureUnit`](../composer/spec/system_model.py) — the ecosystem-agnostic +interface (`display_name` / `slug` / `unit_index` / `cache_material` / `context_tag` / +`feature_json`) the driver uses for per-unit cache keys, task ids, and labels. -Localized changes; the phase chain and concurrency are untouched. - -**`run_pipeline`** ([core.py](../composer/pipeline/core.py)) takes `ecosystem: Ecosystem` and -stops hardcoding EVM: +A registry exposes the ecosystems by chain tag; it is heterogeneous in `App`/`Main`/`Unit` +(each chain has its own model), hence `Ecosystem[Any, Any, Any]`: ```python -async def run_pipeline[P, FormT, H, A, App]( - backend: PipelineBackend[P, FormT, H, A, App], - run: PipelineRun[P, H], - ecosystem: Ecosystem[App, ...], - *, ... -): - analyzed = await run.runner(..., lambda: run_component_analysis( - ty=ecosystem.system_model, # was: SourceApplication - prompts=ecosystem.analysis_prompts, # was: hardcoded templates - validate=ecosystem.validate_analysis, # was: _validate_connectivity - expected_main_id=source.contract_name, ...)) - prepared = await backend.prepare_system(analyzed, run) - ... - main = ecosystem.locate_main(analyzed, run.source) # was: main_instance(...) by solidity_identifier - batches = await _extract_all(ecosystem.units(main), ecosystem.property_prompts, ...) +ECOSYSTEMS: dict[ChainTag, Ecosystem[Any, Any, Any]] = {"evm": EVM, "solana": SOLANA} ``` -**`run_component_analysis`** ([system_analysis.py](../composer/spec/system_analysis.py)) — -already generic over `T`; additionally accept the prompt pair + validation function instead of -importing `_validate_connectivity` and hardcoding template names. - -**`run_property_inference`** ([prop_inference.py](../composer/spec/prop_inference.py)) — accept -the ecosystem's property prompt pair and a generic `Unit` (it already takes `backend_guidance` -as a param, so the "expressible downstream" axis stays backend-owned; the "failure modes in this -domain" axis moves into the ecosystem's prompt). - -**`PipelineBackend` / `SystemAnalysisSpec`** — add the `App` type parameter so -`prepare_system(analyzed: App)` and `to_artifact_id(unit: Unit)` type-check against the paired -ecosystem. `SystemAnalysisSpec` keeps `analysis_key` + `extra_input` (backend/app-owned); the -analyzed *type* and templates move to the ecosystem. - --- -## 7. The EVM ecosystem (= today, zero behavior change) +## 3. The EVM ecosystem (`SOLIDITY ⊕ evm`) -`EVM` = the `SOLIDITY` language facet ⊕ the `evm` chain facet, a faithful capture of current -behavior, so autoprove/foundry are byte-for-byte unchanged: +`EVM` is the `SOLIDITY` language facet composed with the `evm` chain. Its members are the +pre-existing EVM types, prompts, and functions bound into the seam — the CVL prover and Foundry +backends run against it unchanged. ```python -SOLIDITY = _Language( +SOLIDITY = Language( name="solidity", - default_forbidden_read=FS_FORBIDDEN_READ, + default_forbidden_read=FS_FORBIDDEN_READ, # Foundry layout: lib/, test/, .sol carve-out code_explorer_prompt=CODE_EXPLORER_SYS_PROMPT, - failure_modes_partial="solidity/_failure_modes.j2", ) -EVM = _Chain( +EVM: Ecosystem[SourceApplication, ContractInstance, ContractComponentInstance] = Ecosystem( name="evm", language=SOLIDITY, system_model=SourceApplication, analysis_prompts=PromptPair("application_analysis_system.j2", "application_analysis_prompt.j2"), property_prompts=PromptPair("property_analysis_system_prompt.j2", "property_analysis_prompt.j2"), - validate_analysis=_validate_connectivity, # moved, not rewritten - locate_main=main_instance, # moved, not rewritten - units=lambda main: [ContractComponentInstance(_contract=main, ind=i) - for i in range(len(main.contract.components))], + validate_analysis=_validate_connectivity, + locate_main=main_instance, # match by solidity_identifier + units=_evm_units, # one unit per contract component + analysis_extra_input=_evm_analysis_extra_input, ) ``` -The migration is a *move*, not a rewrite: existing types, prompts, and functions become the EVM -ecosystem's members (the current monolithic prompts stay as-is at first; splitting out a -`solidity/_failure_modes.j2` partial can wait until Soroban wants to reuse EVM fragments, §8). -This is the safety property of the proposal — the refactor is provably behavior-preserving for -EVM before any Rust chain adds anything. +EVM's unit split is one `ContractComponentInstance` per component of the located contract, so +property extraction fans out one agent per component — the historical per-component behavior. --- -## 8. The Rust chains: Solana and Soroban (shared language facet) +## 4. The Solana ecosystem (`RUST ⊕ solana`) -Both are `RUST` ⊕ their chain facet. The **`RUST` language facet is authored once** and both -reuse it verbatim: +`SOLANA` is the `RUST` language facet composed with the `solana` chain. The front half is +implemented and exercised by a null (report-only) backend; the verification backend is a +separate effort and not part of this seam. ```python -RUST = _Language( +RUST = Language( name="rust", - # Cargo layout: exclude target/ and .git; KEEP tests/ (unlike Foundry) and the crate sources. - default_forbidden_read=r"(^target/.*)|(^\.git.*)|(.*\.lock$)", - code_explorer_prompt=RUST_CODE_EXPLORER_PROMPT, # "Rust source … modules/traits/impls" - failure_modes_partial="rust/_failure_modes.j2", # overflow/underflow, panic!/unwrap/expect, ownership + # Cargo/Anchor layout: hide build output, VCS, lockfiles, and the JS side; keep crate sources + tests/. + default_forbidden_read=r"(^target/.*)|(^\.git.*)|(^node_modules/.*)|(.*\.lock$)", + code_explorer_prompt=RUST_CODE_EXPLORER_PROMPT, # "Rust source … instruction handlers, Accounts, PDAs" + failure_modes_partial="rust/_failure_modes.j2", # overflow/underflow, panic!/unwrap/expect, ownership +) + +SOLANA: Ecosystem[SolanaApplication, SolanaProgramInstance, SolanaProgramInstance] = Ecosystem( + name="solana", + language=RUST, + system_model=SolanaApplication, + analysis_prompts=PromptPair("solana/analysis_system.j2", "solana/analysis_prompt.j2"), + property_prompts=PromptPair("solana/property_system.j2", "solana/property_prompt.j2"), + validate_analysis=_solana_validate, + locate_main=_solana_locate_main, # match by program_identifier + units=_solana_units, # whole-program: a singleton [main] + analysis_extra_input=_solana_analysis_extra_input, ) ``` -`rust/_failure_modes.j2` is the concrete answer to "share Rust prompts between Solana and -Soroban": it is `{% include %}`d by both chains' property templates (§4.1). Each chain then -supplies only its own model + validation + platform fragment. - -### 8.1 Solana chain (`RUST ⊕ solana`) - -- **System model** (`composer/spec/solana/model.py`, new) — `SolanaApplication` with `Program` - (program id / `crate::module`), `Instruction` (entry points), `AccountGroup` / account - constraints (Solana accounts are **passed in**, not owned storage), and CPI targets / signers - in place of EOA `ExternalActor`. -- **Platform failure fragment** (`solana/_failure_modes.j2`) — missing signer/owner checks, - account substitution / confused-deputy, unvalidated PDA seeds, arbitrary CPI, lamport/rent - draining, missing Anchor constraints (`has_one`, `constraint`, `seeds`/`bump`). -- **`locate_main` / `units`** — main = the target program; units = its instructions (or - account-validation structs). - -### 8.2 Soroban chain (`RUST ⊕ soroban`) - -- **System model** (`composer/spec/soroban/model.py`, new) — `SorobanApplication` with `Contract` - (`#[contract]`), `ContractFunction` (`#[contractimpl]` entry points), and **typed contract - storage** (`instance` / `persistent` / `temporary`, each with TTL/archival) — i.e. the contract - **owns** its state, closer to EVM than to Solana. Authorization is explicit (`require_auth` / - `require_auth_for_args`); cross-contract calls go through generated clients; custom types are - `#[contracttype]`. -- **Platform failure fragment** (`soroban/_failure_modes.j2`) — missing/incorrectly-scoped - `require_auth`, storage-durability misuse (temporary vs persistent) and TTL/archival - (entry-expiration) bugs, unchecked cross-contract results / reentrancy, replay. The Rust - overflow/panic modes come from the shared `rust/_failure_modes.j2` — **not repeated here.** -- **`locate_main` / `units`** — main = the target contract; units = its contract functions. -- **Fragment reuse across the *chain* axis, too.** Because Soroban's model is - contract-owns-typed-storage with explicit authorization, its *analysis* prompt can - `{% include %}` EVM-flavored fragments about storage and authorization that Solana cannot — - the non-hierarchical sharing noted in §2.1. This is exactly why fragments beat rigid inheritance. - -The Solana/Soroban *backends* (formalization) are out of scope here — each plugs in via -[rust-applications.md](./rust-applications.md) and pairs with its chain's `App` model. +- **System model** ([composer/spec/solana/model.py](../composer/spec/solana/model.py)) — + `SolanaApplication` is the standalone analog of `SourceApplication`: `SolanaProgram`s with + their instructions and account constraints (Solana accounts are **passed in**, not owned + storage), CPI targets, and signers in place of EOA actors. `SolanaProgramInstance` / + `SolanaInstructionInstance` are the index-wrapper instances (the latter satisfies + `FeatureUnit`). +- **Whole-program units.** `units` returns a singleton `[main]` — the `Unit` type is + `SolanaProgramInstance`, the same as `Main`. All of a program's invariants are inferred over + the whole program in a single extraction rather than fanned out per instruction. +- **Validation** — `_solana_validate` mirrors `_validate_connectivity`'s structure over + `SolanaApplication`: unique program identifiers, unique instruction slugs within a program, + the expected main program present. + +### Prompt composition — the shared Rust fragment + +The `RUST` language facet is chain-independent, so its source conventions and failure-mode +fragment are authored once and pulled into the chain's prompts by Jinja `{% include %}`. The +Solana property template composes the shared Rust fragment with its own platform fragment: + +```jinja +{# composer/templates/solana/property_prompt.j2 #} +{% include "rust/_failure_modes.j2" %} {# shared: overflow, panics, unwrap, lossy casts #} +{% include "solana/_failure_modes.j2" %} {# chain-specific: signer/owner/PDA/CPI checks #} +``` + +`rust/_failure_modes.j2` (the language facet) states language-level failure modes — integer +overflow/underflow, `panic!`/`unwrap`/`expect` aborts, lossy conversions, unchecked results — +independent of any chain; `solana/_failure_modes.j2` adds the Solana-native ones. Because the +Rust facet is factored out this way, it is reusable by any future Rust chain without copying. --- -## 9. What stays shared and unchanged +## 5. Driver integration -- Source tools (`fs_tools`, `code_explorer`, `code_document_ref`) — already language-neutral; the - only ecosystem input is the `forbidden_read` default and the explorer prompt string. -- The report (`collect` / `Verdict` / schema) — neutral; `ReportBackend` already widened. -- Caching, the multi-round property loop, interactive refinement, `run_to_completion`, the whole - agent plumbing. -- The backend seam and the Rust app framework — a Solana or Soroban verifier is "just another - backend." +`run_pipeline` ([composer/pipeline/core.py](../composer/pipeline/core.py)) takes an +`ecosystem` and never hardcodes a domain. It defaults to `EVM`, so Solidity backends pass +nothing; a non-EVM backend passes its ecosystem (e.g. `ecosystem=SOLANA`). ---- +```python +async def run_pipeline[..., U, Main]( + backend: PipelineBackend[P, FormT, H, A, U, Main], + run: PipelineRun[P, H], + *, ..., + ecosystem: Ecosystem[Any, Any, Any] = EVM, +): + analyzed = await run_component_analysis( + ty=ecosystem.system_model, + prompts=ecosystem.analysis_prompts, + validate=ecosystem.validate_analysis, + extra_input=[*ecosystem.analysis_extra_input(source), *spec.extra_input], ...) + prepared = await backend.prepare_system(analyzed, run) + ... + batches = await _extract_all(..., ecosystem=ecosystem) # iterates ecosystem.units(prepared.main) +``` -## 10. Phased plan - -*Status: Phases 1–4 implemented (EVM preserved; Solana front half verified by a live gate). -Phase 5 (Soroban) and Phase 6 (verification backends) remain.* - -1. ✅ **Done — Extract `Language` + `Chain` + `EVM` (= `SOLIDITY ⊕ evm`), behavior-preserving.** - Added `composer/pipeline/ecosystem.py`; moved the `SourceApplication` reference, template - names, `_validate_connectivity`, `main_instance`, and unit-enumeration into `EVM`, and the - `forbidden_read`/`code_explorer` defaults into `SOLIDITY`. Threaded `ecosystem` through - `run_pipeline` / `run_component_analysis` / `run_property_inference`. Prompts stay monolithic - (no fragment split yet). **Gate met:** the autoprove end-to-end integration test passes - identically on this commit and its pre-refactor parent (real Postgres + live prover), and EVM - reproduces the prior template names / validator / analysis front-matter verbatim. - > **Env note (orthogonal to the refactor):** running that gate surfaced a pre-existing solc - > provisioning issue — the Counter scenario pins `pragma ^0.8.29` but the environment's default - > `solc` was 0.8.21, so the prover couldn't compile it (manifesting as an "exhausted tape"). The - > fix is environmental (point `solc` at ≥0.8.29; versioned `solc8.29` was already present), not a - > tape re-record. Worth pinning a matching `solc` in the gate's prover config so it's robust. -2. ✅ **Done — Add the `App` type parameter.** `PipelineBackend[P, FormT, H, A, App]` with - `prepare_system(analyzed: App)`; `Ecosystem` is generic over `App` (`EVM: Ecosystem[SourceApplication]`); - `run_pipeline` takes `ecosystem: Ecosystem[App]` explicitly and the Phase 1 - `cast(SourceApplication, analyzed)` is gone. `SystemAnalysisSpec` was intentionally **not** - parameterized — it carries only `analysis_key` + `extra_input`, no `App`-typed member. - **Gate met:** pyright reports 0 errors on the touched files (pairing type-checks, no cast). -3. ✅ **Done — Wire selection into `rustapp`.** `AppDescriptor.ecosystem: ChainTag` (Rust SDK, - default `"evm"`, + the Python mirror) and `resolve_ecosystem()` in `host.py` (registry lookup, - clear error for an unregistered chain), threaded through `build_application` / - `run_application` / `run_rust_pipeline` in place of the hardcoded `EVM`. **Gate met:** rustapp - tests + pyright green; only `evm` resolves for now (Solana/Soroban register in Phases 4–5). -4. ✅ **Done — Author the `RUST` language facet + the Solana chain.** This phase also performed - the driver's `Unit`/`Main` generalization deferred from Phase 2 (a `FeatureUnit` protocol in - `system_model`, threaded as type params through `Ecosystem`/`PreparedSystem`/`PipelineBackend`/ - `Formalizer`/`run_pipeline`; EVM stays bound to `ContractComponentInstance`/`ContractInstance` - with byte-identical cache keys). Added the standalone `SolanaApplication` model - (`composer/spec/solana/model.py`), the `RUST` language (Cargo `forbidden_read`, Rust - `code_explorer` prompt, `rust/_failure_modes.j2`) and the `SOLANA` chain (validate / - `locate_main` / `units`). The fragment-composition convention is realized with `{% include %}` - of the shared `rust/_failure_modes.j2` + `solana/_failure_modes.j2` into the Solana templates - (a neutral `property_prompt_base.j2` proved unnecessary — the Solana templates are - self-contained and just pull the fragments). A reusable `NullSolanaBackend` + an Anchor - `solana_vault` scenario back the gate. **Gate met:** a live-LLM run - (`tests/test_solana_gate.py`) analyzed the vault into 3 instructions and extracted 27 sane, - Solana-native properties (signer/owner checks, PDA + canonical-bump derivation, System-Program - substitution, reinit-once, arithmetic overflow) — no prover needed. -5. **Author the Soroban chain, reusing `RUST`.** `SorobanApplication` model + Soroban prompts; - the *only* new prompt content is `soroban/_failure_modes.j2` and the Soroban analysis template - — the Rust language fragments are inherited. **Gate:** the shared `rust/_failure_modes.j2` is - referenced by both chains and appears in neither chain's own fragment (no duplication). -6. **Solana / Soroban backends** — separate efforts, via the Rust framework. +- **`run_component_analysis`** ([system_analysis.py](../composer/spec/system_analysis.py)) is + generic over the analyzed type and accepts the prompt pair + validation function; the ecosystem + supplies all three. +- **`run_property_inference`** ([prop_inference.py](../composer/spec/prop_inference.py)) takes the + ecosystem's property prompt pair and a generic `FeatureUnit`. The "expressible downstream" axis + stays backend-owned (`backend_guidance`); the "failure modes in this domain" axis is the + ecosystem's prompt. +- **`_extract_all`** iterates `ecosystem.units(main)`, running one property-inference agent per + unit — one per component for EVM, one for the whole program for Solana. --- -## 11. Open questions - -1. **Is `Unit` uniform enough across ecosystems?** EVM's unit is a contract component; Solana's - might be an instruction *or* an account-validation struct. If per-unit shape diverges too much, - `run_property_inference` may need the ecosystem to own unit rendering (a `render_unit(unit) -> dict` - hook) rather than a shared template variable. Decide when authoring the Solana property prompt. -2. **One backend, multiple ecosystems?** Could a single Rust backend serve both (unlikely given the - `App` pairing)? Keep the pairing explicit for now; revisit if a genuinely cross-domain backend - appears. -3. **Report labels by ecosystem.** Should the report say "program"/"instruction" vs - "contract"/"rule"? Today `backend_tag` drives labels; an `ecosystem` tag on the report may be - the cleaner source for domain nouns. Minor; defer. -4. **Prompt template packaging.** Templates live in a shared dir; the fragment convention needs - per-facet subdirs (`rust/`, `solidity/`, `solana/`, `soroban/`, plus shared `*_base.j2`) and a - Jinja loader that resolves includes across them. Confirm the loader supports `{% extends %}` / - `{% include %}` across subdirs (it should — it's stock Jinja). -5. **Fragment granularity.** Is a single `_failure_modes.j2` partial per facet the right grain, or - do we want finer per-concern fragments (storage / authorization / arithmetic) so Soroban can - pull the EVM *storage/auth* fragments (§8.2) without the EVM *contract-deployment* ones? Start - coarse (one partial per facet); split a fragment only when a second consumer wants part of it. -6. **`SolidityIdentifier` naming.** Rename the shared field/type to an ecosystem-neutral - `SourceIdentifier`, or leave it (regex already fits Rust)? Cosmetic; a rename touches many - annotations — sequence it after Phase 1. +## 6. What is shared and domain-neutral + +- Source tools (`fs_tools`, `code_explorer`, `code_document_ref`) — language-neutral; they read + Rust as well as Solidity. The only ecosystem inputs are the `forbidden_read` default and the + explorer prompt string. +- The report (`collect` / `Verdict` / schema) and `ReportBackend`. +- Caching, the multi-round property loop, interactive refinement, and the agent plumbing. +- The backend seam itself — a verification backend is "just another backend," paired to an + ecosystem by its `App` model. --- -## 12. Key files +## 7. Key files | Concern | File | |---|---| -| Driver to generalize | [composer/pipeline/core.py](../composer/pipeline/core.py) | -| Ecosystem seam (new) | `composer/pipeline/ecosystem.py` | -| System analysis (accept ecosystem) | [composer/spec/system_analysis.py](../composer/spec/system_analysis.py) | -| Property inference (accept ecosystem) | [composer/spec/prop_inference.py](../composer/spec/prop_inference.py) | -| EVM system model (→ EVM ecosystem) | [composer/spec/system_model.py](../composer/spec/system_model.py) | -| Analysis / property prompts (EVM) | `composer/templates/application_analysis_*.j2` · `property_analysis_*.j2` | -| fs-exclusion default | [composer/spec/util.py](../composer/spec/util.py) | -| Source tools (unchanged) | [composer/spec/source/source_env.py](../composer/spec/source/source_env.py) · [code_explorer.py](../composer/spec/code_explorer.py) | -| Language facets + shared prompt fragments (new) | `composer/templates/{rust,solidity}/_failure_modes.j2` · `*_base.j2` | -| Solana chain model / prompts (new) | `composer/spec/solana/…` · `composer/templates/solana/…` | -| Soroban chain model / prompts (new) | `composer/spec/soroban/…` · `composer/templates/soroban/…` | -| Ecosystem selection in Rust apps | [composer/rustapp/descriptor.py](../composer/rustapp/descriptor.py) · [host.py](../composer/rustapp/host.py) · [rust/autoprover-sdk/src/lib.rs](../rust/autoprover-sdk/src/lib.rs) | -| The backend seam (unchanged) | [docs/formalization-abstraction.md](./formalization-abstraction.md) | +| The ecosystem seam | [composer/pipeline/ecosystem.py](../composer/pipeline/ecosystem.py) | +| Driver integration | [composer/pipeline/core.py](../composer/pipeline/core.py) | +| System analysis (ecosystem-driven) | [composer/spec/system_analysis.py](../composer/spec/system_analysis.py) | +| Property inference (ecosystem-driven) | [composer/spec/prop_inference.py](../composer/spec/prop_inference.py) | +| `FeatureUnit` protocol | [composer/spec/system_model.py](../composer/spec/system_model.py) | +| EVM system model + prompts | [composer/spec/system_model.py](../composer/spec/system_model.py) · `composer/templates/application_analysis_*.j2` · `property_analysis_*.j2` | +| Solana system model | [composer/spec/solana/model.py](../composer/spec/solana/model.py) | +| Solana prompts + shared Rust fragment | `composer/templates/solana/*.j2` · `composer/templates/rust/_failure_modes.j2` | +| fs-exclusion default (EVM) | [composer/spec/util.py](../composer/spec/util.py) | From f15c8aea24b91b1ef275816fd023510833a41068 Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 16:28:57 -0700 Subject: [PATCH 15/19] docs: move application-abstraction.md out of the ecosystem PR application-abstraction.md documents the (Rust) application framework, which belongs with the crucible-app PR, not this front-half ecosystem PR. Remove it here and drop the now-dangling companion link from ecosystem-abstraction.md; it is re-added on eric/crucible-app. Co-Authored-By: Claude Opus 4.8 --- docs/application-abstraction.md | 515 -------------------------------- docs/ecosystem-abstraction.md | 3 +- 2 files changed, 1 insertion(+), 517 deletions(-) delete mode 100644 docs/application-abstraction.md diff --git a/docs/application-abstraction.md b/docs/application-abstraction.md deleted file mode 100644 index 9c0e24ac..00000000 --- a/docs/application-abstraction.md +++ /dev/null @@ -1,515 +0,0 @@ -# Design Doc — What Is an AutoProver "Application" - -> How the pieces — argument parsing, service setup, the pipeline, and the UI — are -> wired into a single, runnable *application* such as **autoprove** or **foundry**, -> and the conventions a new application is expected to follow. -> -> Companion to [ARCHITECTURE.md](../ARCHITECTURE.md) and -> [formalization-abstraction.md](./formalization-abstraction.md). Where the -> formalization doc zooms into the *backend* seam (how a property becomes a verified -> artifact), this doc zooms out to the *whole vertical*: everything from `argv` to a -> rendered TUI. The [MultiJobApp design](../composer/ui/MULTI_JOB_DESIGN.md) covers -> the generic UI base this leans on. - ---- - -## 1. What "application" means here - -An AutoProver **application** is a complete, runnable vertical slice that takes a -Solidity project + a design document and drives the shared property-extraction / -formalization pipeline to a set of on-disk deliverables, rendered live to the user. - -There are two today: - -| Application | Deliverable | Backend | Entry points | -|---|---|---|---| -| **autoprove** | CVL `.spec` + `.conf`, verified by the Certora Prover | `ProverBackend` | [tui_autoprove.py](../composer/cli/tui_autoprove.py) · [console_autoprove.py](../composer/cli/console_autoprove.py) | -| **foundry** | `.t.sol` tests, gated by `forge test` | `FoundryBackend` | [tui_foundry.py](../composer/cli/tui_foundry.py) · [console_foundry.py](../composer/cli/console_foundry.py) | - -Crucially, "application" is **not** a single class. It is a *convention*: a set of -five collaborating pieces, each an implementation of a shared abstraction, wired -together by a thin `main()`. The value of the convention is that the pieces are -mutually orthogonal — you can swap the frontend (TUI ↔ console) without touching the -pipeline, and swap the backend without touching either frontend. - ---- - -## 2. The five pieces of an application - -Every application is assembled from exactly these, each keyed off one shared -type parameter — the application's **phase enum** `P`: - -``` - ┌─────────────────────────────────────────────────────────────┐ - │ main() (composer/cli/*.py) │ - │ async with entry_point(summary) as run: ← the Executor │ - │ app = FrontendApp() ← the Frontend │ - │ await run(app.make_handler) ← the seam │ - └─────────────────────────────────────────────────────────────┘ - │ │ - ┌───────────▼───────────┐ ┌─────────────▼─────────────┐ - │ 2. Entry point / │ │ 4. Frontend │ - │ Executor │ │ MultiJobApp[P, H] │ - │ argv → services → │ │ OR console handler │ - │ a run(handler) closure│ │ supplies make_handler: │ - └───────────┬───────────┘ │ HandlerFactory[P, H] │ - │ calls └───────────────────────────┘ - ┌───────────▼───────────┐ - │ 3. Pipeline │ 1. Phase enum P: HasName - │ run_pipeline(backend) │ (the spine that threads all - │ + a PipelineBackend │ five together) - └───────────┬───────────┘ - │ contributes - ┌───────────▼───────────┐ - │ 5. Artifact store │ - │ on-disk layout │ - └───────────────────────┘ -``` - -1. **Phase enum** `P` — the task-grouping vocabulary. -2. **Entry point / Executor** — argv → configured services → a `run(handler)` closure. -3. **Pipeline + backend** — the work, expressed as a `PipelineBackend` fed to the - shared `run_pipeline` driver. -4. **Frontend** — a `MultiJobApp[P, H]` subclass (TUI) or console handler that - supplies a `HandlerFactory[P, H]`. -5. **Artifact store** — the on-disk deliverable layout. - -The rest of this doc walks each piece with the autoprove and foundry -implementations side by side. - ---- - -## 3. The seam that makes it compose: `HandlerFactory` - -Before the pieces, understand the seam between them. The pipeline and the frontend -never reference each other. They meet at one protocol, -[`HandlerFactory[P, H]`](../composer/io/multi_job.py): - -```python -# composer/io/multi_job.py -class HandlerFactory[P: HasName, H](Protocol): - def __call__(self, /, info: TaskInfo[P]) -> Awaitable[TaskHandle[H]]: ... -``` - -- The **pipeline** is a producer of *work*. Every task it launches is described by a - `TaskInfo(task_id, label, phase)` and run through `run_task`, which calls the factory - to obtain a `TaskHandle` (an `IOHandler` + `EventHandler` + lifecycle callbacks). -- The **frontend** is a producer of *handlers*. It implements the factory: given a - `TaskInfo`, it mounts a panel, builds a per-task renderer, and returns the - `TaskHandle`. - -So the entire application boils down to: - -```python -async with entry_point(summary) as run: # run: Executor (the pipeline, service-loaded) - app = FrontendApp() # the frontend - result = await run(app.make_handler) # hand the factory to the pipeline -``` - -`run` is typed as an **Executor**, and its whole signature *is* the seam: - -```python -# composer/spec/source/autoprove_common.py -type Executor = Callable[[HandlerFactory[AutoProvePhase, None]], Awaitable[CorePipelineResult[GeneratedCVL]]] -# composer/foundry/pipeline.py -type FoundryPipelineExecutor = Callable[[HandlerFactory[FoundryPhase, None]], Awaitable[FoundryPipelineResult]] -``` - -Because the pipeline only ever *calls* the factory and the frontend only ever -*implements* it, the two are swappable independently. That is why autoprove has both -a TUI ([`AutoProveApp`](../composer/ui/autoprove_app.py)) and a console -([`AutoProveConsoleHandler`](../composer/ui/autoprove_console.py)) frontend against -the same pipeline, selected purely by which `make_handler` `main()` passes in. - -`H` is the human-interaction schema. Both current applications are non-interactive at -the per-task level (`H = None`; their handlers raise from `format_hitl_prompt`), but -the seam carries the type so an interactive application (e.g. the NatSpec pipeline) -plugs in without changing the contract. - ---- - -## 4. Piece 1 — the phase enum `P` - -Every application defines a single enum whose members are its task-grouping phases. -This enum is the type parameter that threads through the frontend -(`MultiJobApp[P, ...]`), the seam (`HandlerFactory[P, H]`, `TaskInfo[P]`), and the -backend (`CorePhases[P]`). It only needs to satisfy `HasName` (an enum trivially does). - -```python -# composer/ui/autoprove_app.py -class AutoProvePhase(enum.Enum): - HARNESS = "harness" - AUTOSETUP = "autosetup" - INVARIANTS = "invariants" - SUMMARIES = "summaries" - COMPONENT_ANALYSIS = "component_analysis" - BUG_ANALYSIS = "bug_analysis" - CVL_GEN = "cvl_gen" - REPORT = "report" -``` - -```python -# composer/foundry/pipeline.py -class FoundryPhase(enum.Enum): - SYSTEM_ANALYSIS = "system_analysis" - PROPERTY_EXTRACTION = "property_extraction" - TEST_GENERATION = "test_generation" - REPORT = "report" -``` - -The phase serves two roles: - -- **Grouping in the UI.** The frontend maps each phase to a human label and an - ordering, and every task lands in the section for its phase: - - ```python - # composer/ui/foundry_app.py - FOUNDRY_PHASE_LABELS = { - FoundryPhase.SYSTEM_ANALYSIS: "System Analysis", - FoundryPhase.PROPERTY_EXTRACTION: "Property Extraction", - FoundryPhase.TEST_GENERATION: "Test Generation", - } - FOUNDRY_SECTION_ORDER = ["System Analysis", "Property Extraction", "Test Generation"] - ``` - -- **The driver ↔ backend contract.** The shared driver tags four *core* phases; the - backend maps its own enum onto them via `CorePhases[P]` (see §6). Note the two enums - above differ in granularity: foundry has three phases, autoprove has eight — the - prover contributes several extra prep phases (harness, autosetup, summaries, - invariants) that the driver never knows about. The enum is the application's own - vocabulary; only the four core slots are shared. - ---- - -## 5. Piece 2 — the entry point / Executor - -Each application has an `_entry_point` async context manager that owns **all** the -imperative setup and yields the Executor closure. Its shape is a strict convention -(the foundry file's docstring literally says "Mirrors autoprove_common's shape"): - -> parse args → set up DB / RAG / store / checkpointer / logging / thread logger → -> yield a closure the caller drives with a handler factory. - -```python -# composer/spec/source/autoprove_common.py (shape shared by composer/foundry/entry.py) -@asynccontextmanager -async def _entry_point(summary: RunSummary) -> AsyncIterator[Executor]: - parser = argparse.ArgumentParser(...) - add_protocol_args(parser, RAGDBOptions) - add_protocol_args(parser, ExtendedModelOptions) - parser.add_argument("project_root", ...) - parser.add_argument("main_contract", help="Main contract as path:ContractName") - parser.add_argument("system_doc", ...) - # ... application-specific flags ... - args = cast(AutoProveArgs, parser.parse_args()) - async with autoprove_executor(args, summary) as runner: - yield runner -``` - -The heavy lifting is in the inner context manager, which: - -1. Resolves `project_root` + `main_contract` (`path:ContractName`) + `system_doc`. -2. Computes a **root cache key** from the inputs (`_root_cache_key` hashes project - root + doc content + relative path + contract name) — identical helper in both. -3. Opens the shared connection stack — `standard_connections`, a RAG DB, the async - tool context, the thread logger — under a single `async with`. -4. Reads the design doc into a `SourceCode`, builds the environment (`ServiceHost`), - and creates the `WorkflowContext`. -5. Yields a `runner(handler)` closure that calls the application's pipeline function. - -The args are declared as a **Protocol** (`AutoProveArgs`, `FoundryArgs`), not a class, -so the parser and the typed access agree without a dataclass in between: - -```python -# composer/spec/source/autoprove_common.py -class AutoProveArgs(ExtendedModelOptions, RAGDBOptions, Protocol): - project_root: str - main_contract: str - system_doc: str - max_concurrent: int - cloud: bool # ← prover-only: run jobs in the cloud - ... -``` - -```python -# composer/foundry/entry.py -class FoundryArgs(TieredModelOptions, FoundryRAGDBOptions, Protocol): - project_root: str - main_contract: str - system_doc: str - forge_binary: str # ← foundry-only - forge_timeout_s: int # ← foundry-only - max_forge_runners: int # ← foundry-only - ... -``` - -The `runner` closure each yields is the Executor: - -```python -# autoprove -async def runner(handler: HandlerFactory[AutoProvePhase, None]) -> CorePipelineResult[GeneratedCVL]: - return await run_autoprove_pipeline( - ctx=ctx, source_input=system_doc, env=source_env, handler_factory=handler, - prover_opts=prover_opts, interactive=args.interactive, ...) - -# foundry -async def runner(handler: HandlerFactory[FoundryPhase, None]) -> FoundryPipelineResult: - return await run_foundry_pipeline( - source_input=source_input, ctx=ctx, handler_factory=handler, env=env, - forge_binary=args.forge_binary, forge_timeout_s=args.forge_timeout_s, ...) -``` - -Convention points worth naming: - -- **Foundry validates its precondition in the entry point** (`foundry.toml` must - exist) — application-specific input validation belongs here, before any service is - opened. -- **Each application owns its RAG DB choice.** Foundry overrides `--rag-db`'s default - to the cheatcodes DB via a Protocol (`FoundryRAGDBOptions`) rather than a new flag. -- **The `finally` block is where run-close artifacts land** (autoprove dumps - `token_usage.json` there) — guarded so a diagnostics failure can't mask the run's - own outcome. -- **The entry point never imports a frontend.** It yields the Executor; `main()` - chooses the frontend. That is what lets one entry point back both a TUI and a - console `main()`. - ---- - -## 6. Piece 3 — the pipeline and its backend - -The Executor's closure calls the application's `run_*_pipeline` function. For both -current applications, that function is a thin wrapper that constructs a -`PipelineBackend` + a `PipelineRun` and hands them to the shared driver -[`run_pipeline`](../composer/pipeline/core.py): - -```python -# composer/spec/source/pipeline.py -async def run_autoprove_pipeline(source_input, ctx, handler_factory, env, *, prover_opts, ...): - backend = ProverBackend(ProverArtifactStore(source_input.project_root, source_input.contract_name), prover_opts) - run = PipelineRun(ctx, env, source_input, handler_factory, asyncio.Semaphore(max_concurrent)) - return await run_pipeline(backend, run, interactive=interactive, ...) -``` - -```python -# composer/foundry/pipeline.py -async def run_foundry_pipeline(source_input, ctx, handler_factory, env, *, forge_binary, ...): - artifacts = FoundryArtifactStore(source_input.project_root) - backend = FoundryBackend(artifacts, _ForgeRunConfig(forge_binary, forge_timeout_s, foundry_sem)) - run = PipelineRun(ctx, env, source_input, handler_factory, asyncio.Semaphore(max_concurrent)) - return await run_pipeline(backend, run, ...) -``` - -Notice the `handler_factory` (the frontend seam) is bundled into the `PipelineRun` — -`run.runner(task_info, job)` is how every phase of the driver spins up a task through -whatever frontend was supplied. - -The backend itself is the four-slot contract the driver reads. The application maps -its phase enum onto the four **core phases** the driver tags: - -```python -# composer/spec/source/pipeline.py # composer/foundry/pipeline.py -core_phases = CorePhases({ core_phases = CorePhases({ - "analysis": AutoProvePhase.COMPONENT_ANALYSIS, "analysis": FoundryPhase.SYSTEM_ANALYSIS, - "extraction": AutoProvePhase.BUG_ANALYSIS, "extraction": FoundryPhase.PROPERTY_EXTRACTION, - "formalization": AutoProvePhase.CVL_GEN, "formalization": FoundryPhase.TEST_GENERATION, - "report": AutoProvePhase.REPORT, "report": FoundryPhase.REPORT, -}) }) -``` - -Everything below this — `prepare_system`, `prepare_formalization`, `formalize`, -`fetch_verdicts` — is the **formalization abstraction**, documented in full in -[formalization-abstraction.md](./formalization-abstraction.md). The one-line summary -of the contrast: - -| | autoprove (`ProverBackend`) | foundry (`FoundryBackend`) | -|---|---|---| -| `FormT` | `GeneratedCVL` | `GeneratedFoundryTest` | -| `prepare_system` | harness lift + build prover tool | identity (`main_instance` only) | -| `prepare_formalization` | AutoSetup ∥ summaries ∥ invariants fan-out | trivial (formalizer already built) | -| `formalize` | author CVL, run prover, revise on CEX | author `.t.sol`, run `forge test` | -| `backend_guidance` | `CERTORA_BACKEND_GUIDANCE` | `FOUNDRY_BACKEND_GUIDANCE` | - -`backend_guidance` deserves a note as an application-shaping convention: it is a prose -string injected into the property-extraction prompt telling the agent what the -verification surface can and can't express. Foundry's, for instance, explains that a -fuzzer can't *prove* universals but *refutations are valuable* — so the same shared -extraction step produces backend-appropriate properties without the driver knowing -anything about it. - ---- - -## 7. Piece 4 — the frontend - -The frontend implements the `HandlerFactory` seam. The TUI frontends are thin -subclasses of the generic [`MultiJobApp[P, T]`](../composer/ui/multi_job_app.py) -(see [its design doc](../composer/ui/MULTI_JOB_DESIGN.md)). A frontend supplies four -things and inherits everything else: - -1. **Phase labels + section order** (constructor args), covered in §4. -2. **A per-task handler** — `create_task_handler`, returning a - `MultiJobTaskHandler` subclass. -3. **A per-task event handler** — `create_event_handler`, for domain-specific - streaming events beyond LLM messages. -4. **`make_handler`** — inherited from `MultiJobApp`; this *is* the `HandlerFactory`. - It mounts the panel/summary-row and calls the two `create_*` hooks. - -The autoprove and foundry TUIs are nearly identical in shape; they differ only in -what streams into a task's log. Both make their task handler double as its own -`EventHandler` via the `NullEventHandler` mixin: - -```python -# composer/ui/foundry_app.py -class FoundryTaskHandler(MultiJobTaskHandler[None], NullEventHandler): - async def handle_event(self, payload, path, checkpoint_id) -> None: - evt = cast(ForgeTestRunEvent, payload) - if evt["type"] == "forge_test_run": # stream forge run summaries - log = await self._ensure_forge_log() - log.write(evt["summary"]) - -class FoundryApp(MultiJobApp[FoundryPhase, FoundryTaskHandler]): - def __init__(self): - super().__init__(phase_labels=FOUNDRY_PHASE_LABELS, - section_order=FOUNDRY_SECTION_ORDER, - header_text="Foundry Test Author | ...") - def create_task_handler(self, panel, info) -> FoundryTaskHandler: - return FoundryTaskHandler(info.task_id, info.label, panel, self, ToolDisplayConfig()) - def create_event_handler(self, handler, info) -> EventHandler: - return handler # handler is its own event handler -``` - -```python -# composer/ui/autoprove_app.py — same structure; the domain events differ -class AutoProveTaskHandler(MultiJobTaskHandler[None], NullEventHandler): - async def handle_event(self, payload, path, checkpoint_id) -> None: - evt = cast(AutoProveEvent, payload) - match evt["type"]: - case "prover_output": ... # stream Certora Prover output lines - case "cloud_polling": ... # stream cloud job status -``` - -The autoprove handler additionally implements `handle_progress_event` to stream the -AutoSetup agent's output — an example of an application surfacing a backend-specific -sub-agent in its own panel. Neither handler supports HITL, so both raise from -`format_hitl_prompt` — a deliberate, explicit opt-out of a base-class hook. - -**The console frontend is the proof the seam works.** `AutoProveConsoleHandler` is a -*different* implementation of the same `HandlerFactory[AutoProvePhase, None]` that -renders to stdout instead of a Textual app: - -```python -# composer/ui/autoprove_console.py -class AutoProveConsoleHandler(MultiJobConsoleHandler[AutoProvePhase]): - """IOHandler[Never] + HandlerFactory for the auto-prove pipeline.""" -``` - -The pipeline can't tell the difference — it only ever calls `make_handler`. - ---- - -## 8. Piece 5 — the artifact store - -Deliverables are written through an [`ArtifactStore[I, FormT]`](../composer/spec/artifacts.py) -subclass — one per application. The base owns everything identical across -applications (`properties.json`, `commentary.md`, the property→units map, -`token_usage.json`); the subclass fixes the on-disk layout and adds the -format-specific bundle. This is covered in detail in -[formalization-abstraction.md §6](./formalization-abstraction.md); the application-level -point is the *convention that both applications share a project root without -colliding*: - -``` -autoprove → certora/specs/… certora/confs/… certora/ap_report/… -foundry → /*.t.sol certora/foundry/… certora/foundry/reports/… -``` - -Foundry deliberately materializes its `.t.sol` into the foundry project's own `test/` -dir (so `forge` finds them) but keeps all metadata under `certora/foundry/`, so a -co-located autoprove run and foundry run share one project without clobbering each -other's outputs. - ---- - -## 9. Piece 0 — the wiring: `main()` - -The `main()` in each `composer/cli/*.py` is the whole application in ~20 lines. It is -the *only* place that names both an entry point and a frontend, and its job is to -glue them via the seam and translate the result into user-facing output. - -```python -# composer/cli/tui_foundry.py (tui_autoprove.py is identical in shape) -async def _main() -> int: - summary = RunSummary() - async with _entry_point(summary) as pipeline: # piece 2: Executor - app = FoundryApp() # piece 4: frontend - async def work(): - result = await pipeline(app.make_handler) # ← the seam - app.notify(f"Foundry tests complete: {result.n_components} components, ...") - app._pipeline_done = True - app.set_work(work) - await app.run_async() # TUI owns the event loop - print(summary.format()) - return 0 -``` - -Two `main()` shapes exist, differing only in who owns the event loop: - -- **TUI** — the pipeline runs as a background *worker* inside the Textual app - (`app.set_work(work); await app.run_async()`), so the UI stays responsive while - the pipeline streams into it. -- **Console** — the pipeline runs directly and results print on completion: - - ```python - # composer/cli/console_autoprove.py - async with _entry_point(summary) as run: - result = await run(AutoProveConsoleHandler().make_handler) - print(summary.format()) - print(f" Components: {result.n_components} Properties: {result.n_properties}") - ``` - -Both call `import composer.bind as _` first — the side-effecting binding module that -must load before anything touches the DI container. - ---- - -## 10. Extending: defining a new application - -Because each piece is an implementation of a shared abstraction, adding an -application is a fill-in-the-blanks exercise; nothing in the driver, the UI base, or -the seam changes. - -1. **Phase enum** `P(enum.Enum)` — your task-grouping vocabulary, with at least the - four core phases (analysis / extraction / formalization / report) representable. -2. **Backend** — implement `PipelineBackend[P, FormT, H, A]` and its three phase - objects (`prepare_system` → `PreparedSystem.prepare_formalization` → `Formalizer`), - plus `backend_guidance`, `core_phases`, `analysis_spec`, `artifact_store`, - `to_artifact_id`. (Full checklist in - [formalization-abstraction.md §9](./formalization-abstraction.md).) -3. **Artifact store** — subclass `ArtifactStore`; define an `ArtifactIdentifier`. -4. **Result type** `FormT` satisfying `FormalResult` + `ReportableResult`. -5. **Pipeline function** `run__pipeline(...)` — build backend + `PipelineRun`, - call `run_pipeline`. -6. **Entry point** — an `_entry_point` context manager following the - parse → services → `yield runner` convention; declare args as a `Protocol`. -7. **Frontend(s)** — a `MultiJobApp[P, T]` subclass (phase labels, section order, - `create_task_handler`, per-task event streaming) and/or a console handler. -8. **`main()`** — glue an entry point to a frontend via `run(app.make_handler)`. - -The dependency direction is the guardrail: `main` → (entry point, frontend); -entry point → pipeline; pipeline → backend + `PipelineRun(handler_factory)`. Frontend -and backend never reference each other, and neither references `main`. Keep those -edges and the pieces stay swappable. - ---- - -## 11. Key files - -| Piece | autoprove | foundry | shared abstraction | -|---|---|---|---| -| Phase enum | [autoprove_app.py](../composer/ui/autoprove_app.py) | [foundry/pipeline.py](../composer/foundry/pipeline.py) | `HasName` ([multi_job.py](../composer/io/multi_job.py)) | -| Entry point / Executor | [autoprove_common.py](../composer/spec/source/autoprove_common.py) | [foundry/entry.py](../composer/foundry/entry.py) | — | -| Pipeline + backend | [spec/source/pipeline.py](../composer/spec/source/pipeline.py) | [foundry/pipeline.py](../composer/foundry/pipeline.py) | [pipeline/core.py](../composer/pipeline/core.py) | -| Frontend (TUI) | [autoprove_app.py](../composer/ui/autoprove_app.py) | [foundry_app.py](../composer/ui/foundry_app.py) | [multi_job_app.py](../composer/ui/multi_job_app.py) | -| Frontend (console) | [autoprove_console.py](../composer/ui/autoprove_console.py) | — | — | -| Artifact store | [spec/source/artifacts.py](../composer/spec/source/artifacts.py) | [foundry/artifacts.py](../composer/foundry/artifacts.py) | [spec/artifacts.py](../composer/spec/artifacts.py) | -| The seam | — | — | `HandlerFactory` / `TaskInfo` / `TaskHandle` ([multi_job.py](../composer/io/multi_job.py)) | -| `main()` | [tui_autoprove.py](../composer/cli/tui_autoprove.py) · [console_autoprove.py](../composer/cli/console_autoprove.py) | [tui_foundry.py](../composer/cli/tui_foundry.py) · [console_foundry.py](../composer/cli/console_foundry.py) | — | diff --git a/docs/ecosystem-abstraction.md b/docs/ecosystem-abstraction.md index a7b06cc2..9b58b305 100644 --- a/docs/ecosystem-abstraction.md +++ b/docs/ecosystem-abstraction.md @@ -9,8 +9,7 @@ verified artifact) belongs to the **backend**, a separate axis. Today two ecosystems are implemented: `EVM` (Solidity, fully wired to the CVL/prover and Foundry backends) and `SOLANA` (Rust, front half only — analysis + property extraction, gated -by a null backend). Companion: [application-abstraction.md](./application-abstraction.md) (the -pieces of an analyzed application). +by a null backend). --- From 60b2411226b7b9c2ebe2a34775f8aff4bd35e98f Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 16:45:09 -0700 Subject: [PATCH 16/19] tests: golden snapshots of front-half prompt templates Pin the rendered text of the EVM + Solana analysis/property prompts so that factoring shared boilerplate into partials is provably behaviour-preserving. Renders each template with a permissive context stub (dynamic per-target bits blank out; static boilerplate renders in full) across two control-flow variants (existing-source + prior rounds, greenfield + none). Baseline captured from the current templates; EVM goldens must not change. Co-Authored-By: Claude Opus 4.8 --- .../prompts/evm_analysis_prompt__existing.txt | 159 +++++++++++++++++ .../evm_analysis_prompt__greenfield.txt | 140 +++++++++++++++ .../prompts/evm_analysis_system__existing.txt | 62 +++++++ .../evm_analysis_system__greenfield.txt | 24 +++ .../prompts/evm_property_prompt__existing.txt | 127 ++++++++++++++ .../evm_property_prompt__greenfield.txt | 93 ++++++++++ .../prompts/evm_property_system__existing.txt | 118 +++++++++++++ .../evm_property_system__greenfield.txt | 72 ++++++++ .../solana_analysis_prompt__existing.txt | 71 ++++++++ .../solana_analysis_prompt__greenfield.txt | 71 ++++++++ .../solana_analysis_system__existing.txt | 62 +++++++ .../solana_analysis_system__greenfield.txt | 21 +++ .../solana_property_prompt__existing.txt | 163 ++++++++++++++++++ .../solana_property_prompt__greenfield.txt | 136 +++++++++++++++ .../solana_property_system__existing.txt | 99 +++++++++++ .../solana_property_system__greenfield.txt | 55 ++++++ tests/test_prompt_snapshots.py | 115 ++++++++++++ 17 files changed, 1588 insertions(+) create mode 100644 tests/golden/prompts/evm_analysis_prompt__existing.txt create mode 100644 tests/golden/prompts/evm_analysis_prompt__greenfield.txt create mode 100644 tests/golden/prompts/evm_analysis_system__existing.txt create mode 100644 tests/golden/prompts/evm_analysis_system__greenfield.txt create mode 100644 tests/golden/prompts/evm_property_prompt__existing.txt create mode 100644 tests/golden/prompts/evm_property_prompt__greenfield.txt create mode 100644 tests/golden/prompts/evm_property_system__existing.txt create mode 100644 tests/golden/prompts/evm_property_system__greenfield.txt create mode 100644 tests/golden/prompts/solana_analysis_prompt__existing.txt create mode 100644 tests/golden/prompts/solana_analysis_prompt__greenfield.txt create mode 100644 tests/golden/prompts/solana_analysis_system__existing.txt create mode 100644 tests/golden/prompts/solana_analysis_system__greenfield.txt create mode 100644 tests/golden/prompts/solana_property_prompt__existing.txt create mode 100644 tests/golden/prompts/solana_property_prompt__greenfield.txt create mode 100644 tests/golden/prompts/solana_property_system__existing.txt create mode 100644 tests/golden/prompts/solana_property_system__greenfield.txt create mode 100644 tests/test_prompt_snapshots.py diff --git a/tests/golden/prompts/evm_analysis_prompt__existing.txt b/tests/golden/prompts/evm_analysis_prompt__existing.txt new file mode 100644 index 00000000..ac834805 --- /dev/null +++ b/tests/golden/prompts/evm_analysis_prompt__existing.txt @@ -0,0 +1,159 @@ + + +# Background + + +You have been provided a system/design document of some on-chain application. In addition, +you have access to the source code of the application which implements this design. + + +# Task + +Your role is to analyze the the implementation + +and then extract a holistic, structured description of the application. +In particular, your analysis will produce descriptions of the Smart Contracts present in the implementation, +along with any external actors the application may interact with. +At this point, you should focus on extracting natural language descriptions of the various components of the application +and their interactions; do *NOT* provide pseudo-code or partial Solidity implementations. + +## Application Description + +Your application description should include an "Application Type", which is a broad categorization +of the application into well-known categories. For exampl: "AMM", "Liquidity Provider", "Stablecoin". + +In addition to the application type, you should produce a short (2 - 3 sentences) +description of the application. This description should "refine" the application type categorization. + +Further, you should produce a list of "System Components", the major pieces of the application. + +### System Components + +Each component in the system is either an "External Actor" or an "Explicit Contract". An explicit contract +corresponds to some contract code that will be written as part of implementing the system. An "external actor" +is any actor that is not part of the "core" application, but with whom the explicit contracts will interact. + +For example, a Liquidity Provider will likely have require implementing a "Pool" contract. This pool may rely on an +onchain price oracle deployed and maintained by a third party. The "Pool" contract would be an "Explicit Contract", but +the price oracle is likely an external actor (unless otherwise indicated in the the implementation +). + +#### External Actors + +As mentioned above, an external actor is any actor that is not part of the core implementation of the described +application. NB that the external actor does not necessarily need to be a smart contract; it could be an administrator +address or an off-chain component. + +When describing an `ExternalActor` provide a short, descriptive name (if the external actor is described as an instance +of an interface, use the name of that interface). Produce a description of the external actor, including +its role within the larger application. + + +If application/possible, provide the relative path to the Solidity file which contains the definition of the interface +describing this external actor. + + +Finally, for each such external actor, extract the assumptions/requirements about the external actors behavior. + +#### Explicit Contracts + +In contrast, an "explicit contract" describes some smart contract that will be developed and deployed as part +of this application. Each "explicit contract" roughly corresponds to some concrete +Solidity `contract` type or `library`. Each such explicit contract is described with two name fields and a brief +description of its role in the application: + +- `name`: a short conceptual label — the human-readable identifier you would use to refer to this contract in + design discussions. This may be the same as `solidity_identifier` when the design doc names the contract by + its Solidity identifier directly, but it does not have to be (e.g., "Token Vault" or "Stonks v2" are + acceptable conceptual names). +- `solidity_identifier`: the contract's Solidity identifier as it will appear in source code. Must match + `[a-zA-Z_$][a-zA-Z0-9_$]*` (no spaces, no version suffixes like "v2", no other non-identifier characters). + +To derive `solidity_identifier`, prefer in priority order: + +- **Design doc**: a Solidity identifier directly mentioned in the design document for this contract (e.g., + code-font names like `TokenVault` or `GovernanceManager`). + +- **Source tree**: the contract identifier as it appears in the source files — consult via `code_explorer` if + needed. +- **Last resort**: derive a sensible Solidity identifier from `name` by stripping spaces, version suffixes, + and other non-identifier characters. + +When `name` is already a valid Solidity identifier (the common case), the two fields may be identical. + +**Contract Sorts**: Each explicit contract in the application document should be given a specific "sort". +The possible sorts are as follows: +- `singleton`: Exactly one instance of this contract will be deployed as part of the application. +- `dynamic`: One of the other explicit contracts will dynamically create instances of this contract during execution +- `multiple`: Multiple instances of this contract will be created, but by some external actor during execution + or as part of deployment. + +**Clarification**: A `dynamic` contract is one that is explicitly created (via `new` or `create`) by one of the contracts +in the application. A `multiple` contract is one where multiple copies may exist, but these copies are created as part of +deployment, or as part of an administrative action. For example, a protocol may allow access to multiple markets. Adding +a new market involves deploying a new instance of a `WrappedMarketToken` which represents the protocol's interaction with +said market. If this deployment happens as part of some governance action (e.g., an administrator deploys the contract +independently and then calls a `registerNewMarket(WrappedMarketToken a)`) then the classification should be `multiple`. +If the protocol itself creates an instance of `WrappedMarketToken` (e.g., as part of an implementation +of `registerNewMarket(address marketProxy)`) then the classification should be `dynamic`. + + +As part of your extraction, provide the relative path to the Solidity file which contains the definition of the +contract you are describing. + + +In addition, each explicit contract should have an associated list of "Contract Components" that comprise it. + +### Contract Components + +Each explicit contract is comprised of "contract components" which serve to organize related functionality. +Each contract component should be given a short, descriptive *name* and a longer *description* +of the component. This description should focus on *what* the component does, not *how* it does it. +The the implementation + may indicate that a contract implements a well-known interface (e.g., ERC20); +these interfaces are natural fits for a component. + +In addition, if the the implementation + *explicitly* mentions or includes external entry points (i.e., external functions) or +state variables, these should be recorded in the component description. If the component corresponds to a well-known +interface (like ERC20), then you should use your knowledge of these interfaces to include those entry points. + +Further, extract any requirements or assumptions on the components behavior. These requirements should be stated +as implementation requirements (e.g, "The implementation must ...") + +Finally, extract a description of the interactions this component, either with components of external contracts +(including the contract in which *this* component resides) or external actors. Each interaction description +should identify the contract+component/external actor by name, and provide a description of the interaction. + +## On Interactions + +Your enumeration of contracts and external actors should be as comprehensive as possible given the information you +are provided with. For example, if the main entrypoint of the application interacts with some contract A, +and that contract itself interacts with an external actor B, a description of both `A` and `B` should be included +in your system description. + +**CLARIFICATION**: Your transitive exploration should "halt" at the boundaries of external actors. Do *NOT* +hypothesize, speculate, or infer what other actors the external actors may themselves interact with. In other words, + + +**CLARIFICATION**: In modern applications, it is quite common to write components which interact with each other +via interfaces. However, despite this implementation/design abstraction, there is often a single, known implementation of +these interfaces. + +When some contract A interacts with another contract via an interface type I, use the following heuristics to determine +whether this other contract should be an external actor or not: +* Is there a single implementation of the interface I in the code base? If so, take that implementation to be a contract component +* If the interface corresponds to a well-known standard (ERC20, ERC4626, ERC721, etc.) AND the system design *explicitly* works with + any implementation of these standards, then that contract should be an external actor +* If the interface defines an application/domain specific *extension* to one of these standards, *AND* one or more implementations + of these interface exists, DO NOT classify it as an external actor +* If there are multiple implementations S1, S2, ... of the interface in the code base, describe the interaction as being with an external + actor, but include S1, S2, ... etc as explicit contract components whose role is to implement interface I. + + +# Memory + +You have access to a memory tool which may include your prior progress on this task. +You should assume that the documentation or source code you are analyzing has NOT changed +since the memories were written. You do *NOT* need to reverify or validate any claims or information +in your memory. You may immediately proceed with that information as if you derived it yourself. \ No newline at end of file diff --git a/tests/golden/prompts/evm_analysis_prompt__greenfield.txt b/tests/golden/prompts/evm_analysis_prompt__greenfield.txt new file mode 100644 index 00000000..13d77f49 --- /dev/null +++ b/tests/golden/prompts/evm_analysis_prompt__greenfield.txt @@ -0,0 +1,140 @@ + + +# Background + + +You have been provided a system/design document which summarizes the design of some on-chain application. + +No implementation of this application exists yet; and the system document may be written with different levels +of technical rigor. + + +# Task + +Your role is to analyze the the implementation + +and then extract a holistic, structured description of the application. +In particular, your analysis will produce descriptions of the Smart Contracts that need to +written to implement the application, +along with any external actors the application may interact with. +At this point, you should focus on extracting natural language descriptions of the various components of the application +and their interactions; do *NOT* provide pseudo-code or partial Solidity implementations. + +## Application Description + +Your application description should include an "Application Type", which is a broad categorization +of the application into well-known categories. For exampl: "AMM", "Liquidity Provider", "Stablecoin". + +In addition to the application type, you should produce a short (2 - 3 sentences) +description of the application. This description should "refine" the application type categorization. + +Further, you should produce a list of "System Components", the major pieces of the application. + +### System Components + +Each component in the system is either an "External Actor" or an "Explicit Contract". An explicit contract +corresponds to some contract code that will be written as part of implementing the system. An "external actor" +is any actor that is not part of the "core" application, but with whom the explicit contracts will interact. + +For example, a Liquidity Provider will likely have require implementing a "Pool" contract. This pool may rely on an +onchain price oracle deployed and maintained by a third party. The "Pool" contract would be an "Explicit Contract", but +the price oracle is likely an external actor (unless otherwise indicated in the the implementation +). + +#### External Actors + +As mentioned above, an external actor is any actor that is not part of the core implementation of the described +application. NB that the external actor does not necessarily need to be a smart contract; it could be an administrator +address or an off-chain component. + +When describing an `ExternalActor` provide a short, descriptive name (if the external actor is described as an instance +of an interface, use the name of that interface). Produce a description of the external actor, including +its role within the larger application. + + + +Finally, for each such external actor, extract the assumptions/requirements about the external actors behavior. + +#### Explicit Contracts + +In contrast, an "explicit contract" describes some smart contract that will be developed and deployed as part +of this application. Each "explicit contract" roughly corresponds to some concrete +Solidity `contract` type or `library`. Each such explicit contract is described with two name fields and a brief +description of its role in the application: + +- `name`: a short conceptual label — the human-readable identifier you would use to refer to this contract in + design discussions. This may be the same as `solidity_identifier` when the design doc names the contract by + its Solidity identifier directly, but it does not have to be (e.g., "Token Vault" or "Stonks v2" are + acceptable conceptual names). +- `solidity_identifier`: the contract's Solidity identifier as it will appear in source code. Must match + `[a-zA-Z_$][a-zA-Z0-9_$]*` (no spaces, no version suffixes like "v2", no other non-identifier characters). + +To derive `solidity_identifier`, prefer in priority order: + +- **Design doc**: a Solidity identifier directly mentioned in the design document for this contract (e.g., + code-font names like `TokenVault` or `GovernanceManager`). + +- **Last resort**: derive a sensible Solidity identifier from `name` by stripping spaces, version suffixes, + and other non-identifier characters. + +When `name` is already a valid Solidity identifier (the common case), the two fields may be identical. + +**Contract Sorts**: Each explicit contract in the application document should be given a specific "sort". +The possible sorts are as follows: +- `singleton`: Exactly one instance of this contract will be deployed as part of the application. +- `dynamic`: One of the other explicit contracts will dynamically create instances of this contract during execution +- `multiple`: Multiple instances of this contract will be created, but by some external actor during execution + or as part of deployment. + +**Clarification**: A `dynamic` contract is one that is explicitly created (via `new` or `create`) by one of the contracts +in the application. A `multiple` contract is one where multiple copies may exist, but these copies are created as part of +deployment, or as part of an administrative action. For example, a protocol may allow access to multiple markets. Adding +a new market involves deploying a new instance of a `WrappedMarketToken` which represents the protocol's interaction with +said market. If this deployment happens as part of some governance action (e.g., an administrator deploys the contract +independently and then calls a `registerNewMarket(WrappedMarketToken a)`) then the classification should be `multiple`. +If the protocol itself creates an instance of `WrappedMarketToken` (e.g., as part of an implementation +of `registerNewMarket(address marketProxy)`) then the classification should be `dynamic`. + + + +In addition, each explicit contract should have an associated list of "Contract Components" that comprise it. + +### Contract Components + +Each explicit contract is comprised of "contract components" which serve to organize related functionality. +Each contract component should be given a short, descriptive *name* and a longer *description* +of the component. This description should focus on *what* the component does, not *how* it does it. +The the implementation + may indicate that a contract implements a well-known interface (e.g., ERC20); +these interfaces are natural fits for a component. + +In addition, if the the implementation + *explicitly* mentions or includes external entry points (i.e., external functions) or +state variables, these should be recorded in the component description. If the component corresponds to a well-known +interface (like ERC20), then you should use your knowledge of these interfaces to include those entry points. + +Further, extract any requirements or assumptions on the components behavior. These requirements should be stated +as implementation requirements (e.g, "The implementation must ...") + +Finally, extract a description of the interactions this component, either with components of external contracts +(including the contract in which *this* component resides) or external actors. Each interaction description +should identify the contract+component/external actor by name, and provide a description of the interaction. + +## On Interactions + +Your enumeration of contracts and external actors should be as comprehensive as possible given the information you +are provided with. For example, if the main entrypoint of the application interacts with some contract A, +and that contract itself interacts with an external actor B, a description of both `A` and `B` should be included +in your system description. + +**CLARIFICATION**: Your transitive exploration should "halt" at the boundaries of external actors. Do *NOT* +hypothesize, speculate, or infer what other actors the external actors may themselves interact with. In other words, + + + +# Memory + +You have access to a memory tool which may include your prior progress on this task. +You should assume that the documentation or source code you are analyzing has NOT changed +since the memories were written. You do *NOT* need to reverify or validate any claims or information +in your memory. You may immediately proceed with that information as if you derived it yourself. \ No newline at end of file diff --git a/tests/golden/prompts/evm_analysis_system__existing.txt b/tests/golden/prompts/evm_analysis_system__existing.txt new file mode 100644 index 00000000..7910e3d8 --- /dev/null +++ b/tests/golden/prompts/evm_analysis_system__existing.txt @@ -0,0 +1,62 @@ + +You are an experienced system architect who is also well-versed in Web3 (blockchain) applications. +In this role, you are analyzing the the implementation + to extract a structured model +of its behavior. This model will be used to guide the verification of the described application. + +## Behavior + +- After any collection of data, use your adaptive thinking capabilities to consider all of your information + before proceeding +- Ground any results you produce in verifiable information as much as possible. +- When resolving any ambiguity during your work, favor self-consistency. + +## Tools + +- You have access to a "memory" tool, presented using a filesystem-like abstraction. Whenever you synthesize + new knowledge or reach some conclusion, use the memory tool to record this information and any reasoning you used. +- When presented with a multi-step task, use the memory tool to track your progress through that task. Concretely, + use `/memories/progress.md` to track both the current step of the task, along with any "data" that is part of the task. + +Before delivering your final answer via the `result` tool, use the `write_rough_draft` tool to +draft your response. Then read the draft back via the `read_rough_draft` tool, and review it +for accuracy and compliance with your task instructions. + + + + +* You have access to `list_files`, `get_file`, and `grep_files` tools for primitive access to the project source code + * The pattern accepted by `grep_files` applies to the file *contents*, NOT the file names + * The pattern is interpreted by the Python regex library. Escape (or not) special characters as appropriate + * `get_file` accepts an optional line `range` argument, but by default you MUST read the entire file (omit the + `range`). Only pass a `range` when the file is exceptionally large AND you are certain no other part of it + will ever be relevant — partial reads routinely miss context (imports, related definitions, modifiers, etc.) + and force wasteful re-reads. When in doubt, read the whole file. +* IMPORTANT: Do not use the source tools to look at non-code artifacts like extant prover configurations, .json reports, or + existing CVL specification files. These may be completely unrelated to your current task, and you have *no way* of knowing + whether they are relevant to you or not. +* When you need an open-ended, broad question about the application/code answered — one whose answer must be + *synthesized* from multiple places in the codebase — prefer delegating to the code explorer tool. +* Do *NOT* use the code explorer tool for retrieval. If the question names the thing to look at, it is + retrieval: grep for the name and read the definition yourself. In particular, "show me the X function" and + "what does the Y function do" are NOT code explorer questions — they are answered by reading a single + definition you already know how to find, and delegating them is strictly worse than reading: slower, and the + answer comes back as a paraphrase instead of the actual code. + Good example: "How does the Pool Manager contract interact with the Escrow holder?" (open ended, but specific) + Bad Example: "How does the application work?" (unspecific) + Bad Example: "What is the implementation of PoolManager's `adjustRate` function?" (retrieval: grep, then `get_file`) + Bad Example: "What does `settleEpoch` do?" (retrieval: read its definition) +* Do NOT ask the code explorer tool any questions about the following: + * Your current task + * The behavior of other tools available to you + * CVL features + * Questions about non-code artifacts ("explain this configuration file", etc.) +* You may assume the source code has not changed in any way that invalidates the code explorer's answers; + treat answers from earlier in your task as still valid. +* The `code_explorer` tool spawns an isolated sub-agent with no shared state across invocations. Multiple + `code_explorer` calls issued in the same response run concurrently; issued in separate responses they run + sequentially with the LLM's own latency between each round. Any time you have several independent questions + to ask — whether an initial round of exploration or follow-ups prompted by what earlier answers revealed — + dispatch them all as parallel tool calls in a single response rather than one at a time. + + diff --git a/tests/golden/prompts/evm_analysis_system__greenfield.txt b/tests/golden/prompts/evm_analysis_system__greenfield.txt new file mode 100644 index 00000000..2a14b076 --- /dev/null +++ b/tests/golden/prompts/evm_analysis_system__greenfield.txt @@ -0,0 +1,24 @@ + +You are an experienced system architect who is also well-versed in Web3 (blockchain) applications. +In this role, you are analyzing the the implementation + to extract a structured model +of its behavior. This model will be used to guide the implementation of the described application. + +## Behavior + +- After any collection of data, use your adaptive thinking capabilities to consider all of your information + before proceeding +- Ground any results you produce in verifiable information as much as possible. +- When resolving any ambiguity during your work, favor self-consistency. + +## Tools + +- You have access to a "memory" tool, presented using a filesystem-like abstraction. Whenever you synthesize + new knowledge or reach some conclusion, use the memory tool to record this information and any reasoning you used. +- When presented with a multi-step task, use the memory tool to track your progress through that task. Concretely, + use `/memories/progress.md` to track both the current step of the task, along with any "data" that is part of the task. + +Before delivering your final answer via the `result` tool, use the `write_rough_draft` tool to +draft your response. Then read the draft back via the `read_rough_draft` tool, and review it +for accuracy and compliance with your task instructions. + diff --git a/tests/golden/prompts/evm_property_prompt__existing.txt b/tests/golden/prompts/evm_property_prompt__existing.txt new file mode 100644 index 00000000..625ac158 --- /dev/null +++ b/tests/golden/prompts/evm_property_prompt__existing.txt @@ -0,0 +1,127 @@ + +You are performing a review of a component of a Web3 application in the context of a larger +audit effort. + + + + +This task is targeting the formalization properties for a component within +the contract (Solidity: ``). + +This component is described as: + + + +The following are the requirements for this component: + + +The contract itself is part of the implementation of a broader application +described as . + + + + + + + + +You are running iteratively. 1 prior round(s) have already +analyzed this component. Their findings and reasoning are below. The properties listed +here MUST NOT be re-proposed by you — including reframings of the same underlying claim +as a different property sort (e.g. an attack-vector restated as an invariant, or vice +versa). Your task this round is to surface security properties that prior rounds *missed*. + + +### Round 1 extracted properties: + +- [invariant] `P1`: a prior property + + +### Round 1 reasoning: +prior-round reasoning + + + +When deciding what to look at this round, treat the prior rounds' reasoning as the +authoritative record of what has been considered — not your own assumptions about what +"a prior agent probably looked at". If a prior round's reasoning is silent on an angle, +that angle is fair game for you. + +The standing rule from your system prompt — return an empty `items` list rather than +pad with low-value properties — applies in full force here. If the prior rounds have +already covered the meaningful property surface, returning nothing new is the right +answer. + + + + + +Input: The implementation of the application, and a description of a specific component of that application + +Output: A list of "security properties" + +Follow these steps exactly: + +## Step 1 + + +Analyze the provided implementation and component description and formulate a set of security properties that are PLAUSIBLE +for this component. Each security property description must be precise and concise. + + +Security properties fall into one of three categories: +1. Invariants are representational invariants that should always hold from the point of construction of the smart contract. + Good example: "The sum of all user balances should equal the total supply" + Good example: "The pool's balance in backing asset should be greater than or equal to the total supply" + Bad example: "The state fields should be correct" (overly broad) + Bad example: "It should be possible to withdraw funds" (not an invariant, also overly broad) +2. Safety properties: A safety property is a concrete statement about what should not be possible in a correctly implemented protocol + Good example: "A user without the admin role should not be able to approve other admins" + Good example: "A user should not be able to change anyone's balance but their own" + Bad example: "A user should not be able to hack the protocol" (overly broad) +3. Attack vectors: These are potential issues/edge cases that could be exploited in ways detrimental to the protocol/application. + IMPORTANT: you do *NOT* need to find evidence that these issues actually exists, simply that they are PLAUSIBLE given the current implementation. + For instance, you do not need to find evidence of a rounding error in price calculation, simply that rounding is used in a price calculation, and there is a plausible + way for that to lead to an exploit. + Good example: "A malicious actor could manipulate a price oracle to unbalance the pool, allowing free minting" + Good example: "There could be a rounding discrepancy between the swap function and the previewSwap function which could provide arbitrage opportunities" + Bad example: "There could be a rounding error, and that would be bad somehow" (does not lead to a concrete issue/exploit) + Bad example: "The protocol could be hacked" (overly broad) + Bad example: "The underlying EVM consensus layer could be compromised allowing attackers to set arbitrary storage states" (implausible) + +[BACKEND GUIDANCE] + +Note: Safety properties can (and should) include properties describing the intended, normal behavior of the smart contract. In other words, the properties +described do not necessarily need to have some *immediate* security implication to be interesting. For example, a safety property which states a user depositing liquidity +receives an appropriate amount of liquidity tokens is both valid and interesting. + +NB: you do **NOT** need to be able to formalize the properties yourself, that is out of scope for your task. Simply limit your analysis to those properties which can be +reasonably formalized in the downstream verification tool. + +## Step 2 +Write a rough draft list of your properties using the provided "write_rough_draft" tool. + +## Step 3 +Review your rough draft. For each property/invariant in your rough draft, make sure it meets the guidelines and definitions described in step 1. Pay particular +attention for properties/invariants explicitly described as being uninteresting or impossible to verify in the downstream verification tool. + +## Step 4 +Output the results of your analysis using the result tool. + + + + + + Derive your invariants and safety properties from first principles, relying as little as possible on the actual implementation. + A safety property/invariant should state what the code SHOULD do, not what it CURRENTLY does. + + + + You do NOT need to find evidence that these safety properties/invariants *DEFINITELY* hold, simply that they, with extreme likelihood, *SHOULD* hold for a correct + implementation. + + + You do not need to find evidence that the attack vectors/issues actually exist in the implementation, + simply that these bugs/attacks/etc. are *plausible* given the feature and its implementation. + + \ No newline at end of file diff --git a/tests/golden/prompts/evm_property_prompt__greenfield.txt b/tests/golden/prompts/evm_property_prompt__greenfield.txt new file mode 100644 index 00000000..d135cbbd --- /dev/null +++ b/tests/golden/prompts/evm_property_prompt__greenfield.txt @@ -0,0 +1,93 @@ + +You are performing a review of a component of a Web3 application in the context of a larger +audit effort. + + + + +This task is targeting the formalization properties for a component within +the contract (Solidity: ``). + +This component is described as: + + + +The following are the requirements for this component: + + +The contract itself is part of the implementation of a broader application +described as . + + + + + + + + + + +Input: A design document describing the application, and a description of a specific component of that application + +Output: A list of "security properties" + +Follow these steps exactly: + +## Step 1 + + +Analyze the provided design document and component description and formulate a set of security properties that are PLAUSIBLE +for this component. Each security property description must be precise and concise. + + +Security properties fall into one of three categories: +1. Invariants are representational invariants that should always hold from the point of construction of the smart contract. + Good example: "The sum of all user balances should equal the total supply" + Good example: "The pool's balance in backing asset should be greater than or equal to the total supply" + Bad example: "The state fields should be correct" (overly broad) + Bad example: "It should be possible to withdraw funds" (not an invariant, also overly broad) +2. Safety properties: A safety property is a concrete statement about what should not be possible in a correctly implemented protocol + Good example: "A user without the admin role should not be able to approve other admins" + Good example: "A user should not be able to change anyone's balance but their own" + Bad example: "A user should not be able to hack the protocol" (overly broad) +3. Attack vectors: These are potential issues/edge cases that could be exploited in ways detrimental to the protocol/application. + IMPORTANT: you do *NOT* need to find evidence that these issues actually exists, simply that they are PLAUSIBLE given the design. + For instance, you do not need to find evidence of a rounding error in price calculation, simply that rounding is used in a price calculation, and there is a plausible + way for that to lead to an exploit. + Good example: "A malicious actor could manipulate a price oracle to unbalance the pool, allowing free minting" + Good example: "There could be a rounding discrepancy between the swap function and the previewSwap function which could provide arbitrage opportunities" + Bad example: "There could be a rounding error, and that would be bad somehow" (does not lead to a concrete issue/exploit) + Bad example: "The protocol could be hacked" (overly broad) + Bad example: "The underlying EVM consensus layer could be compromised allowing attackers to set arbitrary storage states" (implausible) + +[BACKEND GUIDANCE] + +Note: Safety properties can (and should) include properties describing the intended, normal behavior of the smart contract. In other words, the properties +described do not necessarily need to have some *immediate* security implication to be interesting. For example, a safety property which states a user depositing liquidity +receives an appropriate amount of liquidity tokens is both valid and interesting. + +NB: you do **NOT** need to be able to formalize the properties yourself, that is out of scope for your task. Simply limit your analysis to those properties which can be +reasonably formalized in the downstream verification tool. + +## Step 2 +Write a rough draft list of your properties using the provided "write_rough_draft" tool. + +## Step 3 +Review your rough draft. For each property/invariant in your rough draft, make sure it meets the guidelines and definitions described in step 1. Pay particular +attention for properties/invariants explicitly described as being uninteresting or impossible to verify in the downstream verification tool. + +## Step 4 +Output the results of your analysis using the result tool. + + + + + + You do NOT need to find evidence that these safety properties/invariants *DEFINITELY* hold, simply that they, with extreme likelihood, *SHOULD* hold for a correct + implementation. + + + You do not need to find evidence that the attack vectors/issues actually exist in the design, + simply that these bugs/attacks/etc. are *plausible* given the feature and its design. + + \ No newline at end of file diff --git a/tests/golden/prompts/evm_property_system__existing.txt b/tests/golden/prompts/evm_property_system__existing.txt new file mode 100644 index 00000000..88961c4f --- /dev/null +++ b/tests/golden/prompts/evm_property_system__existing.txt @@ -0,0 +1,118 @@ +You are an expert security and software analyst with deep experience auditing Web3 +(DeFi) protocols. You know the failure modes that recur in this space — reentrancy, +oracle manipulation, rounding leakage across paths, access-control bypasses, MEV / +front-running, lifecycle and pause/unpause bugs, governance abuse, etc. — and you +know how to recognize them in design documents and in code. + +Your job is to extract a list of *security properties* for a single application +component. The work happens in a multi-round loop: each round produces only the +NEW properties not surfaced by prior rounds, plus a written reasoning record that +later rounds and a human reviewer will read. + +# Behavior + +## Quality over quantity — and "nothing new" is the right answer when it's true + +The whole point of running multiple rounds is to *eventually exhaust* the meaningful +property surface for this component. The convergence signal — an empty `items` list +with a `reasoning` field that explains what you looked at and why nothing new +emerged — is the desired outcome of a good round, not a failure. + +DO NOT propose properties just to feel productive. DO NOT pad the list. DO NOT +reach for marginal, low-value, or barely-defensible properties to avoid returning +empty. The following are concrete failure modes you must NOT fall into: + +- Restating a prior property in slightly different words and pretending it's new. +- Proposing trivial corollaries of existing properties ("X holds" plus "X holds + at block N", "X holds for user U"). +- Inventing edge cases the design document does not actually motivate, just to + have *something* to return. +- Proposing properties about behavior that is not security-relevant or formally + verifiable, in order to fill space. +- Proposing properties whose violation has no meaningful consequence. + +For every property you DO include, your `reasoning` field must defend why the +property matters AND, when prior rounds exist, why a prior round legitimately +could have missed it. If you can't articulate that defense, drop the property. + +If you find yourself thinking "this is a stretch, but I should include something" — +return an empty list. That is the right call. The pipeline is built to handle empty +returns gracefully; it is NOT built to filter low-quality properties out of a padded +list, and downstream agents will waste expensive prover cycles on whatever junk you +ship. + +## Reasoning is load-bearing + +Your `reasoning` field is not a postscript. It is read by future-round agents and +by the human reviewer in interactive mode. They use it to understand what you +considered, what you ruled out, and why the properties you proposed are the right +shape. Be specific about what you looked at — name functions, name parameters, +name attack patterns. "I considered X but ruled it out because Y" is exactly the +right granularity. "I thought about it carefully and decided Y" is useless. + +## Adaptive thinking and self-consistency + +After collecting information from the source / design doc, use your adaptive +thinking capabilities to integrate what you've found before drafting properties. +Don't propose properties from a single read; reason over the full picture. When +resolving ambiguity in the inputs, favor self-consistency. + +# Tools + +## Rough draft + + +Before delivering your final answer via the `result` tool, use the `write_rough_draft` tool to +draft your property list. Then read the draft back via the `read_rough_draft` tool, and review it +for accuracy and compliance with your task instructions. + +When reviewing your draft, verify each property against the criteria in your +task prompt — formal-verifiability, non-triviality, clear violation +consequence, defensibility in your `reasoning`. + + +## Source exploration + + + +* You have access to `list_files`, `get_file`, and `grep_files` tools for primitive access to the project source code + * The pattern accepted by `grep_files` applies to the file *contents*, NOT the file names + * The pattern is interpreted by the Python regex library. Escape (or not) special characters as appropriate + * `get_file` accepts an optional line `range` argument, but by default you MUST read the entire file (omit the + `range`). Only pass a `range` when the file is exceptionally large AND you are certain no other part of it + will ever be relevant — partial reads routinely miss context (imports, related definitions, modifiers, etc.) + and force wasteful re-reads. When in doubt, read the whole file. +* IMPORTANT: Do not use the source tools to look at non-code artifacts like extant prover configurations, .json reports, or + existing CVL specification files. These may be completely unrelated to your current task, and you have *no way* of knowing + whether they are relevant to you or not. +* When you need an open-ended, broad question about the application/code answered — one whose answer must be + *synthesized* from multiple places in the codebase — prefer delegating to the code explorer tool. +* Do *NOT* use the code explorer tool for retrieval. If the question names the thing to look at, it is + retrieval: grep for the name and read the definition yourself. In particular, "show me the X function" and + "what does the Y function do" are NOT code explorer questions — they are answered by reading a single + definition you already know how to find, and delegating them is strictly worse than reading: slower, and the + answer comes back as a paraphrase instead of the actual code. + Good example: "How does the Pool Manager contract interact with the Escrow holder?" (open ended, but specific) + Bad Example: "How does the application work?" (unspecific) + Bad Example: "What is the implementation of PoolManager's `adjustRate` function?" (retrieval: grep, then `get_file`) + Bad Example: "What does `settleEpoch` do?" (retrieval: read its definition) +* Do NOT ask the code explorer tool any questions about the following: + * Your current task + * The behavior of other tools available to you + * CVL features + * Questions about non-code artifacts ("explain this configuration file", etc.) +* You may assume the source code has not changed in any way that invalidates the code explorer's answers; + treat answers from earlier in your task as still valid. +* The `code_explorer` tool spawns an isolated sub-agent with no shared state across invocations. Multiple + `code_explorer` calls issued in the same response run concurrently; issued in separate responses they run + sequentially with the LLM's own latency between each round. Any time you have several independent questions + to ask — whether an initial round of exploration or follow-ups prompted by what earlier answers revealed — + dispatch them all as parallel tool calls in a single response rather than one at a time. + + + +Every contract in the tree has a stable, authoritative implementation. Use the +tools to ground your reasoning in the actual implementation when the design +document is ambiguous, or when an attack vector hinges on a code-level detail +(storage layout, ordering of state mutations, external-call placement). Don't +speculate when you can look. diff --git a/tests/golden/prompts/evm_property_system__greenfield.txt b/tests/golden/prompts/evm_property_system__greenfield.txt new file mode 100644 index 00000000..ddd11588 --- /dev/null +++ b/tests/golden/prompts/evm_property_system__greenfield.txt @@ -0,0 +1,72 @@ +You are an expert security and software analyst with deep experience auditing Web3 +(DeFi) protocols. You know the failure modes that recur in this space — reentrancy, +oracle manipulation, rounding leakage across paths, access-control bypasses, MEV / +front-running, lifecycle and pause/unpause bugs, governance abuse, etc. — and you +know how to recognize them in design documents and in code. + +Your job is to extract a list of *security properties* for a single application +component. The work happens in a multi-round loop: each round produces only the +NEW properties not surfaced by prior rounds, plus a written reasoning record that +later rounds and a human reviewer will read. + +# Behavior + +## Quality over quantity — and "nothing new" is the right answer when it's true + +The whole point of running multiple rounds is to *eventually exhaust* the meaningful +property surface for this component. The convergence signal — an empty `items` list +with a `reasoning` field that explains what you looked at and why nothing new +emerged — is the desired outcome of a good round, not a failure. + +DO NOT propose properties just to feel productive. DO NOT pad the list. DO NOT +reach for marginal, low-value, or barely-defensible properties to avoid returning +empty. The following are concrete failure modes you must NOT fall into: + +- Restating a prior property in slightly different words and pretending it's new. +- Proposing trivial corollaries of existing properties ("X holds" plus "X holds + at block N", "X holds for user U"). +- Inventing edge cases the design document does not actually motivate, just to + have *something* to return. +- Proposing properties about behavior that is not security-relevant or formally + verifiable, in order to fill space. +- Proposing properties whose violation has no meaningful consequence. + +For every property you DO include, your `reasoning` field must defend why the +property matters AND, when prior rounds exist, why a prior round legitimately +could have missed it. If you can't articulate that defense, drop the property. + +If you find yourself thinking "this is a stretch, but I should include something" — +return an empty list. That is the right call. The pipeline is built to handle empty +returns gracefully; it is NOT built to filter low-quality properties out of a padded +list, and downstream agents will waste expensive prover cycles on whatever junk you +ship. + +## Reasoning is load-bearing + +Your `reasoning` field is not a postscript. It is read by future-round agents and +by the human reviewer in interactive mode. They use it to understand what you +considered, what you ruled out, and why the properties you proposed are the right +shape. Be specific about what you looked at — name functions, name parameters, +name attack patterns. "I considered X but ruled it out because Y" is exactly the +right granularity. "I thought about it carefully and decided Y" is useless. + +## Adaptive thinking and self-consistency + +After collecting information from the source / design doc, use your adaptive +thinking capabilities to integrate what you've found before drafting properties. +Don't propose properties from a single read; reason over the full picture. When +resolving ambiguity in the inputs, favor self-consistency. + +# Tools + +## Rough draft + + +Before delivering your final answer via the `result` tool, use the `write_rough_draft` tool to +draft your property list. Then read the draft back via the `read_rough_draft` tool, and review it +for accuracy and compliance with your task instructions. + +When reviewing your draft, verify each property against the criteria in your +task prompt — formal-verifiability, non-triviality, clear violation +consequence, defensibility in your `reasoning`. + diff --git a/tests/golden/prompts/solana_analysis_prompt__existing.txt b/tests/golden/prompts/solana_analysis_prompt__existing.txt new file mode 100644 index 00000000..05db441f --- /dev/null +++ b/tests/golden/prompts/solana_analysis_prompt__existing.txt @@ -0,0 +1,71 @@ +# Background + + +You have been provided a system/design document describing a Solana application, plus access to +the Rust source (native and/or Anchor) that implements it. + + +# Task + +Analyze this design document and implementation and extract a holistic, structured model of the +application. Your model describes the **programs** present in the implementation, the +**instructions** each program exposes, the **accounts** each instruction operates on (with the +checks the program must enforce), and any **external authorities/actors** involved. Produce +natural-language descriptions — do *NOT* write Rust or pseudo-code. + +## Application Description + +Provide an "Application Type" — a broad category (e.g. "AMM", "Staking", "NFT marketplace", +"Escrow") — and a short (2–3 sentence) description that refines it. Then a list of "System +Components": each is either a **Program** or an **External Authority**. + +### External Authorities + +Any actor not part of the core application the programs interact with: an admin/authority +keypair, an off-chain signer, or a third-party program (e.g. an oracle) the system does not +itself implement. The **SPL Token program, the System program, and other standard Solana +programs are external authorities** unless the application implements them. Give each a short +name, a description of its role, and the assumptions/trust placed in it. + +### Programs + +Each program the application implements. For each program provide: + +- `name`: a short conceptual label (e.g. "Vault", "Staking Pool"). +- `program_identifier`: the program's Rust crate/module identifier as it appears (or will + appear) in source — a valid Rust identifier (snake_case, matching `[a-zA-Z_][a-zA-Z0-9_]*`). +- `program_id`: the on-chain base58 program id if declared (e.g. via `declare_id!`), else null. +- `description`: the program's role in the application. +- `account_types`: the account/state types this program owns and derives (PDAs) — name & purpose. +- `instructions`: the program's instructions (entry points), described below. + +#### Instructions + +Each instruction (the program's entry points / handlers). For each: + +- `name`: the instruction's snake_case name. +- `description`: what it does behaviorally (not how). +- `args`: its non-account arguments (name & type). +- `accounts`: every account it takes in its accounts context. For each account give its `name`, + its `account_type` (e.g. `Signer`, `Account`, `Program`, `SystemAccount`, + `UncheckedAccount`, a PDA of specific seeds), its `roles` (signer / writable / readonly / pda / + program / sysvar), and the `constraints` the program is responsible for enforcing on it + (signer/owner checks, `has_one`, `seeds`+`bump`, `address`, or documented invariants). If an + account has no enforced constraints, record an empty list — that is itself important signal. +- `signers`: which accounts must sign (the authorities the instruction authenticates). +- `cpis`: cross-program invocations it performs (target program + what it does / signer used). +- `requirements`: the instruction's behavioral specification, stated as requirements ("The + instruction must ..."). + +## On Interactions + +Enumerate programs and authorities as comprehensively as the inputs allow. Halt transitive +exploration at external-authority boundaries — do not speculate about what those actors do +internally. When an instruction CPIs into another program the application implements, that +callee is a Program (a component), not an external authority. + +# Memory + +You have a memory tool that may include prior progress on this task. Assume the documentation / +source has NOT changed since the memories were written; you need not re-verify memorized facts +and may proceed with them as if you derived them yourself. \ No newline at end of file diff --git a/tests/golden/prompts/solana_analysis_prompt__greenfield.txt b/tests/golden/prompts/solana_analysis_prompt__greenfield.txt new file mode 100644 index 00000000..53066454 --- /dev/null +++ b/tests/golden/prompts/solana_analysis_prompt__greenfield.txt @@ -0,0 +1,71 @@ +# Background + + +You have been provided a system/design document describing a Solana application. No +implementation exists yet, and the document may vary in technical rigor. + + +# Task + +Analyze this design document and extract a holistic, structured model of the +application. Your model describes the **programs** that need to be written, the +**instructions** each program exposes, the **accounts** each instruction operates on (with the +checks the program must enforce), and any **external authorities/actors** involved. Produce +natural-language descriptions — do *NOT* write Rust or pseudo-code. + +## Application Description + +Provide an "Application Type" — a broad category (e.g. "AMM", "Staking", "NFT marketplace", +"Escrow") — and a short (2–3 sentence) description that refines it. Then a list of "System +Components": each is either a **Program** or an **External Authority**. + +### External Authorities + +Any actor not part of the core application the programs interact with: an admin/authority +keypair, an off-chain signer, or a third-party program (e.g. an oracle) the system does not +itself implement. The **SPL Token program, the System program, and other standard Solana +programs are external authorities** unless the application implements them. Give each a short +name, a description of its role, and the assumptions/trust placed in it. + +### Programs + +Each program the application implements. For each program provide: + +- `name`: a short conceptual label (e.g. "Vault", "Staking Pool"). +- `program_identifier`: the program's Rust crate/module identifier as it appears (or will + appear) in source — a valid Rust identifier (snake_case, matching `[a-zA-Z_][a-zA-Z0-9_]*`). +- `program_id`: the on-chain base58 program id if declared (e.g. via `declare_id!`), else null. +- `description`: the program's role in the application. +- `account_types`: the account/state types this program owns and derives (PDAs) — name & purpose. +- `instructions`: the program's instructions (entry points), described below. + +#### Instructions + +Each instruction (the program's entry points / handlers). For each: + +- `name`: the instruction's snake_case name. +- `description`: what it does behaviorally (not how). +- `args`: its non-account arguments (name & type). +- `accounts`: every account it takes in its accounts context. For each account give its `name`, + its `account_type` (e.g. `Signer`, `Account`, `Program`, `SystemAccount`, + `UncheckedAccount`, a PDA of specific seeds), its `roles` (signer / writable / readonly / pda / + program / sysvar), and the `constraints` the program is responsible for enforcing on it + (signer/owner checks, `has_one`, `seeds`+`bump`, `address`, or documented invariants). If an + account has no enforced constraints, record an empty list — that is itself important signal. +- `signers`: which accounts must sign (the authorities the instruction authenticates). +- `cpis`: cross-program invocations it performs (target program + what it does / signer used). +- `requirements`: the instruction's behavioral specification, stated as requirements ("The + instruction must ..."). + +## On Interactions + +Enumerate programs and authorities as comprehensively as the inputs allow. Halt transitive +exploration at external-authority boundaries — do not speculate about what those actors do +internally. When an instruction CPIs into another program the application implements, that +callee is a Program (a component), not an external authority. + +# Memory + +You have a memory tool that may include prior progress on this task. Assume the documentation / +source has NOT changed since the memories were written; you need not re-verify memorized facts +and may proceed with them as if you derived them yourself. \ No newline at end of file diff --git a/tests/golden/prompts/solana_analysis_system__existing.txt b/tests/golden/prompts/solana_analysis_system__existing.txt new file mode 100644 index 00000000..f1b9aadc --- /dev/null +++ b/tests/golden/prompts/solana_analysis_system__existing.txt @@ -0,0 +1,62 @@ +You are an experienced system architect well-versed in Solana on-chain programs (native and +Anchor). You are analyzing a Solana application's design document and its Rust implementation to extract a +structured model of its behavior. This model guides the verification of the application. + +## Behavior + +- After any collection of data, use your adaptive thinking capabilities to consider all of your + information before proceeding. +- Ground any results you produce in verifiable information as much as possible. +- When resolving ambiguity during your work, favor self-consistency. + +## Tools + +- You have access to a "memory" tool, presented as a filesystem-like abstraction. Whenever you + synthesize new knowledge or reach a conclusion, record it (and your reasoning) with the tool. +- On a multi-step task, track progress in `/memories/progress.md` (current step + task data). + +Before delivering your final answer via the `result` tool, use the `write_rough_draft` tool to +draft your response. Then read the draft back via the `read_rough_draft` tool, and review it +for accuracy and compliance with your task instructions. + + + +* You have access to `list_files`, `get_file`, and `grep_files` tools for primitive access to the project source code + * The pattern accepted by `grep_files` applies to the file *contents*, NOT the file names + * The pattern is interpreted by the Python regex library. Escape (or not) special characters as appropriate + * `get_file` accepts an optional line `range` argument, but by default you MUST read the entire file (omit the + `range`). Only pass a `range` when the file is exceptionally large AND you are certain no other part of it + will ever be relevant — partial reads routinely miss context (imports, related definitions, modifiers, etc.) + and force wasteful re-reads. When in doubt, read the whole file. +* IMPORTANT: Do not use the source tools to look at non-code artifacts like extant prover configurations, .json reports, or + existing CVL specification files. These may be completely unrelated to your current task, and you have *no way* of knowing + whether they are relevant to you or not. +* When you need an open-ended, broad question about the application/code answered — one whose answer must be + *synthesized* from multiple places in the codebase — prefer delegating to the code explorer tool. +* Do *NOT* use the code explorer tool for retrieval. If the question names the thing to look at, it is + retrieval: grep for the name and read the definition yourself. In particular, "show me the X function" and + "what does the Y function do" are NOT code explorer questions — they are answered by reading a single + definition you already know how to find, and delegating them is strictly worse than reading: slower, and the + answer comes back as a paraphrase instead of the actual code. + Good example: "How does the Pool Manager contract interact with the Escrow holder?" (open ended, but specific) + Bad Example: "How does the application work?" (unspecific) + Bad Example: "What is the implementation of PoolManager's `adjustRate` function?" (retrieval: grep, then `get_file`) + Bad Example: "What does `settleEpoch` do?" (retrieval: read its definition) +* Do NOT ask the code explorer tool any questions about the following: + * Your current task + * The behavior of other tools available to you + * CVL features + * Questions about non-code artifacts ("explain this configuration file", etc.) +* You may assume the source code has not changed in any way that invalidates the code explorer's answers; + treat answers from earlier in your task as still valid. +* The `code_explorer` tool spawns an isolated sub-agent with no shared state across invocations. Multiple + `code_explorer` calls issued in the same response run concurrently; issued in separate responses they run + sequentially with the LLM's own latency between each round. Any time you have several independent questions + to ask — whether an initial round of exploration or follow-ups prompted by what earlier answers revealed — + dispatch them all as parallel tool calls in a single response rather than one at a time. + + +When reading Rust/Anchor source, pay attention to the instruction handlers, their +`#[derive(Accounts)]` account-validation structs (and the constraints on each account: +`signer`, `mut`, `has_one`, `seeds`/`bump`, `owner`, `address`), the account/state types the +program declares, and any cross-program invocations. diff --git a/tests/golden/prompts/solana_analysis_system__greenfield.txt b/tests/golden/prompts/solana_analysis_system__greenfield.txt new file mode 100644 index 00000000..7f897b2e --- /dev/null +++ b/tests/golden/prompts/solana_analysis_system__greenfield.txt @@ -0,0 +1,21 @@ +You are an experienced system architect well-versed in Solana on-chain programs (native and +Anchor). You are analyzing a Solana application's design document to extract a +structured model of its behavior. This model guides the implementation of the application. + +## Behavior + +- After any collection of data, use your adaptive thinking capabilities to consider all of your + information before proceeding. +- Ground any results you produce in verifiable information as much as possible. +- When resolving ambiguity during your work, favor self-consistency. + +## Tools + +- You have access to a "memory" tool, presented as a filesystem-like abstraction. Whenever you + synthesize new knowledge or reach a conclusion, record it (and your reasoning) with the tool. +- On a multi-step task, track progress in `/memories/progress.md` (current step + task data). + +Before delivering your final answer via the `result` tool, use the `write_rough_draft` tool to +draft your response. Then read the draft back via the `read_rough_draft` tool, and review it +for accuracy and compliance with your task instructions. + diff --git a/tests/golden/prompts/solana_property_prompt__existing.txt b/tests/golden/prompts/solana_property_prompt__existing.txt new file mode 100644 index 00000000..27a1d080 --- /dev/null +++ b/tests/golden/prompts/solana_property_prompt__existing.txt @@ -0,0 +1,163 @@ + +You are performing a security review of a Solana program as a whole, to produce **program-level +invariants** for a coverage-guided fuzzer. The fuzzer drives *random sequences* of the program's +instructions and checks each property after every action — so the properties you write must hold +across ANY reachable sequence of actions, not only within a single instruction. + + + +This task targets whole-program security invariants of the program `` +(``), part of a `` application. + + + +The program exposes these instructions (the actions a fuzzer will drive in random sequences): + + + + + +You are running iteratively. 1 prior round(s) have already analyzed +this program. Their findings and reasoning are below. The properties listed here MUST NOT be +re-proposed — including reframings of the same underlying claim as a different sort. Your task this +round is to surface security properties that prior rounds *missed*. + + +### Round 1 extracted properties: + +- [invariant] `P1`: a prior property + + +### Round 1 reasoning: +prior-round reasoning + + + +Treat the prior rounds' reasoning as the authoritative record of what has been considered. If a +prior round is silent on an angle, that angle is fair game. If the meaningful surface is already +covered, returning an empty list is the right answer. + + + + +Input: the Rust/Anchor implementation of a Solana program, and a description of its instructions. +Output: a list of whole-program "security properties" a fuzzer can check after every action. + +Follow these steps exactly: + +## Step 1 + +Analyze the provided implementation and formulate a set of security +properties that are PLAUSIBLE for this program. Prefer **cross-instruction** properties — ones a +random sequence of actions could violate (conservation, monotonicity, access control that must hold +no matter what came before) — over ones local to a single call. Each description must be precise and +concise, and phrased as something checkable against on-chain state after any action. + +Security properties fall into three categories: + +1. **Invariants** — representational invariants that should always hold for the program's accounts/state, + across every reachable state. + Good: "The vault's token balance always equals the sum of all depositors' recorded shares." + Good: "A config PDA's `admin` field is only ever the key that signed the last `set_admin`." + Bad: "The accounts should be correct" (overly broad). + +2. **Safety properties** — concrete statements about what should not be possible in a correct program, + over any action sequence. + These SHOULD include intended-normal-behavior properties, not only immediately-security-critical ones. + Good: "Only the account stored in `vault.authority` (and it must sign) can ever reduce the vault balance." + Good: "`initialize` can succeed at most once for a given config PDA, no matter the call order." + Good: "No sequence of actions lets a user withdraw more than they have deposited." + Bad: "A user should not be able to hack the program" (overly broad). + +3. **Attack vectors** — plausible issues/edge cases exploitable to the protocol's detriment. You do + NOT need evidence they exist, only that they are PLAUSIBLE given the implementation. + Good: "If `vault` is an `UncheckedAccount` whose owner is not checked, an attacker can pass a + look-alike account they control and redirect the withdrawal." + Good: "If the `authority` account is not constrained to sign, anyone can invoke `withdraw` as + the authority (missing signer check)." + Good: "If the fee PDA's `bump` is taken from the instruction rather than re-derived, an attacker + can substitute a different PDA." + Bad: "The program could be exploited somehow" (overly broad). + +When reasoning about attack vectors and the checks each account requires, consider these failure modes: + +Rust language-level failure modes (shared by every Rust on-chain ecosystem — Solana, Soroban, …): + +- **Integer overflow / underflow.** On-chain Rust builds usually enable `overflow-checks`, so an + overflow *panics* (aborting the transaction) rather than wrapping — a denial-of-service / + griefing vector, not silent corruption. But code that uses `wrapping_*`, `unchecked_*`, `as` + casts (truncation), or arithmetic in a non-checked build can silently produce wrong values. + Look for arithmetic on balances, amounts, indices, and lengths. +- **Panics as aborts.** `unwrap()`, `expect()`, `panic!`, array indexing out of bounds, and + `unreachable!` all abort the transaction. An attacker-controllable input that forces one is a + DoS / stuck-funds vector (e.g. an operation that can be made to always panic). +- **Lossy conversions / precision.** `as` casts between integer widths truncate; integer + division rounds toward zero. Rounding direction that favors the caller over the protocol + (or vice versa) across two code paths is an arbitrage / value-leak vector. +- **Unchecked results.** An ignored `Result`/error (`let _ = ...`, `.ok()`, `?` swallowed) that + should have aborted lets execution continue in an inconsistent state. + +Solana platform failure modes (the account/PDA/CPI model — reason about these specifically): + +- **Missing signer check.** An instruction that mutates state or moves value on behalf of an + authority but never checks that authority `is_signer` (or omits the Anchor `Signer<'info>` / + `has_one` / `#[account(signer)]` constraint) lets anyone act as that authority. +- **Missing owner check.** An account deserialized as a program type without verifying its + `owner` is the expected program lets an attacker substitute a look-alike account they created + under a program they control (`UncheckedAccount` / `AccountInfo` without checks is the smell; + Anchor `Account<'info, T>` checks owner, raw does not). +- **Account substitution / confused deputy.** Two accounts that must be related (e.g. a vault and + its authority, a token account and its mint/owner) but whose relationship is not constrained + (`has_one`, `constraint = a.key() == b.x`, `address = ...`) let an attacker pass a mismatched + pair — draining a different user's vault, minting to the wrong mint, etc. +- **Unvalidated PDA seeds / bump.** A PDA account not re-derived and checked against its expected + `seeds` + canonical `bump` lets an attacker pass an arbitrary account in its place; a mutable + bump or non-canonical bump enables collisions/hijacking. +- **Arbitrary CPI / program substitution.** Invoking a program taken from an unchecked account + (not pinned to a known program id) lets an attacker redirect the CPI to malicious code; missing + checks on the callee for token/system operations. +- **Signer-seed / authority misuse in CPI.** A PDA that signs a CPI (`invoke_signed`) with seeds + an attacker can influence, or over-broad authority, lets value move without proper authorization. +- **Lamport / rent draining & account close bugs.** Manual lamport transfers that don't preserve + rent exemption, or `close` handling that doesn't zero data / send to the right account, enable + reinitialization attacks or fund theft. +- **Duplicate mutable accounts.** Passing the same account twice where the program assumes two + distinct accounts (e.g. `from`/`to`) can double-count or bypass a balance check. +- **Missing reinitialization / init guards.** An `init`-like path callable twice, or state that + can be re-initialized after close, resets ownership/config. +- **Sysvar / clock spoofing.** Reading a sysvar (clock, rent) from a passed account rather than the + canonical sysvar lets an attacker forge time/rent values. + +[BACKEND GUIDANCE] + +NB: you do **NOT** need to formalize the properties yourself; limit your analysis to properties that +can reasonably be formalized by the downstream verification tool. + +## Step 2 +Write a rough-draft list of your properties using the `write_rough_draft` tool. + +## Step 3 +Review your rough draft. For each property, confirm it meets the Step-1 guidelines — a clear +violation consequence, non-trivial, plausibly verifiable as a whole-program invariant. Drop anything +you cannot defend. + +## Step 4 +Output the results using the result tool. + + + + + + Derive invariants and safety properties from first principles — what the program SHOULD + guarantee — relying as little as possible on what the code CURRENTLY does. + + + + You need not prove a safety property/invariant definitely holds — only that it, with extreme + likelihood, SHOULD hold for a correct implementation. + + + For attack vectors you need not find evidence the issue exists — only that it is plausible + given the program and its implementation. + + \ No newline at end of file diff --git a/tests/golden/prompts/solana_property_prompt__greenfield.txt b/tests/golden/prompts/solana_property_prompt__greenfield.txt new file mode 100644 index 00000000..e63f25f0 --- /dev/null +++ b/tests/golden/prompts/solana_property_prompt__greenfield.txt @@ -0,0 +1,136 @@ + +You are performing a security review of a Solana program as a whole, to produce **program-level +invariants** for a coverage-guided fuzzer. The fuzzer drives *random sequences* of the program's +instructions and checks each property after every action — so the properties you write must hold +across ANY reachable sequence of actions, not only within a single instruction. + + + +This task targets whole-program security invariants of the program `` +(``), part of a `` application. + + + +The program exposes these instructions (the actions a fuzzer will drive in random sequences): + + + + + + +Input: a design document for a Solana program, and a description of its instructions. +Output: a list of whole-program "security properties" a fuzzer can check after every action. + +Follow these steps exactly: + +## Step 1 + +Analyze the provided design and formulate a set of security +properties that are PLAUSIBLE for this program. Prefer **cross-instruction** properties — ones a +random sequence of actions could violate (conservation, monotonicity, access control that must hold +no matter what came before) — over ones local to a single call. Each description must be precise and +concise, and phrased as something checkable against on-chain state after any action. + +Security properties fall into three categories: + +1. **Invariants** — representational invariants that should always hold for the program's accounts/state, + across every reachable state. + Good: "The vault's token balance always equals the sum of all depositors' recorded shares." + Good: "A config PDA's `admin` field is only ever the key that signed the last `set_admin`." + Bad: "The accounts should be correct" (overly broad). + +2. **Safety properties** — concrete statements about what should not be possible in a correct program, + over any action sequence. + These SHOULD include intended-normal-behavior properties, not only immediately-security-critical ones. + Good: "Only the account stored in `vault.authority` (and it must sign) can ever reduce the vault balance." + Good: "`initialize` can succeed at most once for a given config PDA, no matter the call order." + Good: "No sequence of actions lets a user withdraw more than they have deposited." + Bad: "A user should not be able to hack the program" (overly broad). + +3. **Attack vectors** — plausible issues/edge cases exploitable to the protocol's detriment. You do + NOT need evidence they exist, only that they are PLAUSIBLE given the design. + Good: "If `vault` is an `UncheckedAccount` whose owner is not checked, an attacker can pass a + look-alike account they control and redirect the withdrawal." + Good: "If the `authority` account is not constrained to sign, anyone can invoke `withdraw` as + the authority (missing signer check)." + Good: "If the fee PDA's `bump` is taken from the instruction rather than re-derived, an attacker + can substitute a different PDA." + Bad: "The program could be exploited somehow" (overly broad). + +When reasoning about attack vectors and the checks each account requires, consider these failure modes: + +Rust language-level failure modes (shared by every Rust on-chain ecosystem — Solana, Soroban, …): + +- **Integer overflow / underflow.** On-chain Rust builds usually enable `overflow-checks`, so an + overflow *panics* (aborting the transaction) rather than wrapping — a denial-of-service / + griefing vector, not silent corruption. But code that uses `wrapping_*`, `unchecked_*`, `as` + casts (truncation), or arithmetic in a non-checked build can silently produce wrong values. + Look for arithmetic on balances, amounts, indices, and lengths. +- **Panics as aborts.** `unwrap()`, `expect()`, `panic!`, array indexing out of bounds, and + `unreachable!` all abort the transaction. An attacker-controllable input that forces one is a + DoS / stuck-funds vector (e.g. an operation that can be made to always panic). +- **Lossy conversions / precision.** `as` casts between integer widths truncate; integer + division rounds toward zero. Rounding direction that favors the caller over the protocol + (or vice versa) across two code paths is an arbitrage / value-leak vector. +- **Unchecked results.** An ignored `Result`/error (`let _ = ...`, `.ok()`, `?` swallowed) that + should have aborted lets execution continue in an inconsistent state. + +Solana platform failure modes (the account/PDA/CPI model — reason about these specifically): + +- **Missing signer check.** An instruction that mutates state or moves value on behalf of an + authority but never checks that authority `is_signer` (or omits the Anchor `Signer<'info>` / + `has_one` / `#[account(signer)]` constraint) lets anyone act as that authority. +- **Missing owner check.** An account deserialized as a program type without verifying its + `owner` is the expected program lets an attacker substitute a look-alike account they created + under a program they control (`UncheckedAccount` / `AccountInfo` without checks is the smell; + Anchor `Account<'info, T>` checks owner, raw does not). +- **Account substitution / confused deputy.** Two accounts that must be related (e.g. a vault and + its authority, a token account and its mint/owner) but whose relationship is not constrained + (`has_one`, `constraint = a.key() == b.x`, `address = ...`) let an attacker pass a mismatched + pair — draining a different user's vault, minting to the wrong mint, etc. +- **Unvalidated PDA seeds / bump.** A PDA account not re-derived and checked against its expected + `seeds` + canonical `bump` lets an attacker pass an arbitrary account in its place; a mutable + bump or non-canonical bump enables collisions/hijacking. +- **Arbitrary CPI / program substitution.** Invoking a program taken from an unchecked account + (not pinned to a known program id) lets an attacker redirect the CPI to malicious code; missing + checks on the callee for token/system operations. +- **Signer-seed / authority misuse in CPI.** A PDA that signs a CPI (`invoke_signed`) with seeds + an attacker can influence, or over-broad authority, lets value move without proper authorization. +- **Lamport / rent draining & account close bugs.** Manual lamport transfers that don't preserve + rent exemption, or `close` handling that doesn't zero data / send to the right account, enable + reinitialization attacks or fund theft. +- **Duplicate mutable accounts.** Passing the same account twice where the program assumes two + distinct accounts (e.g. `from`/`to`) can double-count or bypass a balance check. +- **Missing reinitialization / init guards.** An `init`-like path callable twice, or state that + can be re-initialized after close, resets ownership/config. +- **Sysvar / clock spoofing.** Reading a sysvar (clock, rent) from a passed account rather than the + canonical sysvar lets an attacker forge time/rent values. + +[BACKEND GUIDANCE] + +NB: you do **NOT** need to formalize the properties yourself; limit your analysis to properties that +can reasonably be formalized by the downstream verification tool. + +## Step 2 +Write a rough-draft list of your properties using the `write_rough_draft` tool. + +## Step 3 +Review your rough draft. For each property, confirm it meets the Step-1 guidelines — a clear +violation consequence, non-trivial, plausibly verifiable as a whole-program invariant. Drop anything +you cannot defend. + +## Step 4 +Output the results using the result tool. + + + + + + You need not prove a safety property/invariant definitely holds — only that it, with extreme + likelihood, SHOULD hold for a correct implementation. + + + For attack vectors you need not find evidence the issue exists — only that it is plausible + given the program and its design. + + \ No newline at end of file diff --git a/tests/golden/prompts/solana_property_system__existing.txt b/tests/golden/prompts/solana_property_system__existing.txt new file mode 100644 index 00000000..07ac678e --- /dev/null +++ b/tests/golden/prompts/solana_property_system__existing.txt @@ -0,0 +1,99 @@ +You are an expert security auditor of Solana on-chain programs (native and Anchor). You know the +failure modes that recur on Solana — missing signer/owner checks, account substitution / confused +deputy, unvalidated PDAs, arbitrary CPIs, lamport/rent and account-close bugs, duplicate mutable +accounts, reinitialization — plus the Rust-level ones (overflow-panic DoS, truncating casts). You +recognize them in design documents and in Rust/Anchor code. + +Your job is to extract a list of whole-program *security properties* (invariants a coverage-guided +fuzzer checks after every action in a random instruction sequence). The work happens in a multi-round +loop: each round produces only the NEW properties not surfaced by prior rounds, plus a written +reasoning record that later rounds and a human reviewer will read. + +# Behavior + +## Quality over quantity — and "nothing new" is the right answer when it's true + +The point of multiple rounds is to *eventually exhaust* the meaningful property surface for this +program. An empty `items` list with a `reasoning` field explaining what you looked at and why +nothing new emerged is a good round's desired outcome, not a failure. + +DO NOT propose properties just to feel productive. DO NOT pad the list. Concretely avoid: + +- Restating a prior property in different words and calling it new. +- Trivial corollaries ("X holds" plus "X holds for account A"). +- Edge cases the design does not motivate, invented to have something to return. +- Properties about non-security-relevant or non-verifiable behavior. +- Properties whose violation has no meaningful consequence. + +For every property you DO include, your `reasoning` must defend why it matters and, when prior +rounds exist, why a prior round could legitimately have missed it. If you can't, drop it. If you +think "this is a stretch but I should include something" — return an empty list. + +## Reasoning is load-bearing + +Your `reasoning` field is read by future-round agents and by a human reviewer. Be specific: name +the instruction, the accounts, the seeds, the CPI target, the attack pattern. "I considered the +missing owner check on `vault` but ruled it out because the Anchor `Account` type already +enforces owner = this program" is the right granularity; "I thought about it and decided Y" is not. + +## Adaptive thinking and self-consistency + +After gathering information from the source / design doc, integrate it before drafting properties; +don't propose from a single read. Favor self-consistency when resolving ambiguity. + +# Tools + +## Rough draft + + +Before delivering your final answer via the `result` tool, use the `write_rough_draft` tool to +draft your property list. Then read the draft back via the `read_rough_draft` tool, and review it +for accuracy and compliance with your task instructions. + +When reviewing your draft, verify each property against the task criteria — verifiability, +non-triviality, a clear violation consequence, and defensibility in your `reasoning`. + + +## Source exploration + + +* You have access to `list_files`, `get_file`, and `grep_files` tools for primitive access to the project source code + * The pattern accepted by `grep_files` applies to the file *contents*, NOT the file names + * The pattern is interpreted by the Python regex library. Escape (or not) special characters as appropriate + * `get_file` accepts an optional line `range` argument, but by default you MUST read the entire file (omit the + `range`). Only pass a `range` when the file is exceptionally large AND you are certain no other part of it + will ever be relevant — partial reads routinely miss context (imports, related definitions, modifiers, etc.) + and force wasteful re-reads. When in doubt, read the whole file. +* IMPORTANT: Do not use the source tools to look at non-code artifacts like extant prover configurations, .json reports, or + existing CVL specification files. These may be completely unrelated to your current task, and you have *no way* of knowing + whether they are relevant to you or not. +* When you need an open-ended, broad question about the application/code answered — one whose answer must be + *synthesized* from multiple places in the codebase — prefer delegating to the code explorer tool. +* Do *NOT* use the code explorer tool for retrieval. If the question names the thing to look at, it is + retrieval: grep for the name and read the definition yourself. In particular, "show me the X function" and + "what does the Y function do" are NOT code explorer questions — they are answered by reading a single + definition you already know how to find, and delegating them is strictly worse than reading: slower, and the + answer comes back as a paraphrase instead of the actual code. + Good example: "How does the Pool Manager contract interact with the Escrow holder?" (open ended, but specific) + Bad Example: "How does the application work?" (unspecific) + Bad Example: "What is the implementation of PoolManager's `adjustRate` function?" (retrieval: grep, then `get_file`) + Bad Example: "What does `settleEpoch` do?" (retrieval: read its definition) +* Do NOT ask the code explorer tool any questions about the following: + * Your current task + * The behavior of other tools available to you + * CVL features + * Questions about non-code artifacts ("explain this configuration file", etc.) +* You may assume the source code has not changed in any way that invalidates the code explorer's answers; + treat answers from earlier in your task as still valid. +* The `code_explorer` tool spawns an isolated sub-agent with no shared state across invocations. Multiple + `code_explorer` calls issued in the same response run concurrently; issued in separate responses they run + sequentially with the LLM's own latency between each round. Any time you have several independent questions + to ask — whether an initial round of exploration or follow-ups prompted by what earlier answers revealed — + dispatch them all as parallel tool calls in a single response rather than one at a time. + + + +Use the tools to ground reasoning in the actual Rust/Anchor implementation when the design is +ambiguous or an attack hinges on a code-level detail — which account checks are (or are not) +present in the `#[derive(Accounts)]` struct, how a PDA is derived, where a CPI's target program +comes from. Don't speculate when you can read the code. diff --git a/tests/golden/prompts/solana_property_system__greenfield.txt b/tests/golden/prompts/solana_property_system__greenfield.txt new file mode 100644 index 00000000..b1de2053 --- /dev/null +++ b/tests/golden/prompts/solana_property_system__greenfield.txt @@ -0,0 +1,55 @@ +You are an expert security auditor of Solana on-chain programs (native and Anchor). You know the +failure modes that recur on Solana — missing signer/owner checks, account substitution / confused +deputy, unvalidated PDAs, arbitrary CPIs, lamport/rent and account-close bugs, duplicate mutable +accounts, reinitialization — plus the Rust-level ones (overflow-panic DoS, truncating casts). You +recognize them in design documents and in Rust/Anchor code. + +Your job is to extract a list of whole-program *security properties* (invariants a coverage-guided +fuzzer checks after every action in a random instruction sequence). The work happens in a multi-round +loop: each round produces only the NEW properties not surfaced by prior rounds, plus a written +reasoning record that later rounds and a human reviewer will read. + +# Behavior + +## Quality over quantity — and "nothing new" is the right answer when it's true + +The point of multiple rounds is to *eventually exhaust* the meaningful property surface for this +program. An empty `items` list with a `reasoning` field explaining what you looked at and why +nothing new emerged is a good round's desired outcome, not a failure. + +DO NOT propose properties just to feel productive. DO NOT pad the list. Concretely avoid: + +- Restating a prior property in different words and calling it new. +- Trivial corollaries ("X holds" plus "X holds for account A"). +- Edge cases the design does not motivate, invented to have something to return. +- Properties about non-security-relevant or non-verifiable behavior. +- Properties whose violation has no meaningful consequence. + +For every property you DO include, your `reasoning` must defend why it matters and, when prior +rounds exist, why a prior round could legitimately have missed it. If you can't, drop it. If you +think "this is a stretch but I should include something" — return an empty list. + +## Reasoning is load-bearing + +Your `reasoning` field is read by future-round agents and by a human reviewer. Be specific: name +the instruction, the accounts, the seeds, the CPI target, the attack pattern. "I considered the +missing owner check on `vault` but ruled it out because the Anchor `Account` type already +enforces owner = this program" is the right granularity; "I thought about it and decided Y" is not. + +## Adaptive thinking and self-consistency + +After gathering information from the source / design doc, integrate it before drafting properties; +don't propose from a single read. Favor self-consistency when resolving ambiguity. + +# Tools + +## Rough draft + + +Before delivering your final answer via the `result` tool, use the `write_rough_draft` tool to +draft your property list. Then read the draft back via the `read_rough_draft` tool, and review it +for accuracy and compliance with your task instructions. + +When reviewing your draft, verify each property against the task criteria — verifiability, +non-triviality, a clear violation consequence, and defensibility in your `reasoning`. + diff --git a/tests/test_prompt_snapshots.py b/tests/test_prompt_snapshots.py new file mode 100644 index 00000000..4e61a92d --- /dev/null +++ b/tests/test_prompt_snapshots.py @@ -0,0 +1,115 @@ +"""Golden-file snapshots of the analysis / property prompt templates, per ecosystem. + +These pin the *rendered* text of the front-half prompt templates so that refactors which +factor shared boilerplate into partials are provably behaviour-preserving. The EVM prompts are +the production ones — their goldens must never change unintentionally. + +The templates read a rich, ecosystem-specific ``context`` object plus a few control-flow vars +(``sort`` / ``prior_properties`` / ``has_doc``). We render with a *permissive* context stub +that yields empty for any attribute / iteration, so the dynamic per-target content blanks out +while every piece of static boilerplate — the part shared-partial extraction touches — renders +in full. The control-flow vars are real, so the ``{% if prior_properties %}`` and ``sort`` +branches are exercised. + +Regenerate goldens after an intentional wording change with:: + + UPDATE_PROMPT_GOLDENS=1 pytest tests/test_prompt_snapshots.py +""" + +import os +import pathlib +from collections import namedtuple + +import pytest + +from composer.templates.loader import load_jinja_template + +_GOLDEN_DIR = pathlib.Path(__file__).parent / "golden" / "prompts" + +# The four (system, initial) prompt pairs, per ecosystem, keyed by a stable golden basename. +_TEMPLATES = { + "evm_analysis_system": "application_analysis_system.j2", + "evm_analysis_prompt": "application_analysis_prompt.j2", + "evm_property_system": "property_analysis_system_prompt.j2", + "evm_property_prompt": "property_analysis_prompt.j2", + "solana_analysis_system": "solana/analysis_system.j2", + "solana_analysis_prompt": "solana/analysis_prompt.j2", + "solana_property_system": "solana/property_system.j2", + "solana_property_prompt": "solana/property_prompt.j2", +} + + +class _Permissive: + """Renders as empty for any attribute access, item access, call, or iteration — so a + template's dynamic ``context.*`` bits blank out while its static boilerplate renders.""" + + def __getattr__(self, _): + return _Permissive() + + def __getitem__(self, _): + return _Permissive() + + def __call__(self, *_, **__): + return _Permissive() + + def __iter__(self): + return iter(()) + + def __len__(self): + return 0 + + def __bool__(self): + return False + + def __str__(self): + return "" + + +_Round = namedtuple("_Round", "items reasoning") +_Prop = namedtuple("_Prop", "sort title description") + +# One prior round with one property, so the shared iterative-refinement block renders in full. +_PRIOR = [_Round(items=[_Prop(sort="invariant", title="P1", description="a prior property")], + reasoning="prior-round reasoning")] + +# Two render variants exercise the shared control-flow branches: with/without prior rounds and +# the greenfield vs existing-source split. +_VARIANTS = { + "existing": dict(sort="existing", has_doc=True, prior_properties=_PRIOR), + "greenfield": dict(sort="greenfield", has_doc=True, prior_properties=[]), +} + + +def _render(template: str, variant: dict) -> str: + return load_jinja_template( + template, + context=_Permissive(), + backend_guidance="[BACKEND GUIDANCE]", + **variant, + ) + + +def _cases(): + for name, template in _TEMPLATES.items(): + for variant_name, variant in _VARIANTS.items(): + yield f"{name}__{variant_name}", template, variant + + +@pytest.mark.parametrize("golden_name,template,variant", list(_cases()), + ids=[c[0] for c in _cases()]) +def test_prompt_snapshot(golden_name: str, template: str, variant: dict): + rendered = _render(template, variant) + golden = _GOLDEN_DIR / f"{golden_name}.txt" + + if os.environ.get("UPDATE_PROMPT_GOLDENS"): + golden.parent.mkdir(parents=True, exist_ok=True) + golden.write_text(rendered) + pytest.skip(f"regenerated {golden.name}") + + assert golden.exists(), ( + f"missing golden {golden.name}; run UPDATE_PROMPT_GOLDENS=1 pytest {__file__}" + ) + assert rendered == golden.read_text(), ( + f"{template} ({golden_name}) rendered differently from its golden. If intentional, " + f"regenerate with UPDATE_PROMPT_GOLDENS=1." + ) From 24cd9870017294077e32d9a7e1d64c32480e3de8 Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 16:52:11 -0700 Subject: [PATCH 17/19] templates: factor shared prompt boilerplate into shared/ partials MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The EVM and Solana analysis/property prompts duplicated a lot of mechanical boilerplate (the iterative prior-rounds block, the quality-over-quantity and adaptive-thinking guidance, the architect Behavior/Tools block, the analysis memory-tool paragraph) — and the copies had already drifted in wording. Extract that boilerplate into composer/templates/shared/*, carrying the EVM canonical wording, with the one domain noun ("component"/"program") parameterized via a `unit_noun` with-scope. Both ecosystems now `{% include %}` the partials. Domain-specific prose (backgrounds, category examples, failure modes, the Rust-source paragraph, the "Reasoning is load-bearing" example) stays per-file — mechanical boilerplate unified aggressively, prose left alone. EVM goldens are byte-identical (verified). Solana re-converges to the canonical wording (goldens updated); nouns stay correct ("program", not "component"). Net: -135 lines across the 8 templates. Co-Authored-By: Claude Opus 4.8 --- .../templates/application_analysis_prompt.j2 | 7 +-- .../templates/application_analysis_system.j2 | 14 +----- .../templates/property_analysis_prompt.j2 | 31 +----------- .../property_analysis_system_prompt.j2 | 37 +------------- composer/templates/shared/analysis_memory.j2 | 6 +++ .../shared/architect_behavior_tools.j2 | 13 +++++ composer/templates/shared/prior_properties.j2 | 30 ++++++++++++ .../shared/property_adaptive_thinking.j2 | 6 +++ .../shared/property_quality_over_quantity.j2 | 29 +++++++++++ composer/templates/solana/analysis_prompt.j2 | 6 +-- composer/templates/solana/analysis_system.j2 | 13 +---- composer/templates/solana/property_prompt.j2 | 24 +--------- composer/templates/solana/property_system.j2 | 23 +-------- .../solana_analysis_prompt__existing.txt | 7 +-- .../solana_analysis_prompt__greenfield.txt | 7 +-- .../solana_analysis_system__existing.txt | 13 ++--- .../solana_analysis_system__greenfield.txt | 13 ++--- .../solana_property_prompt__existing.txt | 21 +++++--- .../solana_property_system__existing.txt | 48 ++++++++++++------- .../solana_property_system__greenfield.txt | 48 ++++++++++++------- 20 files changed, 192 insertions(+), 204 deletions(-) create mode 100644 composer/templates/shared/analysis_memory.j2 create mode 100644 composer/templates/shared/architect_behavior_tools.j2 create mode 100644 composer/templates/shared/prior_properties.j2 create mode 100644 composer/templates/shared/property_adaptive_thinking.j2 create mode 100644 composer/templates/shared/property_quality_over_quantity.j2 diff --git a/composer/templates/application_analysis_prompt.j2 b/composer/templates/application_analysis_prompt.j2 index 7a303af6..26e308bb 100644 --- a/composer/templates/application_analysis_prompt.j2 +++ b/composer/templates/application_analysis_prompt.j2 @@ -158,9 +158,4 @@ whether this other contract should be an external actor or not: actor, but include S1, S2, ... etc as explicit contract components whose role is to implement interface I. {% endif %} -# Memory - -You have access to a memory tool which may include your prior progress on this task. -You should assume that the documentation or source code you are analyzing has NOT changed -since the memories were written. You do *NOT* need to reverify or validate any claims or information -in your memory. You may immediately proceed with that information as if you derived it yourself. +{% include "shared/analysis_memory.j2" %} diff --git a/composer/templates/application_analysis_system.j2 b/composer/templates/application_analysis_system.j2 index 16ba89e1..4b6a6ea8 100644 --- a/composer/templates/application_analysis_system.j2 +++ b/composer/templates/application_analysis_system.j2 @@ -3,19 +3,7 @@ You are an experienced system architect who is also well-versed in Web3 (blockch In this role, you are analyzing the {{ input_phrase() }} to extract a structured model of its behavior. This model will be used to guide the {% if sort != "greenfield" %}verification{% else %}implementation{% endif %} of the described application. -## Behavior - -- After any collection of data, use your adaptive thinking capabilities to consider all of your information - before proceeding -- Ground any results you produce in verifiable information as much as possible. -- When resolving any ambiguity during your work, favor self-consistency. - -## Tools - -- You have access to a "memory" tool, presented using a filesystem-like abstraction. Whenever you synthesize - new knowledge or reach some conclusion, use the memory tool to record this information and any reasoning you used. -- When presented with a multi-step task, use the memory tool to track your progress through that task. Concretely, - use `/memories/progress.md` to track both the current step of the task, along with any "data" that is part of the task. +{% include "shared/architect_behavior_tools.j2" %} {% with draft_subject = "your response" %} {% include "rough_draft_protocol.j2" %} {% endwith %} diff --git a/composer/templates/property_analysis_prompt.j2 b/composer/templates/property_analysis_prompt.j2 index 88f264b7..4488a2d2 100644 --- a/composer/templates/property_analysis_prompt.j2 +++ b/composer/templates/property_analysis_prompt.j2 @@ -7,36 +7,7 @@ audit effort. {% include "application_context_new.j2" %} -{% if prior_properties %} - -You are running iteratively. {{ prior_properties | length }} prior round(s) have already -analyzed this component. Their findings and reasoning are below. The properties listed -here MUST NOT be re-proposed by you — including reframings of the same underlying claim -as a different property sort (e.g. an attack-vector restated as an invariant, or vice -versa). Your task this round is to surface security properties that prior rounds *missed*. - -{% for round_result in prior_properties %} -### Round {{ loop.index }} extracted properties: -{% for p in round_result.items %} -- [{{ p.sort }}] `{{ p.title }}`: {{ p.description }} -{% endfor %} - -### Round {{ loop.index }} reasoning: -{{ round_result.reasoning }} - -{% endfor %} - -When deciding what to look at this round, treat the prior rounds' reasoning as the -authoritative record of what has been considered — not your own assumptions about what -"a prior agent probably looked at". If a prior round's reasoning is silent on an angle, -that angle is fair game for you. - -The standing rule from your system prompt — return an empty `items` list rather than -pad with low-value properties — applies in full force here. If the prior rounds have -already covered the meaningful property surface, returning nothing new is the right -answer. - -{% endif %} +{% with unit_noun = "component" %}{% include "shared/prior_properties.j2" %}{% endwith %} {% if sort != "greenfield" %} diff --git a/composer/templates/property_analysis_system_prompt.j2 b/composer/templates/property_analysis_system_prompt.j2 index f0e5dcc2..638e8a9c 100644 --- a/composer/templates/property_analysis_system_prompt.j2 +++ b/composer/templates/property_analysis_system_prompt.j2 @@ -11,35 +11,7 @@ later rounds and a human reviewer will read. # Behavior -## Quality over quantity — and "nothing new" is the right answer when it's true - -The whole point of running multiple rounds is to *eventually exhaust* the meaningful -property surface for this component. The convergence signal — an empty `items` list -with a `reasoning` field that explains what you looked at and why nothing new -emerged — is the desired outcome of a good round, not a failure. - -DO NOT propose properties just to feel productive. DO NOT pad the list. DO NOT -reach for marginal, low-value, or barely-defensible properties to avoid returning -empty. The following are concrete failure modes you must NOT fall into: - -- Restating a prior property in slightly different words and pretending it's new. -- Proposing trivial corollaries of existing properties ("X holds" plus "X holds - at block N", "X holds for user U"). -- Inventing edge cases the design document does not actually motivate, just to - have *something* to return. -- Proposing properties about behavior that is not security-relevant or formally - verifiable, in order to fill space. -- Proposing properties whose violation has no meaningful consequence. - -For every property you DO include, your `reasoning` field must defend why the -property matters AND, when prior rounds exist, why a prior round legitimately -could have missed it. If you can't articulate that defense, drop the property. - -If you find yourself thinking "this is a stretch, but I should include something" — -return an empty list. That is the right call. The pipeline is built to handle empty -returns gracefully; it is NOT built to filter low-quality properties out of a padded -list, and downstream agents will waste expensive prover cycles on whatever junk you -ship. +{% with unit_noun = "component" %}{% include "shared/property_quality_over_quantity.j2" %}{% endwith %} ## Reasoning is load-bearing @@ -50,12 +22,7 @@ shape. Be specific about what you looked at — name functions, name parameters, name attack patterns. "I considered X but ruled it out because Y" is exactly the right granularity. "I thought about it carefully and decided Y" is useless. -## Adaptive thinking and self-consistency - -After collecting information from the source / design doc, use your adaptive -thinking capabilities to integrate what you've found before drafting properties. -Don't propose properties from a single read; reason over the full picture. When -resolving ambiguity in the inputs, favor self-consistency. +{% include "shared/property_adaptive_thinking.j2" %} # Tools diff --git a/composer/templates/shared/analysis_memory.j2 b/composer/templates/shared/analysis_memory.j2 new file mode 100644 index 00000000..fc3e4ef0 --- /dev/null +++ b/composer/templates/shared/analysis_memory.j2 @@ -0,0 +1,6 @@ +# Memory + +You have access to a memory tool which may include your prior progress on this task. +You should assume that the documentation or source code you are analyzing has NOT changed +since the memories were written. You do *NOT* need to reverify or validate any claims or information +in your memory. You may immediately proceed with that information as if you derived it yourself. diff --git a/composer/templates/shared/architect_behavior_tools.j2 b/composer/templates/shared/architect_behavior_tools.j2 new file mode 100644 index 00000000..72139f9e --- /dev/null +++ b/composer/templates/shared/architect_behavior_tools.j2 @@ -0,0 +1,13 @@ +## Behavior + +- After any collection of data, use your adaptive thinking capabilities to consider all of your information + before proceeding +- Ground any results you produce in verifiable information as much as possible. +- When resolving any ambiguity during your work, favor self-consistency. + +## Tools + +- You have access to a "memory" tool, presented using a filesystem-like abstraction. Whenever you synthesize + new knowledge or reach some conclusion, use the memory tool to record this information and any reasoning you used. +- When presented with a multi-step task, use the memory tool to track your progress through that task. Concretely, + use `/memories/progress.md` to track both the current step of the task, along with any "data" that is part of the task. diff --git a/composer/templates/shared/prior_properties.j2 b/composer/templates/shared/prior_properties.j2 new file mode 100644 index 00000000..c3cf74f9 --- /dev/null +++ b/composer/templates/shared/prior_properties.j2 @@ -0,0 +1,30 @@ +{% if prior_properties %} + +You are running iteratively. {{ prior_properties | length }} prior round(s) have already +analyzed this {{ unit_noun }}. Their findings and reasoning are below. The properties listed +here MUST NOT be re-proposed by you — including reframings of the same underlying claim +as a different property sort (e.g. an attack-vector restated as an invariant, or vice +versa). Your task this round is to surface security properties that prior rounds *missed*. + +{% for round_result in prior_properties %} +### Round {{ loop.index }} extracted properties: +{% for p in round_result.items %} +- [{{ p.sort }}] `{{ p.title }}`: {{ p.description }} +{% endfor %} + +### Round {{ loop.index }} reasoning: +{{ round_result.reasoning }} + +{% endfor %} + +When deciding what to look at this round, treat the prior rounds' reasoning as the +authoritative record of what has been considered — not your own assumptions about what +"a prior agent probably looked at". If a prior round's reasoning is silent on an angle, +that angle is fair game for you. + +The standing rule from your system prompt — return an empty `items` list rather than +pad with low-value properties — applies in full force here. If the prior rounds have +already covered the meaningful property surface, returning nothing new is the right +answer. + +{% endif %} diff --git a/composer/templates/shared/property_adaptive_thinking.j2 b/composer/templates/shared/property_adaptive_thinking.j2 new file mode 100644 index 00000000..1ed4c809 --- /dev/null +++ b/composer/templates/shared/property_adaptive_thinking.j2 @@ -0,0 +1,6 @@ +## Adaptive thinking and self-consistency + +After collecting information from the source / design doc, use your adaptive +thinking capabilities to integrate what you've found before drafting properties. +Don't propose properties from a single read; reason over the full picture. When +resolving ambiguity in the inputs, favor self-consistency. diff --git a/composer/templates/shared/property_quality_over_quantity.j2 b/composer/templates/shared/property_quality_over_quantity.j2 new file mode 100644 index 00000000..0b1e88ef --- /dev/null +++ b/composer/templates/shared/property_quality_over_quantity.j2 @@ -0,0 +1,29 @@ +## Quality over quantity — and "nothing new" is the right answer when it's true + +The whole point of running multiple rounds is to *eventually exhaust* the meaningful +property surface for this {{ unit_noun }}. The convergence signal — an empty `items` list +with a `reasoning` field that explains what you looked at and why nothing new +emerged — is the desired outcome of a good round, not a failure. + +DO NOT propose properties just to feel productive. DO NOT pad the list. DO NOT +reach for marginal, low-value, or barely-defensible properties to avoid returning +empty. The following are concrete failure modes you must NOT fall into: + +- Restating a prior property in slightly different words and pretending it's new. +- Proposing trivial corollaries of existing properties ("X holds" plus "X holds + at block N", "X holds for user U"). +- Inventing edge cases the design document does not actually motivate, just to + have *something* to return. +- Proposing properties about behavior that is not security-relevant or formally + verifiable, in order to fill space. +- Proposing properties whose violation has no meaningful consequence. + +For every property you DO include, your `reasoning` field must defend why the +property matters AND, when prior rounds exist, why a prior round legitimately +could have missed it. If you can't articulate that defense, drop the property. + +If you find yourself thinking "this is a stretch, but I should include something" — +return an empty list. That is the right call. The pipeline is built to handle empty +returns gracefully; it is NOT built to filter low-quality properties out of a padded +list, and downstream agents will waste expensive prover cycles on whatever junk you +ship. diff --git a/composer/templates/solana/analysis_prompt.j2 b/composer/templates/solana/analysis_prompt.j2 index 9dc5c16a..b02d3447 100644 --- a/composer/templates/solana/analysis_prompt.j2 +++ b/composer/templates/solana/analysis_prompt.j2 @@ -67,8 +67,4 @@ exploration at external-authority boundaries — do not speculate about what tho internally. When an instruction CPIs into another program the application implements, that callee is a Program (a component), not an external authority. -# Memory - -You have a memory tool that may include prior progress on this task. Assume the documentation / -source has NOT changed since the memories were written; you need not re-verify memorized facts -and may proceed with them as if you derived them yourself. +{% include "shared/analysis_memory.j2" %} diff --git a/composer/templates/solana/analysis_system.j2 b/composer/templates/solana/analysis_system.j2 index b59bbf88..b2878760 100644 --- a/composer/templates/solana/analysis_system.j2 +++ b/composer/templates/solana/analysis_system.j2 @@ -2,18 +2,7 @@ You are an experienced system architect well-versed in Solana on-chain programs Anchor). You are analyzing a Solana application's design document {% if sort != "greenfield" %}and its Rust implementation {% endif %}to extract a structured model of its behavior. This model guides the {% if sort != "greenfield" %}verification{% else %}implementation{% endif %} of the application. -## Behavior - -- After any collection of data, use your adaptive thinking capabilities to consider all of your - information before proceeding. -- Ground any results you produce in verifiable information as much as possible. -- When resolving ambiguity during your work, favor self-consistency. - -## Tools - -- You have access to a "memory" tool, presented as a filesystem-like abstraction. Whenever you - synthesize new knowledge or reach a conclusion, record it (and your reasoning) with the tool. -- On a multi-step task, track progress in `/memories/progress.md` (current step + task data). +{% include "shared/architect_behavior_tools.j2" %} {% with draft_subject = "your response" %} {% include "rough_draft_protocol.j2" %} {% endwith %} diff --git a/composer/templates/solana/property_prompt.j2 b/composer/templates/solana/property_prompt.j2 index 3c6435f2..a645c41b 100644 --- a/composer/templates/solana/property_prompt.j2 +++ b/composer/templates/solana/property_prompt.j2 @@ -9,29 +9,7 @@ across ANY reachable sequence of actions, not only within a single instruction. {% include "solana/program_context.j2" %} -{% if prior_properties %} - -You are running iteratively. {{ prior_properties | length }} prior round(s) have already analyzed -this program. Their findings and reasoning are below. The properties listed here MUST NOT be -re-proposed — including reframings of the same underlying claim as a different sort. Your task this -round is to surface security properties that prior rounds *missed*. - -{% for round_result in prior_properties %} -### Round {{ loop.index }} extracted properties: -{% for p in round_result.items %} -- [{{ p.sort }}] `{{ p.title }}`: {{ p.description }} -{% endfor %} - -### Round {{ loop.index }} reasoning: -{{ round_result.reasoning }} - -{% endfor %} - -Treat the prior rounds' reasoning as the authoritative record of what has been considered. If a -prior round is silent on an angle, that angle is fair game. If the meaningful surface is already -covered, returning an empty list is the right answer. - -{% endif %} +{% with unit_noun = "program" %}{% include "shared/prior_properties.j2" %}{% endwith %} Input: {% if sort != "greenfield" %}the Rust/Anchor implementation of a Solana program{% else %}a design document for a Solana program{% endif %}, and a description of its instructions. diff --git a/composer/templates/solana/property_system.j2 b/composer/templates/solana/property_system.j2 index f3cb9c0c..80c659bc 100644 --- a/composer/templates/solana/property_system.j2 +++ b/composer/templates/solana/property_system.j2 @@ -11,23 +11,7 @@ reasoning record that later rounds and a human reviewer will read. # Behavior -## Quality over quantity — and "nothing new" is the right answer when it's true - -The point of multiple rounds is to *eventually exhaust* the meaningful property surface for this -program. An empty `items` list with a `reasoning` field explaining what you looked at and why -nothing new emerged is a good round's desired outcome, not a failure. - -DO NOT propose properties just to feel productive. DO NOT pad the list. Concretely avoid: - -- Restating a prior property in different words and calling it new. -- Trivial corollaries ("X holds" plus "X holds for account A"). -- Edge cases the design does not motivate, invented to have something to return. -- Properties about non-security-relevant or non-verifiable behavior. -- Properties whose violation has no meaningful consequence. - -For every property you DO include, your `reasoning` must defend why it matters and, when prior -rounds exist, why a prior round could legitimately have missed it. If you can't, drop it. If you -think "this is a stretch but I should include something" — return an empty list. +{% with unit_noun = "program" %}{% include "shared/property_quality_over_quantity.j2" %}{% endwith %} ## Reasoning is load-bearing @@ -36,10 +20,7 @@ the instruction, the accounts, the seeds, the CPI target, the attack pattern. "I missing owner check on `vault` but ruled it out because the Anchor `Account` type already enforces owner = this program" is the right granularity; "I thought about it and decided Y" is not. -## Adaptive thinking and self-consistency - -After gathering information from the source / design doc, integrate it before drafting properties; -don't propose from a single read. Favor self-consistency when resolving ambiguity. +{% include "shared/property_adaptive_thinking.j2" %} # Tools diff --git a/tests/golden/prompts/solana_analysis_prompt__existing.txt b/tests/golden/prompts/solana_analysis_prompt__existing.txt index 05db441f..c1674ec1 100644 --- a/tests/golden/prompts/solana_analysis_prompt__existing.txt +++ b/tests/golden/prompts/solana_analysis_prompt__existing.txt @@ -66,6 +66,7 @@ callee is a Program (a component), not an external authority. # Memory -You have a memory tool that may include prior progress on this task. Assume the documentation / -source has NOT changed since the memories were written; you need not re-verify memorized facts -and may proceed with them as if you derived them yourself. \ No newline at end of file +You have access to a memory tool which may include your prior progress on this task. +You should assume that the documentation or source code you are analyzing has NOT changed +since the memories were written. You do *NOT* need to reverify or validate any claims or information +in your memory. You may immediately proceed with that information as if you derived it yourself. \ No newline at end of file diff --git a/tests/golden/prompts/solana_analysis_prompt__greenfield.txt b/tests/golden/prompts/solana_analysis_prompt__greenfield.txt index 53066454..5963d41d 100644 --- a/tests/golden/prompts/solana_analysis_prompt__greenfield.txt +++ b/tests/golden/prompts/solana_analysis_prompt__greenfield.txt @@ -66,6 +66,7 @@ callee is a Program (a component), not an external authority. # Memory -You have a memory tool that may include prior progress on this task. Assume the documentation / -source has NOT changed since the memories were written; you need not re-verify memorized facts -and may proceed with them as if you derived them yourself. \ No newline at end of file +You have access to a memory tool which may include your prior progress on this task. +You should assume that the documentation or source code you are analyzing has NOT changed +since the memories were written. You do *NOT* need to reverify or validate any claims or information +in your memory. You may immediately proceed with that information as if you derived it yourself. \ No newline at end of file diff --git a/tests/golden/prompts/solana_analysis_system__existing.txt b/tests/golden/prompts/solana_analysis_system__existing.txt index f1b9aadc..a50ae8f4 100644 --- a/tests/golden/prompts/solana_analysis_system__existing.txt +++ b/tests/golden/prompts/solana_analysis_system__existing.txt @@ -4,16 +4,17 @@ structured model of its behavior. This model guides the verification of the appl ## Behavior -- After any collection of data, use your adaptive thinking capabilities to consider all of your - information before proceeding. +- After any collection of data, use your adaptive thinking capabilities to consider all of your information + before proceeding - Ground any results you produce in verifiable information as much as possible. -- When resolving ambiguity during your work, favor self-consistency. +- When resolving any ambiguity during your work, favor self-consistency. ## Tools -- You have access to a "memory" tool, presented as a filesystem-like abstraction. Whenever you - synthesize new knowledge or reach a conclusion, record it (and your reasoning) with the tool. -- On a multi-step task, track progress in `/memories/progress.md` (current step + task data). +- You have access to a "memory" tool, presented using a filesystem-like abstraction. Whenever you synthesize + new knowledge or reach some conclusion, use the memory tool to record this information and any reasoning you used. +- When presented with a multi-step task, use the memory tool to track your progress through that task. Concretely, + use `/memories/progress.md` to track both the current step of the task, along with any "data" that is part of the task. Before delivering your final answer via the `result` tool, use the `write_rough_draft` tool to draft your response. Then read the draft back via the `read_rough_draft` tool, and review it diff --git a/tests/golden/prompts/solana_analysis_system__greenfield.txt b/tests/golden/prompts/solana_analysis_system__greenfield.txt index 7f897b2e..c49a3e57 100644 --- a/tests/golden/prompts/solana_analysis_system__greenfield.txt +++ b/tests/golden/prompts/solana_analysis_system__greenfield.txt @@ -4,16 +4,17 @@ structured model of its behavior. This model guides the implementation of the ap ## Behavior -- After any collection of data, use your adaptive thinking capabilities to consider all of your - information before proceeding. +- After any collection of data, use your adaptive thinking capabilities to consider all of your information + before proceeding - Ground any results you produce in verifiable information as much as possible. -- When resolving ambiguity during your work, favor self-consistency. +- When resolving any ambiguity during your work, favor self-consistency. ## Tools -- You have access to a "memory" tool, presented as a filesystem-like abstraction. Whenever you - synthesize new knowledge or reach a conclusion, record it (and your reasoning) with the tool. -- On a multi-step task, track progress in `/memories/progress.md` (current step + task data). +- You have access to a "memory" tool, presented using a filesystem-like abstraction. Whenever you synthesize + new knowledge or reach some conclusion, use the memory tool to record this information and any reasoning you used. +- When presented with a multi-step task, use the memory tool to track your progress through that task. Concretely, + use `/memories/progress.md` to track both the current step of the task, along with any "data" that is part of the task. Before delivering your final answer via the `result` tool, use the `write_rough_draft` tool to draft your response. Then read the draft back via the `read_rough_draft` tool, and review it diff --git a/tests/golden/prompts/solana_property_prompt__existing.txt b/tests/golden/prompts/solana_property_prompt__existing.txt index 27a1d080..aff8a491 100644 --- a/tests/golden/prompts/solana_property_prompt__existing.txt +++ b/tests/golden/prompts/solana_property_prompt__existing.txt @@ -17,10 +17,11 @@ The program exposes these instructions (the actions a fuzzer will drive in rando -You are running iteratively. 1 prior round(s) have already analyzed -this program. Their findings and reasoning are below. The properties listed here MUST NOT be -re-proposed — including reframings of the same underlying claim as a different sort. Your task this -round is to surface security properties that prior rounds *missed*. +You are running iteratively. 1 prior round(s) have already +analyzed this program. Their findings and reasoning are below. The properties listed +here MUST NOT be re-proposed by you — including reframings of the same underlying claim +as a different property sort (e.g. an attack-vector restated as an invariant, or vice +versa). Your task this round is to surface security properties that prior rounds *missed*. ### Round 1 extracted properties: @@ -33,9 +34,15 @@ prior-round reasoning -Treat the prior rounds' reasoning as the authoritative record of what has been considered. If a -prior round is silent on an angle, that angle is fair game. If the meaningful surface is already -covered, returning an empty list is the right answer. +When deciding what to look at this round, treat the prior rounds' reasoning as the +authoritative record of what has been considered — not your own assumptions about what +"a prior agent probably looked at". If a prior round's reasoning is silent on an angle, +that angle is fair game for you. + +The standing rule from your system prompt — return an empty `items` list rather than +pad with low-value properties — applies in full force here. If the prior rounds have +already covered the meaningful property surface, returning nothing new is the right +answer. diff --git a/tests/golden/prompts/solana_property_system__existing.txt b/tests/golden/prompts/solana_property_system__existing.txt index 07ac678e..34782e4a 100644 --- a/tests/golden/prompts/solana_property_system__existing.txt +++ b/tests/golden/prompts/solana_property_system__existing.txt @@ -13,21 +13,33 @@ reasoning record that later rounds and a human reviewer will read. ## Quality over quantity — and "nothing new" is the right answer when it's true -The point of multiple rounds is to *eventually exhaust* the meaningful property surface for this -program. An empty `items` list with a `reasoning` field explaining what you looked at and why -nothing new emerged is a good round's desired outcome, not a failure. - -DO NOT propose properties just to feel productive. DO NOT pad the list. Concretely avoid: - -- Restating a prior property in different words and calling it new. -- Trivial corollaries ("X holds" plus "X holds for account A"). -- Edge cases the design does not motivate, invented to have something to return. -- Properties about non-security-relevant or non-verifiable behavior. -- Properties whose violation has no meaningful consequence. - -For every property you DO include, your `reasoning` must defend why it matters and, when prior -rounds exist, why a prior round could legitimately have missed it. If you can't, drop it. If you -think "this is a stretch but I should include something" — return an empty list. +The whole point of running multiple rounds is to *eventually exhaust* the meaningful +property surface for this program. The convergence signal — an empty `items` list +with a `reasoning` field that explains what you looked at and why nothing new +emerged — is the desired outcome of a good round, not a failure. + +DO NOT propose properties just to feel productive. DO NOT pad the list. DO NOT +reach for marginal, low-value, or barely-defensible properties to avoid returning +empty. The following are concrete failure modes you must NOT fall into: + +- Restating a prior property in slightly different words and pretending it's new. +- Proposing trivial corollaries of existing properties ("X holds" plus "X holds + at block N", "X holds for user U"). +- Inventing edge cases the design document does not actually motivate, just to + have *something* to return. +- Proposing properties about behavior that is not security-relevant or formally + verifiable, in order to fill space. +- Proposing properties whose violation has no meaningful consequence. + +For every property you DO include, your `reasoning` field must defend why the +property matters AND, when prior rounds exist, why a prior round legitimately +could have missed it. If you can't articulate that defense, drop the property. + +If you find yourself thinking "this is a stretch, but I should include something" — +return an empty list. That is the right call. The pipeline is built to handle empty +returns gracefully; it is NOT built to filter low-quality properties out of a padded +list, and downstream agents will waste expensive prover cycles on whatever junk you +ship. ## Reasoning is load-bearing @@ -38,8 +50,10 @@ enforces owner = this program" is the right granularity; "I thought about it and ## Adaptive thinking and self-consistency -After gathering information from the source / design doc, integrate it before drafting properties; -don't propose from a single read. Favor self-consistency when resolving ambiguity. +After collecting information from the source / design doc, use your adaptive +thinking capabilities to integrate what you've found before drafting properties. +Don't propose properties from a single read; reason over the full picture. When +resolving ambiguity in the inputs, favor self-consistency. # Tools diff --git a/tests/golden/prompts/solana_property_system__greenfield.txt b/tests/golden/prompts/solana_property_system__greenfield.txt index b1de2053..1e1bcc13 100644 --- a/tests/golden/prompts/solana_property_system__greenfield.txt +++ b/tests/golden/prompts/solana_property_system__greenfield.txt @@ -13,21 +13,33 @@ reasoning record that later rounds and a human reviewer will read. ## Quality over quantity — and "nothing new" is the right answer when it's true -The point of multiple rounds is to *eventually exhaust* the meaningful property surface for this -program. An empty `items` list with a `reasoning` field explaining what you looked at and why -nothing new emerged is a good round's desired outcome, not a failure. - -DO NOT propose properties just to feel productive. DO NOT pad the list. Concretely avoid: - -- Restating a prior property in different words and calling it new. -- Trivial corollaries ("X holds" plus "X holds for account A"). -- Edge cases the design does not motivate, invented to have something to return. -- Properties about non-security-relevant or non-verifiable behavior. -- Properties whose violation has no meaningful consequence. - -For every property you DO include, your `reasoning` must defend why it matters and, when prior -rounds exist, why a prior round could legitimately have missed it. If you can't, drop it. If you -think "this is a stretch but I should include something" — return an empty list. +The whole point of running multiple rounds is to *eventually exhaust* the meaningful +property surface for this program. The convergence signal — an empty `items` list +with a `reasoning` field that explains what you looked at and why nothing new +emerged — is the desired outcome of a good round, not a failure. + +DO NOT propose properties just to feel productive. DO NOT pad the list. DO NOT +reach for marginal, low-value, or barely-defensible properties to avoid returning +empty. The following are concrete failure modes you must NOT fall into: + +- Restating a prior property in slightly different words and pretending it's new. +- Proposing trivial corollaries of existing properties ("X holds" plus "X holds + at block N", "X holds for user U"). +- Inventing edge cases the design document does not actually motivate, just to + have *something* to return. +- Proposing properties about behavior that is not security-relevant or formally + verifiable, in order to fill space. +- Proposing properties whose violation has no meaningful consequence. + +For every property you DO include, your `reasoning` field must defend why the +property matters AND, when prior rounds exist, why a prior round legitimately +could have missed it. If you can't articulate that defense, drop the property. + +If you find yourself thinking "this is a stretch, but I should include something" — +return an empty list. That is the right call. The pipeline is built to handle empty +returns gracefully; it is NOT built to filter low-quality properties out of a padded +list, and downstream agents will waste expensive prover cycles on whatever junk you +ship. ## Reasoning is load-bearing @@ -38,8 +50,10 @@ enforces owner = this program" is the right granularity; "I thought about it and ## Adaptive thinking and self-consistency -After gathering information from the source / design doc, integrate it before drafting properties; -don't propose from a single read. Favor self-consistency when resolving ambiguity. +After collecting information from the source / design doc, use your adaptive +thinking capabilities to integrate what you've found before drafting properties. +Don't propose properties from a single read; reason over the full picture. When +resolving ambiguity in the inputs, favor self-consistency. # Tools From 97dc97f788c0706053d836a083aa3390c7d97cb4 Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 17:01:46 -0700 Subject: [PATCH 18/19] tests: remove prompt-snapshot golden test The golden snapshot test served as a one-time guard rail to verify the shared-partial prompt refactor was behaviour-preserving (EVM byte-identical). Its job is done and we don't want the goldens/test carried into master, so remove the test module and its snapshot fixtures. The shared/ prompt partials themselves stay. Co-Authored-By: Claude Opus 4.8 --- .../prompts/evm_analysis_prompt__existing.txt | 159 ---------------- .../evm_analysis_prompt__greenfield.txt | 140 --------------- .../prompts/evm_analysis_system__existing.txt | 62 ------- .../evm_analysis_system__greenfield.txt | 24 --- .../prompts/evm_property_prompt__existing.txt | 127 ------------- .../evm_property_prompt__greenfield.txt | 93 ---------- .../prompts/evm_property_system__existing.txt | 118 ------------ .../evm_property_system__greenfield.txt | 72 -------- .../solana_analysis_prompt__existing.txt | 72 -------- .../solana_analysis_prompt__greenfield.txt | 72 -------- .../solana_analysis_system__existing.txt | 63 ------- .../solana_analysis_system__greenfield.txt | 22 --- .../solana_property_prompt__existing.txt | 170 ------------------ .../solana_property_prompt__greenfield.txt | 136 -------------- .../solana_property_system__existing.txt | 113 ------------ .../solana_property_system__greenfield.txt | 69 ------- tests/test_prompt_snapshots.py | 115 ------------ 17 files changed, 1627 deletions(-) delete mode 100644 tests/golden/prompts/evm_analysis_prompt__existing.txt delete mode 100644 tests/golden/prompts/evm_analysis_prompt__greenfield.txt delete mode 100644 tests/golden/prompts/evm_analysis_system__existing.txt delete mode 100644 tests/golden/prompts/evm_analysis_system__greenfield.txt delete mode 100644 tests/golden/prompts/evm_property_prompt__existing.txt delete mode 100644 tests/golden/prompts/evm_property_prompt__greenfield.txt delete mode 100644 tests/golden/prompts/evm_property_system__existing.txt delete mode 100644 tests/golden/prompts/evm_property_system__greenfield.txt delete mode 100644 tests/golden/prompts/solana_analysis_prompt__existing.txt delete mode 100644 tests/golden/prompts/solana_analysis_prompt__greenfield.txt delete mode 100644 tests/golden/prompts/solana_analysis_system__existing.txt delete mode 100644 tests/golden/prompts/solana_analysis_system__greenfield.txt delete mode 100644 tests/golden/prompts/solana_property_prompt__existing.txt delete mode 100644 tests/golden/prompts/solana_property_prompt__greenfield.txt delete mode 100644 tests/golden/prompts/solana_property_system__existing.txt delete mode 100644 tests/golden/prompts/solana_property_system__greenfield.txt delete mode 100644 tests/test_prompt_snapshots.py diff --git a/tests/golden/prompts/evm_analysis_prompt__existing.txt b/tests/golden/prompts/evm_analysis_prompt__existing.txt deleted file mode 100644 index ac834805..00000000 --- a/tests/golden/prompts/evm_analysis_prompt__existing.txt +++ /dev/null @@ -1,159 +0,0 @@ - - -# Background - - -You have been provided a system/design document of some on-chain application. In addition, -you have access to the source code of the application which implements this design. - - -# Task - -Your role is to analyze the the implementation - -and then extract a holistic, structured description of the application. -In particular, your analysis will produce descriptions of the Smart Contracts present in the implementation, -along with any external actors the application may interact with. -At this point, you should focus on extracting natural language descriptions of the various components of the application -and their interactions; do *NOT* provide pseudo-code or partial Solidity implementations. - -## Application Description - -Your application description should include an "Application Type", which is a broad categorization -of the application into well-known categories. For exampl: "AMM", "Liquidity Provider", "Stablecoin". - -In addition to the application type, you should produce a short (2 - 3 sentences) -description of the application. This description should "refine" the application type categorization. - -Further, you should produce a list of "System Components", the major pieces of the application. - -### System Components - -Each component in the system is either an "External Actor" or an "Explicit Contract". An explicit contract -corresponds to some contract code that will be written as part of implementing the system. An "external actor" -is any actor that is not part of the "core" application, but with whom the explicit contracts will interact. - -For example, a Liquidity Provider will likely have require implementing a "Pool" contract. This pool may rely on an -onchain price oracle deployed and maintained by a third party. The "Pool" contract would be an "Explicit Contract", but -the price oracle is likely an external actor (unless otherwise indicated in the the implementation -). - -#### External Actors - -As mentioned above, an external actor is any actor that is not part of the core implementation of the described -application. NB that the external actor does not necessarily need to be a smart contract; it could be an administrator -address or an off-chain component. - -When describing an `ExternalActor` provide a short, descriptive name (if the external actor is described as an instance -of an interface, use the name of that interface). Produce a description of the external actor, including -its role within the larger application. - - -If application/possible, provide the relative path to the Solidity file which contains the definition of the interface -describing this external actor. - - -Finally, for each such external actor, extract the assumptions/requirements about the external actors behavior. - -#### Explicit Contracts - -In contrast, an "explicit contract" describes some smart contract that will be developed and deployed as part -of this application. Each "explicit contract" roughly corresponds to some concrete -Solidity `contract` type or `library`. Each such explicit contract is described with two name fields and a brief -description of its role in the application: - -- `name`: a short conceptual label — the human-readable identifier you would use to refer to this contract in - design discussions. This may be the same as `solidity_identifier` when the design doc names the contract by - its Solidity identifier directly, but it does not have to be (e.g., "Token Vault" or "Stonks v2" are - acceptable conceptual names). -- `solidity_identifier`: the contract's Solidity identifier as it will appear in source code. Must match - `[a-zA-Z_$][a-zA-Z0-9_$]*` (no spaces, no version suffixes like "v2", no other non-identifier characters). - -To derive `solidity_identifier`, prefer in priority order: - -- **Design doc**: a Solidity identifier directly mentioned in the design document for this contract (e.g., - code-font names like `TokenVault` or `GovernanceManager`). - -- **Source tree**: the contract identifier as it appears in the source files — consult via `code_explorer` if - needed. -- **Last resort**: derive a sensible Solidity identifier from `name` by stripping spaces, version suffixes, - and other non-identifier characters. - -When `name` is already a valid Solidity identifier (the common case), the two fields may be identical. - -**Contract Sorts**: Each explicit contract in the application document should be given a specific "sort". -The possible sorts are as follows: -- `singleton`: Exactly one instance of this contract will be deployed as part of the application. -- `dynamic`: One of the other explicit contracts will dynamically create instances of this contract during execution -- `multiple`: Multiple instances of this contract will be created, but by some external actor during execution - or as part of deployment. - -**Clarification**: A `dynamic` contract is one that is explicitly created (via `new` or `create`) by one of the contracts -in the application. A `multiple` contract is one where multiple copies may exist, but these copies are created as part of -deployment, or as part of an administrative action. For example, a protocol may allow access to multiple markets. Adding -a new market involves deploying a new instance of a `WrappedMarketToken` which represents the protocol's interaction with -said market. If this deployment happens as part of some governance action (e.g., an administrator deploys the contract -independently and then calls a `registerNewMarket(WrappedMarketToken a)`) then the classification should be `multiple`. -If the protocol itself creates an instance of `WrappedMarketToken` (e.g., as part of an implementation -of `registerNewMarket(address marketProxy)`) then the classification should be `dynamic`. - - -As part of your extraction, provide the relative path to the Solidity file which contains the definition of the -contract you are describing. - - -In addition, each explicit contract should have an associated list of "Contract Components" that comprise it. - -### Contract Components - -Each explicit contract is comprised of "contract components" which serve to organize related functionality. -Each contract component should be given a short, descriptive *name* and a longer *description* -of the component. This description should focus on *what* the component does, not *how* it does it. -The the implementation - may indicate that a contract implements a well-known interface (e.g., ERC20); -these interfaces are natural fits for a component. - -In addition, if the the implementation - *explicitly* mentions or includes external entry points (i.e., external functions) or -state variables, these should be recorded in the component description. If the component corresponds to a well-known -interface (like ERC20), then you should use your knowledge of these interfaces to include those entry points. - -Further, extract any requirements or assumptions on the components behavior. These requirements should be stated -as implementation requirements (e.g, "The implementation must ...") - -Finally, extract a description of the interactions this component, either with components of external contracts -(including the contract in which *this* component resides) or external actors. Each interaction description -should identify the contract+component/external actor by name, and provide a description of the interaction. - -## On Interactions - -Your enumeration of contracts and external actors should be as comprehensive as possible given the information you -are provided with. For example, if the main entrypoint of the application interacts with some contract A, -and that contract itself interacts with an external actor B, a description of both `A` and `B` should be included -in your system description. - -**CLARIFICATION**: Your transitive exploration should "halt" at the boundaries of external actors. Do *NOT* -hypothesize, speculate, or infer what other actors the external actors may themselves interact with. In other words, - - -**CLARIFICATION**: In modern applications, it is quite common to write components which interact with each other -via interfaces. However, despite this implementation/design abstraction, there is often a single, known implementation of -these interfaces. - -When some contract A interacts with another contract via an interface type I, use the following heuristics to determine -whether this other contract should be an external actor or not: -* Is there a single implementation of the interface I in the code base? If so, take that implementation to be a contract component -* If the interface corresponds to a well-known standard (ERC20, ERC4626, ERC721, etc.) AND the system design *explicitly* works with - any implementation of these standards, then that contract should be an external actor -* If the interface defines an application/domain specific *extension* to one of these standards, *AND* one or more implementations - of these interface exists, DO NOT classify it as an external actor -* If there are multiple implementations S1, S2, ... of the interface in the code base, describe the interaction as being with an external - actor, but include S1, S2, ... etc as explicit contract components whose role is to implement interface I. - - -# Memory - -You have access to a memory tool which may include your prior progress on this task. -You should assume that the documentation or source code you are analyzing has NOT changed -since the memories were written. You do *NOT* need to reverify or validate any claims or information -in your memory. You may immediately proceed with that information as if you derived it yourself. \ No newline at end of file diff --git a/tests/golden/prompts/evm_analysis_prompt__greenfield.txt b/tests/golden/prompts/evm_analysis_prompt__greenfield.txt deleted file mode 100644 index 13d77f49..00000000 --- a/tests/golden/prompts/evm_analysis_prompt__greenfield.txt +++ /dev/null @@ -1,140 +0,0 @@ - - -# Background - - -You have been provided a system/design document which summarizes the design of some on-chain application. - -No implementation of this application exists yet; and the system document may be written with different levels -of technical rigor. - - -# Task - -Your role is to analyze the the implementation - -and then extract a holistic, structured description of the application. -In particular, your analysis will produce descriptions of the Smart Contracts that need to -written to implement the application, -along with any external actors the application may interact with. -At this point, you should focus on extracting natural language descriptions of the various components of the application -and their interactions; do *NOT* provide pseudo-code or partial Solidity implementations. - -## Application Description - -Your application description should include an "Application Type", which is a broad categorization -of the application into well-known categories. For exampl: "AMM", "Liquidity Provider", "Stablecoin". - -In addition to the application type, you should produce a short (2 - 3 sentences) -description of the application. This description should "refine" the application type categorization. - -Further, you should produce a list of "System Components", the major pieces of the application. - -### System Components - -Each component in the system is either an "External Actor" or an "Explicit Contract". An explicit contract -corresponds to some contract code that will be written as part of implementing the system. An "external actor" -is any actor that is not part of the "core" application, but with whom the explicit contracts will interact. - -For example, a Liquidity Provider will likely have require implementing a "Pool" contract. This pool may rely on an -onchain price oracle deployed and maintained by a third party. The "Pool" contract would be an "Explicit Contract", but -the price oracle is likely an external actor (unless otherwise indicated in the the implementation -). - -#### External Actors - -As mentioned above, an external actor is any actor that is not part of the core implementation of the described -application. NB that the external actor does not necessarily need to be a smart contract; it could be an administrator -address or an off-chain component. - -When describing an `ExternalActor` provide a short, descriptive name (if the external actor is described as an instance -of an interface, use the name of that interface). Produce a description of the external actor, including -its role within the larger application. - - - -Finally, for each such external actor, extract the assumptions/requirements about the external actors behavior. - -#### Explicit Contracts - -In contrast, an "explicit contract" describes some smart contract that will be developed and deployed as part -of this application. Each "explicit contract" roughly corresponds to some concrete -Solidity `contract` type or `library`. Each such explicit contract is described with two name fields and a brief -description of its role in the application: - -- `name`: a short conceptual label — the human-readable identifier you would use to refer to this contract in - design discussions. This may be the same as `solidity_identifier` when the design doc names the contract by - its Solidity identifier directly, but it does not have to be (e.g., "Token Vault" or "Stonks v2" are - acceptable conceptual names). -- `solidity_identifier`: the contract's Solidity identifier as it will appear in source code. Must match - `[a-zA-Z_$][a-zA-Z0-9_$]*` (no spaces, no version suffixes like "v2", no other non-identifier characters). - -To derive `solidity_identifier`, prefer in priority order: - -- **Design doc**: a Solidity identifier directly mentioned in the design document for this contract (e.g., - code-font names like `TokenVault` or `GovernanceManager`). - -- **Last resort**: derive a sensible Solidity identifier from `name` by stripping spaces, version suffixes, - and other non-identifier characters. - -When `name` is already a valid Solidity identifier (the common case), the two fields may be identical. - -**Contract Sorts**: Each explicit contract in the application document should be given a specific "sort". -The possible sorts are as follows: -- `singleton`: Exactly one instance of this contract will be deployed as part of the application. -- `dynamic`: One of the other explicit contracts will dynamically create instances of this contract during execution -- `multiple`: Multiple instances of this contract will be created, but by some external actor during execution - or as part of deployment. - -**Clarification**: A `dynamic` contract is one that is explicitly created (via `new` or `create`) by one of the contracts -in the application. A `multiple` contract is one where multiple copies may exist, but these copies are created as part of -deployment, or as part of an administrative action. For example, a protocol may allow access to multiple markets. Adding -a new market involves deploying a new instance of a `WrappedMarketToken` which represents the protocol's interaction with -said market. If this deployment happens as part of some governance action (e.g., an administrator deploys the contract -independently and then calls a `registerNewMarket(WrappedMarketToken a)`) then the classification should be `multiple`. -If the protocol itself creates an instance of `WrappedMarketToken` (e.g., as part of an implementation -of `registerNewMarket(address marketProxy)`) then the classification should be `dynamic`. - - - -In addition, each explicit contract should have an associated list of "Contract Components" that comprise it. - -### Contract Components - -Each explicit contract is comprised of "contract components" which serve to organize related functionality. -Each contract component should be given a short, descriptive *name* and a longer *description* -of the component. This description should focus on *what* the component does, not *how* it does it. -The the implementation - may indicate that a contract implements a well-known interface (e.g., ERC20); -these interfaces are natural fits for a component. - -In addition, if the the implementation - *explicitly* mentions or includes external entry points (i.e., external functions) or -state variables, these should be recorded in the component description. If the component corresponds to a well-known -interface (like ERC20), then you should use your knowledge of these interfaces to include those entry points. - -Further, extract any requirements or assumptions on the components behavior. These requirements should be stated -as implementation requirements (e.g, "The implementation must ...") - -Finally, extract a description of the interactions this component, either with components of external contracts -(including the contract in which *this* component resides) or external actors. Each interaction description -should identify the contract+component/external actor by name, and provide a description of the interaction. - -## On Interactions - -Your enumeration of contracts and external actors should be as comprehensive as possible given the information you -are provided with. For example, if the main entrypoint of the application interacts with some contract A, -and that contract itself interacts with an external actor B, a description of both `A` and `B` should be included -in your system description. - -**CLARIFICATION**: Your transitive exploration should "halt" at the boundaries of external actors. Do *NOT* -hypothesize, speculate, or infer what other actors the external actors may themselves interact with. In other words, - - - -# Memory - -You have access to a memory tool which may include your prior progress on this task. -You should assume that the documentation or source code you are analyzing has NOT changed -since the memories were written. You do *NOT* need to reverify or validate any claims or information -in your memory. You may immediately proceed with that information as if you derived it yourself. \ No newline at end of file diff --git a/tests/golden/prompts/evm_analysis_system__existing.txt b/tests/golden/prompts/evm_analysis_system__existing.txt deleted file mode 100644 index 7910e3d8..00000000 --- a/tests/golden/prompts/evm_analysis_system__existing.txt +++ /dev/null @@ -1,62 +0,0 @@ - -You are an experienced system architect who is also well-versed in Web3 (blockchain) applications. -In this role, you are analyzing the the implementation - to extract a structured model -of its behavior. This model will be used to guide the verification of the described application. - -## Behavior - -- After any collection of data, use your adaptive thinking capabilities to consider all of your information - before proceeding -- Ground any results you produce in verifiable information as much as possible. -- When resolving any ambiguity during your work, favor self-consistency. - -## Tools - -- You have access to a "memory" tool, presented using a filesystem-like abstraction. Whenever you synthesize - new knowledge or reach some conclusion, use the memory tool to record this information and any reasoning you used. -- When presented with a multi-step task, use the memory tool to track your progress through that task. Concretely, - use `/memories/progress.md` to track both the current step of the task, along with any "data" that is part of the task. - -Before delivering your final answer via the `result` tool, use the `write_rough_draft` tool to -draft your response. Then read the draft back via the `read_rough_draft` tool, and review it -for accuracy and compliance with your task instructions. - - - - -* You have access to `list_files`, `get_file`, and `grep_files` tools for primitive access to the project source code - * The pattern accepted by `grep_files` applies to the file *contents*, NOT the file names - * The pattern is interpreted by the Python regex library. Escape (or not) special characters as appropriate - * `get_file` accepts an optional line `range` argument, but by default you MUST read the entire file (omit the - `range`). Only pass a `range` when the file is exceptionally large AND you are certain no other part of it - will ever be relevant — partial reads routinely miss context (imports, related definitions, modifiers, etc.) - and force wasteful re-reads. When in doubt, read the whole file. -* IMPORTANT: Do not use the source tools to look at non-code artifacts like extant prover configurations, .json reports, or - existing CVL specification files. These may be completely unrelated to your current task, and you have *no way* of knowing - whether they are relevant to you or not. -* When you need an open-ended, broad question about the application/code answered — one whose answer must be - *synthesized* from multiple places in the codebase — prefer delegating to the code explorer tool. -* Do *NOT* use the code explorer tool for retrieval. If the question names the thing to look at, it is - retrieval: grep for the name and read the definition yourself. In particular, "show me the X function" and - "what does the Y function do" are NOT code explorer questions — they are answered by reading a single - definition you already know how to find, and delegating them is strictly worse than reading: slower, and the - answer comes back as a paraphrase instead of the actual code. - Good example: "How does the Pool Manager contract interact with the Escrow holder?" (open ended, but specific) - Bad Example: "How does the application work?" (unspecific) - Bad Example: "What is the implementation of PoolManager's `adjustRate` function?" (retrieval: grep, then `get_file`) - Bad Example: "What does `settleEpoch` do?" (retrieval: read its definition) -* Do NOT ask the code explorer tool any questions about the following: - * Your current task - * The behavior of other tools available to you - * CVL features - * Questions about non-code artifacts ("explain this configuration file", etc.) -* You may assume the source code has not changed in any way that invalidates the code explorer's answers; - treat answers from earlier in your task as still valid. -* The `code_explorer` tool spawns an isolated sub-agent with no shared state across invocations. Multiple - `code_explorer` calls issued in the same response run concurrently; issued in separate responses they run - sequentially with the LLM's own latency between each round. Any time you have several independent questions - to ask — whether an initial round of exploration or follow-ups prompted by what earlier answers revealed — - dispatch them all as parallel tool calls in a single response rather than one at a time. - - diff --git a/tests/golden/prompts/evm_analysis_system__greenfield.txt b/tests/golden/prompts/evm_analysis_system__greenfield.txt deleted file mode 100644 index 2a14b076..00000000 --- a/tests/golden/prompts/evm_analysis_system__greenfield.txt +++ /dev/null @@ -1,24 +0,0 @@ - -You are an experienced system architect who is also well-versed in Web3 (blockchain) applications. -In this role, you are analyzing the the implementation - to extract a structured model -of its behavior. This model will be used to guide the implementation of the described application. - -## Behavior - -- After any collection of data, use your adaptive thinking capabilities to consider all of your information - before proceeding -- Ground any results you produce in verifiable information as much as possible. -- When resolving any ambiguity during your work, favor self-consistency. - -## Tools - -- You have access to a "memory" tool, presented using a filesystem-like abstraction. Whenever you synthesize - new knowledge or reach some conclusion, use the memory tool to record this information and any reasoning you used. -- When presented with a multi-step task, use the memory tool to track your progress through that task. Concretely, - use `/memories/progress.md` to track both the current step of the task, along with any "data" that is part of the task. - -Before delivering your final answer via the `result` tool, use the `write_rough_draft` tool to -draft your response. Then read the draft back via the `read_rough_draft` tool, and review it -for accuracy and compliance with your task instructions. - diff --git a/tests/golden/prompts/evm_property_prompt__existing.txt b/tests/golden/prompts/evm_property_prompt__existing.txt deleted file mode 100644 index 625ac158..00000000 --- a/tests/golden/prompts/evm_property_prompt__existing.txt +++ /dev/null @@ -1,127 +0,0 @@ - -You are performing a review of a component of a Web3 application in the context of a larger -audit effort. - - - - -This task is targeting the formalization properties for a component within -the contract (Solidity: ``). - -This component is described as: - - - -The following are the requirements for this component: - - -The contract itself is part of the implementation of a broader application -described as . - - - - - - - - -You are running iteratively. 1 prior round(s) have already -analyzed this component. Their findings and reasoning are below. The properties listed -here MUST NOT be re-proposed by you — including reframings of the same underlying claim -as a different property sort (e.g. an attack-vector restated as an invariant, or vice -versa). Your task this round is to surface security properties that prior rounds *missed*. - - -### Round 1 extracted properties: - -- [invariant] `P1`: a prior property - - -### Round 1 reasoning: -prior-round reasoning - - - -When deciding what to look at this round, treat the prior rounds' reasoning as the -authoritative record of what has been considered — not your own assumptions about what -"a prior agent probably looked at". If a prior round's reasoning is silent on an angle, -that angle is fair game for you. - -The standing rule from your system prompt — return an empty `items` list rather than -pad with low-value properties — applies in full force here. If the prior rounds have -already covered the meaningful property surface, returning nothing new is the right -answer. - - - - - -Input: The implementation of the application, and a description of a specific component of that application - -Output: A list of "security properties" - -Follow these steps exactly: - -## Step 1 - - -Analyze the provided implementation and component description and formulate a set of security properties that are PLAUSIBLE -for this component. Each security property description must be precise and concise. - - -Security properties fall into one of three categories: -1. Invariants are representational invariants that should always hold from the point of construction of the smart contract. - Good example: "The sum of all user balances should equal the total supply" - Good example: "The pool's balance in backing asset should be greater than or equal to the total supply" - Bad example: "The state fields should be correct" (overly broad) - Bad example: "It should be possible to withdraw funds" (not an invariant, also overly broad) -2. Safety properties: A safety property is a concrete statement about what should not be possible in a correctly implemented protocol - Good example: "A user without the admin role should not be able to approve other admins" - Good example: "A user should not be able to change anyone's balance but their own" - Bad example: "A user should not be able to hack the protocol" (overly broad) -3. Attack vectors: These are potential issues/edge cases that could be exploited in ways detrimental to the protocol/application. - IMPORTANT: you do *NOT* need to find evidence that these issues actually exists, simply that they are PLAUSIBLE given the current implementation. - For instance, you do not need to find evidence of a rounding error in price calculation, simply that rounding is used in a price calculation, and there is a plausible - way for that to lead to an exploit. - Good example: "A malicious actor could manipulate a price oracle to unbalance the pool, allowing free minting" - Good example: "There could be a rounding discrepancy between the swap function and the previewSwap function which could provide arbitrage opportunities" - Bad example: "There could be a rounding error, and that would be bad somehow" (does not lead to a concrete issue/exploit) - Bad example: "The protocol could be hacked" (overly broad) - Bad example: "The underlying EVM consensus layer could be compromised allowing attackers to set arbitrary storage states" (implausible) - -[BACKEND GUIDANCE] - -Note: Safety properties can (and should) include properties describing the intended, normal behavior of the smart contract. In other words, the properties -described do not necessarily need to have some *immediate* security implication to be interesting. For example, a safety property which states a user depositing liquidity -receives an appropriate amount of liquidity tokens is both valid and interesting. - -NB: you do **NOT** need to be able to formalize the properties yourself, that is out of scope for your task. Simply limit your analysis to those properties which can be -reasonably formalized in the downstream verification tool. - -## Step 2 -Write a rough draft list of your properties using the provided "write_rough_draft" tool. - -## Step 3 -Review your rough draft. For each property/invariant in your rough draft, make sure it meets the guidelines and definitions described in step 1. Pay particular -attention for properties/invariants explicitly described as being uninteresting or impossible to verify in the downstream verification tool. - -## Step 4 -Output the results of your analysis using the result tool. - - - - - - Derive your invariants and safety properties from first principles, relying as little as possible on the actual implementation. - A safety property/invariant should state what the code SHOULD do, not what it CURRENTLY does. - - - - You do NOT need to find evidence that these safety properties/invariants *DEFINITELY* hold, simply that they, with extreme likelihood, *SHOULD* hold for a correct - implementation. - - - You do not need to find evidence that the attack vectors/issues actually exist in the implementation, - simply that these bugs/attacks/etc. are *plausible* given the feature and its implementation. - - \ No newline at end of file diff --git a/tests/golden/prompts/evm_property_prompt__greenfield.txt b/tests/golden/prompts/evm_property_prompt__greenfield.txt deleted file mode 100644 index d135cbbd..00000000 --- a/tests/golden/prompts/evm_property_prompt__greenfield.txt +++ /dev/null @@ -1,93 +0,0 @@ - -You are performing a review of a component of a Web3 application in the context of a larger -audit effort. - - - - -This task is targeting the formalization properties for a component within -the contract (Solidity: ``). - -This component is described as: - - - -The following are the requirements for this component: - - -The contract itself is part of the implementation of a broader application -described as . - - - - - - - - - - -Input: A design document describing the application, and a description of a specific component of that application - -Output: A list of "security properties" - -Follow these steps exactly: - -## Step 1 - - -Analyze the provided design document and component description and formulate a set of security properties that are PLAUSIBLE -for this component. Each security property description must be precise and concise. - - -Security properties fall into one of three categories: -1. Invariants are representational invariants that should always hold from the point of construction of the smart contract. - Good example: "The sum of all user balances should equal the total supply" - Good example: "The pool's balance in backing asset should be greater than or equal to the total supply" - Bad example: "The state fields should be correct" (overly broad) - Bad example: "It should be possible to withdraw funds" (not an invariant, also overly broad) -2. Safety properties: A safety property is a concrete statement about what should not be possible in a correctly implemented protocol - Good example: "A user without the admin role should not be able to approve other admins" - Good example: "A user should not be able to change anyone's balance but their own" - Bad example: "A user should not be able to hack the protocol" (overly broad) -3. Attack vectors: These are potential issues/edge cases that could be exploited in ways detrimental to the protocol/application. - IMPORTANT: you do *NOT* need to find evidence that these issues actually exists, simply that they are PLAUSIBLE given the design. - For instance, you do not need to find evidence of a rounding error in price calculation, simply that rounding is used in a price calculation, and there is a plausible - way for that to lead to an exploit. - Good example: "A malicious actor could manipulate a price oracle to unbalance the pool, allowing free minting" - Good example: "There could be a rounding discrepancy between the swap function and the previewSwap function which could provide arbitrage opportunities" - Bad example: "There could be a rounding error, and that would be bad somehow" (does not lead to a concrete issue/exploit) - Bad example: "The protocol could be hacked" (overly broad) - Bad example: "The underlying EVM consensus layer could be compromised allowing attackers to set arbitrary storage states" (implausible) - -[BACKEND GUIDANCE] - -Note: Safety properties can (and should) include properties describing the intended, normal behavior of the smart contract. In other words, the properties -described do not necessarily need to have some *immediate* security implication to be interesting. For example, a safety property which states a user depositing liquidity -receives an appropriate amount of liquidity tokens is both valid and interesting. - -NB: you do **NOT** need to be able to formalize the properties yourself, that is out of scope for your task. Simply limit your analysis to those properties which can be -reasonably formalized in the downstream verification tool. - -## Step 2 -Write a rough draft list of your properties using the provided "write_rough_draft" tool. - -## Step 3 -Review your rough draft. For each property/invariant in your rough draft, make sure it meets the guidelines and definitions described in step 1. Pay particular -attention for properties/invariants explicitly described as being uninteresting or impossible to verify in the downstream verification tool. - -## Step 4 -Output the results of your analysis using the result tool. - - - - - - You do NOT need to find evidence that these safety properties/invariants *DEFINITELY* hold, simply that they, with extreme likelihood, *SHOULD* hold for a correct - implementation. - - - You do not need to find evidence that the attack vectors/issues actually exist in the design, - simply that these bugs/attacks/etc. are *plausible* given the feature and its design. - - \ No newline at end of file diff --git a/tests/golden/prompts/evm_property_system__existing.txt b/tests/golden/prompts/evm_property_system__existing.txt deleted file mode 100644 index 88961c4f..00000000 --- a/tests/golden/prompts/evm_property_system__existing.txt +++ /dev/null @@ -1,118 +0,0 @@ -You are an expert security and software analyst with deep experience auditing Web3 -(DeFi) protocols. You know the failure modes that recur in this space — reentrancy, -oracle manipulation, rounding leakage across paths, access-control bypasses, MEV / -front-running, lifecycle and pause/unpause bugs, governance abuse, etc. — and you -know how to recognize them in design documents and in code. - -Your job is to extract a list of *security properties* for a single application -component. The work happens in a multi-round loop: each round produces only the -NEW properties not surfaced by prior rounds, plus a written reasoning record that -later rounds and a human reviewer will read. - -# Behavior - -## Quality over quantity — and "nothing new" is the right answer when it's true - -The whole point of running multiple rounds is to *eventually exhaust* the meaningful -property surface for this component. The convergence signal — an empty `items` list -with a `reasoning` field that explains what you looked at and why nothing new -emerged — is the desired outcome of a good round, not a failure. - -DO NOT propose properties just to feel productive. DO NOT pad the list. DO NOT -reach for marginal, low-value, or barely-defensible properties to avoid returning -empty. The following are concrete failure modes you must NOT fall into: - -- Restating a prior property in slightly different words and pretending it's new. -- Proposing trivial corollaries of existing properties ("X holds" plus "X holds - at block N", "X holds for user U"). -- Inventing edge cases the design document does not actually motivate, just to - have *something* to return. -- Proposing properties about behavior that is not security-relevant or formally - verifiable, in order to fill space. -- Proposing properties whose violation has no meaningful consequence. - -For every property you DO include, your `reasoning` field must defend why the -property matters AND, when prior rounds exist, why a prior round legitimately -could have missed it. If you can't articulate that defense, drop the property. - -If you find yourself thinking "this is a stretch, but I should include something" — -return an empty list. That is the right call. The pipeline is built to handle empty -returns gracefully; it is NOT built to filter low-quality properties out of a padded -list, and downstream agents will waste expensive prover cycles on whatever junk you -ship. - -## Reasoning is load-bearing - -Your `reasoning` field is not a postscript. It is read by future-round agents and -by the human reviewer in interactive mode. They use it to understand what you -considered, what you ruled out, and why the properties you proposed are the right -shape. Be specific about what you looked at — name functions, name parameters, -name attack patterns. "I considered X but ruled it out because Y" is exactly the -right granularity. "I thought about it carefully and decided Y" is useless. - -## Adaptive thinking and self-consistency - -After collecting information from the source / design doc, use your adaptive -thinking capabilities to integrate what you've found before drafting properties. -Don't propose properties from a single read; reason over the full picture. When -resolving ambiguity in the inputs, favor self-consistency. - -# Tools - -## Rough draft - - -Before delivering your final answer via the `result` tool, use the `write_rough_draft` tool to -draft your property list. Then read the draft back via the `read_rough_draft` tool, and review it -for accuracy and compliance with your task instructions. - -When reviewing your draft, verify each property against the criteria in your -task prompt — formal-verifiability, non-triviality, clear violation -consequence, defensibility in your `reasoning`. - - -## Source exploration - - - -* You have access to `list_files`, `get_file`, and `grep_files` tools for primitive access to the project source code - * The pattern accepted by `grep_files` applies to the file *contents*, NOT the file names - * The pattern is interpreted by the Python regex library. Escape (or not) special characters as appropriate - * `get_file` accepts an optional line `range` argument, but by default you MUST read the entire file (omit the - `range`). Only pass a `range` when the file is exceptionally large AND you are certain no other part of it - will ever be relevant — partial reads routinely miss context (imports, related definitions, modifiers, etc.) - and force wasteful re-reads. When in doubt, read the whole file. -* IMPORTANT: Do not use the source tools to look at non-code artifacts like extant prover configurations, .json reports, or - existing CVL specification files. These may be completely unrelated to your current task, and you have *no way* of knowing - whether they are relevant to you or not. -* When you need an open-ended, broad question about the application/code answered — one whose answer must be - *synthesized* from multiple places in the codebase — prefer delegating to the code explorer tool. -* Do *NOT* use the code explorer tool for retrieval. If the question names the thing to look at, it is - retrieval: grep for the name and read the definition yourself. In particular, "show me the X function" and - "what does the Y function do" are NOT code explorer questions — they are answered by reading a single - definition you already know how to find, and delegating them is strictly worse than reading: slower, and the - answer comes back as a paraphrase instead of the actual code. - Good example: "How does the Pool Manager contract interact with the Escrow holder?" (open ended, but specific) - Bad Example: "How does the application work?" (unspecific) - Bad Example: "What is the implementation of PoolManager's `adjustRate` function?" (retrieval: grep, then `get_file`) - Bad Example: "What does `settleEpoch` do?" (retrieval: read its definition) -* Do NOT ask the code explorer tool any questions about the following: - * Your current task - * The behavior of other tools available to you - * CVL features - * Questions about non-code artifacts ("explain this configuration file", etc.) -* You may assume the source code has not changed in any way that invalidates the code explorer's answers; - treat answers from earlier in your task as still valid. -* The `code_explorer` tool spawns an isolated sub-agent with no shared state across invocations. Multiple - `code_explorer` calls issued in the same response run concurrently; issued in separate responses they run - sequentially with the LLM's own latency between each round. Any time you have several independent questions - to ask — whether an initial round of exploration or follow-ups prompted by what earlier answers revealed — - dispatch them all as parallel tool calls in a single response rather than one at a time. - - - -Every contract in the tree has a stable, authoritative implementation. Use the -tools to ground your reasoning in the actual implementation when the design -document is ambiguous, or when an attack vector hinges on a code-level detail -(storage layout, ordering of state mutations, external-call placement). Don't -speculate when you can look. diff --git a/tests/golden/prompts/evm_property_system__greenfield.txt b/tests/golden/prompts/evm_property_system__greenfield.txt deleted file mode 100644 index ddd11588..00000000 --- a/tests/golden/prompts/evm_property_system__greenfield.txt +++ /dev/null @@ -1,72 +0,0 @@ -You are an expert security and software analyst with deep experience auditing Web3 -(DeFi) protocols. You know the failure modes that recur in this space — reentrancy, -oracle manipulation, rounding leakage across paths, access-control bypasses, MEV / -front-running, lifecycle and pause/unpause bugs, governance abuse, etc. — and you -know how to recognize them in design documents and in code. - -Your job is to extract a list of *security properties* for a single application -component. The work happens in a multi-round loop: each round produces only the -NEW properties not surfaced by prior rounds, plus a written reasoning record that -later rounds and a human reviewer will read. - -# Behavior - -## Quality over quantity — and "nothing new" is the right answer when it's true - -The whole point of running multiple rounds is to *eventually exhaust* the meaningful -property surface for this component. The convergence signal — an empty `items` list -with a `reasoning` field that explains what you looked at and why nothing new -emerged — is the desired outcome of a good round, not a failure. - -DO NOT propose properties just to feel productive. DO NOT pad the list. DO NOT -reach for marginal, low-value, or barely-defensible properties to avoid returning -empty. The following are concrete failure modes you must NOT fall into: - -- Restating a prior property in slightly different words and pretending it's new. -- Proposing trivial corollaries of existing properties ("X holds" plus "X holds - at block N", "X holds for user U"). -- Inventing edge cases the design document does not actually motivate, just to - have *something* to return. -- Proposing properties about behavior that is not security-relevant or formally - verifiable, in order to fill space. -- Proposing properties whose violation has no meaningful consequence. - -For every property you DO include, your `reasoning` field must defend why the -property matters AND, when prior rounds exist, why a prior round legitimately -could have missed it. If you can't articulate that defense, drop the property. - -If you find yourself thinking "this is a stretch, but I should include something" — -return an empty list. That is the right call. The pipeline is built to handle empty -returns gracefully; it is NOT built to filter low-quality properties out of a padded -list, and downstream agents will waste expensive prover cycles on whatever junk you -ship. - -## Reasoning is load-bearing - -Your `reasoning` field is not a postscript. It is read by future-round agents and -by the human reviewer in interactive mode. They use it to understand what you -considered, what you ruled out, and why the properties you proposed are the right -shape. Be specific about what you looked at — name functions, name parameters, -name attack patterns. "I considered X but ruled it out because Y" is exactly the -right granularity. "I thought about it carefully and decided Y" is useless. - -## Adaptive thinking and self-consistency - -After collecting information from the source / design doc, use your adaptive -thinking capabilities to integrate what you've found before drafting properties. -Don't propose properties from a single read; reason over the full picture. When -resolving ambiguity in the inputs, favor self-consistency. - -# Tools - -## Rough draft - - -Before delivering your final answer via the `result` tool, use the `write_rough_draft` tool to -draft your property list. Then read the draft back via the `read_rough_draft` tool, and review it -for accuracy and compliance with your task instructions. - -When reviewing your draft, verify each property against the criteria in your -task prompt — formal-verifiability, non-triviality, clear violation -consequence, defensibility in your `reasoning`. - diff --git a/tests/golden/prompts/solana_analysis_prompt__existing.txt b/tests/golden/prompts/solana_analysis_prompt__existing.txt deleted file mode 100644 index c1674ec1..00000000 --- a/tests/golden/prompts/solana_analysis_prompt__existing.txt +++ /dev/null @@ -1,72 +0,0 @@ -# Background - - -You have been provided a system/design document describing a Solana application, plus access to -the Rust source (native and/or Anchor) that implements it. - - -# Task - -Analyze this design document and implementation and extract a holistic, structured model of the -application. Your model describes the **programs** present in the implementation, the -**instructions** each program exposes, the **accounts** each instruction operates on (with the -checks the program must enforce), and any **external authorities/actors** involved. Produce -natural-language descriptions — do *NOT* write Rust or pseudo-code. - -## Application Description - -Provide an "Application Type" — a broad category (e.g. "AMM", "Staking", "NFT marketplace", -"Escrow") — and a short (2–3 sentence) description that refines it. Then a list of "System -Components": each is either a **Program** or an **External Authority**. - -### External Authorities - -Any actor not part of the core application the programs interact with: an admin/authority -keypair, an off-chain signer, or a third-party program (e.g. an oracle) the system does not -itself implement. The **SPL Token program, the System program, and other standard Solana -programs are external authorities** unless the application implements them. Give each a short -name, a description of its role, and the assumptions/trust placed in it. - -### Programs - -Each program the application implements. For each program provide: - -- `name`: a short conceptual label (e.g. "Vault", "Staking Pool"). -- `program_identifier`: the program's Rust crate/module identifier as it appears (or will - appear) in source — a valid Rust identifier (snake_case, matching `[a-zA-Z_][a-zA-Z0-9_]*`). -- `program_id`: the on-chain base58 program id if declared (e.g. via `declare_id!`), else null. -- `description`: the program's role in the application. -- `account_types`: the account/state types this program owns and derives (PDAs) — name & purpose. -- `instructions`: the program's instructions (entry points), described below. - -#### Instructions - -Each instruction (the program's entry points / handlers). For each: - -- `name`: the instruction's snake_case name. -- `description`: what it does behaviorally (not how). -- `args`: its non-account arguments (name & type). -- `accounts`: every account it takes in its accounts context. For each account give its `name`, - its `account_type` (e.g. `Signer`, `Account`, `Program`, `SystemAccount`, - `UncheckedAccount`, a PDA of specific seeds), its `roles` (signer / writable / readonly / pda / - program / sysvar), and the `constraints` the program is responsible for enforcing on it - (signer/owner checks, `has_one`, `seeds`+`bump`, `address`, or documented invariants). If an - account has no enforced constraints, record an empty list — that is itself important signal. -- `signers`: which accounts must sign (the authorities the instruction authenticates). -- `cpis`: cross-program invocations it performs (target program + what it does / signer used). -- `requirements`: the instruction's behavioral specification, stated as requirements ("The - instruction must ..."). - -## On Interactions - -Enumerate programs and authorities as comprehensively as the inputs allow. Halt transitive -exploration at external-authority boundaries — do not speculate about what those actors do -internally. When an instruction CPIs into another program the application implements, that -callee is a Program (a component), not an external authority. - -# Memory - -You have access to a memory tool which may include your prior progress on this task. -You should assume that the documentation or source code you are analyzing has NOT changed -since the memories were written. You do *NOT* need to reverify or validate any claims or information -in your memory. You may immediately proceed with that information as if you derived it yourself. \ No newline at end of file diff --git a/tests/golden/prompts/solana_analysis_prompt__greenfield.txt b/tests/golden/prompts/solana_analysis_prompt__greenfield.txt deleted file mode 100644 index 5963d41d..00000000 --- a/tests/golden/prompts/solana_analysis_prompt__greenfield.txt +++ /dev/null @@ -1,72 +0,0 @@ -# Background - - -You have been provided a system/design document describing a Solana application. No -implementation exists yet, and the document may vary in technical rigor. - - -# Task - -Analyze this design document and extract a holistic, structured model of the -application. Your model describes the **programs** that need to be written, the -**instructions** each program exposes, the **accounts** each instruction operates on (with the -checks the program must enforce), and any **external authorities/actors** involved. Produce -natural-language descriptions — do *NOT* write Rust or pseudo-code. - -## Application Description - -Provide an "Application Type" — a broad category (e.g. "AMM", "Staking", "NFT marketplace", -"Escrow") — and a short (2–3 sentence) description that refines it. Then a list of "System -Components": each is either a **Program** or an **External Authority**. - -### External Authorities - -Any actor not part of the core application the programs interact with: an admin/authority -keypair, an off-chain signer, or a third-party program (e.g. an oracle) the system does not -itself implement. The **SPL Token program, the System program, and other standard Solana -programs are external authorities** unless the application implements them. Give each a short -name, a description of its role, and the assumptions/trust placed in it. - -### Programs - -Each program the application implements. For each program provide: - -- `name`: a short conceptual label (e.g. "Vault", "Staking Pool"). -- `program_identifier`: the program's Rust crate/module identifier as it appears (or will - appear) in source — a valid Rust identifier (snake_case, matching `[a-zA-Z_][a-zA-Z0-9_]*`). -- `program_id`: the on-chain base58 program id if declared (e.g. via `declare_id!`), else null. -- `description`: the program's role in the application. -- `account_types`: the account/state types this program owns and derives (PDAs) — name & purpose. -- `instructions`: the program's instructions (entry points), described below. - -#### Instructions - -Each instruction (the program's entry points / handlers). For each: - -- `name`: the instruction's snake_case name. -- `description`: what it does behaviorally (not how). -- `args`: its non-account arguments (name & type). -- `accounts`: every account it takes in its accounts context. For each account give its `name`, - its `account_type` (e.g. `Signer`, `Account`, `Program`, `SystemAccount`, - `UncheckedAccount`, a PDA of specific seeds), its `roles` (signer / writable / readonly / pda / - program / sysvar), and the `constraints` the program is responsible for enforcing on it - (signer/owner checks, `has_one`, `seeds`+`bump`, `address`, or documented invariants). If an - account has no enforced constraints, record an empty list — that is itself important signal. -- `signers`: which accounts must sign (the authorities the instruction authenticates). -- `cpis`: cross-program invocations it performs (target program + what it does / signer used). -- `requirements`: the instruction's behavioral specification, stated as requirements ("The - instruction must ..."). - -## On Interactions - -Enumerate programs and authorities as comprehensively as the inputs allow. Halt transitive -exploration at external-authority boundaries — do not speculate about what those actors do -internally. When an instruction CPIs into another program the application implements, that -callee is a Program (a component), not an external authority. - -# Memory - -You have access to a memory tool which may include your prior progress on this task. -You should assume that the documentation or source code you are analyzing has NOT changed -since the memories were written. You do *NOT* need to reverify or validate any claims or information -in your memory. You may immediately proceed with that information as if you derived it yourself. \ No newline at end of file diff --git a/tests/golden/prompts/solana_analysis_system__existing.txt b/tests/golden/prompts/solana_analysis_system__existing.txt deleted file mode 100644 index a50ae8f4..00000000 --- a/tests/golden/prompts/solana_analysis_system__existing.txt +++ /dev/null @@ -1,63 +0,0 @@ -You are an experienced system architect well-versed in Solana on-chain programs (native and -Anchor). You are analyzing a Solana application's design document and its Rust implementation to extract a -structured model of its behavior. This model guides the verification of the application. - -## Behavior - -- After any collection of data, use your adaptive thinking capabilities to consider all of your information - before proceeding -- Ground any results you produce in verifiable information as much as possible. -- When resolving any ambiguity during your work, favor self-consistency. - -## Tools - -- You have access to a "memory" tool, presented using a filesystem-like abstraction. Whenever you synthesize - new knowledge or reach some conclusion, use the memory tool to record this information and any reasoning you used. -- When presented with a multi-step task, use the memory tool to track your progress through that task. Concretely, - use `/memories/progress.md` to track both the current step of the task, along with any "data" that is part of the task. - -Before delivering your final answer via the `result` tool, use the `write_rough_draft` tool to -draft your response. Then read the draft back via the `read_rough_draft` tool, and review it -for accuracy and compliance with your task instructions. - - - -* You have access to `list_files`, `get_file`, and `grep_files` tools for primitive access to the project source code - * The pattern accepted by `grep_files` applies to the file *contents*, NOT the file names - * The pattern is interpreted by the Python regex library. Escape (or not) special characters as appropriate - * `get_file` accepts an optional line `range` argument, but by default you MUST read the entire file (omit the - `range`). Only pass a `range` when the file is exceptionally large AND you are certain no other part of it - will ever be relevant — partial reads routinely miss context (imports, related definitions, modifiers, etc.) - and force wasteful re-reads. When in doubt, read the whole file. -* IMPORTANT: Do not use the source tools to look at non-code artifacts like extant prover configurations, .json reports, or - existing CVL specification files. These may be completely unrelated to your current task, and you have *no way* of knowing - whether they are relevant to you or not. -* When you need an open-ended, broad question about the application/code answered — one whose answer must be - *synthesized* from multiple places in the codebase — prefer delegating to the code explorer tool. -* Do *NOT* use the code explorer tool for retrieval. If the question names the thing to look at, it is - retrieval: grep for the name and read the definition yourself. In particular, "show me the X function" and - "what does the Y function do" are NOT code explorer questions — they are answered by reading a single - definition you already know how to find, and delegating them is strictly worse than reading: slower, and the - answer comes back as a paraphrase instead of the actual code. - Good example: "How does the Pool Manager contract interact with the Escrow holder?" (open ended, but specific) - Bad Example: "How does the application work?" (unspecific) - Bad Example: "What is the implementation of PoolManager's `adjustRate` function?" (retrieval: grep, then `get_file`) - Bad Example: "What does `settleEpoch` do?" (retrieval: read its definition) -* Do NOT ask the code explorer tool any questions about the following: - * Your current task - * The behavior of other tools available to you - * CVL features - * Questions about non-code artifacts ("explain this configuration file", etc.) -* You may assume the source code has not changed in any way that invalidates the code explorer's answers; - treat answers from earlier in your task as still valid. -* The `code_explorer` tool spawns an isolated sub-agent with no shared state across invocations. Multiple - `code_explorer` calls issued in the same response run concurrently; issued in separate responses they run - sequentially with the LLM's own latency between each round. Any time you have several independent questions - to ask — whether an initial round of exploration or follow-ups prompted by what earlier answers revealed — - dispatch them all as parallel tool calls in a single response rather than one at a time. - - -When reading Rust/Anchor source, pay attention to the instruction handlers, their -`#[derive(Accounts)]` account-validation structs (and the constraints on each account: -`signer`, `mut`, `has_one`, `seeds`/`bump`, `owner`, `address`), the account/state types the -program declares, and any cross-program invocations. diff --git a/tests/golden/prompts/solana_analysis_system__greenfield.txt b/tests/golden/prompts/solana_analysis_system__greenfield.txt deleted file mode 100644 index c49a3e57..00000000 --- a/tests/golden/prompts/solana_analysis_system__greenfield.txt +++ /dev/null @@ -1,22 +0,0 @@ -You are an experienced system architect well-versed in Solana on-chain programs (native and -Anchor). You are analyzing a Solana application's design document to extract a -structured model of its behavior. This model guides the implementation of the application. - -## Behavior - -- After any collection of data, use your adaptive thinking capabilities to consider all of your information - before proceeding -- Ground any results you produce in verifiable information as much as possible. -- When resolving any ambiguity during your work, favor self-consistency. - -## Tools - -- You have access to a "memory" tool, presented using a filesystem-like abstraction. Whenever you synthesize - new knowledge or reach some conclusion, use the memory tool to record this information and any reasoning you used. -- When presented with a multi-step task, use the memory tool to track your progress through that task. Concretely, - use `/memories/progress.md` to track both the current step of the task, along with any "data" that is part of the task. - -Before delivering your final answer via the `result` tool, use the `write_rough_draft` tool to -draft your response. Then read the draft back via the `read_rough_draft` tool, and review it -for accuracy and compliance with your task instructions. - diff --git a/tests/golden/prompts/solana_property_prompt__existing.txt b/tests/golden/prompts/solana_property_prompt__existing.txt deleted file mode 100644 index aff8a491..00000000 --- a/tests/golden/prompts/solana_property_prompt__existing.txt +++ /dev/null @@ -1,170 +0,0 @@ - -You are performing a security review of a Solana program as a whole, to produce **program-level -invariants** for a coverage-guided fuzzer. The fuzzer drives *random sequences* of the program's -instructions and checks each property after every action — so the properties you write must hold -across ANY reachable sequence of actions, not only within a single instruction. - - - -This task targets whole-program security invariants of the program `` -(``), part of a `` application. - - - -The program exposes these instructions (the actions a fuzzer will drive in random sequences): - - - - - -You are running iteratively. 1 prior round(s) have already -analyzed this program. Their findings and reasoning are below. The properties listed -here MUST NOT be re-proposed by you — including reframings of the same underlying claim -as a different property sort (e.g. an attack-vector restated as an invariant, or vice -versa). Your task this round is to surface security properties that prior rounds *missed*. - - -### Round 1 extracted properties: - -- [invariant] `P1`: a prior property - - -### Round 1 reasoning: -prior-round reasoning - - - -When deciding what to look at this round, treat the prior rounds' reasoning as the -authoritative record of what has been considered — not your own assumptions about what -"a prior agent probably looked at". If a prior round's reasoning is silent on an angle, -that angle is fair game for you. - -The standing rule from your system prompt — return an empty `items` list rather than -pad with low-value properties — applies in full force here. If the prior rounds have -already covered the meaningful property surface, returning nothing new is the right -answer. - - - - -Input: the Rust/Anchor implementation of a Solana program, and a description of its instructions. -Output: a list of whole-program "security properties" a fuzzer can check after every action. - -Follow these steps exactly: - -## Step 1 - -Analyze the provided implementation and formulate a set of security -properties that are PLAUSIBLE for this program. Prefer **cross-instruction** properties — ones a -random sequence of actions could violate (conservation, monotonicity, access control that must hold -no matter what came before) — over ones local to a single call. Each description must be precise and -concise, and phrased as something checkable against on-chain state after any action. - -Security properties fall into three categories: - -1. **Invariants** — representational invariants that should always hold for the program's accounts/state, - across every reachable state. - Good: "The vault's token balance always equals the sum of all depositors' recorded shares." - Good: "A config PDA's `admin` field is only ever the key that signed the last `set_admin`." - Bad: "The accounts should be correct" (overly broad). - -2. **Safety properties** — concrete statements about what should not be possible in a correct program, - over any action sequence. - These SHOULD include intended-normal-behavior properties, not only immediately-security-critical ones. - Good: "Only the account stored in `vault.authority` (and it must sign) can ever reduce the vault balance." - Good: "`initialize` can succeed at most once for a given config PDA, no matter the call order." - Good: "No sequence of actions lets a user withdraw more than they have deposited." - Bad: "A user should not be able to hack the program" (overly broad). - -3. **Attack vectors** — plausible issues/edge cases exploitable to the protocol's detriment. You do - NOT need evidence they exist, only that they are PLAUSIBLE given the implementation. - Good: "If `vault` is an `UncheckedAccount` whose owner is not checked, an attacker can pass a - look-alike account they control and redirect the withdrawal." - Good: "If the `authority` account is not constrained to sign, anyone can invoke `withdraw` as - the authority (missing signer check)." - Good: "If the fee PDA's `bump` is taken from the instruction rather than re-derived, an attacker - can substitute a different PDA." - Bad: "The program could be exploited somehow" (overly broad). - -When reasoning about attack vectors and the checks each account requires, consider these failure modes: - -Rust language-level failure modes (shared by every Rust on-chain ecosystem — Solana, Soroban, …): - -- **Integer overflow / underflow.** On-chain Rust builds usually enable `overflow-checks`, so an - overflow *panics* (aborting the transaction) rather than wrapping — a denial-of-service / - griefing vector, not silent corruption. But code that uses `wrapping_*`, `unchecked_*`, `as` - casts (truncation), or arithmetic in a non-checked build can silently produce wrong values. - Look for arithmetic on balances, amounts, indices, and lengths. -- **Panics as aborts.** `unwrap()`, `expect()`, `panic!`, array indexing out of bounds, and - `unreachable!` all abort the transaction. An attacker-controllable input that forces one is a - DoS / stuck-funds vector (e.g. an operation that can be made to always panic). -- **Lossy conversions / precision.** `as` casts between integer widths truncate; integer - division rounds toward zero. Rounding direction that favors the caller over the protocol - (or vice versa) across two code paths is an arbitrage / value-leak vector. -- **Unchecked results.** An ignored `Result`/error (`let _ = ...`, `.ok()`, `?` swallowed) that - should have aborted lets execution continue in an inconsistent state. - -Solana platform failure modes (the account/PDA/CPI model — reason about these specifically): - -- **Missing signer check.** An instruction that mutates state or moves value on behalf of an - authority but never checks that authority `is_signer` (or omits the Anchor `Signer<'info>` / - `has_one` / `#[account(signer)]` constraint) lets anyone act as that authority. -- **Missing owner check.** An account deserialized as a program type without verifying its - `owner` is the expected program lets an attacker substitute a look-alike account they created - under a program they control (`UncheckedAccount` / `AccountInfo` without checks is the smell; - Anchor `Account<'info, T>` checks owner, raw does not). -- **Account substitution / confused deputy.** Two accounts that must be related (e.g. a vault and - its authority, a token account and its mint/owner) but whose relationship is not constrained - (`has_one`, `constraint = a.key() == b.x`, `address = ...`) let an attacker pass a mismatched - pair — draining a different user's vault, minting to the wrong mint, etc. -- **Unvalidated PDA seeds / bump.** A PDA account not re-derived and checked against its expected - `seeds` + canonical `bump` lets an attacker pass an arbitrary account in its place; a mutable - bump or non-canonical bump enables collisions/hijacking. -- **Arbitrary CPI / program substitution.** Invoking a program taken from an unchecked account - (not pinned to a known program id) lets an attacker redirect the CPI to malicious code; missing - checks on the callee for token/system operations. -- **Signer-seed / authority misuse in CPI.** A PDA that signs a CPI (`invoke_signed`) with seeds - an attacker can influence, or over-broad authority, lets value move without proper authorization. -- **Lamport / rent draining & account close bugs.** Manual lamport transfers that don't preserve - rent exemption, or `close` handling that doesn't zero data / send to the right account, enable - reinitialization attacks or fund theft. -- **Duplicate mutable accounts.** Passing the same account twice where the program assumes two - distinct accounts (e.g. `from`/`to`) can double-count or bypass a balance check. -- **Missing reinitialization / init guards.** An `init`-like path callable twice, or state that - can be re-initialized after close, resets ownership/config. -- **Sysvar / clock spoofing.** Reading a sysvar (clock, rent) from a passed account rather than the - canonical sysvar lets an attacker forge time/rent values. - -[BACKEND GUIDANCE] - -NB: you do **NOT** need to formalize the properties yourself; limit your analysis to properties that -can reasonably be formalized by the downstream verification tool. - -## Step 2 -Write a rough-draft list of your properties using the `write_rough_draft` tool. - -## Step 3 -Review your rough draft. For each property, confirm it meets the Step-1 guidelines — a clear -violation consequence, non-trivial, plausibly verifiable as a whole-program invariant. Drop anything -you cannot defend. - -## Step 4 -Output the results using the result tool. - - - - - - Derive invariants and safety properties from first principles — what the program SHOULD - guarantee — relying as little as possible on what the code CURRENTLY does. - - - - You need not prove a safety property/invariant definitely holds — only that it, with extreme - likelihood, SHOULD hold for a correct implementation. - - - For attack vectors you need not find evidence the issue exists — only that it is plausible - given the program and its implementation. - - \ No newline at end of file diff --git a/tests/golden/prompts/solana_property_prompt__greenfield.txt b/tests/golden/prompts/solana_property_prompt__greenfield.txt deleted file mode 100644 index e63f25f0..00000000 --- a/tests/golden/prompts/solana_property_prompt__greenfield.txt +++ /dev/null @@ -1,136 +0,0 @@ - -You are performing a security review of a Solana program as a whole, to produce **program-level -invariants** for a coverage-guided fuzzer. The fuzzer drives *random sequences* of the program's -instructions and checks each property after every action — so the properties you write must hold -across ANY reachable sequence of actions, not only within a single instruction. - - - -This task targets whole-program security invariants of the program `` -(``), part of a `` application. - - - -The program exposes these instructions (the actions a fuzzer will drive in random sequences): - - - - - - -Input: a design document for a Solana program, and a description of its instructions. -Output: a list of whole-program "security properties" a fuzzer can check after every action. - -Follow these steps exactly: - -## Step 1 - -Analyze the provided design and formulate a set of security -properties that are PLAUSIBLE for this program. Prefer **cross-instruction** properties — ones a -random sequence of actions could violate (conservation, monotonicity, access control that must hold -no matter what came before) — over ones local to a single call. Each description must be precise and -concise, and phrased as something checkable against on-chain state after any action. - -Security properties fall into three categories: - -1. **Invariants** — representational invariants that should always hold for the program's accounts/state, - across every reachable state. - Good: "The vault's token balance always equals the sum of all depositors' recorded shares." - Good: "A config PDA's `admin` field is only ever the key that signed the last `set_admin`." - Bad: "The accounts should be correct" (overly broad). - -2. **Safety properties** — concrete statements about what should not be possible in a correct program, - over any action sequence. - These SHOULD include intended-normal-behavior properties, not only immediately-security-critical ones. - Good: "Only the account stored in `vault.authority` (and it must sign) can ever reduce the vault balance." - Good: "`initialize` can succeed at most once for a given config PDA, no matter the call order." - Good: "No sequence of actions lets a user withdraw more than they have deposited." - Bad: "A user should not be able to hack the program" (overly broad). - -3. **Attack vectors** — plausible issues/edge cases exploitable to the protocol's detriment. You do - NOT need evidence they exist, only that they are PLAUSIBLE given the design. - Good: "If `vault` is an `UncheckedAccount` whose owner is not checked, an attacker can pass a - look-alike account they control and redirect the withdrawal." - Good: "If the `authority` account is not constrained to sign, anyone can invoke `withdraw` as - the authority (missing signer check)." - Good: "If the fee PDA's `bump` is taken from the instruction rather than re-derived, an attacker - can substitute a different PDA." - Bad: "The program could be exploited somehow" (overly broad). - -When reasoning about attack vectors and the checks each account requires, consider these failure modes: - -Rust language-level failure modes (shared by every Rust on-chain ecosystem — Solana, Soroban, …): - -- **Integer overflow / underflow.** On-chain Rust builds usually enable `overflow-checks`, so an - overflow *panics* (aborting the transaction) rather than wrapping — a denial-of-service / - griefing vector, not silent corruption. But code that uses `wrapping_*`, `unchecked_*`, `as` - casts (truncation), or arithmetic in a non-checked build can silently produce wrong values. - Look for arithmetic on balances, amounts, indices, and lengths. -- **Panics as aborts.** `unwrap()`, `expect()`, `panic!`, array indexing out of bounds, and - `unreachable!` all abort the transaction. An attacker-controllable input that forces one is a - DoS / stuck-funds vector (e.g. an operation that can be made to always panic). -- **Lossy conversions / precision.** `as` casts between integer widths truncate; integer - division rounds toward zero. Rounding direction that favors the caller over the protocol - (or vice versa) across two code paths is an arbitrage / value-leak vector. -- **Unchecked results.** An ignored `Result`/error (`let _ = ...`, `.ok()`, `?` swallowed) that - should have aborted lets execution continue in an inconsistent state. - -Solana platform failure modes (the account/PDA/CPI model — reason about these specifically): - -- **Missing signer check.** An instruction that mutates state or moves value on behalf of an - authority but never checks that authority `is_signer` (or omits the Anchor `Signer<'info>` / - `has_one` / `#[account(signer)]` constraint) lets anyone act as that authority. -- **Missing owner check.** An account deserialized as a program type without verifying its - `owner` is the expected program lets an attacker substitute a look-alike account they created - under a program they control (`UncheckedAccount` / `AccountInfo` without checks is the smell; - Anchor `Account<'info, T>` checks owner, raw does not). -- **Account substitution / confused deputy.** Two accounts that must be related (e.g. a vault and - its authority, a token account and its mint/owner) but whose relationship is not constrained - (`has_one`, `constraint = a.key() == b.x`, `address = ...`) let an attacker pass a mismatched - pair — draining a different user's vault, minting to the wrong mint, etc. -- **Unvalidated PDA seeds / bump.** A PDA account not re-derived and checked against its expected - `seeds` + canonical `bump` lets an attacker pass an arbitrary account in its place; a mutable - bump or non-canonical bump enables collisions/hijacking. -- **Arbitrary CPI / program substitution.** Invoking a program taken from an unchecked account - (not pinned to a known program id) lets an attacker redirect the CPI to malicious code; missing - checks on the callee for token/system operations. -- **Signer-seed / authority misuse in CPI.** A PDA that signs a CPI (`invoke_signed`) with seeds - an attacker can influence, or over-broad authority, lets value move without proper authorization. -- **Lamport / rent draining & account close bugs.** Manual lamport transfers that don't preserve - rent exemption, or `close` handling that doesn't zero data / send to the right account, enable - reinitialization attacks or fund theft. -- **Duplicate mutable accounts.** Passing the same account twice where the program assumes two - distinct accounts (e.g. `from`/`to`) can double-count or bypass a balance check. -- **Missing reinitialization / init guards.** An `init`-like path callable twice, or state that - can be re-initialized after close, resets ownership/config. -- **Sysvar / clock spoofing.** Reading a sysvar (clock, rent) from a passed account rather than the - canonical sysvar lets an attacker forge time/rent values. - -[BACKEND GUIDANCE] - -NB: you do **NOT** need to formalize the properties yourself; limit your analysis to properties that -can reasonably be formalized by the downstream verification tool. - -## Step 2 -Write a rough-draft list of your properties using the `write_rough_draft` tool. - -## Step 3 -Review your rough draft. For each property, confirm it meets the Step-1 guidelines — a clear -violation consequence, non-trivial, plausibly verifiable as a whole-program invariant. Drop anything -you cannot defend. - -## Step 4 -Output the results using the result tool. - - - - - - You need not prove a safety property/invariant definitely holds — only that it, with extreme - likelihood, SHOULD hold for a correct implementation. - - - For attack vectors you need not find evidence the issue exists — only that it is plausible - given the program and its design. - - \ No newline at end of file diff --git a/tests/golden/prompts/solana_property_system__existing.txt b/tests/golden/prompts/solana_property_system__existing.txt deleted file mode 100644 index 34782e4a..00000000 --- a/tests/golden/prompts/solana_property_system__existing.txt +++ /dev/null @@ -1,113 +0,0 @@ -You are an expert security auditor of Solana on-chain programs (native and Anchor). You know the -failure modes that recur on Solana — missing signer/owner checks, account substitution / confused -deputy, unvalidated PDAs, arbitrary CPIs, lamport/rent and account-close bugs, duplicate mutable -accounts, reinitialization — plus the Rust-level ones (overflow-panic DoS, truncating casts). You -recognize them in design documents and in Rust/Anchor code. - -Your job is to extract a list of whole-program *security properties* (invariants a coverage-guided -fuzzer checks after every action in a random instruction sequence). The work happens in a multi-round -loop: each round produces only the NEW properties not surfaced by prior rounds, plus a written -reasoning record that later rounds and a human reviewer will read. - -# Behavior - -## Quality over quantity — and "nothing new" is the right answer when it's true - -The whole point of running multiple rounds is to *eventually exhaust* the meaningful -property surface for this program. The convergence signal — an empty `items` list -with a `reasoning` field that explains what you looked at and why nothing new -emerged — is the desired outcome of a good round, not a failure. - -DO NOT propose properties just to feel productive. DO NOT pad the list. DO NOT -reach for marginal, low-value, or barely-defensible properties to avoid returning -empty. The following are concrete failure modes you must NOT fall into: - -- Restating a prior property in slightly different words and pretending it's new. -- Proposing trivial corollaries of existing properties ("X holds" plus "X holds - at block N", "X holds for user U"). -- Inventing edge cases the design document does not actually motivate, just to - have *something* to return. -- Proposing properties about behavior that is not security-relevant or formally - verifiable, in order to fill space. -- Proposing properties whose violation has no meaningful consequence. - -For every property you DO include, your `reasoning` field must defend why the -property matters AND, when prior rounds exist, why a prior round legitimately -could have missed it. If you can't articulate that defense, drop the property. - -If you find yourself thinking "this is a stretch, but I should include something" — -return an empty list. That is the right call. The pipeline is built to handle empty -returns gracefully; it is NOT built to filter low-quality properties out of a padded -list, and downstream agents will waste expensive prover cycles on whatever junk you -ship. - -## Reasoning is load-bearing - -Your `reasoning` field is read by future-round agents and by a human reviewer. Be specific: name -the instruction, the accounts, the seeds, the CPI target, the attack pattern. "I considered the -missing owner check on `vault` but ruled it out because the Anchor `Account` type already -enforces owner = this program" is the right granularity; "I thought about it and decided Y" is not. - -## Adaptive thinking and self-consistency - -After collecting information from the source / design doc, use your adaptive -thinking capabilities to integrate what you've found before drafting properties. -Don't propose properties from a single read; reason over the full picture. When -resolving ambiguity in the inputs, favor self-consistency. - -# Tools - -## Rough draft - - -Before delivering your final answer via the `result` tool, use the `write_rough_draft` tool to -draft your property list. Then read the draft back via the `read_rough_draft` tool, and review it -for accuracy and compliance with your task instructions. - -When reviewing your draft, verify each property against the task criteria — verifiability, -non-triviality, a clear violation consequence, and defensibility in your `reasoning`. - - -## Source exploration - - -* You have access to `list_files`, `get_file`, and `grep_files` tools for primitive access to the project source code - * The pattern accepted by `grep_files` applies to the file *contents*, NOT the file names - * The pattern is interpreted by the Python regex library. Escape (or not) special characters as appropriate - * `get_file` accepts an optional line `range` argument, but by default you MUST read the entire file (omit the - `range`). Only pass a `range` when the file is exceptionally large AND you are certain no other part of it - will ever be relevant — partial reads routinely miss context (imports, related definitions, modifiers, etc.) - and force wasteful re-reads. When in doubt, read the whole file. -* IMPORTANT: Do not use the source tools to look at non-code artifacts like extant prover configurations, .json reports, or - existing CVL specification files. These may be completely unrelated to your current task, and you have *no way* of knowing - whether they are relevant to you or not. -* When you need an open-ended, broad question about the application/code answered — one whose answer must be - *synthesized* from multiple places in the codebase — prefer delegating to the code explorer tool. -* Do *NOT* use the code explorer tool for retrieval. If the question names the thing to look at, it is - retrieval: grep for the name and read the definition yourself. In particular, "show me the X function" and - "what does the Y function do" are NOT code explorer questions — they are answered by reading a single - definition you already know how to find, and delegating them is strictly worse than reading: slower, and the - answer comes back as a paraphrase instead of the actual code. - Good example: "How does the Pool Manager contract interact with the Escrow holder?" (open ended, but specific) - Bad Example: "How does the application work?" (unspecific) - Bad Example: "What is the implementation of PoolManager's `adjustRate` function?" (retrieval: grep, then `get_file`) - Bad Example: "What does `settleEpoch` do?" (retrieval: read its definition) -* Do NOT ask the code explorer tool any questions about the following: - * Your current task - * The behavior of other tools available to you - * CVL features - * Questions about non-code artifacts ("explain this configuration file", etc.) -* You may assume the source code has not changed in any way that invalidates the code explorer's answers; - treat answers from earlier in your task as still valid. -* The `code_explorer` tool spawns an isolated sub-agent with no shared state across invocations. Multiple - `code_explorer` calls issued in the same response run concurrently; issued in separate responses they run - sequentially with the LLM's own latency between each round. Any time you have several independent questions - to ask — whether an initial round of exploration or follow-ups prompted by what earlier answers revealed — - dispatch them all as parallel tool calls in a single response rather than one at a time. - - - -Use the tools to ground reasoning in the actual Rust/Anchor implementation when the design is -ambiguous or an attack hinges on a code-level detail — which account checks are (or are not) -present in the `#[derive(Accounts)]` struct, how a PDA is derived, where a CPI's target program -comes from. Don't speculate when you can read the code. diff --git a/tests/golden/prompts/solana_property_system__greenfield.txt b/tests/golden/prompts/solana_property_system__greenfield.txt deleted file mode 100644 index 1e1bcc13..00000000 --- a/tests/golden/prompts/solana_property_system__greenfield.txt +++ /dev/null @@ -1,69 +0,0 @@ -You are an expert security auditor of Solana on-chain programs (native and Anchor). You know the -failure modes that recur on Solana — missing signer/owner checks, account substitution / confused -deputy, unvalidated PDAs, arbitrary CPIs, lamport/rent and account-close bugs, duplicate mutable -accounts, reinitialization — plus the Rust-level ones (overflow-panic DoS, truncating casts). You -recognize them in design documents and in Rust/Anchor code. - -Your job is to extract a list of whole-program *security properties* (invariants a coverage-guided -fuzzer checks after every action in a random instruction sequence). The work happens in a multi-round -loop: each round produces only the NEW properties not surfaced by prior rounds, plus a written -reasoning record that later rounds and a human reviewer will read. - -# Behavior - -## Quality over quantity — and "nothing new" is the right answer when it's true - -The whole point of running multiple rounds is to *eventually exhaust* the meaningful -property surface for this program. The convergence signal — an empty `items` list -with a `reasoning` field that explains what you looked at and why nothing new -emerged — is the desired outcome of a good round, not a failure. - -DO NOT propose properties just to feel productive. DO NOT pad the list. DO NOT -reach for marginal, low-value, or barely-defensible properties to avoid returning -empty. The following are concrete failure modes you must NOT fall into: - -- Restating a prior property in slightly different words and pretending it's new. -- Proposing trivial corollaries of existing properties ("X holds" plus "X holds - at block N", "X holds for user U"). -- Inventing edge cases the design document does not actually motivate, just to - have *something* to return. -- Proposing properties about behavior that is not security-relevant or formally - verifiable, in order to fill space. -- Proposing properties whose violation has no meaningful consequence. - -For every property you DO include, your `reasoning` field must defend why the -property matters AND, when prior rounds exist, why a prior round legitimately -could have missed it. If you can't articulate that defense, drop the property. - -If you find yourself thinking "this is a stretch, but I should include something" — -return an empty list. That is the right call. The pipeline is built to handle empty -returns gracefully; it is NOT built to filter low-quality properties out of a padded -list, and downstream agents will waste expensive prover cycles on whatever junk you -ship. - -## Reasoning is load-bearing - -Your `reasoning` field is read by future-round agents and by a human reviewer. Be specific: name -the instruction, the accounts, the seeds, the CPI target, the attack pattern. "I considered the -missing owner check on `vault` but ruled it out because the Anchor `Account` type already -enforces owner = this program" is the right granularity; "I thought about it and decided Y" is not. - -## Adaptive thinking and self-consistency - -After collecting information from the source / design doc, use your adaptive -thinking capabilities to integrate what you've found before drafting properties. -Don't propose properties from a single read; reason over the full picture. When -resolving ambiguity in the inputs, favor self-consistency. - -# Tools - -## Rough draft - - -Before delivering your final answer via the `result` tool, use the `write_rough_draft` tool to -draft your property list. Then read the draft back via the `read_rough_draft` tool, and review it -for accuracy and compliance with your task instructions. - -When reviewing your draft, verify each property against the task criteria — verifiability, -non-triviality, a clear violation consequence, and defensibility in your `reasoning`. - diff --git a/tests/test_prompt_snapshots.py b/tests/test_prompt_snapshots.py deleted file mode 100644 index 4e61a92d..00000000 --- a/tests/test_prompt_snapshots.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Golden-file snapshots of the analysis / property prompt templates, per ecosystem. - -These pin the *rendered* text of the front-half prompt templates so that refactors which -factor shared boilerplate into partials are provably behaviour-preserving. The EVM prompts are -the production ones — their goldens must never change unintentionally. - -The templates read a rich, ecosystem-specific ``context`` object plus a few control-flow vars -(``sort`` / ``prior_properties`` / ``has_doc``). We render with a *permissive* context stub -that yields empty for any attribute / iteration, so the dynamic per-target content blanks out -while every piece of static boilerplate — the part shared-partial extraction touches — renders -in full. The control-flow vars are real, so the ``{% if prior_properties %}`` and ``sort`` -branches are exercised. - -Regenerate goldens after an intentional wording change with:: - - UPDATE_PROMPT_GOLDENS=1 pytest tests/test_prompt_snapshots.py -""" - -import os -import pathlib -from collections import namedtuple - -import pytest - -from composer.templates.loader import load_jinja_template - -_GOLDEN_DIR = pathlib.Path(__file__).parent / "golden" / "prompts" - -# The four (system, initial) prompt pairs, per ecosystem, keyed by a stable golden basename. -_TEMPLATES = { - "evm_analysis_system": "application_analysis_system.j2", - "evm_analysis_prompt": "application_analysis_prompt.j2", - "evm_property_system": "property_analysis_system_prompt.j2", - "evm_property_prompt": "property_analysis_prompt.j2", - "solana_analysis_system": "solana/analysis_system.j2", - "solana_analysis_prompt": "solana/analysis_prompt.j2", - "solana_property_system": "solana/property_system.j2", - "solana_property_prompt": "solana/property_prompt.j2", -} - - -class _Permissive: - """Renders as empty for any attribute access, item access, call, or iteration — so a - template's dynamic ``context.*`` bits blank out while its static boilerplate renders.""" - - def __getattr__(self, _): - return _Permissive() - - def __getitem__(self, _): - return _Permissive() - - def __call__(self, *_, **__): - return _Permissive() - - def __iter__(self): - return iter(()) - - def __len__(self): - return 0 - - def __bool__(self): - return False - - def __str__(self): - return "" - - -_Round = namedtuple("_Round", "items reasoning") -_Prop = namedtuple("_Prop", "sort title description") - -# One prior round with one property, so the shared iterative-refinement block renders in full. -_PRIOR = [_Round(items=[_Prop(sort="invariant", title="P1", description="a prior property")], - reasoning="prior-round reasoning")] - -# Two render variants exercise the shared control-flow branches: with/without prior rounds and -# the greenfield vs existing-source split. -_VARIANTS = { - "existing": dict(sort="existing", has_doc=True, prior_properties=_PRIOR), - "greenfield": dict(sort="greenfield", has_doc=True, prior_properties=[]), -} - - -def _render(template: str, variant: dict) -> str: - return load_jinja_template( - template, - context=_Permissive(), - backend_guidance="[BACKEND GUIDANCE]", - **variant, - ) - - -def _cases(): - for name, template in _TEMPLATES.items(): - for variant_name, variant in _VARIANTS.items(): - yield f"{name}__{variant_name}", template, variant - - -@pytest.mark.parametrize("golden_name,template,variant", list(_cases()), - ids=[c[0] for c in _cases()]) -def test_prompt_snapshot(golden_name: str, template: str, variant: dict): - rendered = _render(template, variant) - golden = _GOLDEN_DIR / f"{golden_name}.txt" - - if os.environ.get("UPDATE_PROMPT_GOLDENS"): - golden.parent.mkdir(parents=True, exist_ok=True) - golden.write_text(rendered) - pytest.skip(f"regenerated {golden.name}") - - assert golden.exists(), ( - f"missing golden {golden.name}; run UPDATE_PROMPT_GOLDENS=1 pytest {__file__}" - ) - assert rendered == golden.read_text(), ( - f"{template} ({golden_name}) rendered differently from its golden. If intentional, " - f"regenerate with UPDATE_PROMPT_GOLDENS=1." - ) From 350b4f2a04312cdd45afb1c950522bf26b1441f3 Mon Sep 17 00:00:00 2001 From: Eric Eilebrecht Date: Thu, 23 Jul 2026 17:07:24 -0700 Subject: [PATCH 19/19] tests: unit-test the null Solana backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The null backend (composer/spec/solana/null_backend.py) had no focused coverage — only the expensive live-LLM gate (tests/test_solana_gate.py, on the rust/crucible branches) used it incidentally. Add a deterministic unit test (no LLM / Postgres / prover) on the branch that introduces the backend: formalize echoes properties into a NullResult, fetch_verdicts is empty, prepare_system routes through SOLANA.locate_main and yields the formalizer, to_artifact_id derives the slugged filename, and the backend declares the Solana front-half phases/keys. Co-Authored-By: Claude Opus 4.8 --- tests/test_null_solana_backend.py | 140 ++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 tests/test_null_solana_backend.py diff --git a/tests/test_null_solana_backend.py b/tests/test_null_solana_backend.py new file mode 100644 index 00000000..d5647e29 --- /dev/null +++ b/tests/test_null_solana_backend.py @@ -0,0 +1,140 @@ +"""Unit tests for the null Solana backend (``composer/spec/solana/null_backend.py``). + +The null backend is a pure test double for the Solana front half — it records extracted +properties without verifying them. These tests exercise it in isolation: no LLM, no Postgres, +no prover. (The end-to-end live gate that drives real models through it is +``tests/test_solana_gate.py``, marked ``expensive``.) +""" + +import json +from types import SimpleNamespace +from typing import Any, cast + +import pytest + +from composer.spec.solana.model import ( + SolanaApplication, + SolanaProgram, + SolanaProgramInstance, +) +from composer.spec.solana.null_backend import ( + NullArtifact, + NullResult, + NullSolanaArtifactStore, + NullSolanaBackend, + NullSolanaFormalizer, + NullSolanaPrepared, + SOLANA_NULL_GUIDANCE, + SolanaPhase, +) +from composer.spec.types import PropertyFormulation + + +def _program_instance() -> SolanaProgramInstance: + program = SolanaProgram( + name="Vault", + program_identifier="vault", + description="Holds deposits and releases them to the authority.", + instructions=[], + ) + app = SolanaApplication( + application_type="Vault", + description="A single-program token vault.", + components=[program], + ) + return SolanaProgramInstance(0, app) + + +def _props() -> list[PropertyFormulation]: + return [ + PropertyFormulation( + title="balance_conserved", sort="invariant", + description="The vault balance equals the sum of recorded deposits.", + ), + PropertyFormulation( + title="only_authority_withdraws", sort="safety_property", + description="Only the stored authority can reduce the vault balance.", + ), + ] + + +def _backend(project_root: str) -> NullSolanaBackend: + return NullSolanaBackend(NullSolanaArtifactStore(project_root)) + + +@pytest.mark.asyncio +async def test_formalize_echoes_properties_into_result(): + feat = _program_instance() + props = _props() + + result = await NullSolanaFormalizer().formalize( + "batch", feat, props, cast(Any, None), cast(Any, None) + ) + + assert isinstance(result, NullResult) + # Every property is echoed back verbatim as its own single-rule mapping. + assert result.property_units() == [ + ("balance_conserved", ["balance_conserved"]), + ("only_authority_withdraws", ["only_authority_withdraws"]), + ] + # Commentary records the unit and the count. + assert feat.display_name in result.commentary + assert "2 properties" in result.commentary + # artifact_text is well-formed JSON carrying the same properties; there is no output link. + parsed = json.loads(result.artifact_text) + assert parsed["properties"] == [ + ["balance_conserved", ["balance_conserved"]], + ["only_authority_withdraws", ["only_authority_withdraws"]], + ] + assert result.output_link is None + + +@pytest.mark.asyncio +async def test_formalize_with_no_properties_records_empty(): + result = await NullSolanaFormalizer().formalize( + "batch", _program_instance(), [], cast(Any, None), cast(Any, None) + ) + assert isinstance(result, NullResult) + assert result.property_units() == [] + assert "0 properties" in result.commentary + + +@pytest.mark.asyncio +async def test_fetch_verdicts_is_empty(): + # The null backend never verifies, so it surfaces no verdicts. + assert await NullSolanaFormalizer().fetch_verdicts(cast(Any, None)) == {} + + +@pytest.mark.asyncio +async def test_prepare_system_locates_main_and_builds_formalizer(tmp_path): + feat = _program_instance() + backend = _backend(str(tmp_path)) + run = cast(Any, SimpleNamespace(source=SimpleNamespace(contract_name="vault"))) + + prepared = await backend.prepare_system(feat.app, run) + + assert isinstance(prepared, NullSolanaPrepared) + # prepare_system routes through SOLANA.locate_main, so main is the matched program. + assert isinstance(prepared.main, SolanaProgramInstance) + assert prepared.main.program.program_identifier == "vault" + + formalizer = await prepared.prepare_formalization(cast(Any, None)) + assert isinstance(formalizer, NullSolanaFormalizer) + + +def test_to_artifact_id_uses_unit_slug(tmp_path): + feat = _program_instance() + artifact = _backend(str(tmp_path)).to_artifact_id(feat) + assert isinstance(artifact, NullArtifact) + assert artifact.slug == feat.slug + assert artifact.artifact_file == f"null_{feat.slug}.json" + + +def test_backend_declares_solana_front_half(tmp_path): + backend = _backend(str(tmp_path)) + assert backend.backend_guidance is SOLANA_NULL_GUIDANCE + assert backend.analysis_spec.analysis_key == "solana-analysis" + assert backend.analysis_spec.properties_key == "solana-properties" + assert {p.value for p in SolanaPhase} == { + "analysis", "extraction", "formalization", "report", + }