diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..aeadc71 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,48 @@ +name: CI + +on: + push: + branches: ["main", "review", "review-1"] + pull_request: + +env: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install toolchain + run: | + pip install uv + uv pip install --system -e . + uv pip install --system ruff pyright typeguard toml-sort yamllint + - name: Lint and format checks + run: make fmt-check + - name: Docs guard + env: + BASE_REF: ${{ github.event.pull_request.base.sha || 'HEAD~1' }} + run: make docs-guard + + tests: + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install dependencies + run: | + pip install uv + uv pip install --system -e . + uv pip install --system pytest + - name: Run pytest + run: make test diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..cf8b3f5 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,18 @@ +# Agent Instructions + +## Documentation Workflow +- After each batch of changes, add a `CHANGELOG.md` entry with an ISO 8601 date/time stamp and developer-facing detail (files, modules, functions, variables, and rationale). Every commit should correspond to a fresh entry. +- Maintain `README.md` as the canonical description of the project; update it whenever behaviour or workflows change. Archive older versions separately when requested. +- Keep the `docs/` wiki and provisioning guides (`SETUP.md`, `ENVIRONMENT_NEEDS.md`) in sync with code updates; add or revise the + relevant page whenever features, modules, or workflows change. +- After each iteration, refresh `ISSUES.md`, `SOT.md`, `PLAN.md`, `RECOMMENDATIONS.md`, `TODO.md`, and related documentation to stay in sync with the codebase. +- Ensure `TODO.md` retains the `Completed`, `Priority Tasks`, and `Recommended Waiting for Approval Tasks` sections, moving finished items under `Completed` at the end of every turn. +- Update `RESUME_NOTES.md` at the end of every turn so the next session starts with accurate context. +- When beginning a turn, review `README.md`, `PROJECT.md`, `PLAN.md`, `RECOMMENDATIONS.md`, `ISSUES.md`, and `SOT.md` to harvest new actionable work. Maintain at least ten quantifiable, prioritised items in the `Priority Tasks` section of `TODO.md`, adding context or links when needed. +- After completing any task, immediately update `TODO.md`, check for the next actionable item, and continue iterating until all unblocked `Priority Tasks` are exhausted for the session. +- Continuously loop through planning and execution: finish a task, document it, surface new follow-ups, and resume implementation so long as environment blockers allow. If extra guidance would improve throughput, extend these instructions proactively. + +## Style Guidelines +- Use descriptive Markdown headings starting at level 1 for top-level documents. +- Keep lines to 120 characters or fewer when practical. +- Prefer bullet lists for enumerations instead of inline commas. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..477188a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,126 @@ +# Changelog + +## [2025-10-17T18:45:00Z] +### Added +- Created a Dockerfile for integration workloads and introduced targeted Compose stacks + under `meshmind/tests/docker/` (Memgraph, Neo4j, Redis, full-stack) alongside a + developer-facing provisioning guide in `SETUP.md` to document service bootstrapping + commands and environment requirements. + +### Changed +- Expanded `pyproject.toml` to install optional dependencies (`fastapi`, + `uvicorn[standard]`, `neo4j`, `mgclient`, `redis`) by default and defined extras + (`dev`, `docs`, `testing`); updated the `Makefile` `install` target accordingly and + regenerated setup documentation across `README.md`, `docs/`, `PROJECT.md`, `PLAN.md`, + `SOT.md`, `NEEDED_FOR_TESTING.md`, `ENVIRONMENT_NEEDS.md`, `FINDINGS.md`, + `RECOMMENDATIONS.md`, and `RESUME_NOTES.md` to reference the new workflow and + credentials. +- Reworked the root `docker-compose.yml` to provision Memgraph, Neo4j, and Redis with + health checks and volumes, added Compose variants in `meshmind/tests/docker/`, and + refreshed onboarding materials (`SETUP.md`, `README.md`, `docs/configuration.md`, + `docs/operations.md`, `docs/testing.md`) to call out the new ports, credentials, and + teardown guidance. +- Replaced references to `pymgclient` with `mgclient` throughout dependency notes and + environment files to match the updated driver import. + +### Fixed +- Patched `meshmind/cli/admin.py` to import `argparse`, restoring CLI admin command + registration after the module refactor. +- Updated `.github/workflows/ci.yml` to pass `--system` to `uv pip install`, resolving + the "No virtual environment found" failure during lint/test setup. + +## [2025-10-16T18:30:00Z] +### Fixed +- Adjusted `meshmind/tests/test_service_interfaces.py::test_memory_service_ingest_and_search` to return a hydrated `Memory` + instance from the monkey-patched `list_memories` stub, ensuring pagination-aware search paths remain asserted while avoiding + empty result sets during verification. + +## [2025-10-16T12:00:00Z] +### Added +- Introduced pagination-aware graph access by adding `search_entities` and `count_entities` to every `GraphDriver` implementation, wiring a new `meshmind admin counts` CLI subcommand and REST `/memories/counts` route through `MemoryManager`, `MemoryService`, and the MeshMind client. +- Added `scripts/check_docs_sync.py` plus a Makefile target, CI step, and pytest coverage to guard documentation updates whenever code under mapped modules changes. + +### Changed +- Extended `MemoryManager.list_memories`, MeshMind client helpers, retrieval graph wrappers, and service adapters to forward `offset`, `limit`, and `query` hints, delegating filtering to the active driver before in-memory scoring. +- Updated examples and tests (`meshmind/tests/test_db_drivers.py`, `test_service_interfaces.py`, `test_graph_retrieval.py`, `test_cli_admin.py`, `test_client.py`, `test_docs_guard.py`) to cover pagination, counts, and driver-side search semantics. + +### Documentation +- Refreshed `README.md`, `PROJECT.md`, `PLAN.md`, `RECOMMENDATIONS.md`, `ISSUES.md`, `SOT.md`, `FINDINGS.md`, `AGENTS.md`, `TODO.md`, and the developer wiki (`docs/api.md`, `docs/development.md`, `docs/operations.md`, `docs/persistence.md`, `docs/retrieval.md`, `docs/troubleshooting.md`) to describe pagination, counts, docs-guard workflows, and updated service interfaces. +## [2025-10-15T15:30:00Z] +### Added +- Created a developer wiki under `docs/` covering architecture, pipelines, persistence, retrieval, configuration, testing, operations, telemetry, and development workflows so code changes stay synchronized with reference material. +- Authored `ENVIRONMENT_NEEDS.md` to request optional dependency installs and external services, plus `RESUME_NOTES.md` for session-to-session continuity. + +### Changed +- Expanded the `GraphDriver` contract to accept namespace and entity-label filters when listing entities, updating the in-memory, SQLite, Neo4j, and Memgraph drivers to push filtering into their native query layers. +- Propagated the new filtering through `MemoryManager`, `MeshMind.list_memories`, graph-backed retrieval wrappers, and service interfaces (REST/gRPC), ensuring hybrid searches hydrate only the required entity types. +- Updated tests (`meshmind/tests/test_graph_retrieval.py`, `test_pipeline_preprocess_store.py`, `test_service_interfaces.py`) to cover entity-label filtering across client, REST, and gRPC paths. + +### Documentation +- Refreshed `README.md`, `PROJECT.md`, `PLAN.md`, `RECOMMENDATIONS.md`, `ISSUES.md`, `SOT.md`, `DISCREPANCIES.md`, `FINDINGS.md`, `TODO.md`, and `AGENTS.md` to describe the new driver filtering, documentation workflow, environment checklist, and wiki requirements. + +## [2025-02-15T00:45:00Z] +### Added +- Introduced `meshmind/retrieval/graph.py` with hybrid/vector/regex/exact/BM25/fuzzy wrappers that hydrate candidates from the active `GraphDriver` before delegating to existing scorers, plus `meshmind/tests/test_graph_retrieval.py` to verify namespace filtering and hybrid integration. +- Added `meshmind/cli/admin.py` and wired `meshmind/cli/__main__.py` to expose `admin` subcommands for predicate management, maintenance telemetry, and graph connectivity checks; created `meshmind/tests/test_cli_admin.py` to cover the new flows. +- Created `meshmind/tests/test_neo4j_driver.py` and a `Neo4jGraphDriver.verify_connectivity` helper to exercise driver-level sanity checks without a live cluster. +- Logged importance score distributions via `meshmind/pipeline/preprocess.summarize_importance` so telemetry captures mean/stddev/recency metrics after scoring. + +### Changed +- Updated `MeshMind` search helpers (`meshmind/client.py`) to auto-load memories from the configured driver when `memories` is `None`, reusing the new graph-backed wrappers. +- Reworked `meshmind/pipeline/consolidate.py` to return a `ConsolidationPlan` with batch/backoff thresholds and skipped-group tracking; `meshmind/tasks/scheduled.consolidate_task` now emits skip counts and returns a structured summary. +- Tuned Python compatibility metadata to `>=3.11,<3.13` in `pyproject.toml` and refreshed docs (`README.md`, `NEEDED_FOR_TESTING.md`, `SOT.md`) accordingly. +- Enhanced `meshmind/pipeline/preprocess.py` to emit telemetry gauges for importance scoring and added `meshmind/tests/test_pipeline_preprocess_store.py::test_score_importance_records_metrics`. +- Expanded retrieval, CLI, and driver test coverage (`meshmind/tests/test_retrieval.py`, `meshmind/tests/test_tasks_scheduled.py`) to account for graph-backed defaults and new return types. + +### Documentation +- Updated `README.md`, `PROJECT.md`, `PLAN.md`, `SOT.md`, `FINDINGS.md`, `DISCREPANCIES.md`, `RECOMMENDATIONS.md`, `NEEDED_FOR_TESTING.md`, `ISSUES.md`, and `TODO.md` to describe graph-backed retrieval wrappers, CLI admin tooling, consolidation backoff behaviour, telemetry metrics, and revised Python support. +- Copied the refreshed README guidance into `README_OLD.md` as an archival reference while keeping `README.md` as the primary source. + +## [2025-10-14T14:57:47Z] +### Added +- Introduced `meshmind/_compat/pydantic.py` to emulate `BaseModel`, `Field`, and `ValidationError` when Pydantic is unavailable, enabling tests to run in constrained environments. +- Added `meshmind/testing/fakes.py` with `FakeMemgraphDriver`, `FakeRedisBroker`, and `FakeEmbeddingEncoder`, plus a package export and dedicated pytest coverage (`meshmind/tests/test_db_drivers.py`, `meshmind/tests/test_tasks_scheduled.py`). +- Created heuristics-focused test cases for consolidation outcomes, maintenance tasks, and the revised retrieval dispatcher to guarantee behaviour without external services. + +### Changed +- Replaced the constant importance assignment in `meshmind/pipeline/preprocess.score_importance` with a heuristic that factors token diversity, recency, metadata richness, and embedding magnitude. +- Rebuilt `meshmind/pipeline/consolidate` around a `ConsolidationOutcome` dataclass that merges metadata, averages embeddings, and surfaces removal IDs; `meshmind/tasks/scheduled.consolidate_task` now applies updates and deletes duplicates lazily via `_get_manager`/`_reset_manager` helpers. +- Hardened Celery maintenance tasks by logging driver initialization failures, tracking update counts, and returning deterministic totals; compression counts now reflect the number of persisted updates. +- Updated `meshmind/core/similarity`, `meshmind/retrieval/bm25`, and `meshmind/retrieval/fuzzy` with pure-Python fallbacks so numpy, scikit-learn, and rapidfuzz remain optional. +- Adjusted `meshmind/pipeline/extract.extract_memories` to defer `openai` imports until a default client is required, unblocking DummyLLM-driven tests. +- Reworked `meshmind/retrieval/search.search` to rerank the original (filtered) candidate ordering, prepend reranked results, and append hybrid-sorted fallbacks, preventing index drift when rerankers return relative positions. +- Normalised SQLite entity hydration in `meshmind/db/sqlite_driver._row_to_dict` so JSON metadata is decoded only when stored as strings. +- Refreshed pytest fixtures (`meshmind/tests/conftest.py`, `meshmind/tests/test_pipeline_preprocess_store.py`) to use deterministic encoders and driver doubles, ensuring CRUD and retrieval suites run without live services. + +### Documentation +- Promoted `README.md` as the single source of truth (archiving the previous copy in `README_OLD.md`) and documented the new heuristics, compatibility shims, and test doubles. +- Updated `NEEDED_FOR_TESTING.md` with notes about the compatibility layer, optional dependencies, and fake drivers. +- Reconciled `PROJECT.md`, `ISSUES.md`, `PLAN.md`, `SOT.md`, `RECOMMENDATIONS.md`, `DISCREPANCIES.md`, `FINDINGS.md`, `TODO.md`, and `CHANGELOG.md` to capture the new persistence behaviour, heuristics, fallbacks, and remaining roadmap items. + +## [Unreleased] - 2025-02-14 +### Added +- Configurable graph driver factory with in-memory, SQLite, Memgraph, and optional Neo4j implementations plus supporting tests. +- REST and gRPC service layers (with FastAPI stub fallback) for ingestion and retrieval, including coverage in the test suite. +- Observability utilities that collect metrics and structured logs across pipelines and scheduled Celery tasks. +- Docker Compose definition provisioning Memgraph, Redis, and a Celery worker for local development. +- Vector-only, regex, exact-match, and optional LLM rerank retrieval helpers with reranker utilities and exports. +- MeshMind client wrappers for hybrid, vector, regex, and exact searches plus driver accessors. +- Example script demonstrating triplet storage and diverse retrieval flows. +- Pytest fixtures for encoder and memory factories alongside new retrieval tests that avoid external services. +- Makefile targets for linting, formatting, type checks, and tests, plus a GitHub Actions workflow running lint and pytest. +- README_LATEST.md capturing the current implementation and CHANGELOG.md for release notes. + +### Changed +- Settings now surface `GRAPH_BACKEND`, Neo4j, and SQLite options while README/NEEDED_FOR_TESTING document the expanded setup. +- README, README_LATEST, and NEW_README were consolidated so the promoted README reflects current behaviour. +- PROJECT, PLAN, SOT, FINDINGS, DISCREPANCIES, ISSUES, RECOMMENDATIONS, and TODO were refreshed to capture new capabilities and + re-homed backlog items under a "Later" section. +- Updated `SearchConfig` to support rerank models and refreshed MeshMind documentation across PROJECT, PLAN, SOT, FINDINGS, + DISCREPANCIES, RECOMMENDATIONS, ISSUES, TODO, and NEEDED_FOR_TESTING files. +- Revised `meshmind.retrieval.search` to apply filters centrally, expose new search helpers, and integrate reranking. +- Exposed graph driver access on MeshMind and refreshed retrieval-facing examples and docs. + +### Fixed +- Example ingestion script now uses MeshMind APIs correctly and illustrates relationship persistence. +- Tests rely on fixtures rather than deprecated hooks, improving portability across environments without Memgraph/OpenAI. diff --git a/DISCREPANCIES.md b/DISCREPANCIES.md new file mode 100644 index 0000000..f992bff --- /dev/null +++ b/DISCREPANCIES.md @@ -0,0 +1,53 @@ +# README vs Implementation Discrepancies + +## Overview +- The legacy README has been superseded by `README.md`, which now reflects the implemented feature set. +- The current codebase delivers extraction, preprocessing, triplet persistence, CRUD helpers, and expanded retrieval strategies + that were missing when the README was written. +- Remaining gaps primarily involve pushing retrieval workloads into the graph backend, exporting observability to external sinks, and automated infrastructure provisioning. + +## API Surface +- ✅ `MeshMind` now exposes CRUD helpers (`create_memory`, `update_memory`, `delete_memory`, `list_memories`, triplet helpers) + that the README referenced implicitly. +- ✅ Triplet storage routes through `store_triplets` and `MemoryManager.add_triplet`, calling `GraphDriver.upsert_edge`. +- ⚠️ The README still references `register_entity`, `register_allowed_predicates`, and `add_predicate`; predicate management is + handled automatically but there is no public API matching those method names. +- ⚠️ README snippets showing `mesh_mind.store_memory(memory)` should be updated to call `store_memories([memory])` or the new + CRUD helpers. + +## Retrieval Capabilities +- ✅ Vector-only, regex, exact-match, hybrid, BM25, fuzzy, and optional LLM rerank searches exist in `meshmind.retrieval.search` + and are surfaced through `MeshMind` helpers. +- ⚠️ README implies retrieval queries the graph directly. Search helpers now fetch candidates from the configured driver when no + list is supplied but still score results in Python; Memgraph/Neo4j-native search remains future work. +- ⚠️ Named helpers like `search_facts` or `search_procedures` never existed; the README should reference the dispatcher plus + specialized helpers now available. + +## Data & Relationship Modeling +- ✅ Predicates are persisted automatically when storing triplets and tracked in `PredicateRegistry`. +- ⚠️ README examples that look up subjects/objects by name still do not match the implementation, which expects UUIDs. Add + documentation explaining how to resolve names to UUIDs before storing edges. +- ⚠️ Consolidation and expiry run via Celery jobs; README narratives should highlight that heuristics require further validation even though persistence is now wired up. + +## Configuration & Dependencies +- ✅ `README.md` and `ENVIRONMENT_NEEDS.md` document required environment variables, dependency guards, and setup steps. +- ⚠️ README still omits optional tooling now required by the Makefile/CI (ruff, pyright, typeguard, toml-sort, yamllint); + highlight these prerequisites more prominently. +- ✅ Python version support in `pyproject.toml` now pins `>=3.11,<3.13`, matching the dependency landscape documented in the README. + +## Example Code Paths +- ✅ Updated example scripts demonstrate extraction, triplet creation, and multiple retrieval strategies. +- ⚠️ Legacy README code that instantiates custom Pydantic entities remains inaccurate; extraction returns `Memory` objects and + validates `entity_label` names only. +- ⚠️ Search examples should be updated to show the new helper functions and optional rerank usage instead of nonexistent + `search_facts`/`search_procedures` calls. + +## Tooling & Operations +- ✅ Makefile and CI workflows now exist, aligning with README promises about automation once the README is refreshed. +- ✅ Docker Compose now provisions Memgraph, Redis, and a Celery worker; README sections should highlight the workflow and + caveats for environments lacking container tooling. +- ⚠️ Celery tasks still depend on optional infrastructure; README should clarify that heuristics and scheduling need production hardening even though persistence now works. + +## Documentation State +- Continue promoting `README.md` as the authoritative guide and propagate updates to supporting docs + (`SOT.md`, `PLAN.md`, `ENVIRONMENT_NEEDS.md`, `docs/`). diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7fe6f4e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.11-slim + +ENV PIP_NO_CACHE_DIR=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + libssl-dev \ + libkrb5-dev \ + curl \ + git \ + && rm -rf /var/lib/apt/lists/* + +COPY . /app + +RUN pip install uv \ + && uv pip install --system -e .[dev,docs,testing] + +CMD ["bash"] diff --git a/ENVIRONMENT_NEEDS.md b/ENVIRONMENT_NEEDS.md new file mode 100644 index 0000000..5acef9e --- /dev/null +++ b/ENVIRONMENT_NEEDS.md @@ -0,0 +1,31 @@ +# Tasks for Human Project Manager + +- Install Python packages required for full optional coverage across CI images and the + shared development environment: + - `neo4j` (official Bolt driver). + - `mgclient` (Memgraph driver). + - `redis` / `redis-py` for caching tasks. + - `celery[redis]` for scheduled/async maintenance workers. + - `fastapi` and `uvicorn[standard]` to exercise the REST API. + - `tiktoken`, `sentence-transformers`, and `openai` for embedding/compression workflows. + - `uv` CLI for reproducible dependency management (used in CI). + - Developer tooling referenced by automation: `ruff`, `pyright`, `typeguard`, + `toml-sort`, `yamllint`, `pytest-cov`, `httpx`, `mkdocs`, `mkdocs-material`. +- Provide system-level build dependencies for the graph drivers (e.g., `build-essential`, + `cmake`, `libssl-dev`, `libkrb5-dev`) so `mgclient` installs cleanly. +- Provision external services and credentials (compose files now exist under the project + root and `meshmind/tests/docker/`): + - Neo4j instance reachable from the execution environment with `NEO4J_URI`, + `NEO4J_USERNAME`, `NEO4J_PASSWORD` (defaults to `neo4j` / `meshminD123`). + - Memgraph instance with `MEMGRAPH_URI`, `MEMGRAPH_USERNAME`, `MEMGRAPH_PASSWORD` + (anonymous access acceptable locally). + - Redis instance with `REDIS_URL`. + - OpenAI API key (`OPENAI_API_KEY`) and, optionally, Azure OpenAI equivalents (future + integration testing). +- Supply datasets/fixtures (future request) representing large knowledge graphs to + stress-test consolidation heuristics and pagination under load. +- Allow outbound package downloads to PyPI (current proxy returns HTTP 403, blocking `uv`/dependency lock generation). +- Enable Docker or container runtime access (future request) so the provided + `docker-compose.yml` files can run inside this environment; alternatively, provision + remote services accessible to CI. +- Document credential management procedures and rotation cadence so secrets stay current. diff --git a/FINDINGS.md b/FINDINGS.md new file mode 100644 index 0000000..aa24c49 --- /dev/null +++ b/FINDINGS.md @@ -0,0 +1,39 @@ +# Findings + +## General Observations +- Core modules are now wired through the `MeshMind` client, including CRUD, triplet storage, and retrieval helpers. Graph-backed wrappers fetch namespace/entity-label filtered candidates from the configured driver automatically; remaining integration work focuses on server-side query optimisation and heuristic evaluation loops. +- Optional dependencies are largely guarded behind lazy imports, compatibility shims, or factory functions, improving portability. Environments still need to install tooling referenced by the Makefile and CI (ruff, pyright, typeguard, toml-sort, yamllint). +- Documentation artifacts (`README.md`, `SOT.md`, `ENVIRONMENT_NEEDS.md`, `docs/`) stay current when updated with each iteration; the legacy README has been archived as `README_OLD.md`. A docs-guard script now enforces synchronized updates during CI. + +## Dependency & Environment Notes +- `MeshMind` defers driver creation until persistence is required, enabling workflows without `mgclient` and selecting between +- Package management via `uv` is currently blocked by proxy restrictions (pip returns HTTP 403), so dependency locks cannot be regenerated until outbound access is granted. + in-memory, SQLite, Memgraph, or Neo4j backends. CLI helpers (`meshmind admin graph`) now expose connectivity sanity checks. +- Project metadata now advertises Python `>=3.11,<3.13`, aligning with available wheels for optional dependencies. +- Encoder registration occurs during bootstrap, but custom deployments must ensure compatible models are registered before + extraction or hybrid search. +- The OpenAI embedding adapter still expects dictionary-like responses; adapting to SDK objects remains on the backlog. +- Celery tasks initialize lazily, yet Redis/Memgraph services are still required at runtime. Docker Compose now provisions + Memgraph, Neo4j, and Redis, while targeted stacks under `meshmind/tests/docker/` support integration testing. `SETUP.md` + explains provisioning and teardown commands. + +## Data Flow & Persistence +- Triplet storage now persists relationships and tracks predicates automatically, closing an earlier data-loss gap. +- Consolidation and compression utilities now persist updates through the maintenance tasks, enforce batch/backoff thresholds, and surface skipped groups; larger-scale validation remains necessary. +- Importance scoring uses heuristics (token diversity, recency, metadata richness, embedding magnitude) and now records telemetry summaries; continued evaluation will raise retrieval quality. + +## CLI & Tooling +- CLI ingestion bootstraps encoders and entities automatically and now ships `admin` subcommands for predicate maintenance, telemetry dumps, graph connectivity checks, and namespace/entity counts. External backends still require valid credentials and running services. +- The Makefile introduces lint, format, type-check, test, and docs-guard targets, plus Docker helpers. External tooling installation is + required before targets succeed. +- GitHub Actions now run formatting checks, the docs guard, and pytest on push/PR, providing basic CI coverage. + +## Testing & Quality +- Pytest suites rely on fixtures (`memory_factory`, `dummy_encoder`) and compatibility shims to run without external services. Coverage now includes graph-backed retrieval wrappers, Neo4j connectivity shims, CLI admin flows, docs guard checks, and Celery consolidation telemetry. +- Type checking via `pyright` and runtime checks via `typeguard` are exposed in the Makefile; dependency installation is + necessary for full validation. + +## Documentation +- `README.md`/`README_LATEST.md` document setup, pipelines, retrieval, tooling, and now highlight service interfaces and observability. +- Supporting docs (`ISSUES.md`, `PLAN.md`, `RECOMMENDATIONS.md`, `SOT.md`) reflect the latest capabilities and highlight remaining + gaps, aiding onboarding and future planning. diff --git a/ISSUES.md b/ISSUES.md new file mode 100644 index 0000000..42d252e --- /dev/null +++ b/ISSUES.md @@ -0,0 +1,35 @@ +# Issues Checklist + +## Blockers +- [x] MeshMind client fails without `mgclient`; introduce lazy driver initialization or documented in-memory fallback. +- [x] Register a default embedding encoder (OpenAI or sentence-transformers) during startup so extraction and hybrid search can run. +- [x] Update OpenAI integration to match the current SDK (Responses API payload, embeddings API response structure). +- [x] Replace eager `tiktoken` imports in `meshmind.core.utils` and `meshmind.pipeline.compress` with guarded, optional imports. +- [x] Align declared Python requirement with supported dependencies (project now pins Python >=3.11,<3.13). + +- [ ] Pip/uv package downloads are blocked by the execution environment (HTTP 403 via proxy), preventing dependency lock regeneration. +## High Priority +- [x] Provide configuration documentation and examples for Memgraph, Redis, and OpenAI environment variables. +- [x] Add automated tests or smoke checks that run without external services (mock OpenAI, stub Memgraph driver). +- [x] Create real docker-compose services for Memgraph and Redis or remove the placeholder file. +- [ ] Document Neo4j driver requirements and verify connectivity against a live cluster (CLI connectivity checks exist but still need validation against a real instance). +- [ ] Exercise the new namespace/entity-label filtering against live Memgraph/Neo4j datasets to confirm Cypher predicates behave as expected. +- [ ] Regenerate `uv.lock` to reflect the updated dependency set (`mgclient`, `fastapi`, `uvicorn`, extras) so CI tooling stays in sync. +## Medium Priority +- [x] Persist results from consolidation and compression tasks back to the database (currently in-memory only). +- [x] Refine `Memory.importance` scoring to reflect actual ranking heuristics instead of a constant. +- [x] Add vector, regex, and exact-match search helpers to match stated feature set or update documentation to demote them. +- [x] Harden Celery tasks to initialize dependencies lazily and log failures when the driver is unavailable. +- [ ] Validate consolidation heuristics on larger datasets and add conflict-resolution strategy when merged metadata conflicts. +- [ ] Revisit the compatibility shim once production environments support Pydantic 2.x so the real models can be restored. +- [ ] Push graph-backed retrieval into Memgraph/Neo4j search capabilities once available (current wrappers now filter/paginate server-side but still score vectors in Python). +- [ ] Reconcile tests that depend on `Memory.pre_init` and outdated OpenAI interfaces with the current implementation. +- [x] Expose `memory_counts` via a gRPC endpoint to keep service interfaces aligned. +- [x] Add linting, formatting, and type-checking tooling to improve code quality. + +- [ ] Validate the new Docker Compose stacks (root and `meshmind/tests/docker/`) on an environment with container support and document host requirements (ports, resources). +## Low Priority / Nice to Have +- [x] Offer alternative storage backends (in-memory driver, SQLite, etc.) for easier local development. +- [x] Provide an administrative dashboard or CLI commands for listing namespaces, counts, and maintenance statistics (CLI admin subcommands now expose predicates, telemetry, and graph checks). +- [ ] Publish onboarding guides and troubleshooting FAQs for contributors. +- [ ] Explore plugin registration for embeddings and retrieval strategies to reduce manual wiring. diff --git a/Makefile b/Makefile index 6693190..09123be 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,34 @@ -.PHONY: install lint fmt test docker +.PHONY: install lint fmt fmt-check typecheck test check docker clean docs-guard install: - pip install -e . + python -m pip install -e .[dev,docs,testing] lint: - ruff . + ruff check . fmt: - isort . - black . + ruff format . + +fmt-check: + ruff format --check . + ruff check . + toml-sort --check pyproject.toml + yamllint .github/workflows + +docs-guard: + python scripts/check_docs_sync.py --base $${BASE_REF:-origin/main} + +typecheck: + pyright + python -m typeguard --check meshmind test: pytest +check: fmt-check lint typecheck test docs-guard + +clean: + rm -rf .pytest_cache .ruff_cache + docker: - docker-compose up \ No newline at end of file + docker compose up diff --git a/NEEDED_FOR_TESTING.md b/NEEDED_FOR_TESTING.md new file mode 100644 index 0000000..71ffade --- /dev/null +++ b/NEEDED_FOR_TESTING.md @@ -0,0 +1,66 @@ +# Needed for Testing MeshMind + +> **Note:** `ENVIRONMENT_NEEDS.md` lists infrastructure and package requests for the human project manager. Use this document for developer-side setup details. + +## Python Runtime +- Python 3.11 or 3.12 is recommended; project metadata now pins `>=3.11,<3.13` because several dependencies (`mgclient`, + `sentence-transformers`) do not yet publish wheels for 3.13. +- Use a virtual environment (`uv`, `venv`, or `conda`) to isolate dependencies. + +## Python Dependencies +- Install the project editable (with extras) using `pip install -e .[dev,docs,testing]` or + `uv pip install --system -e .[dev,docs,testing]` from the repository root. +- Core functionality relies on `openai`, `pydantic`, and `pydantic-settings`, but the repository ships a compatibility shim + (`meshmind/_compat/pydantic.py`) that unlocks tests when Pydantic is unavailable. +- Optional packages improve specific workflows (most now included via the extras above): + - `numpy`, `scikit-learn`, and `rapidfuzz` accelerate similarity and lexical search (pure-Python fallbacks are bundled). + - `sentence-transformers`, `tiktoken`, and `mgclient` enable local embeddings, compression, and Memgraph connectivity. + - `celery[redis]` activates scheduled maintenance with a Redis broker. + - `fastapi` + `uvicorn[standard]` power the REST adapter when exercising HTTP APIs. +- Optional drivers: install `neo4j` if exercising the Neo4j backend; SQLite support ships with the standard library. +- Development tooling referenced by the Makefile and CI (installed via `.[dev,docs,testing]`): + - `ruff` for linting and formatting. + - `pyright` for static type checks. + - `typeguard` for runtime type enforcement (`python -m typeguard --check meshmind`). + - `toml-sort` and `yamllint` for configuration validation. + - `mkdocs`/`mkdocs-material` for future documentation publishing. +- Optional helpers for local workflows: `pytest-cov`, `pre-commit`, `httpx`/`grpcio-tools` (for service interface experimentation). + +## External Services and Infrastructure +- **Graph backend** options: + - In-memory / SQLite require no external services (set `GRAPH_BACKEND=memory` or `sqlite`). +- - **Memgraph** reachable via `MEMGRAPH_URI` with credentials exported in `MEMGRAPH_USERNAME`/`MEMGRAPH_PASSWORD`. +- - **Neo4j** reachable via `NEO4J_URI` with credentials `NEO4J_USERNAME`/`NEO4J_PASSWORD` when the optional driver is installed + (defaults supplied in `docker-compose.yml`). Use `meshmind admin graph --backend neo4j` to verify connectivity once + credentials are configured. +- **Redis** for Celery task queues, referenced through `REDIS_URL`. +- **OpenAI API access** for extraction, embeddings, and LLM reranking (`OPENAI_API_KEY`). +- Recommended: Docker Compose (shipped in repo) to run Memgraph, Neo4j, and Redis together when developing locally. Additional + targeted stacks live under `meshmind/tests/docker/` for integration tests. + +## Environment Variables +- `GRAPH_BACKEND` — `memory`, `sqlite`, `memgraph`, or `neo4j` (defaults to `memory`). +- `OPENAI_API_KEY` — required for extraction, embeddings, and reranking. +- `MEMGRAPH_URI` — e.g., `bolt://localhost:7687` (when using Memgraph). +- `MEMGRAPH_USERNAME` and `MEMGRAPH_PASSWORD` — credentials for the Memgraph database. +- `NEO4J_URI`, `NEO4J_USERNAME`, `NEO4J_PASSWORD` — optional Neo4j connectivity details. +- `SQLITE_PATH` — filesystem path for the SQLite graph backend (defaults to in-memory). +- `REDIS_URL` — optional Redis connection URI (defaults to `redis://localhost:6379/0`). +- `EMBEDDING_MODEL` — encoder key registered with `EncoderRegistry` (defaults to `text-embedding-3-small`). +- Optional overrides for Celery broker/backend if using hosted services. + +## Local Configuration Steps +- Ensure an embedding encoder is registered before extraction or hybrid search. The bootstrap utilities invoked by the CLI and + `MeshMind` constructor handle this, but custom scripts must call `bootstrap_encoders()`. +- For REST/gRPC testing, instantiate the `RestAPIStub`/`GrpcServiceStub` with the in-memory driver to avoid external services. +- Use `meshmind/testing` fakes (`FakeMemgraphDriver`, `FakeRedisBroker`, `FakeEmbeddingEncoder`) in tests or demos to eliminate external infrastructure requirements. +- Invoke `meshmind admin predicates` and `meshmind admin maintenance` during local runs to inspect predicate registries and telemetry without external services. +- Seed demo data as needed using the `examples/extract_preprocess_store_example.py` script after configuring environment + variables. +- Create a `.env` file storing the environment variables above for consistent local configuration. + +## Current Blockers in This Environment +- Neo4j/Memgraph binaries and Docker are unavailable in this workspace, preventing local graph provisioning; use the in-memory or SQLite drivers instead. +- Redis cannot be installed without container or host-level access; Celery tasks remain untestable locally until a remote + instance is provisioned (the fake broker satisfies unit tests but not end-to-end runs). +- External network restrictions may limit installation of proprietary packages or access to OpenAI endpoints. diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..c696e05 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,39 @@ +# Plan of Action + +## Phase 1 – Stabilize Runtime Basics ✅ +1. **Dependency Guards** – Implemented lazy driver factories, optional imports, and clear ImportErrors for missing packages. +2. **Default Encoder Registration** – Bootstraps register encoders/entities automatically and the CLI invokes them on startup. +3. **OpenAI SDK Compatibility** – Extraction and embedding adapters align with the Responses API; remaining polish tracked in + `ISSUES.md`. +4. **Configuration Clarity** – `README.md`, `ENVIRONMENT_NEEDS.md`, and the new `docs/` pages document environment variables and service setup. + +## Phase 2 – Restore Promised API Surface ✅ +1. **Entity & Predicate Registry Wiring** – `MeshMind` now boots registries and storage persists predicates automatically. +2. **CRUD & Triplet Support** – CRUD helpers and triplet APIs live on `MeshMind` and `MemoryManager`, storing relationships via + `GraphDriver.upsert_edge`. +3. **Relationship-Aware Examples** – Updated example script demonstrates triplet creation and retrieval flows. + +## Phase 3 – Retrieval & Maintenance Enhancements (In Progress) +1. **Search Coverage** – Hybrid, vector-only, regex, exact-match, fuzzy, and LLM rerank helpers are implemented and exposed. + Graph-backed wrappers now rely on driver-side filtering, pagination, and aggregation before in-memory scoring. Next: push + similarity computation into Memgraph/Neo4j so vector rankings can execute server-side without Python hydration. +2. **Maintenance Tasks** – Tasks emit telemetry and persist consolidation/compression results. Consolidation planning enforces + batch/backoff thresholds and surfaces skipped groups. Next: validate heuristics on larger datasets and tune the thresholds with + real data. +3. **Importance Scoring Improvements** – Heuristic scoring is live and now records distribution metrics via telemetry. Next: + design data-driven evaluation loops or LLM-assisted ranking to tune weights over time. + +## Phase 4 – Developer Experience & Tooling (In Progress) +1. **Testing Overhaul** – Pytest suites rely on local fixtures, compatibility shims, and Celery workflow coverage. Graph-backed + retrieval, Neo4j connectivity shims, CLI admin helpers, and the documentation guard script now have dedicated tests; continue + adding cross-backend integration coverage. +2. **Automation & CI** – Makefile provides lint/format/type/test/docs-guard targets and CI runs fmt-check, docs guard, and + pytest. Add caching and matrix builds when dependencies stabilize. +3. **Environment Provisioning** – Docker Compose now provisions Memgraph, Neo4j, and Redis (with targeted stacks for tests). + Track multi-backend examples, document the new `SETUP.md`, and ensure docs stay current. + +## Phase 5 – Strategic Enhancements (Planned) +1. **Graph-Backed Retrieval** – Extend the new driver-side filtering/pagination to full vector/lexical execution using backend-native indexes to avoid round-tripping candidate embeddings. +2. **Operational Observability** – Export telemetry to Prometheus/OpenTelemetry and surface dashboards/alerts. +3. **Celery Hardening** – Stress test consolidation/compression heuristics at scale and codify retry/backoff policies. +4. **Model Fidelity** – Replace compatibility shims with production-ready Pydantic models once dependency support catches up. diff --git a/PROJECT.md b/PROJECT.md new file mode 100644 index 0000000..2085de6 --- /dev/null +++ b/PROJECT.md @@ -0,0 +1,89 @@ +# MeshMind Project Overview + +## Vision and Scope +- Transform unstructured text into graph-backed `Memory` records enriched with embeddings and metadata. +- Offer pipelines for extraction, preprocessing, storage, and retrieval that can be orchestrated from CLI tools or bespoke agents. +- Enable background maintenance workflows (expiry, consolidation, compression) once supporting services are provisioned. + +## Current Architecture Snapshot +- **Client façade**: `meshmind.client.MeshMind` composes the OpenAI client, configurable embedding model, registry bootstrap, + and a lazily created graph driver selected via `GRAPH_BACKEND` (memory, SQLite, Memgraph, Neo4j). Retrieval helpers expose + hybrid, vector, regex, exact, and reranked flows. +- **Pipelines**: Extraction (LLM + function calling), preprocessing (deduplicate, score, compress), and storage utilities live in + `meshmind.pipeline`. Maintenance helpers consolidate duplicates and expire stale memories. +- **Graph layer**: `meshmind.db` defines `GraphDriver` and implements in-memory, SQLite, Memgraph, and optional Neo4j drivers + with a shared factory for local testing and production use. +- **Retrieval helpers**: `meshmind.retrieval` now covers BM25, fuzzy, hybrid, vector-only, regex, exact-match, and LLM rerank + workflows with shared filters, reranker utilities, and driver-side filtering/pagination to minimise Python hydration. +- **Task runners**: `meshmind.tasks` configures Celery beat to run expiry, consolidation, and compression. Drivers/managers + initialize lazily so import-time failures are avoided. +- **Support code**: `meshmind.core` provides configuration, data models, embeddings, similarity math, and optional dependency + guards around tokenization. +- **Service adapters**: `meshmind.api.rest` and `.grpc` expose REST/gRPC entry points (with lightweight stubs for tests) so + ingestion and retrieval can run as services, including `/memories/counts` for namespace/label summaries. +- **Observability**: `meshmind.core.observability` collects metrics, gauges, and structured log events across pipelines and + scheduled tasks. +- **Tooling**: The CLI ingest command (`meshmind ingest`), updated example script, Makefile automation, CI workflow, and Docker + Compose file illustrate extraction → preprocessing → storage → retrieval locally. +- **Compatibility & fakes**: `_compat/pydantic` keeps models working when Pydantic is absent, while `meshmind/testing` provides fake Memgraph, Redis, and embedding drivers for offline test runs. + +## Implemented Capabilities +- Serialize knowledge as `Memory` (nodes) and `Triplet` (relationships) Pydantic models with namespaces, metadata, embeddings, + timestamps, TTL, and importance fields. +- Extract structured memories from text via the OpenAI Responses API, with encoder registration handled during bootstrap. +- Deduplicate memories by name and cosine similarity, score importance heuristically (token diversity, recency, metadata, embedding magnitude), and compress metadata when `tiktoken` is installed. +- Persist memory nodes and triplet relationships through the storage pipeline and `MemoryManager` CRUD helpers. +- Perform hybrid, vector-only, regex, exact-match, fuzzy, and BM25 retrieval with optional metadata filters and LLM reranking, leveraging driver-side filtering/pagination to shrink result sets before scoring. +- Summarize stored memories by namespace/entity label via the CLI (`meshmind admin counts`) and REST `/memories/counts` route. +- Provide CRUD surfaces on `MeshMind` for creating, updating, deleting, and listing memories and triplets. +- Run Celery maintenance tasks (expiry, consolidation, compression) that tolerate missing graph drivers until runtime and persist consolidated/compressed memories back to the selected backend. +- Demonstrate ingestion, relationship creation, and retrieval in `examples/extract_preprocess_store_example.py`. +- Automate linting, formatting, type checking, and testing through the Makefile and GitHub Actions. + +## Partially Implemented or Fragile Areas +- The OpenAI embedding wrapper still assumes dictionary-style responses; adjust once SDK models are fully adopted. +- Neo4j driver support is import-guarded; the new CLI connectivity check still needs validation against a live cluster. +- Maintenance tasks rely on in-process heuristics for consolidation summaries; long-term storage and conflict resolution rules need validation. +- Importance scoring now records telemetry but still relies on heuristics; richer scoring logic or LLM-assisted ranking is pending. +- SQLite driver currently stores JSON blobs; future work may normalize columns for structured querying. + +## Missing or Broken Capabilities +- Graph-backed retrieval still hydrates namespace/entity-label filtered candidates client-side; pushing ranking into the graph store is future work. +- Predicate management remains internal to the bootstrap process; external administration APIs are still missing. +- Metrics remain in-memory; external exporters (Prometheus/OpenTelemetry) are not wired up. +- gRPC wiring currently relies on stubs; production-ready servers are still future work. +- Compatibility shims provide minimal validation and should be replaced with real Pydantic models in production builds. + +## External Services & Dependencies +- **Graph backend**: Choose via `GRAPH_BACKEND`. In-memory and SQLite require no external services. Memgraph needs `mgclient`; + Neo4j requires the official driver and a live instance. +- **OpenAI SDK**: Required for extraction, embeddings, and LLM reranking; configure `OPENAI_API_KEY`. +- **tiktoken**: Optional but necessary for compression/token budgeting. +- **RapidFuzz, scikit-learn, numpy**: Support fuzzy and lexical retrieval. +- **Celery + Redis**: Optional but necessary for scheduled maintenance jobs. +- **sentence-transformers**: Optional embedding backend for offline models. +- **ruff, pyright, typeguard, toml-sort, yamllint**: Development tooling invoked by the Makefile and CI workflow. + +## Tooling and Operational State +- `Makefile` exposes `install`, `lint`, `fmt`, `fmt-check`, `typecheck`, `test`, `check`, `docs-guard`, `docker`, and `clean` + targets. `make install` installs the `.[dev,docs,testing]` extras so optional dependencies are present. +- `.github/workflows/ci.yml` runs formatting/linting checks, the documentation guard, and pytest on push and pull requests. +- Tests rely on fixtures (`memory_factory`, `dummy_encoder`, in-memory drivers) and compatibility shims so they pass without external services, though installing optional dependencies improves fidelity. +- Developer-facing documentation now lives in `docs/` alongside the canonical `README.md`; the docs guard (`make docs-guard`) enforces synchronized updates when modules change. +- Docker Compose now provisions Memgraph, Neo4j, and Redis; integration-specific stacks (including the Celery worker) live under + `meshmind/tests/docker/`. See `ENVIRONMENT_NEEDS.md` and `SETUP.md` for enabling optional services locally. + +## Roadmap Highlights +- Push graph-backed retrieval deeper into the drivers (vector similarity, structured filters) so the new server-side filtering/pagination evolves into full backend-native ranking. +- Export observability metrics to external sinks (Prometheus/OpenTelemetry) and surface dashboards. +- Enhance importance scoring with data-driven heuristics or LLM evaluation. +- Validate consolidation heuristics and conflict-resolution rules against real datasets. +- Validate Neo4j driver behaviour against a live cluster and ship official test doubles for Memgraph/Redis/encoders. +- Continue refining documentation to reflect setup, troubleshooting, and architectural decisions. +- Reintroduce full Pydantic models once dependency availability is guaranteed in target environments. + +## Future Potential Extensions +- Plugin-based encoder and retriever registration for runtime extensibility. +- Streaming ingestion workers (queues, webhooks) beyond batch CLI workflows. +- UI or agent-facing dashboards for curation, monitoring, and analytics. +- Automated CI pipelines for release packaging, schema migrations, and integration tests. diff --git a/README.md b/README.md index 04882e7..bd9bf4a 100644 --- a/README.md +++ b/README.md @@ -1,244 +1,196 @@ # MeshMind -MeshMind is a knowledge management system that uses LLMs and graph databases to store and retrieve information. -Adding, Searching, Updating, and Deleting memories is supported. -Retrieval of memories using LLMs and graph databases by comparing embeddings and graph database metadata. - -## Features - -- Adding memories -- Searching memories -- Updating memories -- Deleting memories -- Extracting memories from content -- Expiring memories -- Memory Importance Ranking -- Memory Deduplication -- Memory Consolidation -- Memory Compression - -## Retrieval Methods - -- Embedding Vector Search -- BM25 Retrieval -- ReRanking with LLM -- Fuzzy Search -- Exact Comparison Search -- Regex Search -- Search Filters -- Hybrid Search Methods - -## Components -- OpenAI API -- MemGraphDB (Alternative to Neo4j) -- Embedding Model - -# Types of Memory - -- Long-Term Memory - Persistent Memory - - Explicit Memory - Conscious Memory - Active Hotpath Memory (Triggered in Response to Input) - - Declarative Memory - Conscious Memory - - Semantic Memory - What is known - - Eposodic Memory - What has been experienced - - Procedural Memory - How to perform tasks - - - Implicit Memory - Subconscious Memory - Background Process Memory (Triggered in Intervals) - - Non-Declarative Memory - Subconscious Memory - - Semantic Memory - Implicitly acquired - - Eposodic Memory - Implicitly acquired - - Procedural Memory - Implicitly acquired - -- Short-Term Memory - Transient Memory - - Working Memory (Processing) [reasoning messages, scratchpad, etc] - - Sensory Memory (Input) [user input, system input, etc] - - Log Storage (Output) [assistant responses, tool logs, etc] - -# Methods -- Extract Memory -- Add Memory -- Add Triplet -- Search Memory -- Update Memory -- Delete Memory - -## Usage - +MeshMind is an experimental memory orchestration service that pairs large language models with a property graph. It extracts +structured `Memory` records from unstructured text, enriches them with embeddings and metadata, and stores both nodes and +relationships via a Memgraph driver. Retrieval helpers operate on in-memory collections today, offering hybrid, vector-only, +regex, exact-match, fuzzy, and BM25 scoring with optional LLM reranking. + +## Status at a Glance +- ✅ `meshmind.client.MeshMind` orchestrates extraction, preprocessing, storage, CRUD helpers, and retrieval wrappers. +- ✅ Pipelines deduplicate memories, score importance with token/recency/metadata heuristics, compress metadata, and persist nodes and triplets. +- ✅ Retrieval helpers expose hybrid, vector-only, regex, exact-match, BM25, fuzzy, and rerank workflows with namespace/entity + filters. +- ✅ Celery tasks for expiry, consolidation, and compression initialize lazily and run when Redis and Memgraph are configured. +- ✅ Makefile and GitHub Actions provide linting, formatting, type checking, and pytest automation. +- ✅ Docker Compose provisions Memgraph, Neo4j, and Redis for local orchestration, with + integration-specific stacks under `meshmind/tests/docker/` for Celery workers. +- ✅ Built-in observability surfaces structured events and in-memory metrics for pipelines and scheduled tasks while Celery consolidation/compression flows now persist their updates. +- ✅ Compatibility shims and fake drivers let the suite run without Pydantic, scikit-learn, rapidfuzz, Redis, or live Memgraph instances. +- ✅ Graph-backed retrieval wrappers load memories directly from the configured driver when collections are omitted. +- ✅ Graph drivers filter by namespace and entity label before hydrating candidates, keeping hybrid searches efficient on large graphs. + +## Requirements +- Python 3.11 or 3.12 recommended (`pyproject.toml` pins `>=3.11,<3.13` while third-party packages catch up). +- Configurable graph backend via `GRAPH_BACKEND` (`memory`, `sqlite`, `memgraph`, `neo4j`). +- Memgraph instance reachable via Bolt and the `mgclient` Python package (when `GRAPH_BACKEND=memgraph`). +- Optional Neo4j instance with the official Python driver (when `GRAPH_BACKEND=neo4j`). +- OpenAI API key for extraction, embeddings, and optional reranking. +- Optional: Redis and Celery for scheduled maintenance tasks. +- Optional: `numpy`, `scikit-learn`, and `rapidfuzz` improve similarity and lexical matching but pure-Python fallbacks are bundled. +- Install project dependencies with `pip install -e .[dev,docs,testing]`; see `SETUP.md` for a detailed walkthrough. + +## Installation +1. Create and activate a virtual environment using Python 3.11/3.12 (e.g., `uv venv`, `python -m venv .venv`). +2. Upgrade `pip` and install MeshMind with all optional extras: + ```bash + python -m pip install --upgrade pip + pip install uv + uv pip install --system -e .[dev,docs,testing] # drop --system if you're inside a virtualenv + ``` +3. Export required environment variables (or populate `.env`; see `SETUP.md`): + ```bash + export OPENAI_API_KEY=sk-... + export GRAPH_BACKEND=memory # or memgraph/sqlite/neo4j + export MEMGRAPH_URI=bolt://localhost:7687 + export MEMGRAPH_USERNAME= # optional; Memgraph defaults to anonymous auth + export MEMGRAPH_PASSWORD= + export SQLITE_PATH=/tmp/meshmind.db + export NEO4J_URI=bolt://localhost:7688 + export NEO4J_USERNAME=neo4j + export NEO4J_PASSWORD=meshminD123 + export REDIS_URL=redis://localhost:6379/0 + export EMBEDDING_MODEL=text-embedding-3-small + ``` + +4. Provision Redis, Memgraph, and Neo4j with Docker Compose when you need external + services: + ```bash + docker compose up -d + ``` + Alternate topologies live in `meshmind/tests/docker/`; see `SETUP.md` for guidance on + targeted stacks and teardown commands. + +## Encoder Registration +`MeshMind` bootstraps encoders and entities during initialization, but custom scripts can register additional encoders: ```python -from meshmind import MeshMind -from pydantic import BaseModel, Field - -# Pydantic model's name will be used as the node label in the graph database. -# This defines the attributes of the entity. -# Attributes of the entity are stored in entity metadata['attributes'] -class Person(BaseModel): - first_name: str | None = Field(..., description="First name of the person") - last_name: str | None = Field(None, description="Last name of the person") - description: str | None = Field(None, description="Description of the person") - job_title: str | None = Field(None, description="Job title of the person") - -# Initialize MeshMind -mesh_mind = MeshMind() - -# Register pydantic model as entity -mesh_mind.register_entity(Person) - -# Register allowed relationship predicates labels. -mesh_mind.register_allowed_predicates([ - "employee_of", - "on_team", - "on_project", -]) - -# Add edge - This will create an edge in the graph database. (Alternative to `register_allowed_types`) -mesh_mind.add_predicate("has_skill") - -# Extract memories - [High Level API] - This will create nodes and edges based on the content provided. Extracts memories from the content. -extracted_memories = mesh_mind.extract_memories( - instructions="Extract all memories from the database.", - namespace="Company Employees", - entity_types=[Person], - content=["John Doe, Software Engineer 10 years of experience.", "Jane Doe, Software Engineer 5 years of experience."] - ) +from meshmind.core.embeddings import EncoderRegistry, OpenAIEmbeddingEncoder -for memory in extracted_memories: - # Store memory - This will perform deduplication, add uuid, add timestamps, format memory and store the memory in the graph database using `add_triplet`. - mesh_mind.store_memory(memory) - -# Add memory - [Mid Level API] - This will perform all preprocessing steps i.e. deduplication, add uuid, add timestamps, format memory, etc. and store the memory in the graph database using `add_triplet`. ( Skips extraction and automatically adds memory to the graph database ) **Useful for adding custom memories** -mesh_mind.add_memory( - namespace="Company Employees", - name="John Doe", - entity_label="Person", - entity=Person( - first_name="John", - last_name="Doe", - description="John Doe", - job_title="Software Engineer", - ), - metadata={ - "source": "John Doe Employee Record", - "source_type": "text", - } -) +if not EncoderRegistry.is_registered("text-embedding-3-small"): + EncoderRegistry.register("text-embedding-3-small", OpenAIEmbeddingEncoder("text-embedding-3-small")) +``` +You may register deterministic or local encoders (e.g., sentence-transformers) for offline testing. -# `add_triplet` - [Low Level API] -# -# Add a [node, edge, node] triplet - This will create a pair of nodes and a connecting edge in the graph database using db driver. -# This bypasses deduplication and other preprocessing steps and directly adds the data to the graph database. -# This is useful for adding data that is not in the format of a memory. -# -# subject: The subject of the triplet. ( Source Entity Node Label Name ) -# predicate: The predicate of the triplet. ( Relationship between the subject and object, registered using `register_allowed_predicates` ) -# object: The object of the triplet. ( Target Entity Node Label Name ) -# namespace: The namespace of the triplet. ( Group of related nodes and edges ) -# entity_label: The label of the entity. ( Type of node, registered using `register_entity` ) -# metadata: The metadata of the triplet. ( Additional information about the triplet ) -# reference_time: The time at which the triplet was created. ( Optional ) -# -# If the subject, predicate, object, namespace, or entity_label does not exist, it will be created. -# -# Example: -mesh_mind.add_triplet( - subject="John Doe", - predicate="on_project", - object="Project X", - namespace="Company Employees", - entity_label="Person", - metadata={ - "source": "John Doe Project Record", - "source_type": "text", - "summary": "John Doe is on project X.", - "attributes": { - "first_name": "John", - "last_name": "Doe", - "description": "John Doe", - "job_title": "Software Engineer", - } - }, - reference_time="2025-05-09T23:31:51-04:00" +## Quick Start +```python +from meshmind.client import MeshMind +from meshmind.core.types import Memory, Triplet + +mm = MeshMind() +texts = ["Python is a programming language created by Guido van Rossum."] +memories = mm.extract_memories( + instructions="Extract key facts as Memory objects.", + namespace="demo", + entity_types=[Memory], + content=texts, ) - -# Search - [High Level API] - This will search the graph database for nodes and edges based on the query. -search_results = mesh_mind.search( - query="John Doe", - namespace="Company Employees", - entity_types=[Person], +memories = mm.deduplicate(memories) +memories = mm.score_importance(memories) +memories = mm.compress(memories) +mm.store_memories(memories) + +if len(memories) >= 2: + relation = Triplet( + subject=str(memories[0].uuid), + predicate="RELATED_TO", + object=str(memories[1].uuid), + namespace="demo", + entity_label="Knowledge", ) - -for search_result in search_results: - print(search_result) - -# Search - [Mid Level API] - This will search the graph database for nodes and edges based on the query. -search_results = mesh_mind.search_facts( - query="John Doe", - namespace="Company Employees", - entity_types=[Person], - config=SearchConfig( - encoder="text-embedding-3-small", - ) - ) - -for search_result in search_results: - print(search_result) - -# Search - [Low Level API] - This will search the graph database for nodes and edges based on the query. -search_results = mesh_mind.search_procedures( - query="John Doe", - namespace="Company Employees", - entity_types=[Person], - config=SearchConfig( - encoder="text-embedding-3-small", - ) - ) - -# Update Memory - Same as `add_memory` but updates an existing memory. -mesh_mind.update_memory( - uuid="12345678-1234-1234-1234-123456789012", - namespace="Company Employees", - name="John Doe", - entity_label="Person", - entity=Person( - first_name="John", - last_name="Doe", - description="John Doe", - job_title="Software Engineer", - ), - metadata={ - "source": "John Doe Employee Record", - "source_type": "text", - } -) - -# Delete Memory -mesh_mind.delete_memory( - uuid="12345678-1234-1234-1234-123456789012" -) - -for search_result in search_results: - print(search_result) - + mm.store_triplets([relation]) ``` -## Command-Line Interface (CLI) +## Retrieval +`MeshMind` exposes multiple retrieval helpers that operate on lists of `Memory` objects (e.g., fetched via +`mm.list_memories(namespace="demo")`). When you omit the `memories` argument, the client fetches candidates directly from the +active graph backend configured through `GRAPH_BACKEND` or the driver supplied to `MeshMind`. Driver-backed searches now use +server-side filtering (`query`, `entity_labels`) and pagination (`offset`, `limit`) before in-memory scoring to avoid loading +entire namespaces. +```python +from meshmind.core.types import SearchConfig -MeshMind includes a `meshmind` CLI tool for ingesting content via the extract → preprocess → store pipeline. +memories = mm.list_memories(namespace="demo", entity_labels=["Knowledge"], limit=25) +config = SearchConfig(encoder=mm.embedding_model, top_k=5, rerank_model="gpt-4o-mini") -Usage: -```bash -meshmind --help +hybrid = mm.search("Python", namespace="demo", entity_labels=["Knowledge"], config=config, use_llm_rerank=True) +vector_only = mm.search_vector("programming", namespace="demo", entity_labels=["Knowledge"]) +regex_hits = mm.search_regex(r"Guido", namespace="demo", entity_labels=["Knowledge"]) +exact_hits = mm.search_exact("Python", namespace="demo", entity_labels=["Knowledge"]) ``` -Primary command: +The in-memory and graph-backed search helpers support namespace/entity filters and optional reranking via the OpenAI Responses +API. Entity-label filters are applied at the driver, avoiding unnecessary hydration when graphs contain heterogeneous nodes. +You can still pass explicit lists (e.g., during testing) to bypass graph access when desired. + +## Command-Line Operations ```bash meshmind ingest \ - -n \ - [-e ] \ - [-i ""] \ - [ ...] + --namespace demo \ + --instructions "Extract key facts as Memory objects." \ + ./path/to/text/files ``` -Example: +The CLI bootstraps encoders/entities automatically. Ensure environment variables are set and Memgraph is reachable. + +Administrative helpers expose predicate registry and telemetry insight: ```bash -meshmind ingest -n demo --embedding-model text-embedding-3-small ./data/articles +meshmind admin predicates --list +meshmind admin predicates --add RELATED_TO +meshmind admin maintenance +meshmind admin graph --backend neo4j +meshmind admin counts --namespace demo ``` -This reads text files under `./data/articles`, extracts memories, deduplicates, scores, compresses, -and stores them in your Memgraph database under the `demo` namespace. +## Maintenance Tasks +Celery tasks in `meshmind.tasks.scheduled` provide expiry, consolidation, and compression maintenance with persistence. +```bash +celery -A meshmind.tasks.celery_app.app worker -B +``` +Tasks instantiate the driver lazily, emit structured logs/metrics, and persist consolidated or compressed memories back to the selected graph driver. Provide valid environment variables and ensure Memgraph/Redis are running when using external backends. + +## Tooling +- **Makefile** – `make fmt`, `make lint`, `make typecheck`, `make test`, `make check`, `make docker`, `make clean`, + `make docs-guard`. +- **CI** – `.github/workflows/ci.yml` runs formatting checks (ruff, toml-sort, yamllint), pytest, and the documentation guard + to ensure code changes keep the wiki up to date. +- **Examples** – `examples/extract_preprocess_store_example.py` demonstrates ingestion, triplet creation, and multiple retrieval + strategies. +- **Dockerfile / docker-compose** – Container definition and orchestration files that provision Memgraph, Neo4j, Redis, and the + Celery worker stacks documented in `SETUP.md` and `meshmind/tests/docker/`. + +## Service Interfaces +- **REST** – `meshmind.api.rest.create_app` returns a FastAPI app (or lightweight stub) that exposes `/memories`, `/triplets`, + and `/search` endpoints. +- **gRPC** – `meshmind.api.grpc.GrpcServiceStub` mirrors the ingestion and retrieval RPC surface for integration tests and + future server wiring. + +## Developer Documentation +- `docs/overview.md` – high-level module map. +- `docs/persistence.md` – graph driver behaviours and configuration. +- `docs/retrieval.md` – search strategies and entity-label filtering semantics. +- `docs/pipelines.md` – ingestion lifecycle. +- `docs/api.md` – REST/gRPC payloads and CLI expectations. +- `docs/testing.md` – pytest layout and fake drivers. +- `docs/configuration.md` – environment variables and defaults. +- `docs/operations.md` & `docs/telemetry.md` – operational workflows and observability guidance. +- `docs/development.md` – contribution workflow and coding standards. + +## Observability +- `meshmind.core.observability.telemetry` collects counters, gauges, and durations for pipelines and Celery tasks. +- `meshmind.core.observability.log_event` emits structured log messages that annotate pipeline progress. +- Metrics remain in-memory today; export hooks (Prometheus, OpenTelemetry) are future enhancements. + +## Compatibility & Test Doubles +- `meshmind/_compat/pydantic.py` provides a lightweight `BaseModel` implementation so the codebase functions without installing Pydantic. +- `meshmind/retrieval/bm25.py`, `meshmind/retrieval/fuzzy.py`, and `meshmind/core/similarity.py` include pure-Python fallbacks for scikit-learn, rapidfuzz, and numpy. +- `meshmind/testing` exports fake Memgraph, Redis, and embedding drivers that power the pytest suite and examples without external infrastructure. + +## Testing +- Run `pytest` to execute the suite; tests rely on fixtures, fake drivers, and compatibility shims so they do not require external services or optional libraries. +- `make typecheck` invokes `pyright` and `typeguard`; install the tooling listed above beforehand. +- See `ENVIRONMENT_NEEDS.md` for environment requirements and known blockers (Docker/Memgraph/Redis availability). + +## Known Limitations +- Graph-backed retrieval still hydrates candidates client-side (now filtered by namespace and entity label); server-side vector search remains future work. +- Metrics remain in-memory; no external exporter is wired up yet. +- Importance scoring uses heuristics and telemetry but does not yet incorporate feedback loops or LLM-assisted ranking. + +## Roadmap Snapshot +Consult `PROJECT.md`, `PLAN.md`, and `RECOMMENDATIONS.md` for prioritized enhancements: graph-backed retrieval, metrics exporters, richer importance scoring, and production-ready service deployments. diff --git a/README_OLD.md b/README_OLD.md new file mode 100644 index 0000000..2b3acea --- /dev/null +++ b/README_OLD.md @@ -0,0 +1,169 @@ +# MeshMind + +MeshMind is an experimental memory orchestration service that pairs large language models with a property graph. It extracts +structured `Memory` records from unstructured text, enriches them with embeddings and metadata, and stores both nodes and +relationships via a Memgraph driver. Retrieval helpers operate on in-memory collections today, offering hybrid, vector-only, +regex, exact-match, fuzzy, and BM25 scoring with optional LLM reranking. + +## Status at a Glance +- ✅ `meshmind.client.MeshMind` orchestrates extraction, preprocessing, storage, CRUD helpers, and retrieval wrappers. +- ✅ Pipelines deduplicate memories, score importance with token/recency/metadata heuristics, compress metadata, and persist nodes and triplets. +- ✅ Retrieval helpers expose hybrid, vector-only, regex, exact-match, BM25, fuzzy, and rerank workflows with namespace/entity + filters. +- ✅ Celery tasks for expiry, consolidation, and compression initialize lazily and run when Redis and Memgraph are configured. +- ✅ Makefile and GitHub Actions provide linting, formatting, type checking, and pytest automation. +- ✅ Docker Compose provisions Memgraph, Redis, and a Celery worker for local orchestration. +- ✅ Built-in observability surfaces structured events and in-memory metrics for pipelines and scheduled tasks while Celery consolidation/compression flows now persist their updates. +- ✅ Compatibility shims and fake drivers let the suite run without Pydantic, scikit-learn, rapidfuzz, Redis, or live Memgraph instances. +- ✅ Graph-backed retrieval wrappers load memories directly from the configured driver when collections are omitted. + +## Requirements +- Python 3.11 or 3.12 recommended (`pyproject.toml` pins `>=3.11,<3.13` while third-party packages catch up). +- Configurable graph backend via `GRAPH_BACKEND` (`memory`, `sqlite`, `memgraph`, `neo4j`). +- Memgraph instance reachable via Bolt and the `mgclient` Python package (when `GRAPH_BACKEND=memgraph`). +- Optional Neo4j instance with the official Python driver (when `GRAPH_BACKEND=neo4j`). +- OpenAI API key for extraction, embeddings, and optional reranking. +- Optional: Redis and Celery for scheduled maintenance tasks. +- Optional: `numpy`, `scikit-learn`, and `rapidfuzz` improve similarity and lexical matching but pure-Python fallbacks are bundled. +- Install project dependencies with `pip install -e .`; see `pyproject.toml` for the full list. + +## Installation +1. Create and activate a virtual environment using Python 3.11/3.12 (e.g., `uv venv`, `python -m venv .venv`). +2. Install MeshMind: + ```bash + pip install -e . + ``` +3. Install optional dependencies as needed: + ```bash + pip install mgclient tiktoken sentence-transformers celery[redis] ruff pyright typeguard toml-sort yamllint + ``` +4. Export required environment variables: + ```bash + export OPENAI_API_KEY=sk-... + export GRAPH_BACKEND=memory # or memgraph/sqlite/neo4j + export MEMGRAPH_URI=bolt://localhost:7687 + export MEMGRAPH_USERNAME=neo4j + export MEMGRAPH_PASSWORD=secret + export SQLITE_PATH=/tmp/meshmind.db + export NEO4J_URI=bolt://localhost:7687 + export NEO4J_USERNAME=neo4j + export NEO4J_PASSWORD=secret + export REDIS_URL=redis://localhost:6379/0 + export EMBEDDING_MODEL=text-embedding-3-small + ``` + +## Encoder Registration +`MeshMind` bootstraps encoders and entities during initialization, but custom scripts can register additional encoders: +```python +from meshmind.core.embeddings import EncoderRegistry, OpenAIEmbeddingEncoder + +if not EncoderRegistry.is_registered("text-embedding-3-small"): + EncoderRegistry.register("text-embedding-3-small", OpenAIEmbeddingEncoder("text-embedding-3-small")) +``` +You may register deterministic or local encoders (e.g., sentence-transformers) for offline testing. + +## Quick Start +```python +from meshmind.client import MeshMind +from meshmind.core.types import Memory, Triplet + +mm = MeshMind() +texts = ["Python is a programming language created by Guido van Rossum."] +memories = mm.extract_memories( + instructions="Extract key facts as Memory objects.", + namespace="demo", + entity_types=[Memory], + content=texts, +) +memories = mm.deduplicate(memories) +memories = mm.score_importance(memories) +memories = mm.compress(memories) +mm.store_memories(memories) + +if len(memories) >= 2: + relation = Triplet( + subject=str(memories[0].uuid), + predicate="RELATED_TO", + object=str(memories[1].uuid), + namespace="demo", + entity_label="Knowledge", + ) + mm.store_triplets([relation]) +``` + +## Retrieval +`MeshMind` exposes multiple retrieval helpers that operate on lists of `Memory` objects (e.g., fetched via +`mm.list_memories(namespace="demo")`). When you omit the `memories` argument, the client fetches candidates directly from the +active graph backend configured through `GRAPH_BACKEND` or the driver supplied to `MeshMind`. +```python +from meshmind.core.types import SearchConfig + +memories = mm.list_memories(namespace="demo") +config = SearchConfig(encoder=mm.embedding_model, top_k=5, rerank_model="gpt-4o-mini") + +hybrid = mm.search("Python", namespace="demo", config=config, use_llm_rerank=True) +vector_only = mm.search_vector("programming", namespace="demo") +regex_hits = mm.search_regex(r"Guido", namespace="demo") +exact_hits = mm.search_exact("Python", namespace="demo") +``` +The in-memory and graph-backed search helpers support namespace/entity filters and optional reranking via the OpenAI Responses +API. You can still pass explicit lists (e.g., during testing) to bypass graph access when desired. + +## Command-Line Operations +```bash +meshmind ingest \ + --namespace demo \ + --instructions "Extract key facts as Memory objects." \ + ./path/to/text/files +``` +The CLI bootstraps encoders/entities automatically. Ensure environment variables are set and Memgraph is reachable. + +Administrative helpers expose predicate registry and telemetry insight: +```bash +meshmind admin predicates --list +meshmind admin predicates --add RELATED_TO +meshmind admin maintenance +meshmind admin graph --backend neo4j +``` + +## Maintenance Tasks +Celery tasks in `meshmind.tasks.scheduled` provide expiry, consolidation, and compression maintenance with persistence. +```bash +celery -A meshmind.tasks.celery_app.app worker -B +``` +Tasks instantiate the driver lazily, emit structured logs/metrics, and persist consolidated or compressed memories back to the selected graph driver. Provide valid environment variables and ensure Memgraph/Redis are running when using external backends. + +## Tooling +- **Makefile** – `make fmt`, `make lint`, `make typecheck`, `make test`, `make check`, `make docker`, `make clean`. +- **CI** – `.github/workflows/ci.yml` runs formatting checks (ruff, toml-sort, yamllint) and pytest on push/PR. +- **Examples** – `examples/extract_preprocess_store_example.py` demonstrates ingestion, triplet creation, and multiple retrieval + strategies. + +## Service Interfaces +- **REST** – `meshmind.api.rest.create_app` returns a FastAPI app (or lightweight stub) that exposes `/memories`, `/triplets`, + and `/search` endpoints. +- **gRPC** – `meshmind.api.grpc.GrpcServiceStub` mirrors the ingestion and retrieval RPC surface for integration tests and + future server wiring. + +## Observability +- `meshmind.core.observability.telemetry` collects counters, gauges, and durations for pipelines and Celery tasks. +- `meshmind.core.observability.log_event` emits structured log messages that annotate pipeline progress. +- Metrics remain in-memory today; export hooks (Prometheus, OpenTelemetry) are future enhancements. + +## Compatibility & Test Doubles +- `meshmind/_compat/pydantic.py` provides a lightweight `BaseModel` implementation so the codebase functions without installing Pydantic. +- `meshmind/retrieval/bm25.py`, `meshmind/retrieval/fuzzy.py`, and `meshmind/core/similarity.py` include pure-Python fallbacks for scikit-learn, rapidfuzz, and numpy. +- `meshmind/testing` exports fake Memgraph, Redis, and embedding drivers that power the pytest suite and examples without external infrastructure. + +## Testing +- Run `pytest` to execute the suite; tests rely on fixtures, fake drivers, and compatibility shims so they do not require external services or optional libraries. +- `make typecheck` invokes `pyright` and `typeguard`; install the tooling listed above beforehand. +- See `NEEDED_FOR_TESTING.md` for environment requirements and known blockers (Docker/Memgraph/Redis availability). + +## Known Limitations +- Graph-backed retrieval loads namespace-scoped candidates into memory before scoring; server-side vector search remains future work. +- Metrics remain in-memory; no external exporter is wired up yet. +- Importance scoring uses heuristics and telemetry but does not yet incorporate feedback loops or LLM-assisted ranking. + +## Roadmap Snapshot +Consult `PROJECT.md`, `PLAN.md`, and `RECOMMENDATIONS.md` for prioritized enhancements: graph-backed retrieval, metrics exporters, richer importance scoring, and production-ready service deployments. diff --git a/RECOMMENDATIONS.md b/RECOMMENDATIONS.md new file mode 100644 index 0000000..fc6d0f8 --- /dev/null +++ b/RECOMMENDATIONS.md @@ -0,0 +1,34 @@ +# Recommendations + +## Stabilize the Foundation +- Maintain lazy initialization for optional dependencies and continue testing environments without Memgraph or OpenAI access. +- Maintain declared Python support at `>=3.11,<3.13` and monitor dependency releases before widening the range. +- Harden the OpenAI embedding adapter to consume SDK response objects directly and surface actionable errors for rate limits. +- Add automated smoke tests for the new SQLite/Neo4j drivers to ensure regressions are caught early. + +## Restore and Extend Functionality +- Extend the new server-side filtering and pagination work by pushing similarity ranking into Memgraph/Neo4j so vector scoring runs without loading namespaces in Python. +- Validate consolidation heuristics at scale and tune the new batch/backoff thresholds before enabling automated writes in production. +- Introduce evaluation loops for the new importance heuristic (e.g., LLM-assisted ranking or analytics-driven weights) to tune thresholds over time, leveraging the telemetry stats now emitted. +- Expand predicate/registry management APIs beyond the CLI helper so services can manage vocabularies programmatically. +- Plan for reintroducing full Pydantic models once packaging support is aligned with target Python versions. + +## Improve Developer Experience +- Document usage patterns for each graph backend (memory/sqlite/memgraph/neo4j) inside `docs/` and keep the docs-guard mapping current so contributors know which pages to update when modules change. +- Add Makefile targets for running Celery workers and seeding demo data once infrastructure is provisioned (potentially reusing + the new Docker Compose stacks). +- Broaden pytest coverage with cross-backend integration tests (Memgraph/Neo4j) and failure injection to complement the new graph retrieval, CLI admin, and docs guard unit tests. +- Cache dependencies and split lint/test jobs in CI for faster feedback once the dependency stack stabilizes. + +## Documentation & Onboarding +- Keep `README.md`, `SOT.md`, `docs/`, and onboarding guides synchronized with each release; document rerank, retrieval, and + registry flows with diagrams when possible. +- Maintain the troubleshooting section for optional tooling (ruff, pyright, typeguard, toml-sort, yamllint) now referenced in + the Makefile and expand it as new developer utilities are introduced. Keep `SETUP.md` synchronized when dependencies change. +- Provide walkthroughs for configuring LLM reranking, including sample prompts and response expectations. +- Add onboarding notes for the REST/gRPC service layers with sample payloads and curl/grpcurl snippets. + +## Future Enhancements +- Export telemetry to Prometheus/OpenTelemetry and wire alerts/dashboards around ingestion and maintenance. +- Explore streaming ingestion pipelines (queues, webhooks) for near-real-time updates. +- Investigate lightweight web UI tooling for inspecting memories, triplets, and telemetry snapshots. diff --git a/RESUME_NOTES.md b/RESUME_NOTES.md new file mode 100644 index 0000000..0ee8741 --- /dev/null +++ b/RESUME_NOTES.md @@ -0,0 +1,48 @@ +# Resume Notes + +## Current Context + +- Branch: `work`. +- Optional dependencies are now first-class: `pyproject.toml` installs REST (`fastapi`, `uvicorn`), graph (`neo4j`, `mgclient`, + `redis`), and dev extras by default, and the Makefile’s `install` target installs `.[dev,docs,testing]`. +- Docker assets have been expanded: the root `docker-compose.yml` runs Memgraph, Neo4j, and Redis with health checks, and + `meshmind/tests/docker/` contains targeted stacks for integration scenarios. A new `Dockerfile` supports worker containers. +- Provisioning guidance lives in `SETUP.md`, `ENVIRONMENT_NEEDS.md`, and `NEEDED_FOR_TESTING.md`; docs guard now tracks these + guides when related modules change. + +## Latest Changes + +- Added the dependency extras above, updated `.github/workflows/ci.yml` to use `uv pip install --system`, and refreshed + onboarding docs (`README.md`, `docs/`, `PROJECT.md`, `PLAN.md`, `SOT.md`, `FINDINGS.md`, `RECOMMENDATIONS.md`, + `ENVIRONMENT_NEEDS.md`, `NEEDED_FOR_TESTING.md`) to reference the new workflow and credentials. +- Replaced the old docker-compose stack with multi-service definitions, introduced per-topology Compose files under + `meshmind/tests/docker/`, authored `SETUP.md`, and documented the stack in README/docs. +- Extended `scripts/check_docs_sync.py` and its pytest coverage so changes under `meshmind/tests/docker/`, `docker-compose.yml`, + or the `Dockerfile` require updates to `SETUP.md`, `ENVIRONMENT_NEEDS.md`, or the operations docs. +- Restored the missing `argparse` import in `meshmind/cli/admin.py` so admin subcommands register correctly. +- Ran the full pytest suite (69 tests) to confirm the refactor passes without external services. + +## Environment State + +- External services (Neo4j, Memgraph, Redis) remain unavailable; tests rely on fakes and SQLite/in-memory drivers. +- Outbound package downloads are blocked (`pip install uv` returns HTTP 403 via the proxy), preventing regeneration of + `uv.lock`. The need for PyPI access is recorded in `ISSUES.md` and `ENVIRONMENT_NEEDS.md`. +- Optional packages (`neo4j`, `mgclient`, `redis`, `celery`, `tiktoken`, `sentence-transformers`) are still absent locally. + +## Next Session Starting Points + +1. Address the `TODO.md` priority backlog—top items are regenerating `uv.lock` once network access is restored and validating the + new docs guard paths; remaining work focuses on live Neo4j/Memgraph validation, consolidation tuning, and proto generation. +2. When network/infrastructure unblockers land, install the optional dependencies and spin up the compose stacks to validate + driver connectivity and REST/gRPC smoke tests. +3. Continue expanding backend-native vector search and importance-evaluation loops once larger datasets and live services are + available. +4. Keep `SETUP.md`, `ENVIRONMENT_NEEDS.md`, and the wiki in sync with any further tooling or configuration adjustments and update + this file at the end of each session. + +## Helpful References + +- `docs/overview.md` for the high-level module map. +- `docs/persistence.md` and `docs/operations.md` for backend specifics and operational runbooks. +- `SETUP.md` for provisioning instructions; `ENVIRONMENT_NEEDS.md` for outstanding environment requests. +- `ISSUES.md` and `TODO.md` for the live backlog and blockers. diff --git a/SETUP.md b/SETUP.md new file mode 100644 index 0000000..ff5f3e4 --- /dev/null +++ b/SETUP.md @@ -0,0 +1,114 @@ +# Setup Guide + +This guide walks through preparing a MeshMind development machine, provisioning the +external graph/cache services, and validating that the environment is ready for local +execution and automated testing. + +## 1. Prerequisites + +- **Operating system**: Linux or macOS with Docker Engine ≥ 24 and Docker Compose v2. +- **Python**: CPython 3.11.x (the project currently supports 3.11 and 3.12). +- **System packages** (only required if you intend to install optional graph drivers): + - Build tooling: `build-essential`, `cmake`, `git`. + - Crypto/Kerberos headers: `libssl-dev`, `libkrb5-dev` (for `mgclient`). + - Optional: `libopenblas-dev` for faster `numpy`/`scikit-learn` builds on Debian/Ubuntu. + +## 2. Bootstrap the Python environment + +1. Create and activate a virtual environment: + + ```bash + python3.11 -m venv .venv + source .venv/bin/activate + ``` + +2. Install MeshMind and all optional extras required for full local coverage: + + ```bash + python -m pip install --upgrade pip + pip install uv + uv pip install --system -e .[dev,docs,testing] # omit --system when inside an active virtualenv + ``` + + The editable install pulls in the optional dependencies used by the REST service + (`fastapi`, `uvicorn`), the graph drivers (`neo4j`, `mgclient`, `redis`), LLM tooling + (`openai`, `tiktoken`, `sentence-transformers`), and developer utilities (ruff, + pyright, typeguard, docs tooling, pytest plugins). + +3. Copy the sample environment file and adjust values as needed: + + ```bash + cp .env.example .env + ``` + +## 3. Provision external services + +The repository ships with a root `docker-compose.yml` that starts Redis, Memgraph, and +Neo4j with sane defaults. Run the stack in the background: + +```bash +docker compose up -d +``` + +> The default Neo4j credentials are `neo4j` / `meshminD123`. Memgraph exposes Bolt on +> `bolt://localhost:7687`, the Memgraph Lab UI on `http://localhost:3000`, and an +> additional monitoring endpoint on `http://localhost:7444`. + +### 3.1 Alternate topologies for tests + +For integration scenarios or CI jobs, use the compose files in +`meshmind/tests/docker/`: + +| File | Purpose | +| --- | --- | +| `memgraph.yml` | Runs only Memgraph with local port mapping. | +| `neo4j.yml` | Runs only Neo4j with APOC enabled. | +| `redis.yml` | Runs only Redis with persistence. | +| `full-stack.yml` | Spins up all services plus an optional Celery worker. | + +Example (Memgraph only): + +```bash +docker compose -f meshmind/tests/docker/memgraph.yml up -d +``` + +### 3.2 Cleaning up + +```bash +docker compose down -v +``` + +## 4. Configure environment variables + +Update `.env` (or export variables in your shell) with the credentials for each +service: + +| Variable | Description | Default | +| --- | --- | --- | +| `MEMGRAPH_URI` | Bolt URI for Memgraph. | `bolt://localhost:7687` | +| `NEO4J_URI` | Bolt URI for Neo4j. | `bolt://localhost:7688` | +| `NEO4J_USERNAME` / `NEO4J_PASSWORD` | Neo4j auth values. | `neo4j` / `meshminD123` | +| `REDIS_URL` | Redis connection string. | `redis://localhost:6379/0` | +| `OPENAI_API_KEY` | Key for OpenAI embedding endpoints. | _none_ | +| `SENTENCE_TRANSFORMERS_MODEL` | HuggingFace model ID used for local embeddings. | `all-MiniLM-L6-v2` | +| `DEFAULT_ENCODER` | Preferred encoder alias (`openai` or `sentence-transformers`). | `sentence-transformers` | + +Optional variables for experimentation: + +- `SQLITE_PATH` – override the path for the SQLite prototype driver. +- `GRAPH_BACKEND` – choose between `memory`, `sqlite`, `memgraph`, and `neo4j`. +- `CELERY_BROKER_URL` – override the broker used by Celery (defaults to `REDIS_URL`). + +## 5. Smoke tests + +After installing dependencies and starting the services: + +```bash +make test +meshmind admin graph --backend memgraph +meshmind admin graph --backend neo4j +meshmind admin counts --backend sqlite +``` + +You should see passing pytest output and successful graph connectivity checks. Refer to +`docs/troubleshooting.md` if any of the services fail health checks. diff --git a/SOT.md b/SOT.md new file mode 100644 index 0000000..3c09166 --- /dev/null +++ b/SOT.md @@ -0,0 +1,141 @@ +# MeshMind Source of Truth + +This document summarizes the current architecture, supporting assets, and operational expectations for MeshMind. Update it +whenever workflows or modules change so new contributors can find accurate information in one place. + +## Repository Layout +``` +meshmind/ +├── api/ # MemoryManager CRUD adapter plus REST/gRPC service layers +├── cli/ # CLI entry point, ingest command, and admin utilities +├── client.py # High-level MeshMind façade and orchestration helpers +├── core/ # Configuration, embeddings, types, similarity, shared utilities +├── db/ # Graph driver abstractions plus in-memory, SQLite, Memgraph, and Neo4j implementations +├── models/ # Entity and predicate registries shared across the pipeline +├── pipeline/ # Extract, preprocess, compression, storage, consolidation, expiry stages with telemetry hooks +├── retrieval/ # Search strategies (hybrid, lexical, fuzzy, vector, regex, rerank helpers) +├── tasks/ # Celery beat schedules and maintenance jobs +├── testing/ # Fake drivers (Memgraph, Redis, embedding) for offline tests +├── _compat/ # Compatibility shims (e.g., fallback Pydantic base classes) +├── tests/ # Pytest suites with local fixtures (no external services required) +└── examples/ # Scripts and notebooks showing ingestion and retrieval flows +``` +Supporting assets: +- `Makefile`: Development automation (linting, formatting, type checks, docs guard, docker compose). +- `docker-compose.yml`: Provisions Memgraph, Neo4j, and Redis for local orchestration; targeted stacks for tests live in + `meshmind/tests/docker/` (Memgraph-only, Neo4j-only, Redis-only, and full integration). +- `SETUP.md`: End-to-end provisioning instructions covering Python deps, environment variables, and Compose workflows. +- `.github/workflows/ci.yml`: GitHub Actions workflow running linting/formatting checks and pytest. +- `pyproject.toml`: Project metadata and dependency list (pins Python `>=3.11,<3.13`; see compatibility notes in `ISSUES.md`). +- Documentation (`PROJECT.md`, `PLAN.md`, `SOT.md`, `README.md`, etc.) describing the system and roadmap. + +## Configuration (`meshmind/core/config.py`) +- Loads environment variables for the active graph backend (`GRAPH_BACKEND`), Memgraph (`MEMGRAPH_URI`, `MEMGRAPH_USERNAME`, + `MEMGRAPH_PASSWORD`), SQLite (`SQLITE_PATH`), optional Neo4j (`NEO4J_URI`, `NEO4J_USERNAME`, `NEO4J_PASSWORD`), Redis + (`REDIS_URL`), OpenAI (`OPENAI_API_KEY`), and the default embedding model (`EMBEDDING_MODEL`). +- Uses `python-dotenv` when available to hydrate values from a `.env` file automatically. +- Provides a module-level `settings` instance used across the client, drivers, CLI, and Celery tasks. + +## Core Data Models (`meshmind/core/types.py`) +- `Memory`: Pydantic model (or compatibility fallback) that represents a knowledge record, including embeddings, metadata, and optional TTL. +- `Triplet`: Subject–predicate–object edge connecting two memory UUIDs with namespace and metadata. +- `SearchConfig`: Retrieval configuration (encoder name, `top_k`, `rerank_k`, optional rerank model, metadata filters, + hybrid weights). + +## Client (`meshmind/client.py`) +- `MeshMind` bootstraps: + - Default OpenAI client (Responses API) when the SDK is installed; custom clients can be injected for testing. + - Embedding model from configuration with encoder bootstrap that registers available adapters. + - Graph driver factory that creates the configured backend (memory, SQLite, Memgraph, Neo4j) lazily when persistence is required. +- Provides convenience helpers: + - Pipelines: `extract_memories`, `deduplicate`, `score_importance`, `compress`, `store_memories`, `store_triplets`. + - CRUD: `create_memory`, `update_memory`, `delete_memory`, `get_memory`, + `list_memories(namespace=None, entity_labels=None, offset=0, limit=None, query=None, use_search=None)`, + `memory_counts`, `list_triplets`. + - Retrieval: `search` (hybrid + optional LLM rerank), `search_vector`, `search_regex`, `search_exact` with automatic graph-backed loading when `memories` is omitted. +- Exposes `graph_driver`/`driver` properties that surface the active graph driver instance on demand. + +## Embeddings & Utilities (`meshmind/core/embeddings.py`, `meshmind/core/utils.py`) +- `EncoderRegistry` manages encoder instances (OpenAI embeddings, sentence-transformers, custom fixtures). +- OpenAI and SentenceTransformer adapters provide encoding with retry logic and optional fallbacks. +- Utility functions provide UUIDs, timestamps, hashing, and token counting guarded behind optional `tiktoken` imports. + +## Database Layer (`meshmind/db`) +- `GraphDriver` defines the persistence contract (entity/relationship upserts, querying, deletions, triplet listing) and now standardises pagination (`offset`, `limit`), server-side search (`search_entities`), and aggregated counts (`count_entities`). +- `InMemoryGraphDriver` and `SQLiteGraphDriver` power local development/testing without external services while supporting the extended contract. +- `MemgraphDriver` wraps `mgclient`, handles URI parsing, executes Cypher statements, and exposes Cypher-based filtering, + pagination, and aggregation helpers alongside the Python-side vector search fallback when database-native similarity is + unavailable. +- `Neo4jGraphDriver` mirrors the Memgraph contract using the official driver (optional dependency) and now exposes server-side search and counts. +- `factory.py` exposes helpers (`create_graph_driver`, `graph_driver_factory`) to instantiate backends based on configuration. + +## Pipeline Modules (`meshmind/pipeline`) +1. **Extraction (`extract.py`)** – Orchestrates OpenAI function calling against the `Memory` schema, enforces entity label filters, + and populates embeddings via registered encoders. +2. **Preprocess (`preprocess.py`)** – Deduplicates by name/embedding similarity, ensures memories have importance scores while + recording telemetry statistics, and delegates to compression when available. +3. **Compress (`compress.py`)** – Truncates metadata payloads to configurable token budgets when `tiktoken` is installed and records telemetry counters/durations. +4. **Store (`store.py`)** – Persists memories and triplets using the configured `GraphDriver`, registering predicates as needed and emitting observability events. +5. **Consolidate & Expire (`consolidate.py`, `expire.py`)** – Maintenance utilities triggered by Celery tasks to group memories, + apply batch/backoff settings, surface skipped groups, and remove stale entries. + +## Service Layers (`meshmind/api`) +- `memory_manager.py`: CRUD façade over the active graph driver that forwards namespace/entity-label filters, pagination hints, search strings, and exposes aggregate counts alongside triplet listings. +- `service.py`: Pydantic payloads and orchestration helpers shared by REST/gRPC surfaces. `MemoryService.search` leans on driver-side filtering before ranking and exposes `memory_counts` for CLI/HTTP usage. +- `rest.py`: `create_app` returns a FastAPI application when available or a `RestAPIStub` for tests. Routes support pagination parameters and include `/memories/counts` for namespace/label summaries. +- `grpc.py`: `GrpcServiceStub` plus simple request/response dataclasses mirroring planned RPCs. + +## Retrieval (`meshmind/retrieval`) +- `filters.py`: Namespace, entity label, and metadata filtering helpers. +- `bm25.py`, `fuzzy.py`: Lexical and fuzzy scorers using scikit-learn TF-IDF + cosine and RapidFuzz WRatio, with pure-Python fallbacks when optional dependencies are unavailable. +- `vector.py`: Vector-only search utilities with cosine similarity and optional precomputed query embeddings. +- `hybrid.py`: Combines vector and BM25 scores with configurable weights defined in `SearchConfig`. +- `search.py`: Dispatchers for hybrid, BM25, fuzzy, vector, regex, and exact-match search modes plus metadata filters. +- `rerank.py`: Generic reranker interface and LLM-based rerank helper compatible with the OpenAI Responses API. +- `graph.py`: Wrappers that pull candidates from the active graph driver before delegating to the existing search strategies. + +## CLI (`meshmind/cli`) +- `meshmind.cli.__main__`: Entry point exposing `ingest` and `admin` subcommands for local pipelines and maintenance tooling. +- CLI bootstraps encoder and entity registries, validates configuration early, surfaces actionable errors when optional + dependencies are missing, and routes predicate maintenance, telemetry inspection, and graph connectivity checks through + `meshmind.cli.admin`. + +## Tasks (`meshmind/tasks`) +- `celery_app.py`: Creates the Celery application lazily, returning a shim when Celery is not installed. +- `scheduled.py`: Defines periodic consolidation, compression, and expiry jobs that initialize drivers and managers lazily, + emit observability events, persist updated memories, surface skipped consolidation groups, and tolerate missing dependencies + during import. + +## API Adapter (`meshmind/api/memory_manager.py`) +- Manages CRUD operations against the graph driver, including triplet persistence and deletion helpers. +- Returns Pydantic models for list/get operations and gracefully handles missing records. + +## Models (`meshmind/models/registry.py`) +- `EntityRegistry` and `PredicateRegistry` store class metadata and permitted predicates. +- Registries are populated during bootstrap and extended as new entity/predicate types are defined. + +## Examples & Tests +- `examples/extract_preprocess_store_example.py`: Demonstrates extraction, preprocessing, triplet creation, and multiple + retrieval strategies. +- `meshmind/tests`: Pytest suites rely on fixtures (`memory_factory`, `dummy_encoder`, in-memory drivers, service stubs) and + pure-Python doubles (compatibility BaseModel, fake Memgraph/Redis/embedding drivers), allowing the suite to run without Memgraph, OpenAI, or Redis dependencies. + +- Required: `openai`, `pydantic`, `pydantic-settings`, `python-dotenv`. Install `mgclient` when using Memgraph and `neo4j` + when targeting Neo4j. Pure-Python fallbacks exist for Pydantic, numpy, scikit-learn, and rapidfuzz but production deployments + should install the real packages. +- Optional but supported: `tiktoken`, `sentence-transformers`, `celery[redis]`, `fastapi`, `uvicorn[standard]`, `redis`, `httpx`, + `pytest-cov`. +- Development tooling introduced in the Makefile/CI expects `ruff`, `pyright`, `typeguard`, `toml-sort`, `yamllint`, `mkdocs`, and + `mkdocs-material` (now bundled in the extras). + +## Operational Notes +- Graph persistence requires a configured backend: in-memory/SQLite need no services; Memgraph requires a running instance + reachable via `settings.MEMGRAPH_URI` and `mgclient`; Neo4j requires the official driver and credentials. Use `meshmind admin + graph --backend ` to sanity check connectivity or run the compose stacks. +- Encoder registration occurs during bootstrap; ensure at least one embedding encoder is available before extraction/search. +- LLM reranking uses the OpenAI Responses API. Provide `OPENAI_API_KEY` and confirm the selected `SearchConfig.rerank_model` is + deployed to your account. +- Local development commands rely on external tooling (ruff, pyright, typeguard, toml-sort, yamllint); install them via the + Makefile or the CI workflow instructions in `README.md`. +- Docker Compose can be used to run Memgraph/Neo4j/Redis locally; ensure container tooling is available or provision services + manually. diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..f54b00f --- /dev/null +++ b/TODO.md @@ -0,0 +1,56 @@ +# TODO + +## Completed + +- [x] Implement dependency guards and lazy imports for optional packages (`mgclient`, `tiktoken`, `celery`, `sentence-transformers`). +- [x] Add bootstrap helper for default encoder registration and call it from the CLI. +- [x] Update OpenAI encoder implementation to align with latest SDK responses and retry semantics. +- [x] Improve configuration guidance and automation for environment variables and service setup. +- [x] Wire `EntityRegistry` and `PredicateRegistry` into the storage pipeline and client. +- [x] Implement CRUD and triplet methods on `MeshMind`, including relationship persistence in `GraphDriver`. +- [x] Refresh examples to cover relationship-aware ingestion and retrieval flows. +- [x] Extend retrieval module with vector-only, regex, exact-match, and optional LLM rerank search helpers. +- [x] Modernize pytest suites and add fixtures to run without external services. +- [x] Expand Makefile and add CI workflows for linting, testing, and type checks. +- [x] Document or provision local Memgraph and Redis services (e.g., via docker-compose) for onboarding. +- [x] Abstract `GraphDriver` to support alternative storage backends (Neo4j, in-memory, SQLite prototype). +- [x] Add service interfaces (REST/gRPC) for ingestion and retrieval. +- [x] Introduce observability (logging, metrics) for ingestion and maintenance pipelines. +- [x] Promote the new README, archive the legacy version, and keep SOT diagrams/maps in sync. +- [x] Harden Celery maintenance tasks to initialize drivers lazily and persist consolidation results. +- [x] Replace constant importance scoring with a heuristic driven by token diversity, recency, metadata richness, and embedding magnitude. +- [x] Create fake Memgraph, Redis, and embedding drivers for testing purposes. +- [x] Expand `GraphDriver.list_entities` to support namespace/entity-label filters and propagate the behaviour through `MemoryManager`, graph retrieval wrappers, and the MeshMind client. +- [x] Extend REST/gRPC payloads, CLI helpers, and pytest coverage to exercise the new entity-label filtering semantics. +- [x] Stand up `docs/` wiki pages, `ENVIRONMENT_NEEDS.md`, and `RESUME_NOTES.md` so documentation and session hand-off stay current. +- [x] Add unit tests covering namespace/entity-label filtering for the SQLite driver and fake drivers. +- [x] Update the example pipeline (`examples/extract_preprocess_store_example.py`) to demonstrate entity-label restricted retrieval via the MeshMind client. +- [x] Document REST/gRPC request samples that include `entity_labels` in `docs/api.md`. +- [x] Add a regression test confirming `MeshMind.list_memories` forwards `entity_labels` to the memory manager. +- [x] Push graph-backed retrieval queries deeper into Memgraph/Neo4j backends so search executes without materializing entire namespaces. +- [x] Implement pagination/streaming options in `MemoryManager.list_memories` to avoid loading entire namespaces into memory. +- [x] Add CLI/admin command to report memory counts grouped by namespace and entity label for quick health checks. +- [x] Create developer tooling (pre-commit or CI check) that ensures `docs/` pages are touched when code under corresponding modules changes. +- [x] Draft a troubleshooting section documenting optional tooling installation failures (ruff, pyright, typeguard, toml-sort, yamllint) and recommended fallbacks. +- [x] Expose `memory_counts` via the gRPC stub to keep service interfaces aligned. + +- [x] Extend the docs guard mapping/tests so Docker, setup, and environment guides are enforced when related modules change. +## Priority Tasks + +- [ ] Regenerate `uv.lock` to align with the updated dependency set (`fastapi`, `uvicorn`, `neo4j`, `mgclient`, extras) once package downloads are possible (blocked: pip cannot access PyPI from this environment). +- [ ] Validate Neo4j driver requirements and connectivity against a live cluster (exercise CLI admin checks end-to-end). +- [ ] Validate consolidation heuristics on larger datasets and define conflict-resolution/backoff strategies for maintenance writes. +- [ ] Establish evaluation loops (analytics or LLM-assisted) to tune the new importance heuristic over time. +- [ ] Replace the compatibility shim with production Pydantic models once upstream packaging supports the target Python versions. +- [ ] Verify curl/grpcurl snippets against running REST/gRPC services once infrastructure is available. +- [ ] Create automated smoke tests for REST `/memories/counts` and `meshmind admin counts` against a live backend when infrastructure is ready. +- [ ] Add gRPC proto definitions and generated clients so the Python stubs align with production servers (including `MemoryCounts`). +- [ ] Benchmark driver-side pagination/filtering on large datasets to tune default candidate limits and document recommended overrides. +- [ ] Implement backend-native vector similarity queries for Memgraph/Neo4j to eliminate Python-side scoring when embeddings are present. + +## Recommended Waiting for Approval Tasks + +- [ ] Provision Neo4j, Memgraph, and Redis instances accessible from the development environment to unblock live integration tests (requires infrastructure approval). +- [ ] Approve installation of optional dependencies (`neo4j`, `mgclient`, `redis`, `celery`, `tiktoken`, `sentence-transformers`) across CI and developer machines to exercise full workflows. +- [ ] Source or generate large synthetic datasets for consolidation and retrieval benchmarking to validate heuristics under load. +- [ ] Define a policy for reintroducing Pydantic models (version targets, rollout timeline) so compatibility shims can be retired once approved. diff --git a/docker-compose.yml b/docker-compose.yml index 96be0b4..cdf9fca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,15 +1,73 @@ -version: "3.8" +version: "3.9" + services: memgraph: image: memgraph/memgraph-platform:latest + container_name: meshmind-memgraph + restart: unless-stopped ports: - "7687:7687" + - "7444:7444" - "3000:3000" + healthcheck: + test: ["CMD", "bash", "-c", "cypher-shell --version || exit 1"] + interval: 15s + timeout: 10s + retries: 5 + environment: + MEMGRAPH_MEMORY_LIMIT: 2GB + volumes: + - memgraph-data:/var/lib/memgraph + networks: + - meshmind + + neo4j: + image: neo4j:5.26 + container_name: meshmind-neo4j + restart: unless-stopped + ports: + - "7688:7687" + - "7474:7474" + environment: + NEO4J_AUTH: neo4j/meshminD123 + NEO4J_PLUGINS: '["apoc"]' + NEO4J_apoc_export_file_enabled: "true" + NEO4J_apoc_import_file_enabled: "true" + NEO4J_dbms_security_procedures_unrestricted: apoc.* + healthcheck: + test: ["CMD", "cypher-shell", "-u", "neo4j", "-p", "meshminD123", "RETURN 1"] + interval: 15s + timeout: 10s + retries: 5 + volumes: + - neo4j-data:/data + - neo4j-logs:/logs + networks: + - meshmind + redis: image: redis:7-alpine - worker: - build: . - command: celery -A meshmind.tasks.celery_app worker -B -l info - depends_on: - - memgraph - - redis \ No newline at end of file + container_name: meshmind-redis + restart: unless-stopped + command: redis-server --save 60 1000 --loglevel warning + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + volumes: + - redis-data:/data + networks: + - meshmind + +networks: + meshmind: + name: meshmind + +volumes: + memgraph-data: + neo4j-data: + neo4j-logs: + redis-data: diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..f397039 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,71 @@ +# Service Interfaces & CLI + +MeshMind exposes multiple integration points for ingestion and retrieval workflows. + +## Memory Service (`meshmind.api.service`) + +- `MemoryService` encapsulates ingestion (`ingest_memories`, `ingest_triplets`), search, and CRUD operations. +- `SearchPayload` accepts `entity_labels`, allowing clients to restrict search to specific entity types before hybrid + ranking occurs. +- `MemoryPayload` and `TripletPayload` are Pydantic-compatible models shared by REST and gRPC stubs. + +## REST API (`meshmind.api.rest`) + +- `create_app` dynamically returns a FastAPI application if FastAPI is installed, otherwise a lightweight stub. +- Routes: + - `POST /memories`: ingest a batch of memories. + - `POST /triplets`: ingest relationships. + - `POST /search`: execute hybrid search. + - `GET /memories`: list memories (accepts `namespace`, `entity_labels`, `offset`, `limit`, `query`, `use_search`). + - `GET /triplets`: list triplets. + - `GET /memories/counts`: summarize memory counts grouped by namespace and entity label. +- Sample request bodies: + ```json + { + "query": "python", + "namespace": "demo", + "entity_labels": ["Memory"], + "top_k": 5, + "encoder": "text-embedding-3-small" + } + ``` + ```json + { + "namespace": "demo", + "entity_labels": ["Memory"], + "limit": 20, + "query": "python" + } + ``` + { + "namespace": "demo" + } + ``` +- The `RestAPIStub` mirrors these routes for tests without requiring FastAPI. + +## gRPC Stub (`meshmind.api.grpc`) + +- Provides dataclasses representing typical RPC messages. +- `GrpcServiceStub` implements the service interface in pure Python for unit tests and demos. +- `SearchRequest` mirrors the REST payload including `entity_labels` to ensure consistent filtering semantics. +- `MemoryCountsRequest` returns namespace/entity-label aggregates so service parity with REST/CLI is preserved. + +## CLI (`meshmind/cli`) + +- `meshmind.cli.__main__` bootstraps default encoders, validates configuration, and exposes ingestion commands. +- `meshmind.cli.admin` contains administrative tasks for registry inspection, backend connectivity checks, memory counts, + and maintenance triggers. + +## Client (`meshmind/client.py`) + +- `MeshMind` client wraps all pipelines and persistence methods. +- CRUD and search helpers delegate to the memory manager while respecting namespace/entity label filters. +- Search helpers automatically fetch candidates from the graph when `memories` is omitted, ensuring consistent + behaviour across service surfaces. + +## Example Workflow + +1. Instantiate the client: `mm = MeshMind()` (ensure environment variables are set for your backend). +2. Ingest data using CLI (`meshmind ingest ...`) or the REST API. +3. Retrieve memories via REST (`POST /search`) or the Python client (`mm.search(...)`). +4. Monitor metrics/logs through the observability utilities (`docs/telemetry.md`). diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..794d7c9 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,62 @@ +# Architecture + +MeshMind is organized around a layered architecture that separates extraction, persistence, and retrieval concerns while +keeping the graph storage pluggable. + +## High-Level Flow + +1. **Extraction** (`meshmind.pipeline.extract`) + - Uses instructions plus LLM client and embeddings to convert raw content into `Memory` objects and `Triplet` + relationships. +2. **Preprocessing** (`meshmind.pipeline.preprocess`) + - Deduplicates, scores importance, compresses content, and prepares payloads for persistence. +3. **Storage** (`meshmind.pipeline.store`) + - Persists memories and triplets through the active graph driver, registering entity/predicate schemas along the way. +4. **Retrieval** (`meshmind.retrieval`) + - Provides hybrid search, vector-only, regex, exact, fuzzy, BM25, and optional LLM rerank flows across stored + memories. +5. **Maintenance** (`meshmind.pipeline.consolidate`, `meshmind.tasks.scheduled`) + - Consolidates duplicates, expires stale items, and runs periodic graph hygiene tasks. + +## Key Components + +- **MeshMind Client (`meshmind/client.py`)** + - High-level façade that wires pipelines, storage, and retrieval helpers. Lazily loads a graph driver via the factory + configured by `meshmind.core.config.settings`. + - Provides CRUD utilities, search helpers, and bootstrap logic for registries and encoders. +- **Memory Service Layer (`meshmind/api/service.py`)** + - Encapsulates ingestion/search/triplet operations, enabling REST/gRPC adapters and CLI commands to share behaviour. + - Applies configuration (e.g., `SearchConfig`) and now filters `list_memories` calls by namespace and entity labels to + minimize driver loads. +- **Graph Drivers (`meshmind/db/*`)** + - `InMemoryGraphDriver`: simple dictionaries for tests and demos. + - `SQLiteGraphDriver`: lightweight relational persistence using JSON columns. + - `Neo4jGraphDriver` and `MemgraphDriver`: Bolt-based implementations using Cypher. + - All drivers implement the expanded `GraphDriver.list_entities(namespace, entity_labels)` contract for efficient + filtering at the storage layer. +- **Observability (`meshmind/core/observability.py`)** + - Tracks metrics, counters, and structured logs for pipeline operations. + +## Data Model + +Memories capture: + +- `uuid`: unique identifier. +- `namespace`: logical partition to isolate tenants/workloads. +- `entity_label`: semantic label for type filtering. +- `embedding`: vector representation for similarity search. +- `metadata`: arbitrary JSON payload with content, source, annotations. +- `importance`, `ttl_seconds`, `reference_time`: scheduling and scoring hints. + +Triplets connect memories with: + +- `subject`, `predicate`, `object`: relationship endpoints. +- `entity_label`: stored predicate label to support filtering. +- `metadata`: additional context for the relationship. + +## Extensibility + +- Register new entity or predicate schemas via `EntityRegistry` and `PredicateRegistry` before persistence. +- Provide alternate graph drivers by subclassing `GraphDriver` and implementing the CRUD/search primitives. +- Extend retrieval strategies by adding new modules under `meshmind/retrieval` and wiring them through the client and + service layers. diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..5e14459 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,37 @@ +# Configuration Reference + +MeshMind configuration is derived from environment variables and optional `.env` files (loaded when `python-dotenv` is +installed). + +## Core Settings + +| Variable | Description | Default | +| -------- | ----------- | ------- | +| `GRAPH_BACKEND` | Storage backend (`memory`, `sqlite`, `neo4j`, `memgraph`). | `memory` | +| `MEMGRAPH_URI` | Bolt URI for Memgraph. | `bolt://localhost:7687` | +| `MEMGRAPH_USERNAME` / `MEMGRAPH_PASSWORD` | Credentials for Memgraph. | empty | +| `NEO4J_URI` | Bolt URI for Neo4j. | `bolt://localhost:7688` | +| `NEO4J_USERNAME` / `NEO4J_PASSWORD` | Credentials for Neo4j. | `neo4j` / `meshminD123` | +| `SQLITE_PATH` | File path for SQLite backend. | `:memory:` | +| `REDIS_URL` | Redis connection string for caching/background tasks. | `redis://localhost:6379/0` | +| `OPENAI_API_KEY` | API key for OpenAI client construction. | empty | +| `EMBEDDING_MODEL` | Default embedding model identifier. | `text-embedding-3-small` | + +## Derived Behaviour + +- `Settings.missing()` reports missing variables based on the selected backend (e.g., Memgraph credentials ignored when + not using Memgraph). +- CLI commands (`meshmind.cli.__main__`) surface missing configuration early so ingestion fails fast. +- The MeshMind client reads settings during initialization to configure the default LLM client and graph driver factory. + +## Recommended Secrets Management + +- Store secrets in a `.env` file for local development (loaded automatically when `python-dotenv` is present). +- Use environment-specific secret managers (Vault, AWS Secrets Manager, etc.) for production deployments. +- Rotate API keys/credentials regularly and update the environment variables accordingly. + +## Extending Configuration + +- Add new settings to `meshmind/core/config.py` with sensible defaults. +- Update `Settings.REQUIRED_GROUPS` when additional capabilities require environment validation. +- Document the new variables here, in `README.md`, and in `ENVIRONMENT_NEEDS.md` if they require provisioning support. diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..8be3fad --- /dev/null +++ b/docs/development.md @@ -0,0 +1,44 @@ +# Development Workflow + +This guide summarizes expectations when contributing to MeshMind. + +## Prerequisites + +- Python 3.11 or 3.12 is recommended (see `pyproject.toml`). +- Install dependencies with `pip install -e .[dev,docs,testing]` (or `uv pip install --system -e .[dev,docs,testing]`; drop + `--system` when using an activated virtualenv). +- Optional extras: none—`.[dev,docs,testing]` pulls in REST tooling (`fastapi`, `uvicorn`), graph drivers (`neo4j`, `mgclient`, + `redis`), LLM tooling (`openai`, `tiktoken`, `sentence-transformers`), and developer utilities. Refer to `SETUP.md` for + service provisioning steps. + +## Coding Standards + +- Follow the style rules documented in `AGENTS.md` (120-character lines, meaningful headings, bullet lists for + enumerations). +- Keep code modular and dependency-light to maintain deterministic tests. +- Avoid wrapping imports in `try/except` unless explicitly handling optional dependencies. + +## Documentation + +- Update `README.md`, `CHANGELOG.md`, `docs/`, `SOT.md`, and other root markdown files whenever behaviour changes. +- Each change batch must append a timestamped entry to `CHANGELOG.md` describing modules, functions, and rationale. +- Maintain `RESUME_NOTES.md` at the end of every turn to capture context for future sessions. +- Run `make docs-guard` (optionally with `BASE_REF=`) before pushing to ensure code changes have matching documentation updates. + +## Testing + +- Run `pytest` before committing. +- Add unit tests for new features, especially when touching drivers, pipelines, or retrieval logic. +- Use the fake drivers and fixtures to avoid requiring external services. + +## Git Workflow + +- Work on feature branches derived from the current branch (e.g., `work`). +- Commit logically grouped changes with descriptive messages. +- Generate PR summaries via the automated tooling once commits are ready. + +## Continuous Integration + +- `.github/workflows/ci.yml` runs linting and tests; ensure new code paths are covered. +- The Makefile includes shortcuts (`make test`, `make lint`, `make format`) for local validation. +- CI also executes `make docs-guard` so missing documentation updates will fail the pipeline. diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..84b5ecd --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,48 @@ +# Operations & Maintenance + +This guide covers operational tasks for MeshMind deployments. + +## Configuration + +- Environment variables are read via `meshmind.core.config.Settings`. +- Critical variables: + - `GRAPH_BACKEND`: `memory`, `sqlite`, `neo4j`, or `memgraph`. + - Connection URIs/usernames/passwords for Neo4j/Memgraph. + - `SQLITE_PATH` when using the SQLite backend. + - `REDIS_URL`, `OPENAI_API_KEY`, `EMBEDDING_MODEL` for optional integrations. +- `settings.missing()` groups missing variables by capability to simplify diagnostics. + +## Bootstrap & Registries + +- `meshmind.core.bootstrap.bootstrap_entities` ensures the `Memory` model is registered. +- `meshmind.core.bootstrap.bootstrap_encoders` primes default encoders (OpenAI if available). +- Registry state (entities/predicates) persists in process memory; configure the CLI/admin tasks to inspect/refresh when + adding new schema types. + +## Scheduled Tasks + +- `meshmind.tasks.scheduled.expire_task`: prunes expired memories based on TTL metadata. +- `meshmind.tasks.scheduled.consolidate_task`: merges duplicates, persists consolidation results, and emits telemetry. +- `meshmind.tasks.scheduled.compress_task`: re-compresses long memories and persists updates to the configured backend. +- All tasks rely on `MemoryManager.list_memories` and respect namespace/label filters for efficiency. + +## Observability + +- `meshmind.core.observability.telemetry` exposes counters and gauges with `snapshot()` for inspection. +- The `docs/telemetry.md` page lists the available metrics and logging patterns. + +## Admin CLI + +- `meshmind.cli.admin` provides commands to: + - Validate backend connectivity (Neo4j/Memgraph/SQLite). + - Summarize stored memories via `mesh admin counts --namespace ` grouped by entity label. + - Run consolidation plans manually. + - Inspect registry contents. + - Summarize configuration with sensitive values masked. + +## Deployment Considerations + +- Provision graph databases externally (Docker, managed service) and expose Bolt endpoints reachable from the runtime. +- Ensure optional dependencies (`neo4j`, `mgclient`, `redis`, `celery`, `fastapi`, `uvicorn`, `tiktoken`) are installed where required (or install `.[dev,docs,testing]`). +- Configure logging/metrics sinks to capture telemetry emitted by pipeline stages. +- Use `docker-compose.yml` as a reference for local orchestration (Memgraph, Neo4j, Redis) and the targeted stacks in `meshmind/tests/docker/` for integration testing. diff --git a/docs/overview.md b/docs/overview.md new file mode 100644 index 0000000..560311b --- /dev/null +++ b/docs/overview.md @@ -0,0 +1,34 @@ +# MeshMind Overview + +MeshMind is a modular framework for extracting, storing, and retrieving "memories"—structured knowledge units that +combine embeddings, metadata, and graph relationships. The platform orchestrates LLM-powered extraction pipelines, +pluggable persistence backends, and multiple retrieval strategies so applications can ingest heterogeneous content and +query it with hybrid search techniques. + +## Core Concepts + +- **Memories** (`meshmind.core.types.Memory`): normalized documents with embeddings, importance scores, TTLs, and + metadata. +- **Triplets** (`meshmind.core.types.Triplet`): relationship edges connecting memories by subject/predicate/object. +- **Graph Drivers** (`meshmind.db`): persistence adapters for in-memory usage, SQLite prototyping, and Bolt-compatible + services (Neo4j, Memgraph). +- **Pipelines** (`meshmind.pipeline`): modular stages for extraction, preprocessing, compression, and storage. +- **Retrieval** (`meshmind.retrieval`): text, vector, graph, and rerank utilities that operate on in-memory or driver- + backed memories. +- **Service Interfaces** (`meshmind.api`): REST/gRPC-compatible surfaces and CLI tooling to automate ingestion and + maintenance workflows. + +## Project Layout + +- `meshmind/core`: fundamental types, configuration, bootstrap helpers, encoders, similarity metrics, observability, + and shared utilities. +- `meshmind/pipeline`: ingestion lifecycle components (`extract`, `preprocess`, `store`, `compress`, `consolidate`, + `expire`). +- `meshmind/db`: graph drivers plus the factory that selects the active backend based on environment configuration. +- `meshmind/retrieval`: hybrid search primitives, graph-aware wrappers, and optional LLM rerank integration. +- `meshmind/api`: Memory service layer, REST/gRPC adapters, CLI entry points, and admin tooling. +- `meshmind/tests`: unit and integration-style tests that exercise pipelines, drivers, API stubs, and search flows + using in-memory or fake dependencies. +- `examples/`: runnable demos showing end-to-end extraction, preprocessing, and storage pipelines. + +Consult the other `docs/*.md` pages for deep dives into each subsystem. diff --git a/docs/persistence.md b/docs/persistence.md new file mode 100644 index 0000000..ed1d04e --- /dev/null +++ b/docs/persistence.md @@ -0,0 +1,62 @@ +# Persistence Layer + +MeshMind persists memories and triplets through interchangeable graph drivers. Each driver implements the +`meshmind.db.base_driver.GraphDriver` interface, which now accepts both namespace and entity-label filters for +`list_entities`. + +## Driver Capabilities + +| Driver | Module | Use Case | Notes | +| ------ | ------ | -------- | ----- | +| In-memory | `meshmind.db.in_memory_driver.InMemoryGraphDriver` | Tests and demos | Stores nodes/edges in Python dicts with UUID auto-generation helpers, plus in-process search/pagination implementations. | +| SQLite | `meshmind.db.sqlite_driver.SQLiteGraphDriver` | Lightweight persistence without external services | Uses two tables (`entities`, `triplets`) with JSON columns; supports namespace + label filtering, pagination, and SQL-based search. | +| Neo4j | `meshmind.db.neo4j_driver.Neo4jGraphDriver` | Production Bolt cluster | Requires the `neo4j` Python driver and connectivity to a running instance. Provides `verify_connectivity()` for health checks, server-side search, and aggregated counts. | +| Memgraph | `meshmind.db.memgraph_driver.MemgraphDriver` | Memgraph (Bolt-compatible) | Requires `mgclient`. Filters results using Cypher with optional namespace/label predicates, driver-side search, and count aggregation. | +| Fake drivers | `meshmind.testing.fakes` | Offline testing | Provide stubbed implementations for Redis, embedding models, and Memgraph. | + +## Driver Factory + +`meshmind.db.factory.graph_driver_factory` inspects `meshmind.core.config.settings.GRAPH_BACKEND` and constructs the +corresponding driver. Supported values: + +- `memory` +- `sqlite` +- `neo4j` +- `memgraph` + +Environment variables (see `README.md` and `ENVIRONMENT_NEEDS.md`) provide connection URIs and credentials. + +## MemoryManager + +`meshmind.api.memory_manager.MemoryManager` wraps driver operations and now exposes: + +```python +list_memories( + namespace: str | None = None, + entity_labels: Sequence[str] | None = None, + *, + offset: int = 0, + limit: int | None = None, + query: str | None = None, + use_search: bool | None = None, +) -> List[Memory] + +count_memories(namespace: str | None = None) -> Dict[str, Dict[str, int]] +``` + +Filtering and pagination at this layer keep retrieval queries efficient when large graphs are present. The manager defers +to driver-provided `search_entities` and `count_entities` helpers when available, then handles `add_memory`, +`update_memory`, `delete_memory`, and triplet equivalents by normalizing payloads and delegating to the driver. + +## Maintenance & Consolidation + +Scheduled tasks in `meshmind.tasks.scheduled` use the memory manager to fetch entities before running consolidation or +expiration strategies. Because `list_memories` accepts entity labels, maintenance jobs can focus on specific entity +classes without scanning the full graph. + +## Adding a New Driver + +1. Subclass `GraphDriver` and implement the abstract methods, including the namespace/label-aware `list_entities`. +2. Register the driver in `meshmind/db/factory.py` under a new backend key. +3. Provide fake/test doubles if the backend cannot run inside CI. +4. Update documentation (`docs/persistence.md`, `SOT.md`, `README.md`) and add tests validating CRUD behaviours. diff --git a/docs/pipelines.md b/docs/pipelines.md new file mode 100644 index 0000000..0dc1ea1 --- /dev/null +++ b/docs/pipelines.md @@ -0,0 +1,44 @@ +# Pipelines + +MeshMind pipelines orchestrate the transformation of raw content into stored memories and relationships. + +## Extract (`meshmind.pipeline.extract`) + +- `extract_memories` uses the configured LLM client to parse unstructured content into structured `Memory` objects. +- Supports configurable entity types, instructions, and embedding models. +- Output includes candidate triplets emitted by extraction heuristics. + +## Preprocess (`meshmind.pipeline.preprocess`) + +- `deduplicate`: removes near-duplicate memories based on embedding cosine similarity and metadata hashes. +- `score_importance`: applies heuristic scoring that considers recency, metadata richness, token diversity, and embedding + magnitude. +- `compress`: performs optional content compression if tiktoken is installed; otherwise acts as a no-op while recording + telemetry. + +## Store (`meshmind.pipeline.store`) + +- `store_memories`: upserts each `Memory` through the graph driver after ensuring the entity type is registered. +- `store_triplets`: persists relationships while registering predicates via `PredicateRegistry`. + +## Consolidate (`meshmind.pipeline.consolidate`) + +- `consolidate_memories`: groups similar memories, merges metadata, and produces a plan enumerating consolidation + outcomes for later application. + +## Maintenance (`meshmind.tasks.scheduled`) + +- Provides task stubs for Celery/cron that run consolidation plans, expire TTL-bound memories, and emit importance + telemetry snapshots. + +## Expiration (`meshmind.pipeline.expire`) + +- Contains helpers to remove expired memories (leveraged by maintenance tasks and CLI commands). + +## Usage Patterns + +1. Use the MeshMind client (`MeshMind.extract_memories`) or pipeline functions directly to generate memories. +2. Preprocess the output with `deduplicate`, `score_importance`, and `compress` depending on your quality requirements. +3. Persist via `store_memories` / `store_triplets` using your configured graph backend. +4. Retrieve with hybrid or specialized searches (see `docs/retrieval.md`). +5. Schedule maintenance tasks to keep the graph clean and heuristics calibrated. diff --git a/docs/retrieval.md b/docs/retrieval.md new file mode 100644 index 0000000..eaa6bfa --- /dev/null +++ b/docs/retrieval.md @@ -0,0 +1,57 @@ +# Retrieval Strategies + +MeshMind ships with multiple retrieval strategies that operate on `Memory` collections. Each strategy can run purely in +memory (provided via argument) or load candidates directly from the configured graph driver. + +## Hybrid Search + +`meshmind.retrieval.search.search` + +- Combines BM25 (text), vector similarity, regex, exact-match, and fuzzy signals. +- Accepts `SearchConfig` to tune weights, top-k counts, embedding encoder, and rerank parameters. +- Optional rerank step calls `meshmind.retrieval.llm_rerank.llm_rerank` using the active LLM client. + +Graph wrapper: `graph_hybrid_search(query, driver, namespace=None, entity_labels=None, config=None, reranker=None)` +leans on `MemoryManager.list_memories(..., query=query, use_search=True)` so Memgraph/Neo4j filter and paginate on the +server. The helper automatically expands the candidate window (`top_k * 5`, `rerank_k * 2`) to balance recall with +efficiency. + +## Vector Search + +`meshmind.retrieval.vector.search_vector` + +- Uses cosine similarity between query embedding and stored embeddings. +- Graph wrapper `graph_vector_search` now requests driver-side filtering (`query=`) and pagination, reducing + the number of memories materialised before vector scoring. + +## Textual Search + +- **Regex** (`search_regex` / `graph_regex_search`): applies compiled regular expressions against string fields. +- **Exact** (`search_exact` / `graph_exact_search`): equality comparisons with optional case sensitivity, field + selection, and top-k truncation. +- **Fuzzy** (`search_fuzzy` / `graph_fuzzy_search`): Levenshtein distance using `rapidfuzz` if installed. +- **BM25** (`search_bm25` / `graph_bm25_search`): rank text fields using BM25 weighting. + +## Entity Label Filtering & Pagination + +All graph-backed helpers accept `entity_labels`. The labels and pagination hints (`offset`, `limit`) are forwarded to +`MemoryManager` and ultimately to the graph driver so only relevant entity types are hydrated into Python. This prevents +unnecessary deserialization when a graph contains many heterogeneous memory types and unlocks efficient infinite-scroll or +batch processing patterns. + +## Search Configuration + +`SearchConfig` (`meshmind.core.types.SearchConfig`) controls: + +- `top_k`: maximum results to return. +- `encoder`: embedding model name (default comes from settings and the MeshMind client). +- `rerank_k`: number of documents to rerank with LLM. +- `fields`: optional mapping for textual searches (regex, exact, fuzzy) to target metadata keys. + +## Extending Retrieval + +1. Add a new module under `meshmind/retrieval` with a function that accepts `(query, memories, **kwargs)`. +2. Update `meshmind/retrieval/__init__.py` and the MeshMind client to expose the helper. +3. Create graph wrappers if the strategy benefits from driver-backed loading. +4. Add unit tests in `meshmind/tests/test_retrieval.py` or a dedicated module to ensure determinism. +5. Document the feature here and in `README.md`. diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 0000000..e751a40 --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,31 @@ +# Telemetry & Observability + +MeshMind provides lightweight observability utilities under `meshmind.core.observability` to trace pipeline behaviour. + +## Telemetry API + +- `telemetry.reset()`: clear counters/gauges (used in tests). +- `telemetry.increment(name, value=1.0, tags=None)`: bump a counter. +- `telemetry.gauge(name, value, tags=None)`: record the latest gauge value. +- `telemetry.timer(name, tags=None)`: context manager to measure elapsed time. +- `telemetry.snapshot()`: return a dictionary of counters/gauges/timers for inspection. + +## Instrumented Components + +- `meshmind.pipeline.preprocess.score_importance`: records importance distribution metrics. +- `meshmind.pipeline.compress.compress`: wraps compression attempts to measure latency and failure rates. +- `meshmind.pipeline.store` and `meshmind.tasks.scheduled`: emit counters for stored items and maintenance plans. +- Observability hooks are intentionally dependency-free to keep tests deterministic. + +## Integrations + +- For production use, wrap the telemetry API with adapters that forward metrics to Prometheus, StatsD, or logging + systems. +- Configure logging handlers (see `logging` usage throughout the pipeline modules) to integrate with your preferred log + aggregation stack. + +## Best Practices + +- Tag metrics with `namespace` when running multi-tenant workloads. +- Reset telemetry within tests to avoid cross-test contamination. +- Extend the telemetry module with thread-safe transports if moving beyond single-process deployments. diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..87a37f8 --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,43 @@ +# Testing & Quality + +MeshMind includes an extensive pytest suite that runs without external services by relying on fake drivers and in-memory +backends. + +## Test Topology + +- `meshmind/tests/test_pipeline_*`: validate extraction, preprocessing, storage, and maintenance flows with dummy + drivers. +- `meshmind/tests/test_db_*`: ensure graph drivers implement CRUD semantics (with fakes for optional dependencies). +- `meshmind/tests/test_retrieval.py` and `test_graph_retrieval.py`: exercise hybrid, textual, vector, and graph-based + searches. +- `meshmind/tests/test_service_interfaces.py`: cover REST and gRPC stubs, including entity-label filtering, pagination hints, and memory count routes. +- `meshmind/tests/test_cli_admin.py`: verify administrative CLI commands use the correct driver factory, settings, and counts reporting. +- `meshmind/tests/test_docs_guard.py`: ensure the documentation guard script enforces wiki updates when code modules change. +- `meshmind/tests/test_observability.py`: confirm telemetry metrics/counters update during preprocessing steps. + +## Fakes & Fixtures + +- `meshmind.testing.fakes`: supplies fake Memgraph/Redis/embedding drivers to avoid optional dependency installs. +- `meshmind/tests/conftest.py`: defines reusable fixtures such as `dummy_encoder`, `memory_service`, and telemetry reset + helpers. + +## Running Tests + +```bash +pip install -e .[dev,docs,testing] +pytest +``` + +Optional extras: + +- `PYTHONPATH=.` ensures imports resolve when running tests manually. +- To test the Neo4j/Memgraph drivers, set `GRAPH_BACKEND` appropriately, point `MEMGRAPH_URI` / `NEO4J_URI` at the Docker + services, and export credentials as described in `SETUP.md` and `ENVIRONMENT_NEEDS.md`. + +## Adding Tests + +1. Prefer deterministic unit tests using the in-memory driver or fakes. +2. When touching new modules, add coverage in the closest existing test file or create a dedicated module under + `meshmind/tests`. +3. Mock optional dependencies (OpenAI, Redis, Celery) unless integration coverage is explicitly required. +4. Update documentation (`docs/testing.md`, `README.md`) whenever the testing workflow changes. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000..823c8f5 --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,31 @@ +# Troubleshooting + +## Optional Tooling Installation Failures + +- **Ruff, Pyright, Typeguard** + - Install via `uv pip install ruff pyright typeguard` to match CI tooling. + - If Pyright is unavailable, run `make lint` to exercise Ruff only and document the gap in `ENVIRONMENT_NEEDS.md`. +- **toml-sort / yamllint** + - Provided through `uv pip install toml-sort yamllint`. + - When installing globally, ensure the binaries are on the `PATH` or invoke via `python -m toml_sort` and `python -m yamllint`. + +## Graph Backends + +- **Memgraph / Neo4j connectivity** + - Verify credentials are exported (`MEMGRAPH_URI`, `NEO4J_URI`, etc.) before running `mesh admin graph --backend neo4j`. + - When the native drivers are unavailable, fall back to the in-memory or SQLite backends and note the blocker in `ENVIRONMENT_NEEDS.md`. + +## External Services + +- **Redis** + - Use `docker-compose up redis` to start the local stack, or swap in the `FakeRedisBroker` for unit tests. +- **Embedding providers** + - When OpenAI keys are missing, configure `FAKE_EMBEDDINGS=1` to rely on the deterministic fake encoder. + +## Common Runtime Symptoms + +- **Search returns empty results** + - Confirm that the namespace and `entity_labels` filters align with persisted memories. + - For graph-backed searches, run `mesh admin counts` to validate stored entity counts. +- **Pagination behaves unexpectedly** + - Inspect driver-specific logs (enable via `MESH_LOG_LEVEL=DEBUG`) to confirm the offset and limit values passed downstream. diff --git a/examples/extract_preprocess_store_example.py b/examples/extract_preprocess_store_example.py index 74ac65d..ba803d1 100644 --- a/examples/extract_preprocess_store_example.py +++ b/examples/extract_preprocess_store_example.py @@ -1,40 +1,73 @@ -""" -Example flow: extract → preprocess → store using Meshmind pipeline. -Requires a running Memgraph instance and a valid OPENAI_API_KEY. -""" -from meshmind.core.types import Memory +"""End-to-end MeshMind example covering extraction, storage, and retrieval.""" +from __future__ import annotations + from meshmind.client import MeshMind +from meshmind.core.types import Memory, Triplet + -def main(): - # Initialize MeshMind client (uses OpenAI and default MemgraphDriver) +def main() -> None: mm = MeshMind() - driver = mm.driver - # Sample content for extraction texts = [ "The Eiffel Tower is located in Paris and was built in 1889.", - "Python is a programming language created by Guido van Rossum." + "Python is a programming language created by Guido van Rossum.", ] - # Extract memories via LLM memories = mm.extract_memories( instructions="Extract key facts as Memory objects.", namespace="demo", - entity_types=[Memory], + entity_types=[Memory], content=texts, ) - print(f"Extracted {len(memories)} memories:") - for m in memories: - print(m.json()) - - # Preprocess: deduplicate, score importance, compress - memories = mm.deduplicate(memories, threshold=0.9) + memories = mm.deduplicate(memories) memories = mm.score_importance(memories) memories = mm.compress(memories) - - # Store into graph mm.store_memories(memories) - print("Memories stored to graph.") + print(f"Stored {len(memories)} memories.") + + if len(memories) >= 2: + relation = Triplet( + subject=str(memories[0].uuid), + predicate="RELATED_TO", + object=str(memories[1].uuid), + namespace="demo", + entity_label="Knowledge", + metadata={"confidence": 0.9}, + ) + mm.store_triplets([relation]) + print("Stored relationship between first two memories.") + + stored = mm.list_memories(namespace="demo", entity_labels=["Memory"], limit=10) + + hits = mm.search("Eiffel Tower", namespace="demo", entity_labels=["Memory"]) + print("Hybrid search results:") + for mem in hits: + print(f"- {mem.name} (importance={mem.importance})") + + vector_hits = mm.search_vector("programming", namespace="demo", entity_labels=["Memory"]) + print("Vector-only search results:") + for mem in vector_hits: + print(f"- {mem.name}") + + regex_hits = mm.search_regex(r"Paris", namespace="demo", entity_labels=["Memory"]) + print("Regex search results:") + for mem in regex_hits: + print(f"- {mem.name}") + + exact_hits = mm.search_exact( + "Python", + namespace="demo", + entity_labels=["Memory"], + fields=["name"], + ) + print("Exact match search results:") + for mem in exact_hits: + print(f"- {mem.name}") + + counts = mm.memory_counts(namespace="demo") + print(f"Filtered view returned {len(stored)} memories from the driver.") + print(f"Namespace counts: {counts}") + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/meshmind/_compat/pydantic.py b/meshmind/_compat/pydantic.py new file mode 100644 index 0000000..966e2d1 --- /dev/null +++ b/meshmind/_compat/pydantic.py @@ -0,0 +1,144 @@ +"""Compatibility helpers for environments without :mod:`pydantic`.""" +from __future__ import annotations + +from dataclasses import MISSING +from datetime import datetime +from typing import Any, Dict, List, Optional, TypeVar, Union, get_args, get_origin, get_type_hints +from uuid import UUID + +try: # pragma: no cover - prefer real pydantic when available + from pydantic import BaseModel as _RealBaseModel # type: ignore + from pydantic import Field # type: ignore + from pydantic import ValidationError # type: ignore + + BaseModel = _RealBaseModel +except ImportError: # pragma: no cover - exercised in constrained sandboxes + Field = None # type: ignore + + class ValidationError(Exception): + """Fallback validation error raised by the compatibility layer.""" + + class _FieldInfo: + __slots__ = ("default", "default_factory") + + def __init__(self, default: Any = MISSING, default_factory: Optional[Any] = None) -> None: + self.default = default + self.default_factory = default_factory + + def default_value(self) -> Any: + if self.default_factory is not None: + return self.default_factory() + if self.default is MISSING: + return None + return self.default + + def Field(*, default: Any = MISSING, default_factory: Optional[Any] = None) -> _FieldInfo: # type: ignore[override] + return _FieldInfo(default=default, default_factory=default_factory) + + _T = TypeVar("_T", bound="BaseModel") + + class BaseModel: + """Tiny subset of :class:`pydantic.BaseModel` used in the project.""" + + def __init__(self, **data: Any) -> None: + hints = get_type_hints(self.__class__) + for name, annotation in hints.items(): + value = data.pop(name, MISSING) + if value is MISSING: + default_obj = getattr(self.__class__, name, MISSING) + if isinstance(default_obj, _FieldInfo): + value = default_obj.default_value() + elif default_obj is not MISSING: + value = default_obj + else: + value = None + value = self._coerce(annotation, value) + setattr(self, name, value) + for key, value in data.items(): + setattr(self, key, value) + + @classmethod + def _coerce(cls, annotation: Any, value: Any) -> Any: + if value is None: + return None + + origin = get_origin(annotation) + if origin is Union: + for candidate in get_args(annotation): + try: + return cls._coerce(candidate, value) + except Exception: + continue + return value + + if annotation in (str, int, float, bool): + return annotation(value) + + if annotation is UUID: + if isinstance(value, UUID): + return value + return UUID(str(value)) + + if annotation is datetime: + if isinstance(value, datetime): + return value + return datetime.fromisoformat(str(value)) + + if origin is list: + inner = get_args(annotation)[0] if get_args(annotation) else Any + return [cls._coerce(inner, item) for item in value] + + if origin is dict: + key_type, val_type = (Any, Any) + args = get_args(annotation) + if len(args) == 2: + key_type, val_type = args + return { + cls._coerce(key_type, k): cls._coerce(val_type, v) + for k, v in value.items() + } + + return value + + def dict(self, *, exclude_none: bool = False) -> Dict[str, Any]: + data = dict(self.__dict__) + if exclude_none: + data = {k: v for k, v in data.items() if v is not None} + return data + + def model_dump(self, *, exclude_none: bool = False) -> Dict[str, Any]: + return self.dict(exclude_none=exclude_none) + + def model_copy(self: _T, *, update: Optional[Dict[str, Any]] = None) -> _T: + payload = self.dict() + if update: + payload.update(update) + return self.__class__(**payload) + + @classmethod + def schema(cls) -> Dict[str, Any]: + hints = get_type_hints(cls) + properties: Dict[str, Any] = {} + required: List[str] = [] + for name, annotation in hints.items(): + properties[name] = {"type": _schema_type(annotation)} + required.append(name) + return {"type": "object", "properties": properties, "required": required} + + def __repr__(self) -> str: # pragma: no cover - debug helper + params = ", ".join(f"{k}={v!r}" for k, v in self.dict().items()) + return f"{self.__class__.__name__}({params})" + + def _schema_type(annotation: Any) -> str: + origin = get_origin(annotation) + if origin is list: + return "array" + if origin is dict: + return "object" + if annotation in (int, float): + return "number" + if annotation is bool: + return "boolean" + return "string" + +__all__ = ["BaseModel", "Field", "ValidationError"] diff --git a/meshmind/api/grpc.py b/meshmind/api/grpc.py new file mode 100644 index 0000000..4bf8914 --- /dev/null +++ b/meshmind/api/grpc.py @@ -0,0 +1,98 @@ +"""gRPC-style adapters for MeshMind.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, Iterable, List + +from meshmind.api.service import MemoryPayload, MemoryService, SearchPayload, TripletPayload + + +@dataclass +class IngestMemoriesRequest: + memories: List[dict] = field(default_factory=list) + + +@dataclass +class IngestMemoriesResponse: + uuids: List[str] = field(default_factory=list) + + +@dataclass +class IngestTripletsRequest: + triplets: List[dict] = field(default_factory=list) + + +@dataclass +class IngestTripletsResponse: + stored: int = 0 + + +@dataclass +class SearchRequest: + query: str + namespace: str | None = None + top_k: int = 10 + encoder: str | None = None + rerank_k: int | None = None + entity_labels: List[str] | None = None + + +@dataclass +class SearchResponse: + results: List[dict] = field(default_factory=list) + + +@dataclass +class MemoryCountsRequest: + namespace: str | None = None + + +@dataclass +class MemoryCountsResponse: + counts: Dict[str, Dict[str, int]] = field(default_factory=dict) + + +class GrpcServiceStub: + """Simple callable object that mirrors gRPC service behaviour for tests.""" + + def __init__(self, service: MemoryService) -> None: + self.service = service + + def IngestMemories(self, request: IngestMemoriesRequest) -> IngestMemoriesResponse: # noqa: N802 + payloads = [MemoryPayload(**item) for item in request.memories] + uuids = self.service.ingest_memories(payloads) + return IngestMemoriesResponse(uuids=uuids) + + def IngestTriplets(self, request: IngestTripletsRequest) -> IngestTripletsResponse: # noqa: N802 + payloads = [TripletPayload(**item) for item in request.triplets] + stored = self.service.ingest_triplets(payloads) + return IngestTripletsResponse(stored=stored) + + def Search(self, request: SearchRequest) -> SearchResponse: # noqa: N802 + payload = SearchPayload( + query=request.query, + namespace=request.namespace, + top_k=request.top_k, + encoder=request.encoder, + rerank_k=request.rerank_k, + entity_labels=request.entity_labels, + ) + results = self.service.search(payload) + return SearchResponse(results=[mem.dict() for mem in results]) + + def MemoryCounts(self, request: MemoryCountsRequest) -> MemoryCountsResponse: # noqa: N802 + counts = self.service.memory_counts(namespace=request.namespace) + return MemoryCountsResponse(counts=counts) + + +__all__ = [ + "GrpcServiceStub", + "IngestMemoriesRequest", + "IngestMemoriesResponse", + "IngestTripletsRequest", + "IngestTripletsResponse", + "SearchRequest", + "SearchResponse", + "MemoryCountsRequest", + "MemoryCountsResponse", +] diff --git a/meshmind/api/memory_manager.py b/meshmind/api/memory_manager.py index 935d270..c22f49d 100644 --- a/meshmind/api/memory_manager.py +++ b/meshmind/api/memory_manager.py @@ -1,39 +1,50 @@ -from typing import Any, List, Optional +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Sequence from uuid import UUID +from meshmind._compat.pydantic import BaseModel + +from meshmind.core.types import Memory, Triplet + + class MemoryManager: - """ - Mid-level CRUD interface for Memory objects, delegating to an underlying graph driver. - """ + """Mid-level CRUD interface for ``Memory`` and ``Triplet`` objects.""" + def __init__(self, graph_driver: Any): # pragma: no cover self.driver = graph_driver - def add_memory(self, memory: Any) -> UUID: + @staticmethod + def _props(model: Any) -> Dict[str, Any]: + if isinstance(model, BaseModel): + return model.dict(exclude_none=True) + if hasattr(model, "dict"): + try: + return model.dict(exclude_none=True) # type: ignore[attr-defined] + except TypeError: + pass + if isinstance(model, dict): + return {k: v for k, v in model.items() if v is not None} + return {k: v for k, v in model.__dict__.items() if v is not None} + + def add_memory(self, memory: Memory) -> UUID: """ Add a new Memory object to the graph. :param memory: A Memory-like object to be stored. :return: The UUID of the newly added memory. """ - # Upsert the memory object into the graph - try: - props = memory.dict(exclude_none=True) - except Exception: - props = memory.__dict__ + props = self._props(memory) self.driver.upsert_entity(memory.entity_label, memory.name, props) return memory.uuid - def update_memory(self, memory: Any) -> None: + def update_memory(self, memory: Memory) -> None: """ Update an existing Memory object in the graph. :param memory: A Memory-like object with updated fields. """ - # Update an existing memory via upsert - try: - props = memory.dict(exclude_none=True) - except Exception: - props = memory.__dict__ + props = self._props(memory) self.driver.upsert_entity(memory.entity_label, memory.name, props) def delete_memory(self, memory_id: UUID) -> None: @@ -52,44 +63,104 @@ def get_memory(self, memory_id: UUID) -> Optional[Any]: :param memory_id: UUID of the memory to retrieve. :return: Memory-like object or None if not found. """ - # Retrieve a memory by UUID - from meshmind.core.types import Memory - - cypher = "MATCH (m) WHERE m.uuid = $uuid RETURN m" - params = {"uuid": str(memory_id)} - records = self.driver.find(cypher, params) - if not records: + payload = self.driver.get_entity(str(memory_id)) + if not payload: return None - # Extract node properties - record = records[0] - data = record.get('m', record) try: - return Memory(**data) + return Memory(**payload) except Exception: return None - def list_memories(self, namespace: Optional[str] = None) -> List[Any]: + def list_memories( + self, + namespace: Optional[str] = None, + entity_labels: Optional[Sequence[str]] = None, + *, + offset: int = 0, + limit: Optional[int] = None, + query: Optional[str] = None, + use_search: bool | None = None, + ) -> List[Memory]: """ - List Memory objects, optionally filtered by namespace. + List Memory objects, optionally filtered by namespace and entity label. :param namespace: If provided, only return memories in this namespace. + :param entity_labels: Optional entity labels to restrict the results. + :param offset: Offset into the result set for pagination. + :param limit: Maximum number of memories to return. + :param query: Optional search string to hint server-side filtering. + :param use_search: Force enabling/disabling the driver's search routine. :return: List of Memory-like objects. """ - # List memories, optionally filtered by namespace - from meshmind.core.types import Memory - - if namespace: - cypher = "MATCH (m) WHERE m.namespace = $namespace RETURN m" - params = {"namespace": namespace} + should_search = use_search if use_search is not None else bool(query) + if should_search and hasattr(self.driver, "search_entities"): + search_fn = getattr(self.driver, "search_entities") + records = search_fn( # type: ignore[operator] + query=query, + namespace=namespace, + entity_labels=entity_labels, + offset=offset, + limit=limit, + ) else: - cypher = "MATCH (m) RETURN m" - params = {} - records = self.driver.find(cypher, params) - result: List[Any] = [] - for record in records: - data = record.get('m', record) + records = self.driver.list_entities( + namespace, + entity_labels, + offset=offset, + limit=limit, + ) + result: List[Memory] = [] + for data in records: try: result.append(Memory(**data)) except Exception: continue - return result \ No newline at end of file + return result + + def count_memories(self, namespace: Optional[str] = None) -> Dict[str, Dict[str, int]]: + """Aggregate memory counts grouped by namespace and entity label.""" + + if not hasattr(self.driver, "count_entities"): + raise NotImplementedError("Graph driver does not implement count_entities") + counter = getattr(self.driver, "count_entities") + return counter(namespace) # type: ignore[misc] + + def add_triplet(self, triplet: Triplet) -> None: + """Persist or update a ``Triplet`` relationship.""" + + props = self._props(triplet) + namespace = props.pop("namespace", None) + if namespace is not None: + props["namespace"] = namespace + self.driver.upsert_edge( + triplet.subject, + triplet.predicate, + triplet.object, + props, + ) + + def delete_triplet(self, subj: str, predicate: str, obj: str) -> None: + """Remove a relationship identified by subject/predicate/object.""" + + self.driver.delete_triplet(subj, predicate, obj) + + def list_triplets(self, namespace: Optional[str] = None) -> List[Triplet]: + """Return stored ``Triplet`` objects, optionally filtered by namespace.""" + + records = self.driver.list_triplets(namespace) + result: List[Triplet] = [] + for record in records: + data = { + "subject": record.get("subject"), + "predicate": record.get("predicate"), + "object": record.get("object"), + "namespace": record.get("namespace") or namespace, + "entity_label": record.get("predicate", "Relation"), + "metadata": record.get("metadata") or {}, + "reference_time": record.get("reference_time"), + } + try: + result.append(Triplet(**data)) + except Exception: + continue + return result diff --git a/meshmind/api/rest.py b/meshmind/api/rest.py new file mode 100644 index 0000000..0d2dcd2 --- /dev/null +++ b/meshmind/api/rest.py @@ -0,0 +1,123 @@ +"""REST adapters for the :mod:`meshmind` service layer.""" +from __future__ import annotations + +from typing import Any, Dict, Iterable, List + +from meshmind.api.service import MemoryPayload, MemoryService, SearchPayload, TripletPayload + + +class RestAPIStub: + """Fallback handler that emulates REST routes without FastAPI.""" + + def __init__(self, service: MemoryService) -> None: + self.service = service + + def dispatch(self, method: str, path: str, payload: Dict[str, Any] | None = None) -> Dict[str, Any]: + method = method.upper() + payload = payload or {} + if method == "POST" and path == "/memories": + memories = [MemoryPayload(**item) for item in payload.get("memories", [])] + uuids = self.service.ingest_memories(memories) + return {"uuids": uuids} + if method == "POST" and path == "/triplets": + triplets = [TripletPayload(**item) for item in payload.get("triplets", [])] + count = self.service.ingest_triplets(triplets) + return {"stored": count} + if method == "POST" and path == "/search": + request = SearchPayload(**payload) + results = self.service.search(request) + return {"results": [mem.dict() for mem in results]} + if method == "GET" and path == "/memories": + namespace = payload.get("namespace") + entity_labels = payload.get("entity_labels") + offset = int(payload.get("offset", 0)) + limit_value = payload.get("limit") + limit = int(limit_value) if limit_value is not None else None + query = payload.get("query") + use_search = payload.get("use_search") + memories = self.service.list_memories( + namespace, + entity_labels, + offset=offset, + limit=limit, + query=query, + use_search=use_search, + ) + return {"memories": [mem.dict() for mem in memories]} + if method == "GET" and path == "/memories/counts": + namespace = payload.get("namespace") + counts = self.service.memory_counts(namespace) + return {"counts": counts} + if method == "GET" and path == "/triplets": + namespace = payload.get("namespace") + triplets = self.service.list_triplets(namespace) + return {"triplets": [triplet.dict() for triplet in triplets]} + raise ValueError(f"Unsupported route {method} {path}") + + +def create_app(service: MemoryService) -> Any: + """Create a FastAPI application if FastAPI is installed, otherwise return a stub.""" + + try: # pragma: no cover - optional dependency path + from fastapi import FastAPI, HTTPException + except ImportError: # pragma: no cover - executed in tests without fastapi + return RestAPIStub(service) + + app = FastAPI(title="MeshMind API") + + @app.post("/memories") + def create_memories(payload: Dict[str, Iterable[Dict[str, Any]]]): + try: + items = [MemoryPayload(**item) for item in payload.get("memories", [])] + except Exception as exc: # pragma: no cover - FastAPI handles validation + raise HTTPException(status_code=400, detail=str(exc)) + uuids = service.ingest_memories(items) + return {"uuids": uuids} + + @app.post("/triplets") + def create_triplets(payload: Dict[str, Iterable[Dict[str, Any]]]): + try: + items = [TripletPayload(**item) for item in payload.get("triplets", [])] + except Exception as exc: # pragma: no cover + raise HTTPException(status_code=400, detail=str(exc)) + stored = service.ingest_triplets(items) + return {"stored": stored} + + @app.post("/search") + def search(payload: Dict[str, Any]): + try: + request = SearchPayload(**payload) + except Exception as exc: # pragma: no cover + raise HTTPException(status_code=400, detail=str(exc)) + results = service.search(request) + return {"results": [mem.dict() for mem in results]} + + @app.get("/memories") + def list_memories( + namespace: str | None = None, + entity_labels: List[str] | None = None, + offset: int = 0, + limit: int | None = None, + query: str | None = None, + use_search: bool | None = None, + ): + memories = service.list_memories( + namespace, + entity_labels, + offset=offset, + limit=limit, + query=query, + use_search=use_search, + ) + return {"memories": [mem.dict() for mem in memories]} + + @app.get("/triplets") + def list_triplets(namespace: str | None = None): + triplets = service.list_triplets(namespace) + return {"triplets": [triplet.dict() for triplet in triplets]} + + @app.get("/memories/counts") + def memory_counts(namespace: str | None = None): + return {"counts": service.memory_counts(namespace)} + + return app diff --git a/meshmind/api/service.py b/meshmind/api/service.py new file mode 100644 index 0000000..5e9edac --- /dev/null +++ b/meshmind/api/service.py @@ -0,0 +1,129 @@ +"""Service layer abstractions for REST and gRPC adapters.""" +from __future__ import annotations + +from typing import Dict, Iterable, List, Sequence + +from meshmind._compat.pydantic import BaseModel, Field + +from meshmind.api.memory_manager import MemoryManager +from meshmind.core.types import Memory, SearchConfig, Triplet +from meshmind.retrieval import search as retrieval_search + + +class MemoryPayload(BaseModel): + """Serializable payload for creating or updating a memory.""" + + uuid: str | None = None + namespace: str + name: str + entity_label: str = "Memory" + embedding: List[float] | None = None + metadata: dict[str, object] = Field(default_factory=dict) + reference_time: str | None = None + importance: float | None = None + ttl_seconds: int | None = None + + def to_memory(self) -> Memory: + payload = self.model_dump(exclude_none=True) + if self.uuid: + payload["uuid"] = self.uuid + return Memory(**payload) + + +class TripletPayload(BaseModel): + subject: str + predicate: str + object: str + namespace: str + entity_label: str = "Relation" + metadata: dict[str, object] = Field(default_factory=dict) + reference_time: str | None = None + + def to_triplet(self) -> Triplet: + return Triplet(**self.model_dump(exclude_none=True)) + + +class SearchPayload(BaseModel): + query: str + namespace: str | None = None + top_k: int = 10 + encoder: str | None = None + rerank_k: int | None = None + entity_labels: Sequence[str] | None = None + + def to_config(self) -> SearchConfig: + config = SearchConfig(top_k=self.top_k) + if self.encoder: + config.encoder = self.encoder + if self.rerank_k is not None: + config.rerank_k = self.rerank_k + return config + + +class MemoryService: + """Business logic for ingestion, retrieval, and triplet persistence.""" + + def __init__(self, manager: MemoryManager) -> None: + self.manager = manager + + # ------------------------------------------------------------------ + # Ingestion + # ------------------------------------------------------------------ + def ingest_memories(self, payloads: Sequence[MemoryPayload]) -> List[str]: + uuids: List[str] = [] + for payload in payloads: + memory = payload.to_memory() + self.manager.add_memory(memory) + uuids.append(str(memory.uuid)) + return uuids + + def ingest_triplets(self, payloads: Iterable[TripletPayload]) -> int: + count = 0 + for payload in payloads: + self.manager.add_triplet(payload.to_triplet()) + count += 1 + return count + + # ------------------------------------------------------------------ + # Retrieval + # ------------------------------------------------------------------ + def search(self, request: SearchPayload) -> List[Memory]: + config = request.to_config() + candidate_limit = max(config.top_k * 5, (config.rerank_k or 0) * 2) + limit = candidate_limit or None + memories = self.manager.list_memories( + namespace=request.namespace, + entity_labels=request.entity_labels, + limit=limit, + query=request.query, + use_search=True, + ) + return retrieval_search(request.query, memories, config=config) + + # ------------------------------------------------------------------ + # CRUD proxies + # ------------------------------------------------------------------ + def list_memories( + self, + namespace: str | None = None, + entity_labels: Sequence[str] | None = None, + *, + offset: int = 0, + limit: int | None = None, + query: str | None = None, + use_search: bool | None = None, + ) -> List[Memory]: + return self.manager.list_memories( + namespace, + entity_labels, + offset=offset, + limit=limit, + query=query, + use_search=use_search, + ) + + def list_triplets(self, namespace: str | None = None) -> List[Triplet]: + return self.manager.list_triplets(namespace) + + def memory_counts(self, namespace: str | None = None) -> Dict[str, Dict[str, int]]: + return self.manager.count_memories(namespace) diff --git a/meshmind/cli/__main__.py b/meshmind/cli/__main__.py index 24dab53..933c8da 100644 --- a/meshmind/cli/__main__.py +++ b/meshmind/cli/__main__.py @@ -5,7 +5,10 @@ import argparse import sys +from meshmind.cli.admin import register_admin_subcommands from meshmind.cli.ingest import ingest_command +from meshmind.core.bootstrap import bootstrap_encoders, bootstrap_entities +from meshmind.core.config import settings def main(): @@ -32,15 +35,32 @@ def main(): ingest_parser.add_argument( "paths", nargs="+", help="Paths to files or directories to ingest" ) + ingest_parser.set_defaults(func=ingest_command) + + register_admin_subcommands(subparsers) args = parser.parse_args() - if args.command == "ingest": - ingest_command(args) - else: - parser.print_help() - sys.exit(1) + # Ensure default encoders and entities are registered before executing commands + bootstrap_entities() + bootstrap_encoders() + + missing = settings.missing() + if missing: + for group, keys in missing.items(): + print( + f"Warning: missing configuration for {group}: {', '.join(keys)}", + file=sys.stderr, + ) + + func = getattr(args, "func", None) + if callable(func): + result = func(args) + return result + + parser.print_help() + sys.exit(1) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/meshmind/cli/admin.py b/meshmind/cli/admin.py new file mode 100644 index 0000000..41b1cd7 --- /dev/null +++ b/meshmind/cli/admin.py @@ -0,0 +1,147 @@ +"""Administrative CLI helpers for MeshMind.""" +from __future__ import annotations + +import argparse +import json +import sys +from typing import TextIO + +from meshmind.api.memory_manager import MemoryManager +from meshmind.core.config import settings +from meshmind.core.observability import telemetry +from meshmind.db.factory import create_graph_driver +from meshmind.models.registry import PredicateRegistry + + +def register_admin_subcommands(subparsers: argparse._SubParsersAction) -> None: + """Attach admin subcommands to the CLI parser.""" + + admin_parser = subparsers.add_parser("admin", help="Administrative commands") + admin_sub = admin_parser.add_subparsers(dest="admin_command") + + predicate_parser = admin_sub.add_parser( + "predicates", help="Manage the predicate registry" + ) + predicate_parser.add_argument("--list", action="store_true", help="List predicates") + predicate_parser.add_argument("--add", metavar="LABEL", help="Add a predicate label") + predicate_parser.add_argument( + "--remove", metavar="LABEL", help="Remove a predicate label if present" + ) + predicate_parser.set_defaults(func=handle_predicates) + + maintenance_parser = admin_sub.add_parser( + "maintenance", help="Inspect maintenance telemetry" + ) + maintenance_parser.add_argument( + "--reset", action="store_true", help="Reset telemetry after printing" + ) + maintenance_parser.set_defaults(func=handle_maintenance) + + graph_parser = admin_sub.add_parser( + "graph", help="Validate graph backend connectivity" + ) + graph_parser.add_argument( + "--backend", + default=settings.GRAPH_BACKEND, + help="Backend to test (memory, sqlite, memgraph, neo4j)", + ) + graph_parser.set_defaults(func=handle_graph_check) + + counts_parser = admin_sub.add_parser( + "counts", help="Summarise memory counts by namespace and entity label" + ) + counts_parser.add_argument( + "--backend", + default=settings.GRAPH_BACKEND, + help="Graph backend to inspect", + ) + counts_parser.add_argument( + "--namespace", + default=None, + help="Optional namespace filter", + ) + counts_parser.set_defaults(func=handle_counts) + + +def handle_predicates(args: argparse.Namespace, stream: TextIO | None = None) -> None: + """Apply predicate registry operations based on CLI flags.""" + + stream = stream or sys.stdout + updated = False + if args.add: + PredicateRegistry.add(args.add) + print(f"Registered predicate: {args.add}", file=stream) + updated = True + if args.remove: + removed = PredicateRegistry.remove(args.remove) + status = "Removed" if removed else "Not present" + print(f"{status} predicate: {args.remove}", file=stream) + updated = True + if args.list or not updated: + labels = sorted(PredicateRegistry.all()) + print(json.dumps({"predicates": labels}, indent=2), file=stream) + + +def handle_maintenance(args: argparse.Namespace, stream: TextIO | None = None) -> None: + """Print maintenance telemetry and optionally reset it.""" + + stream = stream or sys.stdout + snapshot = telemetry.snapshot() + print(json.dumps(snapshot, indent=2, sort_keys=True), file=stream) + if args.reset: + telemetry.reset() + print("Telemetry reset", file=stream) + + +def handle_graph_check(args: argparse.Namespace, stream: TextIO | None = None) -> int: + """Try to instantiate the requested graph driver and verify connectivity.""" + + stream = stream or sys.stdout + try: + driver = create_graph_driver(backend=args.backend) + except Exception as exc: # pragma: no cover - best effort logging + print(f"Failed to create driver: {exc}", file=sys.stderr) + return 1 + + verify = getattr(driver, "verify_connectivity", None) + if callable(verify): + try: + ok = bool(verify()) + except Exception as exc: # pragma: no cover - depends on backend state + print(f"Connectivity check failed: {exc}", file=sys.stderr) + return 2 + else: + result = "ok" if ok else "unknown" + print(json.dumps({"backend": args.backend, "status": result}), file=stream) + return 0 + + print( + json.dumps({"backend": args.backend, "status": "unsupported"}), + file=stream, + ) + return 0 + + +def handle_counts(args: argparse.Namespace, stream: TextIO | None = None) -> int: + """Print aggregated memory counts grouped by namespace and label.""" + + stream = stream or sys.stdout + try: + driver = create_graph_driver(backend=args.backend) + except Exception as exc: # pragma: no cover - backend instantiation failures + print(f"Failed to create driver: {exc}", file=sys.stderr) + return 1 + + manager = MemoryManager(driver) + try: + counts = manager.count_memories(namespace=args.namespace) + except NotImplementedError as exc: + print(str(exc), file=sys.stderr) + return 2 + finally: + closer = getattr(driver, "close", None) + if callable(closer): # pragma: no cover - depends on backend + closer() + + print(json.dumps(counts, indent=2, sort_keys=True), file=stream) + return 0 diff --git a/meshmind/client.py b/meshmind/client.py index 365eac0..89bfa7d 100644 --- a/meshmind/client.py +++ b/meshmind/client.py @@ -1,80 +1,333 @@ -""" -MeshMind client combining LLM, embedding, and graph driver. -""" -from openai import OpenAI -from typing import Any, List, Type -from meshmind.db.memgraph_driver import MemgraphDriver +"""High-level MeshMind client orchestrating ingestion and storage flows.""" +from __future__ import annotations + +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Type +from uuid import UUID + +try: # pragma: no cover - optional dependency + from openai import OpenAI +except ImportError: # pragma: no cover - optional dependency + OpenAI = None # type: ignore + +from meshmind.api.memory_manager import MemoryManager +from meshmind.core.bootstrap import bootstrap_entities, bootstrap_encoders from meshmind.core.config import settings +from meshmind.core.types import Memory, Triplet, SearchConfig +from meshmind.db.base_driver import GraphDriver +from meshmind.db.factory import graph_driver_factory as make_graph_driver_factory +from meshmind.models.registry import EntityRegistry, PredicateRegistry class MeshMind: - """ - High-level client to manage extraction, preprocessing, and storage of memories. - """ + """High-level orchestration client for extraction, preprocessing, and persistence.""" + def __init__( self, llm_client: Any = None, embedding_model: str | None = None, - graph_driver: Any = None, + graph_driver: Optional[GraphDriver] = None, + graph_driver_factory: Callable[[], GraphDriver] | None = None, ): - # Initialize LLM client - self.llm_client = llm_client or OpenAI() - # Set embedding model name + if llm_client is None: + if OpenAI is None: + raise ImportError( + "openai package is required to construct a default MeshMind LLM client." + ) + client_kwargs: dict[str, Any] = {} + if settings.OPENAI_API_KEY: + client_kwargs["api_key"] = settings.OPENAI_API_KEY + llm_client = OpenAI(**client_kwargs) + + self.llm_client = llm_client self.embedding_model = embedding_model or settings.EMBEDDING_MODEL - # Initialize graph driver - self.driver = graph_driver or MemgraphDriver( - settings.MEMGRAPH_URI, - settings.MEMGRAPH_USERNAME, - settings.MEMGRAPH_PASSWORD, + + self._graph_driver: Optional[GraphDriver] = graph_driver + self._graph_driver_factory = graph_driver_factory + if self._graph_driver is None and self._graph_driver_factory is None: + self._graph_driver_factory = make_graph_driver_factory() + + self._memory_manager: Optional[MemoryManager] = ( + MemoryManager(self._graph_driver) if self._graph_driver else None ) + self.entity_registry = EntityRegistry + self.predicate_registry = PredicateRegistry + bootstrap_entities([Memory]) + bootstrap_encoders() + @property + def graph_driver(self) -> GraphDriver: + """Expose the active graph driver, creating it on demand.""" + return self._ensure_driver() + + @property + def driver(self) -> GraphDriver: + """Backward compatible alias for :attr:`graph_driver`.""" + return self.graph_driver + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + def _ensure_driver(self) -> GraphDriver: + if self._graph_driver is None: + if self._graph_driver_factory is None: + raise RuntimeError("No graph driver factory available for MeshMind") + self._graph_driver = self._graph_driver_factory() + return self._graph_driver + + def _ensure_manager(self) -> MemoryManager: + if self._memory_manager is None: + self._memory_manager = MemoryManager(self._ensure_driver()) + return self._memory_manager + + # ------------------------------------------------------------------ + # Pipelines + # ------------------------------------------------------------------ def extract_memories( self, instructions: str, namespace: str, - entity_types: List[Type[Any]], - content: List[str], + entity_types: Sequence[Type[Any]], + content: Sequence[str], ) -> List[Any]: from meshmind.pipeline.extract import extract_memories return extract_memories( instructions=instructions, namespace=namespace, - entity_types=entity_types, + entity_types=list(entity_types), embedding_model=self.embedding_model, - content=content, + content=list(content), llm_client=self.llm_client, ) def deduplicate( self, - memories: List[Any], + memories: Sequence[Any], threshold: float = 0.95, ) -> List[Any]: from meshmind.pipeline.preprocess import deduplicate - return deduplicate(memories, threshold) + return deduplicate(list(memories), threshold) def score_importance( self, - memories: List[Any], + memories: Sequence[Any], ) -> List[Any]: from meshmind.pipeline.preprocess import score_importance - return score_importance(memories) + return score_importance(list(memories)) def compress( self, - memories: List[Any], + memories: Sequence[Any], ) -> List[Any]: from meshmind.pipeline.preprocess import compress - return compress(memories) + return compress(list(memories)) def store_memories( self, - memories: List[Any], + memories: Iterable[Any], ) -> None: from meshmind.pipeline.store import store_memories - store_memories(memories, self.driver) + store_memories( + memories, + self._ensure_driver(), + entity_registry=self.entity_registry, + ) + + def store_triplets( + self, + triplets: Iterable[Triplet], + ) -> None: + from meshmind.pipeline.store import store_triplets + + store_triplets( + triplets, + self._ensure_driver(), + predicate_registry=self.predicate_registry, + ) + + # ------------------------------------------------------------------ + # CRUD helpers + # ------------------------------------------------------------------ + def create_memory(self, memory: Memory) -> UUID: + return self._ensure_manager().add_memory(memory) + + def update_memory(self, memory: Memory) -> None: + self._ensure_manager().update_memory(memory) + + def delete_memory(self, memory_id: UUID) -> None: + self._ensure_manager().delete_memory(memory_id) + + def get_memory(self, memory_id: UUID) -> Optional[Memory]: + return self._ensure_manager().get_memory(memory_id) + + def list_memories( + self, + namespace: str | None = None, + entity_labels: Sequence[str] | None = None, + *, + offset: int = 0, + limit: int | None = None, + query: str | None = None, + use_search: bool | None = None, + ) -> List[Memory]: + return self._ensure_manager().list_memories( + namespace, + entity_labels, + offset=offset, + limit=limit, + query=query, + use_search=use_search, + ) + + def memory_counts(self, namespace: str | None = None) -> Dict[str, Dict[str, int]]: + return self._ensure_manager().count_memories(namespace) + + def create_triplet(self, triplet: Triplet) -> None: + self.predicate_registry.add(triplet.predicate) + self._ensure_manager().add_triplet(triplet) + + def delete_triplet(self, triplet: Triplet) -> None: + self._ensure_manager().delete_triplet( + triplet.subject, triplet.predicate, triplet.object + ) + + def list_triplets(self, namespace: str | None = None) -> List[Triplet]: + return self._ensure_manager().list_triplets(namespace) + + # ------------------------------------------------------------------ + # Retrieval helpers + # ------------------------------------------------------------------ + def search( + self, + query: str, + memories: Sequence[Memory] | None = None, + namespace: str | None = None, + entity_labels: Sequence[str] | None = None, + config: SearchConfig | None = None, + use_llm_rerank: bool = False, + reranker: Callable[[str, Sequence[Memory], int], Sequence[Memory]] | None = None, + ) -> List[Memory]: + from meshmind.retrieval import llm_rerank, search as hybrid_search + from meshmind.retrieval.graph import graph_hybrid_search + + cfg = config or SearchConfig(encoder=self.embedding_model) + active_reranker = reranker + if use_llm_rerank: + active_reranker = lambda q, c, k: llm_rerank( + q, c, self.llm_client, k, model=cfg.rerank_model + ) + + if memories is None: + return graph_hybrid_search( + query, + self._ensure_driver(), + namespace=namespace, + entity_labels=entity_labels, + config=cfg, + reranker=active_reranker, + ) + + return hybrid_search( + query, + list(memories), + namespace=namespace, + entity_labels=list(entity_labels) if entity_labels else None, + config=cfg, + reranker=active_reranker, + ) + + def search_vector( + self, + query: str, + memories: Sequence[Memory] | None = None, + namespace: str | None = None, + entity_labels: Sequence[str] | None = None, + config: SearchConfig | None = None, + ) -> List[Memory]: + from meshmind.retrieval import search_vector + from meshmind.retrieval.graph import graph_vector_search + + cfg = config or SearchConfig(encoder=self.embedding_model) + if memories is None: + return graph_vector_search( + query, + self._ensure_driver(), + namespace=namespace, + entity_labels=entity_labels, + config=cfg, + ) + return search_vector( + query, + list(memories), + namespace=namespace, + entity_labels=list(entity_labels) if entity_labels else None, + config=cfg, + ) + + def search_regex( + self, + pattern: str, + memories: Sequence[Memory] | None = None, + namespace: str | None = None, + entity_labels: Sequence[str] | None = None, + flags: int | None = None, + top_k: int = 10, + ) -> List[Memory]: + from meshmind.retrieval import search_regex + from meshmind.retrieval.graph import graph_regex_search + + if memories is None: + return graph_regex_search( + pattern, + self._ensure_driver(), + namespace=namespace, + entity_labels=entity_labels, + flags=flags, + top_k=top_k, + ) + return search_regex( + pattern, + list(memories), + namespace=namespace, + entity_labels=list(entity_labels) if entity_labels else None, + flags=flags, + top_k=top_k, + ) + + def search_exact( + self, + query: str, + memories: Sequence[Memory] | None = None, + namespace: str | None = None, + entity_labels: Sequence[str] | None = None, + fields: Sequence[str] | None = None, + case_sensitive: bool = False, + top_k: int = 10, + ) -> List[Memory]: + from meshmind.retrieval import search_exact + from meshmind.retrieval.graph import graph_exact_search + + if memories is None: + return graph_exact_search( + query, + self._ensure_driver(), + namespace=namespace, + entity_labels=entity_labels, + fields=fields, + case_sensitive=case_sensitive, + top_k=top_k, + ) + return search_exact( + query, + list(memories), + namespace=namespace, + entity_labels=list(entity_labels) if entity_labels else None, + fields=list(fields) if fields else None, + case_sensitive=case_sensitive, + top_k=top_k, + ) + diff --git a/meshmind/core/bootstrap.py b/meshmind/core/bootstrap.py new file mode 100644 index 0000000..1c1b243 --- /dev/null +++ b/meshmind/core/bootstrap.py @@ -0,0 +1,45 @@ +"""Bootstrap helpers for wiring encoders and registries.""" +from __future__ import annotations + +import warnings +from typing import Iterable, Sequence, Type + +from meshmind._compat.pydantic import BaseModel + +from meshmind.core.config import settings +from meshmind.core.embeddings import EncoderRegistry, OpenAIEmbeddingEncoder +from meshmind.core.types import Memory +from meshmind.models.registry import EntityRegistry, PredicateRegistry + + +def bootstrap_encoders(default_models: Sequence[str] | None = None) -> None: + """Ensure a default set of embedding encoders are registered.""" + + models = list(default_models) if default_models else [settings.EMBEDDING_MODEL] + for model_name in models: + if EncoderRegistry.is_registered(model_name): + continue + try: + EncoderRegistry.register(model_name, OpenAIEmbeddingEncoder(model_name)) + except ImportError as exc: + warnings.warn( + f"Skipping registration of OpenAI encoder '{model_name}': {exc}", + RuntimeWarning, + stacklevel=2, + ) + + +def bootstrap_entities(entity_models: Iterable[Type[BaseModel]] | None = None) -> None: + """Register default entity models used throughout the application.""" + + models = list(entity_models) if entity_models else [Memory] + for model in models: + EntityRegistry.register(model) + + +def register_predicates(predicates: Iterable[str]) -> None: + """Register predicate labels in the global predicate registry.""" + + for predicate in predicates: + PredicateRegistry.add(predicate) + diff --git a/meshmind/core/config.py b/meshmind/core/config.py index 9b14ffd..da78b2e 100644 --- a/meshmind/core/config.py +++ b/meshmind/core/config.py @@ -13,20 +13,73 @@ class Settings: """Application settings loaded from environment variables.""" + GRAPH_BACKEND: str = os.getenv("GRAPH_BACKEND", "memory") MEMGRAPH_URI: str = os.getenv("MEMGRAPH_URI", "bolt://localhost:7687") MEMGRAPH_USERNAME: str = os.getenv("MEMGRAPH_USERNAME", "") MEMGRAPH_PASSWORD: str = os.getenv("MEMGRAPH_PASSWORD", "") + NEO4J_URI: str = os.getenv("NEO4J_URI", "bolt://localhost:7687") + NEO4J_USERNAME: str = os.getenv("NEO4J_USERNAME", "neo4j") + NEO4J_PASSWORD: str = os.getenv("NEO4J_PASSWORD", "") + SQLITE_PATH: str = os.getenv("SQLITE_PATH", ":memory:") REDIS_URL: str = os.getenv("REDIS_URL", "redis://localhost:6379/0") OPENAI_API_KEY: str = os.getenv("OPENAI_API_KEY", "") EMBEDDING_MODEL: str = os.getenv("EMBEDDING_MODEL", "text-embedding-3-small") + REQUIRED_GROUPS = { + "memgraph": ("MEMGRAPH_URI",), + "neo4j": ("NEO4J_URI",), + "openai": ("OPENAI_API_KEY",), + "redis": ("REDIS_URL",), + } + def __repr__(self) -> str: return ( - f"Settings(MEMGRAPH_URI={self.MEMGRAPH_URI}, " + f"Settings(GRAPH_BACKEND={self.GRAPH_BACKEND}, " + f"MEMGRAPH_URI={self.MEMGRAPH_URI}, " f"MEMGRAPH_USERNAME={self.MEMGRAPH_USERNAME}, " + f"NEO4J_URI={self.NEO4J_URI}, " f"REDIS_URL={self.REDIS_URL}, " f"EMBEDDING_MODEL={self.EMBEDDING_MODEL})" ) + @staticmethod + def _mask(value: str) -> str: + if not value: + return "" + if len(value) <= 4: + return "*" * len(value) + return f"{value[:2]}***{value[-2:]}" + + def missing(self) -> dict[str, list[str]]: + """Return missing environment variables grouped by capability.""" + + missing: dict[str, list[str]] = {} + for group, keys in self.REQUIRED_GROUPS.items(): + if group == "memgraph" and self.GRAPH_BACKEND != "memgraph": + continue + if group == "neo4j" and self.GRAPH_BACKEND != "neo4j": + continue + absent = [key for key in keys if not getattr(self, key)] + if absent: + missing[group] = absent + return missing + + def summary(self) -> dict[str, str]: + """Return a sanitized summary of active configuration values.""" + + return { + "MEMGRAPH_URI": self.MEMGRAPH_URI, + "MEMGRAPH_USERNAME": self.MEMGRAPH_USERNAME, + "MEMGRAPH_PASSWORD": self._mask(self.MEMGRAPH_PASSWORD), + "NEO4J_URI": self.NEO4J_URI, + "NEO4J_USERNAME": self.NEO4J_USERNAME, + "NEO4J_PASSWORD": self._mask(self.NEO4J_PASSWORD), + "GRAPH_BACKEND": self.GRAPH_BACKEND, + "SQLITE_PATH": self.SQLITE_PATH, + "REDIS_URL": self.REDIS_URL, + "OPENAI_API_KEY": self._mask(self.OPENAI_API_KEY), + "EMBEDDING_MODEL": self.EMBEDDING_MODEL, + } + -settings = Settings() \ No newline at end of file +settings = Settings() diff --git a/meshmind/core/embeddings.py b/meshmind/core/embeddings.py index 84a67e8..ed2eb56 100644 --- a/meshmind/core/embeddings.py +++ b/meshmind/core/embeddings.py @@ -1,18 +1,16 @@ -""" -Embedding encoders and registry for MeshMind. -""" -from typing import List, Dict, Any +"""Embedding encoder implementations and registry utilities.""" +from __future__ import annotations + import time +from typing import Any, Dict, List _OPENAI_AVAILABLE = True -try: +try: # pragma: no cover - environment dependent from openai import OpenAI - from openai.error import RateLimitError -except ImportError: + from openai import RateLimitError +except ImportError: # pragma: no cover - environment dependent _OPENAI_AVAILABLE = False - openai = None # type: ignore - class RateLimitError(Exception): # type: ignore - pass + OpenAI = None # type: ignore from .config import settings @@ -31,12 +29,11 @@ def __init__( raise ImportError( "openai package is required for OpenAIEmbeddingEncoder" ) - try: - openai.api_key = settings.OPENAI_API_KEY - except Exception: - pass + client_kwargs: Dict[str, Any] = {} + if settings.OPENAI_API_KEY: + client_kwargs["api_key"] = settings.OPENAI_API_KEY - self.llm_client = OpenAI() + self.llm_client = OpenAI(**client_kwargs) self.RateLimitError = RateLimitError self.model_name = model_name self.max_retries = max_retries @@ -49,14 +46,23 @@ def encode(self, texts: List[str] | str) -> List[List[float]]: """ if isinstance(texts, str): texts = [texts] - + for attempt in range(self.max_retries): try: response = self.llm_client.embeddings.create( model=self.model_name, input=texts, ) - return [item['embedding'] for item in response['data']] + data = getattr(response, "data", None) + if data is None: + data = response.get("data", []) # type: ignore[assignment] + embeddings: List[List[float]] = [] + for item in data: + if hasattr(item, "embedding"): + embeddings.append(list(getattr(item, "embedding"))) + else: + embeddings.append(list(item["embedding"])) + return embeddings except self.RateLimitError: time.sleep(self.backoff_factor * (2 ** attempt)) except Exception: @@ -71,7 +77,13 @@ class SentenceTransformerEncoder: Encoder that uses a local SentenceTransformer model. """ def __init__(self, model_name: str): - from sentence_transformers import SentenceTransformer + try: # pragma: no cover - optional dependency + from sentence_transformers import SentenceTransformer + except ImportError as exc: + raise ImportError( + "sentence-transformers is required for SentenceTransformerEncoder." + " Install the optional 'sentence-transformers' extra to enable this encoder." + ) from exc self.model = SentenceTransformer(model_name) @@ -106,4 +118,22 @@ def get(cls, name: str) -> Any: encoder = cls._encoders.get(name) if encoder is None: raise KeyError(f"Encoder '{name}' not found in registry") - return encoder \ No newline at end of file + return encoder + + @classmethod + def is_registered(cls, name: str) -> bool: + """Return True if an encoder ``name`` has been registered.""" + + return name in cls._encoders + + @classmethod + def available(cls) -> List[str]: + """Return the list of registered encoder identifiers.""" + + return list(cls._encoders.keys()) + + @classmethod + def clear(cls) -> None: + """Remove all registered encoders. Intended for testing.""" + + cls._encoders.clear() diff --git a/meshmind/core/observability.py b/meshmind/core/observability.py new file mode 100644 index 0000000..6c36fe1 --- /dev/null +++ b/meshmind/core/observability.py @@ -0,0 +1,80 @@ +"""Shared logging and metrics utilities for MeshMind pipelines.""" +from __future__ import annotations + +import logging +from collections import defaultdict +from contextlib import contextmanager +from time import perf_counter +from typing import Any, Dict, Iterable + +_LOGGER_NAME = "meshmind" +logger = logging.getLogger(_LOGGER_NAME) +if not logger.handlers: # pragma: no cover - avoid duplicate handlers in tests + handler = logging.StreamHandler() + formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s") + handler.setFormatter(formatter) + logger.addHandler(handler) +logger.setLevel(logging.INFO) + + +class Telemetry: + """Lightweight in-memory metrics collector.""" + + def __init__(self) -> None: + self._counters: Dict[str, int] = defaultdict(int) + self._durations: Dict[str, list[float]] = defaultdict(list) + self._gauges: Dict[str, float] = {} + + # ------------------------------------------------------------------ + # Counter helpers + # ------------------------------------------------------------------ + def increment(self, metric: str, value: int = 1) -> None: + self._counters[metric] += value + + def gauge(self, metric: str, value: float) -> None: + self._gauges[metric] = value + + def observe(self, metric: str, value: float) -> None: + self._durations[metric].append(value) + + @contextmanager + def track_duration(self, metric: str): + start = perf_counter() + try: + yield + finally: + elapsed = perf_counter() - start + self.observe(metric, elapsed) + + def extend_counter(self, metric: str, values: Iterable[Any]) -> None: + count = sum(1 for _ in values) + self.increment(metric, count) + + # ------------------------------------------------------------------ + # Snapshot helpers + # ------------------------------------------------------------------ + def snapshot(self) -> Dict[str, Any]: + return { + "counters": dict(self._counters), + "durations": {k: list(v) for k, v in self._durations.items()}, + "gauges": dict(self._gauges), + } + + def reset(self) -> None: + self._counters.clear() + self._durations.clear() + self._gauges.clear() + + +telemetry = Telemetry() + + +def log_event(event: str, **fields: Any) -> None: + """Emit a structured log entry and update a counter for the event.""" + + telemetry.increment(f"events.{event}") + if fields: + formatted = " ".join(f"{key}={value}" for key, value in fields.items()) + logger.info("event=%s %s", event, formatted) + else: + logger.info("event=%s", event) diff --git a/meshmind/core/similarity.py b/meshmind/core/similarity.py index 78fa6c9..7e47130 100644 --- a/meshmind/core/similarity.py +++ b/meshmind/core/similarity.py @@ -1,8 +1,13 @@ -""" -Similarity and distance metrics for MeshMind. -""" +"""Similarity and distance metrics for MeshMind.""" +from __future__ import annotations + +from math import sqrt from typing import Sequence -import numpy as np + +try: # pragma: no cover - optional dependency + import numpy as np +except ImportError: # pragma: no cover - fallback path for test sandboxes + np = None # type: ignore def cosine_similarity(vec1: Sequence[float], vec2: Sequence[float]) -> float: @@ -10,23 +15,38 @@ def cosine_similarity(vec1: Sequence[float], vec2: Sequence[float]) -> float: Compute the cosine similarity between two vectors. Returns 0.0 if either vector has zero magnitude. """ - a = np.array(vec1, dtype=float) - b = np.array(vec2, dtype=float) - if a.shape != b.shape: + if np is not None: + a = np.array(vec1, dtype=float) + b = np.array(vec2, dtype=float) + if a.shape != b.shape: + raise ValueError("Vectors must be the same length") + norm_a = np.linalg.norm(a) + norm_b = np.linalg.norm(b) + if norm_a == 0.0 or norm_b == 0.0: + return 0.0 + return float(np.dot(a, b) / (norm_a * norm_b)) + + if len(vec1) != len(vec2): raise ValueError("Vectors must be the same length") - norm_a = np.linalg.norm(a) - norm_b = np.linalg.norm(b) + dot = sum(float(a) * float(b) for a, b in zip(vec1, vec2)) + norm_a = sqrt(sum(float(a) ** 2 for a in vec1)) + norm_b = sqrt(sum(float(b) ** 2 for b in vec2)) if norm_a == 0.0 or norm_b == 0.0: return 0.0 - return float(np.dot(a, b) / (norm_a * norm_b)) + return float(dot / (norm_a * norm_b)) def euclidean_distance(vec1: Sequence[float], vec2: Sequence[float]) -> float: """ Compute the Euclidean distance between two vectors. """ - a = np.array(vec1, dtype=float) - b = np.array(vec2, dtype=float) - if a.shape != b.shape: + if np is not None: + a = np.array(vec1, dtype=float) + b = np.array(vec2, dtype=float) + if a.shape != b.shape: + raise ValueError("Vectors must be the same length") + return float(np.linalg.norm(a - b)) + + if len(vec1) != len(vec2): raise ValueError("Vectors must be the same length") - return float(np.linalg.norm(a - b)) \ No newline at end of file + return float(sqrt(sum((float(a) - float(b)) ** 2 for a, b in zip(vec1, vec2)))) \ No newline at end of file diff --git a/meshmind/core/types.py b/meshmind/core/types.py index 112d31e..1ef6779 100644 --- a/meshmind/core/types.py +++ b/meshmind/core/types.py @@ -3,7 +3,7 @@ from typing import Any, Optional, Tuple from uuid import UUID, uuid4 -from pydantic import BaseModel, Field +from meshmind._compat.pydantic import BaseModel, Field class Memory(BaseModel): @@ -43,5 +43,6 @@ class SearchConfig(BaseModel): encoder: str = "text-embedding-3-small" top_k: int = 20 rerank_k: int = 10 + rerank_model: Optional[str] = None filters: Optional[dict[str, Any]] = None hybrid_weights: Tuple[float, float] = (0.5, 0.5) \ No newline at end of file diff --git a/meshmind/core/utils.py b/meshmind/core/utils.py index 74e90dc..1884212 100644 --- a/meshmind/core/utils.py +++ b/meshmind/core/utils.py @@ -1,9 +1,44 @@ -"""Utility functions for MeshMind.""" +"""Utility helpers for MeshMind with optional dependency guards.""" +from __future__ import annotations + +import hashlib import uuid from datetime import datetime -import hashlib -from typing import Any -import tiktoken +from functools import lru_cache +from typing import Any, Optional + +_TIKTOKEN = None + + +def _ensure_tiktoken() -> Any: + """Return the ``tiktoken`` module if it is installed.""" + + global _TIKTOKEN + if _TIKTOKEN is None: + try: + import tiktoken # type: ignore + except ImportError as exc: # pragma: no cover - exercised in minimal envs + raise RuntimeError( + "tiktoken is required for token counting but is not installed." + " Install the optional 'tiktoken' extra to enable compression features." + ) from exc + _TIKTOKEN = tiktoken + return _TIKTOKEN + + +@lru_cache(maxsize=8) +def get_token_encoder(encoding_name: str = "o200k_base", optional: bool = False) -> Optional[Any]: + """Return a cached tiktoken encoder or ``None`` when optional.""" + + try: + module = _ensure_tiktoken() + except RuntimeError: + if optional: + return None + raise + return module.get_encoding(encoding_name) + + def generate_uuid() -> str: """Generate a UUID4 string.""" return str(uuid.uuid4()) @@ -21,13 +56,7 @@ def hash_dict(data: Any) -> str: return hash_string(str(data)) def num_tokens_from_string(string: str, encoding_name: str = "o200k_base") -> int: - """Returns the number of tokens in a text string. - Args: - string: The text string to count tokens for. - encoding_name: The name of the encoding to use. Defaults to "o200k_base". - Returns: - The number of tokens in the text string. - """ - encoding = tiktoken.get_encoding(encoding_name) - num_tokens = len(encoding.encode(string)) - return num_tokens \ No newline at end of file + """Return the number of tokens in ``string`` for ``encoding_name``.""" + + encoder = get_token_encoder(encoding_name, optional=False) + return len(encoder.encode(string)) diff --git a/meshmind/db/__init__.py b/meshmind/db/__init__.py index e69de29..e5c249e 100644 --- a/meshmind/db/__init__.py +++ b/meshmind/db/__init__.py @@ -0,0 +1,14 @@ +"""Graph driver implementations exposed for convenience.""" +from .base_driver import GraphDriver +from .in_memory_driver import InMemoryGraphDriver +from .memgraph_driver import MemgraphDriver +from .neo4j_driver import Neo4jGraphDriver +from .sqlite_driver import SQLiteGraphDriver + +__all__ = [ + "GraphDriver", + "InMemoryGraphDriver", + "MemgraphDriver", + "Neo4jGraphDriver", + "SQLiteGraphDriver", +] diff --git a/meshmind/db/base_driver.py b/meshmind/db/base_driver.py index 1e1366c..5fb6145 100644 --- a/meshmind/db/base_driver.py +++ b/meshmind/db/base_driver.py @@ -1,6 +1,8 @@ """Abstract base class for graph database drivers.""" +from __future__ import annotations + from abc import ABC, abstractmethod -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional, Sequence import uuid @@ -22,7 +24,61 @@ def find(self, cypher: str, params: Dict[str, Any]) -> List[Dict[str, Any]]: """Execute a Cypher query and return results.""" raise NotImplementedError + @abstractmethod + def get_entity(self, uid: str) -> Optional[Dict[str, Any]]: + """Return a single entity by UUID, if it exists.""" + raise NotImplementedError + + @abstractmethod + def list_entities( + self, + namespace: Optional[str] = None, + entity_labels: Optional[Sequence[str]] = None, + *, + offset: int = 0, + limit: Optional[int] = None, + ) -> List[Dict[str, Any]]: + """Return entities, optionally filtered by namespace and label.""" + raise NotImplementedError + + def search_entities( + self, + query: Optional[str] = None, + namespace: Optional[str] = None, + entity_labels: Optional[Sequence[str]] = None, + *, + offset: int = 0, + limit: Optional[int] = None, + ) -> List[Dict[str, Any]]: + """Server-side memory search. Defaults to ``list_entities`` when unsupported.""" + + return self.list_entities( + namespace=namespace, + entity_labels=entity_labels, + offset=offset, + limit=limit, + ) + @abstractmethod def delete(self, uuid: uuid.UUID) -> None: """Delete a node or relationship by UUID.""" - raise NotImplementedError \ No newline at end of file + raise NotImplementedError + + @abstractmethod + def delete_triplet(self, subj: str, pred: str, obj: str) -> None: + """Delete a relationship identified by subject/predicate/object.""" + + raise NotImplementedError + + @abstractmethod + def list_triplets(self, namespace: Optional[str] = None) -> List[Dict[str, Any]]: + """Return stored triplets, optionally filtered by namespace.""" + + raise NotImplementedError + + def count_entities( + self, namespace: Optional[str] = None + ) -> Dict[str, Dict[str, int]]: + """Return memory counts grouped by namespace and entity label.""" + + raise NotImplementedError diff --git a/meshmind/db/factory.py b/meshmind/db/factory.py new file mode 100644 index 0000000..e7d18cd --- /dev/null +++ b/meshmind/db/factory.py @@ -0,0 +1,66 @@ +"""Factory helpers for constructing :class:`GraphDriver` instances.""" +from __future__ import annotations + +from typing import Callable, Dict, Type + +from meshmind.core.config import settings +from meshmind.db.base_driver import GraphDriver +from meshmind.db.in_memory_driver import InMemoryGraphDriver +from meshmind.db.memgraph_driver import MemgraphDriver +from meshmind.db.neo4j_driver import Neo4jGraphDriver +from meshmind.db.sqlite_driver import SQLiteGraphDriver + + +def _normalize_backend(name: str) -> str: + return name.replace("-", "_").lower() + + +def available_backends() -> Dict[str, Type[GraphDriver]]: + """Return the mapping of backend names to driver classes.""" + + return { + "memory": InMemoryGraphDriver, + "in_memory": InMemoryGraphDriver, + "inmemory": InMemoryGraphDriver, + "sqlite": SQLiteGraphDriver, + "memgraph": MemgraphDriver, + "neo4j": Neo4jGraphDriver, + } + + +def create_graph_driver(backend: str | None = None, **kwargs) -> GraphDriver: + """Instantiate a :class:`GraphDriver` for the requested backend.""" + + backend_name = _normalize_backend(backend or settings.GRAPH_BACKEND) + drivers = available_backends() + if backend_name not in drivers: + raise ValueError(f"Unsupported graph backend '{backend_name}'") + + driver_cls: Type[GraphDriver] = drivers[backend_name] + if driver_cls is InMemoryGraphDriver: + return driver_cls() + if driver_cls is SQLiteGraphDriver: + path = kwargs.get("path") or settings.SQLITE_PATH + return driver_cls(path) + if driver_cls is MemgraphDriver: + return driver_cls( + settings.MEMGRAPH_URI, + settings.MEMGRAPH_USERNAME, + settings.MEMGRAPH_PASSWORD, + ) + if driver_cls is Neo4jGraphDriver: + return driver_cls( + settings.NEO4J_URI, + settings.NEO4J_USERNAME, + settings.NEO4J_PASSWORD, + ) + return driver_cls(**kwargs) + + +def graph_driver_factory(backend: str | None = None, **kwargs) -> Callable[[], GraphDriver]: + """Return a callable that lazily constructs the configured driver.""" + + def _factory() -> GraphDriver: + return create_graph_driver(backend=backend, **kwargs) + + return _factory diff --git a/meshmind/db/in_memory_driver.py b/meshmind/db/in_memory_driver.py new file mode 100644 index 0000000..191758b --- /dev/null +++ b/meshmind/db/in_memory_driver.py @@ -0,0 +1,188 @@ +"""In-memory implementation of :class:`GraphDriver` for tests and local development.""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Sequence, Tuple +from uuid import UUID, uuid4 + +from meshmind.db.base_driver import GraphDriver + + +class InMemoryGraphDriver(GraphDriver): + """A lightweight graph driver that stores entities and triplets in dictionaries.""" + + def __init__(self) -> None: + self._nodes: Dict[str, Dict[str, Any]] = {} + self._labels: Dict[str, str] = {} + self._triplets: Dict[Tuple[str, str, str], Dict[str, Any]] = {} + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + def _ensure_uuid(self, props: Dict[str, Any]) -> str: + uid = props.get("uuid") + if isinstance(uid, UUID): + props["uuid"] = str(uid) + return str(uid) + if isinstance(uid, str): + return uid + new_uid = str(uuid4()) + props["uuid"] = new_uid + return new_uid + + # ------------------------------------------------------------------ + # GraphDriver API + # ------------------------------------------------------------------ + def upsert_entity(self, label: str, name: str, props: Dict[str, Any]) -> None: + props = dict(props) + uid = self._ensure_uuid(props) + props.setdefault("name", name) + props.setdefault("entity_label", label) + self._nodes[uid] = props + self._labels[uid] = label + + def upsert_edge(self, subj: str, pred: str, obj: str, props: Dict[str, Any]) -> None: + key = (str(subj), pred, str(obj)) + payload = dict(props) + payload.setdefault("subject", str(subj)) + payload.setdefault("predicate", pred) + payload.setdefault("object", str(obj)) + self._triplets[key] = payload + + def find(self, cypher: str, params: Dict[str, Any]) -> List[Dict[str, Any]]: + cypher = cypher.lower().strip() + if "where m.uuid" in cypher: + uid = str(params.get("uuid", "")) + node = self._nodes.get(uid) + if node is None: + return [] + return [{"m": dict(node)}] + if "where m.namespace" in cypher: + namespace = params.get("namespace") + results = [ + {"m": dict(node)} + for node in self._nodes.values() + if node.get("namespace") == namespace + ] + return results + if cypher.startswith("match (m) return m"): + return [{"m": dict(node)} for node in self._nodes.values()] + if cypher.startswith("match (a)-[r"): + namespace = params.get("namespace") + results: List[Dict[str, Any]] = [] + for payload in self._triplets.values(): + if namespace and payload.get("namespace") != namespace: + continue + record = { + "subject": payload.get("subject"), + "predicate": payload.get("predicate"), + "object": payload.get("object"), + "namespace": payload.get("namespace"), + "metadata": payload.get("metadata"), + "reference_time": payload.get("reference_time"), + } + results.append(record) + return results + return [] + + def get_entity(self, uid: str) -> Optional[Dict[str, Any]]: + node = self._nodes.get(str(uid)) + return dict(node) if node else None + + def _filtered_nodes( + self, + namespace: Optional[str], + entity_labels: Optional[Sequence[str]], + ) -> List[Dict[str, Any]]: + labels = set(entity_labels or []) + results: List[Dict[str, Any]] = [] + for node in self._nodes.values(): + if namespace is not None and node.get("namespace") != namespace: + continue + label = node.get("entity_label") or self._labels.get(str(node.get("uuid"))) + if labels and label not in labels: + continue + results.append(dict(node)) + return results + + def _slice(self, items: List[Dict[str, Any]], offset: int, limit: Optional[int]) -> List[Dict[str, Any]]: + offset = max(offset, 0) + if limit is None: + return items[offset:] + if limit <= 0: + return [] + return items[offset : offset + limit] + + def list_entities( + self, + namespace: Optional[str] = None, + entity_labels: Optional[Sequence[str]] = None, + *, + offset: int = 0, + limit: Optional[int] = None, + ) -> List[Dict[str, Any]]: + nodes = self._filtered_nodes(namespace, entity_labels) + return self._slice(nodes, offset, limit) + + def search_entities( + self, + query: Optional[str] = None, + namespace: Optional[str] = None, + entity_labels: Optional[Sequence[str]] = None, + *, + offset: int = 0, + limit: Optional[int] = None, + ) -> List[Dict[str, Any]]: + nodes = self._filtered_nodes(namespace, entity_labels) + if query: + needle = query.lower() + filtered: List[Dict[str, Any]] = [] + for node in nodes: + haystack = " ".join( + str(node.get(key, "")) for key in ("name", "content", "description") + ).lower() + metadata = node.get("metadata") or {} + if isinstance(metadata, dict): + haystack += " " + " ".join(str(value).lower() for value in metadata.values()) + if needle in haystack: + filtered.append(node) + nodes = filtered + return self._slice(nodes, offset, limit) + + def delete(self, uuid: UUID) -> None: + uid = str(uuid) + self._nodes.pop(uid, None) + self._labels.pop(uid, None) + to_delete = [key for key in self._triplets if key[0] == uid or key[2] == uid] + for key in to_delete: + self._triplets.pop(key, None) + + def delete_triplet(self, subj: str, pred: str, obj: str) -> None: + self._triplets.pop((str(subj), pred, str(obj)), None) + + def list_triplets(self, namespace: Optional[str] = None) -> List[Dict[str, Any]]: + results: List[Dict[str, Any]] = [] + for payload in self._triplets.values(): + if namespace and payload.get("namespace") != namespace: + continue + results.append( + { + "subject": payload.get("subject"), + "predicate": payload.get("predicate"), + "object": payload.get("object"), + "namespace": payload.get("namespace"), + "metadata": payload.get("metadata"), + "reference_time": payload.get("reference_time"), + } + ) + return results + + def count_entities(self, namespace: Optional[str] = None) -> Dict[str, Dict[str, int]]: + counts: Dict[str, Dict[str, int]] = {} + for node in self._nodes.values(): + ns = node.get("namespace") + if namespace is not None and ns != namespace: + continue + label = node.get("entity_label") or self._labels.get(str(node.get("uuid"))) or "Unknown" + bucket = counts.setdefault(ns or "default", {}) + bucket[label] = bucket.get(label, 0) + 1 + return counts diff --git a/meshmind/db/memgraph_driver.py b/meshmind/db/memgraph_driver.py index f4e8c4e..a8d79b0 100644 --- a/meshmind/db/memgraph_driver.py +++ b/meshmind/db/memgraph_driver.py @@ -1,110 +1,255 @@ -"""Memgraph implementation of GraphDriver.""" -from typing import Any, Dict, List -from .base_driver import GraphDriver +"""Memgraph implementation of :class:`GraphDriver` using ``mgclient``.""" +from __future__ import annotations - -"""Memgraph implementation of GraphDriver using mgclient.""" -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Sequence from urllib.parse import urlparse -try: +from meshmind.db.base_driver import GraphDriver + +try: # pragma: no cover - optional dependency import mgclient -except ImportError: +except ImportError: # pragma: no cover - optional dependency mgclient = None # type: ignore -from .base_driver import GraphDriver - class MemgraphDriver(GraphDriver): - """Memgraph driver implementation of GraphDriver using mgclient.""" + """Memgraph driver implementation backed by ``mgclient``.""" - def __init__(self, uri: str, username: str = None, password: str = None) -> None: - """Initialize Memgraph driver with Bolt URI and credentials.""" + def __init__(self, uri: str, username: str = "", password: str = "") -> None: if mgclient is None: raise ImportError("mgclient is required for MemgraphDriver") + self.uri = uri self.username = username self.password = password - # Parse URI: bolt://host:port + parsed = urlparse(uri) - host = parsed.hostname or 'localhost' + host = parsed.hostname or "localhost" port = parsed.port or 7687 - # Establish connection - self._conn = mgclient.connect( + + self._conn = mgclient.connect( # type: ignore[union-attr] host=host, port=port, - username=username, - password=password, + username=username or None, + password=password or None, ) self._cursor = self._conn.cursor() - def _execute(self, cypher: str, params: Optional[Dict[str, Any]] = None): - if params is None: - params = {} + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + def _execute(self, cypher: str, params: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]: + params = params or {} self._cursor.execute(cypher, params) try: rows = self._cursor.fetchall() cols = [col[0] for col in self._cursor.description] - results: List[Dict[str, Any]] = [] - for row in rows: - rec: Dict[str, Any] = {} - for idx, val in enumerate(row): - rec[cols[idx]] = val - results.append(rec) - return results except Exception: return [] + results: List[Dict[str, Any]] = [] + for row in rows: + record: Dict[str, Any] = {} + for idx, value in enumerate(row): + record[cols[idx]] = value + results.append(record) + return results + + @staticmethod + def _sanitize_predicate(predicate: str) -> str: + return predicate.replace("`", "") + + @staticmethod + def _normalize_node(node: Any) -> Dict[str, Any]: + if hasattr(node, "properties"): + try: + return dict(node.properties) # type: ignore[attr-defined] + except Exception: + pass + if isinstance(node, dict): + return dict(node) + if hasattr(node, "_properties"): + return dict(getattr(node, "_properties")) + return {k: v for k, v in getattr(node, "__dict__", {}).items() if not k.startswith("_")} + + # ------------------------------------------------------------------ + # GraphDriver API + # ------------------------------------------------------------------ def upsert_entity(self, label: str, name: str, props: Dict[str, Any]) -> None: - """Insert or update an entity node by uuid.""" - uid = props.get('uuid') + uid = props.get("uuid") cypher = ( f"MERGE (n:{label} {{uuid: $uuid}})\n" f"SET n += $props" ) - params = {'uuid': str(uid), 'props': props} + params = {"uuid": str(uid), "props": props} self._execute(cypher, params) self._conn.commit() def upsert_edge(self, subj: str, pred: str, obj: str, props: Dict[str, Any]) -> None: - """Insert or update an edge between two entities identified by uuid.""" + predicate = self._sanitize_predicate(pred) cypher = ( - f"MATCH (a {{uuid: $subj}}), (b {{uuid: $obj}})\n" - f"MERGE (a)-[r:`{pred}`]->(b)\n" - f"SET r += $props" + "MATCH (a {uuid: $subj}), (b {uuid: $obj})\n" + f"MERGE (a)-[r:`{predicate}`]->(b)\n" + "SET r += $props" ) - params = {'subj': str(subj), 'obj': str(obj), 'props': props} + params = {"subj": str(subj), "obj": str(obj), "props": props} self._execute(cypher, params) self._conn.commit() def find(self, cypher: str, params: Dict[str, Any]) -> List[Dict[str, Any]]: - """Execute a Cypher query and return results as list of dicts.""" return self._execute(cypher, params) + def get_entity(self, uid: str) -> Optional[Dict[str, Any]]: + records = self.find( + "MATCH (m) WHERE m.uuid = $uuid RETURN m", + {"uuid": str(uid)}, + ) + if not records: + return None + node = records[0].get("m", records[0]) + return self._normalize_node(node) + + def list_entities( + self, + namespace: Optional[str] = None, + entity_labels: Optional[Sequence[str]] = None, + *, + offset: int = 0, + limit: Optional[int] = None, + ) -> List[Dict[str, Any]]: + if limit is not None and limit <= 0: + return [] + clauses = ["($namespace IS NULL OR m.namespace = $namespace)"] + clauses.append("($labels IS NULL OR m.entity_label IN $labels)") + cypher = ["MATCH (m)"] + cypher.append("WHERE " + " AND ".join(clauses)) + cypher.append("RETURN m") + cypher.append("ORDER BY m.namespace, m.entity_label, m.name") + params = { + "namespace": namespace, + "labels": list(entity_labels) if entity_labels else None, + "offset": max(offset, 0), + } + cypher.append("SKIP $offset") + if limit is not None: + cypher.append("LIMIT $limit") + params["limit"] = limit + records = self.find("\n".join(cypher), params) + entities: List[Dict[str, Any]] = [] + for record in records: + node = record.get("m", record) + entities.append(self._normalize_node(node)) + return entities + + def search_entities( + self, + query: Optional[str] = None, + namespace: Optional[str] = None, + entity_labels: Optional[Sequence[str]] = None, + *, + offset: int = 0, + limit: Optional[int] = None, + ) -> List[Dict[str, Any]]: + if limit is not None and limit <= 0: + return [] + clauses = ["($namespace IS NULL OR m.namespace = $namespace)"] + clauses.append("($labels IS NULL OR m.entity_label IN $labels)") + params = { + "namespace": namespace, + "labels": list(entity_labels) if entity_labels else None, + "offset": max(offset, 0), + "search": query.lower() if query else None, + } + text_clause = ( + "($search IS NULL OR " + "toLower(coalesce(m.name, '')) CONTAINS $search OR " + "toLower(coalesce(m.content, '')) CONTAINS $search OR " + "toLower(coalesce(m.description, '')) CONTAINS $search OR " + "($search IS NOT NULL AND exists(m.metadata) AND " + "any(value IN values(m.metadata) WHERE toLower(toString(value)) CONTAINS $search))" + ")" + ) + clauses.append(text_clause) + cypher = ["MATCH (m)"] + cypher.append("WHERE " + " AND ".join(clauses)) + cypher.append("RETURN m") + cypher.append("ORDER BY m.reference_time DESC, m.name") + cypher.append("SKIP $offset") + if limit is not None: + cypher.append("LIMIT $limit") + params["limit"] = limit + records = self.find("\n".join(cypher), params) + entities: List[Dict[str, Any]] = [] + for record in records: + node = record.get("m", record) + entities.append(self._normalize_node(node)) + return entities + def delete(self, uuid: Any) -> None: - """Delete a node (and detach relationships) by uuid.""" cypher = "MATCH (n {uuid: $uuid}) DETACH DELETE n" - params = {'uuid': str(uuid)} + params = {"uuid": str(uuid)} + self._execute(cypher, params) + self._conn.commit() + + def delete_triplet(self, subj: str, pred: str, obj: str) -> None: + predicate = self._sanitize_predicate(pred) + cypher = ( + "MATCH (a {uuid: $subj})-[r:`{predicate}`]->(b {uuid: $obj}) " + "DELETE r" + ) + params = {"subj": str(subj), "obj": str(obj)} self._execute(cypher, params) self._conn.commit() + def list_triplets(self, namespace: Optional[str] = None) -> List[Dict[str, Any]]: + cypher = ( + "MATCH (a)-[r]->(b)\n" + "WHERE $namespace IS NULL OR r.namespace = $namespace\n" + "RETURN a.uuid AS subject, type(r) AS predicate, b.uuid AS object, " + "r.namespace AS namespace, r.metadata AS metadata, r.reference_time AS reference_time" + ) + params = {"namespace": namespace} + return self._execute(cypher, params) + + def count_entities(self, namespace: Optional[str] = None) -> Dict[str, Dict[str, int]]: + clauses = ["($namespace IS NULL OR m.namespace = $namespace)"] + cypher = ( + "MATCH (m)\n" + "WHERE " + + " AND ".join(clauses) + + "\nRETURN coalesce(m.namespace, '') AS namespace, " + "m.entity_label AS label, count(m) AS count" + ) + params = {"namespace": namespace} + rows = self.find(cypher, params) + results: Dict[str, Dict[str, int]] = {} + for row in rows: + ns = row.get("namespace") or "default" + label = row.get("label") or "Unknown" + count = int(row.get("count", 0)) + bucket = results.setdefault(ns, {}) + bucket[label] = count + return results + + # ------------------------------------------------------------------ + # Convenience helpers + # ------------------------------------------------------------------ def vector_search(self, embedding: List[float], top_k: int = 10) -> List[Dict[str, Any]]: - """ - Fallback vector search: loads all embeddings and ranks by cosine similarity. - """ from meshmind.core.similarity import cosine_similarity - # Load all entities with embeddings - records = self.find("MATCH (n) WHERE exists(n.embedding) RETURN n.embedding AS emb, n AS node", {}) + + records = self.find( + "MATCH (n) WHERE exists(n.embedding) RETURN n.embedding AS emb, n AS node", + {}, + ) scored = [] for rec in records: - emb = rec.get('emb') + emb = rec.get("emb") if not isinstance(emb, list): continue try: score = cosine_similarity(embedding, emb) except Exception: score = 0.0 - scored.append({'node': rec.get('node'), 'score': float(score)}) - # Sort and take top_k - scored.sort(key=lambda x: x['score'], reverse=True) - return scored[:top_k] \ No newline at end of file + scored.append({"node": rec.get("node"), "score": float(score)}) + scored.sort(key=lambda item: item["score"], reverse=True) + return scored[:top_k] diff --git a/meshmind/db/neo4j_driver.py b/meshmind/db/neo4j_driver.py new file mode 100644 index 0000000..cfd2214 --- /dev/null +++ b/meshmind/db/neo4j_driver.py @@ -0,0 +1,200 @@ +"""Neo4j implementation of :class:`GraphDriver` using the official driver.""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Sequence + +from meshmind.db.base_driver import GraphDriver + +try: # pragma: no cover - optional dependency + from neo4j import GraphDatabase # type: ignore +except ImportError: # pragma: no cover - optional dependency + GraphDatabase = None # type: ignore + + +class Neo4jGraphDriver(GraphDriver): + """GraphDriver backed by Neo4j via the ``neo4j`` Python driver.""" + + def __init__(self, uri: str, username: str = "neo4j", password: str = "") -> None: + if GraphDatabase is None: + raise ImportError("neo4j driver is required for Neo4jGraphDriver") + auth = None + if username or password: + auth = (username or None, password or None) + self._driver = GraphDatabase.driver(uri, auth=auth) + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + def _run(self, cypher: str, params: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]: + params = params or {} + with self._driver.session() as session: # type: ignore[attr-defined] + result = session.run(cypher, **params) + records = [] + for record in result: + records.append(record.data()) + return records + + @staticmethod + def _normalize_node(node: Any) -> Dict[str, Any]: + if hasattr(node, "_properties"): + return dict(node._properties) # type: ignore[attr-defined] + if isinstance(node, dict): + return dict(node) + return {k: v for k, v in getattr(node, "__dict__", {}).items() if not k.startswith("_")} + + # ------------------------------------------------------------------ + # GraphDriver API + # ------------------------------------------------------------------ + def upsert_entity(self, label: str, name: str, props: Dict[str, Any]) -> None: + cypher = ( + f"MERGE (n:{label} {{uuid: $uuid}})\n" + "SET n += $props" + ) + params = {"uuid": str(props.get("uuid")), "props": props} + self._run(cypher, params) + + def upsert_edge(self, subj: str, pred: str, obj: str, props: Dict[str, Any]) -> None: + predicate = pred.replace("`", "") + cypher = ( + "MATCH (a {uuid: $subj}), (b {uuid: $obj})\n" + f"MERGE (a)-[r:`{predicate}`]->(b)\n" + "SET r += $props" + ) + params = {"subj": str(subj), "obj": str(obj), "props": props} + self._run(cypher, params) + + def find(self, cypher: str, params: Dict[str, Any]) -> List[Dict[str, Any]]: + return self._run(cypher, params) + + def get_entity(self, uid: str) -> Optional[Dict[str, Any]]: + records = self.find("MATCH (m) WHERE m.uuid = $uuid RETURN m", {"uuid": str(uid)}) + if not records: + return None + node = records[0].get("m", records[0]) + return self._normalize_node(node) + + def list_entities( + self, + namespace: Optional[str] = None, + entity_labels: Optional[Sequence[str]] = None, + *, + offset: int = 0, + limit: Optional[int] = None, + ) -> List[Dict[str, Any]]: + if limit is not None and limit <= 0: + return [] + clauses = ["($namespace IS NULL OR m.namespace = $namespace)"] + clauses.append("($labels IS NULL OR m.entity_label IN $labels)") + cypher = ["MATCH (m)"] + cypher.append("WHERE " + " AND ".join(clauses)) + cypher.append("RETURN m") + cypher.append("ORDER BY m.namespace, m.entity_label, m.name") + cypher.append("SKIP $offset") + if limit is not None: + cypher.append("LIMIT $limit") + params = { + "namespace": namespace, + "labels": list(entity_labels) if entity_labels else None, + "offset": max(offset, 0), + } + if limit is not None: + params["limit"] = limit + records = self.find("\n".join(cypher), params) + return [self._normalize_node(rec.get("m", rec)) for rec in records] + + def search_entities( + self, + query: Optional[str] = None, + namespace: Optional[str] = None, + entity_labels: Optional[Sequence[str]] = None, + *, + offset: int = 0, + limit: Optional[int] = None, + ) -> List[Dict[str, Any]]: + if limit is not None and limit <= 0: + return [] + clauses = ["($namespace IS NULL OR m.namespace = $namespace)"] + clauses.append("($labels IS NULL OR m.entity_label IN $labels)") + params = { + "namespace": namespace, + "labels": list(entity_labels) if entity_labels else None, + "offset": max(offset, 0), + "search": query.lower() if query else None, + } + text_clause = ( + "($search IS NULL OR " + "toLower(coalesce(m.name, '')) CONTAINS $search OR " + "toLower(coalesce(m.content, '')) CONTAINS $search OR " + "toLower(coalesce(m.description, '')) CONTAINS $search OR " + "($search IS NOT NULL AND exists(m.metadata) AND " + "any(value IN values(m.metadata) WHERE toLower(toString(value)) CONTAINS $search))" + ")" + ) + clauses.append(text_clause) + cypher = ["MATCH (m)"] + cypher.append("WHERE " + " AND ".join(clauses)) + cypher.append("RETURN m") + cypher.append("ORDER BY m.reference_time DESC, m.name") + cypher.append("SKIP $offset") + if limit is not None: + cypher.append("LIMIT $limit") + params["limit"] = limit + records = self.find("\n".join(cypher), params) + return [self._normalize_node(rec.get("m", rec)) for rec in records] + + def delete(self, uuid: Any) -> None: + self._run("MATCH (m {uuid: $uuid}) DETACH DELETE m", {"uuid": str(uuid)}) + + def delete_triplet(self, subj: str, pred: str, obj: str) -> None: + predicate = pred.replace("`", "") + cypher = ( + f"MATCH (a {{uuid: $subj}})-[r:`{predicate}`]->(b {{uuid: $obj}})" + " DELETE r" + ) + self._run(cypher, {"subj": str(subj), "obj": str(obj)}) + + def list_triplets(self, namespace: Optional[str] = None) -> List[Dict[str, Any]]: + cypher = ( + "MATCH (a)-[r]->(b)\n" + "WHERE $namespace IS NULL OR r.namespace = $namespace\n" + "RETURN a.uuid AS subject, type(r) AS predicate, b.uuid AS object, " + "r.namespace AS namespace, r.metadata AS metadata, r.reference_time AS reference_time" + ) + params = {"namespace": namespace} + return self.find(cypher, params) + + def close(self) -> None: + self._driver.close() + + def count_entities(self, namespace: Optional[str] = None) -> Dict[str, Dict[str, int]]: + clauses = ["($namespace IS NULL OR m.namespace = $namespace)"] + cypher = ( + "MATCH (m)\n" + "WHERE " + + " AND ".join(clauses) + + "\nRETURN coalesce(m.namespace, '') AS namespace, " + "m.entity_label AS label, count(m) AS count" + ) + params = {"namespace": namespace} + rows = self.find(cypher, params) + results: Dict[str, Dict[str, int]] = {} + for row in rows: + ns = row.get("namespace") or "default" + label = row.get("label") or "Unknown" + count = int(row.get("count", 0)) + bucket = results.setdefault(ns, {}) + bucket[label] = count + return results + + def verify_connectivity(self) -> bool: + """Use the Neo4j driver to verify connectivity.""" + + checker = getattr(self._driver, "verify_connectivity", None) + if callable(checker): + checker() + return True + try: + self._run("RETURN 1 AS ok") + except Exception: + return False + return True diff --git a/meshmind/db/sqlite_driver.py b/meshmind/db/sqlite_driver.py new file mode 100644 index 0000000..24bb468 --- /dev/null +++ b/meshmind/db/sqlite_driver.py @@ -0,0 +1,308 @@ +"""SQLite implementation of :class:`GraphDriver` for lightweight persistence.""" +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence + +from meshmind.db.base_driver import GraphDriver + + +class SQLiteGraphDriver(GraphDriver): + """GraphDriver backed by SQLite tables using simple JSON columns.""" + + def __init__(self, path: str | Path = ":memory:") -> None: + self._path = str(path) + self._conn = sqlite3.connect(self._path) + self._conn.row_factory = sqlite3.Row + self._ensure_schema() + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + def _ensure_schema(self) -> None: + cur = self._conn.cursor() + cur.execute( + """ + CREATE TABLE IF NOT EXISTS entities ( + uuid TEXT PRIMARY KEY, + label TEXT NOT NULL, + name TEXT NOT NULL, + namespace TEXT, + props TEXT NOT NULL + ) + """ + ) + cur.execute( + """ + CREATE TABLE IF NOT EXISTS triplets ( + subject TEXT NOT NULL, + predicate TEXT NOT NULL, + object TEXT NOT NULL, + namespace TEXT, + metadata TEXT, + reference_time TEXT, + PRIMARY KEY (subject, predicate, object) + ) + """ + ) + self._conn.commit() + + def _row_to_dict(self, row: sqlite3.Row) -> Dict[str, Any]: + payload = dict(row) + if "props" in payload and payload["props"]: + props = payload.pop("props") + if isinstance(props, str): + payload.update(json.loads(props)) + elif isinstance(props, dict): + payload.update(props) + if "metadata" in payload and isinstance(payload["metadata"], str): + payload["metadata"] = json.loads(payload["metadata"]) + return payload + + # ------------------------------------------------------------------ + # GraphDriver API + # ------------------------------------------------------------------ + def upsert_entity(self, label: str, name: str, props: Dict[str, Any]) -> None: + payload = dict(props) + uid = str(payload.get("uuid")) + if not uid: + raise ValueError("Memory props must include a UUID for SQLiteGraphDriver") + payload.setdefault("entity_label", label) + payload.setdefault("name", name) + namespace = payload.get("namespace") + cur = self._conn.cursor() + cur.execute( + """ + INSERT INTO entities (uuid, label, name, namespace, props) + VALUES (:uuid, :label, :name, :namespace, :props) + ON CONFLICT(uuid) DO UPDATE SET + label=excluded.label, + name=excluded.name, + namespace=excluded.namespace, + props=excluded.props + """, + { + "uuid": uid, + "label": payload.get("entity_label", label), + "name": payload.get("name", name), + "namespace": namespace, + "props": json.dumps(payload), + }, + ) + self._conn.commit() + + def upsert_edge(self, subj: str, pred: str, obj: str, props: Dict[str, Any]) -> None: + payload = dict(props) + payload.setdefault("subject", subj) + payload.setdefault("predicate", pred) + payload.setdefault("object", obj) + metadata = payload.get("metadata") or {} + cur = self._conn.cursor() + cur.execute( + """ + INSERT INTO triplets (subject, predicate, object, namespace, metadata, reference_time) + VALUES (:subject, :predicate, :object, :namespace, :metadata, :reference_time) + ON CONFLICT(subject, predicate, object) DO UPDATE SET + namespace=excluded.namespace, + metadata=excluded.metadata, + reference_time=excluded.reference_time + """, + { + "subject": payload["subject"], + "predicate": payload["predicate"], + "object": payload["object"], + "namespace": payload.get("namespace"), + "metadata": json.dumps(metadata), + "reference_time": payload.get("reference_time"), + }, + ) + self._conn.commit() + + def find(self, cypher: str, params: Dict[str, Any]) -> List[Dict[str, Any]]: + # Provide compatibility for simple MATCH queries used by MemoryManager. + cypher_lower = cypher.lower().strip() + cur = self._conn.cursor() + if "where m.uuid" in cypher_lower: + cur.execute("SELECT * FROM entities WHERE uuid = :uuid", {"uuid": params.get("uuid")}) + row = cur.fetchone() + if not row: + return [] + return [{"m": self._row_to_dict(row)}] + if "where m.namespace" in cypher_lower: + cur.execute( + "SELECT * FROM entities WHERE namespace = :namespace", + {"namespace": params.get("namespace")}, + ) + rows = cur.fetchall() + return [{"m": self._row_to_dict(row)} for row in rows] + if cypher_lower.startswith("match (m) return m"): + cur.execute("SELECT * FROM entities") + rows = cur.fetchall() + return [{"m": self._row_to_dict(row)} for row in rows] + if cypher_lower.startswith("match (a)-[r"): + namespace = params.get("namespace") + if namespace: + cur.execute( + "SELECT * FROM triplets WHERE namespace = :namespace", + {"namespace": namespace}, + ) + else: + cur.execute("SELECT * FROM triplets") + rows = cur.fetchall() + return [dict(row) for row in rows] + return [] + + def get_entity(self, uid: str) -> Optional[Dict[str, Any]]: + cur = self._conn.cursor() + cur.execute("SELECT * FROM entities WHERE uuid = :uuid", {"uuid": uid}) + row = cur.fetchone() + return self._row_to_dict(row) if row else None + + def list_entities( + self, + namespace: Optional[str] = None, + entity_labels: Optional[Sequence[str]] = None, + *, + offset: int = 0, + limit: Optional[int] = None, + ) -> List[Dict[str, Any]]: + cur = self._conn.cursor() + clauses: List[str] = [] + params: Dict[str, Any] = {} + if namespace: + clauses.append("namespace = :namespace") + params["namespace"] = namespace + if entity_labels: + placeholders = [] + for idx, label in enumerate(entity_labels): + key = f"label_{idx}" + placeholders.append(f":{key}") + params[key] = label + clauses.append(f"label IN ({', '.join(placeholders)})") + query = "SELECT * FROM entities" + if clauses: + query += " WHERE " + " AND ".join(clauses) + query += " ORDER BY namespace, label, name" + if limit is not None: + if limit <= 0: + return [] + query += " LIMIT :limit" + params["limit"] = limit + if offset > 0: + query += " OFFSET :offset" + params["offset"] = offset + elif offset > 0: + query += " LIMIT -1 OFFSET :offset" + params["offset"] = offset + cur.execute(query, params) + rows = cur.fetchall() + return [self._row_to_dict(row) for row in rows] + + def search_entities( + self, + query: Optional[str] = None, + namespace: Optional[str] = None, + entity_labels: Optional[Sequence[str]] = None, + *, + offset: int = 0, + limit: Optional[int] = None, + ) -> List[Dict[str, Any]]: + cur = self._conn.cursor() + clauses: List[str] = [] + params: Dict[str, Any] = {} + if namespace: + clauses.append("namespace = :namespace") + params["namespace"] = namespace + if entity_labels: + placeholders = [] + for idx, label in enumerate(entity_labels): + key = f"label_{idx}" + placeholders.append(f":{key}") + params[key] = label + clauses.append(f"label IN ({', '.join(placeholders)})") + if query: + params["needle"] = f"%{query.lower()}%" + clauses.append( + "(LOWER(name) LIKE :needle OR LOWER(props) LIKE :needle)" + ) + sql = "SELECT * FROM entities" + if clauses: + sql += " WHERE " + " AND ".join(clauses) + sql += " ORDER BY namespace, label, name" + if limit is not None: + if limit <= 0: + return [] + sql += " LIMIT :limit" + params["limit"] = limit + if offset > 0: + sql += " OFFSET :offset" + params["offset"] = offset + elif offset > 0: + sql += " LIMIT -1 OFFSET :offset" + params["offset"] = offset + cur.execute(sql, params) + rows = cur.fetchall() + return [self._row_to_dict(row) for row in rows] + + def delete(self, uuid: Any) -> None: + cur = self._conn.cursor() + cur.execute("DELETE FROM entities WHERE uuid = :uuid", {"uuid": str(uuid)}) + cur.execute( + "DELETE FROM triplets WHERE subject = :uuid OR object = :uuid", + {"uuid": str(uuid)}, + ) + self._conn.commit() + + def delete_triplet(self, subj: str, pred: str, obj: str) -> None: + cur = self._conn.cursor() + cur.execute( + "DELETE FROM triplets WHERE subject = :subject AND predicate = :predicate AND object = :object", + {"subject": str(subj), "predicate": pred, "object": str(obj)}, + ) + self._conn.commit() + + def list_triplets(self, namespace: Optional[str] = None) -> List[Dict[str, Any]]: + cur = self._conn.cursor() + if namespace: + cur.execute( + "SELECT * FROM triplets WHERE namespace = :namespace", + {"namespace": namespace}, + ) + else: + cur.execute("SELECT * FROM triplets") + rows = cur.fetchall() + result = [] + for row in rows: + payload = dict(row) + metadata = payload.get("metadata") + payload["metadata"] = json.loads(metadata) if metadata else {} + result.append(payload) + return result + + def count_entities(self, namespace: Optional[str] = None) -> Dict[str, Dict[str, int]]: + cur = self._conn.cursor() + params: Dict[str, Any] = {} + sql = ( + "SELECT COALESCE(namespace, '') AS namespace, label, COUNT(*) AS count " + "FROM entities" + ) + if namespace: + sql += " WHERE namespace = :namespace" + params["namespace"] = namespace + sql += " GROUP BY namespace, label" + cur.execute(sql, params) + rows = cur.fetchall() + results: Dict[str, Dict[str, int]] = {} + for row in rows: + payload = dict(row) + ns = payload.get("namespace") or "default" + label = payload.get("label") or "Unknown" + count = int(payload.get("count", 0)) + bucket = results.setdefault(ns, {}) + bucket[label] = count + return results + + def close(self) -> None: + self._conn.close() diff --git a/meshmind/models/registry.py b/meshmind/models/registry.py index bd98f87..eea632b 100644 --- a/meshmind/models/registry.py +++ b/meshmind/models/registry.py @@ -1,6 +1,6 @@ """Registry for entity and predicate models.""" from typing import Type, Optional, Dict, Set -from pydantic import BaseModel +from meshmind._compat.pydantic import BaseModel class EntityRegistry: @@ -30,4 +30,25 @@ def add(cls, label: str) -> None: @classmethod def allowed(cls, label: str) -> bool: """Check if a predicate label is allowed.""" - return label in cls._predicates \ No newline at end of file + return label in cls._predicates + + @classmethod + def all(cls) -> Set[str]: + """Return all registered predicate labels.""" + + return set(cls._predicates) + + @classmethod + def clear(cls) -> None: + """Remove all registered predicates (testing helper).""" + + cls._predicates.clear() + + @classmethod + def remove(cls, label: str) -> bool: + """Remove a predicate label if it exists.""" + + if label in cls._predicates: + cls._predicates.remove(label) + return True + return False diff --git a/meshmind/pipeline/compress.py b/meshmind/pipeline/compress.py index 9ded746..c7eb995 100644 --- a/meshmind/pipeline/compress.py +++ b/meshmind/pipeline/compress.py @@ -1,13 +1,11 @@ -""" -Pipeline for token-aware compression/summarization of memories. -""" +"""Token-aware compression helpers for memory metadata.""" +from __future__ import annotations + from typing import List -from meshmind.core.types import Memory -try: - import tiktoken -except ImportError: - tiktoken = None # type: ignore +from meshmind.core.observability import log_event, telemetry +from meshmind.core.types import Memory +from meshmind.core.utils import get_token_encoder def compress_memories( @@ -20,19 +18,29 @@ def compress_memories( :param max_tokens: Maximum number of tokens allowed per memory. :return: List of Memory objects with content possibly shortened. """ - encoder = tiktoken.get_encoding('o200k_base') + log_event("pipeline.compress.start", items=len(memories)) + encoder = get_token_encoder("o200k_base", optional=True) + if encoder is None: + telemetry.increment("pipeline.compress.skipped", len(memories)) + return memories compressed = [] - for mem in memories: - content = mem.metadata.get('content') - if not isinstance(content, str): - compressed.append(mem) - continue - tokens = encoder.encode(content) - if len(tokens) <= max_tokens: + modified = 0 + with telemetry.track_duration("pipeline.compress.duration"): + for mem in memories: + content = mem.metadata.get('content') + if not isinstance(content, str): + compressed.append(mem) + continue + tokens = encoder.encode(content) + if len(tokens) <= max_tokens: + compressed.append(mem) + continue + # Truncate tokens and decode back to string + truncated = encoder.decode(tokens[:max_tokens]) + mem.metadata['content'] = truncated compressed.append(mem) - continue - # Truncate tokens and decode back to string - truncated = encoder.decode(tokens[:max_tokens]) - mem.metadata['content'] = truncated - compressed.append(mem) - return compressed \ No newline at end of file + modified += 1 + telemetry.increment("pipeline.compress.processed", len(memories)) + telemetry.increment("pipeline.compress.modified", modified) + log_event("pipeline.compress.complete", modified=modified) + return compressed diff --git a/meshmind/pipeline/consolidate.py b/meshmind/pipeline/consolidate.py index e3f4dc3..398007e 100644 --- a/meshmind/pipeline/consolidate.py +++ b/meshmind/pipeline/consolidate.py @@ -1,29 +1,157 @@ -""" -Pipeline for consolidating and summarizing duplicate memories. -""" -from typing import List, Any +"""Pipeline helpers for consolidating and summarising duplicate memories.""" +from __future__ import annotations + +from dataclasses import dataclass +from dataclasses import dataclass, field +from typing import Dict, Iterable, List, Optional from meshmind.core.types import Memory -def consolidate_memories(memories: List[Memory]) -> List[Memory]: - """ - Consolidate duplicate memories by name, preferring high importance. +@dataclass +class ConsolidationOutcome: + """Result of merging a group of related memories.""" + + updated: Memory + removed_ids: List[str] + skipped_ids: List[str] = field(default_factory=list) + + +@dataclass +class ConsolidationSettings: + """Constraints used when planning consolidation batches.""" + + max_group_size: int = 50 + max_updates: int = 200 + max_updates_per_namespace: int = 40 + + +@dataclass +class ConsolidationPlan: + """Container for consolidation outcomes and any skipped groups.""" + + outcomes: List[ConsolidationOutcome] = field(default_factory=list) + skipped_groups: Dict[str, int] = field(default_factory=dict) + + def add_skipped(self, namespace: str, count: int) -> None: + self.skipped_groups[namespace] = self.skipped_groups.get(namespace, 0) + count + + def __iter__(self): + return iter(self.outcomes) + + def __len__(self) -> int: + return len(self.outcomes) - :param memories: List of Memory objects. - :return: List of consolidated Memory objects. - """ - # Group memories by name - grouped: dict[str, List[Memory]] = {} + +def _merge_metadata(group: Iterable[Memory]) -> Dict[str, object]: + merged: Dict[str, object] = {} + for mem in group: + metadata = getattr(mem, "metadata", {}) or {} + if not isinstance(metadata, dict): + continue + for key, value in metadata.items(): + if key not in merged: + merged[key] = value + continue + existing = merged[key] + if existing == value: + continue + if not isinstance(existing, list): + existing = [existing] + if isinstance(value, list): + for item in value: + if item not in existing: + existing.append(item) + else: + if value not in existing: + existing.append(value) + merged[key] = existing + return merged + + +def _combine_embeddings(group: Iterable[Memory]) -> List[float] | None: + embeddings: List[List[float]] = [] + for mem in group: + emb = getattr(mem, "embedding", None) + if isinstance(emb, list) and emb: + embeddings.append([float(x) for x in emb]) + if not embeddings: + return None + length = len(embeddings[0]) + sums = [0.0] * length + for vector in embeddings: + if len(vector) != length: + continue + for idx, value in enumerate(vector): + sums[idx] += value + count = max(len(embeddings), 1) + return [round(total / count, 6) for total in sums] + + +def _summary_from_group(group: Iterable[Memory]) -> str: + seen: List[str] = [] + for mem in group: + text = "" + metadata = getattr(mem, "metadata", {}) or {} + if isinstance(metadata, dict): + text = str(metadata.get("content") or metadata.get("summary") or "") + if not text: + text = getattr(mem, "name", "") + text = text.strip() + if text and text not in seen: + seen.append(text) + return " \n".join(seen[:3]) + + +def consolidate_memories( + memories: List[Memory], + settings: Optional[ConsolidationSettings] = None, +) -> ConsolidationPlan: + """Consolidate duplicate memories and describe the merge plan.""" + + grouped: Dict[tuple[str, str, str], List[Memory]] = {} for mem in memories: - grouped.setdefault(mem.name, []).append(mem) + key = (getattr(mem, "namespace", ""), getattr(mem, "entity_label", ""), getattr(mem, "name", "")) + grouped.setdefault(key, []).append(mem) + + cfg = settings or ConsolidationSettings() + plan = ConsolidationPlan() + namespace_counts: Dict[str, int] = {} + + for group in grouped.values(): + if len(group) == 1: + continue + namespace = getattr(group[0], "namespace", "") or "default" + if len(group) > cfg.max_group_size: + plan.add_skipped(namespace, len(group)) + continue + if len(plan) >= cfg.max_updates: + plan.add_skipped(namespace, len(group)) + continue + if namespace_counts.get(namespace, 0) >= cfg.max_updates_per_namespace: + plan.add_skipped(namespace, len(group)) + continue - consolidated: List[Memory] = [] - for name, group in grouped.items(): - # Choose the memory with highest importance (fallback to first) - selected = max( + primary = max( group, - key=lambda m: (m.importance or 0.0), + key=lambda m: ( + getattr(m, "importance", 0.0) or 0.0, + getattr(m, "updated_at", getattr(m, "created_at", None)), + ), ) - consolidated.append(selected) - return consolidated \ No newline at end of file + metadata = _merge_metadata(group) + summary = _summary_from_group(group) + if summary: + metadata.setdefault("consolidated_summary", summary) + embedding = _combine_embeddings(group) + update: Dict[str, object] = {"metadata": metadata} + if embedding is not None: + update["embedding"] = embedding + if metadata and "importance" not in metadata: + importance_values = [getattr(mem, "importance", 0.0) or 0.0 for mem in group] + update["importance"] = round(max(importance_values), 3) + updated = primary.model_copy(update=update) + removed_ids = [str(getattr(mem, "uuid", "")) for mem in group if mem is not primary] + namespace_counts[namespace] = namespace_counts.get(namespace, 0) + 1 + plan.outcomes.append(ConsolidationOutcome(updated=updated, removed_ids=removed_ids)) + return plan diff --git a/meshmind/pipeline/extract.py b/meshmind/pipeline/extract.py index 613073b..34a131d 100644 --- a/meshmind/pipeline/extract.py +++ b/meshmind/pipeline/extract.py @@ -1,11 +1,13 @@ -from typing import Any, List, Type +from typing import Any, List, Sequence, Type + +from meshmind.core.observability import log_event, telemetry def extract_memories( instructions: str, namespace: str, - entity_types: List[Type[Any]], + entity_types: Sequence[Type[Any]], embedding_model: str, - content: List[str], + content: Sequence[str], llm_client: Any = None, ) -> List[Any]: """ @@ -20,15 +22,18 @@ def extract_memories( :return: List of extracted Memory-like dicts or objects. """ import json - try: - from openai import OpenAI - except ImportError: - raise RuntimeError("openai package is required for extraction pipeline") from meshmind.core.types import Memory from meshmind.core.embeddings import EncoderRegistry + from meshmind.models.registry import EntityRegistry + + log_event("pipeline.extract.start", segments=len(content)) # Initialize default LLM client if not provided if llm_client is None: + try: + from openai import OpenAI + except ImportError as exc: + raise RuntimeError("openai package is required for extraction pipeline") from exc llm_client = OpenAI() # Prepare function schema for Memory items @@ -46,6 +51,9 @@ def extract_memories( } # Build system prompt using a default template and user instructions + entity_types = list(entity_types) or [Memory] + for model in entity_types: + EntityRegistry.register(model) allowed_labels = [cls.__name__ for cls in entity_types] default_prompt = ( "You are an agent that extracts structured memories from text segments. " @@ -58,15 +66,16 @@ def extract_memories( prompt += f"\nAllowed entity labels: {', '.join(allowed_labels)}." messages = [{"role": "system", "content": prompt}] # Add each text segment as a user message - messages += [{"role": "user", "content": text} for text in content] + messages += [{"role": "user", "content": text} for text in list(content)] # Call chat completion with function-calling - response = llm_client.responses.create( - model="gpt-4.1-mini", - messages=messages, - functions=[function_spec], - function_call={"name": "extract_memories"}, - ) + with telemetry.track_duration("pipeline.extract.duration"): + response = llm_client.responses.create( + model="gpt-4.1-mini", + messages=messages, + functions=[function_spec], + function_call={"name": "extract_memories"}, + ) msg = response.choices[0].message # Parse function call arguments or direct JSON if msg.get("function_call"): @@ -82,7 +91,7 @@ def extract_memories( memories = [] # Instantiate Memory objects, validate entity labels, and compute embeddings - from pydantic import ValidationError + from meshmind._compat.pydantic import ValidationError encoder = EncoderRegistry.get(embedding_model) for entry in items: label = entry.get("entity_label") @@ -99,4 +108,8 @@ def extract_memories( emb = encoder.encode([mem.name])[0] mem.embedding = emb memories.append(mem) - return memories \ No newline at end of file + + telemetry.increment("pipeline.extract.segments", len(content)) + telemetry.increment("pipeline.extract.memories", len(memories)) + log_event("pipeline.extract.complete", memories=len(memories)) + return memories diff --git a/meshmind/pipeline/preprocess.py b/meshmind/pipeline/preprocess.py index fde0077..53d7a39 100644 --- a/meshmind/pipeline/preprocess.py +++ b/meshmind/pipeline/preprocess.py @@ -1,4 +1,12 @@ -from typing import Any, List +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Dict, List + +import math +import re + +from meshmind.core.observability import log_event, telemetry def deduplicate(memories: List[Any], threshold: float = 0.95) -> List[Any]: """ @@ -40,22 +48,106 @@ def deduplicate(memories: List[Any], threshold: float = 0.95) -> List[Any]: unique.append(mem) return unique +def _text_from_memory(mem: Any) -> str: + chunks: List[str] = [] + name = getattr(mem, "name", None) + if isinstance(name, str): + chunks.append(name) + metadata = getattr(mem, "metadata", None) or {} + if isinstance(metadata, dict): + for value in metadata.values(): + if isinstance(value, str): + chunks.append(value) + return " ".join(chunks) + + +def _recent_bonus(reference_time: Any, now: datetime) -> float: + if reference_time is None: + return 0.0 + try: + if isinstance(reference_time, datetime): + ref = reference_time + else: + ref = datetime.fromisoformat(str(reference_time)) + except Exception: + return 0.0 + if ref.tzinfo is None: + ref = ref.replace(tzinfo=timezone.utc) + delta = now - ref + seconds = max(delta.total_seconds(), 0.0) + week = 7 * 24 * 60 * 60 + return max(0.0, 1.0 - min(seconds / week, 1.0)) + + def score_importance(memories: List[Any]) -> List[Any]: - """ - Assign or update importance scores for each Memory. + """Assign a heuristic importance score for each memory.""" - :param memories: List of Memory-like objects. - :return: List of Memory-like objects with updated importance. - """ - # Assign a default importance score if missing + now = datetime.now(timezone.utc) for mem in memories: - if getattr(mem, 'importance', None) is None: - try: - mem.importance = 1.0 - except Exception: + try: + if getattr(mem, "importance", None) not in (None, 0): continue + + text = _text_from_memory(mem) + tokens = re.findall(r"\w+", text.lower()) + unique_ratio = len(set(tokens)) / max(len(tokens), 1) + length_factor = min(len(tokens) / 25.0, 1.5) + digit_bonus = 0.3 if any(ch.isdigit() for ch in text) else 0.0 + recency_bonus = _recent_bonus(getattr(mem, "reference_time", None), now) + metadata_size = len(getattr(mem, "metadata", {}) or {}) + metadata_bonus = min(metadata_size * 0.05, 0.25) + + embedding = getattr(mem, "embedding", None) or [] + magnitude = math.sqrt(sum((float(x) ** 2 for x in embedding))) if embedding else 0.0 + magnitude_bonus = min(magnitude / 10.0, 0.4) + + score = 0.5 + (0.8 * unique_ratio) + length_factor + digit_bonus + score += recency_bonus + metadata_bonus + magnitude_bonus + mem.importance = round(min(score, 5.0), 3) + except Exception: + continue + metrics = summarize_importance(memories) + telemetry.gauge("importance.mean", metrics["mean"]) + telemetry.gauge("importance.stddev", metrics["stddev"]) + telemetry.gauge("importance.count", float(metrics["count"])) + log_event( + "importance.scored", + mean=metrics["mean"], + stddev=metrics["stddev"], + count=metrics["count"], + recent=metrics["recent_bonus"], + ) return memories + +def summarize_importance(memories: List[Any]) -> Dict[str, float]: + """Return descriptive statistics about importance assignments.""" + + values: List[float] = [] + recency_bonus_total = 0.0 + now = datetime.now(timezone.utc) + for mem in memories: + val = getattr(mem, "importance", None) + if val is None: + continue + try: + values.append(float(val)) + recency_bonus_total += _recent_bonus(getattr(mem, "reference_time", None), now) + except Exception: + continue + count = len(values) + if not values: + return {"mean": 0.0, "stddev": 0.0, "count": 0.0, "recent_bonus": 0.0} + mean_value = sum(values) / count + variance = sum((value - mean_value) ** 2 for value in values) / count + stddev = math.sqrt(variance) + return { + "mean": round(mean_value, 3), + "stddev": round(stddev, 3), + "count": float(count), + "recent_bonus": round(recency_bonus_total / count, 3), + } + def compress(memories: List[Any]) -> List[Any]: """ Compress long text fields in Memory objects to save space or reduce tokens. diff --git a/meshmind/pipeline/store.py b/meshmind/pipeline/store.py index 3d0b07c..f4e1b6c 100644 --- a/meshmind/pipeline/store.py +++ b/meshmind/pipeline/store.py @@ -1,23 +1,71 @@ +"""Persistence helpers for storing memories and triplets.""" +from __future__ import annotations + from typing import Any, Iterable + +from meshmind._compat.pydantic import BaseModel + +from meshmind.core.observability import log_event, telemetry +from meshmind.core.types import Triplet from meshmind.db.base_driver import GraphDriver +from meshmind.models.registry import EntityRegistry, PredicateRegistry + + +def _props(obj: Any) -> dict[str, Any]: + if isinstance(obj, BaseModel): + return obj.dict(exclude_none=True) + if hasattr(obj, "dict"): + try: + return obj.dict(exclude_none=True) # type: ignore[attr-defined] + except TypeError: + pass + if isinstance(obj, dict): + return {k: v for k, v in obj.items() if v is not None} + return {k: v for k, v in obj.__dict__.items() if v is not None} + def store_memories( memories: Iterable[Any], graph_driver: GraphDriver, + *, + entity_registry: type[EntityRegistry] | None = None, ) -> None: - """ - Persist a sequence of Memory objects into the graph database. - - :param memories: An iterable of Memory-like objects with attributes for upsert. - :param graph_driver: An instance of GraphDriver to perform database operations. - """ - # Iterate over Memory-like objects and upsert into graph - for mem in memories: - # Use Pydantic-like dict to extract properties - try: - props = mem.dict(exclude_none=True) - except Exception: - # Fallback for non-Pydantic objects - props = mem.__dict__ - # Upsert entity node with label and name - graph_driver.upsert_entity(mem.entity_label, mem.name, props) \ No newline at end of file + """Persist a sequence of Memory objects into the graph database.""" + + registry = entity_registry or EntityRegistry + stored = 0 + with telemetry.track_duration("pipeline.store.memories.duration"): + for mem in memories: + props = _props(mem) + label = getattr(mem, "entity_label", None) + if label and registry.model_for_label(label) is None and isinstance(mem, BaseModel): + registry.register(type(mem)) + graph_driver.upsert_entity(label or "Memory", getattr(mem, "name", ""), props) + stored += 1 + telemetry.increment("pipeline.store.memories.stored", stored) + log_event("pipeline.store.memories", count=stored) + + +def store_triplets( + triplets: Iterable[Triplet], + graph_driver: GraphDriver, + *, + predicate_registry: type[PredicateRegistry] | None = None, +) -> None: + """Persist a collection of ``Triplet`` relationships.""" + + registry = predicate_registry or PredicateRegistry + stored = 0 + with telemetry.track_duration("pipeline.store.triplets.duration"): + for triplet in triplets: + registry.add(triplet.predicate) + props = _props(triplet) + graph_driver.upsert_edge( + triplet.subject, + triplet.predicate, + triplet.object, + props, + ) + stored += 1 + telemetry.increment("pipeline.store.triplets.stored", stored) + log_event("pipeline.store.triplets", count=stored) diff --git a/meshmind/retrieval/__init__.py b/meshmind/retrieval/__init__.py index e69de29..4239b7f 100644 --- a/meshmind/retrieval/__init__.py +++ b/meshmind/retrieval/__init__.py @@ -0,0 +1,39 @@ +"""Retrieval helpers exposed for external consumers.""" + +from .search import ( + search, + search_bm25, + search_exact, + search_fuzzy, + search_regex, + search_vector, +) +from .graph import ( + graph_bm25_search, + graph_exact_search, + graph_fuzzy_search, + graph_hybrid_search, + graph_regex_search, + graph_vector_search, +) +from .vector import vector_search, vector_search_from_embeddings +from .rerank import llm_rerank, apply_reranker + +__all__ = [ + "search", + "search_bm25", + "search_exact", + "search_fuzzy", + "search_regex", + "search_vector", + "graph_hybrid_search", + "graph_vector_search", + "graph_regex_search", + "graph_exact_search", + "graph_bm25_search", + "graph_fuzzy_search", + "vector_search", + "vector_search_from_embeddings", + "llm_rerank", + "apply_reranker", +] diff --git a/meshmind/retrieval/bm25.py b/meshmind/retrieval/bm25.py index 6158b03..a68c1d5 100644 --- a/meshmind/retrieval/bm25.py +++ b/meshmind/retrieval/bm25.py @@ -1,9 +1,17 @@ -""" -TF-IDF based retrieval (approximate BM25) using scikit-learn. -""" +"""TF-IDF based retrieval (approximate BM25) using scikit-learn or fallbacks.""" +from __future__ import annotations + +import math +import re +from collections import Counter from typing import List, Tuple -from sklearn.feature_extraction.text import TfidfVectorizer -from sklearn.metrics.pairwise import cosine_similarity + +try: # pragma: no cover - optional dependency + from sklearn.feature_extraction.text import TfidfVectorizer + from sklearn.metrics.pairwise import cosine_similarity +except ImportError: # pragma: no cover - exercised when sklearn is unavailable + TfidfVectorizer = None # type: ignore + cosine_similarity = None # type: ignore from meshmind.core.types import Memory @@ -21,24 +29,52 @@ def bm25_search( :param top_k: Number of top results to return. :return: List of (Memory, similarity_score) tuples. """ - # Prepare document texts docs = [mem.name for mem in memories] - # Vectorize documents and query - vectorizer = TfidfVectorizer() - tfidf_matrix = vectorizer.fit_transform(docs) - query_vec = vectorizer.transform([query]) - # Compute cosine similarity scores - scores = cosine_similarity(query_vec, tfidf_matrix)[0] - # Rank scores - ranked = sorted( - enumerate(scores), key=lambda x: x[1], reverse=True - ) - # Collect top_k non-zero scores + + if TfidfVectorizer is not None and cosine_similarity is not None: + vectorizer = TfidfVectorizer() + tfidf_matrix = vectorizer.fit_transform(docs) + query_vec = vectorizer.transform([query]) + scores = cosine_similarity(query_vec, tfidf_matrix)[0] + ranked = sorted(enumerate(scores), key=lambda x: x[1], reverse=True) + results: List[Tuple[Memory, float]] = [] + for idx, score in ranked: + if score <= 0: + break + results.append((memories[idx], float(score))) + if len(results) >= top_k: + break + return results + + tokens = [_tokenize(doc) for doc in docs] + query_tokens = Counter(_tokenize(query)) + doc_freq: Counter[str] = Counter() + for tok_set in map(set, tokens): + for token in tok_set: + doc_freq[token] += 1 + + scores: List[Tuple[int, float]] = [] + total_docs = max(len(tokens), 1) + for idx, doc_tokens in enumerate(tokens): + doc_count = Counter(doc_tokens) + doc_len = len(doc_tokens) or 1 + score = 0.0 + for token, q_tf in query_tokens.items(): + tf = doc_count.get(token, 0) / doc_len + idf = math.log((total_docs + 1) / (doc_freq.get(token, 0) + 1)) + 1.0 + score += tf * idf * q_tf + scores.append((idx, score)) + + ranked = sorted(scores, key=lambda x: x[1], reverse=True) results: List[Tuple[Memory, float]] = [] for idx, score in ranked: if score <= 0: - break + continue results.append((memories[idx], float(score))) if len(results) >= top_k: break - return results \ No newline at end of file + return results + + +def _tokenize(text: str) -> List[str]: + return re.findall(r"\w+", text.lower()) diff --git a/meshmind/retrieval/fuzzy.py b/meshmind/retrieval/fuzzy.py index bd762e0..461da73 100644 --- a/meshmind/retrieval/fuzzy.py +++ b/meshmind/retrieval/fuzzy.py @@ -1,8 +1,14 @@ -""" -Fuzzy string matching retrieval using rapidfuzz. -""" -from typing import List, Tuple -from rapidfuzz import process, fuzz +"""Fuzzy string matching retrieval with optional ``rapidfuzz`` acceleration.""" +from __future__ import annotations + +from difflib import SequenceMatcher +from typing import Callable, List, Tuple + +try: # pragma: no cover - optional dependency + from rapidfuzz import fuzz, process +except ImportError: # pragma: no cover - fallback for environments without rapidfuzz + fuzz = None # type: ignore + process = None # type: ignore from meshmind.core.types import Memory @@ -22,19 +28,31 @@ def fuzzy_search( :param score_cutoff: Minimum score (0-1) to include in results. :return: List of (Memory, normalized_score) tuples. """ - # Build choices mapping choices = [mem.name for mem in memories] - # rapidfuzz returns scores in 0-100 range - raw_results = process.extract( - query, - choices, - scorer=fuzz.WRatio, - limit=top_k, - score_cutoff=score_cutoff * 100, - ) + + if process is not None and fuzz is not None: + raw_results = process.extract( + query, + choices, + scorer=fuzz.WRatio, + limit=top_k, + score_cutoff=score_cutoff * 100, + ) + results: List[Tuple[Memory, float]] = [] + for match, score, idx in raw_results: + results.append((memories[idx], score / 100.0)) + return results + + scorer: Callable[[str, str], float] = _sequence_ratio results: List[Tuple[Memory, float]] = [] - for match, score, idx in raw_results: - # Normalize score to [0,1] - norm = score / 100.0 - results.append((memories[idx], norm)) - return results \ No newline at end of file + for idx, name in enumerate(choices): + score = scorer(query, name) + if score < score_cutoff: + continue + results.append((memories[idx], score)) + results.sort(key=lambda item: item[1], reverse=True) + return results[:top_k] + + +def _sequence_ratio(a: str, b: str) -> float: + return SequenceMatcher(None, a.lower(), b.lower()).ratio() diff --git a/meshmind/retrieval/graph.py b/meshmind/retrieval/graph.py new file mode 100644 index 0000000..20fc96d --- /dev/null +++ b/meshmind/retrieval/graph.py @@ -0,0 +1,225 @@ +"""Graph-backed retrieval helpers that fetch memories directly from a driver.""" +from __future__ import annotations + +from typing import Callable, Iterable, List, Optional, Sequence + +from meshmind.api.memory_manager import MemoryManager +from meshmind.core.types import Memory, SearchConfig +from meshmind.db.base_driver import GraphDriver +from meshmind.retrieval.search import ( + search as hybrid_search, + search_bm25, + search_exact, + search_fuzzy, + search_regex, + search_vector, +) + +Reranker = Callable[[str, Sequence[Memory], int], Sequence[Memory]] + + +def _load_memories( + driver: GraphDriver, + namespace: Optional[str] = None, + entity_labels: Optional[Iterable[str]] = None, + *, + query: Optional[str] = None, + config: Optional[SearchConfig] = None, + top_k: Optional[int] = None, + use_search: bool = True, +) -> List[Memory]: + manager = MemoryManager(driver) + labels = _ensure_sequence(entity_labels) + candidate_limit: Optional[int] = None + if use_search: + limit_hint = 0 + if config is not None: + limit_hint = max(limit_hint, config.top_k * 5) + if config.rerank_k: + limit_hint = max(limit_hint, config.rerank_k * 2) + if top_k: + limit_hint = max(limit_hint, top_k * 5) + candidate_limit = limit_hint or None + return manager.list_memories( + namespace, + labels, + limit=candidate_limit, + query=query if use_search else None, + use_search=use_search, + ) + + +def _ensure_sequence(values: Iterable[str] | None) -> List[str] | None: + if values is None: + return None + return list(values) + + +def graph_hybrid_search( + query: str, + driver: GraphDriver, + namespace: Optional[str] = None, + entity_labels: Optional[Iterable[str]] = None, + config: Optional[SearchConfig] = None, + reranker: Reranker | None = None, +) -> List[Memory]: + """Run the standard hybrid search against memories fetched from the graph.""" + + labels = _ensure_sequence(entity_labels) + memories = _load_memories( + driver, + namespace, + labels, + query=query, + config=config, + use_search=True, + ) + return hybrid_search( + query, + memories, + namespace=namespace, + entity_labels=labels, + config=config, + reranker=reranker, + ) + + +def graph_vector_search( + query: str, + driver: GraphDriver, + namespace: Optional[str] = None, + entity_labels: Optional[Iterable[str]] = None, + config: Optional[SearchConfig] = None, +) -> List[Memory]: + """Run vector search against graph-backed memories.""" + + labels = _ensure_sequence(entity_labels) + memories = _load_memories( + driver, + namespace, + labels, + query=query, + config=config, + use_search=True, + ) + return search_vector( + query, + memories, + namespace=namespace, + entity_labels=labels, + config=config, + ) + + +def graph_regex_search( + pattern: str, + driver: GraphDriver, + namespace: Optional[str] = None, + entity_labels: Optional[Iterable[str]] = None, + flags: int | None = None, + top_k: int = 10, +) -> List[Memory]: + """Execute regex search with memories loaded from the graph.""" + + labels = _ensure_sequence(entity_labels) + memories = _load_memories( + driver, + namespace, + labels, + top_k=top_k, + use_search=False, + ) + return search_regex( + pattern, + memories, + namespace=namespace, + entity_labels=labels, + flags=flags, + top_k=top_k, + ) + + +def graph_exact_search( + query: str, + driver: GraphDriver, + namespace: Optional[str] = None, + entity_labels: Optional[Iterable[str]] = None, + fields: Optional[Iterable[str]] = None, + case_sensitive: bool = False, + top_k: int = 10, +) -> List[Memory]: + """Execute exact-match search using graph-backed memories.""" + + labels = _ensure_sequence(entity_labels) + memories = _load_memories( + driver, + namespace, + labels, + query=query, + top_k=top_k, + use_search=True, + ) + target_fields = list(fields) if fields else None + return search_exact( + query, + memories, + namespace=namespace, + entity_labels=labels, + fields=target_fields, + case_sensitive=case_sensitive, + top_k=top_k, + ) + + +def graph_bm25_search( + query: str, + driver: GraphDriver, + namespace: Optional[str] = None, + entity_labels: Optional[Iterable[str]] = None, + top_k: int = 10, +) -> List[Memory]: + """Execute BM25 search using graph-backed memories.""" + + labels = _ensure_sequence(entity_labels) + memories = _load_memories( + driver, + namespace, + labels, + query=query, + top_k=top_k, + use_search=True, + ) + return search_bm25( + query, + memories, + namespace=namespace, + entity_labels=labels, + top_k=top_k, + ) + + +def graph_fuzzy_search( + query: str, + driver: GraphDriver, + namespace: Optional[str] = None, + entity_labels: Optional[Iterable[str]] = None, + top_k: int = 10, +) -> List[Memory]: + """Execute fuzzy search against graph-backed memories.""" + + labels = _ensure_sequence(entity_labels) + memories = _load_memories( + driver, + namespace, + labels, + query=query, + top_k=top_k, + use_search=True, + ) + return search_fuzzy( + query, + memories, + namespace=namespace, + entity_labels=labels, + top_k=top_k, + ) diff --git a/meshmind/retrieval/rerank.py b/meshmind/retrieval/rerank.py new file mode 100644 index 0000000..71085ec --- /dev/null +++ b/meshmind/retrieval/rerank.py @@ -0,0 +1,80 @@ +"""Helpers for reranking retrieval results.""" +from __future__ import annotations + +from typing import Callable, List, Sequence + +from meshmind.core.types import Memory + +Reranker = Callable[[str, Sequence[Memory], int], Sequence[Memory]] + + +def llm_rerank( + query: str, + memories: Sequence[Memory], + llm_client: object | None, + top_k: int, + model: str | None = None, +) -> List[Memory]: + """Rerank results using an LLM client that supports the Responses API.""" + if llm_client is None or not memories: + return list(memories)[:top_k] + + model_name = model or "gpt-4o-mini" + prompt = "\n".join( + [ + "You are a ranking assistant.", + "Given the query and numbered memory summaries, return a JSON array of memory indexes", + "sorted from best to worst match.", + f"Query: {query}", + "Memories:", + ] + ) + for idx, memory in enumerate(memories): + prompt += f"\n{idx}: {memory.name}" + + try: # pragma: no cover - network interaction mocked in tests + response = llm_client.responses.create( # type: ignore[attr-defined] + model=model_name, + input=[{"role": "user", "content": prompt}], + response_format={"type": "json_schema", "json_schema": { + "name": "rankings", + "schema": { + "type": "object", + "properties": { + "order": { + "type": "array", + "items": {"type": "integer"}, + } + }, + "required": ["order"], + }, + }}, + ) + content = response.output[0].content[0].text # type: ignore[index] + except Exception: + return list(memories)[:top_k] + + try: + import json + + data = json.loads(content) + indexes = [idx for idx in data.get("order", []) if 0 <= idx < len(memories)] + except Exception: + return list(memories)[:top_k] + + ranked = [memories[idx] for idx in indexes] + remaining = [mem for mem in memories if mem not in ranked] + ranked.extend(remaining) + return ranked[:top_k] + + +def apply_reranker( + query: str, + candidates: Sequence[Memory], + top_k: int, + reranker: Reranker | None = None, +) -> List[Memory]: + if reranker is None: + return list(candidates)[:top_k] + ranked = reranker(query, candidates, top_k) + return list(ranked)[:top_k] diff --git a/meshmind/retrieval/search.py b/meshmind/retrieval/search.py index 666d7b1..f9da468 100644 --- a/meshmind/retrieval/search.py +++ b/meshmind/retrieval/search.py @@ -1,7 +1,8 @@ -""" -Unified dispatcher for various retrieval strategies. -""" -from typing import List, Optional +"""Unified dispatcher for various retrieval strategies.""" +from __future__ import annotations + +import re +from typing import Callable, List, Optional, Sequence from meshmind.core.types import Memory, SearchConfig from meshmind.retrieval.bm25 import bm25_search @@ -12,6 +13,23 @@ filter_by_entity_labels, filter_by_metadata, ) +from meshmind.retrieval.rerank import apply_reranker +from meshmind.retrieval.vector import vector_search + +Reranker = Callable[[str, Sequence[Memory], int], Sequence[Memory]] + + +def _apply_filters( + memories: Sequence[Memory], + namespace: Optional[str], + entity_labels: Optional[List[str]], + config: Optional[SearchConfig], +) -> List[Memory]: + mems = filter_by_namespace(list(memories), namespace) + mems = filter_by_entity_labels(mems, entity_labels) + if config and config.filters: + mems = filter_by_metadata(mems, config.filters) + return mems def search( @@ -20,28 +38,29 @@ def search( namespace: Optional[str] = None, entity_labels: Optional[List[str]] = None, config: Optional[SearchConfig] = None, + reranker: Reranker | None = None, ) -> List[Memory]: - """ - Perform hybrid search over memories with optional filters. - - :param query: Query string. - :param memories: List of Memory objects. - :param namespace: Filter by namespace. - :param entity_labels: Filter by entity labels. - :param config: SearchConfig overriding defaults. - :return: Ranked list of Memory objects. - """ - # Apply filters - mems = filter_by_namespace(memories, namespace) - mems = filter_by_entity_labels(mems, entity_labels) - if config and config.filters: - mems = filter_by_metadata(mems, config.filters) - - # Use hybrid search by default + """Perform hybrid search with optional reranking.""" cfg = config or SearchConfig() + mems = _apply_filters(memories, namespace, entity_labels, cfg) ranked = hybrid_search(query, mems, cfg) - # Return only Memory objects - return [m for m, _ in ranked] + baseline = [m for m, _ in ranked] + if not baseline: + return [] + + if reranker is None: + return baseline[: cfg.top_k] + + subset = mems[: cfg.rerank_k] + reranked_subset = apply_reranker(query, subset, cfg.rerank_k, reranker) + ordered: List[Memory] = [] + for mem in reranked_subset: + if mem not in ordered: + ordered.append(mem) + for mem in baseline: + if mem not in ordered: + ordered.append(mem) + return ordered[: cfg.top_k] def search_bm25( @@ -51,8 +70,7 @@ def search_bm25( entity_labels: Optional[List[str]] = None, top_k: int = 10, ) -> List[Memory]: - mems = filter_by_namespace(memories, namespace) - mems = filter_by_entity_labels(mems, entity_labels) + mems = _apply_filters(memories, namespace, entity_labels, None) results = bm25_search(query, mems, top_k=top_k) return [m for m, _ in results] @@ -64,7 +82,74 @@ def search_fuzzy( entity_labels: Optional[List[str]] = None, top_k: int = 10, ) -> List[Memory]: - mems = filter_by_namespace(memories, namespace) - mems = filter_by_entity_labels(mems, entity_labels) + mems = _apply_filters(memories, namespace, entity_labels, None) results = fuzzy_search(query, mems, top_k=top_k) - return [m for m, _ in results] \ No newline at end of file + return [m for m, _ in results] + + +def search_vector( + query: str, + memories: List[Memory], + namespace: Optional[str] = None, + entity_labels: Optional[List[str]] = None, + config: Optional[SearchConfig] = None, +) -> List[Memory]: + cfg = config or SearchConfig() + mems = _apply_filters(memories, namespace, entity_labels, cfg) + results = vector_search(query, mems, cfg) + return [m for m, _ in results] + + +def search_regex( + pattern: str, + memories: List[Memory], + namespace: Optional[str] = None, + entity_labels: Optional[List[str]] = None, + flags: int | None = None, + top_k: int = 10, +) -> List[Memory]: + mems = _apply_filters(memories, namespace, entity_labels, None) + regex = re.compile(pattern, flags or re.IGNORECASE) + scored: List[tuple[Memory, int]] = [] + for mem in mems: + haystacks = [mem.name] + [str(value) for value in mem.metadata.values()] + matches = [len(regex.findall(h)) for h in haystacks] + score = max(matches, default=0) + if score > 0: + scored.append((mem, score)) + scored.sort(key=lambda item: item[1], reverse=True) + return [mem for mem, _ in scored[:top_k]] + + +def search_exact( + query: str, + memories: List[Memory], + namespace: Optional[str] = None, + entity_labels: Optional[List[str]] = None, + fields: Optional[List[str]] = None, + case_sensitive: bool = False, + top_k: int = 10, +) -> List[Memory]: + mems = _apply_filters(memories, namespace, entity_labels, None) + needle = query if case_sensitive else query.lower() + fields = fields or ["name"] + + def normalize(value: object) -> str: + text = "" if value is None else str(value) + return text if case_sensitive else text.lower() + + matched: List[Memory] = [] + for mem in mems: + for field in fields: + if field == "metadata": + metadata = getattr(mem, "metadata", {}) + if isinstance(metadata, dict): + if any(normalize(meta_val) == needle for meta_val in metadata.values()): + matched.append(mem) + break + continue + value = getattr(mem, field, None) + if normalize(value) == needle: + matched.append(mem) + break + return matched[:top_k] diff --git a/meshmind/retrieval/vector.py b/meshmind/retrieval/vector.py new file mode 100644 index 0000000..2ca83a6 --- /dev/null +++ b/meshmind/retrieval/vector.py @@ -0,0 +1,57 @@ +"""Vector-only retrieval helpers.""" +from __future__ import annotations + +from typing import Iterable, List, Sequence, Tuple + +from meshmind.core.embeddings import EncoderRegistry +from meshmind.core.similarity import cosine_similarity +from meshmind.core.types import Memory, SearchConfig + + +def vector_search( + query: str, + memories: Sequence[Memory], + config: SearchConfig | None = None, +) -> List[Tuple[Memory, float]]: + """Rank memories using cosine similarity against the query embedding.""" + if not memories: + return [] + + cfg = config or SearchConfig() + encoder = EncoderRegistry.get(cfg.encoder) + query_embedding = encoder.encode([query])[0] + + scored: List[Tuple[Memory, float]] = [] + for memory in memories: + embedding = getattr(memory, "embedding", None) + if embedding is None: + continue + try: + score = cosine_similarity(query_embedding, embedding) + except Exception: + score = 0.0 + scored.append((memory, float(score))) + + scored.sort(key=lambda item: item[1], reverse=True) + return scored[: cfg.top_k] + + +def vector_search_from_embeddings( + query_embedding: Sequence[float], + memories: Iterable[Memory], + top_k: int = 10, +) -> List[Tuple[Memory, float]]: + """Rank memories when the query embedding is precomputed.""" + scored: List[Tuple[Memory, float]] = [] + for memory in memories: + embedding = getattr(memory, "embedding", None) + if embedding is None: + continue + try: + score = cosine_similarity(query_embedding, embedding) + except Exception: + score = 0.0 + scored.append((memory, float(score))) + + scored.sort(key=lambda item: item[1], reverse=True) + return scored[:top_k] diff --git a/meshmind/tasks/celery_app.py b/meshmind/tasks/celery_app.py index 988f355..46fe7f9 100644 --- a/meshmind/tasks/celery_app.py +++ b/meshmind/tasks/celery_app.py @@ -36,4 +36,4 @@ def decorator(fn): app.conf.timezone = 'UTC' app.conf.enable_utc = True else: - app = _DummyCeleryApp() \ No newline at end of file + app = _DummyCeleryApp() diff --git a/meshmind/tasks/scheduled.py b/meshmind/tasks/scheduled.py index eaa4ce2..7814801 100644 --- a/meshmind/tasks/scheduled.py +++ b/meshmind/tasks/scheduled.py @@ -1,6 +1,8 @@ """ Scheduled Celery tasks for expiry, consolidation, and compression. """ +from __future__ import annotations + try: from celery.schedules import crontab _CELERY_BEAT = True @@ -9,25 +11,40 @@ _CELERY_BEAT = False def crontab(*args, **kwargs): # type: ignore return None -from meshmind.tasks.celery_app import app -from meshmind.pipeline.expire import expire_memories -from meshmind.pipeline.consolidate import consolidate_memories -from meshmind.pipeline.compress import compress_memories from meshmind.api.memory_manager import MemoryManager -from meshmind.db.memgraph_driver import MemgraphDriver from meshmind.core.config import settings +from meshmind.core.observability import log_event, telemetry +from meshmind.db.factory import create_graph_driver +from meshmind.pipeline.compress import compress_memories +from meshmind.pipeline.consolidate import ( + ConsolidationOutcome, + ConsolidationPlan, + consolidate_memories, +) +from meshmind.pipeline.expire import expire_memories +from meshmind.tasks.celery_app import app + +_MANAGER: MemoryManager | None = None -# Initialize database driver and memory manager (fallback if mgclient missing) -try: - driver = MemgraphDriver( - settings.MEMGRAPH_URI, - settings.MEMGRAPH_USERNAME, - settings.MEMGRAPH_PASSWORD, - ) - manager = MemoryManager(driver) -except Exception: - driver = None # type: ignore - manager = None # type: ignore + +def _reset_manager() -> None: + global _MANAGER + _MANAGER = None + + +def _get_manager() -> MemoryManager | None: + global _MANAGER + if _MANAGER is not None: + return _MANAGER + + try: + driver = create_graph_driver() + except Exception as exc: + log_event("task.manager.error", error=str(exc)) + return None + + _MANAGER = MemoryManager(driver) + return _MANAGER # Define periodic task schedule if Celery is available if _CELERY_BEAT and hasattr(app, 'conf'): @@ -50,30 +67,69 @@ def crontab(*args, **kwargs): # type: ignore @app.task(name='meshmind.tasks.scheduled.expire_task') def expire_task(): """Delete expired memories based on TTL.""" + manager = _get_manager() if manager is None: return [] - return expire_memories(manager) + log_event("task.expire.start") + with telemetry.track_duration("task.expire.duration"): + results = expire_memories(manager) + telemetry.increment("task.expire.runs") + log_event("task.expire.complete", removed=len(results)) + return results @app.task(name='meshmind.tasks.scheduled.consolidate_task') def consolidate_task(): """Merge duplicate memories and summarise.""" + manager = _get_manager() if manager is None: return 0 memories = manager.list_memories() - consolidated = consolidate_memories(memories) - for mem in consolidated: - manager.update_memory(mem) - return len(consolidated) + log_event("task.consolidate.start", memories=len(memories)) + merged_count = 0 + removed_total = 0 + with telemetry.track_duration("task.consolidate.duration"): + plan = consolidate_memories(memories) + if isinstance(plan, ConsolidationPlan): + skipped = dict(plan.skipped_groups) + else: # backward compatibility + skipped = {} + for outcome in plan: + _apply_consolidation(manager, outcome) + merged_count += 1 + removed_total += len(outcome.removed_ids) + telemetry.increment("task.consolidate.runs") + telemetry.gauge("task.consolidate.skipped_groups", float(len(skipped))) + log_event( + "task.consolidate.complete", + merged=merged_count, + removed=removed_total, + skipped=skipped, + ) + return {"merged": merged_count, "removed": removed_total, "skipped": skipped} @app.task(name='meshmind.tasks.scheduled.compress_task') def compress_task(): """Compress long memories to respect token limits.""" + manager = _get_manager() if manager is None: return 0 memories = manager.list_memories() - compressed = compress_memories(memories) - for mem in compressed: - manager.update_memory(mem) - return len(compressed) \ No newline at end of file + log_event("task.compress.start", memories=len(memories)) + updated = 0 + with telemetry.track_duration("task.compress.duration"): + compressed = compress_memories(memories) + for mem in compressed: + manager.update_memory(mem) + updated += 1 + telemetry.increment("task.compress.runs") + log_event("task.compress.complete", updated=updated) + return updated + + +def _apply_consolidation(manager: MemoryManager, outcome: ConsolidationOutcome) -> None: + manager.update_memory(outcome.updated) + for uid in outcome.removed_ids: + if uid: + manager.delete_memory(uid) diff --git a/meshmind/testing/__init__.py b/meshmind/testing/__init__.py new file mode 100644 index 0000000..d714220 --- /dev/null +++ b/meshmind/testing/__init__.py @@ -0,0 +1,8 @@ +"""Testing utilities and doubles for MeshMind.""" +from .fakes import FakeEmbeddingEncoder, FakeMemgraphDriver, FakeRedisBroker + +__all__ = [ + "FakeEmbeddingEncoder", + "FakeMemgraphDriver", + "FakeRedisBroker", +] diff --git a/meshmind/testing/fakes.py b/meshmind/testing/fakes.py new file mode 100644 index 0000000..e6d8972 --- /dev/null +++ b/meshmind/testing/fakes.py @@ -0,0 +1,86 @@ +"""Testing doubles for MeshMind components and external services.""" +from __future__ import annotations + +from collections import defaultdict, deque +from typing import Any, Deque, Dict, Iterable, List, Optional + +from meshmind.db.in_memory_driver import InMemoryGraphDriver + + +class FakeMemgraphDriver(InMemoryGraphDriver): + """In-memory substitute that records Cypher interactions for assertions.""" + + def __init__(self) -> None: + super().__init__() + self.cypher_calls: List[tuple[str, Dict[str, Any]]] = [] + + def find(self, cypher: str, params: Dict[str, Any]) -> List[Dict[str, Any]]: + self.cypher_calls.append((cypher, dict(params))) + return super().find(cypher, params) + + +class FakeRedisBroker: + """Lightweight Redis replacement implementing the few operations we use.""" + + def __init__(self) -> None: + self._kv: Dict[str, Any] = {} + self._lists: Dict[str, Deque[Any]] = defaultdict(deque) + self._pubsub: Dict[str, List[Any]] = defaultdict(list) + + # Key/value helpers ------------------------------------------------- + def get(self, key: str) -> Any: + return self._kv.get(key) + + def set(self, key: str, value: Any, ex: Optional[int] = None) -> None: # noqa: ARG002 - expiry unused + self._kv[key] = value + + def delete(self, *keys: str) -> int: + removed = 0 + for key in keys: + if key in self._kv: + del self._kv[key] + removed += 1 + if key in self._lists: + del self._lists[key] + removed += 1 + return removed + + # List helpers ------------------------------------------------------ + def lpush(self, key: str, *values: Any) -> int: + lst = self._lists[key] + for value in values: + lst.appendleft(value) + return len(lst) + + def rpush(self, key: str, *values: Any) -> int: + lst = self._lists[key] + for value in values: + lst.append(value) + return len(lst) + + def lrange(self, key: str, start: int, stop: int) -> List[Any]: + lst = list(self._lists[key]) + if stop == -1: + stop = len(lst) + return lst[start:stop + 1] + + # Pub/Sub helpers --------------------------------------------------- + def publish(self, channel: str, message: Any) -> int: + self._pubsub[channel].append(message) + return len(self._pubsub[channel]) + + +class FakeEmbeddingEncoder: + """Deterministic encoder that hashes text into simple float vectors.""" + + def __init__(self, scale: float = 1.0) -> None: + self.scale = scale + + def encode(self, texts: Iterable[str] | str) -> List[List[float]]: + if isinstance(texts, str): + texts = [texts] + vectors: List[List[float]] = [] + for text in texts: + total = sum(ord(ch) for ch in text) + vectors.append([self.scale * (total % 101) / 100.0]) + return vectors diff --git a/meshmind/tests/conftest.py b/meshmind/tests/conftest.py new file mode 100644 index 0000000..06fb370 --- /dev/null +++ b/meshmind/tests/conftest.py @@ -0,0 +1,57 @@ +import pytest + +from meshmind.api.memory_manager import MemoryManager +from meshmind.api.service import MemoryService +from meshmind.core.embeddings import EncoderRegistry +from meshmind.core.types import Memory +from meshmind.db.in_memory_driver import InMemoryGraphDriver +from meshmind.testing import FakeMemgraphDriver, FakeRedisBroker + + +@pytest.fixture +def memory_factory(): + def _factory(name: str, **overrides): + payload = {"namespace": "ns", "name": name, "entity_label": "Test"} + payload.update(overrides) + return Memory(**payload) + + return _factory + + +@pytest.fixture +def dummy_encoder(): + EncoderRegistry.clear() + + class _Encoder: + def encode(self, texts): + return [[1.0 if "apple" in text else 0.0] for text in texts] + + name = "dummy-encoder" + EncoderRegistry.register(name, _Encoder()) + yield name + EncoderRegistry.clear() + + +@pytest.fixture +def fake_memgraph_driver(): + return FakeMemgraphDriver() + + +@pytest.fixture +def fake_redis(): + return FakeRedisBroker() + + +@pytest.fixture +def in_memory_driver(): + return InMemoryGraphDriver() + + +@pytest.fixture +def memory_manager(in_memory_driver): + return MemoryManager(in_memory_driver) + + +@pytest.fixture +def memory_service(memory_manager): + return MemoryService(memory_manager) diff --git a/meshmind/tests/docker/full-stack.yml b/meshmind/tests/docker/full-stack.yml new file mode 100644 index 0000000..2b396b2 --- /dev/null +++ b/meshmind/tests/docker/full-stack.yml @@ -0,0 +1,49 @@ +version: "3.9" + +services: + memgraph: + extends: + file: ./memgraph.yml + service: memgraph + container_name: meshmind-int-memgraph + ports: + - "27687:7687" + - "23000:3000" + + neo4j: + extends: + file: ./neo4j.yml + service: neo4j + container_name: meshmind-int-neo4j + ports: + - "27474:7474" + - "27688:7687" + + redis: + extends: + file: ./redis.yml + service: redis + container_name: meshmind-int-redis + ports: + - "26379:6379" + + celery-worker: + build: + context: ../../.. + command: celery -A meshmind.tasks.celery_app worker -B -l info + environment: + MEMGRAPH_URI: bolt://memgraph:7687 + REDIS_URL: redis://redis:6379/0 + PYTHONUNBUFFERED: "1" + depends_on: + memgraph: + condition: service_healthy + redis: + condition: service_healthy + volumes: + - ../../..:/app + working_dir: /app + +networks: + default: + name: meshmind-integration diff --git a/meshmind/tests/docker/memgraph.yml b/meshmind/tests/docker/memgraph.yml new file mode 100644 index 0000000..d9d250e --- /dev/null +++ b/meshmind/tests/docker/memgraph.yml @@ -0,0 +1,22 @@ +version: "3.9" + +services: + memgraph: + image: memgraph/memgraph-platform:latest + container_name: meshmind-test-memgraph + restart: unless-stopped + ports: + - "17687:7687" + - "13000:3000" + environment: + MEMGRAPH_MEMORY_LIMIT: 1GB + healthcheck: + test: ["CMD", "bash", "-c", "cypher-shell --version || exit 1"] + interval: 15s + timeout: 10s + retries: 5 + volumes: + - memgraph-test-data:/var/lib/memgraph + +volumes: + memgraph-test-data: diff --git a/meshmind/tests/docker/neo4j.yml b/meshmind/tests/docker/neo4j.yml new file mode 100644 index 0000000..d38bf73 --- /dev/null +++ b/meshmind/tests/docker/neo4j.yml @@ -0,0 +1,28 @@ +version: "3.9" + +services: + neo4j: + image: neo4j:5.26 + container_name: meshmind-test-neo4j + restart: unless-stopped + ports: + - "17474:7474" + - "17688:7687" + environment: + NEO4J_AUTH: neo4j/meshminD123 + NEO4J_PLUGINS: '["apoc"]' + NEO4J_dbms_security_procedures_unrestricted: apoc.* + NEO4J_apoc_import_file_enabled: "true" + NEO4J_apoc_export_file_enabled: "true" + healthcheck: + test: ["CMD", "cypher-shell", "-u", "neo4j", "-p", "meshminD123", "RETURN 1"] + interval: 15s + timeout: 10s + retries: 5 + volumes: + - neo4j-test-data:/data + - neo4j-test-logs:/logs + +volumes: + neo4j-test-data: + neo4j-test-logs: diff --git a/meshmind/tests/docker/redis.yml b/meshmind/tests/docker/redis.yml new file mode 100644 index 0000000..2466d99 --- /dev/null +++ b/meshmind/tests/docker/redis.yml @@ -0,0 +1,20 @@ +version: "3.9" + +services: + redis: + image: redis:7-alpine + container_name: meshmind-test-redis + restart: unless-stopped + ports: + - "16379:6379" + command: redis-server --save 60 1000 --loglevel warning + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + volumes: + - redis-test-data:/data + +volumes: + redis-test-data: diff --git a/meshmind/tests/test_cli_admin.py b/meshmind/tests/test_cli_admin.py new file mode 100644 index 0000000..4af2aee --- /dev/null +++ b/meshmind/tests/test_cli_admin.py @@ -0,0 +1,68 @@ +from argparse import Namespace +from io import StringIO + +import pytest + +from meshmind.cli import admin +from meshmind.core.observability import telemetry +from meshmind.models.registry import PredicateRegistry + + +@pytest.fixture(autouse=True) +def reset_registry(): + PredicateRegistry.clear() + telemetry.reset() + yield + PredicateRegistry.clear() + telemetry.reset() + + +def test_handle_predicates_add_list_remove(): + stream = StringIO() + admin.handle_predicates(Namespace(add="LIKES", remove=None, list=False), stream=stream) + assert "LIKES" in stream.getvalue() + stream = StringIO() + admin.handle_predicates(Namespace(add=None, remove="LIKES", list=False), stream=stream) + assert "Removed" in stream.getvalue() + stream = StringIO() + admin.handle_predicates(Namespace(add=None, remove=None, list=True), stream=stream) + assert "predicates" in stream.getvalue() + + +def test_handle_maintenance_outputs_snapshot(): + telemetry.increment("events.test") + stream = StringIO() + admin.handle_maintenance(Namespace(reset=False), stream=stream) + output = stream.getvalue() + assert "events.test" in output + + +def test_handle_graph_check_with_verify(monkeypatch): + class DummyDriver: + def verify_connectivity(self): + return True + + monkeypatch.setattr(admin, "create_graph_driver", lambda **kwargs: DummyDriver()) + stream = StringIO() + status = admin.handle_graph_check(Namespace(backend="neo4j"), stream=stream) + assert status == 0 + assert "ok" in stream.getvalue() + + +def test_handle_counts_outputs_grouped(monkeypatch): + class DummyDriver: + def __init__(self): + self.closed = False + + def count_entities(self, namespace=None): # noqa: ARG002 - testing helper + return {"demo": {"Note": 3}} + + def close(self): + self.closed = True + + monkeypatch.setattr(admin, "create_graph_driver", lambda **kwargs: DummyDriver()) + stream = StringIO() + status = admin.handle_counts(Namespace(backend="memory", namespace=None), stream=stream) + + assert status == 0 + assert "\"Note\": 3" in stream.getvalue() diff --git a/meshmind/tests/test_client.py b/meshmind/tests/test_client.py new file mode 100644 index 0000000..5a9946d --- /dev/null +++ b/meshmind/tests/test_client.py @@ -0,0 +1,41 @@ +from meshmind.api.memory_manager import MemoryManager +from meshmind.client import MeshMind +from meshmind.db.in_memory_driver import InMemoryGraphDriver + + +def test_meshmind_list_memories_forwards_filters(monkeypatch): + driver = InMemoryGraphDriver() + client = MeshMind(llm_client=object(), graph_driver=driver) + + captured: dict[str, object] = {} + + def fake_list_memories(self, namespace=None, entity_labels=None, **kwargs): # noqa: ANN001 + captured["namespace"] = namespace + captured["entity_labels"] = entity_labels + captured["kwargs"] = kwargs + return [] + + monkeypatch.setattr(MemoryManager, "list_memories", fake_list_memories) + + client.list_memories( + namespace="demo", + entity_labels=["Note"], + offset=5, + limit=10, + query="alpha", + use_search=True, + ) + + assert captured["namespace"] == "demo" + assert captured["entity_labels"] == ["Note"] + assert captured["kwargs"] == {"offset": 5, "limit": 10, "query": "alpha", "use_search": True} + + +def test_meshmind_memory_counts_delegates(monkeypatch): + driver = InMemoryGraphDriver() + client = MeshMind(llm_client=object(), graph_driver=driver) + + monkeypatch.setattr(MemoryManager, "count_memories", lambda self, namespace=None: {"ns": {"Note": 1}}) + + counts = client.memory_counts(namespace="ns") + assert counts["ns"]["Note"] == 1 diff --git a/meshmind/tests/test_db_drivers.py b/meshmind/tests/test_db_drivers.py new file mode 100644 index 0000000..32fc61a --- /dev/null +++ b/meshmind/tests/test_db_drivers.py @@ -0,0 +1,184 @@ +from uuid import uuid4 + +from meshmind.api.memory_manager import MemoryManager +from meshmind.core.types import Memory, Triplet +from meshmind.db.in_memory_driver import InMemoryGraphDriver +from meshmind.db.sqlite_driver import SQLiteGraphDriver +from meshmind.testing import FakeEmbeddingEncoder, FakeMemgraphDriver, FakeRedisBroker + + +def _memory_payload(name: str) -> dict: + return { + "uuid": str(uuid4()), + "namespace": "test", + "name": name, + "entity_label": "Note", + "metadata": {"content": name}, + } + + +def test_in_memory_driver_roundtrip(): + driver = InMemoryGraphDriver() + payload = _memory_payload("alpha") + driver.upsert_entity("Note", payload["name"], payload) + + entity = driver.get_entity(payload["uuid"]) + assert entity and entity["name"] == "alpha" + + triplet = Triplet( + subject=payload["uuid"], + predicate="related_to", + object=str(uuid4()), + namespace="test", + entity_label="Relation", + ) + driver.upsert_edge(triplet.subject, triplet.predicate, triplet.object, triplet.dict()) + records = driver.list_triplets("test") + assert records and records[0]["predicate"] == "related_to" + + driver.delete(uuid4()) # deleting unknown should not error + driver.delete_triplet(triplet.subject, triplet.predicate, triplet.object) + assert not driver.list_triplets("test") + + driver.delete(payload["uuid"]) + assert driver.get_entity(payload["uuid"]) is None + + +def test_in_memory_driver_pagination_and_search(): + driver = InMemoryGraphDriver() + for idx in range(5): + payload = _memory_payload(f"item-{idx}") + payload["metadata"]["description"] = f"important note {idx}" + driver.upsert_entity("Note", payload["name"], payload) + + limited = driver.list_entities(namespace="test", offset=1, limit=2) + assert len(limited) == 2 + + searched = driver.search_entities(query="important note 3", namespace="test") + assert len(searched) == 1 + assert searched[0]["name"].endswith("3") + + +def test_sqlite_driver_roundtrip(): + driver = SQLiteGraphDriver(":memory:") + payload = _memory_payload("beta") + driver.upsert_entity("Note", payload["name"], payload) + + entity = driver.get_entity(payload["uuid"]) + assert entity and entity["name"] == "beta" + + triplet = Triplet( + subject=payload["uuid"], + predicate="mentions", + object=str(uuid4()), + namespace="test", + entity_label="Relation", + ) + driver.upsert_edge(triplet.subject, triplet.predicate, triplet.object, triplet.dict()) + records = driver.list_triplets("test") + assert records and records[0]["predicate"] == "mentions" + + driver.delete(payload["uuid"]) + assert driver.get_entity(payload["uuid"]) is None + assert not driver.list_triplets("test") + + +def test_sqlite_list_entities_filters_namespace_and_label(): + driver = SQLiteGraphDriver(":memory:") + note = _memory_payload("note") + task = _memory_payload("task") + task["entity_label"] = "Task" + driver.upsert_entity("Note", note["name"], note) + driver.upsert_entity("Task", task["name"], task) + + filtered = driver.list_entities(namespace="test", entity_labels=["Note"]) + assert len(filtered) == 1 + assert filtered[0]["entity_label"] == "Note" + + everything = driver.list_entities(namespace="test") + assert len(everything) == 2 + + +def test_sqlite_search_entities_and_pagination(): + driver = SQLiteGraphDriver(":memory:") + for idx in range(6): + payload = _memory_payload(f"row-{idx}") + payload["metadata"]["summary"] = "lorem ipsum" + driver.upsert_entity("Note", payload["name"], payload) + + chunk = driver.list_entities(namespace="test", limit=3) + assert len(chunk) == 3 + + second_page = driver.list_entities(namespace="test", offset=3, limit=3) + assert len(second_page) == 3 + + match = driver.search_entities(query="ipsum", namespace="test") + assert len(match) == 6 + + none = driver.search_entities(query="missing", namespace="test", limit=1) + assert none == [] + + +def test_fake_memgraph_driver_behaviour(): + driver = FakeMemgraphDriver() + payload = _memory_payload("gamma") + driver.upsert_entity("Note", payload["name"], payload) + records = driver.find("MATCH (m) RETURN m", {}) + assert records + assert driver.cypher_calls + + +def test_fake_memgraph_entity_label_filtering(): + driver = FakeMemgraphDriver() + note = _memory_payload("alpha") + other = _memory_payload("beta") + other["entity_label"] = "Task" + driver.upsert_entity("Note", note["name"], note) + driver.upsert_entity("Task", other["name"], other) + + filtered = driver.list_entities(namespace="test", entity_labels=["Note"]) + assert len(filtered) == 1 + assert filtered[0]["entity_label"] == "Note" + + +def test_fake_memgraph_counts(): + driver = FakeMemgraphDriver() + alpha = _memory_payload("alpha") + beta = _memory_payload("beta") + beta["entity_label"] = "Task" + driver.upsert_entity("Note", alpha["name"], alpha) + driver.upsert_entity("Task", beta["name"], beta) + + counts = driver.count_entities() + assert counts["test"]["Note"] == 1 + assert counts["test"]["Task"] == 1 + + +def test_memory_manager_count_memories(): + driver = InMemoryGraphDriver() + manager = MemoryManager(driver) + first = _memory_payload("first") + second = _memory_payload("second") + second["entity_label"] = "Task" + manager.add_memory(Memory(**first)) + manager.add_memory(Memory(**second)) + + counts = manager.count_memories() + assert counts["test"]["Note"] == 1 + assert counts["test"]["Task"] == 1 + + +def test_fake_redis_broker_roundtrip(): + broker = FakeRedisBroker() + broker.set("key", "value") + assert broker.get("key") == "value" + broker.lpush("queue", "a", "b") + assert broker.lrange("queue", 0, -1) == ["b", "a"] + broker.publish("events", {"message": "hello"}) + assert broker.delete("key") == 1 + + +def test_fake_embedding_encoder_hashing(): + encoder = FakeEmbeddingEncoder(scale=2.0) + vec = encoder.encode(["alpha"]) + assert isinstance(vec, list) and isinstance(vec[0][0], float) diff --git a/meshmind/tests/test_docs_guard.py b/meshmind/tests/test_docs_guard.py new file mode 100644 index 0000000..aadbbb1 --- /dev/null +++ b/meshmind/tests/test_docs_guard.py @@ -0,0 +1,25 @@ +from scripts.check_docs_sync import check_docs + + +def test_docs_guard_passes_when_docs_cover_changes(monkeypatch): + monkeypatch.setattr("scripts.check_docs_sync._git_diff", lambda base: ["meshmind/api/memory_manager.py", "docs/api.md"]) + assert check_docs("HEAD") == 0 + + +def test_docs_guard_fails_when_docs_missing(monkeypatch): + monkeypatch.setattr("scripts.check_docs_sync._git_diff", lambda base: ["meshmind/core/config.py"]) + result = check_docs("HEAD") + assert result == 1 + + +def test_docs_guard_requires_setup_for_compose(monkeypatch): + monkeypatch.setattr("scripts.check_docs_sync._git_diff", lambda base: ["docker-compose.yml"]) + assert check_docs("HEAD") == 1 + + +def test_docs_guard_passes_when_setup_updated(monkeypatch): + monkeypatch.setattr( + "scripts.check_docs_sync._git_diff", + lambda base: ["docker-compose.yml", "SETUP.md"], + ) + assert check_docs("HEAD") == 0 diff --git a/meshmind/tests/test_driver_factory.py b/meshmind/tests/test_driver_factory.py new file mode 100644 index 0000000..3bc6c62 --- /dev/null +++ b/meshmind/tests/test_driver_factory.py @@ -0,0 +1,30 @@ +import pytest + +from meshmind.db.factory import create_graph_driver, graph_driver_factory +from meshmind.db.in_memory_driver import InMemoryGraphDriver +from meshmind.db.sqlite_driver import SQLiteGraphDriver + + +def test_create_graph_driver_memory(): + driver = create_graph_driver(backend="memory") + assert isinstance(driver, InMemoryGraphDriver) + + +def test_create_graph_driver_sqlite(tmp_path): + path = tmp_path / "graph.db" + driver = create_graph_driver(backend="sqlite", path=str(path)) + try: + assert isinstance(driver, SQLiteGraphDriver) + finally: + driver.close() + + +def test_graph_driver_factory_callable(): + factory = graph_driver_factory(backend="memory") + driver = factory() + assert isinstance(driver, InMemoryGraphDriver) + + +def test_create_graph_driver_invalid_backend(): + with pytest.raises(ValueError): + create_graph_driver(backend="unknown") diff --git a/meshmind/tests/test_graph_retrieval.py b/meshmind/tests/test_graph_retrieval.py new file mode 100644 index 0000000..1696376 --- /dev/null +++ b/meshmind/tests/test_graph_retrieval.py @@ -0,0 +1,121 @@ +from meshmind.api.memory_manager import MemoryManager +from meshmind.core.types import Memory, SearchConfig +from meshmind.db.in_memory_driver import InMemoryGraphDriver +from meshmind.retrieval.graph import ( + graph_exact_search, + graph_hybrid_search, + graph_regex_search, + graph_vector_search, +) + + +class TrackingDriver(InMemoryGraphDriver): + def __init__(self) -> None: + super().__init__() + self.search_calls: list[dict[str, object]] = [] + self.list_calls = 0 + + def search_entities(self, *args, **kwargs): # noqa: ANN002 - passthrough to super + self.search_calls.append(dict(kwargs)) + return super().search_entities(*args, **kwargs) + + def list_entities(self, *args, **kwargs): # noqa: ANN002 - passthrough to super + self.list_calls += 1 + return super().list_entities(*args, **kwargs) + + +def test_graph_hybrid_search_uses_driver(dummy_encoder): + driver = TrackingDriver() + manager = MemoryManager(driver) + memory = Memory( + namespace="ns", + name="Apple Pie", + entity_label="Recipe", + embedding=[1.0], + ) + manager.add_memory(memory) + config = SearchConfig(encoder=dummy_encoder, top_k=1) + + results = graph_hybrid_search("apple", driver, namespace="ns", config=config) + + assert results and results[0].name == "Apple Pie" + assert driver.search_calls + assert driver.search_calls[0]["query"] == "apple" + + +def test_graph_vector_search_filters_namespace(dummy_encoder): + driver = TrackingDriver() + manager = MemoryManager(driver) + include = Memory( + namespace="keep", + name="Keep", + entity_label="Note", + embedding=[1.0], + ) + exclude = Memory( + namespace="skip", + name="Skip", + entity_label="Note", + embedding=[0.0], + ) + manager.add_memory(include) + manager.add_memory(exclude) + config = SearchConfig(encoder=dummy_encoder, top_k=5) + + results = graph_vector_search("keep", driver, namespace="keep", config=config) + + assert len(results) == 1 + assert results[0].name == include.name + + +def test_graph_exact_search_filters_entity_labels(dummy_encoder): + driver = TrackingDriver() + manager = MemoryManager(driver) + keep = Memory( + namespace="ns", + name="Retain", + entity_label="Note", + metadata={"content": "keep me"}, + ) + skip = Memory( + namespace="ns", + name="Skip", + entity_label="Task", + metadata={"content": "ignore"}, + ) + manager.add_memory(keep) + manager.add_memory(skip) + + results = graph_exact_search( + "Retain", + driver, + namespace="ns", + entity_labels=["Note"], + fields=["name"], + top_k=5, + ) + + assert len(results) == 1 + assert results[0].entity_label == "Note" + + +def test_graph_regex_search_falls_back_to_list(dummy_encoder): + driver = TrackingDriver() + manager = MemoryManager(driver) + memory = Memory( + namespace="ns", + name="Alpha", + entity_label="Note", + metadata={"content": "Value"}, + ) + manager.add_memory(memory) + + results = graph_regex_search( + "Alpha", + driver, + namespace="ns", + entity_labels=["Note"], + ) + assert results + assert driver.list_calls >= 1 + assert not driver.search_calls # regex skips driver search diff --git a/meshmind/tests/test_memgraph_driver.py b/meshmind/tests/test_memgraph_driver.py index 03f7e79..66de8cf 100644 --- a/meshmind/tests/test_memgraph_driver.py +++ b/meshmind/tests/test_memgraph_driver.py @@ -55,8 +55,23 @@ def commit(self): # Test upsert_edge does not raise edge_props = {'rel': 'value'} driver.upsert_edge('id1', 'REL', 'id2', edge_props) + # Test delete_triplet uses predicate sanitisation + driver.delete_triplet('id1', 'REL', 'id2') + assert 'DELETE r' in driver._cursor._last_query + # Test list_triplets returns parsed dicts + driver._cursor.description = [ + ('subject',), + ('predicate',), + ('object',), + ('namespace',), + ('metadata',), + ('reference_time',), + ] + driver._cursor._rows = [(('s',), ('p',), ('o',), ('ns',), ({'k': 'v'},), (None,))] + triplets = driver.list_triplets() + assert triplets and triplets[0]['subject'] == ('s',) # Test vector_search returns list # Use dummy record driver._cursor._rows = [([1.0], {'uuid': 'id1'})] out = driver.vector_search([1.0], top_k=1) - assert isinstance(out, list) \ No newline at end of file + assert isinstance(out, list) diff --git a/meshmind/tests/test_neo4j_driver.py b/meshmind/tests/test_neo4j_driver.py new file mode 100644 index 0000000..20cf3f7 --- /dev/null +++ b/meshmind/tests/test_neo4j_driver.py @@ -0,0 +1,40 @@ +import pytest + +from meshmind.db import neo4j_driver + + +def test_verify_connectivity_uses_driver(monkeypatch): + class DummySession: + def run(self, cypher, **params): + return [] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + class DummyNeo4jDriver: + def __init__(self, uri, auth=None): + self.uri = uri + self.auth = auth + self.connected = False + + def session(self): + return DummySession() + + def verify_connectivity(self): + self.connected = True + return True + + def close(self): + pass + + class DummyGraphDatabase: + @staticmethod + def driver(uri, auth=None): + return DummyNeo4jDriver(uri, auth) + + monkeypatch.setattr(neo4j_driver, "GraphDatabase", DummyGraphDatabase) + driver = neo4j_driver.Neo4jGraphDriver("bolt://localhost:7687", "neo4j", "pass") + assert driver.verify_connectivity() is True diff --git a/meshmind/tests/test_observability.py b/meshmind/tests/test_observability.py new file mode 100644 index 0000000..f191d7b --- /dev/null +++ b/meshmind/tests/test_observability.py @@ -0,0 +1,24 @@ +import pytest + +from meshmind.core.observability import log_event, telemetry +from meshmind.pipeline.store import store_memories + + +@pytest.fixture(autouse=True) +def reset_telemetry(): + telemetry.reset() + yield + telemetry.reset() + + +def test_log_event_increments_counter(): + log_event("unit.test", value=1) + snapshot = telemetry.snapshot() + assert snapshot["counters"]["events.unit.test"] == 1 + + +def test_store_memories_tracks_metrics(memory_factory, in_memory_driver): + memories = [memory_factory("one"), memory_factory("two")] + store_memories(memories, in_memory_driver) + snapshot = telemetry.snapshot() + assert snapshot["counters"]["pipeline.store.memories.stored"] == 2 diff --git a/meshmind/tests/test_pipeline_extract.py b/meshmind/tests/test_pipeline_extract.py index 6b72966..30ac3ff 100644 --- a/meshmind/tests/test_pipeline_extract.py +++ b/meshmind/tests/test_pipeline_extract.py @@ -1,11 +1,10 @@ import json + import pytest -import openai from meshmind.client import MeshMind from meshmind.core.types import Memory from meshmind.core.embeddings import EncoderRegistry -from meshmind.db.memgraph_driver import MemgraphDriver class DummyEncoder: @@ -14,39 +13,26 @@ def encode(self, texts): return [[len(text)] for text in texts] -class DummyChoice: - def __init__(self, message): - self.message = message - class DummyResponse: - def __init__(self, arg_json): - func_call = {'arguments': arg_json} - self.choices = [DummyChoice({'function_call': func_call})] + def __init__(self, payload): + self.choices = [type('Choice', (), {'message': payload})] -@pytest.fixture(autouse=True) -def patch_openai(monkeypatch): - """Patch OpenAI client to use DummyChat for responses.""" - class DummyChat: +class DummyLLMClient: + class responses: # type: ignore[assignment] @staticmethod def create(model, messages, functions, function_call): names = [m['content'] for m in messages if m['role'] == 'user'] items = [{'name': n, 'entity_label': 'Memory'} for n in names] arg_json = json.dumps({'memories': items}) - return DummyResponse(arg_json) - class DummyModelClient: - def __init__(self): - self.responses = DummyChat - monkeypatch.setattr(openai, 'OpenAI', lambda *args, **kwargs: DummyModelClient()) - return None + return DummyResponse({'function_call': {'arguments': arg_json}}) def test_extract_memories_basic(tmp_path): # Register dummy encoder + EncoderRegistry.clear() EncoderRegistry.register('text-embedding-3-small', DummyEncoder()) - mm = MeshMind() - # override default llm_client to use dummy - mm.llm_client = openai.OpenAI() + mm = MeshMind(llm_client=DummyLLMClient()) # Run extraction texts = ['alpha', 'beta'] results = mm.extract_memories( @@ -64,16 +50,17 @@ def test_extract_memories_basic(tmp_path): assert mem.embedding == [len(text)] -def test_extract_invalid_label(monkeypatch): - # Monkeypatch to return an entry with invalid label - def bad_create(*args, **kwargs): - arg_json = json.dumps({'memories': [{'name': 'x', 'entity_label': 'Bad'}]}) - return DummyResponse(arg_json) - from openai import OpenAI - llm_client = OpenAI() - monkeypatch.setattr(llm_client.responses, 'create', bad_create) +def test_extract_invalid_label(): + class BadLLMClient: + class responses: # type: ignore[assignment] + @staticmethod + def create(*args, **kwargs): + arg_json = json.dumps({'memories': [{'name': 'x', 'entity_label': 'Bad'}]}) + return DummyResponse({'function_call': {'arguments': arg_json}}) + + EncoderRegistry.clear() EncoderRegistry.register('text-embedding-3-small', DummyEncoder()) - mm = MeshMind(llm_client=llm_client) + mm = MeshMind(llm_client=BadLLMClient()) with pytest.raises(ValueError) as e: mm.extract_memories( instructions='Extract:', @@ -81,4 +68,4 @@ def bad_create(*args, **kwargs): entity_types=[Memory], content=['x'], ) - assert 'Invalid entity_label' in str(e.value) \ No newline at end of file + assert 'Invalid entity_label' in str(e.value) diff --git a/meshmind/tests/test_pipeline_maintenance.py b/meshmind/tests/test_pipeline_maintenance.py index fccb78d..83e6e7d 100644 --- a/meshmind/tests/test_pipeline_maintenance.py +++ b/meshmind/tests/test_pipeline_maintenance.py @@ -1,71 +1,62 @@ -try: - import pytest -except ImportError: - # Define fallback for pytest.approx - class pytest: - @staticmethod - def approx(x): - return x -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone + +import pytest -from meshmind.pipeline.expire import expire_memories -from meshmind.pipeline.consolidate import consolidate_memories -from meshmind.pipeline.compress import compress_memories from meshmind.core.types import Memory +from meshmind.pipeline.consolidate import consolidate_memories +from meshmind.pipeline.expire import expire_memories class DummyManager: def __init__(self, memories): self._memories = memories - self.deleted = [] + self.deleted: list[str] = [] def list_memories(self): - return self._memories + return list(self._memories) def delete_memory(self, memory_id): self.deleted.append(str(memory_id)) - # Methods to satisfy manager interface - def update_memory(self, memory): + def update_memory(self, memory): # pragma: no cover - interface placeholder pass -def make_memory(name, created_at, ttl): +def make_memory(name: str, created_at: datetime, ttl: int | None = None) -> Memory: mem = Memory(namespace="ns", name=name, entity_label="Test") mem.created_at = created_at - mem.reference_time = None mem.ttl_seconds = ttl return mem def test_expire_memories(monkeypatch): - now = datetime(2025, 1, 1, 12, 0, 0) - # Create memories: one expired, one not - m1 = make_memory("a", now - timedelta(seconds=3600), ttl=1800) - m2 = make_memory("b", now - timedelta(seconds=600), ttl=1800) - manager = DummyManager([m1, m2]) - # Monkeypatch datetime.utcnow + now = datetime(2025, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + expired = make_memory("old", now - timedelta(hours=2), ttl=1800) + active = make_memory("fresh", now - timedelta(minutes=5), ttl=1800) + manager = DummyManager([expired, active]) + class DummyDateTime: @classmethod def utcnow(cls): return now - monkeypatch.setattr('meshmind.pipeline.expire.datetime', DummyDateTime) - expired = expire_memories(manager) - assert str(m1.uuid) in expired - assert str(m2.uuid) not in expired - assert str(m1.uuid) in manager.deleted - - -def test_consolidate_memories(): - # Two memories with same name, different importance - m1 = Memory(namespace="ns", name="x", entity_label="Test") - m2 = Memory(namespace="ns", name="x", entity_label="Test") - m1.importance = 0.5 - m2.importance = 1.0 - m3 = Memory(namespace="ns", name="y", entity_label="Test") - result = consolidate_memories([m1, m2, m3]) - # Expect one for 'x' with higher importance and one for 'y' - names = {m.name for m in result} - assert names == {"x", "y"} - selected_x = next(m for m in result if m.name == "x") - assert selected_x.importance == pytest.approx(1.0) + + monkeypatch.setattr("meshmind.pipeline.expire.datetime", DummyDateTime) + + removed = expire_memories(manager) + + assert str(expired.uuid) in removed + assert str(active.uuid) not in removed + assert str(expired.uuid) in manager.deleted + + +def test_consolidate_memories_returns_outcomes(): + primary = Memory(namespace="ns", name="Alice", entity_label="Test", metadata={"content": "likes tea"}) + duplicate = Memory(namespace="ns", name="Alice", entity_label="Test", metadata={"content": "likes coffee"}) + primary.importance = 0.8 + duplicate.importance = 0.2 + + plan = consolidate_memories([primary, duplicate]) + assert len(plan.outcomes) == 1 + outcome = plan.outcomes[0] + assert outcome.updated.metadata["consolidated_summary"] + assert outcome.removed_ids diff --git a/meshmind/tests/test_pipeline_preprocess_store.py b/meshmind/tests/test_pipeline_preprocess_store.py index 2d10abd..3ebef61 100644 --- a/meshmind/tests/test_pipeline_preprocess_store.py +++ b/meshmind/tests/test_pipeline_preprocess_store.py @@ -1,26 +1,66 @@ import pytest -from meshmind.pipeline.preprocess import deduplicate, score_importance, compress -from meshmind.pipeline.store import store_memories +from datetime import datetime, timezone + +from meshmind.pipeline.consolidate import consolidate_memories +from meshmind.pipeline.preprocess import ( + compress, + deduplicate, + score_importance, +) +from meshmind.pipeline.store import store_memories, store_triplets from meshmind.api.memory_manager import MemoryManager -from meshmind.core.types import Memory +from meshmind.core.types import Memory, Triplet +from meshmind.models.registry import PredicateRegistry +from meshmind.core.observability import telemetry class DummyDriver: def __init__(self): self.entities = [] self.deleted = [] + self.edges = [] + self.deleted_edges = [] def upsert_entity(self, label, name, props): - self.entities.append((label, name, props)) + self.entities.append((label, name, dict(props))) + + def upsert_edge(self, subj, pred, obj, props): + self.edges.append((subj, pred, obj, props)) def delete(self, uuid): self.deleted.append(uuid) + self.entities = [entry for entry in self.entities if str(entry[2].get("uuid")) != str(uuid)] + + def delete_triplet(self, subj, pred, obj): + self.deleted_edges.append((subj, pred, obj)) def find(self, cypher, params): # Return empty for simplicity return [] + def list_triplets(self, namespace=None): + return [] + + def get_entity(self, uuid): + for _, _, props in self.entities: + if str(props.get("uuid")) == str(uuid): + return dict(props) + return None + + def list_entities(self, namespace=None, entity_labels=None, *, offset=0, limit=None): + results = [] + for _, _, props in self.entities: + if namespace is None or props.get("namespace") == namespace: + if entity_labels and props.get("entity_label") not in set(entity_labels): + continue + results.append(dict(props)) + if limit is None: + return results[offset:] + if limit <= 0: + return [] + return results[offset : offset + limit] + def make_memory(name: str) -> Memory: return Memory(namespace="ns", name=name, entity_label="Test") @@ -35,11 +75,31 @@ def test_deduplicate_removes_duplicates(): assert {m.name for m in result} == {"a", "b"} -def test_score_importance_sets_default(): - m = make_memory("x") - m.importance = None - scored = score_importance([m]) - assert scored[0].importance == pytest.approx(1.0) +def test_score_importance_heuristic_variation(): + recent = make_memory("Urgent server outage 500 error") + recent.metadata = {"content": "Service unavailable 500"} + recent.reference_time = datetime.now(timezone.utc) + + mundane = make_memory("Note") + mundane.metadata = {"content": "write unit tests"} + + scored = score_importance([recent, mundane]) + values = [mem.importance for mem in scored] + assert all(val is not None for val in values) + assert scored[0].importance != scored[1].importance + assert max(values) <= 5.0 + + +def test_score_importance_records_metrics(): + telemetry.reset() + m1 = make_memory("Outage 500") + m1.metadata = {"content": "Service returned 500 error"} + m1.reference_time = datetime.now(timezone.utc) + score_importance([m1]) + snapshot = telemetry.snapshot() + gauges = snapshot["gauges"] + assert "importance.mean" in gauges + assert gauges["importance.count"] >= 1.0 def test_compress_noop(): @@ -47,6 +107,21 @@ def test_compress_noop(): assert compress([m])[0] is m +def test_consolidate_memories_merges_duplicates(): + base = make_memory("Alice") + duplicate = make_memory("Alice") + duplicate.metadata = {"content": "Alice likes tea"} + base.metadata = {"content": "Alice likes coffee"} + duplicate.importance = 0.2 + base.importance = 0.9 + + plan = consolidate_memories([base, duplicate]) + assert len(plan.outcomes) == 1 + outcome = plan.outcomes[0] + assert "consolidated_summary" in outcome.updated.metadata + assert outcome.removed_ids + + def test_store_memories_calls_driver(): d = DummyDriver() m1 = make_memory("node1") @@ -57,6 +132,21 @@ def test_store_memories_calls_driver(): assert d.entities[0][1] == "node1" +def test_store_triplets_registers_predicate(): + PredicateRegistry.clear() + d = DummyDriver() + triplet = Triplet( + subject="s", + predicate="RELATES", + object="o", + namespace="ns", + entity_label="Relation", + ) + store_triplets([triplet], d) + assert d.edges and d.edges[0][1] == "RELATES" + assert "RELATES" in PredicateRegistry.all() + + def test_memory_manager_add_update_delete(): d = DummyDriver() mgr = MemoryManager(d) @@ -76,6 +166,24 @@ def test_memory_manager_add_update_delete(): # list returns empty or list lst = mgr.list_memories() assert isinstance(lst, list) + assert mgr.list_memories(entity_labels=["Other"]) == [] + + +def test_memory_manager_triplet_roundtrip(): + d = DummyDriver() + mgr = MemoryManager(d) + triplet = Triplet( + subject="s", + predicate="RELATES", + object="o", + namespace="ns", + entity_label="Relation", + ) + mgr.add_triplet(triplet) + assert d.edges + mgr.delete_triplet(triplet.subject, triplet.predicate, triplet.object) + assert d.deleted_edges + assert mgr.list_triplets() == [] def test_deduplicate_by_embedding_similarity(): # Two memories with similar embeddings should be deduplicated @@ -89,4 +197,4 @@ def test_deduplicate_by_embedding_similarity(): assert len(result_high) == 1 # With low threshold, keep both result_low = deduplicate([m1, m2], threshold=0.1) - assert len(result_low) == 2 \ No newline at end of file + assert len(result_low) == 2 diff --git a/meshmind/tests/test_retrieval.py b/meshmind/tests/test_retrieval.py index 66abd33..728243e 100644 --- a/meshmind/tests/test_retrieval.py +++ b/meshmind/tests/test_retrieval.py @@ -1,76 +1,127 @@ import pytest -from meshmind.core.types import Memory, SearchConfig -from meshmind.retrieval.bm25 import bm25_search -from meshmind.retrieval.fuzzy import fuzzy_search +from meshmind.client import MeshMind +from meshmind.core.types import SearchConfig +from meshmind.db.in_memory_driver import InMemoryGraphDriver +from meshmind.retrieval import ( + apply_reranker, + llm_rerank, + search, + search_bm25, + search_exact, + search_fuzzy, + search_regex, + search_vector, +) from meshmind.retrieval.hybrid import hybrid_search -from meshmind.retrieval.search import search, search_bm25, search_fuzzy - - -def make_memory(name: str) -> Memory: - return Memory(namespace="ns", name=name, entity_label="Test") - - -@pytest.fixture(autouse=True) -def add_embeddings(): - # Assign dummy embeddings equal to length of name - def _hook(mem: Memory): - mem.embedding = [len(mem.name)] - return mem - Memory.pre_init = _hook - yield - delattr(Memory, 'pre_init') - - -def test_bm25_search(): - docs = [make_memory("apple pie"), make_memory("banana split"), make_memory("cherry tart")] - results = bm25_search("apple", docs, top_k=2) - # Expect 'apple pie' first - assert results and results[0][0].name == "apple pie" - assert results[0][1] > 0 - - -def test_fuzzy_search(): - docs = [make_memory("apple pie"), make_memory("banana split")] - results = fuzzy_search("apple pie", docs, top_k=2) - assert results and results[0][0].name == "apple pie" - assert 0 < results[0][1] <= 1.0 - - -def test_hybrid_search(): - # Setup memories - m1 = make_memory("apple") - m2 = make_memory("banana") - m1.embedding = [1.0] - m2.embedding = [0.0] - docs = [m1, m2] - config = SearchConfig(encoder="dummy", top_k=2, hybrid_weights=(0.5, 0.5)) - # Register dummy encoder that returns [1] for 'apple' and [0] for 'banana' - class DummyEncoder: - def encode(self, texts): - return [[1.0] if "apple" in t else [0.0] for t in texts] - from meshmind.core.embeddings import EncoderRegistry - EncoderRegistry.register("dummy", DummyEncoder()) - results = hybrid_search("apple", docs, config) - # apple should have highest hybrid score - assert results[0][0].name == "apple" - - -def test_search_dispatcher(): - m1 = make_memory("apple") - m2 = make_memory("banana") - m1.embedding = [1.0] - m2.embedding = [0.0] - docs = [m1, m2] - from meshmind.core.embeddings import EncoderRegistry - class DummyEncoder: - def encode(self, texts): return [[1.0] if "apple" in t else [0.0] for t in texts] - EncoderRegistry.register("dummy", DummyEncoder()) - config = SearchConfig(encoder="dummy", top_k=1, hybrid_weights=(0.5,0.5)) - res = search("apple", docs, namespace="ns", entity_labels=["Test"], config=config) - assert len(res) == 1 and res[0].name == "apple" - # BM25 and fuzzy via dispatcher - res2 = search_bm25("banana", docs) - assert res2 and res2[0].name == "banana" - res3 = search_fuzzy("banana", docs) - assert res3 and res3[0].name == "banana" \ No newline at end of file + + +def test_bm25_search(memory_factory): + docs = [ + memory_factory("apple pie"), + memory_factory("banana split"), + memory_factory("cherry tart"), + ] + results = search_bm25("apple", docs, top_k=2) + assert results and results[0].name == "apple pie" + + +def test_fuzzy_search(memory_factory): + docs = [memory_factory("apple pie"), memory_factory("banana split")] + results = search_fuzzy("apple pie", docs, top_k=2) + assert results and results[0].name == "apple pie" + + +def test_hybrid_search(memory_factory, dummy_encoder): + m1 = memory_factory("apple", embedding=[1.0]) + m2 = memory_factory("banana", embedding=[0.0]) + config = SearchConfig(encoder=dummy_encoder, top_k=2, hybrid_weights=(0.5, 0.5)) + ranked = hybrid_search("apple", [m1, m2], config) + assert ranked[0][0].name == "apple" + + +def test_vector_search(memory_factory, dummy_encoder): + m1 = memory_factory("apple", embedding=[1.0]) + m2 = memory_factory("banana", embedding=[0.0]) + config = SearchConfig(encoder=dummy_encoder, top_k=1) + results = search_vector("apple", [m1, m2], config=config) + assert results == [m1] + + +def test_regex_search(memory_factory): + docs = [ + memory_factory("Visit Paris", metadata={"city": "Paris"}), + memory_factory("Visit Berlin", metadata={"city": "Berlin"}), + ] + results = search_regex("paris", docs, top_k=5) + assert len(results) == 1 and results[0].name == "Visit Paris" + + +def test_exact_search(memory_factory): + docs = [ + memory_factory("Python"), + memory_factory("Rust", metadata={"language": "Rust"}), + ] + results = search_exact("rust", docs, fields=["metadata"], case_sensitive=False) + assert results and results[0].name == "Rust" + + +def test_search_dispatcher_with_rerank(memory_factory, dummy_encoder): + m1 = memory_factory("apple", embedding=[1.0]) + m2 = memory_factory("banana", embedding=[0.1]) + docs = [m2, m1] + config = SearchConfig(encoder=dummy_encoder, top_k=2, rerank_k=2) + + class DummyLLM: + class responses: + @staticmethod + def create(**kwargs): + return type( + "Resp", + (), + { + "output": [ + type( + "Out", + (), + { + "content": [ + type("Text", (), {"text": '{"order": [1, 0]}'}) + ] + }, + ) + ] + }, + ) + + reranked = search( + "apple", + docs, + config=config, + reranker=lambda q, c, k: llm_rerank(q, c, DummyLLM(), k, model="dummy"), + ) + assert reranked[0].name == "apple" + + +def test_apply_reranker_default(memory_factory): + docs = [memory_factory("alpha"), memory_factory("beta")] + ranked = apply_reranker("alpha", docs, top_k=1) + assert ranked == [docs[0]] + + +def test_llm_rerank_failure(memory_factory): + docs = [memory_factory("alpha"), memory_factory("beta")] + result = llm_rerank("alpha", docs, llm_client=None, top_k=2) + assert len(result) == 2 + + +def test_client_search_uses_graph_when_memories_none(dummy_encoder, memory_factory): + driver = InMemoryGraphDriver() + client = MeshMind(llm_client=object(), embedding_model=dummy_encoder, graph_driver=driver) + memory = memory_factory("apple", embedding=[1.0]) + client.create_memory(memory) + config = SearchConfig(encoder=dummy_encoder, top_k=1) + + results = client.search("apple", memories=None, namespace="ns", config=config) + + assert results and results[0].name == "apple" diff --git a/meshmind/tests/test_service_interfaces.py b/meshmind/tests/test_service_interfaces.py new file mode 100644 index 0000000..2301d2a --- /dev/null +++ b/meshmind/tests/test_service_interfaces.py @@ -0,0 +1,150 @@ +from uuid import UUID + +from meshmind.api.grpc import ( + GrpcServiceStub, + IngestMemoriesRequest, + IngestTripletsRequest, + MemoryCountsRequest, + SearchRequest, +) +from meshmind.api.rest import RestAPIStub +from meshmind.api.service import MemoryPayload, SearchPayload, TripletPayload +from meshmind.core.types import Memory + + +def _memory(name: str) -> MemoryPayload: + return MemoryPayload(namespace="test", name=name, entity_label="Note") + + +def test_memory_service_ingest_and_search(memory_service, dummy_encoder): + payloads = [_memory("apple"), _memory("banana")] + uuids = memory_service.ingest_memories(payloads) + assert len(uuids) == 2 + + captured: dict[str, object] = {} + + sample = Memory( + uuid=UUID(uuids[0]), + namespace="test", + name="apple", + entity_label="Note", + content="apple", + ) + + def fake_list(namespace=None, entity_labels=None, **kwargs): # noqa: ANN001 + captured["namespace"] = namespace + captured["entity_labels"] = entity_labels + captured["kwargs"] = kwargs + return [sample] + + original = memory_service.manager.list_memories + memory_service.manager.list_memories = fake_list # type: ignore[assignment] + + request = SearchPayload(query="apple", namespace="test", encoder=dummy_encoder, top_k=1) + results = memory_service.search(request) + memory_service.manager.list_memories = original # restore + + assert results and isinstance(results[0], Memory) + assert captured["namespace"] == "test" + assert captured["entity_labels"] is None or captured["entity_labels"] == [] + assert captured["kwargs"]["query"] == "apple" + assert captured["kwargs"]["use_search"] is True + assert captured["kwargs"]["limit"] is not None + + +def test_rest_stub_routes(memory_service, dummy_encoder): + app = RestAPIStub(memory_service) + response = app.dispatch( + "POST", + "/memories", + {"memories": [_memory("alpha").model_dump()]}, + ) + assert "uuids" in response + + search_response = app.dispatch( + "POST", + "/search", + {"query": "alpha", "namespace": "test", "encoder": dummy_encoder, "top_k": 1}, + ) + assert search_response["results"] + + filtered = app.dispatch( + "GET", + "/memories", + {"namespace": "test", "entity_labels": ["Note"]}, + ) + assert filtered["memories"] + + filtered_none = app.dispatch( + "GET", + "/memories", + {"namespace": "test", "entity_labels": ["Task"]}, + ) + assert filtered_none["memories"] == [] + + counts = app.dispatch("GET", "/memories/counts", {"namespace": "test"}) + assert counts["counts"]["test"]["Note"] >= 1 + + +def test_memory_service_list_memories_forwards_kwargs(memory_service): + captured: dict[str, object] = {} + + def fake_list(namespace=None, entity_labels=None, **kwargs): # noqa: ANN001 + captured["namespace"] = namespace + captured["entity_labels"] = entity_labels + captured["kwargs"] = kwargs + return [] + + original = memory_service.manager.list_memories + memory_service.manager.list_memories = fake_list # type: ignore[assignment] + + memory_service.list_memories( + namespace="demo", + entity_labels=["Note"], + offset=5, + limit=10, + query="python", + use_search=False, + ) + + memory_service.manager.list_memories = original # restore + + assert captured["namespace"] == "demo" + assert captured["entity_labels"] == ["Note"] + assert captured["kwargs"] == {"offset": 5, "limit": 10, "query": "python", "use_search": False} + + +def test_grpc_stub(memory_service, dummy_encoder): + grpc = GrpcServiceStub(memory_service) + request = IngestMemoriesRequest(memories=[_memory("cherry").model_dump()]) + resp = grpc.IngestMemories(request) + assert resp.uuids + + search_resp = grpc.Search( + SearchRequest( + query="cherry", + namespace="test", + encoder=dummy_encoder, + top_k=1, + entity_labels=["Note"], + ) + ) + assert search_resp.results + + counts_resp = grpc.MemoryCounts(MemoryCountsRequest(namespace="test")) + assert counts_resp.counts["test"]["Note"] >= 1 + + triplet_resp = grpc.IngestTriplets( + IngestTripletsRequest( + triplets=[ + TripletPayload( + subject=resp.uuids[0], + predicate="linked_to", + object=resp.uuids[0], + namespace="test", + entity_label="Relation", + ).model_dump() + ] + ) + ) + assert triplet_resp.stored == 1 diff --git a/meshmind/tests/test_tasks_scheduled.py b/meshmind/tests/test_tasks_scheduled.py new file mode 100644 index 0000000..16bb7a1 --- /dev/null +++ b/meshmind/tests/test_tasks_scheduled.py @@ -0,0 +1,44 @@ +import pytest +import pytest + +from meshmind.core.types import Memory +from meshmind.tasks import scheduled + + +class StubManager: + def __init__(self, memories): + self._memories = memories + self.updated = [] + self.deleted = [] + + def list_memories(self): + return list(self._memories) + + def update_memory(self, memory): + self.updated.append(memory) + + def delete_memory(self, memory_id): + self.deleted.append(str(memory_id)) + + +@pytest.fixture(autouse=True) +def reset_manager(): + scheduled._reset_manager() + yield + scheduled._reset_manager() + + +def test_consolidate_task_persists_updates(monkeypatch): + mem_a = Memory(namespace="ns", name="Alice", entity_label="Person", metadata={"content": "Alice likes tea"}) + mem_b = Memory(namespace="ns", name="Alice", entity_label="Person", metadata={"content": "Alice likes coffee"}) + mem_b.importance = 0.9 + + manager = StubManager([mem_a, mem_b]) + monkeypatch.setattr(scheduled, "_MANAGER", manager, raising=False) + + stats = scheduled.consolidate_task() + + assert stats["merged"] == 1 + assert stats["removed"] >= 1 + assert manager.updated + assert manager.deleted diff --git a/pyproject.toml b/pyproject.toml index 062f6d7..aa1d168 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "meshmind" version = "0.1.0" description = "AI Agent Memory management system using LLMs and graph databases" readme = "README.md" -requires-python = ">=3.13" +requires-python = ">=3.11,<3.13" dependencies = [ "openai>=1.78.0", "pydantic>=2.11.4", @@ -17,8 +17,29 @@ dependencies = [ "sentence-transformers>=4.1.0", "typing-extensions>=4.13.2", "tiktoken>=0.9.0", - "pymgclient>=1.4.0", + "fastapi>=0.115.6", + "uvicorn[standard]>=0.34.0", + "neo4j>=5.26.0", + "mgclient>=1.4.0", + "redis>=5.0.0", ] [project.scripts] meshmind = "meshmind.cli.__main__:main" + +[project.optional-dependencies] +dev = [ + "ruff>=0.7.3", + "pyright>=1.1.386", + "typeguard>=4.3.0", + "toml-sort>=0.23.1", + "yamllint>=1.35.1", +] +docs = [ + "mkdocs>=1.6.1", + "mkdocs-material>=9.5.39", +] +testing = [ + "pytest-cov>=5.0.0", + "httpx>=0.28.1", +] diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/check_docs_sync.py b/scripts/check_docs_sync.py new file mode 100755 index 0000000..91ff6e4 --- /dev/null +++ b/scripts/check_docs_sync.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Ensure documentation updates accompany relevant code changes.""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path +from typing import Dict, Iterable, List, Set + +PROJECT_ROOT = Path(__file__).resolve().parent.parent + +DOC_MAP: Dict[str, List[str]] = { + "meshmind/api": ["docs/api.md", "docs/operations.md"], + "meshmind/retrieval": ["docs/retrieval.md"], + "meshmind/db": ["docs/persistence.md"], + "meshmind/pipeline": ["docs/pipelines.md"], + "meshmind/core": ["docs/architecture.md", "docs/development.md"], + "meshmind/cli": ["docs/operations.md"], + "meshmind/tasks": ["docs/operations.md", "docs/telemetry.md"], + "meshmind/tests/docker": ["SETUP.md", "docs/operations.md"], + "docker-compose": ["SETUP.md", "docs/operations.md", "ENVIRONMENT_NEEDS.md"], + "Dockerfile": ["SETUP.md", "ENVIRONMENT_NEEDS.md"], +} + +DOC_PREFIXES = ("docs/",) +ALWAYS_DOC_FILES = { + "README.md", + "PROJECT.md", + "SOT.md", + "PLAN.md", + "RECOMMENDATIONS.md", + "RESUME_NOTES.md", + "SETUP.md", + "ENVIRONMENT_NEEDS.md", + "NEEDED_FOR_TESTING.md", +} + + +def _git_diff(base: str) -> List[str]: + cmd = ["git", "diff", "--name-only", f"{base}..HEAD"] + result = subprocess.run(cmd, capture_output=True, text=True, check=False) + if result.returncode != 0: + raise RuntimeError(result.stderr.strip() or "git diff failed") + return [line.strip() for line in result.stdout.splitlines() if line.strip()] + + +def _docs_touched(files: Iterable[str]) -> Set[str]: + docs: Set[str] = set() + for path in files: + if path.startswith(DOC_PREFIXES) or path in ALWAYS_DOC_FILES: + docs.add(path) + return docs + + +def check_docs(base: str) -> int: + changed = _git_diff(base) + if not changed: + return 0 + + docs_changed = _docs_touched(changed) + missing: Dict[str, Dict[str, Iterable[str]]] = {} + + for prefix, required_docs in DOC_MAP.items(): + touched = [path for path in changed if path.startswith(prefix)] + if not touched: + continue + if any(doc in docs_changed for doc in required_docs): + continue + missing[prefix] = {"files": touched, "docs": required_docs} + + if missing: + print("Documentation updates required for the following areas:", file=sys.stderr) + for prefix, info in missing.items(): + print(f"- {prefix}", file=sys.stderr) + print(f" touched: {', '.join(info['files'])}", file=sys.stderr) + print(f" expected docs: {', '.join(info['docs'])}", file=sys.stderr) + return 1 + + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--base", + default="origin/main", + help="Git reference to diff against (default: origin/main)", + ) + args = parser.parse_args() + + try: + return check_docs(args.base) + except RuntimeError as exc: # pragma: no cover - git errors depend on CI setup + print(str(exc), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main())