diff --git a/apps/server/src/modules/agent/agenetes/conversation-stores.ts b/apps/server/src/modules/agent/agenetes/conversation-stores.ts new file mode 100644 index 000000000..2c235ed4d --- /dev/null +++ b/apps/server/src/modules/agent/agenetes/conversation-stores.ts @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * Which Agenetes conversation stores this deployment runs on. + * + * Agenetes takes its three storage ports at mount, once, while the storage + * profile is only known at runtime and the active Workspace can change under + * a running process. So the mounted stores are dispatchers: each call picks + * the implementation that suits the namespace it was handed. + * + * The choice is made per namespace rather than per process because that is + * where the answer actually lives. A namespace carries a `storage.root` when + * the Space it belongs to is a directory, and does not when it is rows — the + * same fact the Space facade reports, arriving here through Agenetes's own + * vocabulary. + * + * The in-memory fall-through is not a backend choice. It is what an *unnamed* + * namespace has always got: a conversation with no Space to belong to, which + * Agenetes explicitly treats as non-persistent. + */ + +import { + FileEventLogStore, + FileThreadStore, + FileTurnStore, + InMemoryEventLogStore, + InMemoryThreadStore, + InMemoryTurnStore, +} from '@agenetes/agenetes'; + +import { + conversationTables, + SqliteEventLogStore, + SqliteThreadStore, + SqliteTurnStore, +} from './sqlite-stores.js'; + +import type { + EventLogEntry, + EventLogRecord, + EventLogStore, + PersistedTurn, + ThreadRecord, + ThreadStore, + TurnStartLogEntry, + TurnStore, +} from '@agenetes/agenetes'; +import type { AgentSubmission, Namespace } from '@agenetes/protocol'; + +interface Backing { + readonly threads: ThreadStore; + readonly events: EventLogStore; + readonly turns: TurnStore; +} + +const file: Backing = { + threads: new FileThreadStore(), + events: new FileEventLogStore(), + turns: new FileTurnStore(), +}; + +const sqlite: Backing = { + threads: new SqliteThreadStore(), + events: new SqliteEventLogStore(), + turns: new SqliteTurnStore(), +}; + +/** + * Shared, so an unnamed namespace keeps one conversation for the life of the + * process instead of a fresh empty one per port. + */ +const memory: Backing = { + threads: new InMemoryThreadStore(), + events: new InMemoryEventLogStore(), + turns: new InMemoryTurnStore(), +}; + +/** The stores that own this namespace's durable conversation state. */ +function backingFor(namespace: Namespace): Backing { + // A directory to write into settles it: that is the Disk profile, and the + // file stores are what wrote whatever is already there. + if (namespace.storage?.root) return file; + if (namespace.name && conversationTables(namespace) !== null) return sqlite; + return memory; +} + +export const conversationThreadStore: ThreadStore = { + upsert: (namespace, threadId, record: ThreadRecord) => + backingFor(namespace).threads.upsert(namespace, threadId, record), + get: (namespace, threadId) => + backingFor(namespace).threads.get(namespace, threadId), + list: (namespace) => backingFor(namespace).threads.list(namespace), + delete: (namespace, threadId) => + backingFor(namespace).threads.delete(namespace, threadId), +}; + +export const conversationEventLogStore: EventLogStore = { + appendTurnStart: ( + namespace, + threadId, + request: AgentSubmission | null, + ): TurnStartLogEntry => + backingFor(namespace).events.appendTurnStart(namespace, threadId, request), + append: (namespace, threadId, event): EventLogEntry => + backingFor(namespace).events.append(namespace, threadId, event), + read: (namespace, threadId, sinceSeq) => + backingFor(namespace).events.read(namespace, threadId, sinceSeq), + readRecords: (namespace, threadId, sinceSeq) => + backingFor(namespace).events.readRecords(namespace, threadId, sinceSeq), + maxSeq: (namespace, threadId) => + backingFor(namespace).events.maxSeq(namespace, threadId), + replace: (namespace, threadId, records: readonly EventLogRecord[]) => + backingFor(namespace).events.replace(namespace, threadId, records), + delete: (namespace, threadId) => + backingFor(namespace).events.delete(namespace, threadId), +}; + +export const conversationTurnStore: TurnStore = { + append: (namespace, threadId, persisted: PersistedTurn) => + backingFor(namespace).turns.append(namespace, threadId, persisted), + list: (namespace, threadId) => + backingFor(namespace).turns.list(namespace, threadId), + count: (namespace, threadId) => + backingFor(namespace).turns.count(namespace, threadId), + fence: (namespace, threadId) => + backingFor(namespace).turns.fence(namespace, threadId), + replace: (namespace, threadId, persisted: readonly PersistedTurn[]) => + backingFor(namespace).turns.replace(namespace, threadId, persisted), + delete: (namespace, threadId) => + backingFor(namespace).turns.delete(namespace, threadId), +}; diff --git a/apps/server/src/modules/agent/agenetes/drivers.ts b/apps/server/src/modules/agent/agenetes/drivers.ts index e514dffcc..6292ae0fa 100644 --- a/apps/server/src/modules/agent/agenetes/drivers.ts +++ b/apps/server/src/modules/agent/agenetes/drivers.ts @@ -7,15 +7,15 @@ import { type AcpCreateSpec, type AcpTurnCtx, } from '@agenetes/acp-driver'; -import { - FileEventLogStore, - FileThreadStore, - FileTurnStore, - mountAgenetes, -} from '@agenetes/agenetes'; +import { mountAgenetes } from '@agenetes/agenetes'; import { getAgentTeamRegistry } from '@agenetes/agentlet-host'; import { piDriverFactory, type PiTurnCtx } from '@agenetes/pi-driver'; +import { + conversationEventLogStore, + conversationThreadStore, + conversationTurnStore, +} from './conversation-stores.js'; import { type AgentHandle } from './handle.js'; import { HISTORY_LOAD_SANITY_LIMIT } from './history-replay.js'; import { huabuPiDriverPorts } from './pi-driver.js'; @@ -62,9 +62,11 @@ export const agenetes: Agenetes = mountAgenetes({ [INTERNAL_DRIVER_KIND]: piDriverFactory({ ports: huabuPiDriverPorts }), [EXTERNAL_DRIVER_KIND]: externalDriver, }, - threadStore: new FileThreadStore(), - eventLogStore: new FileEventLogStore(), - turnStore: new FileTurnStore(), + // Dispatchers, not one backing: which store owns a conversation depends on + // where its Space lives, and that is a runtime fact (`conversation-stores`). + threadStore: conversationThreadStore, + eventLogStore: conversationEventLogStore, + turnStore: conversationTurnStore, // Corruption guard, not a context budget: replay restores whatever the // live handle would still be holding, and trimming that is the // conversation's problem, not recovery's. diff --git a/apps/server/src/modules/agent/agenetes/sqlite-stores.test.ts b/apps/server/src/modules/agent/agenetes/sqlite-stores.test.ts new file mode 100644 index 000000000..f3999e8ae --- /dev/null +++ b/apps/server/src/modules/agent/agenetes/sqlite-stores.test.ts @@ -0,0 +1,218 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * The Agenetes conversation stores against a real SQLite profile. + * + * The claim under test is the one a user would notice: a conversation held in + * a Space that has no directory survives a restart, and goes away with its + * Space. Everything is driven through the mounted profile rather than a stub, + * so a broken extension substrate or a missing cascade fails here. + */ + +import { afterEach, describe, expect, it } from 'vitest'; + +import { + conversationEventLogStore, + conversationThreadStore, + conversationTurnStore, +} from './conversation-stores.js'; +import { deleteSpace } from '../../storage/index.js'; +import { + mountTestWorkspace, + type MountedTestStorage, +} from '../../storage/testing.js'; +import { canvasAcpNamespace } from '../../workspace/paths.js'; + +import type { StorageProfile } from '../../storage/profile.js'; +import type { ThreadRecord } from '@agenetes/agenetes'; +import type { AgentStateSnapshot, WorkloadSpec } from '@agenetes/protocol'; + +const SQLITE: StorageProfile = { + structured: { kind: 'sqlite' }, + blobs: { kind: 'disk' }, +}; + +const CANVAS_ID = 'canvas-conversation'; +const THREAD_ID = 'thread-1'; + +let mounted: MountedTestStorage | null = null; + +afterEach(async () => { + await mounted?.close(); + mounted = null; +}); + +/** Open the profile and create the Space the conversation belongs to. */ +async function openWithSpace(): Promise { + const opened = await mountTestWorkspace(SQLITE, 'huabu-agenetes-sqlite-'); + mounted = opened; + const created = await opened.storage.structured + .spaces() + .create({ canvasId: CANVAS_ID, title: 'Conversation Space' }); + if (!created.ok) throw new Error('Expected to create the Space'); + return opened; +} + +function threadRecord(threadId = THREAD_ID): ThreadRecord { + return { + driverSchemaVersion: 1, + spec: { + kind: 'internal', + threadId, + namespace: { name: CANVAS_ID }, + } as unknown as WorkloadSpec, + state: { status: 'idle' } as unknown as AgentStateSnapshot, + }; +} + +describe('Agenetes conversation stores on SQLite', () => { + it('keeps a Space with no directory out of the file stores', async () => { + await openWithSpace(); + const namespace = canvasAcpNamespace(CANVAS_ID); + + // The absence of `storage.root` is the whole signal: it is what tells the + // dispatcher this Space is not a folder. + expect(namespace.storage).toBeUndefined(); + expect(namespace.name).toBe(CANVAS_ID); + }); + + it('round-trips threads, events, and folded turns', async () => { + await openWithSpace(); + const namespace = canvasAcpNamespace(CANVAS_ID); + + conversationThreadStore.upsert(namespace, THREAD_ID, threadRecord()); + expect(conversationThreadStore.get(namespace, THREAD_ID)).toEqual( + threadRecord(), + ); + expect(conversationThreadStore.list(namespace)).toHaveLength(1); + + const start = conversationEventLogStore.appendTurnStart( + namespace, + THREAD_ID, + null, + ); + expect(start).toMatchObject({ seq: 1, kind: 'turn_start', request: null }); + const appended = conversationEventLogStore.append(namespace, THREAD_ID, { + type: 'text', + text: 'hello', + } as never); + expect(appended.seq).toBe(2); + expect(conversationEventLogStore.maxSeq(namespace, THREAD_ID)).toBe(2); + + // `read` is the streamed frames only; `readRecords` includes the internal + // turn boundary. + expect(conversationEventLogStore.read(namespace, THREAD_ID)).toHaveLength( + 1, + ); + expect( + conversationEventLogStore.readRecords(namespace, THREAD_ID), + ).toHaveLength(2); + expect( + conversationEventLogStore.read(namespace, THREAD_ID, 2), + ).toHaveLength(0); + + conversationTurnStore.append(namespace, THREAD_ID, { + turn: { id: 'turn-1' } as never, + seqStart: 1, + seqEnd: 2, + }); + expect(conversationTurnStore.count(namespace, THREAD_ID)).toBe(1); + expect(conversationTurnStore.fence(namespace, THREAD_ID)).toBe(2); + expect(conversationTurnStore.list(namespace, THREAD_ID)).toEqual([ + { turn: { id: 'turn-1' }, seqStart: 1, seqEnd: 2 }, + ]); + }); + + it('isolates one Space from another', async () => { + const opened = await openWithSpace(); + const other = 'canvas-conversation-other'; + const created = await opened.storage.structured + .spaces() + .create({ canvasId: other, title: 'Other Space' }); + if (!created.ok) throw new Error('Expected to create the second Space'); + + conversationThreadStore.upsert( + canvasAcpNamespace(CANVAS_ID), + THREAD_ID, + threadRecord(), + ); + + expect( + conversationThreadStore.get(canvasAcpNamespace(other), THREAD_ID), + ).toBeUndefined(); + expect(conversationThreadStore.list(canvasAcpNamespace(other))).toEqual([]); + }); + + it('destroys a conversation with the Space that held it', async () => { + await openWithSpace(); + const namespace = canvasAcpNamespace(CANVAS_ID); + conversationThreadStore.upsert(namespace, THREAD_ID, threadRecord()); + conversationEventLogStore.append(namespace, THREAD_ID, { + type: 'text', + text: 'hello', + } as never); + conversationTurnStore.append(namespace, THREAD_ID, { + turn: { id: 'turn-1' } as never, + seqStart: 1, + seqEnd: 1, + }); + + await expect(deleteSpace(CANVAS_ID)).resolves.toEqual({ + ok: true, + reason: 'deleted', + }); + + // The Space is gone, so there is no substrate to answer from — which is + // the port's rule, and is also what the foreign-key cascade leaves behind. + expect(conversationThreadStore.get(namespace, THREAD_ID)).toBeUndefined(); + expect(conversationEventLogStore.maxSeq(namespace, THREAD_ID)).toBe(0); + expect(conversationTurnStore.count(namespace, THREAD_ID)).toBe(0); + }); + + it('survives a restart', async () => { + const opened = await openWithSpace(); + const namespace = canvasAcpNamespace(CANVAS_ID); + conversationThreadStore.upsert(namespace, THREAD_ID, threadRecord()); + conversationEventLogStore.appendTurnStart(namespace, THREAD_ID, null); + conversationEventLogStore.append(namespace, THREAD_ID, { + type: 'text', + text: 'hello', + } as never); + conversationTurnStore.append(namespace, THREAD_ID, { + turn: { id: 'turn-1' } as never, + seqStart: 1, + seqEnd: 2, + }); + + await opened.reopen(); + + // Same namespace, new connection: this is the whole reason these stores + // exist rather than the in-memory defaults. + const after = canvasAcpNamespace(CANVAS_ID); + expect(conversationThreadStore.get(after, THREAD_ID)).toEqual( + threadRecord(), + ); + expect(conversationEventLogStore.maxSeq(after, THREAD_ID)).toBe(2); + expect( + conversationEventLogStore.readRecords(after, THREAD_ID), + ).toHaveLength(2); + expect(conversationTurnStore.fence(after, THREAD_ID)).toBe(2); + }); + + it('reports an unnamed namespace as having no durable place', async () => { + await openWithSpace(); + const anonymous = canvasAcpNamespace(''); + + // Agenetes's own rule: a namespace with no name is non-persistent. It must + // not fall through to some other Space's tables. + expect(conversationThreadStore.list(anonymous)).toEqual([]); + conversationThreadStore.upsert(anonymous, THREAD_ID, threadRecord()); + expect(conversationThreadStore.get(anonymous, THREAD_ID)).toEqual( + threadRecord(), + ); + expect( + conversationThreadStore.get(canvasAcpNamespace(CANVAS_ID), THREAD_ID), + ).toBeUndefined(); + }); +}); diff --git a/apps/server/src/modules/agent/agenetes/sqlite-stores.ts b/apps/server/src/modules/agent/agenetes/sqlite-stores.ts new file mode 100644 index 000000000..95d58cf16 --- /dev/null +++ b/apps/server/src/modules/agent/agenetes/sqlite-stores.ts @@ -0,0 +1,427 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * The Agenetes conversation stores, for a Space that lives in SQLite. + * + * Agenetes ships three narrow storage ports — the durable thread table, the + * Tier-1 event log, and the Tier-2 folded turn log — plus an in-memory and a + * file implementation of each. A host picks. On Disk we pick the file ones and + * they write under the Space's `.history/`. Where a Space is rows there is no + * such directory, and the honest choice is not "in memory": a conversation + * that vanishes on restart is a worse answer than the one this file gives. + * + * These are the storage proposal's §6.4.4 arrangement in practice. The port + * hands over a *place* — a connection plus a Space-owned parent row — and the + * owner brings its own tables and its own queries. Every table below hangs off + * `space_extensions` with `ON DELETE CASCADE`, so deleting a Space takes its + * conversations with it without storage knowing what a conversation is, and + * without this module knowing where a Space is stored. + * + * The ports are synchronous, which is why they reach for `sqliteTree` rather + * than the async `extension()`: `node:sqlite` is synchronous all the way down, + * so nothing is lost by saying so. + */ + +import { space } from '../../storage/index.js'; + +import type { SqliteSpaceSubstrate } from '../../storage/index.js'; +import type { + EventLogEntry, + EventLogRecord, + EventLogStore, + PersistedTurn, + ThreadRecord, + ThreadStore, + TurnStartLogEntry, + TurnStore, +} from '@agenetes/agenetes'; +import type { AgentSubmission, Namespace } from '@agenetes/protocol'; +import type { DatabaseSync } from 'node:sqlite'; + +/** + * One namespace for all three stores. + * + * They are one owner — the conversation — split into three ports for reasons + * that belong to Agenetes, not to storage. Giving each its own namespace would + * buy three parent rows and no isolation that matters. + */ +const CONVERSATION_NAMESPACE = 'agenetes.conversations'; + +const SCHEMA = ` + CREATE TABLE IF NOT EXISTS agenetes_threads ( + extension_id INTEGER NOT NULL, + thread_id TEXT NOT NULL, + record_json TEXT NOT NULL CHECK (json_valid(record_json)), + PRIMARY KEY (extension_id, thread_id), + FOREIGN KEY (extension_id) REFERENCES space_extensions(extension_id) + ON DELETE CASCADE + ) STRICT; + + CREATE TABLE IF NOT EXISTS agenetes_events ( + extension_id INTEGER NOT NULL, + thread_id TEXT NOT NULL, + seq INTEGER NOT NULL, + record_json TEXT NOT NULL CHECK (json_valid(record_json)), + PRIMARY KEY (extension_id, thread_id, seq), + FOREIGN KEY (extension_id) REFERENCES space_extensions(extension_id) + ON DELETE CASCADE + ) STRICT; + + CREATE TABLE IF NOT EXISTS agenetes_turns ( + extension_id INTEGER NOT NULL, + thread_id TEXT NOT NULL, + ordinal INTEGER NOT NULL, + seq_start INTEGER NOT NULL, + seq_end INTEGER NOT NULL, + turn_json TEXT NOT NULL CHECK (json_valid(turn_json)), + PRIMARY KEY (extension_id, thread_id, ordinal), + FOREIGN KEY (extension_id) REFERENCES space_extensions(extension_id) + ON DELETE CASCADE + ) STRICT; +`; + +/** Namespaces whose tables have been created on this connection. */ +const prepared = new WeakSet(); + +/** + * The place this Space's conversations live, or `null` when there is none. + * + * `null` covers three ordinary situations that all mean the same thing to a + * caller: the profile is not SQLite, the namespace has no Space (an unnamed + * Agenetes namespace), or the Space has been deleted. + */ +export function conversationTables( + namespace: Namespace, +): SqliteSpaceSubstrate | null { + if (!namespace.name) return null; + const tree = space(namespace.name).sqliteTree; + if (!tree) return null; + const substrate = tree.extension(CONVERSATION_NAMESPACE); + if (!substrate) return null; + if (!prepared.has(substrate.database)) { + substrate.database.exec(SCHEMA); + prepared.add(substrate.database); + } + return substrate; +} + +function encode(value: unknown, what: string): string { + const encoded = JSON.stringify(value); + if (encoded === undefined) { + throw new TypeError(`${what} is not representable as JSON`); + } + return encoded; +} + +function decode(value: unknown, what: string): T { + if (typeof value !== 'string') { + throw new SyntaxError(`${what} is not stored as JSON text`); + } + return JSON.parse(value) as T; +} + +function requireSubstrate(namespace: Namespace): SqliteSpaceSubstrate { + const substrate = conversationTables(namespace); + if (!substrate) { + throw new Error( + `No SQLite conversation store for namespace ${JSON.stringify(namespace.name)}`, + ); + } + return substrate; +} + +export class SqliteThreadStore implements ThreadStore { + upsert(namespace: Namespace, threadId: string, record: ThreadRecord): void { + const { database, extensionId } = requireSubstrate(namespace); + database + .prepare( + `INSERT INTO agenetes_threads (extension_id, thread_id, record_json) + VALUES (?, ?, ?) + ON CONFLICT(extension_id, thread_id) DO UPDATE SET + record_json = excluded.record_json`, + ) + .run(extensionId, threadId, encode(record, `Thread ${threadId}`)); + } + + get(namespace: Namespace, threadId: string): ThreadRecord | undefined { + const substrate = conversationTables(namespace); + if (!substrate) return undefined; + const row = substrate.database + .prepare( + `SELECT record_json FROM agenetes_threads + WHERE extension_id = ? AND thread_id = ?`, + ) + .get(substrate.extensionId, threadId); + return row === undefined + ? undefined + : decode(row['record_json'], `Thread ${threadId}`); + } + + list(namespace: Namespace): ThreadRecord[] { + const substrate = conversationTables(namespace); + if (!substrate) return []; + return substrate.database + .prepare( + `SELECT thread_id, record_json FROM agenetes_threads + WHERE extension_id = ? + ORDER BY thread_id`, + ) + .all(substrate.extensionId) + .map((row) => + decode( + row['record_json'], + `Thread ${String(row['thread_id'])}`, + ), + ); + } + + delete(namespace: Namespace, threadId: string): void { + const substrate = conversationTables(namespace); + if (!substrate) return; + substrate.database + .prepare( + `DELETE FROM agenetes_threads + WHERE extension_id = ? AND thread_id = ?`, + ) + .run(substrate.extensionId, threadId); + } +} + +export class SqliteEventLogStore implements EventLogStore { + appendTurnStart( + namespace: Namespace, + threadId: string, + request: AgentSubmission | null, + ): TurnStartLogEntry { + const entry: TurnStartLogEntry = { + seq: this.maxSeq(namespace, threadId) + 1, + ts: Date.now(), + kind: 'turn_start' as const, + request, + }; + this.#insert(namespace, threadId, entry.seq, entry); + return entry; + } + + append( + namespace: Namespace, + threadId: string, + event: EventLogEntry['event'], + ): EventLogEntry { + const entry: EventLogEntry = { + seq: this.maxSeq(namespace, threadId) + 1, + ts: Date.now(), + event, + }; + this.#insert(namespace, threadId, entry.seq, entry); + return entry; + } + + read(namespace: Namespace, threadId: string, sinceSeq = 0): EventLogEntry[] { + return this.readRecords(namespace, threadId, sinceSeq).filter( + (record): record is EventLogEntry => !('kind' in record), + ); + } + + readRecords( + namespace: Namespace, + threadId: string, + sinceSeq = 0, + ): EventLogRecord[] { + const substrate = conversationTables(namespace); + if (!substrate) return []; + return substrate.database + .prepare( + `SELECT record_json FROM agenetes_events + WHERE extension_id = ? AND thread_id = ? AND seq > ? + ORDER BY seq`, + ) + .all(substrate.extensionId, threadId, sinceSeq) + .map((row) => + decode( + row['record_json'], + `Event log for thread ${threadId}`, + ), + ); + } + + maxSeq(namespace: Namespace, threadId: string): number { + const substrate = conversationTables(namespace); + if (!substrate) return 0; + const value = substrate.database + .prepare( + `SELECT COALESCE(MAX(seq), 0) AS max_seq FROM agenetes_events + WHERE extension_id = ? AND thread_id = ?`, + ) + .get(substrate.extensionId, threadId)?.['max_seq']; + return typeof value === 'number' ? value : 0; + } + + replace( + namespace: Namespace, + threadId: string, + records: readonly EventLogRecord[], + ): void { + const { database, extensionId } = requireSubstrate(namespace); + // One statement batch, not a transaction: `rehome()` calls this while the + // instance holds its own ordering, and an adapter that opened a nested + // transaction here would collide with a caller that already has one. + database + .prepare( + 'DELETE FROM agenetes_events WHERE extension_id = ? AND thread_id = ?', + ) + .run(extensionId, threadId); + const insert = database.prepare( + `INSERT INTO agenetes_events (extension_id, thread_id, seq, record_json) + VALUES (?, ?, ?, ?)`, + ); + for (const record of records) { + insert.run( + extensionId, + threadId, + record.seq, + encode(record, `Event log for thread ${threadId}`), + ); + } + } + + delete(namespace: Namespace, threadId: string): void { + const substrate = conversationTables(namespace); + if (!substrate) return; + substrate.database + .prepare( + 'DELETE FROM agenetes_events WHERE extension_id = ? AND thread_id = ?', + ) + .run(substrate.extensionId, threadId); + } + + #insert( + namespace: Namespace, + threadId: string, + seq: number, + record: EventLogRecord, + ): void { + const { database, extensionId } = requireSubstrate(namespace); + database + .prepare( + `INSERT INTO agenetes_events (extension_id, thread_id, seq, record_json) + VALUES (?, ?, ?, ?)`, + ) + .run( + extensionId, + threadId, + seq, + encode(record, `Event log for thread ${threadId}`), + ); + } +} + +export class SqliteTurnStore implements TurnStore { + append( + namespace: Namespace, + threadId: string, + persisted: PersistedTurn, + ): void { + const { database, extensionId } = requireSubstrate(namespace); + const ordinal = this.count(namespace, threadId) + 1; + database + .prepare( + `INSERT INTO agenetes_turns ( + extension_id, thread_id, ordinal, seq_start, seq_end, turn_json + ) VALUES (?, ?, ?, ?, ?, ?)`, + ) + .run( + extensionId, + threadId, + ordinal, + persisted.seqStart, + persisted.seqEnd, + encode(persisted.turn, `Turn for thread ${threadId}`), + ); + } + + list(namespace: Namespace, threadId: string): PersistedTurn[] { + const substrate = conversationTables(namespace); + if (!substrate) return []; + return substrate.database + .prepare( + `SELECT seq_start, seq_end, turn_json FROM agenetes_turns + WHERE extension_id = ? AND thread_id = ? + ORDER BY ordinal`, + ) + .all(substrate.extensionId, threadId) + .map((row) => ({ + turn: decode( + row['turn_json'], + `Turn for thread ${threadId}`, + ), + seqStart: Number(row['seq_start']), + seqEnd: Number(row['seq_end']), + })); + } + + count(namespace: Namespace, threadId: string): number { + const substrate = conversationTables(namespace); + if (!substrate) return 0; + const value = substrate.database + .prepare( + `SELECT COUNT(*) AS turns FROM agenetes_turns + WHERE extension_id = ? AND thread_id = ?`, + ) + .get(substrate.extensionId, threadId)?.['turns']; + return typeof value === 'number' ? value : 0; + } + + fence(namespace: Namespace, threadId: string): number { + const substrate = conversationTables(namespace); + if (!substrate) return 0; + const value = substrate.database + .prepare( + `SELECT seq_end FROM agenetes_turns + WHERE extension_id = ? AND thread_id = ? + ORDER BY ordinal DESC + LIMIT 1`, + ) + .get(substrate.extensionId, threadId)?.['seq_end']; + return typeof value === 'number' ? value : 0; + } + + replace( + namespace: Namespace, + threadId: string, + persisted: readonly PersistedTurn[], + ): void { + const { database, extensionId } = requireSubstrate(namespace); + database + .prepare( + 'DELETE FROM agenetes_turns WHERE extension_id = ? AND thread_id = ?', + ) + .run(extensionId, threadId); + const insert = database.prepare( + `INSERT INTO agenetes_turns ( + extension_id, thread_id, ordinal, seq_start, seq_end, turn_json + ) VALUES (?, ?, ?, ?, ?, ?)`, + ); + persisted.forEach((record, index) => { + insert.run( + extensionId, + threadId, + index + 1, + record.seqStart, + record.seqEnd, + encode(record.turn, `Turn for thread ${threadId}`), + ); + }); + } + + delete(namespace: Namespace, threadId: string): void { + const substrate = conversationTables(namespace); + if (!substrate) return; + substrate.database + .prepare( + 'DELETE FROM agenetes_turns WHERE extension_id = ? AND thread_id = ?', + ) + .run(substrate.extensionId, threadId); + } +} diff --git a/apps/server/src/modules/agent/conversation/prompt/debug-prompt.ts b/apps/server/src/modules/agent/conversation/prompt/debug-prompt.ts index b923b2098..2e91df4d7 100644 --- a/apps/server/src/modules/agent/conversation/prompt/debug-prompt.ts +++ b/apps/server/src/modules/agent/conversation/prompt/debug-prompt.ts @@ -14,11 +14,8 @@ * Gated and fully wrapped in try/catch so it can never affect a request. */ -import { appendFileSync } from 'node:fs'; -import path from 'node:path'; - -import { sanitizeId } from '../../../../utils/fs.js'; import { space } from '../../../storage/index.js'; +import { appendSubstrateLog } from '../../substrate-store.js'; import type { SpaceSubstrate } from '../../../storage/index.js'; import type { Context } from '@earendil-works/pi-ai'; @@ -151,13 +148,8 @@ type SubstrateResolution = | { readonly ok: true; readonly substrate: SpaceSubstrate | null } | { readonly ok: false; readonly error: unknown }; -/** Where this module keeps one log per thread on a Disk substrate. */ -function diskLogPath(substrate: SpaceSubstrate, threadId: string): string { - return path.join( - substrate.directory, - `${sanitizeId(threadId, 'threadId')}.prompt.log`, - ); -} +/** The suffix one thread's log carries, whatever the substrate stores it in. */ +const LOG_SUFFIX = '.prompt.log'; /** * Append a readable dump of the assembled prompt for one turn. No-op @@ -212,7 +204,7 @@ export function dumpAssembledPrompt(params: DumpPromptParams): void { if (!resolved.ok) throw resolved.error; const { substrate } = resolved; if (!substrate) return; - appendFileSync(diskLogPath(substrate, params.threadId), block, 'utf-8'); + appendSubstrateLog(substrate, params.threadId, LOG_SUFFIX, block); }) .catch((err: unknown) => { params.logger.warn( diff --git a/apps/server/src/modules/agent/memory/analyzer.test.ts b/apps/server/src/modules/agent/memory/analyzer.test.ts index 6aaa95b7a..0c75db5a3 100644 --- a/apps/server/src/modules/agent/memory/analyzer.test.ts +++ b/apps/server/src/modules/agent/memory/analyzer.test.ts @@ -33,6 +33,7 @@ vi.mock('../../storage/index.js', () => ({ SPACE_MEMORY_BLOB_NAME: 'space.md', })); vi.mock('../../workspace/paths.js', () => ({ + hasWorkspaceSettingDirectory: () => true, workspaceMemoryPath: () => `${physicalState.root}/setting/user.md`, })); diff --git a/apps/server/src/modules/agent/memory/analyzer.ts b/apps/server/src/modules/agent/memory/analyzer.ts index e5da5d82f..c99f2d0a3 100644 --- a/apps/server/src/modules/agent/memory/analyzer.ts +++ b/apps/server/src/modules/agent/memory/analyzer.ts @@ -33,7 +33,10 @@ import { type CanvasFile, type SpaceHandle, } from '../../storage/index.js'; -import { workspaceMemoryPath } from '../../workspace/paths.js'; +import { + hasWorkspaceSettingDirectory, + workspaceMemoryPath, +} from '../../workspace/paths.js'; import { runAgent } from '../agent.service.js'; import { readCanvasMemory } from './read.js'; @@ -280,7 +283,12 @@ function readEventsDigest(events: readonly CanvasEvent[]): EventsDigest | null { async function readMemorySnapshot(canvasId: string): Promise { const parts: string[] = []; - const longTerm = readFileSafe(workspaceMemoryPath()); + // Empty rather than missing on a backend with no Workspace folder: the + // curator's prompt keeps its shape, and the tier it cannot write to simply + // reads as empty (`workspace-user-memory` capability). + const longTerm = hasWorkspaceSettingDirectory() + ? readFileSafe(workspaceMemoryPath()) + : ''; parts.push('## Long-term memory'); parts.push(longTerm.trim().length > 0 ? longTerm.trim() : '(empty)'); diff --git a/apps/server/src/modules/agent/memory/read.ts b/apps/server/src/modules/agent/memory/read.ts index 281d16292..638e664f4 100644 --- a/apps/server/src/modules/agent/memory/read.ts +++ b/apps/server/src/modules/agent/memory/read.ts @@ -16,7 +16,10 @@ import { existsSync, readFileSync } from 'node:fs'; import { space, SPACE_MEMORY_BLOB_NAME } from '../../storage/index.js'; -import { workspaceMemoryPath } from '../../workspace/paths.js'; +import { + hasWorkspaceSettingDirectory, + workspaceMemoryPath, +} from '../../workspace/paths.js'; /** * Read the user memory body. @@ -28,6 +31,10 @@ import { workspaceMemoryPath } from '../../workspace/paths.js'; * zero-information `(empty)` line. */ export function readWorkspaceMemory(): string | null { + // A backend with no Workspace folder has no user memory document. Absence, + // not failure: the preamble is optional context either way, and the + // limitation is stated up front as the `workspace-user-memory` capability. + if (!hasWorkspaceSettingDirectory()) return null; return readNonEmpty(workspaceMemoryPath()); } diff --git a/apps/server/src/modules/agent/memory/trigger.ts b/apps/server/src/modules/agent/memory/trigger.ts index f2d4b1ced..08b870785 100644 --- a/apps/server/src/modules/agent/memory/trigger.ts +++ b/apps/server/src/modules/agent/memory/trigger.ts @@ -24,13 +24,12 @@ * one analysis pass, which is harmless. */ -import path from 'node:path'; - -import { atomicWriteJson, readJson } from '../../../utils/fs.js'; import { createKeyedMutex } from '../../../utils/keyed-mutex.js'; import { space } from '../../storage/index.js'; - -import type { SpaceSubstrate } from '../../storage/index.js'; +import { + readSubstrateDocument, + writeSubstrateDocument, +} from '../substrate-store.js'; /** This module's namespace on the substrate. */ const MEMORY_NAMESPACE = 'huabu.memory'; @@ -44,9 +43,7 @@ const MEMORY_NAMESPACE = 'huabu.memory'; * (§6.4.4). An owner that later wants the same shape extracts a helper *over* * the substrate, never a port member. */ -function diskStatePath(substrate: SpaceSubstrate): string { - return path.join(substrate.directory, 'state.json'); -} +const STATE_DOCUMENT = 'state'; /** Op-count threshold that triggers a memory analysis pass. */ export const OP_THRESHOLD = 50; @@ -81,7 +78,10 @@ const EMPTY_STATE: MemoryState = { export async function readMemoryState(canvasId: string): Promise { const substrate = await space(canvasId).extension(MEMORY_NAMESPACE); if (!substrate) return { ...EMPTY_STATE }; - const raw = readJson>(diskStatePath(substrate)); + const raw = readSubstrateDocument>( + substrate, + STATE_DOCUMENT, + ); if (!raw || typeof raw !== 'object') return { ...EMPTY_STATE }; return { counter: typeof raw.counter === 'number' ? raw.counter : 0, @@ -111,7 +111,7 @@ export async function writeMemoryState( ): Promise { const substrate = await space(canvasId).extension(MEMORY_NAMESPACE); if (!substrate) return; - atomicWriteJson(diskStatePath(substrate), state); + writeSubstrateDocument(substrate, STATE_DOCUMENT, state); } /** diff --git a/apps/server/src/modules/agent/substrate-store.ts b/apps/server/src/modules/agent/substrate-store.ts new file mode 100644 index 000000000..0f94fc004 --- /dev/null +++ b/apps/server/src/modules/agent/substrate-store.ts @@ -0,0 +1,137 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * Two shapes an extension namespace can be stored in, on either substrate. + * + * The storage port hands a namespace a *place* and nothing else — a directory + * on Disk, a connection plus a parent row on SQLite (proposal §6.4.4). That is + * deliberate: a key/value member on the port would have fixed one access shape + * for every owner forever. What the port's own commentary anticipates instead + * is a helper *over* the substrate, written by owners who happen to want the + * same shape. This is that helper, for the two shapes the agent module needs: + * + * - a whole JSON document, rewritten each time (memory bookkeeping); + * - an append-only text log per key (the debug prompt dump). + * + * Nothing here is a port. It is one owner's storage code, kept in one file + * because two owners wanted the same thing rather than because storage said + * they should. + */ + +import { appendFileSync, mkdirSync } from 'node:fs'; +import path from 'node:path'; + +import { atomicWriteJson, readJson, sanitizeId } from '../../utils/fs.js'; + +import type { SpaceSubstrate } from '../storage/index.js'; +import type { DatabaseSync } from 'node:sqlite'; + +/** Tables created on demand, once per connection. */ +const prepared = new WeakSet(); + +const SCHEMA = ` + CREATE TABLE IF NOT EXISTS extension_documents ( + extension_id INTEGER NOT NULL, + name TEXT NOT NULL, + body TEXT NOT NULL, + PRIMARY KEY (extension_id, name), + FOREIGN KEY (extension_id) REFERENCES space_extensions(extension_id) + ON DELETE CASCADE + ) STRICT; +`; + +function ensureTables(database: DatabaseSync): void { + if (prepared.has(database)) return; + database.exec(SCHEMA); + prepared.add(database); +} + +/** + * Read one JSON document from a namespace, or `null` when it is not there. + * + * Absence and damage are the same answer on purpose: both callers treat a + * missing document as "start from nothing", and a bookkeeping file a user can + * corrupt by hand must not be able to fail a request. + */ +export function readSubstrateDocument( + substrate: SpaceSubstrate, + name: string, +): T | null { + const safe = sanitizeId(name, 'document name'); + if (substrate.kind === 'disk') { + return readJson(path.join(substrate.directory, `${safe}.json`)); + } + ensureTables(substrate.database); + const row = substrate.database + .prepare( + `SELECT body FROM extension_documents + WHERE extension_id = ? AND name = ?`, + ) + .get(substrate.extensionId, safe); + if (row === undefined || typeof row['body'] !== 'string') return null; + try { + return JSON.parse(row['body']) as T; + } catch { + return null; + } +} + +/** Replace one JSON document in a namespace. */ +export function writeSubstrateDocument( + substrate: SpaceSubstrate, + name: string, + value: unknown, +): void { + const safe = sanitizeId(name, 'document name'); + if (substrate.kind === 'disk') { + atomicWriteJson(path.join(substrate.directory, `${safe}.json`), value); + return; + } + ensureTables(substrate.database); + const body = JSON.stringify(value); + if (body === undefined) { + throw new TypeError(`Document ${safe} is not representable as JSON`); + } + substrate.database + .prepare( + `INSERT INTO extension_documents (extension_id, name, body) + VALUES (?, ?, ?) + ON CONFLICT(extension_id, name) DO UPDATE SET body = excluded.body`, + ) + .run(substrate.extensionId, safe, body); +} + +/** + * Append to one text log in a namespace. + * + * On Disk this is a real file, which is the point of the debug log: a + * developer tails it. Elsewhere it is a row that grows, which keeps the same + * feature working without pretending there is a file to tail. + */ +export function appendSubstrateLog( + substrate: SpaceSubstrate, + name: string, + suffix: string, + block: string, +): void { + const safe = sanitizeId(name, 'log name'); + if (substrate.kind === 'disk') { + mkdirSync(substrate.directory, { recursive: true }); + appendFileSync( + path.join(substrate.directory, `${safe}${suffix}`), + block, + 'utf8', + ); + return; + } + ensureTables(substrate.database); + substrate.database + .prepare( + `INSERT INTO extension_documents (extension_id, name, body) + VALUES (?, ?, ?) + ON CONFLICT(extension_id, name) DO UPDATE SET + body = extension_documents.body || excluded.body`, + ) + .run(substrate.extensionId, `${safe}${suffix}`, block); +} diff --git a/apps/server/src/modules/agent/tools/handlers/fs-sandbox.ts b/apps/server/src/modules/agent/tools/handlers/fs-sandbox.ts index 076739824..c1cc6438f 100644 --- a/apps/server/src/modules/agent/tools/handlers/fs-sandbox.ts +++ b/apps/server/src/modules/agent/tools/handlers/fs-sandbox.ts @@ -30,7 +30,11 @@ import { readFileSync, readdirSync, statSync, type Dirent } from 'node:fs'; import path from 'node:path'; import { parseFrontmatter } from '../../../../utils/markdown-frontmatter.js'; -import { space, unavailableCapabilityMessage } from '../../../storage/index.js'; +import { + space, + storageServes, + unavailableCapabilityMessage, +} from '../../../storage/index.js'; // ─── Always-skipped directory names ───────────────────────────────────────── @@ -147,7 +151,12 @@ export function safeResolve(canvasId: string, rel: string): string { // capability-matrix entry, so an operator learns this when they select a // profile rather than when an agent calls a tool. Refusing here is the // backstop behind that declaration, phrased the same way. - const tree = space(canvasId).diskTree; + // The matrix decides; `diskTree` supplies the root. These tools address the + // byte areas through `upload/` and `artifacts/` aliases as well as `nodes/`, + // so the requirement spans both axes and must not be re-derived here. + const tree = storageServes('builtin-file-tools') + ? space(canvasId).diskTree + : null; if (!tree) throw new Error(unavailableCapabilityMessage('builtin-file-tools')); const root = tree.directory(); diff --git a/apps/server/src/modules/agent/tools/handlers/fs-write.ts b/apps/server/src/modules/agent/tools/handlers/fs-write.ts index ab2de9a28..dbe2a5346 100644 --- a/apps/server/src/modules/agent/tools/handlers/fs-write.ts +++ b/apps/server/src/modules/agent/tools/handlers/fs-write.ts @@ -32,7 +32,12 @@ import { existsSync } from 'node:fs'; import path from 'node:path'; import { normalizeRel } from './fs-sandbox.js'; -import { space, SPACE_MEMORY_BLOB_NAME } from '../../../storage/index.js'; +import { + space, + SPACE_MEMORY_BLOB_NAME, + storageServes, + unavailableCapabilityMessage, +} from '../../../storage/index.js'; import { settingDir, userSkillsDir } from '../../../workspace/paths.js'; import { resolveLongTermPath, @@ -110,6 +115,15 @@ function resolveTarget( const rel = normalizeRel(args.path); if (rel === 'memory/user.md') { + // Refused in the words the profile declared, rather than crashing on a + // path the backend cannot build. A Space's own memory body still works — + // it is a blob, not a Workspace file. + if (!storageServes('workspace-user-memory')) { + return { + path: rel, + error: unavailableCapabilityMessage('workspace-user-memory'), + }; + } return { tier: 'workspace', document: fileDocument(resolveLongTermPath(), settingDir()), @@ -154,6 +168,12 @@ function resolveTarget( error: `fs_write only accepts skill paths of the form "skills//SKILL.md"`, }; } + if (!storageServes('workspace-user-skills')) { + return { + path: rel, + error: unavailableCapabilityMessage('workspace-user-skills'), + }; + } const skillId = segs[1]; try { const absPath = resolveUserSkillPath(skillId); diff --git a/apps/server/src/modules/agent/tools/world-target-read.test.ts b/apps/server/src/modules/agent/tools/world-target-read.test.ts index bd7a6eff7..12a02ce9b 100644 --- a/apps/server/src/modules/agent/tools/world-target-read.test.ts +++ b/apps/server/src/modules/agent/tools/world-target-read.test.ts @@ -11,6 +11,7 @@ const workspaceState = vi.hoisted(() => ({ path: '', leases: 0 })); vi.mock('../../workspace.js', () => ({ getWorkspacePath: () => workspaceState.path, + getWorkspaceKey: () => workspaceState.path, acquireWorkspaceOperationLease: () => { const workspacePath = workspaceState.path; workspaceState.leases += 1; diff --git a/apps/server/src/modules/canvas/canvas-command-router.test.ts b/apps/server/src/modules/canvas/canvas-command-router.test.ts index f9b6b1072..c2972dab5 100644 --- a/apps/server/src/modules/canvas/canvas-command-router.test.ts +++ b/apps/server/src/modules/canvas/canvas-command-router.test.ts @@ -39,6 +39,16 @@ interface TestStoredNode { data?: Record; } +/** + * The live Spaces the World's rules are checked against. + * + * The rules are pure: they need to know which Portal targets still exist, + * and a test says so directly rather than standing up a catalogue. + */ +function liveSpaceIds(): ReadonlySet { + return new Set(['canvas-a', 'canvas-b']); +} + function writeCanvas( directory: string, canvasId: string, @@ -681,6 +691,7 @@ describe.skip('legacy World Portal pin command routing', () => { 'canvas-world', brokenTopology, convertedDescendant, + liveSpaceIds(), ), ).toThrow('A node reference cannot change node type'); @@ -804,6 +815,7 @@ describe.skip('legacy World Portal pin command routing', () => { 'canvas-world', directlyLocked, directlyLocked, + liveSpaceIds(), ), ).not.toThrow(); @@ -833,6 +845,7 @@ describe.skip('legacy World Portal pin command routing', () => { 'canvas-world', portalLocked, portalLocked, + liveSpaceIds(), ), ).not.toThrow(); @@ -892,6 +905,7 @@ describe.skip('legacy World Portal pin command routing', () => { 'canvas-world', styled ?? [], copiedTarget, + liveSpaceIds(), ), ).toThrow('contains unsupported source-owned data'); @@ -933,7 +947,12 @@ describe.skip('legacy World Portal pin command routing', () => { if (!previous) throw new Error('Missing World state'); const canonical = structuredClone(previous); expect(() => - assertWorldPortalTopologyAllowed('canvas-world', previous, canonical), + assertWorldPortalTopologyAllowed( + 'canvas-world', + previous, + canonical, + liveSpaceIds(), + ), ).not.toThrow(); const resized = structuredClone(previous) as Array<{ @@ -944,7 +963,12 @@ describe.skip('legacy World Portal pin command routing', () => { if (!portal?.style) throw new Error('Missing Portal'); portal.style.width = (portal.style.width ?? 0) + 100; expect(() => - assertWorldPortalTopologyAllowed('canvas-world', previous, resized), + assertWorldPortalTopologyAllowed( + 'canvas-world', + previous, + resized, + liveSpaceIds(), + ), ).toThrow(WorldPortalMutationError); const withoutNodeRef = ( @@ -955,6 +979,7 @@ describe.skip('legacy World Portal pin command routing', () => { 'canvas-world', previous, withoutNodeRef, + liveSpaceIds(), ), ).toThrow('Node references must be removed with SET_PORTAL_NODE_PINS'); }); diff --git a/apps/server/src/modules/canvas/canvas-executor.ts b/apps/server/src/modules/canvas/canvas-executor.ts index 5202ddcd7..3ce60b4f8 100644 --- a/apps/server/src/modules/canvas/canvas-executor.ts +++ b/apps/server/src/modules/canvas/canvas-executor.ts @@ -61,9 +61,11 @@ import { importForeignNodeSources } from './import-node-src.js'; import { assertWorldPortalMutationsAllowed, assertWorldPortalResultAllowed, + readLiveSpaceIds, } from './world-portal-policy.js'; import { getLogger } from '../../utils/logger.js'; import { + isWorldCanvasId, space, withCanvasMutex, type BlobScope, @@ -74,6 +76,9 @@ import { type SpaceNodeMutation, } from '../storage/index.js'; +/** Reused for every Space that cannot hold a Portal, which is all but one. */ +const EMPTY_CANVAS_IDS: ReadonlySet = new Set(); + const log = getLogger('canvas.executor'); function insertedNodeIds(deltas: readonly Delta[]): Set { @@ -731,11 +736,18 @@ export async function executeOnServerAlreadyLocked( ); const prestateEdges = (canvas.state.edges ?? []) as CanvasEdge[]; + // Only the World's rules consult it, and only the World can hold Portals, + // so an ordinary Space never pays for the catalogue read. + const liveCanvasIds = isWorldCanvasId(canvasId) + ? await readLiveSpaceIds() + : EMPTY_CANVAS_IDS; + assertWorldPortalMutationsAllowed( canvasId, commands, prestateNodes, originator.source, + liveCanvasIds, ); if (originator.source === 'agent') { @@ -834,7 +846,12 @@ export async function executeOnServerAlreadyLocked( const sharedOut = applySharedPostEffectsFromWriteResult(writeResult); const finalNodes = writeResult.nodes; const finalEdges = sharedOut.edges; - assertWorldPortalResultAllowed(canvasId, prestateNodes, finalNodes); + assertWorldPortalResultAllowed( + canvasId, + prestateNodes, + finalNodes, + liveCanvasIds, + ); const deltas = diffCanvasState( { nodes: prestateNodes, edges: prestateEdges }, diff --git a/apps/server/src/modules/canvas/canvas.route.test.ts b/apps/server/src/modules/canvas/canvas.route.test.ts index fbe276855..1cd744e90 100644 --- a/apps/server/src/modules/canvas/canvas.route.test.ts +++ b/apps/server/src/modules/canvas/canvas.route.test.ts @@ -711,6 +711,69 @@ function useTablesProfile(): () => void { } describe('Disk-only capability refusals', () => { + it('preflights Disk export without sending an archive, then still downloads it', async () => { + createCanvas('c1', 'Disk Space'); + const app = await buildApp(); + try { + const checked = await app.inject({ + method: 'GET', + url: '/canvas/c1/export?check=true', + }); + expect(checked.statusCode).toBe(204); + expect(checked.body).toBe(''); + expect(checked.headers['content-disposition']).toBeUndefined(); + const download = await app.inject({ + method: 'GET', + url: '/canvas/c1/export', + }); + expect(download.statusCode).toBe(200); + expect(download.headers['content-type']).toBe('application/zip'); + expect(download.rawPayload.subarray(0, 2).toString()).toBe('PK'); + const missing = await app.inject({ + method: 'GET', + url: '/canvas/missing/export?check=true', + }); + expect(missing.statusCode).toBe(404); + } finally { + await app.close(); + } + }); + + it('refuses an unsupported export during preflight using the same policy as download', async () => { + createCanvas('c1', 'Tables Space'); + const restore = useTablesProfile(); + const app = await buildApp(); + try { + const checked = await app.inject({ + method: 'GET', + url: '/canvas/c1/export?check=true', + }); + expect(checked.statusCode).toBe(400); + expect(checked.json()).toEqual({ + code: 'STORAGE_CAPABILITY_UNAVAILABLE', + message: unavailableCapabilityMessage('space-bundle-export'), + }); + const body = multipartBody( + 'space.zip', + 'application/zip', + Buffer.from('zip'), + ); + const imported = await app.inject({ + method: 'POST', + url: '/canvas/import', + ...body, + }); + expect(imported.statusCode).toBe(400); + expect(imported.json()).toEqual({ + code: 'STORAGE_CAPABILITY_UNAVAILABLE', + message: unavailableCapabilityMessage('space-bundle-import'), + }); + } finally { + await app.close(); + restore(); + } + }); + it('refuses in the same words the profile declared at startup', async () => { createCanvas('c1', 'Tables Space'); const restore = useTablesProfile(); @@ -769,7 +832,9 @@ describe('Space export/import persistence', () => { createCanvas('c1', 'Private Export'); const promptStore = await space('c1').extension('huabu.prompt.log'); const memoryStore = await space('c1').extension('huabu.memory'); - if (!promptStore || !memoryStore) throw new Error('Expected Disk stores'); + if (promptStore?.kind !== 'disk' || memoryStore?.kind !== 'disk') { + throw new Error('Expected Disk stores'); + } writeFileSync( join(promptStore.directory, 'thread.prompt.log'), 'private system and user prompt', @@ -806,7 +871,7 @@ describe('Space export/import persistence', () => { const importedPrompt = await space(importedId).extension('huabu.prompt.log'); const importedMemory = await space(importedId).extension('huabu.memory'); - if (!importedPrompt || !importedMemory) { + if (importedPrompt?.kind !== 'disk' || importedMemory?.kind !== 'disk') { throw new Error('Expected imported Disk stores'); } expect( diff --git a/apps/server/src/modules/canvas/canvas.route.ts b/apps/server/src/modules/canvas/canvas.route.ts index 337cf76d1..d4cae905c 100644 --- a/apps/server/src/modules/canvas/canvas.route.ts +++ b/apps/server/src/modules/canvas/canvas.route.ts @@ -41,6 +41,7 @@ import { } from './space-preview-scene.js'; import { assertWorldPortalTopologyAllowed, + readLiveSpaceIds, WorldPortalMutationError, } from './world-portal-policy.js'; import { reconcileWorldPortals } from './world-portals.js'; @@ -52,12 +53,13 @@ import { MAX_UPLOAD_BYTES } from '../../upload-limits.js'; import { ARTIFACT_URL_REGEX } from '../artifact/utils.js'; import { getPreprocessDispatcher, getProfile } from '../preprocessing/index.js'; import { stripOfficeparserPreamble } from '../preprocessing/loaders/office-strip.js'; -import { isWorldCanvasId } from '../storage/canvas-dirs.js'; import { space, createSpace, deleteSpace, + isWorldCanvasId, stageSpaceImport, + storageServes, unavailableCapabilityMessage, getStructuredStore, type CanvasFile, @@ -1184,6 +1186,9 @@ const canvasRoutes: FastifyPluginAsync = async (fastify) => { canvasId, (existing?.state.nodes ?? []) as NodeLike[], incomingState.nodes ?? [], + isWorldCanvasId(canvasId) + ? await readLiveSpaceIds() + : new Set(), ); } catch (error) { if (error instanceof WorldPortalMutationError) { @@ -1611,13 +1616,17 @@ const canvasRoutes: FastifyPluginAsync = async (fastify) => { if (!(await handle.read())) { return reply.code(404).send({ message: 'Canvas not found' }); } - // Disk-only, declared as `reveal-space-folder`: the feature *is* "show me - // this in Finder", so a backend without a folder has nothing to show. + // Declared as `reveal-space-folder`: what this opens is the `nodes/` + // folder, and off Disk a node is a row, so there is no folder of node + // documents to open and no hand-editable collision to resolve in one. + // The matrix decides and `diskTree` only supplies the path — asking + // `diskTree` directly would re-derive the requirement here. + // // A profile that cannot serve the feature and a Space whose folder is // missing are different problems with different remedies, so they get // different answers — the first repeats the matrix sentence the operator // read when they chose the profile. - const tree = handle.diskTree; + const tree = storageServes('reveal-space-folder') ? handle.diskTree : null; if (!tree) { return reply.code(400).send({ message: unavailableCapabilityMessage('reveal-space-folder'), @@ -1638,11 +1647,12 @@ const canvasRoutes: FastifyPluginAsync = async (fastify) => { fastify.get<{ Params: { canvasId: string }; Querystring: ExportCanvasQuery; - // Success path streams a zip archive (Readable). Failure path is the + // Success streams a ZIP archive or returns 204 after an eligibility check. + // Failure is the // canonical ApiErrorBody — declared here so the 400/404 branches // type-check via the same `reply.send(...)` machinery the JSON // routes use. - Reply: ApiResult; + Reply: ApiResult; }>('/:canvasId/export', async function (request, reply) { const { canvasId } = request.params; const parsedQuery = exportCanvasQuerySchema.safeParse(request.query); @@ -1659,13 +1669,17 @@ const canvasRoutes: FastifyPluginAsync = async (fastify) => { return reply.code(404).send({ message: 'Canvas not found' }); } - // Disk-only, declared as `space-bundle-export` in the capability matrix; - // a portable export generated from records plus reachable blob references - // is a separate later design. Refuse in the matrix's own words, and keep - // that distinct from a Space whose directory has gone missing. - const tree = handle.diskTree; + // Declared as `space-bundle-export`; a portable export generated from + // records plus reachable blob references is a separate later design. The + // matrix decides and `diskTree` only supplies the path: this requirement + // spans both axes — the bundle is the Space folder archived, so it needs + // the bytes in it — and asking `diskTree` would re-derive only half. + // Refuse in the matrix's own words, and keep that distinct from a Space + // whose directory has gone missing. + const tree = storageServes('space-bundle-export') ? handle.diskTree : null; if (!tree) { return reply.code(400).send({ + code: 'STORAGE_CAPABILITY_UNAVAILABLE', message: unavailableCapabilityMessage('space-bundle-export'), }); } @@ -1674,6 +1688,12 @@ const canvasRoutes: FastifyPluginAsync = async (fastify) => { return reply.code(404).send({ message: 'Canvas directory not found' }); } + // The browser checks eligibility before following the native download link. + // Keep the checks above shared so preflight uses the same storage policy. + if (parsedQuery.data.check === 'true') { + return reply.code(204).send(undefined); + } + const manifest = { version: '2', exportedAt: new Date().toISOString(), @@ -1735,9 +1755,15 @@ const canvasRoutes: FastifyPluginAsync = async (fastify) => { // staging location, the title-derived directory, the record filename, // and the index entry are all layout. This route owns the `.huabu.zip` // format and nothing else (proposal §12.6.2). - const staged = stageSpaceImport(targetCanvasId); + // Same rule as export: the matrix decides, and staging only supplies + // the place. Import needs the bytes to land in the folder too, so the + // requirement spans both axes and re-deriving it here would miss that. + const staged = storageServes('space-bundle-import') + ? stageSpaceImport(targetCanvasId) + : null; if (!staged) { return reply.code(400).send({ + code: 'STORAGE_CAPABILITY_UNAVAILABLE', message: unavailableCapabilityMessage('space-bundle-import'), }); } diff --git a/apps/server/src/modules/canvas/external-watcher.test.ts b/apps/server/src/modules/canvas/external-watcher.test.ts index 8203416e8..5c919b1f1 100644 --- a/apps/server/src/modules/canvas/external-watcher.test.ts +++ b/apps/server/src/modules/canvas/external-watcher.test.ts @@ -72,6 +72,26 @@ const spaceHandle = vi.hoisted(() => ({ read: vi.fn(async () => ({ state: { nodes: [] } })), })); +/** + * The Space facade the watcher actually consults. + * + * `diskTree` is resolved per call from the same directory index the real one + * uses, so the "renamed outside the server" case still moves the watched path + * — and a Space with no directory is `null`, the way a non-Disk backend + * reports it. + */ +const spaceFacade = vi.hoisted(() => (canvasId: string) => ({ + ...spaceHandle, + diskTree: (() => { + const entry = canvasDirs + .list() + .find((candidate) => candidate.id === canvasId); + return entry + ? { canvasId, directory: () => `/ws/${entry.filename}` } + : null; + })(), +})); + // The facade is stubbed for the Space handle, but the directory-handle // helpers must stay the real ones: these cases drive // `withSpaceDirHandlesReleased` and assert the watcher released its handles, @@ -80,7 +100,7 @@ const spaceHandle = vi.hoisted(() => ({ vi.mock('../storage/index.js', async () => { const handles = await import('../storage/backends/disk/space-dir-handles.js'); return { - space: () => spaceHandle, + space: spaceFacade, registerSpaceDirHandleOwner: handles.registerSpaceDirHandleOwner, withSpaceDirHandlesReleased: handles.withSpaceDirHandlesReleased, }; diff --git a/apps/server/src/modules/canvas/external-watcher.ts b/apps/server/src/modules/canvas/external-watcher.ts index c20430521..bd40c8eb5 100644 --- a/apps/server/src/modules/canvas/external-watcher.ts +++ b/apps/server/src/modules/canvas/external-watcher.ts @@ -32,10 +32,8 @@ import path from 'node:path'; import { getLogger } from '../../utils/logger.js'; import { parseFrontmatter } from '../../utils/markdown-frontmatter.js'; -import { listAllCanvasDirEntries } from '../storage/canvas-dirs.js'; -import { space } from '../storage/index.js'; -import { registerSpaceDirHandleOwner } from '../storage/index.js'; -import { getWorkspacePath, isWorkspaceConfigured } from '../workspace.js'; +import { registerSpaceDirHandleOwner, space } from '../storage/index.js'; +import { isWorkspaceConfigured } from '../workspace.js'; import type { CanvasFile } from '../storage/index.js'; import type { ExternalNoteEvent, ExternalNoteItem } from '@huabu/shared'; @@ -112,11 +110,13 @@ function isSessionCurrent(session: ActiveSpaceWatch, stamp?: string): boolean { function nodesPathFor(canvasId: string): string | null { if (!isWorkspaceConfigured()) return null; - const entry = listAllCanvasDirEntries().find( - (candidate) => candidate.id === canvasId, - ); - if (!entry) return null; - return path.join(getWorkspacePath(), entry.filename, 'nodes'); + // `null` when the Space has no directory to watch, which covers both an + // unknown id and a backend that keeps Spaces in tables. Watching for + // documents that arrived without going through the application is the + // declared `external-note-discovery` capability, and this is where its + // absence becomes "there is nothing to watch". + const directory = space(canvasId).diskTree?.directory(); + return directory === undefined ? null : path.join(directory, 'nodes'); } function noteIdsFromCanvas(canvas: CanvasFile | null): Set { diff --git a/apps/server/src/modules/canvas/external.route.ts b/apps/server/src/modules/canvas/external.route.ts index 600c4144c..a2a7e97bb 100644 --- a/apps/server/src/modules/canvas/external.route.ts +++ b/apps/server/src/modules/canvas/external.route.ts @@ -17,7 +17,11 @@ import { takeExternalNote, } from './external-watcher.js'; import { parseFrontmatter } from '../../utils/markdown-frontmatter.js'; -import { space, unavailableCapabilityMessage } from '../storage/index.js'; +import { + space, + storageServes, + unavailableCapabilityMessage, +} from '../storage/index.js'; import type { FastifyPluginAsync } from 'fastify'; @@ -98,7 +102,9 @@ const externalRoutes: FastifyPluginAsync = async (fastify): Promise => { // Disk-only, declared as `external-note-discovery` in the capability // matrix: it adopts documents that arrived without going through the // application, and no database backend has such an arrival path. - const tree = space(canvasId).diskTree; + const tree = storageServes('external-note-discovery') + ? space(canvasId).diskTree + : null; if (!tree) { return reply.code(400).send({ message: unavailableCapabilityMessage('external-note-discovery'), diff --git a/apps/server/src/modules/canvas/persistence-validation.ts b/apps/server/src/modules/canvas/persistence-validation.ts new file mode 100644 index 000000000..20b3613f4 --- /dev/null +++ b/apps/server/src/modules/canvas/persistence-validation.ts @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** Runtime validation shared by structured storage adapters. */ + +function finiteNumber(value: unknown): boolean { + return typeof value === 'number' && Number.isFinite(value); +} + +/** Return the first minimal CanvasFile shape violation, if any. */ +export function canvasFileShapeError( + value: unknown, + expectedCanvasId: string, +): string | null { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return 'must be an object'; + } + + const record = value as Record; + if (record['canvasId'] !== expectedCanvasId) { + return `canvasId must equal ${JSON.stringify(expectedCanvasId)}`; + } + if (record['title'] !== null && typeof record['title'] !== 'string') { + return 'title must be a string or null'; + } + if (!finiteNumber(record['version'])) + return 'version must be a finite number'; + if (!finiteNumber(record['createdAt'])) { + return 'createdAt must be a finite number'; + } + if (!finiteNumber(record['updatedAt'])) { + return 'updatedAt must be a finite number'; + } + + const state = record['state']; + if (typeof state !== 'object' || state === null || Array.isArray(state)) { + return 'state must be an object'; + } + const stateRecord = state as Record; + if (!Array.isArray(stateRecord['nodes'])) + return 'state.nodes must be an array'; + if (!Array.isArray(stateRecord['edges'])) + return 'state.edges must be an array'; + return null; +} diff --git a/apps/server/src/modules/canvas/world-portal-policy.ts b/apps/server/src/modules/canvas/world-portal-policy.ts index cd74e7d74..f1336b1c2 100644 --- a/apps/server/src/modules/canvas/world-portal-policy.ts +++ b/apps/server/src/modules/canvas/world-portal-policy.ts @@ -3,10 +3,7 @@ import { fitPortals, getDescendantIds } from '@huabu/shared/canvas-engine'; -import { - isWorldCanvasId, - listCanvasDirEntries, -} from '../storage/canvas-dirs.js'; +import { getStructuredStore, isWorldCanvasId } from '../storage/index.js'; import type { CanvasCommand } from '@huabu/shared'; import type { NestableNode } from '@huabu/shared/canvas-engine'; @@ -24,6 +21,22 @@ function storedNodes(nodes: readonly unknown[]): StoredNode[] { ); } +/** + * Every ordinary Space in the active Workspace, by id. + * + * The World's Portals point at Spaces, so the rules below need to know which + * of those targets still exist. Read through the catalogue rather than a + * directory listing: the answer is the same on every backend, and the World is + * the one place in the product that asks it. + * + * The checks that consume this set take it as an argument, so the rules + * themselves stay pure and testable without a live backend. + */ +export async function readLiveSpaceIds(): Promise> { + const summaries = await getStructuredStore().spaces().list(); + return new Set(summaries.map((summary) => summary.canvasId)); +} + export class WorldPortalMutationError extends Error { constructor(message: string) { super(message); @@ -130,6 +143,7 @@ export function assertWorldPortalTopologyAllowed( canvasId: string, previousNodesInput: readonly unknown[], nextNodesInput: readonly unknown[], + liveCanvasIds: ReadonlySet, ): void { const previousNodes = storedNodes(previousNodesInput); const nextNodes = storedNodes(nextNodesInput); @@ -308,9 +322,6 @@ export function assertWorldPortalTopologyAllowed( } } - const liveCanvasIds = new Set( - listCanvasDirEntries().map((entry) => entry.id), - ); for (const previous of previousNodes) { const previousNodeRef = nodeRefTarget(previous); if (previousNodeRef) { @@ -372,12 +383,10 @@ export function assertWorldPortalResultAllowed( canvasId: string, previousNodesInput: readonly unknown[], nextNodesInput: readonly unknown[], + liveCanvasIds: ReadonlySet, ): void { if (!isWorldCanvasId(canvasId)) return; - const liveCanvasIds = new Set( - listCanvasDirEntries().map((entry) => entry.id), - ); const nextById = new Map( storedNodes(nextNodesInput).map((node) => [node.id, node]), ); @@ -418,6 +427,7 @@ export function assertWorldPortalMutationsAllowed( commands: readonly CanvasCommand[], nodes: readonly StoredNode[], source: 'ui' | 'agent' | 'system', + liveCanvasIds: ReadonlySet, ): void { if (source === 'system') return; @@ -439,9 +449,6 @@ export function assertWorldPortalMutationsAllowed( if (!isWorldCanvasId(canvasId)) return; - const liveCanvasIds = new Set( - listCanvasDirEntries().map((entry) => entry.id), - ); const byId = new Map(nodes.map((node) => [node.id, node])); for (const command of commands) { diff --git a/apps/server/src/modules/canvas/world-portals.test.ts b/apps/server/src/modules/canvas/world-portals.test.ts index 3af3c2bc1..16df35aff 100644 --- a/apps/server/src/modules/canvas/world-portals.test.ts +++ b/apps/server/src/modules/canvas/world-portals.test.ts @@ -13,6 +13,7 @@ const workspaceState = vi.hoisted(() => ({ path: '' })); vi.mock('../workspace.js', () => ({ getWorkspacePath: () => workspaceState.path, + getWorkspaceKey: () => workspaceState.path, })); import { executeOnServer } from './canvas-executor.js'; @@ -52,6 +53,19 @@ function writeCanvas( ); } +/** + * The live Spaces the World's rules are checked against. + * + * Passed in rather than read from a backend, because the rules are pure: what + * they need to know is which Portal targets still exist, and a test says so + * directly instead of standing up a catalogue to be asked. + */ +function liveSpaceIds( + ids: readonly string[] = ['canvas-a', 'canvas-b'], +): ReadonlySet { + return new Set(ids); +} + function portals(): Array<{ id: string; position: { x: number; y: number }; @@ -237,7 +251,12 @@ describe('World Space preview reconciliation', () => { if (!previous) throw new Error('Missing World topology'); expect(() => - assertWorldPortalTopologyAllowed('canvas-world', previous, []), + assertWorldPortalTopologyAllowed( + 'canvas-world', + previous, + [], + liveSpaceIds(), + ), ).toThrow(WorldPortalMutationError); const moved = structuredClone(previous) as Array<{ @@ -248,7 +267,12 @@ describe('World Space preview reconciliation', () => { if (!portal) throw new Error('Missing Space preview'); portal.position = { x: 999, y: 999 }; expect(() => - assertWorldPortalTopologyAllowed('canvas-world', previous, moved), + assertWorldPortalTopologyAllowed( + 'canvas-world', + previous, + moved, + liveSpaceIds(), + ), ).not.toThrow(); expect(() => @@ -263,6 +287,7 @@ describe('World Space preview reconciliation', () => { data: { targetCanvasId: 'canvas-b' }, }, ], + liveSpaceIds(), ), ).toThrow(WorldPortalMutationError); }); @@ -308,8 +333,15 @@ describe('World Space preview reconciliation', () => { }); refreshCanvasDirIndex(); + // `canvas-a` is gone, so its Portal is broken and the subtree under it may + // be removed. expect(() => - assertWorldPortalTopologyAllowed('canvas-world', previous, []), + assertWorldPortalTopologyAllowed( + 'canvas-world', + previous, + [], + liveSpaceIds(['canvas-b']), + ), ).not.toThrow(); }); @@ -354,6 +386,7 @@ describe('World Space preview reconciliation', () => { 'canvas-world', canonical, structuredClone(canonical), + liveSpaceIds(), ), ).not.toThrow(); @@ -375,6 +408,7 @@ describe('World Space preview reconciliation', () => { 'canvas-world', canonical, apparentlyFitted, + liveSpaceIds(), ), ).toThrow('Frame reference size is managed by its contents'); }); @@ -417,6 +451,7 @@ describe('World Space preview reconciliation', () => { 'canvas-world', canonical, apparentlyFitted, + liveSpaceIds(), ), ).toThrow('Frame reference size is managed by its contents'); }); @@ -460,6 +495,7 @@ describe('World Space preview reconciliation', () => { 'canvas-world', cyclic, structuredClone(cyclic), + liveSpaceIds(), ), ).toThrow('World reference hierarchy is cyclic'); }); diff --git a/apps/server/src/modules/canvas/world-reference-resolver.test.ts b/apps/server/src/modules/canvas/world-reference-resolver.test.ts index 277f0b86e..df79eb6a1 100644 --- a/apps/server/src/modules/canvas/world-reference-resolver.test.ts +++ b/apps/server/src/modules/canvas/world-reference-resolver.test.ts @@ -11,6 +11,7 @@ const workspaceState = vi.hoisted(() => ({ path: '', leases: 0 })); vi.mock('../workspace.js', () => ({ getWorkspacePath: () => workspaceState.path, + getWorkspaceKey: () => workspaceState.path, acquireWorkspaceOperationLease: () => { const workspacePath = workspaceState.path; workspaceState.leases += 1; diff --git a/apps/server/src/modules/desktop-workspace-upgrade.test.ts b/apps/server/src/modules/desktop-workspace-upgrade.test.ts index 24765eda8..41a44c515 100644 --- a/apps/server/src/modules/desktop-workspace-upgrade.test.ts +++ b/apps/server/src/modules/desktop-workspace-upgrade.test.ts @@ -39,7 +39,7 @@ import path from 'node:path'; import fastify from 'fastify'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { workspaceRegistryPath } from './storage/backends/disk/workspace-repository.js'; +import { workspaceRegistryPath } from './storage/backends/disk/data-dir.js'; import { resetStorageCache } from './storage/index.js'; import { setWorkspacePath } from './workspace.js'; import workspaceRoutes from './workspace.route.js'; diff --git a/apps/server/src/modules/interactive-view/interactive-view.service.ts b/apps/server/src/modules/interactive-view/interactive-view.service.ts index 917f64e82..491693976 100644 --- a/apps/server/src/modules/interactive-view/interactive-view.service.ts +++ b/apps/server/src/modules/interactive-view/interactive-view.service.ts @@ -1,9 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -import { existsSync } from 'node:fs'; -import path from 'node:path'; - import { createId, interactiveViewDefinitionV1Schema, @@ -27,7 +24,6 @@ import { agentThreadService, type ExternalAgentThreadTarget, } from '../agent/agent-thread.service.js'; -import { safeResolve } from '../agent/tools/handlers/fs-sandbox.js'; import { executeOnServer, type InteractiveViewConflict, @@ -88,17 +84,21 @@ async function resolveOwnerThread( } } -function stagedRendererPath( +/** + * Whether a `upload/` renderer has actually been staged. + * + * Asked of the Space's uploads scope rather than of a directory: the scratch + * an upload lands in is a blob area on every backend, and it is the same place + * on Disk that this used to resolve by hand. + */ +async function stagedRendererExists( canvasId: string, rendererArtifact: string, -): string | null { +): Promise { const match = STAGED_RENDERER_ARTIFACT_RE.exec(rendererArtifact); const filename = match?.[1]; - if (!filename) return null; - const uploadRoot = safeResolve(canvasId, '.upload'); - const candidate = path.resolve(uploadRoot, filename); - if (!candidate.startsWith(uploadRoot + path.sep)) return null; - return candidate; + if (!filename) return false; + return (await space(canvasId).uploads.head(filename)) !== null; } function validateDefinition(definition: InteractiveViewDefinitionV1): void { @@ -358,11 +358,8 @@ export class InteractiveViewService { `Owner thread ${request.ownerThreadId} is not an external Agent thread in this Canvas`, ); } - const stagedPath = request.rendererArtifact.startsWith('upload/') - ? stagedRendererPath(canvasId, request.rendererArtifact) - : null; const rendererExists = request.rendererArtifact.startsWith('upload/') - ? stagedPath !== null && existsSync(stagedPath) + ? await stagedRendererExists(canvasId, request.rendererArtifact) : Boolean(await space(canvasId).artifacts.head(request.rendererArtifact)); if (!rendererExists) { throw new InteractiveViewServiceError( diff --git a/apps/server/src/modules/remote_fs/rfs.route.ts b/apps/server/src/modules/remote_fs/rfs.route.ts index d72d38465..a399d3498 100644 --- a/apps/server/src/modules/remote_fs/rfs.route.ts +++ b/apps/server/src/modules/remote_fs/rfs.route.ts @@ -112,6 +112,10 @@ import { InteractiveViewServiceError, interactiveViewService, } from '../interactive-view/interactive-view.service.js'; +import { + storageServes, + unavailableCapabilityMessage, +} from '../storage/index.js'; import { RunCompletionError, runCompletionService, @@ -255,6 +259,18 @@ function logReachbackEvent( // ── Route plugin ── const rfsRoutes: FastifyPluginAsync = async (app) => { + // RFS is the Space *as files*. A backend that keeps Spaces in tables has no + // tree to project, and the honest answer is the declared refusal rather + // than a partial projection assembled from records — see the + // `space-file-plane` capability. One hook, because every route below + // resolves a real path sooner or later. + app.addHook('onRequest', async (_request, reply) => { + if (storageServes('space-file-plane')) return; + return reply + .code(409) + .send(rfsError(unavailableCapabilityMessage('space-file-plane'))); + }); + // Consume every request body as raw bytes within this plugin: uploads are // arbitrary binary, and the `agent` endpoint accepts either a JSON body or a // raw text prompt. Handlers interpret the Buffer per Content-Type. diff --git a/apps/server/src/modules/storage/backends/disk/blob-store.rename-retry.test.ts b/apps/server/src/modules/storage/backends/disk/blob-store.rename-retry.test.ts index 142e0806f..75219305f 100644 --- a/apps/server/src/modules/storage/backends/disk/blob-store.rename-retry.test.ts +++ b/apps/server/src/modules/storage/backends/disk/blob-store.rename-retry.test.ts @@ -8,6 +8,7 @@ import path from 'node:path'; import { describe, expect, it, vi } from 'vitest'; import { DiskBlobStore } from './blob-store.js'; +import { canvasRoot } from './layout.js'; import type * as NodeFsPromises from 'node:fs/promises'; @@ -23,6 +24,7 @@ vi.mock('node:fs/promises', async (importOriginal) => { vi.mock('../../../workspace.js', () => ({ getWorkspacePath: () => testState.workspacePath, + getWorkspaceKey: () => testState.workspacePath, })); function errno(code: string): NodeJS.ErrnoException { @@ -38,7 +40,9 @@ describe('DiskBlobStore retry cleanup', () => { testState.renameAsync.mockRejectedValue(error); try { - const scope = new DiskBlobStore().space('canvas-under-test').artifacts; + const scope = new DiskBlobStore(canvasRoot).space( + 'canvas-under-test', + ).artifacts; await expect(scope.put('blocked.bin', Buffer.from('bytes'))).rejects.toBe( error, diff --git a/apps/server/src/modules/storage/backends/disk/blob-store.test.ts b/apps/server/src/modules/storage/backends/disk/blob-store.test.ts index ba7317708..8cf608260 100644 --- a/apps/server/src/modules/storage/backends/disk/blob-store.test.ts +++ b/apps/server/src/modules/storage/backends/disk/blob-store.test.ts @@ -18,16 +18,18 @@ const workspaceState = vi.hoisted(() => ({ path: '' })); vi.mock('../../../workspace.js', () => ({ getWorkspacePath: () => workspaceState.path, + getWorkspaceKey: () => workspaceState.path, })); import { DiskBlobStore } from './blob-store.js'; +import { canvasRoot } from './layout.js'; import { describeBlobStoreContract } from '../../ports/contracts/blob-store.contract.js'; describeBlobStoreContract('DiskBlobStore', () => { const root = mkdtempSync(path.join(tmpdir(), 'huabu-blob-')); workspaceState.path = root; return { - store: new DiskBlobStore(), + store: new DiskBlobStore(canvasRoot), canvasId: 'canvas-under-test', cleanup: () => rmSync(root, { recursive: true, force: true }), }; @@ -54,7 +56,7 @@ describe('DiskBlobStore temp file hygiene', () => { }); it('cleans up after both successful and failed writes', async () => { - const scope = new DiskBlobStore().space(canvasId).artifacts; + const scope = new DiskBlobStore(canvasRoot).space(canvasId).artifacts; await scope.put('kept.bin', Buffer.from('fine')); await scope.put('streamed.bin', Readable.from([Buffer.from('also fine')])); @@ -77,7 +79,9 @@ describe('DiskBlobStore temp file hygiene', () => { }); it('cleans up siblings from concurrent writers to one key', async () => { - const scope = new DiskBlobStore().space('concurrent-canvas').artifacts; + const scope = new DiskBlobStore(canvasRoot).space( + 'concurrent-canvas', + ).artifacts; await Promise.all( Array.from({ length: 8 }, (_, i) => @@ -92,7 +96,7 @@ describe('DiskBlobStore temp file hygiene', () => { it('binds in-flight paths to their original workspace and rejects a held scope after activation', async () => { const otherRoot = mkdtempSync(path.join(tmpdir(), 'huabu-blob-switched-')); - const scope = new DiskBlobStore().space(canvasId).artifacts; + const scope = new DiskBlobStore(canvasRoot).space(canvasId).artifacts; let signalStarted = (): void => {}; const started = new Promise((resolve) => { signalStarted = resolve; diff --git a/apps/server/src/modules/storage/backends/disk/blob-store.ts b/apps/server/src/modules/storage/backends/disk/blob-store.ts index 9404f22a0..30f72d469 100644 --- a/apps/server/src/modules/storage/backends/disk/blob-store.ts +++ b/apps/server/src/modules/storage/backends/disk/blob-store.ts @@ -4,13 +4,19 @@ /** * Disk implementation of the blob port. * - * Maps each area of a Space to a directory under its Space folder, preserving + * Maps each area of a Space to a directory under that Space's root, preserving * the layout the workspace format has always used: one file per blob, named by * the URL key, no manifest indirection. * + * The root is a constructor argument, because this adapter does not know where + * a Space is. `blobs=disk` names a *medium* — bytes are local files — and + * composition names the place (`storage.ts::buildBlobStore`, which carries the + * rule and the reason). Everything below the root is identical whichever place + * that turns out to be. + * * Each scope is bound to the workspace active when it is created. A fresh - * scope follows a free-mode workspace switch; a retained scope rejects the - * next operation instead of silently redirecting it into the new workspace. + * scope follows a workspace switch; a retained scope rejects the next + * operation instead of silently redirecting it into the new workspace. */ import { randomUUID } from 'node:crypto'; @@ -27,13 +33,12 @@ import path from 'node:path'; import { pipeline } from 'node:stream/promises'; import { - artifactsDir, - canvasRoot, - spaceMemoryDir, - spaceUploadDir, + ARTIFACTS_DIR_NAME, + MEMORY_DIR_NAME, + UPLOAD_DIR_NAME, } from './layout.js'; import { renameOverWithRetry } from '../../../../utils/fs.js'; -import { getWorkspacePath } from '../../../workspace.js'; +import { getWorkspaceKey } from '../../../workspace.js'; import { BlobNameError, createBlobLease, @@ -69,6 +74,17 @@ function isTempEntry(entry: string): boolean { /** The areas of a Space, as this adapter places them. */ type SpaceBlobArea = keyof SpaceBlobs; +/** + * Where this Space keeps its bytes. + * + * Supplied, never defaulted. Which directory a Space's bytes belong in is a + * fact about the whole deployment — it depends on whether the *structured* + * backend gives that Space a directory of its own — and this adapter is not + * the layer that knows. A default here would be that cross-axis decision made + * silently, by whichever caller forgot to pass one. + */ +export type SpaceBlobRoot = (canvasId: string) => string; + /** * Where one area's bytes sit, and which names it owns there. * @@ -81,21 +97,18 @@ interface ScopePlacement { readonly members: readonly string[] | null; } -function scopePlacement(area: SpaceBlobArea, canvasId: string): ScopePlacement { +function scopePlacement(area: SpaceBlobArea, root: string): ScopePlacement { switch (area) { case 'artifacts': - return { directory: artifactsDir(canvasId), members: null }; + return { directory: path.join(root, ARTIFACTS_DIR_NAME), members: null }; case 'memory': - return { directory: spaceMemoryDir(canvasId), members: null }; + return { directory: path.join(root, MEMORY_DIR_NAME), members: null }; case 'uploads': - return { directory: spaceUploadDir(canvasId), members: null }; + return { directory: path.join(root, UPLOAD_DIR_NAME), members: null }; case 'guide': - // The Space root, which also holds `space.json` and every node - // directory — so this area is the guide names, not the folder. - return { - directory: canvasRoot(canvasId), - members: SPACE_GUIDE_BLOB_NAMES, - }; + // The Space root itself, which on Disk also holds `space.json` and every + // node directory — so this area is the guide names, not the folder. + return { directory: root, members: SPACE_GUIDE_BLOB_NAMES }; } } @@ -112,17 +125,21 @@ function isMissing(err: unknown): boolean { class DiskBlobScope implements BlobScope { readonly #area: SpaceBlobArea; readonly #canvasId: string; - readonly #workspacePath: string; + readonly #root: SpaceBlobRoot; + readonly #workspaceKey: string; - constructor(area: SpaceBlobArea, canvasId: string) { + constructor(area: SpaceBlobArea, canvasId: string, root: SpaceBlobRoot) { this.#area = area; this.#canvasId = canvasId; - this.#workspacePath = path.resolve(getWorkspacePath()); + this.#root = root; + // The Workspace as an identity rather than a location: a Workspace that is + // a row has no path to compare, and the binding means the same thing + // either way. + this.#workspaceKey = getWorkspaceKey(); } #placement(): ScopePlacement { - const active = path.resolve(getWorkspacePath()); - if (active !== this.#workspacePath) { + if (getWorkspaceKey() !== this.#workspaceKey) { throw new Error( `DiskBlobScope(${this.#canvasId}) belongs to an inactive workspace. ` + `Resolve a fresh scope after workspace activation.`, @@ -131,7 +148,7 @@ class DiskBlobScope implements BlobScope { // Resolve once per operation, before its first await. Every later path in // that operation is derived from this absolute directory, so a workspace // switch cannot combine a temp in A with a destination in B. - return scopePlacement(this.#area, this.#canvasId); + return scopePlacement(this.#area, this.#root(this.#canvasId)); } /** Names this scope owns in `dir`, given what is actually there. */ @@ -312,6 +329,12 @@ class DiskBlobScope implements BlobScope { export class DiskBlobStore implements BlobStore { readonly kind = 'disk' as const; + readonly #root: SpaceBlobRoot; + + constructor(root: SpaceBlobRoot) { + this.#root = root; + } + async init(): Promise { // Area directories are created on first write; nothing to prepare. } @@ -323,11 +346,13 @@ export class DiskBlobStore implements BlobStore { async close(): Promise {} space(canvasId: string): SpaceBlobs { + const scope = (area: SpaceBlobArea): BlobScope => + new DiskBlobScope(area, canvasId, this.#root); return { - artifacts: new DiskBlobScope('artifacts', canvasId), - guide: new DiskBlobScope('guide', canvasId), - memory: new DiskBlobScope('memory', canvasId), - uploads: new DiskBlobScope('uploads', canvasId), + artifacts: scope('artifacts'), + guide: scope('guide'), + memory: scope('memory'), + uploads: scope('uploads'), }; } } diff --git a/apps/server/src/modules/storage/backends/disk/data-dir.ts b/apps/server/src/modules/storage/backends/disk/data-dir.ts new file mode 100644 index 000000000..f250f8827 --- /dev/null +++ b/apps/server/src/modules/storage/backends/disk/data-dir.ts @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * Where the Disk backend puts state that belongs to no Workspace folder. + * + * `layout.ts` answers "where inside a Workspace does a Space go". This file + * answers the other half: the Disk adapters also keep things in the Server's + * own data directory — a registry of Workspaces that is *about* folders rather + * than in one, and Space bytes for a deployment whose records live in a + * database and therefore has no folder at all. + * + * Both are under `/storage/disk/`, which names the backend the way + * `storage/sqlite/` names the other one. Two adapters share that directory, so + * each gets a subtree of its own and neither may grow into the other's: + * + * storage/disk/ + * workspaces.json the *structured* store's membership registry + * blobs//… the *blob* store's Space byte roots + * + * That separation is not cosmetic. The blob store deletes whole directories — + * an area on `deleteAll()`, a Space's root when its record goes — and the + * registry is not its to delete. Keeping the registry out of `blobs/` is what + * makes "sweep this Space's bytes" unable to reach it, and putting both paths + * in one file is what keeps that true when either moves. + * + * Nothing outside `storage/` may depend on these names (§12.5.2). The SQLite + * backend answers the same question for itself in `backends/sqlite/database.ts`. + */ + +import path from 'node:path'; + +import { getDataDir } from '../../../../data-dir.js'; +import { sanitizeId } from '../../../../utils/fs.js'; + +/** The Disk backend's own area in the Server data directory. */ +export function diskDataDir(dataDir: string = getDataDir()): string { + return path.join(dataDir, 'storage', 'disk'); +} + +export const WORKSPACE_REGISTRY_FILENAME = 'workspaces.json'; + +/** Structured store: the `workspaceId -> workspacePath` discovery index. */ +export function workspaceRegistryPath(dataDir: string = getDataDir()): string { + return path.join(diskDataDir(dataDir), WORKSPACE_REGISTRY_FILENAME); +} + +/** + * Blob store: the root its Space byte directories sit under. + * + * Only reached when the structured backend gives a Space no folder of its own; + * where Disk keeps the records too, a Space's bytes stay inside the Space + * folder the user can see and this path is never built. + * + * `HUABU_BLOB_ROOT` replaces it wholesale, for a deployment that keeps bytes + * on another volume. It moves the bytes and nothing else — the registry above + * is the structured store's and stays where it is. + */ +export function diskBlobRoot(dataDir: string = getDataDir()): string { + const configured = process.env['HUABU_BLOB_ROOT']?.trim(); + return configured ? configured : path.join(diskDataDir(dataDir), 'blobs'); +} + +/** + * Blob store: where one Space's areas go, Workspace-scoped. + * + * A Space belongs to exactly one Workspace, so its bytes are filed under that + * Workspace and removed with it. The directory holds bytes and nothing else: + * it is not a Workspace folder and not a Space tree, which is why none of the + * Disk-only capabilities become available because it exists. + */ +export function diskSpaceBlobRoot( + workspaceId: string, + canvasId: string, + dataDir: string = getDataDir(), +): string { + return path.join( + diskBlobRoot(dataDir), + sanitizeId(workspaceId, 'workspaceId'), + sanitizeId(canvasId, 'canvasId'), + ); +} diff --git a/apps/server/src/modules/storage/backends/disk/layout.ts b/apps/server/src/modules/storage/backends/disk/layout.ts index 9201c4505..a6ea321a7 100644 --- a/apps/server/src/modules/storage/backends/disk/layout.ts +++ b/apps/server/src/modules/storage/backends/disk/layout.ts @@ -83,23 +83,17 @@ export function artifactsDir(canvasId: string): string { /** * Hidden directory holding the agent's private memory document. * - * Named here rather than in the workspace module because it is now a blob - * scope's placement — where Disk puts the bytes of one user-visible area — - * and every other such placement already lives beside this one. + * Named here rather than in the workspace module because it is a blob scope's + * placement — what Disk calls the folder holding one user-visible area — and + * every other such placement already lives beside this one. The blob adapter + * joins these names onto whichever Space root it was given, so they are + * constants rather than resolvers. */ export const MEMORY_DIR_NAME = '.memory'; -export function spaceMemoryDir(canvasId: string): string { - return path.join(canvasRoot(canvasId), MEMORY_DIR_NAME); -} - /** Hidden scratch an upload lands in before anything claims it. */ export const UPLOAD_DIR_NAME = '.upload'; -export function spaceUploadDir(canvasId: string): string { - return path.join(canvasRoot(canvasId), UPLOAD_DIR_NAME); -} - export function artifactPath(canvasId: string, filename: string): string { const base = path.basename(filename); if (!base || base === '.' || base === '..') { diff --git a/apps/server/src/modules/storage/backends/disk/space-nodes.test.ts b/apps/server/src/modules/storage/backends/disk/space-nodes.test.ts index 19cc23b7c..ebf7d6760 100644 --- a/apps/server/src/modules/storage/backends/disk/space-nodes.test.ts +++ b/apps/server/src/modules/storage/backends/disk/space-nodes.test.ts @@ -54,10 +54,12 @@ describeSpaceNodesContract('Disk', async () => { if (!created.ok) throw new Error('Node contract Space already exists'); const store = new DiskStructuredStore(); + const space = store.space('node-space'); return { - repository: store.space('node-space').nodes, + repository: space.nodes, missingRepository: store.space('missing-node-space').nodes, expectedCanvasId: 'node-space', + deletedNodePut: 'write-suppressed', cleanup: () => { vi.restoreAllMocks(); resetStorageCache(); diff --git a/apps/server/src/modules/storage/backends/disk/space-record-validation.ts b/apps/server/src/modules/storage/backends/disk/space-record-validation.ts index 1792458c4..e31bb697c 100644 --- a/apps/server/src/modules/storage/backends/disk/space-record-validation.ts +++ b/apps/server/src/modules/storage/backends/disk/space-record-validation.ts @@ -1,55 +1,14 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -/** Runtime validation shared by strict Disk Space-record boundaries. */ +/** Runtime validation and strict reads for Disk Space-record boundaries. */ import { readJsonStrict } from '../../../../utils/fs.js'; +import { canvasFileShapeError } from '../../../canvas/persistence-validation.js'; import type { CanvasFile } from '../../../canvas/persistence-types.js'; -function finiteNumber(value: unknown): boolean { - return typeof value === 'number' && Number.isFinite(value); -} - -/** Return the first minimal {@link CanvasFile} shape violation, if any. */ -export function canvasFileShapeError( - value: unknown, - expectedCanvasId: string, -): string | null { - if (typeof value !== 'object' || value === null || Array.isArray(value)) { - return 'must be an object'; - } - - const record = value as Record; - if (record['canvasId'] !== expectedCanvasId) { - return `canvasId must equal ${JSON.stringify(expectedCanvasId)}`; - } - if (record['title'] !== null && typeof record['title'] !== 'string') { - return 'title must be a string or null'; - } - if (!finiteNumber(record['version'])) { - return 'version must be a finite number'; - } - if (!finiteNumber(record['createdAt'])) { - return 'createdAt must be a finite number'; - } - if (!finiteNumber(record['updatedAt'])) { - return 'updatedAt must be a finite number'; - } - - const state = record['state']; - if (typeof state !== 'object' || state === null || Array.isArray(state)) { - return 'state must be an object'; - } - const stateRecord = state as Record; - if (!Array.isArray(stateRecord['nodes'])) { - return 'state.nodes must be an array'; - } - if (!Array.isArray(stateRecord['edges'])) { - return 'state.edges must be an array'; - } - return null; -} +export { canvasFileShapeError } from '../../../canvas/persistence-validation.js'; /** * Strictly read and validate one indexed `space.json` path. diff --git a/apps/server/src/modules/storage/backends/disk/space-repository.test.ts b/apps/server/src/modules/storage/backends/disk/space-repository.test.ts index 250a61022..033cbb0b6 100644 --- a/apps/server/src/modules/storage/backends/disk/space-repository.test.ts +++ b/apps/server/src/modules/storage/backends/disk/space-repository.test.ts @@ -141,9 +141,12 @@ describeSpaceExtensionContract('Disk', () => { // An owner of a Disk namespace writes files into its directory; nothing // about the shape is storage's business, so the suite borrows the // simplest one an owner could pick. - write: (substrate, value) => - writeFileSync(path.join(substrate.directory, 'value'), value, 'utf8'), + write: (substrate, value) => { + if (substrate.kind !== 'disk') throw new Error('Expected Disk substrate'); + writeFileSync(path.join(substrate.directory, 'value'), value, 'utf8'); + }, read: (substrate) => { + if (substrate.kind !== 'disk') throw new Error('Expected Disk substrate'); const file = path.join(substrate.directory, 'value'); return existsSync(file) ? readFileSync(file, 'utf8') : null; }, diff --git a/apps/server/src/modules/storage/backends/disk/storage-recovery.test.ts b/apps/server/src/modules/storage/backends/disk/storage-recovery.test.ts index 47858cbd2..1022b4b21 100644 --- a/apps/server/src/modules/storage/backends/disk/storage-recovery.test.ts +++ b/apps/server/src/modules/storage/backends/disk/storage-recovery.test.ts @@ -23,6 +23,7 @@ const workspaceState = vi.hoisted(() => ({ path: '' })); vi.mock('../../../workspace.js', () => ({ getWorkspacePath: () => workspaceState.path, + getWorkspaceKey: () => workspaceState.path, })); import { refreshCanvasDirIndex } from './canvas-dirs.js'; diff --git a/apps/server/src/modules/storage/backends/disk/structured-store.test.ts b/apps/server/src/modules/storage/backends/disk/structured-store.test.ts index 0f3dadd8d..ef0ad0772 100644 --- a/apps/server/src/modules/storage/backends/disk/structured-store.test.ts +++ b/apps/server/src/modules/storage/backends/disk/structured-store.test.ts @@ -28,6 +28,7 @@ import { import { DiskStructuredStore } from './structured-store.js'; import { toSafeFilename } from '../../../../utils/naming.js'; import { describeSpaceLogsContract } from '../../ports/contracts/space-logs.contract.js'; +import { describeSpaceTasksContract } from '../../ports/contracts/space-tasks.contract.js'; import { describeStructuredStoreContract } from '../../ports/contracts/structured-store.contract.js'; import type { CanvasFile } from '../../../canvas/persistence-types.js'; @@ -111,6 +112,8 @@ describe('Disk Space extension workspace binding', () => { try { const substrate = await pending; expect(substrate?.kind).toBe('disk'); + if (substrate?.kind !== 'disk') + throw new Error('Expected Disk substrate'); expect(substrate?.directory.startsWith(`${firstRoot}${path.sep}`)).toBe( true, ); @@ -125,6 +128,30 @@ describe('Disk Space extension workspace binding', () => { }); }); +describeSpaceTasksContract('Disk', () => { + const root = freshWorkspace('huabu-task-contract-'); + seedSpace(root, 'canvas-task', 'Canvas Task'); + const store = new DiskStructuredStore(); + return { + tasks: store.space('canvas-task').tasks, + concurrent: store.space('canvas-task').tasks, + canvasId: 'canvas-task', + missing: store.space('missing-canvas').tasks, + missingCanvasId: 'missing-canvas', + beginDelete: async () => { + const result = await store.spaces().beginDelete({ + canvasId: 'canvas-task', + }); + if (!result.ok) throw new Error('Ordinary Space must be deletable'); + return result.session; + }, + cleanup: () => { + resetStorageCache(); + rmSync(root, { recursive: true, force: true }); + }, + }; +}); + describe('Disk Space Tasks', () => { let root = ''; let store: DiskStructuredStore; @@ -141,132 +168,8 @@ describe('Disk Space Tasks', () => { rmSync(root, { recursive: true, force: true }); }); - it('serializes Task and Run mutations across independent handles', async () => { - const first = store.space('canvas-task').tasks; - const second = store.space('canvas-task').tasks; - await Promise.all([ - first.create({ - taskId: 'task-a', - canvasId: 'canvas-task', - goal: 'Goal A', - defaultRootProfileId: 'profile-a', - anchorNodeId: 'node-a', - createdAt: 1, - }), - second.create({ - taskId: 'task-b', - canvasId: 'canvas-task', - goal: 'Goal B', - defaultRootProfileId: 'profile-b', - anchorNodeId: 'node-b', - createdAt: 2, - }), - ]); - await first.runs.create({ - runId: 'run-a', - taskId: 'task-a', - canvasIdSnapshot: 'canvas-task', - goalSnapshot: 'Goal A', - rootProfileIdSnapshot: 'profile-a', - status: 'pending', - createdAt: 3, - }); - const updated = await second.runs.update('run-a', { - rootNodeId: 'node-root', - rootThreadId: 'thread-root', - status: 'running', - startedAt: 4, - }); - - expect(updated.status).toBe('running'); - await expect(first.read()).resolves.toMatchObject({ - version: 1, - tasks: [ - expect.objectContaining({ taskId: 'task-a' }), - expect.objectContaining({ taskId: 'task-b' }), - ], - runs: [ - expect.objectContaining({ - runId: 'run-a', - rootNodeId: 'node-root', - rootThreadId: 'thread-root', - }), - ], - }); - }); - - it('returns an empty versioned snapshot when no Task store exists', async () => { - await expect(store.space('canvas-empty').tasks.read()).resolves.toEqual({ - version: 1, - tasks: [], - runs: [], - }); - }); - - it('completes a running Run atomically and keeps its message immutable', async () => { - const runs = store.space('canvas-task').tasks.runs; - await expect( - runs.complete('task-a', 'run-a', { - completedAt: 5, - message: 'PR merged', - }), - ).resolves.toMatchObject({ - outcome: 'completed', - run: { - status: 'completed', - completion: { completedAt: 5, message: 'PR merged' }, - }, - }); - await expect( - runs.complete('task-a', 'run-a', { - completedAt: 6, - message: 'PR merged', - }), - ).resolves.toMatchObject({ - outcome: 'unchanged', - run: { completion: { completedAt: 5, message: 'PR merged' } }, - }); - await expect( - runs.complete('task-a', 'run-a', { - completedAt: 7, - message: 'Different result', - }), - ).resolves.toMatchObject({ outcome: 'completion_conflict' }); - - await runs.create({ - runId: 'run-pending', - taskId: 'task-b', - canvasIdSnapshot: 'canvas-task', - goalSnapshot: 'Goal B', - rootProfileIdSnapshot: 'profile-b', - status: 'pending', - createdAt: 8, - }); - await expect( - runs.complete('task-b', 'run-pending', { completedAt: 9 }), - ).resolves.toMatchObject({ outcome: 'run_not_running' }); - await expect( - runs.complete('missing-task', 'run-a', { completedAt: 9 }), - ).resolves.toEqual({ outcome: 'task_not_found' }); - await expect( - runs.complete('task-a', 'missing-run', { completedAt: 9 }), - ).resolves.toEqual({ outcome: 'run_not_found' }); - }); - - it('rejects mutations for a missing Space', async () => { - await expect( - store.space('missing-canvas').tasks.create({ - taskId: 'task-missing', - canvasId: 'missing-canvas', - goal: 'Missing', - defaultRootProfileId: 'profile-a', - anchorNodeId: 'node-missing', - createdAt: 1, - }), - ).rejects.toThrow(/cannot write a missing Space/); - }); - it('fails fast on malformed and internally inconsistent Task stores', async () => { + mkdirSync(path.dirname(tasksPath('canvas-task')), { recursive: true }); writeFileSync(tasksPath('canvas-task'), '{"version":1,"tasks":{}}'); await expect(store.space('canvas-task').tasks.read()).rejects.toThrow( /Invalid Task store/, diff --git a/apps/server/src/modules/storage/backends/disk/workspace-repository.test.ts b/apps/server/src/modules/storage/backends/disk/workspace-repository.test.ts index 7fae730b9..f1260aaea 100644 --- a/apps/server/src/modules/storage/backends/disk/workspace-repository.test.ts +++ b/apps/server/src/modules/storage/backends/disk/workspace-repository.test.ts @@ -13,10 +13,10 @@ import { import { tmpdir } from 'node:os'; import path from 'node:path'; +import { workspaceRegistryPath as registryPath } from './data-dir.js'; import { DiskWorkspaceRepository, WORKSPACE_MANIFEST_FILENAME, - WORKSPACE_REGISTRY_FILENAME, } from './workspace-repository.js'; import { describeWorkspaceRepositoryContract } from '../../ports/contracts/workspace-repository.contract.js'; import { adoptWorkspaceDirectory } from '../../storage.js'; @@ -34,10 +34,6 @@ describe('DiskWorkspaceRepository', () => { return path.join(root, WORKSPACE_MANIFEST_FILENAME); } - function registryPath(dataDir: string): string { - return path.join(dataDir, 'storage', 'disk', WORKSPACE_REGISTRY_FILENAME); - } - afterAll(() => { for (const root of roots) { rmSync(root, { recursive: true, force: true }); diff --git a/apps/server/src/modules/storage/backends/disk/workspace-repository.ts b/apps/server/src/modules/storage/backends/disk/workspace-repository.ts index 90f65fa9e..745f0e17c 100644 --- a/apps/server/src/modules/storage/backends/disk/workspace-repository.ts +++ b/apps/server/src/modules/storage/backends/disk/workspace-repository.ts @@ -12,6 +12,7 @@ * * The Server data directory holds a separate discovery index containing * `workspaceId -> workspacePath` plus the last time that Workspace was opened. + * Where that file sits is `data-dir.ts`'s to say, not this adapter's. * Array order has no meaning: listings sort by the explicit timestamp, and * adopting/activating a Workspace updates its timestamp in place. That * deliberate duplication is the minimum needed to recognize an externally @@ -48,7 +49,6 @@ import type { } from '../../ports/workspace.js'; export const WORKSPACE_MANIFEST_FILENAME = '.workspace.json'; -export const WORKSPACE_REGISTRY_FILENAME = 'workspaces.json'; const WORKSPACE_MANIFEST_SCHEMA_VERSION = 1; const WORKSPACE_REGISTRY_SCHEMA_VERSION = 1; @@ -85,11 +85,6 @@ type WorkspaceRegistryEntry = z.infer< typeof workspaceRegistrySchema >['workspaces'][number]; -/** Where the Disk backend keeps its discovery index inside the data dir. */ -export function workspaceRegistryPath(dataDir: string): string { - return path.join(dataDir, 'storage', 'disk', WORKSPACE_REGISTRY_FILENAME); -} - function manifestPath(workspacePath: string): string { return path.join(workspacePath, WORKSPACE_MANIFEST_FILENAME); } diff --git a/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts b/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts new file mode 100644 index 000000000..a35d2ecc0 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts @@ -0,0 +1,219 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { SqliteStoreContext } from './database.js'; +import { + createSqliteTestFile, + installDeltaAbortTrigger, + openEmptySqliteTestStore, + openSqliteTestStore, + readSqliteDeltaLog, +} from './test-support.js'; +import { SqliteWorkspaceRepository } from './workspace-repository.js'; +import { describeSpaceExtensionContract } from '../../ports/contracts/space-extension.contract.js'; +import { describeSpaceLogsContract } from '../../ports/contracts/space-logs.contract.js'; +import { describeSpaceNodesContract } from '../../ports/contracts/space-nodes.contract.js'; +import { describeSpaceRepositoryContract } from '../../ports/contracts/space-repository.contract.js'; +import { describeSpaceTasksContract } from '../../ports/contracts/space-tasks.contract.js'; +import { describeSpaceWriteContract } from '../../ports/contracts/space-write.contract.js'; +import { describeStructuredStoreContract } from '../../ports/contracts/structured-store.contract.js'; +import { describeWorkspaceRepositoryContract } from '../../ports/contracts/workspace-repository.contract.js'; + +import type { SqliteStructuredStore } from './structured-store.js'; +import type { NodeContent } from '../../../canvas/persistence-types.js'; + +function note(nodeId: string, label: string, content: string): NodeContent { + return { nodeId, type: 'note', label, content }; +} + +async function createOrdinarySpace( + store: SqliteStructuredStore, + canvasId: string, + title: string, +): Promise { + const created = await store.spaces().create({ canvasId, title }); + if (!created.ok) throw new Error(`Could not create test Space ${canvasId}`); +} + +describeStructuredStoreContract('SQLite', async () => { + // Through the same lifecycle a Server uses: open the connection, then select + // a Workspace. A handle resolved before one is active has no namespace to + // address, which is the SQL twin of the Disk adapter refusing before a + // workspace path is committed. + const harness = await openEmptySqliteTestStore( + 'huabu-sqlite-structured-contract-', + ); + return { store: harness.store, cleanup: harness.cleanup }; +}); + +describeSpaceRepositoryContract('SQLite', async () => { + const harness = await openSqliteTestStore( + 'huabu-sqlite-space-repository-contract-', + ); + const emptyStores: Array< + Awaited> + > = []; + return { + repository: harness.store.spaces(), + read: (canvasId: string) => harness.store.space(canvasId).read(), + worldCanvasId: harness.world.canvasId, + attemptMutation: (canvasId: string) => + harness.store.space(canvasId).nodes.put({ + nodeId: 'contract-delete-fence-node', + record: note( + 'contract-delete-fence-node', + 'Deletion fence node', + 'body', + ), + }), + openEmptyNamespace: async () => { + const empty = await openEmptySqliteTestStore( + 'huabu-sqlite-empty-namespace-contract-', + ); + emptyStores.push(empty); + return { + repository: empty.store.spaces(), + read: (canvasId: string) => empty.store.space(canvasId).read(), + }; + }, + cleanup: async () => { + for (const empty of emptyStores.splice(0)) await empty.cleanup(); + await harness.cleanup(); + }, + }; +}); + +describeSpaceNodesContract('SQLite', async () => { + const harness = await openSqliteTestStore('huabu-sqlite-nodes-contract-'); + const canvasId = 'sqlite-nodes-contract'; + await createOrdinarySpace(harness.store, canvasId, 'SQLite Nodes Contract'); + const space = harness.store.space(canvasId); + return { + repository: space.nodes, + missingRepository: harness.store.space('sqlite-nodes-missing').nodes, + expectedCanvasId: canvasId, + deletedNodePut: 'allowed', + cleanup: harness.cleanup, + }; +}); + +describeSpaceExtensionContract('SQLite', async () => { + const harness = await openSqliteTestStore('huabu-sqlite-extension-contract-'); + const table = 'contract_extension_values'; + return { + repository: harness.store.spaces(), + space: (canvasId: string) => harness.store.space(canvasId), + write: (substrate, value: string) => { + if (substrate.kind !== 'sqlite') { + throw new Error('Expected a SQLite substrate'); + } + substrate.database.exec( + `CREATE TABLE IF NOT EXISTS ${table} ( + extension_id INTEGER PRIMARY KEY, + value TEXT NOT NULL, + FOREIGN KEY (extension_id) REFERENCES space_extensions(extension_id) + ON DELETE CASCADE + ) STRICT`, + ); + substrate.database + .prepare( + `INSERT INTO ${table} (extension_id, value) VALUES (?, ?) + ON CONFLICT(extension_id) DO UPDATE SET value = excluded.value`, + ) + .run(substrate.extensionId, value); + }, + read: (substrate) => { + if (substrate.kind !== 'sqlite') { + throw new Error('Expected a SQLite substrate'); + } + const row = substrate.database + .prepare(`SELECT value FROM ${table} WHERE extension_id = ?`) + .get(substrate.extensionId); + return typeof row?.['value'] === 'string' ? row['value'] : null; + }, + cleanup: harness.cleanup, + }; +}); + +describeSpaceWriteContract('SQLite', async () => { + const harness = await openSqliteTestStore('huabu-sqlite-write-contract-'); + const canvasId = 'sqlite-write-contract'; + await createOrdinarySpace(harness.store, canvasId, 'SQLite Write Contract'); + const existingNode = note( + 'contract-existing-node', + 'Existing contract node', + 'before', + ); + const space = harness.store.space(canvasId); + const put = await space.nodes.put({ + nodeId: existingNode.nodeId, + record: existingNode, + }); + if (!put.ok) { + throw new Error(`Could not seed SQLite write contract: ${put.reason}`); + } + + return { + space, + concurrent: harness.store.space(canvasId), + missing: harness.store.space('sqlite-write-missing'), + existingNode, + newNode: note('contract-new-node', 'New contract node', 'after'), + readJournal: async () => readSqliteDeltaLog(harness.filename, canvasId), + failNextDeltaAppend: (error: Error) => + installDeltaAbortTrigger(harness.filename, error.message), + cleanup: harness.cleanup, + }; +}); + +describeSpaceLogsContract('SQLite', async () => { + const harness = await openSqliteTestStore('huabu-sqlite-logs-contract-'); + const canvasId = 'sqlite-logs-contract'; + await createOrdinarySpace(harness.store, canvasId, 'SQLite Logs Contract'); + const first = harness.store.space(canvasId); + const second = harness.store.space(canvasId); + return { + events: first.events, + changes: first.changes, + concurrent: { + events: second.events, + changes: second.changes, + }, + cleanup: harness.cleanup, + }; +}); + +describeSpaceTasksContract('SQLite', async () => { + const harness = await openSqliteTestStore('huabu-sqlite-tasks-contract-'); + const canvasId = 'sqlite-tasks-contract'; + const missingCanvasId = 'sqlite-tasks-missing'; + await createOrdinarySpace(harness.store, canvasId, 'SQLite Tasks Contract'); + return { + tasks: harness.store.space(canvasId).tasks, + concurrent: harness.store.space(canvasId).tasks, + canvasId, + missing: harness.store.space(missingCanvasId).tasks, + missingCanvasId, + beginDelete: async () => { + const result = await harness.store.spaces().beginDelete({ canvasId }); + if (!result.ok) throw new Error('Ordinary Space must be deletable'); + return result.session; + }, + cleanup: harness.cleanup, + }; +}); + +describeWorkspaceRepositoryContract('SQLite', async () => { + const file = createSqliteTestFile('huabu-sqlite-workspace-contract-'); + const context = new SqliteStoreContext(file.filename); + context.init(); + const repository = new SqliteWorkspaceRepository(context); + return { + repository, + create: (name: string) => repository.create(name), + cleanup: () => { + context.close(); + file.remove(); + }, + }; +}); diff --git a/apps/server/src/modules/storage/backends/sqlite/database.ts b/apps/server/src/modules/storage/backends/sqlite/database.ts new file mode 100644 index 000000000..5740be4b4 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/database.ts @@ -0,0 +1,311 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * The one SQLite connection a process holds, and the state that lives as long + * as it does. + * + * The structured store and the Workspace repository share this object. That is + * not a convenience: they are one database file, so two connections would be + * two writers to it, and SQLite's answer to that is a lock error rather than a + * queue. One connection also makes the ordered Space write a real transaction + * across everything it touches. + * + * The active Workspace is held here for the same reason the Disk adapters hold + * the active workspace path: it is the namespace every query is scoped to. + * Switching Workspaces re-points this field and reopens nothing — the settled + * "Backend selection scope" decision in proposal §2. + */ + +import { mkdirSync } from 'node:fs'; +import path from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; + +import { SQLITE_MIGRATIONS, type SqliteMigration } from './schema.js'; +import { getDataDir } from '../../../../data-dir.js'; +import { + assertSpaceMutationAllowed, + beginSpaceDeleteAdmission, +} from '../../space-lifecycle-admission.js'; + +import type { StorageHealth } from '../../ports/common.js'; + +export { SQLITE_MIGRATIONS, SQLITE_SCHEMA_VERSION } from './schema.js'; +export type { SqliteMigration } from './schema.js'; + +/** + * The collision key the hidden World Space is filed under. + * + * Unreachable from any user title: `toSafeFilename` strips leading dots, so + * no requested name normalizes to it and the World slot cannot be taken by + * an ordinary Space. + */ +export const SQLITE_WORLD_COLLISION_KEY = '.world'; + +/** Milliseconds a statement waits for a lock before reporting SQLITE_BUSY. */ +const BUSY_TIMEOUT_MS = 5_000; + +/** + * Where this backend keeps its records in the Server data directory. + * + * One file, under a directory named for the backend the way + * `storage/disk/` names the other one, so an operator finds both in the same + * place. `HUABU_SQLITE_PATH` replaces it for a deployment that keeps its + * database elsewhere. + * + * Records only. A Space's *bytes* are the blob axis's business wherever this + * backend is selected, and this backend never learns where they went — see + * `backends/disk/data-dir.ts`. + */ +export function sqliteDatabasePath(dataDir: string = getDataDir()): string { + const configured = process.env['HUABU_SQLITE_PATH']?.trim(); + if (configured) return configured; + return path.join(dataDir, 'storage', 'sqlite', 'huabu.sqlite'); +} + +function readUserVersion(database: DatabaseSync): number { + const row = database.prepare('PRAGMA user_version').get(); + const version = row?.['user_version']; + if (typeof version !== 'number' || !Number.isSafeInteger(version)) { + throw new Error('SQLite returned an invalid PRAGMA user_version'); + } + return version; +} + +export function applySqliteMigrations( + database: DatabaseSync, + migrations: readonly SqliteMigration[] = SQLITE_MIGRATIONS, +): void { + for (let index = 0; index < migrations.length; index += 1) { + const expectedVersion = index + 1; + if (migrations[index]?.version !== expectedVersion) { + throw new Error( + `SQLite migrations must be contiguous from version 1; expected ${expectedVersion}`, + ); + } + } + const targetVersion = migrations.at(-1)?.version ?? 0; + const current = readUserVersion(database); + if (current > targetVersion) { + throw new Error( + `SQLite schema version ${current} is newer than supported version ${targetVersion}`, + ); + } + if (current === targetVersion) return; + + database.exec('BEGIN IMMEDIATE'); + try { + let version = readUserVersion(database); + for (const migration of migrations) { + if (migration.version <= version) continue; + if (migration.version !== version + 1) { + throw new Error( + `No SQLite migration path from schema version ${version} to ${targetVersion}`, + ); + } + database.exec(migration.sql); + database.exec(`PRAGMA user_version = ${migration.version}`); + version = migration.version; + } + if (version !== targetVersion) { + throw new Error( + `No SQLite migration path from schema version ${version} to ${targetVersion}`, + ); + } + database.exec('COMMIT'); + } catch (error) { + if (database.isTransaction) database.exec('ROLLBACK'); + throw error; + } +} + +/** Raised when a handle outlives the Workspace it was resolved in. */ +export class SqliteWorkspaceScopeError extends Error { + override name = 'SqliteWorkspaceScopeError'; +} + +/** One connection and all adapter-lifetime process-local state. */ +export class SqliteStoreContext { + readonly now: () => number; + + readonly #database: DatabaseSync; + readonly #filename: string; + readonly #admissionScope: string; + #state: 'new' | 'open' | 'closed' = 'new'; + #workspaceId: string | null = null; + + constructor(filename: string, now: () => number = Date.now) { + if (typeof filename !== 'string' || filename.length === 0) { + throw new TypeError('SQLite filename must be a non-empty string'); + } + this.now = now; + this.#filename = filename; + this.#admissionScope = `sqlite:${filename}`; + this.#database = new DatabaseSync(filename, { open: false }); + } + + get filename(): string { + return this.#filename; + } + + init(): void { + if (this.#state === 'open') return; + if (this.#state === 'closed') { + throw new Error('SQLite store is closed'); + } + + try { + // A database file names a directory that may not exist yet — the whole + // point of this profile is that the operator never had to create one. + // In-memory and URI filenames name no directory at all. + const directory = path.dirname(this.#filename); + if ( + !this.#filename.startsWith(':') && + !this.#filename.startsWith('file:') + ) { + mkdirSync(directory, { recursive: true }); + } + this.#database.open(); + // Write-ahead logging so a reader is never blocked by the writer, and a + // bounded wait so a second connection (an external tool, a stale + // process) reports a busy database instead of failing instantly. + this.#database.exec(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS}`); + this.#database.exec('PRAGMA journal_mode = WAL'); + // NORMAL is the documented pairing for WAL: durable across a process + // crash, and only a machine-level crash can lose the most recent + // commits — which is the same guarantee the Disk adapter's atomic + // renames give, stated rather than assumed. + this.#database.exec('PRAGMA synchronous = NORMAL'); + this.#database.exec('PRAGMA foreign_keys = ON'); + const foreignKeys = this.#database.prepare('PRAGMA foreign_keys').get()?.[ + 'foreign_keys' + ]; + if (foreignKeys !== 1) { + throw new Error('Could not enable SQLite foreign key enforcement'); + } + applySqliteMigrations(this.#database); + this.#state = 'open'; + } catch (error) { + if (this.#database.isOpen) this.#database.close(); + this.#state = 'closed'; + throw error; + } + } + + health(kind: string): StorageHealth { + this.assertOpen(); + try { + const value = this.#database.prepare('SELECT 1 AS ok').get()?.['ok']; + return value === 1 + ? { ok: true, kind } + : { ok: false, kind, detail: 'SQLite liveness query returned no row' }; + } catch (error) { + return { + ok: false, + kind, + detail: error instanceof Error ? error.message : String(error), + }; + } + } + + close(): void { + if (this.#state === 'closed') return; + this.#state = 'closed'; + if (this.#database.isOpen) this.#database.close(); + } + + database(): DatabaseSync { + this.assertOpen(); + return this.#database; + } + + assertOpen(): void { + if (this.#state !== 'open') { + throw new Error( + this.#state === 'closed' + ? 'SQLite store is closed' + : 'SQLite store is not initialized', + ); + } + } + + // ─── The active Workspace ──────────────────────────────────────────────── + + /** Point every subsequent query at one Workspace. Reopens nothing. */ + useWorkspace(workspaceId: string | null): void { + this.#workspaceId = workspaceId; + } + + /** The active Workspace id, or `null` when none has been selected. */ + activeWorkspaceId(): string | null { + return this.#workspaceId; + } + + /** The active Workspace id, or a refusal when none has been selected. */ + workspaceId(): string { + this.assertOpen(); + if (this.#workspaceId === null) { + throw new SqliteWorkspaceScopeError( + 'No Workspace is active on the SQLite backend. Activate one before ' + + 'reading or writing Spaces.', + ); + } + return this.#workspaceId; + } + + /** + * The Workspace a retained handle was resolved in, or a refusal. + * + * A handle keeps the id it was built with and re-checks it here, so a + * Workspace switch makes the stale handle reject rather than silently + * addressing rows in the newly active namespace. That is the same rule the + * Disk adapters apply to a retained workspace path. + */ + assertBoundWorkspace(boundWorkspaceId: string, what: string): string { + const active = this.workspaceId(); + if (active !== boundWorkspaceId) { + throw new SqliteWorkspaceScopeError( + `${what} belongs to an inactive Workspace. Resolve a fresh handle ` + + 'after Workspace activation.', + ); + } + return active; + } + + // ─── Space lifecycle admission ─────────────────────────────────────────── + + assertMutationAllowed(canvasId: string): void { + this.assertOpen(); + assertSpaceMutationAllowed(this.#admissionScope, canvasId); + } + + async acquireDelete(canvasId: string): Promise<() => void> { + this.assertOpen(); + const releaseGate = await beginSpaceDeleteAdmission( + this.#admissionScope, + canvasId, + ); + try { + this.assertOpen(); + } catch (error) { + releaseGate(); + throw error; + } + return releaseGate; + } +} + +export function withImmediateTransaction( + database: DatabaseSync, + operation: () => T, +): T { + database.exec('BEGIN IMMEDIATE'); + try { + const result = operation(); + database.exec('COMMIT'); + return result; + } catch (error) { + if (database.isTransaction) database.exec('ROLLBACK'); + throw error; + } +} diff --git a/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql b/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql new file mode 100644 index 000000000..adcaa7ec1 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql @@ -0,0 +1,137 @@ +-- Immutable SQLite storage schema v1 fixture. +-- +-- Hand-written to match `schema.ts`'s version 1 exactly, and never rewritten +-- once a version ships: the point of the fixture is to prove that opening an +-- existing database migrates and reads it rather than reshaping it. A later +-- schema version gets its own fixture beside this one. + +PRAGMA foreign_keys = ON; +BEGIN IMMEDIATE; + +CREATE TABLE workspaces ( + workspace_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + created_at REAL NOT NULL, + last_opened_at REAL NOT NULL, + forgotten_at REAL +) STRICT; + +CREATE TABLE spaces ( + canvas_id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + title TEXT, + collision_key TEXT NOT NULL, + version INTEGER NOT NULL, + state_json TEXT NOT NULL CHECK (json_valid(state_json)), + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + is_world INTEGER NOT NULL DEFAULT 0 CHECK (is_world IN (0, 1)), + UNIQUE (workspace_id, collision_key), + FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) + ON DELETE CASCADE +) STRICT; + +CREATE UNIQUE INDEX spaces_single_world + ON spaces(workspace_id) + WHERE is_world = 1; + +CREATE TABLE nodes ( + canvas_id TEXT NOT NULL, + node_id TEXT NOT NULL, + record_json TEXT NOT NULL CHECK (json_valid(record_json)), + revision TEXT NOT NULL CHECK (length(revision) > 0), + label_collision_key TEXT NOT NULL, + PRIMARY KEY (canvas_id, node_id), + UNIQUE (canvas_id, label_collision_key), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE +) STRICT; + +CREATE TABLE events ( + event_id INTEGER PRIMARY KEY AUTOINCREMENT, + canvas_id TEXT NOT NULL, + event_json TEXT NOT NULL CHECK (json_valid(event_json)), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE +) STRICT; + +CREATE INDEX events_by_canvas_order + ON events(canvas_id, event_id); + +CREATE TABLE changes ( + canvas_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + snapshot_json TEXT NOT NULL CHECK (json_valid(snapshot_json)), + PRIMARY KEY (canvas_id, thread_id), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE +) STRICT; + +CREATE TABLE tasks ( + canvas_id TEXT PRIMARY KEY, + snapshot_json TEXT NOT NULL CHECK (json_valid(snapshot_json)), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE +) STRICT; + +CREATE TABLE space_extensions ( + extension_id INTEGER PRIMARY KEY AUTOINCREMENT, + canvas_id TEXT NOT NULL, + namespace TEXT NOT NULL, + UNIQUE (canvas_id, namespace), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE +) STRICT; + +CREATE TABLE delta_log ( + canvas_id TEXT NOT NULL, + version INTEGER NOT NULL, + entry_json TEXT NOT NULL CHECK (json_valid(entry_json)), + PRIMARY KEY (canvas_id, version), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE +) STRICT; + +INSERT INTO workspaces ( + workspace_id, name, created_at, last_opened_at, forgotten_at +) VALUES ('fixture-workspace', 'Fixture Workspace', 1, 1, NULL); + +INSERT INTO spaces ( + canvas_id, workspace_id, title, collision_key, version, state_json, + created_at, updated_at, is_world +) VALUES ( + 'fixture-world', 'fixture-workspace', 'World', '.world', 0, + '{"nodes":[],"edges":[]}', 1, 1, 1 +); + +INSERT INTO spaces ( + canvas_id, workspace_id, title, collision_key, version, state_json, + created_at, updated_at, is_world +) VALUES ( + 'fixture-space', 'fixture-workspace', 'Fixture Space', 'fixture space', 3, + '{"nodes":[{"id":"fixture-node","type":"note"}],"edges":[]}', + 10, 13, 0 +); + +INSERT INTO nodes ( + canvas_id, node_id, record_json, revision, label_collision_key +) VALUES ( + 'fixture-space', 'fixture-node', + '{"nodeId":"fixture-node","type":"note","label":"Fixture Node","content":"fixture body"}', + 'fixture-revision', 'fixture node' +); + +INSERT INTO events (canvas_id, event_json) VALUES ( + 'fixture-space', + '{"payload":{"action":"node_selected","node":{"id":"fixture-node","type":"note","label":"Fixture Node"}},"ts":12}' +); + +INSERT INTO changes (canvas_id, thread_id, snapshot_json) VALUES ( + 'fixture-space', 'fixture-thread', '[]' +); + +INSERT INTO tasks (canvas_id, snapshot_json) VALUES ( + 'fixture-space', '{"version":1,"tasks":[],"runs":[]}' +); + +INSERT INTO delta_log (canvas_id, version, entry_json) VALUES ( + 'fixture-space', 3, + '{"version":3,"ts":13,"commands":[],"deltas":[],"originator":{"source":"system"}}' +); + +PRAGMA user_version = 1; +COMMIT; diff --git a/apps/server/src/modules/storage/backends/sqlite/identity.ts b/apps/server/src/modules/storage/backends/sqlite/identity.ts new file mode 100644 index 000000000..3a84a7497 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/identity.ts @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * Allocation of the names a Space or Node is filed under. + * + * The `collision_key` columns carry a UNIQUE constraint, so a title or label + * has to be de-duplicated before it reaches the database rather than after a + * failed insert. These rules are pure and share `utils/naming` with Disk, so + * both backends hand out the same ` (2)` suffixes for the same inputs — see + * `backends/disk/space-title.ts` for the directory-locator half. + */ + +import { + dedupeName, + normalizeForCompare, + toSafeFilename, +} from '../../../../utils/naming.js'; + +import type { NodeContent } from '../../../canvas/persistence-types.js'; + +function allocatedSpaceTitle( + requested: string | null, + canvasId: string, + allocatedName: string, +): string | null { + if (requested === null) return null; + const base = toSafeFilename(requested, canvasId); + if (allocatedName === base) return requested; + const candidate = `${requested}${allocatedName.slice(base.length)}`; + return toSafeFilename(candidate, canvasId) === allocatedName + ? candidate + : allocatedName; +} + +export function allocateSpaceIdentity( + requestedTitle: string | null, + canvasId: string, + occupiedCollisionKeys: Iterable, +): { readonly title: string | null; readonly collisionKey: string } { + const base = toSafeFilename(requestedTitle, canvasId); + const allocated = dedupeName(base, occupiedCollisionKeys); + return { + title: allocatedSpaceTitle(requestedTitle, canvasId, allocated), + collisionKey: normalizeForCompare(allocated), + }; +} + +export function collisionKeyForTitle( + title: string | null, + canvasId: string, +): string { + return normalizeForCompare(toSafeFilename(title, canvasId)); +} + +export function allocateNodeIdentity( + record: NodeContent, + nodeId: string, + existingCollisionKey: string | null, + occupiedCollisionKeys: Iterable, +): { + readonly record: NodeContent; + readonly collisionKey: string; + readonly desiredCollisionKey: string; +} { + const trimmedLabel = + typeof record.label === 'string' && record.label.trim().length > 0 + ? record.label + : null; + if (trimmedLabel === null && existingCollisionKey !== null) { + return { + record, + collisionKey: existingCollisionKey, + desiredCollisionKey: existingCollisionKey, + }; + } + + const desired = toSafeFilename(trimmedLabel, nodeId); + const allocated = dedupeName(desired, occupiedCollisionKeys); + const suffix = + allocated.length > desired.length && allocated.startsWith(desired) + ? allocated.slice(desired.length) + : ''; + return { + record: + suffix && trimmedLabel + ? { ...record, label: `${trimmedLabel}${suffix}` } + : record, + collisionKey: normalizeForCompare(allocated), + desiredCollisionKey: normalizeForCompare(desired), + }; +} diff --git a/apps/server/src/modules/storage/backends/sqlite/integration.test.ts b/apps/server/src/modules/storage/backends/sqlite/integration.test.ts new file mode 100644 index 000000000..6a13e9a62 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/integration.test.ts @@ -0,0 +1,896 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { readFileSync } from 'node:fs'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { extractCanvasChanges } from '@huabu/shared/canvas-engine'; + +import { + applySqliteMigrations, + SqliteStoreContext, + SQLITE_SCHEMA_VERSION, +} from './database.js'; +import { SqliteStructuredStore } from './structured-store.js'; +import { + createSqliteTestFile, + installDeltaAbortTrigger, + openSqliteTestStore, + readSqliteDeltaLog, + withTestDatabase, +} from './test-support.js'; +import { SqliteWorkspaceRepository } from './workspace-repository.js'; + +import type { + CanvasFile, + DeltaLogEntry, + NodeContent, +} from '../../../canvas/persistence-types.js'; +import type { NodeSnapshot } from '../../ports/structured.js'; +import type { TaskRecord } from '@huabu/shared'; +import type { CanvasNode } from '@huabu/shared/canvas-engine'; + +const cleanups: Array<() => Promise | void> = []; + +afterEach(async () => { + for (const cleanup of cleanups.splice(0).reverse()) { + await cleanup(); + } +}); + +function trackedFile(prefix: string) { + const file = createSqliteTestFile(prefix); + cleanups.push(file.remove); + return file; +} + +function trackedStore(filename: string): SqliteStructuredStore { + const store = new SqliteStructuredStore(filename); + cleanups.push(() => store.close()); + return store; +} + +/** A tracked context, so a test can drive the shared connection directly. */ +function trackedContext(filename: string): SqliteStoreContext { + const context = new SqliteStoreContext(filename); + cleanups.push(() => context.close()); + return context; +} + +/** + * Open a store on an existing file and activate a Workspace on it. + * + * Reopening is the interesting half of persistence, and every Space query is + * Workspace-scoped, so a reopened store has to select one before it can read + * anything — exactly as a restarted Server does. + */ +async function reopenWithWorkspace( + filename: string, +): Promise { + const context = trackedContext(filename); + context.init(); + const workspaces = new SqliteWorkspaceRepository(context); + const [first] = await workspaces.list(); + if (!first) throw new Error('Reopened SQLite database holds no Workspace'); + context.useWorkspace(first.workspaceId); + return new SqliteStructuredStore(context); +} + +async function trackedOpenStore(prefix: string) { + const harness = await openSqliteTestStore(prefix); + cleanups.push(harness.cleanup); + return harness; +} + +function note(nodeId: string, label: string, content: string): NodeContent { + return { nodeId, type: 'note', label, content }; +} + +function nextRecord(current: CanvasFile): CanvasFile { + return { + ...current, + version: current.version + 1, + updatedAt: current.updatedAt + 1, + }; +} + +function delta(version: number, marker: string): DeltaLogEntry { + return { + version, + ts: version + 100, + commands: [{ marker }], + deltas: [{ marker }], + originator: { source: 'system' }, + }; +} + +async function createSpace( + store: SqliteStructuredStore, + canvasId: string, + title: string, +): Promise { + const result = await store.spaces().create({ canvasId, title }); + if (!result.ok) throw new Error(`Could not create test Space ${canvasId}`); + return result.record; +} + +describe('SqliteStructuredStore lifecycle and schema', () => { + it('rejects an empty database filename', () => { + expect(() => new SqliteStructuredStore('')).toThrow(/filename.*empty/i); + }); + + it('rejects before init and after close while lifecycle operations stay idempotent', async () => { + const file = trackedFile('huabu-sqlite-lifecycle-'); + const store = trackedStore(file.filename); + + await expect(store.health()).rejects.toThrow(/not initialized/); + await expect( + Promise.resolve().then(() => store.spaces().list()), + ).rejects.toThrow(/not initialized/); + await expect( + Promise.resolve().then(() => store.space('lifecycle-space').read()), + ).rejects.toThrow(/not initialized/); + await expect( + Promise.resolve().then(() => + store.space('lifecycle-space').nodes.readMany([]), + ), + ).rejects.toThrow(/not initialized/); + + await expect(store.init()).resolves.toBeUndefined(); + await expect(store.init()).resolves.toBeUndefined(); + await expect(store.health()).resolves.toEqual({ ok: true, kind: 'sqlite' }); + await expect(store.health()).resolves.toEqual({ ok: true, kind: 'sqlite' }); + + // Open is not the same as ready: a Space query needs a Workspace, and an + // open store with none says so rather than answering for an arbitrary one. + await expect( + Promise.resolve().then(() => store.spaces().list()), + ).rejects.toThrow(/No Workspace is active/); + + await expect(store.close()).resolves.toBeUndefined(); + await expect(store.close()).resolves.toBeUndefined(); + await expect(store.health()).rejects.toThrow(/closed/); + await expect( + Promise.resolve().then(() => store.spaces().list()), + ).rejects.toThrow(/closed/); + await expect( + Promise.resolve().then(() => store.space('lifecycle-space').read()), + ).rejects.toThrow(/closed/); + await expect( + Promise.resolve().then(() => + store.space('lifecycle-space').nodes.readMany([]), + ), + ).rejects.toThrow(/closed/); + await expect(store.init()).rejects.toThrow(/closed/); + }); + + it('creates the complete STRICT v1 schema in a fresh database', async () => { + const file = trackedFile('huabu-sqlite-fresh-schema-'); + const store = trackedStore(file.filename); + await store.init(); + + withTestDatabase(file.filename, (database) => { + expect(database.prepare('PRAGMA user_version').get()).toEqual({ + user_version: SQLITE_SCHEMA_VERSION, + }); + const expectedTables = [ + 'changes', + 'delta_log', + 'events', + 'nodes', + 'space_extensions', + 'spaces', + 'tasks', + 'workspaces', + ]; + const tableRows = database.prepare('PRAGMA table_list').all(); + const productionTables = tableRows.filter((row) => + expectedTables.includes(String(row['name'])), + ); + expect(productionTables.map((row) => row['name']).sort()).toEqual( + expectedTables, + ); + expect(productionTables.every((row) => row['strict'] === 1)).toBe(true); + expect( + database + .prepare('PRAGMA foreign_key_list(nodes)') + .all() + .map((row) => ({ + table: row['table'], + from: row['from'], + to: row['to'], + onDelete: row['on_delete'], + })), + ).toContainEqual({ + table: 'spaces', + from: 'canvas_id', + to: 'canvas_id', + onDelete: 'CASCADE', + }); + expect( + database + .prepare('PRAGMA foreign_key_list(spaces)') + .all() + .map((row) => ({ + table: row['table'], + from: row['from'], + to: row['to'], + onDelete: row['on_delete'], + })), + ).toContainEqual({ + table: 'workspaces', + from: 'workspace_id', + to: 'workspace_id', + onDelete: 'CASCADE', + }); + }); + }); + + it('opens the immutable v1 SQL fixture without rewriting its records', async () => { + const file = trackedFile('huabu-sqlite-v1-fixture-'); + const fixtureSql = readFileSync( + new URL('./fixtures/v1.sql', import.meta.url), + 'utf8', + ); + withTestDatabase(file.filename, (database) => database.exec(fixtureSql)); + + const store = await reopenWithWorkspace(file.filename); + await expect(store.spaces().worldId()).resolves.toBe('fixture-world'); + await expect(store.spaces().list()).resolves.toEqual([ + { + canvasId: 'fixture-space', + title: 'Fixture Space', + nodeCount: 1, + createdAt: 10, + updatedAt: 13, + }, + ]); + const space = store.space('fixture-space'); + await expect(space.read()).resolves.toEqual({ + canvasId: 'fixture-space', + title: 'Fixture Space', + version: 3, + state: { + nodes: [{ id: 'fixture-node', type: 'note' }], + edges: [], + }, + createdAt: 10, + updatedAt: 13, + }); + await expect(space.nodes.read('fixture-node')).resolves.toEqual({ + record: note('fixture-node', 'Fixture Node', 'fixture body'), + revision: 'fixture-revision', + }); + await expect(space.events.read()).resolves.toEqual([ + { + payload: { + action: 'node_selected', + node: { id: 'fixture-node', type: 'note', label: 'Fixture Node' }, + }, + ts: 12, + }, + ]); + await expect(space.changes.read('fixture-thread')).resolves.toEqual([]); + await expect(space.tasks.read()).resolves.toEqual({ + version: 1, + tasks: [], + runs: [], + }); + expect(readSqliteDeltaLog(file.filename, 'fixture-space')).toEqual([ + { + version: 3, + ts: 13, + commands: [], + deltas: [], + originator: { source: 'system' }, + }, + ]); + }); + + it('rejects a database whose user_version is from the future', async () => { + const file = trackedFile('huabu-sqlite-future-schema-'); + withTestDatabase(file.filename, (database) => { + database.exec(`PRAGMA user_version = ${SQLITE_SCHEMA_VERSION + 1}`); + }); + const store = trackedStore(file.filename); + + await expect(store.init()).rejects.toThrow(/newer than supported/); + await expect(store.health()).rejects.toThrow(/closed/); + }); + + it('rolls every migration step and user_version back when a later step fails', () => { + const file = trackedFile('huabu-sqlite-migration-rollback-'); + withTestDatabase(file.filename, (database) => { + expect(() => + applySqliteMigrations(database, [ + { + version: 1, + sql: 'CREATE TABLE migration_v1 (id INTEGER PRIMARY KEY) STRICT;', + }, + { + version: 2, + sql: ` + CREATE TABLE migration_v2 (id INTEGER PRIMARY KEY) STRICT; + INSERT INTO missing_migration_table (id) VALUES (1); + `, + }, + ]), + ).toThrow(/missing_migration_table|no such table/); + + expect(database.prepare('PRAGMA user_version').get()).toEqual({ + user_version: 0, + }); + expect( + database + .prepare( + `SELECT name + FROM sqlite_schema + WHERE type = 'table' AND name LIKE 'migration_%'`, + ) + .all(), + ).toEqual([]); + }); + }); +}); + +describe('SqliteStructuredStore persistence and transactions', () => { + it('persists Space and Node records across close and reopen', async () => { + const harness = await trackedOpenStore('huabu-sqlite-reopen-'); + const canvasId = 'reopen-space'; + const created = await createSpace(harness.store, canvasId, 'Reopen Space'); + const record = note('reopen-node', 'Reopen Node', 'persisted body'); + const put = await harness.store.space(canvasId).nodes.put({ + nodeId: record.nodeId, + record, + }); + expect(put).toMatchObject({ ok: true, record }); + + harness.closeConnection(); + const reopened = await reopenWithWorkspace(harness.filename); + + await expect(reopened.spaces().worldId()).resolves.toBe( + harness.world.canvasId, + ); + await expect(reopened.space(canvasId).read()).resolves.toEqual(created); + await expect( + reopened.space(canvasId).nodes.read(record.nodeId), + ).resolves.toEqual(put.ok ? { record, revision: put.revision } : null); + }); + + it('rolls node, record, and delta state back on a real trigger abort', async () => { + const harness = await trackedOpenStore('huabu-sqlite-trigger-rollback-'); + const canvasId = 'trigger-rollback-space'; + const baseline = await createSpace( + harness.store, + canvasId, + 'Trigger Rollback Space', + ); + const oldNode = note('old-node', 'Old Node', 'before'); + const newNode = note('new-node', 'New Node', 'after'); + const oldPut = await harness.store.space(canvasId).nodes.put({ + nodeId: oldNode.nodeId, + record: oldNode, + }); + if (!oldPut.ok) throw new Error('Could not seed rollback node'); + + const next: CanvasFile = { + ...nextRecord(baseline), + state: { + nodes: [{ id: newNode.nodeId, type: newNode.type }], + edges: [], + }, + }; + const restore = installDeltaAbortTrigger( + harness.filename, + 'forced delta abort', + ); + try { + await expect( + harness.store.space(canvasId).write({ + expectedVersion: baseline.version, + nextRecord: next, + nodeMutations: [ + { kind: 'delete', nodeId: oldNode.nodeId }, + { + kind: 'put', + nodeId: newNode.nodeId, + record: newNode, + authoritativeInsert: true, + }, + ], + delta: delta(next.version, 'trigger-abort'), + }), + ).rejects.toThrow('forced delta abort'); + } finally { + restore(); + } + + const space = harness.store.space(canvasId); + await expect(space.read()).resolves.toEqual(baseline); + await expect(space.nodes.read(oldNode.nodeId)).resolves.toEqual({ + record: oldPut.record, + revision: oldPut.revision, + }); + await expect(space.nodes.read(newNode.nodeId)).resolves.toBeNull(); + expect(readSqliteDeltaLog(harness.filename, canvasId)).toEqual([]); + await expect( + space.nodes.put({ + nodeId: oldNode.nodeId, + expectedRevision: oldPut.revision, + record: { ...oldNode, content: 'still writable' }, + }), + ).resolves.toMatchObject({ ok: true }); + }); + + it('rejects sparse JSON arrays without changing the exact persisted Node', async () => { + const harness = await trackedOpenStore('huabu-sqlite-sparse-json-'); + const canvasId = 'sparse-json-space'; + await createSpace(harness.store, canvasId, 'Sparse JSON Space'); + const nodes = harness.store.space(canvasId).nodes; + const record = note('sparse-json-node', 'Sparse JSON Node', 'before'); + const baseline = await nodes.put({ nodeId: record.nodeId, record }); + if (!baseline.ok) throw new Error('Could not seed sparse JSON node'); + const sparse: unknown[] = []; + sparse[1] = 'present'; + expect(0 in sparse).toBe(false); + + await expect( + nodes.put({ + nodeId: record.nodeId, + expectedRevision: baseline.revision, + record: { ...record, metadata: sparse }, + }), + ).rejects.toThrow(/sparse array/i); + await expect(nodes.read(record.nodeId)).resolves.toEqual({ + record, + revision: baseline.revision, + }); + }); + + it('recovers malformed stored Node content through every read shape', async () => { + const harness = await trackedOpenStore('huabu-sqlite-node-recovery-'); + const canvasId = 'node-recovery-space'; + await createSpace(harness.store, canvasId, 'Node Recovery Space'); + const nodes = harness.store.space(canvasId).nodes; + const record = note('recoverable-node', 'Recoverable Node', 'before'); + const baseline = await nodes.put({ nodeId: record.nodeId, record }); + if (!baseline.ok) throw new Error('Could not seed recoverable Node'); + + withTestDatabase(harness.filename, (database) => { + database + .prepare( + `UPDATE nodes + SET record_json = ? + WHERE canvas_id = ? AND node_id = ?`, + ) + .run('{"content":"recoverable body"}', canvasId, record.nodeId); + }); + + const recovered: NodeSnapshot = { + record: { + nodeId: record.nodeId, + type: 'note', + label: null, + content: 'recoverable body', + }, + revision: baseline.revision, + }; + await expect(nodes.read(record.nodeId)).resolves.toEqual(recovered); + await expect(nodes.readMany([record.nodeId])).resolves.toEqual( + new Map([[record.nodeId, recovered]]), + ); + await expect(nodes.list()).resolves.toEqual( + new Map([[record.nodeId, recovered]]), + ); + const delivered: NodeSnapshot[] = []; + await expect( + nodes.stream((snapshot) => delivered.push(snapshot)), + ).resolves.toEqual(new Map([[record.nodeId, recovered]])); + expect(delivered).toEqual([recovered]); + + await expect( + nodes.put({ + nodeId: record.nodeId, + expectedRevision: baseline.revision, + record: { ...record, content: 'repaired' }, + }), + ).resolves.toMatchObject({ ok: true }); + }); + + it('releases deletion admission when post-acquire Space setup throws', async () => { + const harness = await trackedOpenStore('huabu-sqlite-delete-setup-'); + const canvasId = 'delete-setup-space'; + const record = await createSpace( + harness.store, + canvasId, + 'Delete Setup Space', + ); + const repository = harness.store.spaces(); + + const malformedAttempt = repository.beginDelete({ canvasId }); + withTestDatabase(harness.filename, (database) => { + database + .prepare('UPDATE spaces SET state_json = ? WHERE canvas_id = ?') + .run('[]', canvasId); + }); + await expect(malformedAttempt).rejects.toThrow(/Invalid Space/); + withTestDatabase(harness.filename, (database) => { + database + .prepare('UPDATE spaces SET state_json = ? WHERE canvas_id = ?') + .run(JSON.stringify(record.state), canvasId); + }); + + let secondResult: + | Awaited> + | undefined; + let secondError: unknown; + const secondSettled = repository.beginDelete({ canvasId }).then( + (result) => { + secondResult = result; + }, + (error: unknown) => { + secondError = error; + }, + ); + await new Promise((resolve) => setImmediate(resolve)); + + expect(secondError).toBeUndefined(); + expect(secondResult).toMatchObject({ ok: true }); + if (!secondResult?.ok) { + throw new Error('Deletion gate remained occupied after setup failure'); + } + await secondResult.session.abort(); + await secondSettled; + }); + + it('cascades every child record when a deletion session finishes', async () => { + const harness = await trackedOpenStore('huabu-sqlite-delete-session-'); + const canvasId = 'delete-session-space'; + const baseline = await createSpace( + harness.store, + canvasId, + 'Delete Session Space', + ); + const record = note('deleted-node', 'Deleted Node', 'stale body'); + const handle = harness.store.space(canvasId); + await handle.nodes.put({ nodeId: record.nodeId, record }); + await handle.events.append([ + { + payload: { + action: 'node_selected', + node: { + id: record.nodeId, + type: 'note', + label: record.label ?? undefined, + }, + }, + ts: 2, + }, + ]); + const changeNode: CanvasNode = { + id: 'change-node', + type: 'note', + position: { x: 0, y: 0 }, + data: { label: 'Change Node', content: 'change body' }, + } as CanvasNode; + await handle.changes.append( + 'delete-thread', + extractCanvasChanges([{ type: 'INSERT_NODE', node: changeNode }]), + ); + const task: TaskRecord = { + taskId: 'delete-task', + canvasId, + goal: 'Delete this fixture', + defaultRootProfileId: 'profile-delete', + anchorNodeId: record.nodeId, + createdAt: 3, + }; + await handle.tasks.create(task); + const next = nextRecord(baseline); + await expect( + handle.write({ + expectedVersion: baseline.version, + nextRecord: next, + nodeMutations: [], + delta: delta(next.version, 'delete-session'), + }), + ).resolves.toEqual({ ok: true }); + + withTestDatabase(harness.filename, (database) => { + for (const table of [ + 'nodes', + 'events', + 'changes', + 'tasks', + 'delta_log', + ]) { + expect( + database + .prepare( + `SELECT count(*) AS count FROM ${table} WHERE canvas_id = ?`, + ) + .get(canvasId)?.['count'], + ).toBe(1); + } + }); + + const started = await harness.store.spaces().beginDelete({ canvasId }); + if (!started.ok) throw new Error('Ordinary Space must be deletable'); + await expect(handle.read()).resolves.toEqual(next); + await expect(handle.nodes.read(record.nodeId)).resolves.toMatchObject({ + record, + }); + await expect(started.session.finish()).resolves.toEqual({ + ok: true, + reason: 'deleted', + }); + + withTestDatabase(harness.filename, (database) => { + for (const table of [ + 'nodes', + 'events', + 'changes', + 'tasks', + 'delta_log', + ]) { + expect( + database + .prepare( + `SELECT count(*) AS count FROM ${table} WHERE canvas_id = ?`, + ) + .get(canvasId)?.['count'], + ).toBe(0); + } + }); + + await expect(handle.read()).resolves.toBeNull(); + }); + + it('allows a first write after deleting an already absent node', async () => { + const harness = await trackedOpenStore('huabu-sqlite-absent-delete-'); + const canvasId = 'absent-delete-space'; + await createSpace(harness.store, canvasId, 'Absent Delete Space'); + const nodes = harness.store.space(canvasId).nodes; + const record = note('not-yet-created', 'Not Yet Created', 'body'); + + await expect(nodes.delete(record.nodeId)).resolves.toBe('absent'); + await expect( + nodes.put({ nodeId: record.nodeId, record }), + ).resolves.toMatchObject({ ok: true, record }); + }); + + it('allows immediate reuse of a deleted primary key across reopen', async () => { + const harness = await trackedOpenStore('huabu-sqlite-delete-reopen-'); + const canvasId = 'tombstone-reopen-space'; + await createSpace(harness.store, canvasId, 'Tombstone Reopen Space'); + const record = note('tombstoned-node', 'Tombstoned Node', 'before'); + const nodes = harness.store.space(canvasId).nodes; + const initial = await nodes.put({ nodeId: record.nodeId, record }); + if (!initial.ok) throw new Error('Could not create initial test Node'); + + await expect(nodes.delete(record.nodeId)).resolves.toBe('deleted'); + const recreated = await nodes.put({ + nodeId: record.nodeId, + record: { ...record, content: 'immediate replacement' }, + }); + if (!recreated.ok) throw new Error('Could not recreate test Node'); + expect(recreated.record).toEqual({ + ...record, + content: 'immediate replacement', + }); + expect(recreated.revision).not.toBe(initial.revision); + await expect( + nodes.put({ + nodeId: record.nodeId, + expectedRevision: initial.revision, + record: { ...record, content: 'stale replacement' }, + }), + ).resolves.toEqual({ + ok: false, + reason: 'revision-conflict', + currentRevision: recreated.revision, + }); + + harness.closeConnection(); + const reopened = await reopenWithWorkspace(harness.filename); + await expect( + reopened.space(canvasId).nodes.delete(record.nodeId), + ).resolves.toBe('deleted'); + await expect( + reopened.space(canvasId).nodes.put({ + nodeId: record.nodeId, + record: { ...record, content: 'after reopen' }, + }), + ).resolves.toMatchObject({ + ok: true, + record: { ...record, content: 'after reopen' }, + }); + }); +}); + +describe('SqliteStructuredStore durability and encoding', () => { + it('opens in WAL with a bounded busy wait and foreign keys enforced', async () => { + const harness = await trackedOpenStore('huabu-sqlite-pragmas-'); + + withTestDatabase(harness.filename, (database) => { + // Read on a *second* connection: `journal_mode` is a property of the + // database file, so this proves the mode was actually persisted rather + // than set on the adapter's own handle and forgotten. + expect(database.prepare('PRAGMA journal_mode').get()).toEqual({ + journal_mode: 'wal', + }); + }); + const database = harness.context.database(); + expect(database.prepare('PRAGMA foreign_keys').get()).toEqual({ + foreign_keys: 1, + }); + expect( + Number(database.prepare('PRAGMA busy_timeout').get()?.['timeout']), + ).toBeGreaterThan(0); + }); + + it('accepts an undefined field the way JSON.stringify does', async () => { + const harness = await trackedOpenStore('huabu-sqlite-undefined-'); + const canvasId = 'undefined-field-space'; + const base = await createSpace(harness.store, canvasId, 'Undefined Space'); + const handle = harness.store.space(canvasId); + + // Disk persists through `JSON.stringify`, which drops an undefined own + // property. A record it accepts must not become a rejected write here — + // that divergence is invisible until a caller happens to spread an + // optional field onto a node. + const next = { + ...base, + version: 1, + state: { + nodes: [ + { + id: 'node-undefined', + type: 'note', + position: { x: 0, y: 0 }, + data: { kept: 'yes', dropped: undefined }, + }, + ], + edges: [], + }, + } as unknown as CanvasFile; + await expect( + handle.write({ expectedVersion: 0, nextRecord: next, nodeMutations: [] }), + ).resolves.toEqual({ ok: true }); + const stored = await handle.read(); + expect( + (stored?.state.nodes[0] as { data: Record }).data, + ).toEqual({ kept: 'yes' }); + + // What is genuinely unrepresentable still rejects. + const cyclic: Record = { id: 'node-cyclic' }; + cyclic['self'] = cyclic; + await expect( + handle.write({ + expectedVersion: 1, + nextRecord: { + ...base, + version: 2, + state: { nodes: [cyclic], edges: [] }, + } as unknown as CanvasFile, + nodeMutations: [], + }), + ).rejects.toThrow(/cycle/); + await expect( + handle.write({ + expectedVersion: 1, + nextRecord: { + ...base, + version: 2, + state: { nodes: [{ id: 'n', size: Number.NaN }], edges: [] }, + } as unknown as CanvasFile, + nodeMutations: [], + }), + ).rejects.toThrow(/non-finite/); + }); + + it('delivers streamed nodes before the scan finishes and stops on abort', async () => { + const harness = await trackedOpenStore('huabu-sqlite-stream-'); + const canvasId = 'stream-space'; + await createSpace(harness.store, canvasId, 'Stream Space'); + const nodes = harness.store.space(canvasId).nodes; + for (let index = 0; index < 6; index += 1) { + const put = await nodes.put({ + nodeId: `node-${index}`, + record: note(`node-${index}`, `Node ${index}`, `body ${index}`), + }); + if (!put.ok) throw new Error('Could not seed a stream node'); + } + + const signal = { aborted: false }; + const seen: NodeSnapshot[] = []; + const delivered = await nodes.stream( + (snapshot) => { + seen.push(snapshot); + if (seen.length === 2) signal.aborted = true; + }, + { signal }, + ); + + // An aborted scan stops reading rather than materializing the whole + // collection first and discarding it, so the map it settles with is the + // partial one the port describes. + expect(seen).toHaveLength(2); + expect(delivered.size).toBe(2); + await expect(nodes.list()).resolves.toHaveProperty('size', 6); + + const complete: string[] = []; + const all = await nodes.stream((snapshot) => + complete.push(snapshot.record.nodeId), + ); + expect(complete).toHaveLength(6); + expect(all.size).toBe(6); + }); + + it('reads a batch of nodes in one pass, including duplicates and absences', async () => { + const harness = await trackedOpenStore('huabu-sqlite-readmany-'); + const canvasId = 'readmany-space'; + await createSpace(harness.store, canvasId, 'ReadMany Space'); + const nodes = harness.store.space(canvasId).nodes; + for (const nodeId of ['a', 'b', 'c']) { + await nodes.put({ + nodeId, + record: note(nodeId, `Node ${nodeId}`, nodeId), + }); + } + + const selection = await nodes.readMany(['a', 'a', 'missing', 'c']); + expect([...selection.keys()].sort()).toEqual(['a', 'c']); + expect(selection.get('a')).toEqual(await nodes.read('a')); + }); + + it('scopes every Space operation to the active Workspace', async () => { + const harness = await trackedOpenStore('huabu-sqlite-workspaces-'); + const first = harness.workspaceId; + await createSpace(harness.store, 'workspace-a-space', 'Space A'); + + const workspaces = new SqliteWorkspaceRepository(harness.context); + const second = await workspaces.create('Second Workspace'); + const retained = harness.store.space('workspace-a-space'); + + harness.context.useWorkspace(second.workspaceId); + // A Space in another Workspace is not visible, and a handle resolved + // before the switch refuses rather than answering for the new namespace. + await expect(harness.store.spaces().list()).resolves.toEqual([]); + await expect( + harness.store.space('workspace-a-space').read(), + ).resolves.toBeNull(); + await expect(retained.read()).rejects.toThrow(/inactive Workspace/); + + // Same title, different Workspace: no collision, no suffix. + const created = await harness.store + .spaces() + .create({ canvasId: 'workspace-b-space', title: 'Space A' }); + expect(created).toMatchObject({ ok: true, record: { title: 'Space A' } }); + + harness.context.useWorkspace(first); + await expect(harness.store.spaces().list()).resolves.toHaveLength(1); + }); + + it('keeps a forgotten Workspace out of listings without destroying it', async () => { + const harness = await trackedOpenStore('huabu-sqlite-forget-'); + const workspaces = new SqliteWorkspaceRepository(harness.context); + await createSpace(harness.store, 'forgotten-space', 'Forgotten Space'); + + await expect(workspaces.remove(harness.workspaceId)).resolves.toBe(true); + await expect(workspaces.list()).resolves.toEqual([]); + await expect(workspaces.get(harness.workspaceId)).resolves.toBeNull(); + await expect(workspaces.remove(harness.workspaceId)).resolves.toBe(false); + + // "Forget" is not "delete": the port's wording is deliberate, and on a + // backend with no folder left behind the rows have to be what honours it. + expect( + withTestDatabase(harness.filename, (database) => + database + .prepare('SELECT canvas_id FROM spaces WHERE canvas_id = ?') + .all('forgotten-space'), + ), + ).toHaveLength(1); + }); +}); diff --git a/apps/server/src/modules/storage/backends/sqlite/rows.ts b/apps/server/src/modules/storage/backends/sqlite/rows.ts new file mode 100644 index 000000000..94faf3b96 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/rows.ts @@ -0,0 +1,366 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * Movement of persisted values between domain records and SQLite rows. + * + * Every column this backend stores is either JSON text or a scalar, so the + * codecs here are the single place that decides what a well-formed stored + * value looks like. Space and log reads reject malformed domain values. Node + * reads preserve the port's repair path by recovering malformed JSON values + * into a valid record whose content still exposes the stored value. + * + * The encoder's job is to refuse what SQLite could not faithfully return — + * cycles, non-finite numbers, values `JSON.stringify` would silently reshape + * into something else. It deliberately does **not** refuse what + * `JSON.stringify` already handles by rule, because Disk persists through + * that same function: a record it accepts must not become a rejected write + * here. `undefined` is the case that matters in practice — an optional field + * spread onto a node makes an own property whose value is `undefined`, and + * Disk drops it. See §13's "silent divergence" risk: a portable contract that + * only holds where the adapters already agree certifies both sides of a + * disagreement. + */ + +import { SQLITE_WORLD_COLLISION_KEY } from './database.js'; +import { canvasFileShapeError } from '../../../canvas/persistence-validation.js'; + +import type { + CanvasFile, + NodeContent, +} from '../../../canvas/persistence-types.js'; +import type { DatabaseSync } from 'node:sqlite'; + +type JsonPrimitive = null | boolean | number | string; +type JsonValue = JsonPrimitive | JsonValue[] | { [key: string]: JsonValue }; + +function assertJsonValue( + value: unknown, + context: string, + seen: Set, +): asserts value is JsonValue { + if ( + value === null || + typeof value === 'string' || + typeof value === 'boolean' + ) { + return; + } + if (typeof value === 'number') { + if (!Number.isFinite(value)) { + throw new TypeError(`${context} contains a non-finite number`); + } + return; + } + // `JSON.stringify` drops an `undefined` object property and encodes an + // `undefined` array element as null, so Disk already accepts both. Matching + // that rule keeps one record from being writable on one backend only. + if (value === undefined) return; + if (typeof value !== 'object') { + throw new TypeError(`${context} contains a non-JSON value`); + } + if (seen.has(value)) throw new TypeError(`${context} contains a cycle`); + seen.add(value); + try { + if (Array.isArray(value)) { + for (let index = 0; index < value.length; index += 1) { + if (!Object.prototype.hasOwnProperty.call(value, index)) { + throw new TypeError(`${context} contains a sparse array`); + } + assertJsonValue(value[index], `${context}[${index}]`, seen); + } + return; + } + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) { + throw new TypeError(`${context} contains a non-plain object`); + } + for (const [key, entry] of Object.entries(value)) { + assertJsonValue(entry, `${context}.${key}`, seen); + } + } finally { + seen.delete(value); + } +} + +export function stringifyJson(value: unknown, context: string): string { + assertJsonValue(value, context, new Set()); + const encoded = JSON.stringify(value); + if (encoded === undefined) { + throw new TypeError(`${context} is not representable as JSON`); + } + return encoded; +} + +export function parseJson(value: unknown, context: string): unknown { + if (typeof value !== 'string') { + throw new SyntaxError(`${context} is not stored as JSON text`); + } + try { + return JSON.parse(value) as unknown; + } catch (error) { + throw new SyntaxError( + `Invalid JSON in ${context}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} + +function rowObject(value: unknown, context: string): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new SyntaxError(`Missing or malformed SQLite row for ${context}`); + } + return value as Record; +} + +function stringColumn( + row: Record, + column: string, + context: string, +): string { + const value = row[column]; + if (typeof value !== 'string') { + throw new SyntaxError(`Invalid ${column} in ${context}`); + } + return value; +} + +function nullableStringColumn( + row: Record, + column: string, + context: string, +): string | null { + const value = row[column]; + if (value !== null && typeof value !== 'string') { + throw new SyntaxError(`Invalid ${column} in ${context}`); + } + return value; +} + +function numberColumn( + row: Record, + column: string, + context: string, +): number { + const value = row[column]; + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new SyntaxError(`Invalid ${column} in ${context}`); + } + return value; +} + +export interface PersistedSpace { + readonly record: CanvasFile; + readonly workspaceId: string; + readonly collisionKey: string; + readonly isWorld: boolean; +} + +export function decodeSpaceRow(value: unknown): PersistedSpace { + const row = rowObject(value, 'Space'); + const canvasId = stringColumn(row, 'canvas_id', 'Space'); + const context = `Space ${JSON.stringify(canvasId)}`; + const record: CanvasFile = { + canvasId, + title: nullableStringColumn(row, 'title', context), + version: numberColumn(row, 'version', context), + state: parseJson( + row['state_json'], + `${context} state`, + ) as CanvasFile['state'], + createdAt: numberColumn(row, 'created_at', context), + updatedAt: numberColumn(row, 'updated_at', context), + }; + const shapeError = canvasFileShapeError(record, canvasId); + if (shapeError) throw new SyntaxError(`Invalid ${context}: ${shapeError}`); + const world = numberColumn(row, 'is_world', context); + if (world !== 0 && world !== 1) { + throw new SyntaxError(`Invalid is_world in ${context}`); + } + return { + record, + workspaceId: stringColumn(row, 'workspace_id', context), + collisionKey: stringColumn(row, 'collision_key', context), + isWorld: world === 1, + }; +} + +export const SPACE_COLUMNS = + 'canvas_id, workspace_id, title, collision_key, version, state_json, ' + + 'created_at, updated_at, is_world'; + +/** + * Read one Space, scoped to the Workspace that owns it. + * + * The Workspace predicate is not an optimization. `canvas_id` is unique across + * the whole database, so without it a handle resolved in one Workspace would + * answer for a Space in another — which is exactly the confusion the Disk + * adapters prevent by binding to a workspace path. + */ +export function readSpaceRow( + database: DatabaseSync, + workspaceId: string, + canvasId: string, +): PersistedSpace | null { + const row = database + .prepare( + `SELECT ${SPACE_COLUMNS} + FROM spaces + WHERE workspace_id = ? AND canvas_id = ?`, + ) + .get(workspaceId, canvasId); + return row === undefined ? null : decodeSpaceRow(row); +} + +/** Whether the named Space exists in this Workspace. */ +export function spaceRowExists( + database: DatabaseSync, + workspaceId: string, + canvasId: string, +): boolean { + return ( + database + .prepare( + `SELECT 1 AS present + FROM spaces + WHERE workspace_id = ? AND canvas_id = ?`, + ) + .get(workspaceId, canvasId)?.['present'] === 1 + ); +} + +/** Collision keys already taken in one Workspace, for name allocation. */ +export function occupiedCollisionKeys( + database: DatabaseSync, + workspaceId: string, +): string[] { + return database + .prepare('SELECT collision_key FROM spaces WHERE workspace_id = ?') + .all(workspaceId) + .map((row) => row['collision_key']) + .filter((value): value is string => typeof value === 'string'); +} + +export function validateCanvasFile(record: CanvasFile, canvasId: string): void { + const shapeError = canvasFileShapeError(record, canvasId); + if (shapeError) { + throw new TypeError(`Invalid Space record: ${shapeError}`); + } + stringifyJson(record.state, `Space ${JSON.stringify(canvasId)} state`); +} + +export function insertSpaceRow( + database: DatabaseSync, + workspaceId: string, + record: CanvasFile, + collisionKey: string, + isWorld = false, +): void { + validateCanvasFile(record, record.canvasId); + database + .prepare( + `INSERT INTO spaces ( + canvas_id, workspace_id, title, collision_key, version, state_json, + created_at, updated_at, is_world + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .run( + record.canvasId, + workspaceId, + record.title, + isWorld ? SQLITE_WORLD_COLLISION_KEY : collisionKey, + record.version, + stringifyJson(record.state, `Space ${record.canvasId} state`), + record.createdAt, + record.updatedAt, + isWorld ? 1 : 0, + ); +} + +export function updateSpaceRow( + database: DatabaseSync, + workspaceId: string, + record: CanvasFile, + expectedVersion: number, +): number { + validateCanvasFile(record, record.canvasId); + const result = database + .prepare( + `UPDATE spaces + SET version = ?, state_json = ?, updated_at = ? + WHERE workspace_id = ? AND canvas_id = ? AND version = ?`, + ) + .run( + record.version, + stringifyJson(record.state, `Space ${record.canvasId} state`), + record.updatedAt, + workspaceId, + record.canvasId, + expectedVersion, + ); + return Number(result.changes); +} + +export function validateNodeContent( + record: NodeContent, + expectedNodeId: string, +): void { + if (typeof record !== 'object' || record === null || Array.isArray(record)) { + throw new TypeError('Node record must be an object'); + } + if (record.nodeId !== expectedNodeId) { + throw new Error( + `Node id mismatch: argument=${JSON.stringify(expectedNodeId)} ` + + `record=${JSON.stringify(record.nodeId)}`, + ); + } + if (typeof record.type !== 'string') { + throw new TypeError('Node record type must be a string'); + } + if (record.label !== null && typeof record.label !== 'string') { + throw new TypeError('Node record label must be a string or null'); + } + if (typeof record.content !== 'string') { + throw new TypeError('Node record content must be a string'); + } + stringifyJson(record, `Node ${JSON.stringify(expectedNodeId)} record`); +} + +export function decodeNodeRecord( + value: unknown, + expectedNodeId: string, +): NodeContent { + const parsed = parseJson(value, `Node ${JSON.stringify(expectedNodeId)}`); + try { + validateNodeContent(parsed as NodeContent, expectedNodeId); + return parsed as NodeContent; + } catch { + // A valid JSON value can still have a damaged Node shape after an + // out-of-band database edit. Keep it reachable so a normal put can repair + // it, matching the lenient content rule used by the Disk adapter. + const fields = + typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed) + ? (parsed as Record) + : {}; + return { + ...fields, + nodeId: expectedNodeId, + type: typeof fields['type'] === 'string' ? fields['type'] : 'note', + label: typeof fields['label'] === 'string' ? fields['label'] : null, + content: + typeof fields['content'] === 'string' + ? fields['content'] + : stringifyJson(parsed, `Malformed Node ${expectedNodeId}`), + } as NodeContent; + } +} + +export function requireRevision(value: unknown, nodeId: string): string { + if (typeof value !== 'string' || value.length === 0) { + throw new SyntaxError( + `Invalid persisted revision for Node ${JSON.stringify(nodeId)}`, + ); + } + return value; +} diff --git a/apps/server/src/modules/storage/backends/sqlite/schema.ts b/apps/server/src/modules/storage/backends/sqlite/schema.ts new file mode 100644 index 000000000..d8b41fbb3 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/schema.ts @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * The SQLite schema, and the rule for changing it. + * + * One file per version, applied in order, never edited once released. The + * migration runner in `database.ts` enforces that shape; this file is only the + * SQL. Everything is `STRICT` so a column's declared type is a real + * constraint, and every child row reaches its owner through a foreign key so + * deleting a Space or a Workspace cannot leave the rest behind. + * + * Two collections sit at the top: Workspaces, which are the namespaces a + * deployment holds, and Spaces, which belong to exactly one of them. That is + * the whole reason a SQL profile needs no Workspace directory — a Workspace is + * a row, not a folder, and switching to another one re-scopes queries through + * the same connection rather than reopening anything (proposal §2, "Backend + * selection scope"). + * + * Nothing here holds bytes. Blobs are files on whichever file system the blob + * axis names, so a Space's uploads, artifacts, guide and memory body are never + * rows in this database; the two lifecycles are joined only by the deletion + * saga in `storage.ts`, which sweeps the byte areas before the record goes. + */ + +/** + * Version 1 — Workspaces, Spaces, and everything a Space owns. + * + * `collision_key` is the de-duplicated, case-folded name a Space or node is + * filed under. It exists because titles and labels collide and the product + * resolves that with " (2)" suffixes; the UNIQUE constraints are what make + * the allocation in `identity.ts` authoritative rather than advisory. + */ +const SCHEMA_V1 = ` + CREATE TABLE workspaces ( + workspace_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + created_at REAL NOT NULL, + last_opened_at REAL NOT NULL, + -- Membership is forgettable without being destructive: the port's + -- remove() drops a Workspace from the listing and keeps everything it + -- owns, the way forgetting a Disk Workspace leaves its folder on disk. + forgotten_at REAL + ) STRICT; + + CREATE TABLE spaces ( + canvas_id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + title TEXT, + collision_key TEXT NOT NULL, + version INTEGER NOT NULL, + state_json TEXT NOT NULL CHECK (json_valid(state_json)), + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + is_world INTEGER NOT NULL DEFAULT 0 CHECK (is_world IN (0, 1)), + UNIQUE (workspace_id, collision_key), + FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) + ON DELETE CASCADE + ) STRICT; + + CREATE UNIQUE INDEX spaces_single_world + ON spaces(workspace_id) + WHERE is_world = 1; + + CREATE TABLE nodes ( + canvas_id TEXT NOT NULL, + node_id TEXT NOT NULL, + record_json TEXT NOT NULL CHECK (json_valid(record_json)), + revision TEXT NOT NULL CHECK (length(revision) > 0), + label_collision_key TEXT NOT NULL, + PRIMARY KEY (canvas_id, node_id), + UNIQUE (canvas_id, label_collision_key), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE TABLE events ( + event_id INTEGER PRIMARY KEY AUTOINCREMENT, + canvas_id TEXT NOT NULL, + event_json TEXT NOT NULL CHECK (json_valid(event_json)), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE INDEX events_by_canvas_order + ON events(canvas_id, event_id); + + CREATE TABLE changes ( + canvas_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + snapshot_json TEXT NOT NULL CHECK (json_valid(snapshot_json)), + PRIMARY KEY (canvas_id, thread_id), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE TABLE tasks ( + canvas_id TEXT PRIMARY KEY, + snapshot_json TEXT NOT NULL CHECK (json_valid(snapshot_json)), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE TABLE space_extensions ( + extension_id INTEGER PRIMARY KEY AUTOINCREMENT, + canvas_id TEXT NOT NULL, + namespace TEXT NOT NULL, + UNIQUE (canvas_id, namespace), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE TABLE delta_log ( + canvas_id TEXT NOT NULL, + version INTEGER NOT NULL, + entry_json TEXT NOT NULL CHECK (json_valid(entry_json)), + PRIMARY KEY (canvas_id, version), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; +`; + +export interface SqliteMigration { + readonly version: number; + readonly sql: string; +} + +export const SQLITE_MIGRATIONS: readonly SqliteMigration[] = Object.freeze([ + Object.freeze({ version: 1, sql: SCHEMA_V1 }), +]); + +export const SQLITE_SCHEMA_VERSION = + SQLITE_MIGRATIONS[SQLITE_MIGRATIONS.length - 1]?.version ?? 0; diff --git a/apps/server/src/modules/storage/backends/sqlite/space-extension.ts b/apps/server/src/modules/storage/backends/sqlite/space-extension.ts new file mode 100644 index 000000000..6f66b34d9 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/space-extension.ts @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * SQLite connection point for one extension namespace in one Space. + * + * The port's `extension()` is async because a backend may have to go and open + * something. SQLite does not: `node:sqlite` is synchronous, so the whole + * operation is a row read and possibly one insert. That matters beyond + * tidiness — an owner whose own interface is synchronous (the Agenetes + * conversation stores are the live example) cannot await, and would otherwise + * have to keep its own cache warmed by an unrelated code path. So the work + * lives in a synchronous function and the port member wraps it. + */ + +import { withImmediateTransaction } from './database.js'; +import { spaceRowExists } from './rows.js'; +import { assertValidNamespace } from '../../ports/namespace.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { SpaceHandle, SpaceSubstrate } from '../../ports/structured.js'; + +/** The SQLite arm of {@link SpaceSubstrate}, for callers that narrowed already. */ +export type SqliteSpaceSubstrate = Extract; + +/** + * Resolve — creating if absent — the namespace's connection point. + * + * `null` when the Space does not exist, which is the port's rule: refusing a + * substrate for a Space that is gone is what stops an owner from resurrecting + * one through an ad-hoc write. + */ +export function resolveSqliteSpaceExtension( + context: SqliteStoreContext, + boundWorkspaceId: string, + canvasId: string, + namespaceInput: string, +): SqliteSpaceSubstrate | null { + const namespace = assertValidNamespace(namespaceInput); + const workspaceId = context.assertBoundWorkspace( + boundWorkspaceId, + `SQLite Space extension(${canvasId})`, + ); + context.assertMutationAllowed(canvasId); + const database = context.database(); + + return withImmediateTransaction(database, () => { + if (!spaceRowExists(database, workspaceId, canvasId)) return null; + + database + .prepare( + `INSERT INTO space_extensions (canvas_id, namespace) + VALUES (?, ?) + ON CONFLICT(canvas_id, namespace) DO NOTHING`, + ) + .run(canvasId, namespace); + const extensionId = database + .prepare( + `SELECT extension_id + FROM space_extensions + WHERE canvas_id = ? AND namespace = ?`, + ) + .get(canvasId, namespace)?.['extension_id']; + if ( + typeof extensionId !== 'number' || + !Number.isSafeInteger(extensionId) || + extensionId <= 0 + ) { + throw new Error( + `Could not resolve SQLite extension ${JSON.stringify(namespace)}`, + ); + } + return Object.freeze({ + kind: 'sqlite' as const, + database, + extensionId, + }); + }); +} + +export function createSqliteSpaceExtension( + context: SqliteStoreContext, + boundWorkspaceId: string, + canvasId: string, +): SpaceHandle['extension'] { + return async function extension(namespaceInput: string) { + return resolveSqliteSpaceExtension( + context, + boundWorkspaceId, + canvasId, + namespaceInput, + ); + }; +} diff --git a/apps/server/src/modules/storage/backends/sqlite/space-logs.ts b/apps/server/src/modules/storage/backends/sqlite/space-logs.ts new file mode 100644 index 000000000..a948d9920 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/space-logs.ts @@ -0,0 +1,289 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { canvasEventInputSchema, canvasEventRecordSchema } from '@huabu/shared'; +import { + coalesceChanges, + type CanvasChangeRecord, +} from '@huabu/shared/canvas-engine'; + +import { withImmediateTransaction } from './database.js'; +import { parseJson, spaceRowExists, stringifyJson } from './rows.js'; +import { sanitizeId } from '../../../../utils/fs.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { CanvasEvent } from '../../../canvas/persistence-types.js'; +import type { + NewCanvasEvent, + SpaceChanges, + SpaceEvents, +} from '../../ports/structured.js'; +import type { z } from 'zod'; + +function firstIssue(error: z.ZodError): string { + const issue = error.issues[0]; + if (!issue) return 'unknown schema violation'; + const location = issue.path.length > 0 ? issue.path.join('.') : ''; + return `${location}: ${issue.message}`; +} + +function requireSpace( + context: SqliteStoreContext, + workspaceId: string, + canvasId: string, +): void { + context.assertMutationAllowed(canvasId); + if (!spaceRowExists(context.database(), workspaceId, canvasId)) { + throw new Error( + `SQLite Space logs(${canvasId}) cannot mutate a missing Space`, + ); + } +} + +function decodeEvents(rows: readonly Record[]): CanvasEvent[] { + return rows.map((row, index) => { + const parsedJson = parseJson( + row['event_json'], + `Canvas event ${index + 1}`, + ); + const parsed = canvasEventRecordSchema.safeParse(parsedJson); + if (!parsed.success) { + throw new SyntaxError( + `Invalid persisted Canvas event ${index + 1}: ${firstIssue(parsed.error)}`, + ); + } + return parsedJson as CanvasEvent; + }); +} + +function decodeChanges( + value: unknown, + canvasId: string, + threadId: string, +): CanvasChangeRecord[] { + const parsed = parseJson( + value, + `changes for Space ${JSON.stringify(canvasId)} thread ${JSON.stringify(threadId)}`, + ); + if (!Array.isArray(parsed)) { + throw new SyntaxError( + `Persisted changes for Space ${canvasId} thread ${threadId} must be an array`, + ); + } + return coalesceChanges(parsed as CanvasChangeRecord[]); +} + +export interface SqliteSpaceLogs { + readonly events: SpaceEvents; + readonly changes: SpaceChanges; +} + +class SqliteSpaceLogCoordinator { + readonly #context: SqliteStoreContext; + readonly #workspaceId: string; + readonly #canvasId: string; + + constructor( + context: SqliteStoreContext, + workspaceId: string, + canvasId: string, + ) { + this.#context = context; + this.#workspaceId = workspaceId; + this.#canvasId = canvasId; + } + + #workspace(): string { + return this.#context.assertBoundWorkspace( + this.#workspaceId, + `SQLite Space logs(${this.#canvasId})`, + ); + } + + async readEvents(limit?: number): Promise { + this.#workspace(); + const database = this.#context.database(); + if (limit !== undefined && !(limit > 0)) return []; + if (limit === undefined || !Number.isFinite(limit)) { + return decodeEvents( + database + .prepare( + `SELECT event_json + FROM events + WHERE canvas_id = ? + ORDER BY event_id ASC`, + ) + .all(this.#canvasId), + ); + } + const rows = database + .prepare( + `SELECT event_json + FROM events + WHERE canvas_id = ? + ORDER BY event_id DESC + LIMIT ?`, + ) + .all(this.#canvasId, Math.ceil(limit)) + .reverse(); + return decodeEvents(rows); + } + + async appendEvents(events: readonly NewCanvasEvent[]): Promise { + this.#context.assertOpen(); + const workspaceId = this.#workspace(); + if (events.length === 0) return; + const records: CanvasEvent[] = events.map((event, index) => { + const input = canvasEventInputSchema.safeParse(event); + if (!input.success) { + throw new TypeError( + `Invalid Canvas event append input at index ${index}: ${firstIssue(input.error)}`, + ); + } + const record = { + payload: event.payload, + ts: event.ts ?? this.#context.now(), + }; + const parsed = canvasEventRecordSchema.safeParse(record); + if (!parsed.success) { + throw new TypeError( + `Invalid Canvas event append record at index ${index}: ${firstIssue(parsed.error)}`, + ); + } + stringifyJson(record, `Canvas event append input ${index}`); + return record; + }); + + requireSpace(this.#context, workspaceId, this.#canvasId); + const database = this.#context.database(); + withImmediateTransaction(database, () => { + const insert = database.prepare( + 'INSERT INTO events (canvas_id, event_json) VALUES (?, ?)', + ); + for (const record of records) { + insert.run( + this.#canvasId, + stringifyJson(record, `Canvas event for ${this.#canvasId}`), + ); + } + }); + } + + async readChanges(threadIdInput: string): Promise { + const threadId = sanitizeId(threadIdInput, 'threadId'); + this.#workspace(); + const row = this.#context + .database() + .prepare( + `SELECT snapshot_json + FROM changes + WHERE canvas_id = ? AND thread_id = ?`, + ) + .get(this.#canvasId, threadId); + return row === undefined + ? [] + : decodeChanges(row['snapshot_json'], this.#canvasId, threadId); + } + + async appendChanges( + threadIdInput: string, + records: readonly CanvasChangeRecord[], + ): Promise { + const threadId = sanitizeId(threadIdInput, 'threadId'); + stringifyJson(records, `Changes for thread ${JSON.stringify(threadId)}`); + requireSpace(this.#context, this.#workspace(), this.#canvasId); + const database = this.#context.database(); + return withImmediateTransaction(database, () => { + const current = database + .prepare( + `SELECT snapshot_json + FROM changes + WHERE canvas_id = ? AND thread_id = ?`, + ) + .get(this.#canvasId, threadId); + const existing = + current === undefined + ? [] + : decodeChanges(current['snapshot_json'], this.#canvasId, threadId); + const merged = coalesceChanges([...existing, ...records]); + database + .prepare( + `INSERT INTO changes (canvas_id, thread_id, snapshot_json) + VALUES (?, ?, ?) + ON CONFLICT(canvas_id, thread_id) DO UPDATE SET + snapshot_json = excluded.snapshot_json`, + ) + .run( + this.#canvasId, + threadId, + stringifyJson(merged, `Changes for thread ${threadId}`), + ); + return merged; + }); + } + + async deleteChange( + threadIdInput: string, + changeId: string, + ): Promise { + const threadId = sanitizeId(threadIdInput, 'threadId'); + requireSpace(this.#context, this.#workspace(), this.#canvasId); + const database = this.#context.database(); + return withImmediateTransaction(database, () => { + const current = database + .prepare( + `SELECT snapshot_json + FROM changes + WHERE canvas_id = ? AND thread_id = ?`, + ) + .get(this.#canvasId, threadId); + if (current === undefined) return null; + const existing = decodeChanges( + current['snapshot_json'], + this.#canvasId, + threadId, + ); + const index = existing.findIndex((record) => record.id === changeId); + if (index < 0) return null; + const [removed] = existing.splice(index, 1); + database + .prepare( + `UPDATE changes + SET snapshot_json = ? + WHERE canvas_id = ? AND thread_id = ?`, + ) + .run( + stringifyJson(existing, `Changes for thread ${threadId}`), + this.#canvasId, + threadId, + ); + return removed ?? null; + }); + } +} + +export function createSqliteSpaceLogs( + context: SqliteStoreContext, + workspaceId: string, + canvasId: string, +): SqliteSpaceLogs { + const coordinator = new SqliteSpaceLogCoordinator( + context, + workspaceId, + canvasId, + ); + return Object.freeze({ + events: Object.freeze({ + read: (limit?: number) => coordinator.readEvents(limit), + append: (events: readonly NewCanvasEvent[]) => + coordinator.appendEvents(events), + }), + changes: Object.freeze({ + read: (threadId: string) => coordinator.readChanges(threadId), + append: (threadId: string, records: readonly CanvasChangeRecord[]) => + coordinator.appendChanges(threadId, records), + delete: (threadId: string, changeId: string) => + coordinator.deleteChange(threadId, changeId), + }), + }); +} diff --git a/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts b/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts new file mode 100644 index 000000000..48ac2cc63 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts @@ -0,0 +1,338 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { randomUUID } from 'node:crypto'; + +import { withImmediateTransaction } from './database.js'; +import { allocateNodeIdentity } from './identity.js'; +import { + decodeNodeRecord, + requireRevision, + spaceRowExists, + stringifyJson, + validateNodeContent, +} from './rows.js'; +import { sanitizeId } from '../../../../utils/fs.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { + NodeDeleteResult, + NodePutInput, + NodePutResult, + NodeSnapshot, + NodeStreamOptions, + SpaceNodes, +} from '../../ports/structured.js'; +import type { DatabaseSync } from 'node:sqlite'; + +interface NodeRow { + readonly record: NodeSnapshot['record']; + readonly revision: string; + readonly collisionKey: string; +} + +function decodeNodeRow(value: unknown, nodeId: string): NodeRow { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new SyntaxError(`Malformed persisted Node ${JSON.stringify(nodeId)}`); + } + const row = value as Record; + const collisionKey = row['label_collision_key']; + if (typeof collisionKey !== 'string') { + throw new SyntaxError( + `Invalid collision key for Node ${JSON.stringify(nodeId)}`, + ); + } + return { + record: decodeNodeRecord(row['record_json'], nodeId), + revision: requireRevision(row['revision'], nodeId), + collisionKey, + }; +} + +function readNodeRow( + database: DatabaseSync, + canvasId: string, + nodeId: string, +): NodeRow | null { + const row = database + .prepare( + `SELECT record_json, revision, label_collision_key + FROM nodes + WHERE canvas_id = ? AND node_id = ?`, + ) + .get(canvasId, nodeId); + return row === undefined ? null : decodeNodeRow(row, nodeId); +} + +/** + * Ids per `readMany` statement. + * + * Comfortably under SQLite's default 999-parameter ceiling with room for the + * `canvas_id` bind, so a caller never has to know the limit exists. + */ +const READ_MANY_CHUNK = 500; + +/** Decode one scanned row into the id the port keys collections by. */ +function decodeIdentifiedNodeRow(value: unknown): [string, NodeSnapshot] { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new SyntaxError('Malformed persisted SQLite Node row'); + } + const nodeId = (value as Record)['node_id']; + if (typeof nodeId !== 'string') { + throw new SyntaxError('Invalid node_id in persisted SQLite Node'); + } + const row = decodeNodeRow(value, nodeId); + return [nodeId, { record: row.record, revision: row.revision }]; +} + +function collectNodeRow(value: unknown, into: Map): void { + const [nodeId, snapshot] = decodeIdentifiedNodeRow(value); + into.set(nodeId, snapshot); +} + +function validatePut(input: NodePutInput): string { + const nodeId = sanitizeId(input.nodeId, 'nodeId'); + validateNodeContent(input.record, nodeId); + if ( + input.expectedRevision !== undefined && + input.expectedRevision !== null && + typeof input.expectedRevision !== 'string' + ) { + throw new TypeError('expectedRevision must be a string, null, or omitted'); + } + return nodeId; +} + +/** Apply one node put inside the caller's active transaction. */ +export function putSqliteNodeInTransaction( + database: DatabaseSync, + workspaceId: string, + canvasId: string, + input: NodePutInput, +): NodePutResult { + const nodeId = validatePut(input); + if (!spaceRowExists(database, workspaceId, canvasId)) { + return { ok: false, reason: 'not-found' }; + } + + const current = readNodeRow(database, canvasId, nodeId); + const currentRevision = current?.revision ?? null; + if ( + input.expectedRevision !== undefined && + input.expectedRevision !== currentRevision + ) { + return { + ok: false, + reason: 'revision-conflict', + currentRevision, + }; + } + + const occupied = database + .prepare( + `SELECT label_collision_key + FROM nodes + WHERE canvas_id = ? AND node_id <> ?`, + ) + .all(canvasId, nodeId) + .map((row) => row['label_collision_key']) + .filter((value): value is string => typeof value === 'string'); + const allocation = allocateNodeIdentity( + input.record, + nodeId, + current?.collisionKey ?? null, + input.strictLabel === true ? [] : occupied, + ); + + if (input.strictLabel === true) { + const conflict = database + .prepare( + `SELECT node_id, record_json, label_collision_key + FROM nodes + WHERE canvas_id = ? + AND label_collision_key = ? + AND node_id <> ?`, + ) + .get(canvasId, allocation.desiredCollisionKey, nodeId); + if (conflict !== undefined) { + const conflictingNodeId = conflict['node_id']; + const collisionKey = conflict['label_collision_key']; + if (typeof conflictingNodeId !== 'string') { + throw new SyntaxError('Invalid conflicting SQLite Node id'); + } + const conflicting = decodeNodeRecord( + conflict['record_json'], + conflictingNodeId, + ); + return { + ok: false, + reason: 'label-conflict', + conflictingNodeId, + conflictingLabel: + typeof conflicting.label === 'string' + ? conflicting.label + : typeof collisionKey === 'string' + ? collisionKey + : conflictingNodeId, + }; + } + } + + const revision = randomUUID(); + database + .prepare( + `INSERT INTO nodes ( + canvas_id, node_id, record_json, revision, label_collision_key + ) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(canvas_id, node_id) DO UPDATE SET + record_json = excluded.record_json, + revision = excluded.revision, + label_collision_key = excluded.label_collision_key`, + ) + .run( + canvasId, + nodeId, + stringifyJson(allocation.record, `Node ${JSON.stringify(nodeId)} record`), + revision, + allocation.collisionKey, + ); + return { + ok: true, + record: allocation.record, + revision, + }; +} + +export class SqliteSpaceNodes implements SpaceNodes { + readonly canvasId: string; + + readonly #context: SqliteStoreContext; + readonly #workspaceId: string; + + constructor( + context: SqliteStoreContext, + workspaceId: string, + canvasId: string, + ) { + this.#context = context; + this.#workspaceId = workspaceId; + this.canvasId = canvasId; + } + + #workspace(): string { + return this.#context.assertBoundWorkspace( + this.#workspaceId, + `SQLite Space nodes(${this.canvasId})`, + ); + } + + async read(nodeIdInput: string): Promise { + const nodeId = sanitizeId(nodeIdInput, 'nodeId'); + this.#workspace(); + const current = readNodeRow( + this.#context.database(), + this.canvasId, + nodeId, + ); + return current === null + ? null + : { record: current.record, revision: current.revision }; + } + + async readMany( + nodeIds: readonly string[], + ): Promise> { + const wanted = [...new Set(nodeIds)].map((nodeId) => + sanitizeId(nodeId, 'nodeId'), + ); + // Before the empty-batch shortcut: asking a closed store for nothing is + // still asking a closed store. + this.#workspace(); + const database = this.#context.database(); + const snapshots = new Map(); + if (wanted.length === 0) return snapshots; + + // One statement per batch rather than one per id: a neighbourhood read + // asks for tens of nodes, and the port exists so that cost stays + // proportional to the request. SQLite caps a statement at + // SQLITE_MAX_VARIABLE_NUMBER parameters, so the batch is chunked rather + // than assumed to fit. + for (let start = 0; start < wanted.length; start += READ_MANY_CHUNK) { + const chunk = wanted.slice(start, start + READ_MANY_CHUNK); + const placeholders = chunk.map(() => '?').join(', '); + const rows = database + .prepare( + `SELECT node_id, record_json, revision, label_collision_key + FROM nodes + WHERE canvas_id = ? AND node_id IN (${placeholders})`, + ) + .all(this.canvasId, ...chunk); + for (const value of rows) collectNodeRow(value, snapshots); + } + return snapshots; + } + + async list(): Promise> { + const snapshots = new Map(); + for (const value of this.#scan()) collectNodeRow(value, snapshots); + return snapshots; + } + + async stream( + onNode: (snapshot: NodeSnapshot) => void, + options?: NodeStreamOptions, + ): Promise> { + const delivered = new Map(); + // Decoded row by row off a live cursor, so a reader that renders partial + // results sees the first node without waiting for the last, and an + // aborted scan stops reading rather than discarding rows it already + // materialized. + for (const value of this.#scan()) { + if (options?.signal?.aborted) break; + const [nodeId, snapshot] = decodeIdentifiedNodeRow(value); + onNode(snapshot); + delivered.set(nodeId, snapshot); + } + return delivered; + } + + #scan(): Iterable { + this.#workspace(); + return this.#context + .database() + .prepare( + `SELECT node_id, record_json, revision, label_collision_key + FROM nodes + WHERE canvas_id = ?`, + ) + .iterate(this.canvasId); + } + + async put(input: NodePutInput): Promise { + validatePut(input); + const workspaceId = this.#workspace(); + this.#context.assertMutationAllowed(this.canvasId); + const database = this.#context.database(); + return withImmediateTransaction(database, () => + putSqliteNodeInTransaction(database, workspaceId, this.canvasId, input), + ); + } + + async delete(nodeIdInput: string): Promise { + const nodeId = sanitizeId(nodeIdInput, 'nodeId'); + const workspaceId = this.#workspace(); + this.#context.assertMutationAllowed(this.canvasId); + const database = this.#context.database(); + return withImmediateTransaction(database, () => { + if (!spaceRowExists(database, workspaceId, this.canvasId)) { + return 'absent' as const; + } + const deleted = Number( + database + .prepare('DELETE FROM nodes WHERE canvas_id = ? AND node_id = ?') + .run(this.canvasId, nodeId).changes, + ); + return deleted === 1 ? ('deleted' as const) : ('absent' as const); + }); + } +} diff --git a/apps/server/src/modules/storage/backends/sqlite/space-repository.ts b/apps/server/src/modules/storage/backends/sqlite/space-repository.ts new file mode 100644 index 000000000..019d0136d --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/space-repository.ts @@ -0,0 +1,336 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { randomUUID } from 'node:crypto'; + +import { withImmediateTransaction } from './database.js'; +import { allocateSpaceIdentity, collisionKeyForTitle } from './identity.js'; +import { + decodeSpaceRow, + insertSpaceRow, + occupiedCollisionKeys, + readSpaceRow, + SPACE_COLUMNS, +} from './rows.js'; +import { sanitizeId } from '../../../../utils/fs.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { CanvasFile } from '../../../canvas/persistence-types.js'; +import type { + SpaceBeginDeleteResult, + SpaceCreateInput, + SpaceCreateResult, + SpaceDeleteInput, + SpaceDeleteSession, + SpaceRenameInput, + SpaceRenameResult, + SpaceRepository, +} from '../../ports/structured.js'; +import type { CanvasSummary } from '@huabu/shared'; +import type { DatabaseSync } from 'node:sqlite'; + +/** + * Whether this Space id is taken anywhere in the database. + * + * Space ids are the primary key across every Workspace, so creation has to ask + * globally even though everything else is scoped. + */ +function spaceRowExistsAnywhere( + database: DatabaseSync, + canvasId: string, +): boolean { + return ( + database + .prepare('SELECT 1 AS present FROM spaces WHERE canvas_id = ?') + .get(canvasId)?.['present'] === 1 + ); +} + +function validateTitle(title: unknown): asserts title is string | null { + if (title !== null && typeof title !== 'string') { + throw new TypeError('Space title must be a string or null'); + } +} + +export class SqliteSpaceRepository implements SpaceRepository { + readonly #context: SqliteStoreContext; + readonly #workspaceId: string; + + constructor(context: SqliteStoreContext) { + this.#context = context; + // Bound at construction, like the Disk repository binds the workspace + // path: one repository instance spans a caller's read and its follow-up + // write, and a Workspace switch in between must reject rather than + // silently retarget the write. + this.#workspaceId = context.workspaceId(); + } + + /** The Workspace every query below is scoped to, re-checked per call. */ + #workspace(): string { + return this.#context.assertBoundWorkspace( + this.#workspaceId, + 'SQLite Space repository', + ); + } + + async list(): Promise { + const workspaceId = this.#workspace(); + const database = this.#context.database(); + return database + .prepare( + `SELECT ${SPACE_COLUMNS} + FROM spaces + WHERE workspace_id = ? AND is_world = 0`, + ) + .all(workspaceId) + .map((row) => { + const { record } = decodeSpaceRow(row); + return { + canvasId: record.canvasId, + title: record.title, + nodeCount: record.state.nodes.length, + createdAt: record.createdAt, + updatedAt: record.updatedAt, + }; + }); + } + + async worldId(): Promise { + const workspaceId = this.#workspace(); + const database = this.#context.database(); + const rows = database + .prepare( + `SELECT ${SPACE_COLUMNS} + FROM spaces + WHERE workspace_id = ? AND is_world = 1`, + ) + .all(workspaceId); + if (rows.length !== 1) { + throw new Error( + rows.length === 0 + ? 'SQLite namespace has no World Space' + : 'SQLite namespace has multiple World Spaces', + ); + } + const world = decodeSpaceRow(rows[0]); + if (!world.isWorld) throw new Error('SQLite World Space is malformed'); + return sanitizeId(world.record.canvasId, 'world canvasId'); + } + + async ensureWorld(): Promise { + const workspaceId = this.#workspace(); + const database = this.#context.database(); + return withImmediateTransaction(database, () => { + const existing = database + .prepare( + `SELECT ${SPACE_COLUMNS} + FROM spaces + WHERE workspace_id = ? AND is_world = 1`, + ) + .all(workspaceId); + if (existing.length > 1) { + throw new Error('SQLite namespace has multiple World Spaces'); + } + if (existing.length === 1) { + const world = decodeSpaceRow(existing[0]); + if (!world.isWorld) throw new Error('SQLite World Space is malformed'); + return sanitizeId(world.record.canvasId, 'world canvasId'); + } + + const canvasId = randomUUID(); + const timestamp = this.#context.now(); + if (!Number.isFinite(timestamp)) { + throw new TypeError('SQLite Space clock returned a non-finite value'); + } + insertSpaceRow( + database, + workspaceId, + { + canvasId, + title: 'World', + version: 0, + state: { nodes: [], edges: [] }, + createdAt: timestamp, + updatedAt: timestamp, + }, + '', + true, + ); + return canvasId; + }); + } + + async create(input: SpaceCreateInput): Promise { + const canvasId = sanitizeId(input.canvasId, 'canvasId'); + validateTitle(input.title); + const workspaceId = this.#workspace(); + this.#context.assertMutationAllowed(canvasId); + const database = this.#context.database(); + + return withImmediateTransaction(database, () => { + // Existence is checked across every Workspace, not just the active one: + // `canvas_id` is the primary key, so an id already used elsewhere is + // taken here too, and reporting it as free would fail on INSERT. + if (spaceRowExistsAnywhere(database, canvasId)) { + return { ok: false as const, reason: 'already-exists' as const }; + } + const identity = allocateSpaceIdentity( + input.title, + canvasId, + occupiedCollisionKeys(database, workspaceId), + ); + const timestamp = this.#context.now(); + if (!Number.isFinite(timestamp)) { + throw new TypeError('SQLite Space clock returned a non-finite value'); + } + const record: CanvasFile = { + canvasId, + title: identity.title, + version: 0, + state: { nodes: [], edges: [] }, + createdAt: timestamp, + updatedAt: timestamp, + }; + insertSpaceRow(database, workspaceId, record, identity.collisionKey); + return { ok: true as const, record }; + }); + } + + async beginDelete(input: SpaceDeleteInput): Promise { + const canvasId = sanitizeId(input.canvasId, 'canvasId'); + const workspaceId = this.#workspace(); + const beforeAdmission = readSpaceRow( + this.#context.database(), + workspaceId, + canvasId, + ); + if (beforeAdmission?.isWorld) { + return { ok: false, reason: 'world-forbidden' }; + } + + const release = await this.#context.acquireDelete(canvasId); + let sessionOwnsGate = false; + try { + const afterAdmission = readSpaceRow( + this.#context.database(), + workspaceId, + canvasId, + ); + if (afterAdmission?.isWorld) { + return { ok: false, reason: 'world-forbidden' }; + } + + let state: 'open' | 'finishing' | 'closed' = 'open'; + const close = (): void => { + if (state === 'closed') return; + state = 'closed'; + release(); + }; + const session: SpaceDeleteSession = Object.freeze({ + finish: async () => { + if (state !== 'open') { + throw new Error(`Space deletion session for ${canvasId} is closed`); + } + state = 'finishing'; + try { + this.#context.assertOpen(); + const database = this.#context.database(); + const result = withImmediateTransaction(database, () => { + const current = readSpaceRow(database, workspaceId, canvasId); + if (current?.isWorld) { + throw new Error(`Refusing to delete World Space ${canvasId}`); + } + if (current === null) { + return { + deleted: false, + }; + } + const deleted = Number( + database + .prepare( + 'DELETE FROM spaces WHERE workspace_id = ? AND canvas_id = ?', + ) + .run(workspaceId, canvasId).changes, + ); + return { deleted: deleted === 1 }; + }); + if (result.deleted) + return { ok: true as const, reason: 'deleted' as const }; + return { ok: false as const, reason: 'not-found' as const }; + } finally { + close(); + } + }, + abort: async () => { + if (state === 'finishing') { + throw new Error( + `Space deletion session for ${canvasId} is already finishing`, + ); + } + if (state === 'closed') return; + try { + this.#context.assertOpen(); + } finally { + close(); + } + }, + }); + sessionOwnsGate = true; + return { ok: true, session }; + } finally { + if (!sessionOwnsGate) release(); + } + } + + async rename(input: SpaceRenameInput): Promise { + const canvasId = sanitizeId(input.canvasId, 'canvasId'); + validateTitle(input.title); + const workspaceId = this.#workspace(); + this.#context.assertMutationAllowed(canvasId); + const database = this.#context.database(); + + return withImmediateTransaction(database, () => { + const current = readSpaceRow(database, workspaceId, canvasId); + if (current === null) return { ok: false, reason: 'not-found' } as const; + if (current.isWorld) { + return { ok: false, reason: 'world-forbidden' } as const; + } + if (current.record.title === input.title) { + return { ok: true, record: current.record } as const; + } + + const collisionKey = collisionKeyForTitle(input.title, canvasId); + if (collisionKey !== current.collisionKey) { + const conflict = database + .prepare( + `SELECT ${SPACE_COLUMNS} + FROM spaces + WHERE workspace_id = ? AND collision_key = ? AND canvas_id <> ?`, + ) + .get(workspaceId, collisionKey, canvasId); + if (conflict !== undefined) { + return { + ok: false, + reason: 'title-conflict', + conflictingTitle: decodeSpaceRow(conflict).record.title, + } as const; + } + } + + const result = database + .prepare( + `UPDATE spaces + SET title = ?, collision_key = ? + WHERE workspace_id = ? AND canvas_id = ?`, + ) + .run(input.title, collisionKey, workspaceId, canvasId); + if (Number(result.changes) !== 1) { + throw new Error(`Could not rename SQLite Space ${canvasId}`); + } + return { + ok: true, + record: { ...current.record, title: input.title }, + } as const; + }); + } +} diff --git a/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts b/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts new file mode 100644 index 000000000..57f82cba2 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts @@ -0,0 +1,261 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { + taskRecordSchema, + taskRunCompletionSchema, + taskRunRecordSchema, + taskStoreSnapshotSchema, + type TaskRecord, + type TaskRunCompletion, + type TaskRunRecord, + type TaskStoreSnapshot, +} from '@huabu/shared'; + +import { withImmediateTransaction } from './database.js'; +import { parseJson, spaceRowExists, stringifyJson } from './rows.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { + SpaceTaskRuns, + SpaceTasks, + TaskRunCompletionResult, + TaskRunUpdate, +} from '../../ports/structured.js'; + +const EMPTY_TASKS: TaskStoreSnapshot = { + version: 1, + tasks: [], + runs: [], +}; + +function validateSnapshot(value: unknown, canvasId: string): TaskStoreSnapshot { + const parsed = taskStoreSnapshotSchema.safeParse(value); + if (!parsed.success) { + throw new SyntaxError( + `Invalid Task store for Canvas ${canvasId}: ${parsed.error.issues[0]?.message ?? 'schema violation'}`, + ); + } + const taskIds = new Set(); + for (const task of parsed.data.tasks) { + if (task.canvasId !== canvasId) { + throw new SyntaxError( + `Invalid Task store for Canvas ${canvasId}: Task ${task.taskId} belongs to Canvas ${task.canvasId}`, + ); + } + if (taskIds.has(task.taskId)) { + throw new SyntaxError( + `Invalid Task store for Canvas ${canvasId}: duplicate Task ${task.taskId}`, + ); + } + taskIds.add(task.taskId); + } + const runIds = new Set(); + for (const run of parsed.data.runs) { + if (run.canvasIdSnapshot !== canvasId) { + throw new SyntaxError( + `Invalid Task store for Canvas ${canvasId}: Run ${run.runId} belongs to Canvas ${run.canvasIdSnapshot}`, + ); + } + if (runIds.has(run.runId)) { + throw new SyntaxError( + `Invalid Task store for Canvas ${canvasId}: duplicate Run ${run.runId}`, + ); + } + if (!taskIds.has(run.taskId)) { + throw new SyntaxError( + `Invalid Task store for Canvas ${canvasId}: Run ${run.runId} references missing Task ${run.taskId}`, + ); + } + runIds.add(run.runId); + } + return parsed.data; +} + +function readSnapshot( + context: SqliteStoreContext, + canvasId: string, +): TaskStoreSnapshot { + const row = context + .database() + .prepare('SELECT snapshot_json FROM tasks WHERE canvas_id = ?') + .get(canvasId); + if (row === undefined) { + return { ...EMPTY_TASKS, tasks: [], runs: [] }; + } + return validateSnapshot( + parseJson(row['snapshot_json'], `Task store for Canvas ${canvasId}`), + canvasId, + ); +} + +export class SqliteSpaceTasks implements SpaceTasks { + readonly runs: SpaceTaskRuns; + + readonly #context: SqliteStoreContext; + readonly #workspaceId: string; + readonly #canvasId: string; + + constructor( + context: SqliteStoreContext, + workspaceId: string, + canvasId: string, + ) { + this.#context = context; + this.#workspaceId = workspaceId; + this.#canvasId = canvasId; + this.runs = Object.freeze({ + create: (run: TaskRunRecord) => this.#createRun(run), + update: (runId: string, update: TaskRunUpdate) => + this.#updateRun(runId, update), + complete: ( + taskId: string, + runId: string, + completion: TaskRunCompletion, + ) => this.#completeRun(taskId, runId, completion), + }); + } + + #workspace(): string { + return this.#context.assertBoundWorkspace( + this.#workspaceId, + `SQLite Space Tasks(${this.#canvasId})`, + ); + } + + async read(): Promise { + this.#context.assertOpen(); + this.#workspace(); + return readSnapshot(this.#context, this.#canvasId); + } + + async create(task: TaskRecord): Promise { + const parsed = taskRecordSchema.safeParse(task); + if (!parsed.success || parsed.data.canvasId !== this.#canvasId) { + throw new TypeError(`Invalid Task record for Canvas ${this.#canvasId}`); + } + this.#mutate((snapshot) => { + if ( + snapshot.tasks.some( + (candidate) => candidate.taskId === parsed.data.taskId, + ) + ) { + throw new Error(`Task ${parsed.data.taskId} already exists`); + } + snapshot.tasks.push(parsed.data); + }); + } + + async #createRun(run: TaskRunRecord): Promise { + const parsed = taskRunRecordSchema.safeParse(run); + if (!parsed.success || parsed.data.canvasIdSnapshot !== this.#canvasId) { + throw new TypeError(`Invalid Run record for Canvas ${this.#canvasId}`); + } + this.#mutate((snapshot) => { + if ( + snapshot.runs.some((candidate) => candidate.runId === parsed.data.runId) + ) { + throw new Error(`Run ${parsed.data.runId} already exists`); + } + if ( + !snapshot.tasks.some( + (candidate) => candidate.taskId === parsed.data.taskId, + ) + ) { + throw new Error(`Task ${parsed.data.taskId} does not exist`); + } + snapshot.runs.push(parsed.data); + }); + } + + async #updateRun( + runId: string, + update: TaskRunUpdate, + ): Promise { + return this.#mutate((snapshot) => { + const index = snapshot.runs.findIndex((run) => run.runId === runId); + if (index < 0) throw new Error(`Run ${runId} does not exist`); + const parsed = taskRunRecordSchema.safeParse({ + ...snapshot.runs[index], + ...update, + }); + if (!parsed.success) { + throw new TypeError(`Invalid update for Run ${runId}`); + } + snapshot.runs[index] = parsed.data; + return parsed.data; + }); + } + + async #completeRun( + taskId: string, + runId: string, + completion: TaskRunCompletion, + ): Promise { + const parsedCompletion = taskRunCompletionSchema.safeParse(completion); + if (!parsedCompletion.success) { + throw new TypeError(`Invalid completion for Run ${runId}`); + } + return this.#mutate((snapshot) => { + if (!snapshot.tasks.some((task) => task.taskId === taskId)) { + return { outcome: 'task_not_found' }; + } + const index = snapshot.runs.findIndex((run) => run.runId === runId); + if (index < 0 || snapshot.runs[index]?.taskId !== taskId) { + return { outcome: 'run_not_found' }; + } + const current = snapshot.runs[index]; + if (!current) return { outcome: 'run_not_found' }; + if (current.status === 'completed') { + return current.completion?.message === parsedCompletion.data.message + ? { outcome: 'unchanged', run: current } + : { outcome: 'completion_conflict', run: current }; + } + if (current.status !== 'running') { + return { outcome: 'run_not_running', run: current }; + } + const parsedRun = taskRunRecordSchema.safeParse({ + ...current, + status: 'completed', + completion: parsedCompletion.data, + }); + if (!parsedRun.success) { + throw new TypeError(`Invalid completion update for Run ${runId}`); + } + snapshot.runs[index] = parsedRun.data; + return { outcome: 'completed', run: parsedRun.data }; + }); + } + + #mutate(apply: (snapshot: TaskStoreSnapshot) => T): T { + const workspaceId = this.#workspace(); + this.#context.assertMutationAllowed(this.#canvasId); + const database = this.#context.database(); + return withImmediateTransaction(database, () => { + if (!spaceRowExists(database, workspaceId, this.#canvasId)) { + throw new Error( + `Space Tasks(${this.#canvasId}) cannot write a missing Space`, + ); + } + const current = readSnapshot(this.#context, this.#canvasId); + const next: TaskStoreSnapshot = { + version: 1, + tasks: [...current.tasks], + runs: [...current.runs], + }; + const result = apply(next); + database + .prepare( + `INSERT INTO tasks (canvas_id, snapshot_json) + VALUES (?, ?) + ON CONFLICT(canvas_id) DO UPDATE SET + snapshot_json = excluded.snapshot_json`, + ) + .run( + this.#canvasId, + stringifyJson(next, `Task store for Canvas ${this.#canvasId}`), + ); + return result; + }); + } +} diff --git a/apps/server/src/modules/storage/backends/sqlite/space-write.ts b/apps/server/src/modules/storage/backends/sqlite/space-write.ts new file mode 100644 index 000000000..774fb381c --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/space-write.ts @@ -0,0 +1,193 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { withImmediateTransaction } from './database.js'; +import { allocateSpaceIdentity } from './identity.js'; +import { + insertSpaceRow, + occupiedCollisionKeys, + readSpaceRow, + stringifyJson, + updateSpaceRow, + validateCanvasFile, + validateNodeContent, +} from './rows.js'; +import { putSqliteNodeInTransaction } from './space-nodes.js'; +import { sanitizeId } from '../../../../utils/fs.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { + NodePutResult, + SpaceHandle, + SpaceNodeMutation, + SpaceWriteInput, + SpaceWriteResult, +} from '../../ports/structured.js'; + +function mutationError( + mutation: SpaceNodeMutation, + result: NodePutResult, +): Error { + const prefix = `Space write failed for node ${JSON.stringify(mutation.nodeId)}`; + if (result.ok) return new Error(`${prefix}: unexpected success result`); + switch (result.reason) { + case 'not-found': + return new Error(`${prefix}: Space does not exist`); + case 'revision-conflict': + return new Error(`${prefix}: unexpected revision conflict`); + case 'label-conflict': + return new Error( + `${prefix}: label conflicts with node ${JSON.stringify(result.conflictingNodeId)}`, + ); + case 'duplicate-node': + return new Error(`${prefix}: duplicate persisted node`); + case 'write-suppressed': + return new Error(`${prefix}: write is suppressed after deletion`); + } +} + +function validateInput(canvasId: string, input: SpaceWriteInput): void { + if (!Number.isFinite(input.expectedVersion)) { + throw new TypeError('expectedVersion must be a finite number'); + } + validateCanvasFile(input.nextRecord, canvasId); + if (input.nextRecord.version !== input.expectedVersion + 1) { + throw new Error( + `SpaceWrite(${canvasId}) expected nextRecord.version ` + + `${input.expectedVersion + 1}, received ${input.nextRecord.version}`, + ); + } + if ( + input.allowCreate === true && + (input.nodeMutations.length > 0 || input.delta !== undefined) + ) { + throw new Error( + 'allowCreate is valid only for a record-only structural write', + ); + } + if ( + input.delta !== undefined && + input.delta.version !== input.nextRecord.version + ) { + throw new Error( + 'delta.version must equal the committed Space record version', + ); + } + if (input.delta !== undefined) { + stringifyJson(input.delta, `Space ${JSON.stringify(canvasId)} delta`); + } + for (const mutation of input.nodeMutations) { + sanitizeId(mutation.nodeId, 'nodeId'); + if (mutation.kind === 'put') { + validateNodeContent(mutation.record, mutation.nodeId); + } + } +} + +/** Bind the atomic SQLite record/node/delta write to one Space. */ +export function createSqliteSpaceWrite( + context: SqliteStoreContext, + boundWorkspaceId: string, + canvasId: string, +): SpaceHandle['write'] { + return async function write( + input: SpaceWriteInput, + ): Promise { + const workspaceId = context.assertBoundWorkspace( + boundWorkspaceId, + `SpaceWrite(${canvasId})`, + ); + context.assertMutationAllowed(canvasId); + validateInput(canvasId, input); + const database = context.database(); + + const completed = withImmediateTransaction(database, () => { + const current = readSpaceRow(database, workspaceId, canvasId); + if (current === null) { + if (!input.allowCreate) { + return { ok: false, reason: 'not-found' } as const; + } + if (input.expectedVersion !== 0) { + throw new Error( + `SpaceWrite(${canvasId}) can create only from version 0`, + ); + } + const identity = allocateSpaceIdentity( + input.nextRecord.title, + canvasId, + occupiedCollisionKeys(database, workspaceId), + ); + insertSpaceRow( + database, + workspaceId, + { ...input.nextRecord, title: identity.title }, + identity.collisionKey, + ); + return { ok: true } as const; + } + + if (current.record.version !== input.expectedVersion) { + return { + ok: false, + reason: 'version-conflict', + actualVersion: current.record.version, + } as const; + } + if (input.nextRecord.createdAt !== current.record.createdAt) { + throw new Error(`SpaceWrite(${canvasId}) refusing to change createdAt`); + } + if (input.nextRecord.title !== current.record.title) { + throw new Error( + `SpaceWrite(${canvasId}) cannot change title; ` + + 'use SpaceRepository.rename first', + ); + } + + for (const mutation of input.nodeMutations) { + if (mutation.kind === 'delete') { + database + .prepare('DELETE FROM nodes WHERE canvas_id = ? AND node_id = ?') + .run(canvasId, mutation.nodeId); + continue; + } + + const result = putSqliteNodeInTransaction( + database, + workspaceId, + canvasId, + { + nodeId: mutation.nodeId, + record: mutation.record, + strictLabel: mutation.strictLabel, + }, + ); + if (!result.ok) throw mutationError(mutation, result); + } + + if ( + updateSpaceRow( + database, + workspaceId, + input.nextRecord, + input.expectedVersion, + ) !== 1 + ) { + throw new Error(`SpaceWrite(${canvasId}) lost its version race`); + } + if (input.delta !== undefined) { + database + .prepare( + `INSERT INTO delta_log (canvas_id, version, entry_json) + VALUES (?, ?, ?)`, + ) + .run( + canvasId, + input.delta.version, + stringifyJson(input.delta, `Space ${canvasId} delta`), + ); + } + return { ok: true } as const; + }); + return completed; + }; +} diff --git a/apps/server/src/modules/storage/backends/sqlite/structured-store.ts b/apps/server/src/modules/storage/backends/sqlite/structured-store.ts new file mode 100644 index 000000000..267facb5e --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/structured-store.ts @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { SqliteStoreContext } from './database.js'; +import { readSpaceRow } from './rows.js'; +import { + createSqliteSpaceExtension, + resolveSqliteSpaceExtension, +} from './space-extension.js'; +import { createSqliteSpaceLogs } from './space-logs.js'; +import { SqliteSpaceNodes } from './space-nodes.js'; +import { SqliteSpaceRepository } from './space-repository.js'; +import { SqliteSpaceTasks } from './space-tasks.js'; +import { createSqliteSpaceWrite } from './space-write.js'; +import { sanitizeId } from '../../../../utils/fs.js'; + +import type { SqliteSpaceSubstrate } from './space-extension.js'; +import type { StorageHealth } from '../../ports/common.js'; +import type { + SpaceHandle, + SpaceRepository, + StructuredStore, +} from '../../ports/structured.js'; + +/** + * Structured-store adapter over one `node:sqlite` connection. + * + * The connection is shared with the Workspace repository — one database file + * cannot have two writers — so this class does not assume it owns the + * lifecycle. Constructed with a filename it opens and closes its own + * connection; constructed with an existing context it borrows one, and + * `init`/`close` become the shared owner's business. + */ +export class SqliteStructuredStore implements StructuredStore { + readonly kind = 'sqlite' as const; + + readonly #context: SqliteStoreContext; + readonly #ownsContext: boolean; + + constructor( + source: string | SqliteStoreContext, + now: () => number = Date.now, + ) { + if (source instanceof SqliteStoreContext) { + this.#context = source; + this.#ownsContext = false; + return; + } + if (typeof source !== 'string') { + throw new TypeError('SQLite filename must be a string'); + } + if (source.length === 0) { + throw new TypeError('SQLite filename must not be empty'); + } + this.#context = new SqliteStoreContext(source, now); + this.#ownsContext = true; + } + + /** The shared connection, for composition that wires a second port on it. */ + get context(): SqliteStoreContext { + return this.#context; + } + + async init(): Promise { + if (this.#ownsContext) this.#context.init(); + else this.#context.assertOpen(); + } + + async health(): Promise { + return this.#context.health(this.kind); + } + + async close(): Promise { + if (this.#ownsContext) this.#context.close(); + } + + spaces(): SpaceRepository { + return Object.freeze(new SqliteSpaceRepository(this.#context)); + } + + /** + * The synchronous form of `space(canvasId).extension(namespace)`. + * + * Off the port on purpose: it is a SQLite capability, and the composition + * root hands it to owners the same way it hands out `diskTree` — named for + * the backend that has it, absent everywhere else. + */ + extensionSync( + canvasIdInput: string, + namespace: string, + ): SqliteSpaceSubstrate | null { + const canvasId = sanitizeId(canvasIdInput, 'canvasId'); + return resolveSqliteSpaceExtension( + this.#context, + this.#context.workspaceId(), + canvasId, + namespace, + ); + } + + space(canvasIdInput: string): SpaceHandle { + const canvasId = sanitizeId(canvasIdInput, 'canvasId'); + // Bound once, here, so every part of this handle answers for the same + // Workspace and a switch invalidates all of them together. + const workspaceId = this.#context.workspaceId(); + const { events, changes } = createSqliteSpaceLogs( + this.#context, + workspaceId, + canvasId, + ); + const nodes = Object.freeze( + new SqliteSpaceNodes(this.#context, workspaceId, canvasId), + ); + const tasks = Object.freeze( + new SqliteSpaceTasks(this.#context, workspaceId, canvasId), + ); + return Object.freeze({ + canvasId, + read: async () => { + this.#context.assertBoundWorkspace( + workspaceId, + `SQLite Space(${canvasId})`, + ); + return ( + readSpaceRow(this.#context.database(), workspaceId, canvasId) + ?.record ?? null + ); + }, + write: createSqliteSpaceWrite(this.#context, workspaceId, canvasId), + nodes, + changes, + tasks, + events, + extension: createSqliteSpaceExtension( + this.#context, + workspaceId, + canvasId, + ), + }); + } +} diff --git a/apps/server/src/modules/storage/backends/sqlite/test-support.ts b/apps/server/src/modules/storage/backends/sqlite/test-support.ts new file mode 100644 index 000000000..50155664c --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/test-support.ts @@ -0,0 +1,212 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; + +import { SqliteStoreContext, SQLITE_SCHEMA_VERSION } from './database.js'; +import { collisionKeyForTitle } from './identity.js'; +import { insertSpaceRow, parseJson } from './rows.js'; +import { SqliteStructuredStore } from './structured-store.js'; +import { SqliteWorkspaceRepository } from './workspace-repository.js'; + +import type { + CanvasFile, + DeltaLogEntry, +} from '../../../canvas/persistence-types.js'; + +export const SQLITE_TEST_WORLD_ID = 'sqlite-test-world'; +export const SQLITE_TEST_WORKSPACE_NAME = 'Test Workspace'; + +export interface SqliteTestFile { + readonly directory: string; + readonly filename: string; + readonly remove: () => void; +} + +export interface EmptySqliteTestStore extends SqliteTestFile { + readonly store: SqliteStructuredStore; + readonly context: SqliteStoreContext; + readonly workspaceId: string; + /** Drop the connection but keep the file, so a test can reopen it. */ + readonly closeConnection: () => void; + readonly cleanup: () => Promise; +} + +export interface OpenSqliteTestStore extends EmptySqliteTestStore { + readonly world: CanvasFile; +} + +export function createSqliteTestFile(prefix = 'huabu-sqlite-'): SqliteTestFile { + const directory = mkdtempSync(path.join(tmpdir(), prefix)); + const filename = path.join(directory, 'structured.sqlite'); + let removed = false; + return { + directory, + filename, + remove: () => { + if (removed) return; + removed = true; + rmSync(directory, { recursive: true, force: true }); + }, + }; +} + +/** Run a short test-only query through a connection independent of the store. */ +export function withTestDatabase( + filename: string, + operation: (database: DatabaseSync) => T, +): T { + const database = new DatabaseSync(filename); + try { + database.exec('PRAGMA foreign_keys = ON'); + return operation(database); + } finally { + database.close(); + } +} + +/** + * Seed World without reaching through the adapter under test. + * + * The store first creates the production schema. This helper then opens a + * separate node:sqlite connection and uses the production row encoder, so a + * contract cannot pass because World creation accidentally shares private + * adapter state with the operation being exercised. + */ +export function seedSqliteWorld( + filename: string, + workspaceId: string, + canvasId = SQLITE_TEST_WORLD_ID, +): CanvasFile { + const record: CanvasFile = { + canvasId, + title: 'World', + version: 0, + state: { nodes: [], edges: [] }, + createdAt: 1, + updatedAt: 1, + }; + withTestDatabase(filename, (database) => { + const version = database.prepare('PRAGMA user_version').get()?.[ + 'user_version' + ]; + if (version !== SQLITE_SCHEMA_VERSION) { + throw new Error( + `Expected production SQLite schema v${SQLITE_SCHEMA_VERSION}, got ${String(version)}`, + ); + } + insertSpaceRow( + database, + workspaceId, + record, + collisionKeyForTitle(record.title, record.canvasId), + true, + ); + }); + return record; +} + +/** + * Open a store on a fresh file with one activated Workspace. + * + * Every Space query is Workspace-scoped, so a store with no active Workspace + * refuses — the same way a Disk adapter refuses before a workspace path is + * committed. Tests get one activated Workspace so they can address Spaces + * without repeating the lifecycle. + */ +export async function openEmptySqliteTestStore( + prefix = 'huabu-sqlite-empty-', + now?: () => number, +): Promise { + const file = createSqliteTestFile(prefix); + const context = new SqliteStoreContext(file.filename, now); + const store = new SqliteStructuredStore(context); + try { + context.init(); + const workspace = await new SqliteWorkspaceRepository(context).create( + SQLITE_TEST_WORKSPACE_NAME, + ); + context.useWorkspace(workspace.workspaceId); + return { + ...file, + store, + context, + workspaceId: workspace.workspaceId, + closeConnection: () => context.close(), + cleanup: async () => { + context.close(); + file.remove(); + }, + }; + } catch (error) { + context.close(); + file.remove(); + throw error; + } +} + +export async function openSqliteTestStore( + prefix = 'huabu-sqlite-', + now?: () => number, +): Promise { + const opened = await openEmptySqliteTestStore(prefix, now); + try { + const world = seedSqliteWorld(opened.filename, opened.workspaceId); + return { ...opened, world }; + } catch (error) { + await opened.cleanup(); + throw error; + } +} + +export function readSqliteDeltaLog( + filename: string, + canvasId: string, +): DeltaLogEntry[] { + return withTestDatabase(filename, (database) => + database + .prepare( + `SELECT entry_json + FROM delta_log + WHERE canvas_id = ? + ORDER BY version`, + ) + .all(canvasId) + .map( + (row, index) => + parseJson( + row['entry_json'], + `test delta row ${index} for ${canvasId}`, + ) as DeltaLogEntry, + ), + ); +} + +/** Install a real SQLite failure immediately before a delta row is inserted. */ +export function installDeltaAbortTrigger( + filename: string, + message: string, +): () => void { + const quotedMessage = message.split("'").join("''"); + withTestDatabase(filename, (database) => { + database.exec('DROP TRIGGER IF EXISTS test_abort_delta_insert'); + database.exec(` + CREATE TRIGGER test_abort_delta_insert + BEFORE INSERT ON delta_log + BEGIN + SELECT RAISE(ABORT, '${quotedMessage}'); + END + `); + }); + let restored = false; + return () => { + if (restored) return; + restored = true; + withTestDatabase(filename, (database) => { + database.exec('DROP TRIGGER IF EXISTS test_abort_delta_insert'); + }); + }; +} diff --git a/apps/server/src/modules/storage/backends/sqlite/workspace-repository.ts b/apps/server/src/modules/storage/backends/sqlite/workspace-repository.ts new file mode 100644 index 000000000..546850701 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/workspace-repository.ts @@ -0,0 +1,222 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * SQLite implementation of the Workspace storage port. + * + * A Workspace here is a row, not a folder. That is the whole difference + * between this adapter and the Disk one, and it is why selecting a SQL profile + * asks the operator for no directory: the port never promised a location, only + * an identity and a name (`ports/workspace.ts`), and the Disk repository's + * path index is a materialization fact that lives beside it rather than in it. + * + * `remove()` is a **forget**, not a delete. The port's wording is deliberate — + * "forget one member without deleting any Workspace-owned data" — and on Disk + * that is easy to honour because the folder outlives the registry entry. A + * database has no such second copy, so forgetting is recorded as a timestamp + * and the rows stay: a listing skips them, and nothing a user authored is + * destroyed by an operation whose name does not say "delete". + */ + +import { randomUUID } from 'node:crypto'; + +import { withImmediateTransaction } from './database.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { + WorkspaceHandle, + WorkspaceRepository, +} from '../../ports/workspace.js'; +import type { DatabaseSync } from 'node:sqlite'; + +const WORKSPACE_COLUMNS = 'workspace_id, name, created_at, last_opened_at'; + +function decodeWorkspaceRow(value: unknown): WorkspaceHandle { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new SyntaxError('Malformed persisted SQLite Workspace row'); + } + const row = value as Record; + const workspaceId = row['workspace_id']; + const name = row['name']; + if (typeof workspaceId !== 'string' || workspaceId.length === 0) { + throw new SyntaxError('Invalid workspace_id in persisted SQLite Workspace'); + } + if (typeof name !== 'string') { + throw new SyntaxError('Invalid name in persisted SQLite Workspace'); + } + return { workspaceId, name }; +} + +function requireName(name: unknown): string { + if (typeof name !== 'string') { + throw new TypeError('Workspace name must be a string'); + } + const trimmed = name.trim(); + if (trimmed.length === 0) { + throw new TypeError('Workspace name must not be empty'); + } + return trimmed; +} + +function readWorkspaceRow( + database: DatabaseSync, + workspaceId: string, +): WorkspaceHandle | null { + const row = database + .prepare( + `SELECT ${WORKSPACE_COLUMNS} + FROM workspaces + WHERE workspace_id = ? AND forgotten_at IS NULL`, + ) + .get(workspaceId); + return row === undefined ? null : decodeWorkspaceRow(row); +} + +function insertWorkspaceRow( + database: DatabaseSync, + workspaceId: string, + name: string, + timestamp: number, +): void { + if (!Number.isFinite(timestamp)) { + throw new TypeError('SQLite Workspace clock returned a non-finite value'); + } + database + .prepare( + `INSERT INTO workspaces ( + workspace_id, name, created_at, last_opened_at, forgotten_at + ) VALUES (?, ?, ?, ?, NULL)`, + ) + .run(workspaceId, name, timestamp, timestamp); +} + +function markOpenedIn( + database: DatabaseSync, + workspaceId: string, + timestamp: number, +): void { + database + .prepare('UPDATE workspaces SET last_opened_at = ? WHERE workspace_id = ?') + .run(timestamp, workspaceId); +} + +export class SqliteWorkspaceRepository implements WorkspaceRepository { + readonly #context: SqliteStoreContext; + + constructor(context: SqliteStoreContext) { + this.#context = context; + } + + async get(workspaceId: string): Promise { + if (typeof workspaceId !== 'string' || workspaceId.length === 0) { + return null; + } + return readWorkspaceRow(this.#context.database(), workspaceId); + } + + async list(): Promise { + // Most recently opened first, matching the Disk registry's ordering, so a + // client rendering the picker gets the same list on either backend. + return this.#context + .database() + .prepare( + `SELECT ${WORKSPACE_COLUMNS} + FROM workspaces + WHERE forgotten_at IS NULL + ORDER BY last_opened_at DESC, created_at DESC`, + ) + .all() + .map(decodeWorkspaceRow); + } + + async rename( + workspaceId: string, + name: string, + ): Promise { + const trimmed = requireName(name); + const database = this.#context.database(); + return withImmediateTransaction(database, () => { + if (readWorkspaceRow(database, workspaceId) === null) return null; + database + .prepare( + `UPDATE workspaces + SET name = ? + WHERE workspace_id = ? AND forgotten_at IS NULL`, + ) + .run(trimmed, workspaceId); + return readWorkspaceRow(database, workspaceId); + }); + } + + async remove(workspaceId: string): Promise { + const database = this.#context.database(); + return withImmediateTransaction(database, () => { + if (readWorkspaceRow(database, workspaceId) === null) return false; + database + .prepare( + 'UPDATE workspaces SET forgotten_at = ? WHERE workspace_id = ?', + ) + .run(this.#context.now(), workspaceId); + return true; + }); + } + + // ─── Beyond the port ───────────────────────────────────────────────────── + // + // Creating a Workspace and recording that one was opened are lifecycle + // operations the port deliberately leaves out: on Disk they are "adopt this + // directory", which is a materialization fact. They are named for what this + // backend actually does instead of being bent into the shared shape. + + /** Register a new Workspace and return its identity. */ + async create(name: string): Promise { + const trimmed = requireName(name); + const workspaceId = randomUUID(); + insertWorkspaceRow( + this.#context.database(), + workspaceId, + trimmed, + this.#context.now(), + ); + return { workspaceId, name: trimmed }; + } + + /** + * The Workspace a fresh deployment starts in. + * + * A database nobody has opened before holds no Workspace, and a Server with + * no Workspace has nothing to show. The Disk profile answers this by asking + * the user for a folder; a SQL profile has nothing to ask for, so it starts + * one. Idempotent, and narrower than "create if absent": it mints a + * Workspace only when the database holds none at all, so forgetting the last + * one does not silently mint a second. + */ + async ensureDefault(name: string): Promise { + const trimmed = requireName(name); + const database = this.#context.database(); + return withImmediateTransaction(database, () => { + const existing = database + .prepare( + `SELECT ${WORKSPACE_COLUMNS} + FROM workspaces + WHERE forgotten_at IS NULL + ORDER BY last_opened_at DESC, created_at DESC + LIMIT 1`, + ) + .get(); + if (existing !== undefined) { + const workspace = decodeWorkspaceRow(existing); + markOpenedIn(database, workspace.workspaceId, this.#context.now()); + return workspace; + } + const workspaceId = randomUUID(); + insertWorkspaceRow(database, workspaceId, trimmed, this.#context.now()); + return { workspaceId, name: trimmed }; + }); + } + + /** Record that a Workspace was activated, for recency ordering. */ + markOpened(workspaceId: string): void { + markOpenedIn(this.#context.database(), workspaceId, this.#context.now()); + } +} diff --git a/apps/server/src/modules/storage/capabilities.test.ts b/apps/server/src/modules/storage/capabilities.test.ts index 527dd8b2c..c7f0330f9 100644 --- a/apps/server/src/modules/storage/capabilities.test.ts +++ b/apps/server/src/modules/storage/capabilities.test.ts @@ -27,13 +27,7 @@ const DISK: StorageProfile = { blobs: { kind: 'disk' }, }; -/** - * A profile naming a structured backend that has no adapter. - * - * The matrix has to answer for one before it exists — that is the point of - * declaring rather than discovering — so this stands in for the first backend - * that keeps Spaces in tables. - */ +/** The profile that keeps Spaces in tables and their bytes in files. */ const TABLES: StorageProfile = { structured: { kind: 'sqlite' }, blobs: { kind: 'disk' }, @@ -45,9 +39,17 @@ describe('storage capability matrix', () => { expect(new Set(ids).size).toBe(ids.length); for (const capability of STORAGE_CAPABILITIES) { - expect(capability.backends.length).toBeGreaterThan(0); // A capability nothing can serve is not a limitation, it is a removed - // feature; a capability every backend serves does not belong here. + // feature; one that names no axis at all is served everywhere and does + // not belong on an exception list. + const clauses = [ + capability.requires.structured, + capability.requires.blobs, + ].filter((clause) => clause !== undefined); + // A requirement with no clauses is met by every profile. + expect(clauses.length).toBeGreaterThan(0); + // A clause no backend satisfies is a removed feature, not a limitation. + for (const clause of clauses) expect(clause.length).toBeGreaterThan(0); expect(capability.summary).not.toHaveLength(0); expect(capability.rationale).not.toHaveLength(0); } @@ -58,16 +60,89 @@ describe('storage capability matrix', () => { expect(describeUnavailableCapabilities(DISK)).toEqual([]); }); - it('answers for a backend that has no adapter yet', () => { + it('answers for the backend that keeps Spaces in tables', () => { const missing = unavailableCapabilities(TABLES); // Every entry is Disk-only today, so a structured backend that is not // Disk loses all of them. The assertion is the shape, not the count. + // + // `TABLES` pairs SQLite records with Disk *bytes*, which is the profile + // this deployment actually runs, so this is also the answer to "does a + // real file system for bytes give any of these back". It does not: every + // entry needs the Space's record and node documents to be files, and + // those are rows whatever holds the bytes. expect(missing).toEqual(STORAGE_CAPABILITIES); expect(hasStorageCapability(TABLES, 'reveal-space-folder')).toBe(false); expect(hasStorageCapability(DISK, 'reveal-space-folder')).toBe(true); }); + /** + * The reason the matrix is keyed on the profile rather than on one axis. + * + * `disk`/`azure` has no adapter and `validateStorageProfile` would refuse + * it, which is exactly why it is the right shape to assert against: the + * matrix must already answer correctly for the pairing before anyone can + * select it. Records are files here and Spaces are real directories — a + * structured-only matrix would call the bundle exportable, and it would + * archive a Space folder whose artifacts had never been written to it. + */ + it('takes the bundle with a blob backend that cannot co-locate', () => { + const OFFSITE_BYTES: StorageProfile = { + structured: { kind: 'disk' }, + blobs: { kind: 'azure' }, + }; + + expect(hasStorageCapability(OFFSITE_BYTES, 'space-bundle-export')).toBe( + false, + ); + expect(hasStorageCapability(OFFSITE_BYTES, 'space-bundle-import')).toBe( + false, + ); + expect(hasStorageCapability(OFFSITE_BYTES, 'builtin-file-tools')).toBe( + false, + ); + expect(hasStorageCapability(OFFSITE_BYTES, 'space-file-plane')).toBe(false); + + // What survives: the Space folder still holds the record and the node + // documents, so showing it to a user is still showing them the Space, and + // a note dropped into `nodes/` still arrives. Those rows name no blob + // axis, which is how they say they do not care where the bytes went. + expect(hasStorageCapability(OFFSITE_BYTES, 'reveal-space-folder')).toBe( + true, + ); + expect(hasStorageCapability(OFFSITE_BYTES, 'external-note-discovery')).toBe( + true, + ); + expect(hasStorageCapability(OFFSITE_BYTES, 'workspace-directory')).toBe( + true, + ); + }); + + it('requires every clause, and any backend within one', () => { + // `and` across axes: the hybrid profile satisfies the blob clause of + // `space-bundle-export` and fails its structured one, and half a + // requirement is not a requirement met. + expect(hasStorageCapability(TABLES, 'space-bundle-export')).toBe(false); + // ...and the mirror image, which is the disk/azure case above. + expect( + hasStorageCapability( + { structured: { kind: 'disk' }, blobs: { kind: 'azure' } }, + 'space-bundle-export', + ), + ).toBe(false); + + // `or` within a clause: the one row that names a blob backend is met by + // that backend, and an absent clause is met by anything — which is what + // lets `reveal-space-folder` survive a blob backend it never named. + expect(hasStorageCapability(DISK, 'space-bundle-export')).toBe(true); + expect( + hasStorageCapability( + { structured: { kind: 'disk' }, blobs: { kind: 'azure' } }, + 'reveal-space-folder', + ), + ).toBe(true); + }); + it('treats an unknown id as available rather than guessing', () => { // The matrix is an exception list. A feature nobody wrote down is // portable by construction, and inventing a refusal for it would make @@ -75,13 +150,13 @@ describe('storage capability matrix', () => { expect(hasStorageCapability(TABLES, 'something-portable')).toBe(true); }); - it('states a limitation without making it a misconfiguration', () => { - // A profile that merely offers fewer features must not fail validation — - // that is reserved for a backend that cannot serve at all. `sqlite` has - // no adapter yet, so it does fail; the distinction is which check - // rejects it. + it('reports capability gaps without making them a misconfiguration', () => { + // The two gates are separate on purpose. A profile that offers fewer + // features is a stated limitation and must still start; only a profile + // that cannot serve at all is rejected. Conflating them would refuse a + // legitimate deployment. expect(describeUnavailableCapabilities(TABLES).length).toBeGreaterThan(0); - expect(() => validateStorageProfile(TABLES)).toThrow(/not implemented/); + expect(() => validateStorageProfile(TABLES)).not.toThrow(); expect(() => validateStorageProfile(DISK)).not.toThrow(); }); @@ -93,7 +168,8 @@ describe('storage capability matrix', () => { expect(line).toBeDefined(); // The id to search for, what is lost, and why it cannot be emulated. expect(line).toContain(capability.summary); - expect(line).toContain('sqlite'); + // The whole profile, because a row may be unavailable for either axis. + expect(line).toContain('sqlite/disk'); } }); }); diff --git a/apps/server/src/modules/storage/capabilities.ts b/apps/server/src/modules/storage/capabilities.ts index 68c412685..989c9f965 100644 --- a/apps/server/src/modules/storage/capabilities.ts +++ b/apps/server/src/modules/storage/capabilities.ts @@ -22,111 +22,234 @@ * This is a *declaration*, not an enforcement point. Each listed feature also * refuses at its own call site, because a matrix nobody consults at runtime is * documentation. What the matrix adds is the up-front answer. + * + * Two rules keep those call sites honest: + * + * - **A refusal asks the matrix.** `storageServes(id)` on the composition + * root, never a re-derivation of the requirement such as "is there a + * `diskTree`". A gate that re-derives is a second copy of the rule, and it + * is how a row could grow a blob-axis requirement its own call site never + * learned about. + * - **A degradation does not.** Code that renders absence rather than + * refusing — the memory preamble reading as empty — asks the concrete + * predicate, because it is not making the profile's promise, only reading + * what is there. + * + * Every row must therefore be refusable. A property that nothing can ask about + * is not a capability: it is a fact about a backend, and it belongs in that + * backend's own commentary. Windows directory-handle coordination was listed + * here and removed for exactly that reason — a Space with no directory + * registers no handle owner, so nothing ever asks and nothing is lost. */ +import type { BlobBackendKind } from './ports/blob.js'; import type { StructuredBackendKind } from './ports/structured.js'; import type { StorageProfile } from './profile.js'; /** - * A product feature whose availability depends on the structured backend. + * A product feature some storage profiles cannot serve. * - * Keyed by structured kind alone: every entry here needs a Space to be a real - * directory, which is a structured-backend property. A feature that turned on - * the blob backend instead would be a second matrix, and there are none. + * Keyed on the **profile**, not on one axis. Most entries need a Space or a + * Workspace to be a real directory, which is a structured-backend property — + * but several need more than that: they need the Space's *bytes* to be in that + * directory too, and that is the blob backend's business. A matrix that asked + * only the structured axis would call a bundle exportable on a profile that + * archives a Space folder its artifacts had never been written to. + * + * Each row therefore states a {@link StorageRequirement} rather than a list of + * backends: every axis it names must hold, and an axis it does not name is one + * it does not depend on. */ export interface StorageCapability { /** Stable id, for a diagnostic an operator can search for. */ readonly id: string; /** What a user loses, in their vocabulary rather than the port's. */ readonly summary: string; - /** Structured backends that serve it. */ - readonly backends: readonly StructuredBackendKind[]; + /** What a deployment must be for this feature to work. */ + readonly requires: StorageRequirement; /** Why it cannot be served elsewhere, and what remains instead. */ readonly rationale: string; } /** - * Every feature that is not available on every backend. + * The condition a profile has to meet, one clause per storage axis. + * + * **Every clause present must hold** — the axes are an `and`, because a + * feature that needs both a Space directory and the Space's bytes inside it + * needs both, not either. **Within a clause the backends are an `or`**: the + * configured backend has to be one of them. * - * Deliberately not "every feature" — a matrix that listed the portable ones - * too would need updating whenever anything was built, and would go stale - * silently. What must stay accurate is the exception list. + * So `{ structured: ['disk'], blobs: ['disk'] }` reads "the structured backend + * must be Disk *and* the blob backend must be Disk", and a future + * `{ structured: ['disk', 'postgres'] }` would read "the structured backend + * must be Disk *or* Postgres, and the blob backend may be anything". + * + * **An absent clause is not a requirement**, so every backend on that axis + * passes. That default is the design rather than a shortcut: a feature that + * does not touch a Space's bytes must not need editing when a blob backend is + * added, and the features that do are exactly the ones that should be forced + * to decide then. A requirement with no clauses at all requires nothing, which + * is not a limitation — `capabilities.test.ts` rejects one. */ +export interface StorageRequirement { + /** Structured backends that satisfy it; absent means any does. */ + readonly structured?: readonly StructuredBackendKind[]; + /** Blob backends that satisfy it; absent means any does. */ + readonly blobs?: readonly BlobBackendKind[]; +} + export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ { id: 'space-bundle-export', summary: 'Export a Space as a .huabu.zip bundle', - backends: ['disk'], + requires: { structured: ['disk'], blobs: ['disk'] }, rationale: - 'The bundle is a Disk projection — the Space directory, archived. A ' + + 'The bundle is a Disk projection — the Space directory, archived — so ' + + 'it needs both halves of that directory: the records and the bytes. A ' + 'portable export generated from records plus reachable blob references ' + 'is a separate design.', }, { id: 'space-bundle-import', summary: 'Import a Space from a .huabu.zip bundle', - backends: ['disk'], - rationale: 'Pairs with export; unzips into place.', + requires: { structured: ['disk'], blobs: ['disk'] }, + rationale: + 'Pairs with export; unzips into place, which is only the whole Space ' + + 'where the whole Space is in that place.', }, { id: 'reveal-space-folder', - summary: 'Reveal a Space in the OS file manager', - backends: ['disk'], + summary: "Open a Space's nodes folder in the OS file manager", + requires: { structured: ['disk'] }, rationale: - 'The feature is "show me this in Finder". Without a folder there is ' + - 'nothing to show.', + 'It opens the folder of node documents so a user can settle a ' + + 'duplicate-markdown collision by hand. Off Disk a node is a row: there ' + + 'is no folder of documents to open, and the collision it exists to ' + + 'settle cannot arise, because label uniqueness is a constraint rather ' + + "than a filename. The Space's byte areas are files, but they are " + + 'hidden, Server-owned, and hold artifacts rather than documents — ' + + 'opening those would answer a question nobody asked. No blob clause: ' + + 'the folder is still the documents wherever the bytes went.', }, { id: 'builtin-file-tools', summary: 'Built-in agent file tools (read, write, glob, grep)', - backends: ['disk'], + requires: { structured: ['disk'], blobs: ['disk'] }, rationale: - 'They sandbox on the Space directory. Off Disk the first-party agent ' + - 'reaches a Space over RFS/HTTP, which is what external agents already ' + - 'use.', + 'They sandbox on the Space directory and the documents they exist to ' + + 'edit are the node sidecars under `nodes/`, which are rows here. A ' + + "Space's byte areas are files on every profile, but they hold " + + 'artifacts and uploads, not the documents an agent reads and writes. ' + + 'Off Disk the first-party agent goes through the Canvas tools instead, ' + + 'which is the portable surface it already prefers for structured edits.', + }, + { + id: 'space-file-plane', + summary: 'Reach a Space as files over RFS, the plane external agents mount', + requires: { structured: ['disk'], blobs: ['disk'] }, + rationale: + 'RFS projects the Space directory over HTTP — the record and the node ' + + 'sidecars, reachable from another machine. Those are rows here, and a ' + + 'projection of the byte areas alone would be a different plane wearing ' + + "this one's name. It is listed apart from the built-in file tools " + + 'because it is what those tools were said to fall back to: a Space ' + + 'with no file plane has neither, and an external agent bound to a ' + + 'Space on this backend reaches it through the Canvas API.', }, { id: 'external-note-discovery', summary: 'Adopt Markdown files dropped into a Space from outside the app', - backends: ['disk'], + requires: { structured: ['disk'] }, + rationale: + 'It watches `nodes/` for documents that arrived without going through ' + + 'the application. That tier is rows here, and no byte area is a place ' + + 'a user would drop a note into: they are hidden, Server-owned, and ' + + 'hold artifacts. Inventing an arrival path would buy nothing.', + }, + { + id: 'workspace-directory', + summary: 'Choose, create, or reveal a Workspace folder on this machine', + requires: { structured: ['disk'] }, rationale: - 'It watches for documents that arrived without going through the ' + - 'application. A database backend has no such arrival path unless ' + - 'someone writes to the store out of band, and inventing one would buy ' + - 'nothing.', + 'A Workspace is a folder the user picks. Where Workspaces are rows ' + + 'there is nothing to browse to: the Server opens its own on first ' + + 'start and Workspaces are created and managed by name instead of by ' + + 'path. The per-Workspace directory under the blob root is Server-owned ' + + 'storage for bytes, not a Workspace a user could choose or move.', }, { - id: 'space-directory-handle-coordination', - summary: 'Windows: rename or delete a Space while a watcher holds it open', - backends: ['disk'], + id: 'workspace-user-memory', + summary: 'The cross-Space user memory document (setting/user.md)', + requires: { structured: ['disk'] }, rationale: - 'Exists so a directory rename can succeed against a live `fs.watch` ' + - 'handle. No directory, no problem.', + 'A user-editable file at the Workspace root, deliberately outside any ' + + 'Space so it applies to all of them. The blob port has no ' + + 'Workspace-level scope — every area it vends belongs to a Space — so ' + + 'the document has no scope to live in, whatever directories happen to ' + + "exist. A Space's own memory body is unaffected; it is a blob.", + }, + { + id: 'workspace-user-skills', + summary: 'User-authored skills under the Workspace setting/skills folder', + requires: { structured: ['disk'] }, + rationale: + 'Skills are read as files a user can edit and drop in by hand, which ' + + 'is the same arrival path external notes rely on. Bundled and Agent ' + + 'Team skills are unaffected.', }, ]; +/** + * Whether one profile serves one capability. + * + * An axis the capability does not name is an axis it does not depend on, so + * every backend there passes. A profile may request a backend that has no + * adapter — `validateStorageProfile` is what rejects those — and such a kind + * appears in no list, which is the right answer: an unwritten backend serves + * nothing. + */ +function serves( + capability: StorageCapability, + profile: StorageProfile, +): boolean { + /** One clause: absent requires nothing, present is met by any member. */ + const satisfied = ( + allowed: readonly string[] | undefined, + configured: string, + ): boolean => allowed === undefined || allowed.includes(configured); + + const { structured, blobs } = capability.requires; + // Every clause, not any: a feature needing a Space directory *and* the + // Space's bytes inside it is not served by half of that. + return ( + satisfied(structured, profile.structured.kind) && + satisfied(blobs, profile.blobs.kind) + ); +} + /** Capabilities this profile cannot serve. */ export function unavailableCapabilities( profile: StorageProfile, ): readonly StorageCapability[] { return STORAGE_CAPABILITIES.filter( - (capability) => - !(capability.backends as readonly string[]).includes( - profile.structured.kind, - ), + (capability) => !serves(capability, profile), ); } -/** Whether this profile serves `id`. Unknown ids are available by omission. */ +/** + * Whether this profile serves `id`. Unknown ids are available by omission. + * + * Application code should not reach this directly — the only profile worth + * asking about is the one storage was opened with, and the composition root's + * `storageServes(id)` is bound to it. This form exists for the matrix's own + * tests, which need to ask about profiles the process is not running. + */ export function hasStorageCapability( profile: StorageProfile, id: string, ): boolean { const capability = STORAGE_CAPABILITIES.find((entry) => entry.id === id); - if (!capability) return true; - return (capability.backends as readonly string[]).includes( - profile.structured.kind, - ); + return capability === undefined || serves(capability, profile); } /** @@ -159,9 +282,10 @@ export function unavailableCapabilityMessage(id: string): string { export function describeUnavailableCapabilities( profile: StorageProfile, ): readonly string[] { + const label = `${profile.structured.kind}/${profile.blobs.kind}`; return unavailableCapabilities(profile).map( (capability) => `${capability.id}: ${capability.summary} — unavailable on the ` + - `"${profile.structured.kind}" structured backend. ${capability.rationale}`, + `"${label}" storage profile. ${capability.rationale}`, ); } diff --git a/apps/server/src/modules/storage/compatibility/delete-canvas.test.ts b/apps/server/src/modules/storage/compatibility/delete-canvas.test.ts index 8467909f2..d7239dd3d 100644 --- a/apps/server/src/modules/storage/compatibility/delete-canvas.test.ts +++ b/apps/server/src/modules/storage/compatibility/delete-canvas.test.ts @@ -16,7 +16,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { executeOnServer } from '../../canvas/canvas-executor.js'; import { DiskBlobStore } from '../backends/disk/blob-store.js'; import { refreshCanvasDirIndex } from '../backends/disk/canvas-dirs.js'; -import { artifactPath, canvasJsonPath } from '../backends/disk/layout.js'; +import { + artifactPath, + canvasJsonPath, + canvasRoot, +} from '../backends/disk/layout.js'; import { resetStorageCache } from '../backends/disk/legacy/canvas-store-cache.js'; import { DiskStructuredStore } from '../backends/disk/structured-store.js'; import { getCanvasStore } from '../index.js'; @@ -42,6 +46,7 @@ const workspaceState = vi.hoisted(() => ({ path: '', leaseCount: 0 })); vi.mock('../../workspace.js', () => ({ getWorkspacePath: () => workspaceState.path, + getWorkspaceKey: () => workspaceState.path, acquireWorkspaceOperationLease: () => { const workspacePath = workspaceState.path; workspaceState.leaseCount += 1; @@ -72,7 +77,7 @@ function wrapAreas( /** How many sweeps one Space deletion must perform. */ const SPACE_AREA_COUNT = spaceBlobAreas( - new DiskBlobStore().space('probe'), + new DiskBlobStore(canvasRoot).space('probe'), ).length; function writeCanvas(directory: string, canvasId: string, title: string): void { @@ -105,7 +110,7 @@ class OrderRecordingBlobStore implements BlobStore { readonly kind = 'disk' as const; readonly recordPresentAtSweep: boolean[] = []; - private readonly inner = new DiskBlobStore(); + private readonly inner = new DiskBlobStore(canvasRoot); init(): Promise { return this.inner.init(); @@ -154,7 +159,7 @@ class ControllableBlobStore implements BlobStore { readonly deleteStarted = deferred(); readonly #putsReleased = deferred(); readonly #deletesReleased = deferred(); - readonly #inner = new DiskBlobStore(); + readonly #inner = new DiskBlobStore(canvasRoot); blockPuts = false; blockDeletes = false; diff --git a/apps/server/src/modules/storage/detached-blobs.test.ts b/apps/server/src/modules/storage/detached-blobs.test.ts new file mode 100644 index 000000000..9c81e6cd1 --- /dev/null +++ b/apps/server/src/modules/storage/detached-blobs.test.ts @@ -0,0 +1,186 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * Where a Space's bytes go when its records are rows. + * + * The portable behaviour — put, read, sweep on delete — is already proven for + * every profile by `product-boundary.test.ts`, and naming a directory there + * would stop it being evidence of anything portable. What is left is the part + * that *is* about placement, and it belongs here: bytes are files on every + * profile, so a backend with no Space folder still needs one, and it has to be + * scoped to the Workspace that owns the Space and removed with it. + */ + +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { + diskBlobRoot, + diskDataDir, + diskSpaceBlobRoot, + workspaceRegistryPath, +} from './backends/disk/data-dir.js'; +import { ARTIFACTS_DIR_NAME } from './backends/disk/layout.js'; +import { + activateWorkspace, + createNamedWorkspace, + createSpace, + deleteSpace, + space, +} from './storage.js'; +import { mountTestWorkspace, type MountedTestStorage } from './testing.js'; +import { getWorkspaceHandle } from '../workspace.js'; + +import type { StorageProfile } from './profile.js'; + +/** Records in SQLite, bytes on the file system — the hybrid this covers. */ +const HYBRID: StorageProfile = { + structured: { kind: 'sqlite' }, + blobs: { kind: 'disk' }, +}; + +const CANVAS_ID = 'canvas-detached-blobs'; + +let mounted: MountedTestStorage | null = null; + +afterEach(async () => { + await mounted?.close(); + mounted = null; +}); + +async function mount(): Promise { + mounted = await mountTestWorkspace(HYBRID, 'huabu-detached-blobs-'); + return mounted; +} + +/** The directory this profile puts one Space's artifacts in. */ +function artifactsDirectory(canvasId: string): string { + const workspace = getWorkspaceHandle(); + if (!workspace) throw new Error('Expected an active Workspace'); + return path.join( + diskSpaceBlobRoot(workspace.workspaceId, canvasId), + ARTIFACTS_DIR_NAME, + ); +} + +function isInside(parent: string, child: string): boolean { + return path.resolve(child).startsWith(`${path.resolve(parent)}${path.sep}`); +} + +describe('Space bytes on a backend with no Space folder', () => { + it('writes real files under the Workspace-scoped byte root', async () => { + await mount(); + await createSpace(CANVAS_ID, 'Detached'); + + await space(CANVAS_ID).artifacts.put('art.bin', Buffer.from('real bytes')); + + const file = path.join(artifactsDirectory(CANVAS_ID), 'art.bin'); + expect(readFileSync(file)).toEqual(Buffer.from('real bytes')); + }); + + it("files each Workspace's bytes under its own root", async () => { + await mount(); + const firstWorkspace = getWorkspaceHandle(); + if (!firstWorkspace) throw new Error('Expected an active Workspace'); + await createSpace(CANVAS_ID, 'First'); + await space(CANVAS_ID).artifacts.put('art.bin', Buffer.from('first')); + const first = artifactsDirectory(CANVAS_ID); + + const second = await createNamedWorkspace('Second Workspace'); + await activateWorkspace(second); + const otherCanvasId = 'canvas-detached-blobs-second'; + await createSpace(otherCanvasId, 'Second'); + await space(otherCanvasId).artifacts.put('art.bin', Buffer.from('second')); + + // Two Workspaces served by one connection and one blob root, and neither + // can reach into the other's bytes: the Workspace segment is what keeps + // them apart, the same way a Workspace folder does on Disk. + const secondDirectory = artifactsDirectory(otherCanvasId); + expect(path.dirname(path.dirname(first))).not.toBe( + path.dirname(path.dirname(secondDirectory)), + ); + expect(readFileSync(path.join(first, 'art.bin'))).toEqual( + Buffer.from('first'), + ); + + await activateWorkspace(firstWorkspace); + expect(await space(CANVAS_ID).artifacts.read('art.bin')).toEqual( + Buffer.from('first'), + ); + }); + + it('is not a Space tree, so no Disk-only capability turns on', async () => { + await mount(); + await createSpace(CANVAS_ID, 'Detached'); + await space(CANVAS_ID).artifacts.put('art.bin', Buffer.from('bytes')); + + // The byte directory exists and holds real files, and the Space still has + // no `diskTree`. That single `null` is what every Disk-only feature keys + // on — bundle export, reveal-in-file-manager, the built-in file tools, + // RFS, external-note claim — so it is the fact worth pinning: a file + // system for bytes is not a Space directory and grants none of them. + expect(existsSync(artifactsDirectory(CANVAS_ID))).toBe(true); + expect(space(CANVAS_ID).diskTree).toBeNull(); + }); + + it('leaves no directory behind when the Space is deleted', async () => { + await mount(); + await createSpace(CANVAS_ID, 'Detached'); + await space(CANVAS_ID).artifacts.put('art.bin', Buffer.from('bytes')); + const spaceRoot = path.dirname(artifactsDirectory(CANVAS_ID)); + expect(existsSync(spaceRoot)).toBe(true); + + await expect(deleteSpace(CANVAS_ID)).resolves.toMatchObject({ ok: true }); + + // Sweeping the areas is the blob port's contract; removing the directory + // composition put them under is this module's, and nothing else would. + expect(existsSync(spaceRoot)).toBe(false); + }); +}); + +/** + * The two Disk adapters share `storage/disk/`, so the line between them is a + * path fact and is tested as one. + * + * They are never both in use — the registry belongs to the Disk *structured* + * store and the byte roots appear only when some other backend holds the + * records — but one data directory can see both across a backend switch. The + * blob store deletes whole directories; the registry is not its to delete. + */ +describe('the Disk backend area in the data directory', () => { + const DATA_DIR = '/var/lib/huabu'; + + it('gives the registry and the byte roots separate subtrees', () => { + const registry = workspaceRegistryPath(DATA_DIR); + const blobs = diskBlobRoot(DATA_DIR); + + expect(isInside(diskDataDir(DATA_DIR), registry)).toBe(true); + expect(isInside(diskDataDir(DATA_DIR), blobs)).toBe(true); + // The one that matters: no sweep of a Space's bytes, an area, or the whole + // blob root can reach the structured store's registry. + expect(isInside(blobs, registry)).toBe(false); + expect(isInside(blobs, diskSpaceBlobRoot('ws', 'canvas', DATA_DIR))).toBe( + true, + ); + }); + + it('moves only the bytes when HUABU_BLOB_ROOT is set', () => { + const previous = process.env['HUABU_BLOB_ROOT']; + process.env['HUABU_BLOB_ROOT'] = '/mnt/bulk/huabu-bytes'; + try { + expect(diskSpaceBlobRoot('ws', 'canvas', DATA_DIR)).toBe( + path.join('/mnt/bulk/huabu-bytes', 'ws', 'canvas'), + ); + // The registry is the structured store's and does not follow. + expect(workspaceRegistryPath(DATA_DIR)).toBe( + path.join(diskDataDir(DATA_DIR), 'workspaces.json'), + ); + } finally { + if (previous === undefined) delete process.env['HUABU_BLOB_ROOT']; + else process.env['HUABU_BLOB_ROOT'] = previous; + } + }); +}); diff --git a/apps/server/src/modules/storage/index.ts b/apps/server/src/modules/storage/index.ts index 0a12f64c6..78a123395 100644 --- a/apps/server/src/modules/storage/index.ts +++ b/apps/server/src/modules/storage/index.ts @@ -24,11 +24,18 @@ export { } from './compatibility/canvas.js'; export type { RenameResult, RenameSelfResult } from './compatibility/canvas.js'; +/** + * World identity, answered for whichever backend is configured. + * + * These used to come straight from the Disk directory index. They are on the + * composition root now because the World is a Space like any other and every + * backend has one — the index is just how Disk finds it. + */ export { getWorldCanvasId, isWorldCanvasId, requireWorldCanvasId, -} from './backends/disk/canvas-dirs.js'; +} from './storage.js'; /** * Materialization-tier capabilities, re-exported so consumers that need a @@ -71,9 +78,11 @@ export type { // ─── Storage ports and composition ───────────────────────────────────────── export { + activateWorkspace, adoptWorkspaceDirectory, closeStorage, composeStorage, + createNamedWorkspace, createSpace, createStorage, deleteSpace, @@ -83,6 +92,7 @@ export { getWorkspaceRepository, hasWorkspaceRegistry, initStorage, + materializesWorkspaces, setStorageForTesting, space, stageSpaceImport, @@ -90,7 +100,13 @@ export { workspaceAtDirectory, workspaceDirectory, } from './storage.js'; -export type { Space, SpaceDeleteOutcome, Storage } from './storage.js'; +export type { + Space, + SpaceDeleteOutcome, + SqliteSpaceTree, + Storage, +} from './storage.js'; +export type { SqliteSpaceSubstrate } from './backends/sqlite/space-extension.js'; export type { DiskSpaceTree } from './backends/disk/space-tree.js'; export type { DiskSpaceImport } from './backends/disk/space-import.js'; export { @@ -100,11 +116,16 @@ export { } from './profile.js'; export { describeUnavailableCapabilities, - hasStorageCapability, STORAGE_CAPABILITIES, unavailableCapabilities, unavailableCapabilityMessage, } from './capabilities.js'; +/** + * Capability questions are asked of the profile in force, never of one the + * caller assembled — so the bound accessor is what leaves the module and + * `hasStorageCapability` stays inside it. + */ +export { storageServes } from './storage.js'; export type { StorageCapability } from './capabilities.js'; export type { StorageProfile } from './profile.js'; export { diff --git a/apps/server/src/modules/storage/module-boundaries.test.ts b/apps/server/src/modules/storage/module-boundaries.test.ts index aa9c3ead2..5e027b421 100644 --- a/apps/server/src/modules/storage/module-boundaries.test.ts +++ b/apps/server/src/modules/storage/module-boundaries.test.ts @@ -21,6 +21,8 @@ import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; +import { STORAGE_CAPABILITIES } from './capabilities.js'; + const HERE = path.dirname(fileURLToPath(import.meta.url)); const STORAGE_DIR = HERE; const SRC_DIR = path.resolve(HERE, '../..'); @@ -87,6 +89,7 @@ describe('storage module tree', () => { 'canvas-dirs.ts', 'capabilities.test.ts', 'capabilities.ts', + 'detached-blobs.test.ts', 'index.ts', 'module-boundaries.test.ts', 'paths.ts', @@ -174,6 +177,49 @@ describe('storage dependency direction', () => { expect(violations).toEqual([]); }); + /** + * A declared capability is refused somewhere, or it is not a capability. + * + * `capabilities.ts` promises that every row also refuses at its own call + * site, "because a matrix nobody consults at runtime is documentation". This + * is that promise, checked. It catches the two ways it rots: a row added for + * an operator's benefit that no feature ever asks about, and a refusal + * deleted while its row stays behind, still printed at boot. + */ + it('refuses every capability it declares, outside the storage module', () => { + const consumers = sourceFiles + .filter((f) => !f.startsWith('modules/storage/')) + .filter((f) => !f.endsWith('.test.ts')) + .map((f) => read(f)); + + const unenforced = STORAGE_CAPABILITIES.filter( + (capability) => + !consumers.some((source) => source.includes(`'${capability.id}'`)), + ).map((capability) => capability.id); + + expect(unenforced).toEqual([]); + }); + + /** + * Each backend owns its own area of the Server data directory. + * + * `storage/disk/` and `storage/sqlite/` are backend-shaped names, so the + * only files allowed to build them are those backends'. The composition root + * asks; it does not know. Two adapters share `storage/disk/` — the + * structured store's Workspace registry and the blob store's Space byte + * roots — and one file deciding both is what keeps them from overlapping. + */ + it('lets each backend own its area of the data directory', () => { + const owners = sourceFiles + .filter((f) => !f.endsWith('.test.ts')) + .filter((f) => /'storage',\s*'(disk|sqlite)'/.test(read(f))); + + expect(owners.sort()).toEqual([ + 'modules/storage/backends/disk/data-dir.ts', + 'modules/storage/backends/sqlite/database.ts', + ]); + }); + it('selects a backend only in the composition root', () => { const importers = storageFiles // Tests construct adapters directly — that is how an adapter gets @@ -300,6 +346,10 @@ describe('workspace module names no backend', () => { */ describe('Disk Space tree capability', () => { const EXPECTED_CONSUMERS = [ + // A — external-note discovery. The watcher asks whether this Space has a + // directory to watch at all; `null` is the whole of its behaviour off + // Disk. + 'modules/canvas/external-watcher.ts', // A — the built-in file tools' sandbox root. 'modules/agent/tools/handlers/fs-sandbox.ts', // A — bundle export. @@ -316,6 +366,25 @@ describe('Disk Space tree capability', () => { 'modules/workspace/paths.ts', ].sort(); + /** + * `sqliteTree` is the same kind of thing as `diskTree` and gets the same + * fence. It is narrower on purpose: the *only* reason it exists rather than + * the port's async `extension()` is that Agenetes's storage ports are + * synchronous, so exactly one owner should ever appear here. + */ + const EXPECTED_SQLITE_CONSUMERS = [ + 'modules/agent/agenetes/sqlite-stores.ts', + ].sort(); + + it('keeps the exact synchronous SQLite substrate census', () => { + const consumers = sourceFiles + .filter((file) => !file.startsWith('modules/storage/')) + .filter((file) => !file.endsWith('.test.ts')) + .filter((file) => /\bsqliteTree\b/.test(read(file))); + + expect(consumers.sort()).toEqual(EXPECTED_SQLITE_CONSUMERS); + }); + it('keeps the exact production consumer census', () => { // Matched as a bare word, not as `.diskTree`: destructuring the member // off a handle (`const { diskTree } = space(id)`) or reaching it by @@ -606,10 +675,7 @@ describe('root forwarding shims', () => { 'storage/canvas-dirs.js': [ 'modules/agent/tools/world-target-read.test.ts', 'modules/canvas/canvas-command-router.test.ts', - 'modules/canvas/canvas.route.ts', 'modules/canvas/external-watcher.test.ts', - 'modules/canvas/external-watcher.ts', - 'modules/canvas/world-portal-policy.ts', 'modules/canvas/world-portals.test.ts', 'modules/canvas/world-reference-resolver.test.ts', 'modules/workspace.ts', diff --git a/apps/server/src/modules/storage/ports/blob.ts b/apps/server/src/modules/storage/ports/blob.ts index 3f324dd75..54bdabf5d 100644 --- a/apps/server/src/modules/storage/ports/blob.ts +++ b/apps/server/src/modules/storage/ports/blob.ts @@ -22,7 +22,19 @@ import type { StorageHealth } from './common.js'; import type { Readable } from 'node:stream'; -export type BlobBackendKind = 'disk' | 'azure'; +/** + * Backends with a blob adapter today. + * + * Like {@link StructuredBackendKind}, this names only what exists. The wider + * vocabulary a profile may *request* — including `azure`, which is a settled + * direction with no adapter — belongs to `profile.ts`. + * + * Every member of that wider vocabulary is a **file system**: a local + * directory now, an object store later. Bytes are not records, and a + * structured backend is never asked to hold them — which is what lets a + * deployment pair SQL records with ordinary files (proposal §6.2). + */ +export type BlobBackendKind = 'disk'; /** * Every area of one Space that holds bytes. diff --git a/apps/server/src/modules/storage/ports/contracts/space-nodes.contract.ts b/apps/server/src/modules/storage/ports/contracts/space-nodes.contract.ts index 3d70bbd8f..f5649985a 100644 --- a/apps/server/src/modules/storage/ports/contracts/space-nodes.contract.ts +++ b/apps/server/src/modules/storage/ports/contracts/space-nodes.contract.ts @@ -14,6 +14,8 @@ export interface SpaceNodesContractHarness { /** Repository scoped to a Space whose structural record is absent. */ readonly missingRepository: SpaceNodes; readonly expectedCanvasId: string; + /** Whether this adapter fences a deleted id against late standalone puts. */ + readonly deletedNodePut: 'allowed' | 'write-suppressed'; readonly cleanup?: () => Promise | void; } @@ -336,20 +338,24 @@ export function describeSpaceNodesContract( await expect(repository.delete(nodeId)).resolves.toBe('absent'); }); - it('suppresses a late standalone put after deletion', async () => { - const { repository } = await open(); + it('reports the adapter-defined result for a standalone put after deletion', async () => { + const { repository, deletedNodePut } = await open(); const nodeId = 'contract-late-put'; const record = note(nodeId, 'Contract late put', 'before'); await putSuccessfully(repository, { nodeId, record }); await repository.delete(nodeId); - await expect( - repository.put({ - nodeId, - record: { ...record, content: 'late resurrection' }, - }), - ).resolves.toEqual({ ok: false, reason: 'write-suppressed' }); - await expect(repository.read(nodeId)).resolves.toBeNull(); + const late = { ...record, content: 'late resurrection' }; + const result = await repository.put({ nodeId, record: late }); + if (deletedNodePut === 'write-suppressed') { + expect(result).toEqual({ ok: false, reason: 'write-suppressed' }); + await expect(repository.read(nodeId)).resolves.toBeNull(); + } else { + expect(result).toMatchObject({ ok: true, record: late }); + await expect(repository.read(nodeId)).resolves.toMatchObject({ + record: late, + }); + } }); }); } diff --git a/apps/server/src/modules/storage/ports/contracts/space-tasks.contract.ts b/apps/server/src/modules/storage/ports/contracts/space-tasks.contract.ts new file mode 100644 index 000000000..a02aba86f --- /dev/null +++ b/apps/server/src/modules/storage/ports/contracts/space-tasks.contract.ts @@ -0,0 +1,441 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** Reusable behavioral contract for {@link SpaceTasks} and its Runs. */ + +import { afterEach, describe, expect, it } from 'vitest'; + +import type { + SpaceDeleteSession, + SpaceTasks, + TaskRunUpdate, +} from '../structured.js'; +import type { TaskRecord, TaskRunRecord } from '@huabu/shared'; + +export interface SpaceTasksContractHarness { + /** Task ledger for an existing Space, initially empty. */ + readonly tasks: SpaceTasks; + /** A second retained handle for the same existing Space. */ + readonly concurrent: SpaceTasks; + readonly canvasId: string; + /** Task ledger scoped to a Space whose structural record is absent. */ + readonly missing: SpaceTasks; + readonly missingCanvasId: string; + /** Open a structured-deletion fence for `canvasId`. */ + readonly beginDelete: () => Promise; + readonly cleanup?: () => Promise | void; +} + +function task(canvasId: string, taskId: string, createdAt: number): TaskRecord { + return { + taskId, + canvasId, + goal: `Goal for ${taskId}`, + defaultRootProfileId: `profile-${taskId}`, + anchorNodeId: `anchor-${taskId}`, + createdAt, + }; +} + +function run( + canvasId: string, + taskId: string, + runId: string, + createdAt: number, +): TaskRunRecord { + return { + runId, + taskId, + canvasIdSnapshot: canvasId, + goalSnapshot: `Goal snapshot for ${taskId}`, + rootProfileIdSnapshot: `profile-${taskId}`, + status: 'pending', + createdAt, + }; +} + +export function describeSpaceTasksContract( + name: string, + createHarness: () => + | Promise + | SpaceTasksContractHarness, +): void { + describe(`SpaceTasks contract: ${name}`, () => { + let harness: SpaceTasksContractHarness | null = null; + + async function open(): Promise { + harness = await createHarness(); + return harness; + } + + afterEach(async () => { + await harness?.cleanup?.(); + harness = null; + }); + + it('reads an empty versioned snapshot', async () => { + const { tasks } = await open(); + + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [], + runs: [], + }); + }); + + it('creates a Task and rejects a duplicate id without replacing it', async () => { + const { tasks, canvasId } = await open(); + const original = task(canvasId, 'task-duplicate', 1); + await tasks.create(original); + + await expect( + tasks.create({ ...original, goal: 'Replacement goal', createdAt: 2 }), + ).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [original], + runs: [], + }); + }); + + it('requires an existing Task before creating its Run', async () => { + const { tasks, canvasId } = await open(); + const owner = task(canvasId, 'task-owner', 1); + const ownedRun = run(canvasId, owner.taskId, 'run-owned', 2); + + await expect(tasks.runs.create(ownedRun)).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [], + runs: [], + }); + + await tasks.create(owner); + await tasks.runs.create(ownedRun); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [owner], + runs: [ownedRun], + }); + }); + + it('rejects a duplicate Run id without replacing it', async () => { + const { tasks, canvasId } = await open(); + const owner = task(canvasId, 'task-run-duplicate', 1); + const original = run(canvasId, owner.taskId, 'run-duplicate', 2); + await tasks.create(owner); + await tasks.runs.create(original); + + await expect( + tasks.runs.create({ + ...original, + status: 'running', + startedAt: 3, + }), + ).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [owner], + runs: [original], + }); + }); + + it('updates an existing Run and rejects a missing Run id', async () => { + const { tasks, canvasId } = await open(); + const owner = task(canvasId, 'task-update', 1); + const original = run(canvasId, owner.taskId, 'run-update', 2); + await tasks.create(owner); + await tasks.runs.create(original); + const update: TaskRunUpdate = { + rootNodeId: 'root-node', + rootThreadId: 'root-thread', + status: 'running', + startedAt: 3, + }; + + await expect(tasks.runs.update(original.runId, update)).resolves.toEqual({ + ...original, + ...update, + }); + await expect( + tasks.runs.update('run-missing', { status: 'running' }), + ).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [owner], + runs: [{ ...original, ...update }], + }); + }); + + it('completes only a running Run and keeps the first completion immutable', async () => { + const { tasks, concurrent, canvasId } = await open(); + const owner = task(canvasId, 'task-complete', 1); + const other = task(canvasId, 'task-complete-other', 2); + const original = run(canvasId, owner.taskId, 'run-complete', 3); + await tasks.create(owner); + await tasks.create(other); + await tasks.runs.create(original); + + await expect( + tasks.runs.complete(owner.taskId, original.runId, { completedAt: 4 }), + ).resolves.toMatchObject({ outcome: 'run_not_running', run: original }); + + await tasks.runs.update(original.runId, { + status: 'running', + startedAt: 5, + }); + await expect( + tasks.runs.complete(owner.taskId, original.runId, { + completedAt: 6, + message: 'Done', + }), + ).resolves.toMatchObject({ + outcome: 'completed', + run: { + status: 'completed', + completion: { completedAt: 6, message: 'Done' }, + }, + }); + await expect( + tasks.runs.complete(owner.taskId, original.runId, { + completedAt: 7, + message: 'Done', + }), + ).resolves.toMatchObject({ + outcome: 'unchanged', + run: { completion: { completedAt: 6, message: 'Done' } }, + }); + await expect( + tasks.runs.complete(owner.taskId, original.runId, { + completedAt: 8, + message: 'Different', + }), + ).resolves.toMatchObject({ outcome: 'completion_conflict' }); + await expect( + tasks.runs.complete('task-missing', original.runId, { completedAt: 9 }), + ).resolves.toEqual({ outcome: 'task_not_found' }); + await expect( + tasks.runs.complete(other.taskId, original.runId, { completedAt: 9 }), + ).resolves.toEqual({ outcome: 'run_not_found' }); + await expect( + tasks.runs.complete(owner.taskId, 'run-missing', { completedAt: 9 }), + ).resolves.toEqual({ outcome: 'run_not_found' }); + await expect(concurrent.read()).resolves.toEqual({ + version: 1, + tasks: [owner, other], + runs: [ + { + ...original, + status: 'completed', + startedAt: 5, + completion: { completedAt: 6, message: 'Done' }, + }, + ], + }); + }); + + it('serializes competing completions and persists exactly one winner', async () => { + const { tasks, concurrent, canvasId } = await open(); + const owner = task(canvasId, 'task-competing-completion', 1); + const original = run( + canvasId, + owner.taskId, + 'run-competing-completion', + 2, + ); + await tasks.create(owner); + await tasks.runs.create(original); + await tasks.runs.update(original.runId, { + status: 'running', + startedAt: 3, + }); + + const results = await Promise.all([ + tasks.runs.complete(owner.taskId, original.runId, { + completedAt: 4, + message: 'First candidate', + }), + concurrent.runs.complete(owner.taskId, original.runId, { + completedAt: 5, + message: 'Second candidate', + }), + ]); + expect(results.map((result) => result.outcome).sort()).toEqual([ + 'completed', + 'completion_conflict', + ]); + const completed = results.find( + (result) => result.outcome === 'completed', + ); + const conflict = results.find( + (result) => result.outcome === 'completion_conflict', + ); + if (completed?.outcome !== 'completed') { + throw new Error('Expected one completion winner'); + } + if (conflict?.outcome !== 'completion_conflict') { + throw new Error('Expected one completion conflict'); + } + expect(conflict.run).toEqual(completed.run); + await expect(concurrent.read()).resolves.toEqual({ + version: 1, + tasks: [owner], + runs: [completed.run], + }); + }); + + it('rejects Task and Run records scoped to another Space', async () => { + const { tasks, canvasId } = await open(); + const owner = task(canvasId, 'task-scope', 1); + + await expect( + tasks.create({ ...owner, canvasId: 'another-space' }), + ).rejects.toThrow(); + await tasks.create(owner); + await expect( + tasks.runs.create({ + ...run(canvasId, owner.taskId, 'run-scope', 2), + canvasIdSnapshot: 'another-space', + }), + ).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [owner], + runs: [], + }); + }); + + it('rejects malformed Task, Run, and Run-update input', async () => { + const { tasks, canvasId } = await open(); + const owner = task(canvasId, 'task-validation', 1); + const ownedRun = run(canvasId, owner.taskId, 'run-validation', 2); + + await expect(tasks.create({ ...owner, goal: '' })).rejects.toThrow(); + await tasks.create(owner); + await expect( + tasks.runs.create({ ...ownedRun, goalSnapshot: '' }), + ).rejects.toThrow(); + await tasks.runs.create(ownedRun); + await expect( + tasks.runs.update(ownedRun.runId, { startedAt: -1 }), + ).rejects.toThrow(); + await expect( + tasks.runs.complete(owner.taskId, ownedRun.runId, { + completedAt: -1, + }), + ).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [owner], + runs: [ownedRun], + }); + }); + + it('preserves concurrent mutations through two retained handles', async () => { + const { tasks, concurrent, canvasId } = await open(); + const taskA = task(canvasId, 'task-concurrent-a', 1); + const taskB = task(canvasId, 'task-concurrent-b', 2); + await Promise.all([tasks.create(taskA), concurrent.create(taskB)]); + + const runA = run(canvasId, taskA.taskId, 'run-concurrent-a', 3); + const runB = run(canvasId, taskB.taskId, 'run-concurrent-b', 4); + await Promise.all([ + tasks.runs.create(runA), + concurrent.runs.create(runB), + ]); + await Promise.all([ + concurrent.runs.update(runA.runId, { + status: 'running', + startedAt: 5, + }), + tasks.runs.update(runB.runId, { + status: 'running', + startedAt: 6, + }), + ]); + + const snapshot = await tasks.read(); + expect(snapshot.tasks.map((record) => record.taskId).sort()).toEqual([ + taskA.taskId, + taskB.taskId, + ]); + expect(snapshot.runs.map((record) => record.runId).sort()).toEqual([ + runA.runId, + runB.runId, + ]); + expect(snapshot.runs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + runId: runA.runId, + status: 'running', + startedAt: 5, + }), + expect.objectContaining({ + runId: runB.runId, + status: 'running', + startedAt: 6, + }), + ]), + ); + }); + + it('rejects every mutation for a missing Space', async () => { + const { missing, missingCanvasId } = await open(); + const owner = task(missingCanvasId, 'task-missing-space', 1); + const ownedRun = run( + missingCanvasId, + owner.taskId, + 'run-missing-space', + 2, + ); + + await expect(missing.create(owner)).rejects.toThrow(); + await expect(missing.runs.create(ownedRun)).rejects.toThrow(); + await expect( + missing.runs.update(ownedRun.runId, { status: 'running' }), + ).rejects.toThrow(); + await expect( + missing.runs.complete(owner.taskId, ownedRun.runId, { + completedAt: 3, + }), + ).rejects.toThrow(); + }); + + it('rejects mutations while structured deletion is fenced', async () => { + const { tasks, canvasId, beginDelete } = await open(); + const owner = task(canvasId, 'task-delete-fence', 1); + const original = run(canvasId, owner.taskId, 'run-delete-fence', 2); + await tasks.create(owner); + await tasks.runs.create(original); + const before = await tasks.read(); + const session = await beginDelete(); + + try { + await expect( + tasks.create(task(canvasId, 'task-too-late', 3)), + ).rejects.toThrow(); + await expect( + tasks.runs.create(run(canvasId, owner.taskId, 'run-too-late', 4)), + ).rejects.toThrow(); + await expect( + tasks.runs.update(original.runId, { status: 'running' }), + ).rejects.toThrow(); + await expect( + tasks.runs.complete(owner.taskId, original.runId, { + completedAt: 5, + }), + ).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual(before); + } finally { + await session.abort(); + } + + await expect( + tasks.runs.update(original.runId, { + status: 'running', + startedAt: 5, + }), + ).resolves.toMatchObject({ status: 'running', startedAt: 5 }); + }); + }); +} diff --git a/apps/server/src/modules/storage/ports/contracts/structured-store.contract.ts b/apps/server/src/modules/storage/ports/contracts/structured-store.contract.ts index 9ede9188f..42c08e6a9 100644 --- a/apps/server/src/modules/storage/ports/contracts/structured-store.contract.ts +++ b/apps/server/src/modules/storage/ports/contracts/structured-store.contract.ts @@ -67,6 +67,7 @@ export function describeStructuredStoreContract( for (const method of [ 'list', 'worldId', + 'ensureWorld', 'create', 'beginDelete', 'rename', diff --git a/apps/server/src/modules/storage/ports/structured.ts b/apps/server/src/modules/storage/ports/structured.ts index 60332eebd..ee69234a0 100644 --- a/apps/server/src/modules/storage/ports/structured.ts +++ b/apps/server/src/modules/storage/ports/structured.ts @@ -52,6 +52,7 @@ import type { TaskStoreSnapshot, } from '@huabu/shared'; import type { CanvasChangeRecord } from '@huabu/shared/canvas-engine'; +import type { DatabaseSync } from 'node:sqlite'; /** * Backends with a structured adapter today. @@ -61,7 +62,7 @@ import type { CanvasChangeRecord } from '@huabu/shared/canvas-engine'; * that are configurable but unimplemented — belongs to `profile.ts`, which * owns rejecting them with an actionable message. */ -export type StructuredBackendKind = 'disk'; +export type StructuredBackendKind = 'disk' | 'sqlite'; /** A connection to a structured backend. Process-wide; handles are derived. */ export interface StructuredStore { @@ -299,14 +300,23 @@ export interface SpaceHandle { * * One member per backend that exists, like {@link StructuredBackendKind} and * for the same reason: a union that named `sqlite` today would advertise a - * substrate no adapter can supply. It grows with each adapter — a table prefix - * for SQLite, a schema for Postgres — and an owner switches on `kind`. + * substrate no adapter can supply. It grows with each adapter — a scoped + * connection and parent id for SQLite, a schema for Postgres — and an owner + * switches on `kind`. */ -export type SpaceSubstrate = { - readonly kind: 'disk'; - /** A directory reserved for this namespace, created and ready to write. */ - readonly directory: string; -}; +export type SpaceSubstrate = + | { + readonly kind: 'disk'; + /** A directory reserved for this namespace, created and ready to write. */ + readonly directory: string; + } + | { + readonly kind: 'sqlite'; + /** The adapter connection on which the owner creates its own tables. */ + readonly database: DatabaseSync; + /** Stable parent row for owner tables to reference with ON DELETE CASCADE. */ + readonly extensionId: number; + }; // ─── The ordered Space write ───────────────────────────────────────────────── diff --git a/apps/server/src/modules/storage/product-boundary.test.ts b/apps/server/src/modules/storage/product-boundary.test.ts index eea943b4b..7ef7cf767 100644 --- a/apps/server/src/modules/storage/product-boundary.test.ts +++ b/apps/server/src/modules/storage/product-boundary.test.ts @@ -347,6 +347,43 @@ forEachProductProfile((profile: StorageProfile, label: string) => { await expect(m.storage.structured.spaces().list()).resolves.toEqual([]); }); + it('serves the same Space after a restart', async () => { + const canvasId = 'space-product-restart'; + const m = await seedSpace(canvasId); + const before = m.storage.space(canvasId); + await before.artifacts.put('kept.bin', Buffer.from('durable bytes')); + await before.events.append([ + { payload: { action: 'node_created', nodes: [] }, ts: 7 }, + ]); + const record = await before.read(); + const nodes = await before.nodes.list(); + const worldId = await m.storage.structured.spaces().worldId(); + + // The restart is the point. Everything above is in whatever the backend + // calls durable; nothing about this case says which. + const storage = await m.reopen(); + + await expect(storage.structured.spaces().worldId()).resolves.toBe( + worldId, + ); + const after = storage.space(canvasId); + await expect(after.read()).resolves.toEqual(record); + // Revisions are opaque tokens, so the records are compared rather than + // the snapshots: a backend may mint a new token for the same content. + expect( + [...(await after.nodes.list())].map(([id, snapshot]) => [ + id, + snapshot.record, + ]), + ).toEqual([...nodes].map(([id, snapshot]) => [id, snapshot.record])); + expect(await after.artifacts.read('kept.bin')).toEqual( + Buffer.from('durable bytes'), + ); + await expect(after.events.read()).resolves.toEqual([ + { payload: { action: 'node_created', nodes: [] }, ts: 7 }, + ]); + }); + it('refuses to delete the World', async () => { const m = await open(); const spaces = m.storage.structured.spaces(); diff --git a/apps/server/src/modules/storage/profile.test.ts b/apps/server/src/modules/storage/profile.test.ts index 18be15c91..bd388d8a1 100644 --- a/apps/server/src/modules/storage/profile.test.ts +++ b/apps/server/src/modules/storage/profile.test.ts @@ -64,7 +64,20 @@ describe('validateStorageProfile', () => { structured: { kind: 'postgres' }, blobs: { kind: 'disk' }, }), - ).toThrow(/not implemented yet.*disk/s); + ).toThrow(/not implemented yet.*disk, sqlite/s); + }); + + // Fewer features is a stated limitation, not a misconfiguration: a profile + // may lose capabilities as long as the matrix declares them. Only an + // unimplemented backend fails here — the axes share nothing, so every + // pairing of implemented backends is a valid deployment. + it('accepts sqlite records beside disk blobs', () => { + expect(() => + validateStorageProfile({ + structured: { kind: 'sqlite' }, + blobs: { kind: 'disk' }, + }), + ).not.toThrow(); }); it('rejects a known but unimplemented blob backend', () => { @@ -94,7 +107,6 @@ describe('requiresExplicitInit', () => { it.each([ { structured: { kind: 'postgres' }, blobs: { kind: 'disk' } }, { structured: { kind: 'sqlite' }, blobs: { kind: 'disk' } }, - { structured: { kind: 'disk' }, blobs: { kind: 'azure' } }, ] as const)('requires an awaited init for %j', (profile) => { expect(requiresExplicitInit(profile)).toBe(true); }); diff --git a/apps/server/src/modules/storage/profile.ts b/apps/server/src/modules/storage/profile.ts index f962e8717..4eca6a547 100644 --- a/apps/server/src/modules/storage/profile.ts +++ b/apps/server/src/modules/storage/profile.ts @@ -6,12 +6,12 @@ * * Structured and blob storage are independent configuration axes — the * settled direction of docs/proposals/multi-backend-storage.md §6.3. A - * profile names one backend on each axis; not every pairing is a valid - * deployment, so profiles are validated before any connection is opened. + * profile names one backend on each axis and every pairing of implemented + * backends is a valid deployment, because the axes share nothing: records go + * to the structured backend, bytes go to a file system. `sqlite` records with + * `disk` bytes is an ordinary profile, not a special case. */ -import type { BlobBackendKind } from './ports/blob.js'; - /** * Structured backend families a profile may name. * @@ -23,24 +23,45 @@ import type { BlobBackendKind } from './ports/blob.js'; */ export type RequestedStructuredKind = 'disk' | 'sqlite' | 'postgres'; +/** + * Blob backend families a profile may name. + * + * Wider than the port's {@link BlobBackendKind} for the same reason + * {@link RequestedStructuredKind} is wider than the structured one. + * + * Every member is a file system. Bytes are files wherever they live — a local + * directory today, an object store later — and never rows in the structured + * database, so the two axes stay genuinely independent and a deployment may + * pair SQL records with ordinary files. + */ +export type RequestedBlobKind = 'disk' | 'azure'; + export interface StorageProfile { structured: { kind: RequestedStructuredKind }; - blobs: { kind: BlobBackendKind }; + blobs: { kind: RequestedBlobKind }; } /** - * Backends that exist today. Naming one that is not written yet must fail - * loudly rather than half-work. + * Backends with an adapter, and therefore selectable. + * + * Selectable is not "identical to Disk". A profile may offer fewer features, + * as long as every one it does not offer is declared in `capabilities.ts` and + * refused where a user would reach for it. What disqualifies a backend is an + * *undeclared* gap — a feature that would fail with a stack trace rather than + * a sentence. */ -const IMPLEMENTED_STRUCTURED: readonly RequestedStructuredKind[] = ['disk']; -const IMPLEMENTED_BLOBS: readonly BlobBackendKind[] = ['disk']; +const AVAILABLE_STRUCTURED: readonly RequestedStructuredKind[] = [ + 'disk', + 'sqlite', +]; +const AVAILABLE_BLOBS: readonly RequestedBlobKind[] = ['disk']; const STRUCTURED_KINDS: readonly RequestedStructuredKind[] = [ 'disk', 'sqlite', 'postgres', ]; -const BLOB_KINDS: readonly string[] = ['disk', 'azure']; +const BLOB_KINDS: readonly RequestedBlobKind[] = ['disk', 'azure']; export class StorageProfileError extends Error { override name = 'StorageProfileError'; @@ -77,7 +98,7 @@ export function parseStorageProfile( 'HUABU_BLOB_BACKEND', env['HUABU_BLOB_BACKEND'], BLOB_KINDS, - ) as BlobBackendKind, + ) as RequestedBlobKind, }, }; } @@ -85,10 +106,11 @@ export function parseStorageProfile( /** * Reject profiles that cannot serve correctly, before any connection opens. * - * Today that means "named but not implemented". This is also where - * cross-axis rules belong as backends land — for example, Postgres paired - * with a node-local disk blob root is unsafe across replicas unless the - * path is a deliberately shared filesystem. + * Today that means either "named but not implemented" or "implemented only as + * an isolated preview". This is also where cross-axis rules belong as + * backends land — for example, Postgres paired with a node-local disk blob + * root is unsafe across replicas unless the path is a deliberately shared + * filesystem. * * A profile that merely offers *fewer features* is not rejected here. Those * are stated limitations rather than misconfigurations, and they are declared @@ -98,37 +120,36 @@ export function parseStorageProfile( * warning. */ export function validateStorageProfile(profile: StorageProfile): void { - if (!IMPLEMENTED_STRUCTURED.includes(profile.structured.kind)) { + if (!AVAILABLE_STRUCTURED.includes(profile.structured.kind)) { throw new StorageProfileError( `Structured backend "${profile.structured.kind}" is not implemented yet. ` + - `Available: ${IMPLEMENTED_STRUCTURED.join(', ')}.`, + `Adapters available: ${AVAILABLE_STRUCTURED.join(', ')}.`, ); } - if (!IMPLEMENTED_BLOBS.includes(profile.blobs.kind)) { + if (!AVAILABLE_BLOBS.includes(profile.blobs.kind)) { throw new StorageProfileError( `Blob backend "${profile.blobs.kind}" is not implemented yet. ` + - `Available: ${IMPLEMENTED_BLOBS.join(', ')}.`, + `Available: ${AVAILABLE_BLOBS.join(', ')}.`, ); } } /** - * Backends whose `init()` has nothing to open, so building them on demand is - * safe. + * Structured backends whose `init()` has nothing to open, so building them on + * demand is safe. * * The lazy accessor in `storage.ts` is synchronous and therefore cannot - * `await init()`. That is harmless for backends which have no connection to - * establish, and silently wrong for any that do — they would be handed to + * `await init()`. That is harmless for a backend which has no connection to + * establish, and silently wrong for any that does — it would be handed to * callers unopened. Keeping the list here, next to the other backend facts, * means adding an adapter forces a decision about it. + * + * Only the structured axis appears: every blob backend is a file system, and + * a file system has no connection to open. */ const LAZY_SAFE_STRUCTURED: readonly RequestedStructuredKind[] = ['disk']; -const LAZY_SAFE_BLOBS: readonly BlobBackendKind[] = ['disk']; /** Whether this profile may only be built through an awaited `initStorage()`. */ export function requiresExplicitInit(profile: StorageProfile): boolean { - return ( - !LAZY_SAFE_STRUCTURED.includes(profile.structured.kind) || - !LAZY_SAFE_BLOBS.includes(profile.blobs.kind) - ); + return !LAZY_SAFE_STRUCTURED.includes(profile.structured.kind); } diff --git a/apps/server/src/modules/storage/storage.ts b/apps/server/src/modules/storage/storage.ts index dc8ede82f..7d4e5918f 100644 --- a/apps/server/src/modules/storage/storage.ts +++ b/apps/server/src/modules/storage/storage.ts @@ -23,21 +23,32 @@ * through it. */ -import path from 'node:path'; +import { rm } from 'node:fs/promises'; -import { getDataDir } from '../../data-dir.js'; import { acquireWorkspaceOperationLease, - getWorkspacePath, + commitWorkspaceIdentity, + getWorkspaceHandle, + getWorkspaceKey, } from '../workspace.js'; import { DiskBlobStore } from './backends/disk/blob-store.js'; +import { getWorldCanvasId as diskWorldCanvasId } from './backends/disk/canvas-dirs.js'; +import { + diskSpaceBlobRoot, + workspaceRegistryPath, +} from './backends/disk/data-dir.js'; +import { canvasRoot } from './backends/disk/layout.js'; import { stageDiskSpaceImport } from './backends/disk/space-import.js'; import { diskSpaceTree } from './backends/disk/space-tree.js'; import { DiskStructuredStore } from './backends/disk/structured-store.js'; +import { DiskWorkspaceRepository } from './backends/disk/workspace-repository.js'; import { - DiskWorkspaceRepository, - workspaceRegistryPath, -} from './backends/disk/workspace-repository.js'; + SqliteStoreContext, + sqliteDatabasePath, +} from './backends/sqlite/database.js'; +import { SqliteStructuredStore } from './backends/sqlite/structured-store.js'; +import { SqliteWorkspaceRepository } from './backends/sqlite/workspace-repository.js'; +import { hasStorageCapability } from './capabilities.js'; import { spaceBlobAreas } from './ports/blob.js'; import { parseStorageProfile, @@ -50,6 +61,7 @@ import { withSpacePutAdmission } from './space-lifecycle-admission.js'; import type { DiskSpaceImport } from './backends/disk/space-import.js'; import type { DiskSpaceTree } from './backends/disk/space-tree.js'; +import type { SqliteSpaceSubstrate } from './backends/sqlite/space-extension.js'; import type { BlobInfo, BlobLease, @@ -83,12 +95,20 @@ export type SpaceDeleteOutcome = | SpaceDeleteFinishResult | { readonly ok: false; readonly reason: 'world-forbidden' }; -function activeWorkspacePath(): string { - return path.resolve(getWorkspacePath()); +/** + * The active Workspace as an identity to compare, not a location. + * + * The blob put saga has to prove that the Workspace has not changed under an + * awaited operation. On Disk that comparison was the resolved path; a + * Workspace that is a row has no path, so the key is what both backends can + * answer with. + */ +function activeWorkspaceKey(): string { + return getWorkspaceKey(); } -function assertActiveWorkspace(workspacePath: string, canvasId: string): void { - if (activeWorkspacePath() !== workspacePath) { +function assertActiveWorkspace(workspaceKey: string, canvasId: string): void { + if (activeWorkspaceKey() !== workspaceKey) { throw new Error( `Blob scope for Space "${canvasId}" belongs to an inactive workspace. ` + `Resolve a fresh scope after workspace activation.`, @@ -96,6 +116,27 @@ function assertActiveWorkspace(workspacePath: string, canvasId: string): void { } } +/** + * Where a Space keeps its bytes when the structured backend has no folder for + * it. + * + * Blobs are always files (`ports/blob.ts`), so a profile whose records are + * rows still needs somewhere on a file system for uploads, artifacts, the + * guide document and the memory body. *Which* directory is the Disk blob + * adapter's own business — this only supplies the Workspace the Space belongs + * to, which is the one part of the answer the adapter cannot know. + */ +function detachedSpaceRoot(canvasId: string): string { + const workspace = getWorkspaceHandle(); + if (!workspace) { + throw new Error( + `Blob scope for Space "${canvasId}" needs an active Workspace. ` + + 'Activate one before reading or writing bytes.', + ); + } + return diskSpaceBlobRoot(workspace.workspaceId, canvasId); +} + /** * Release a rejected streaming body that storage never fully consumed. * @@ -156,6 +197,29 @@ export interface Space extends SpaceHandle, SpaceBlobs { * this module's internal topology. */ readonly diskTree: DiskSpaceTree | null; + /** + * SQLite's connection point for an extension namespace, without awaiting. + * `null` on every other backend. + * + * The same shape as {@link diskTree} and there for the same reason: a + * capability one backend has, named for it and typed by its absence. What + * makes it worth its own member rather than the port's `extension()` is + * that it is *synchronous*. An owner whose own interface is synchronous — + * the Agenetes conversation stores — can resolve its place at the moment it + * needs it instead of keeping a cache primed from somewhere else. + */ + readonly sqliteTree: SqliteSpaceTree | null; +} + +/** What a namespace can ask of the SQLite backend for one Space. */ +export interface SqliteSpaceTree { + /** + * This namespace's connection point, created on demand. + * + * `null` when the Space does not exist — the same refusal the port's + * `extension()` makes, and for the same reason. + */ + extension(namespace: string): SqliteSpaceSubstrate | null; } function composeSpace(storage: Storage, canvasId: string): Space { @@ -180,23 +244,61 @@ function composeSpace(storage: Storage, canvasId: string): Space { storage.profile.structured.kind === 'disk' ? diskSpaceTree(canvasId) : null, + sqliteTree: + storage.structured instanceof SqliteStructuredStore + ? { + extension: (namespace: string) => + (storage.structured as SqliteStructuredStore).extensionSync( + canvasId, + namespace, + ), + } + : null, }; } +/** + * The blob connection for this profile, and where it puts a Space's bytes. + * + * `blobs=disk` names a *medium* — bytes are local files — so there is one + * adapter. The place is composition's to choose, and the rule is one sentence: + * **a Space's bytes live with the Space.** + * + * Where the structured backend files a Space as a directory, that directory is + * where the Space *is*, so the bytes go inside it. That is not only + * backward-compatibility with every Workspace that already exists: a Space + * folder being self-contained is what several declared capabilities are made + * of. `.huabu.zip` export is that folder archived, reveal-in-file-manager + * shows it, RFS projects it, and the built-in file tools sandbox on it. + * Relocating artifacts to a Server-owned root would quietly hollow out all + * four while every one of them still reported as available. + * + * Where a Space is a row it has no directory to be inside, so the adapter gets + * a root of its own under the Disk backend's data-directory area. + * + * One rule, two outcomes, because a Space has two possible homes — not two + * meanings for `blobs=disk`. The corollary is a genuine cross-axis constraint + * for the day a blob backend cannot co-locate: an object store would put bytes + * outside the Space folder even on Disk records, and the four capabilities + * above would then depend on both axes rather than the structured one alone + * (see `capabilities.ts`). + */ function buildBlobStore(profile: StorageProfile): BlobStore { - switch (profile.blobs.kind) { - case 'disk': - return new DiskBlobStore(); - default: - // Unreachable: validateStorageProfile rejects unimplemented kinds. - throw new Error(`Unsupported blob backend: ${profile.blobs.kind}`); + if (profile.blobs.kind !== 'disk') { + // Unreachable: validateStorageProfile rejects unimplemented kinds. + throw new Error(`Unsupported blob backend: ${profile.blobs.kind}`); } + return new DiskBlobStore( + profile.structured.kind === 'disk' ? canvasRoot : detachedSpaceRoot, + ); } function buildStructuredStore(profile: StorageProfile): StructuredStore { switch (profile.structured.kind) { case 'disk': return new DiskStructuredStore(); + case 'sqlite': + return new SqliteStructuredStore(sqliteConnection()); default: throw new Error( `Unsupported structured backend: ${profile.structured.kind}`, @@ -245,9 +347,27 @@ export function createStorage(profile: StorageProfile): Storage { // ─── Process-wide holder ──────────────────────────────────────────────────── let current: Storage | null = null; -let workspaces: DiskWorkspaceRepository | null = null; +let workspaces: WorkspaceRepository | null = null; +let sqlite: SqliteStoreContext | null = null; +let activeWorldCanvasId: string | null = null; let spaceCreateTail: Promise = Promise.resolve(); +/** + * The one SQLite connection this process holds, opened on first need. + * + * Opening it is synchronous, which is why the on-demand path stays legal for + * this profile: there is no `await` to skip. The structured store and the + * Workspace repository both borrow it, because they are one database file and + * a second connection would be a second writer. + */ +function sqliteConnection(): SqliteStoreContext { + if (sqlite) return sqlite; + const context = new SqliteStoreContext(sqliteDatabasePath()); + context.init(); + sqlite = context; + return context; +} + /** * The Workspace repository for the configured structured backend. * @@ -267,14 +387,68 @@ let spaceCreateTail: Promise = Promise.resolve(); * wired during awaited startup rather than through the on-demand path. */ export function getWorkspaceRepository(): WorkspaceRepository { - return materializedWorkspaces(); + if (workspaces) return workspaces; + const profile = activeProfile(); + workspaces = + profile.structured.kind === 'sqlite' + ? new SqliteWorkspaceRepository(sqliteConnection()) + : new DiskWorkspaceRepository(workspaceRegistryPath()); + return workspaces; +} + +/** + * Whether the profile in force serves `id`. + * + * The one form application code should use. `hasStorageCapability` takes a + * profile, and the only profile worth asking about is the one storage was + * actually opened with — a call site that parses the environment instead gets + * a different answer the moment a test or an embedder mounts an explicit + * profile. Binding it here removes the choice. + * + * Ask this in a refusal. Code that degrades to absence instead of refusing + * should keep asking the concrete predicate it depends on; see + * `capabilities.ts`. + */ +export function storageServes(id: string): boolean { + return hasStorageCapability(activeProfile(), id); } -/** Whether the Disk Workspace membership registry already exists on disk. */ +/** + * Whether the Disk Workspace membership registry already exists on disk. + * + * `false` off Disk, where there is no such registry to import into: the one + * caller is the deprecated desktop-store import, which is a Disk migration. + */ export function hasWorkspaceRegistry(): boolean { + if (!materializesWorkspaces()) return false; return materializedWorkspaces().hasDurableRegistry(); } +/** + * Whether the configured backend gives a Workspace a real directory. + * + * The one question the rest of the Server should ask before reaching for a + * Workspace path: everything that follows from "no" — no folder picker, no + * bundle import, no user skills directory — is a stated capability rather + * than a runtime surprise. + */ +export function materializesWorkspaces(): boolean { + return activeProfile().structured.kind === 'disk'; +} + +/** + * The profile in force, preferring the one storage was actually opened with. + * + * The environment answers before startup — managed mode adopts its Workspace + * while `app.ts` is still evaluating — but once `initStorage` has run, the + * profile it was handed is the truth. A test that mounts an explicit profile + * would otherwise get a Workspace repository for whatever the environment + * happened to say. + */ +function activeProfile(): StorageProfile { + return current?.profile ?? parseStorageProfile(); +} + /** * The Workspace repository, narrowed to a backend that materializes * Workspaces as real directories. @@ -287,18 +461,17 @@ export function hasWorkspaceRegistry(): boolean { * refuses outright rather than handing back a path that does not exist. */ function materializedWorkspaces(): DiskWorkspaceRepository { - if (workspaces) return workspaces; - - const profile = parseStorageProfile(); - if (profile.structured.kind !== 'disk') { + const repository = getWorkspaceRepository(); + if (!(repository instanceof DiskWorkspaceRepository)) { + const profile = parseStorageProfile(); throw new StorageProfileError( `The "${profile.structured.kind}" structured backend does not materialize ` + - `Workspaces as directories. Implement a locator for it before using ` + - `directory-shaped Workspace activation.`, + `Workspaces as directories, so there is no folder to adopt, reveal, or ` + + `resolve. Select the disk structured backend for directory-shaped ` + + `Workspace activation.`, ); } - workspaces = new DiskWorkspaceRepository(workspaceRegistryPath(getDataDir())); - return workspaces; + return repository; } /** @@ -311,6 +484,32 @@ export function adoptWorkspaceDirectory( return materializedWorkspaces().adopt(workspacePath); } +/** + * Create a Workspace that has no directory. + * + * The counterpart to {@link adoptWorkspaceDirectory} for a backend where a + * Workspace is a row: nothing to adopt, so a name is the whole of it. It is + * not a port member for the same reason locating a Workspace is not — Disk + * could only serve it by inventing a folder the user never picked, and the + * point of the port is that it says nothing about where a Workspace is. + * + * A deployment that keeps Workspaces in a database needs this to hold more + * than the one the Server opens for itself, which is the whole of multi- + * Workspace support there: every other operation — list, activate, rename, + * forget — is already on the port. + */ +export function createNamedWorkspace(name: string): Promise { + const repository = getWorkspaceRepository(); + if (!(repository instanceof SqliteWorkspaceRepository)) { + throw new StorageProfileError( + `The "${activeProfile().structured.kind}" structured backend keeps ` + + 'Workspaces as directories, so a Workspace is created by adopting a ' + + 'folder rather than by name.', + ); + } + return repository.create(name); +} + /** The registered Workspace materialized at a directory, if there is one. */ export function workspaceAtDirectory( workspacePath: string, @@ -318,8 +517,16 @@ export function workspaceAtDirectory( return materializedWorkspaces().at(workspacePath); } -/** The directory backing a registered Workspace, or null if it is not one. */ +/** + * The directory backing a registered Workspace, or `null` if there is none. + * + * `null` covers both "not a registered Workspace" and "this backend does not + * put Workspaces in folders". Callers already handle the first, and treating + * the second the same way is what lets a listing render on either backend + * instead of failing whole. + */ export function workspaceDirectory(workspaceId: string): string | null { + if (!materializesWorkspaces()) return null; return materializedWorkspaces().directoryOf(workspaceId); } @@ -372,12 +579,90 @@ function ensure(): Storage { export async function initStorage( profile: StorageProfile = parseStorageProfile(), ): Promise { + // Rebuild the Workspace repository against this profile: a repository + // memoized from the environment before an explicit profile was chosen would + // answer for the wrong backend. + workspaces = null; const storage = createStorage(profile); await Promise.all([storage.structured.init(), storage.blobs.init()]); current = storage; + await ensureActiveWorkspace(profile); return storage; } +/** + * Make sure a Workspace is active, for a backend that can decide by itself. + * + * On Disk the Workspace is a folder the user chooses, so the Server waits. + * Where a Workspace is a row there is nothing to choose and nothing to ask + * for: the first start creates one and activates it, and the app is usable + * without a setup step. A Workspace already activated — by managed mode, or + * by a previous call — is left alone. + */ +async function ensureActiveWorkspace(profile: StorageProfile): Promise { + if (profile.structured.kind !== 'sqlite') return; + const repository = getWorkspaceRepository(); + if (!(repository instanceof SqliteWorkspaceRepository)) return; + // The question is whether *this connection* is pointed at a Workspace, not + // whether the process remembers one. A handle left over from a previous + // profile is a name without a namespace behind it. + if (sqliteConnection().activeWorkspaceId() !== null) return; + const workspace = await repository.ensureDefault(DEFAULT_WORKSPACE_NAME); + await activateWorkspace(workspace); +} + +/** The name a SQL deployment's first Workspace is given. */ +const DEFAULT_WORKSPACE_NAME = 'Workspace'; + +/** + * Select one Workspace as the process's active namespace. + * + * Two things have to agree: the Server's own active-Workspace state and the + * namespace the backend scopes its queries to. Doing both here keeps them + * from drifting — a connection still pointed at the previous Workspace would + * answer confidently with the wrong Spaces. + */ +export async function activateWorkspace( + workspace: WorkspaceHandle, +): Promise { + if (sqlite) sqlite.useWorkspace(workspace.workspaceId); + activeWorldCanvasId = null; + commitWorkspaceIdentity(workspace); + if (workspaces instanceof SqliteWorkspaceRepository) { + workspaces.markOpened(workspace.workspaceId); + } + // A Workspace with no World has no Portal target and no home view. On Disk + // the World is written by workspace preparation; here the same step belongs + // to activation, because activation is the whole of "open a Workspace". + activeWorldCanvasId = await ensure().structured.spaces().ensureWorld(); +} + +/** + * The hidden World Space of the active Workspace, or `null` before one is + * opened. + * + * Disk answers from its directory index, which re-scans, so a Workspace edited + * from outside the app stays correct. Elsewhere the id is remembered from + * activation: it is minted once per Workspace and never changes, and reading + * it is synchronous in call sites that cannot await. + */ +export function getWorldCanvasId(): string | null { + return materializesWorkspaces() ? diskWorldCanvasId() : activeWorldCanvasId; +} + +export function requireWorldCanvasId(): string { + const canvasId = getWorldCanvasId(); + if (!canvasId) { + throw new Error('Configured workspace has no World canvas'); + } + return canvasId; +} + +export function isWorldCanvasId(canvasId: string): boolean { + const world = getWorldCanvasId(); + return world !== null && world === canvasId; +} + export function getStorage(): Storage { return ensure(); } @@ -396,10 +681,17 @@ export function getStorage(): Storage { */ export async function closeStorage(): Promise { const storage = current; + const connection = sqlite; current = null; workspaces = null; - if (!storage) return; - await Promise.all([storage.structured.close(), storage.blobs.close()]); + sqlite = null; + activeWorldCanvasId = null; + if (storage) { + await Promise.all([storage.structured.close(), storage.blobs.close()]); + } + // The shared connection outlives either store, so closing it is this + // module's job rather than whichever adapter happens to hold it. + connection?.close(); } export function getBlobStore(): BlobStore { @@ -468,6 +760,16 @@ export async function deleteSpace( area.deleteAll(), ), ); + // Where the record is a row, nothing else will ever remove the + // directory those areas sat in. Sweeping the areas is the port's + // contract; removing what composition placed them under is this + // module's, and it is what stops a deleted Space leaving a husk behind. + if (storage.profile.structured.kind !== 'disk') { + await rm(detachedSpaceRoot(canvasId), { + recursive: true, + force: true, + }); + } return await started.session.finish(); } catch (error) { await started.session.abort(); @@ -495,7 +797,7 @@ function guardedBlobScope( canvasId: string, delegate: BlobScope, ): BlobScope { - const workspacePath = activeWorkspacePath(); + const workspaceKey = activeWorkspaceKey(); async function requireSpace(): Promise { const record = await storage.structured.space(canvasId).read(); @@ -507,16 +809,12 @@ function guardedBlobScope( return { async put(name: string, body: Readable | Buffer): Promise { try { - return await withSpacePutAdmission( - workspacePath, - canvasId, - async () => { - assertActiveWorkspace(workspacePath, canvasId); - await requireSpace(); - assertActiveWorkspace(workspacePath, canvasId); - return delegate.put(name, body); - }, - ); + return await withSpacePutAdmission(workspaceKey, canvasId, async () => { + assertActiveWorkspace(workspaceKey, canvasId); + await requireSpace(); + assertActiveWorkspace(workspaceKey, canvasId); + return delegate.put(name, body); + }); } catch (error) { drainRejectedBody(body); throw error; diff --git a/apps/server/src/modules/storage/testing.ts b/apps/server/src/modules/storage/testing.ts index a52183145..658f2a96b 100644 --- a/apps/server/src/modules/storage/testing.ts +++ b/apps/server/src/modules/storage/testing.ts @@ -36,6 +36,7 @@ import type { Storage } from './storage.js'; */ export const PRODUCT_STORAGE_PROFILES: readonly StorageProfile[] = [ { structured: { kind: 'disk' }, blobs: { kind: 'disk' } }, + { structured: { kind: 'sqlite' }, blobs: { kind: 'disk' } }, ]; /** Readable name for a profile, for test titles. */ @@ -43,11 +44,32 @@ export function describeProfile(profile: StorageProfile): string { return `${profile.structured.kind}/${profile.blobs.kind}`; } +function restoreEnv(key: string, previous: string | undefined): void { + if (previous === undefined) delete process.env[key]; + else process.env[key] = previous; +} + export interface MountedTestStorage { readonly profile: StorageProfile; readonly storage: Storage; - /** The temporary Workspace. Only the harness itself should name paths. */ + /** + * The temporary directory this mount owns. + * + * For a Disk profile it is the Workspace itself; for a profile that keeps + * Workspaces in a database it is only where the harness put that database + * and the Space byte root. Either way it is the harness's own business — a + * case that reads it has stopped being evidence of anything portable. + */ readonly workspacePath: string; + /** + * Close the connections and open them again on the same durable state. + * + * What a restart actually is, for a suite that needs to prove something + * survives one. Returns the fresh {@link Storage}; the mount's own + * `storage` field still refers to the closed one, so a caller uses the + * value this returns. + */ + reopen(): Promise; close(): Promise; } @@ -67,11 +89,24 @@ export async function mountTestWorkspace( // A profile label reads as `disk/disk`, which is not a directory name. const safePrefix = prefix.replace(/[^a-zA-Z0-9._-]/g, '-'); const workspacePath = mkdtempSync(path.join(tmpdir(), safePrefix)); - // Prepares and commits the Workspace, exactly as a synchronous activation - // does. Workspace selection precedes storage here for the same reason it - // does at boot: the backend is process-wide and the Workspace is the - // namespace selected inside it. - setWorkspacePath(workspacePath); + const previousSqlitePath = process.env['HUABU_SQLITE_PATH']; + const previousBlobRoot = process.env['HUABU_BLOB_ROOT']; + + if (profile.structured.kind === 'disk') { + // Prepares and commits the Workspace, exactly as a synchronous activation + // does. Workspace selection precedes storage here for the same reason it + // does at boot: the backend is process-wide and the Workspace is the + // namespace selected inside it. + setWorkspacePath(workspacePath); + } else { + // No Workspace folder to pick. The Workspace is a row the backend creates + // on first start, and `initStorage` activates it — which is exactly the + // behaviour that lets this profile run without one. The temp directory + // only gives this mount its own database file and its own byte root, so + // parallel suites do not share either. + process.env['HUABU_SQLITE_PATH'] = path.join(workspacePath, 'huabu.sqlite'); + process.env['HUABU_BLOB_ROOT'] = path.join(workspacePath, 'blobs'); + } const storage = await initStorage(profile); // A namespace nobody has opened before has no World, and a Workspace @@ -82,8 +117,17 @@ export async function mountTestWorkspace( profile, storage, workspacePath, + async reopen(): Promise { + await closeStorage(); + if (profile.structured.kind === 'disk') setWorkspacePath(workspacePath); + const reopened = await initStorage(profile); + await reopened.structured.spaces().ensureWorld(); + return reopened; + }, async close(): Promise { await closeStorage(); + restoreEnv('HUABU_SQLITE_PATH', previousSqlitePath); + restoreEnv('HUABU_BLOB_ROOT', previousBlobRoot); rmSync(workspacePath, { recursive: true, force: true }); }, }; diff --git a/apps/server/src/modules/workspace.route.test.ts b/apps/server/src/modules/workspace.route.test.ts index 7973ca1fa..8a8ec8bb1 100644 --- a/apps/server/src/modules/workspace.route.test.ts +++ b/apps/server/src/modules/workspace.route.test.ts @@ -23,6 +23,7 @@ vi.mock('./workspace.js', async (importOriginal) => { name: workspaceState.name, } : null, + getWorkspaceDirectory: () => workspaceState.path, getWorkspacePath: () => workspaceState.path, isManagedMode: () => workspaceState.managed, isWorkspaceConfigured: () => workspaceState.configured, diff --git a/apps/server/src/modules/workspace.route.ts b/apps/server/src/modules/workspace.route.ts index e89c4fd50..173f05e6e 100644 --- a/apps/server/src/modules/workspace.route.ts +++ b/apps/server/src/modules/workspace.route.ts @@ -9,15 +9,21 @@ import path from 'node:path'; import { validatePathSchema, workspacePathSchema } from '@huabu/shared'; import { resetPreprocessDispatcher } from './preprocessing/index.js'; -import { getStructuredStore, resetStorageCache } from './storage/index.js'; +import { + getStructuredStore, + materializesWorkspaces, + resetStorageCache, + storageServes, + unavailableCapabilityMessage, +} from './storage/index.js'; import { activateWorkspacePath, WorkspaceActivationInProgressError, WorkspaceActivationTimeoutError, } from './workspace-activation.js'; import { + getWorkspaceDirectory, getWorkspaceHandle, - getWorkspacePath, isManagedMode, } from './workspace.js'; @@ -161,15 +167,21 @@ async function buildWorkspaceState(): Promise { configured, workspaceId: workspace?.workspaceId ?? null, // Free-mode active absolute path. Never exposed in managed mode. - path: workspace && !managed ? getWorkspacePath() : null, + // Null in managed mode, and null wherever a Workspace has no folder at + // all. The client already renders a Workspace with no path. + path: workspace && !managed ? getWorkspaceDirectory() : null, // Persisted display label. Safe to send in either mode. name: workspace?.name ?? null, worldCanvasId: configured ? await getStructuredStore().spaces().worldId() : null, capabilities: { - canChangeWorkspace: !managed, - nativePicker: !managed && canShowNativePicker(), + // Switching Workspaces means picking a folder in this API. A backend + // that keeps Workspaces as rows has one already open and no folder to + // offer, so the client stops showing a picker it could not honour. + canChangeWorkspace: !managed && materializesWorkspaces(), + nativePicker: + !managed && materializesWorkspaces() && canShowNativePicker(), }, }; } @@ -194,6 +206,14 @@ const workspaceRoutes: FastifyPluginAsync = async (app) => { if (isManagedMode()) { return sendError(reply, 403, 'Workspace is locked'); } + if (!storageServes('workspace-directory')) { + return sendError( + reply, + 409, + unavailableCapabilityMessage('workspace-directory'), + 'STORAGE_CAPABILITY_UNAVAILABLE', + ); + } if (!isLocalhost(request.ip)) { return sendError( reply, @@ -253,6 +273,14 @@ const workspaceRoutes: FastifyPluginAsync = async (app) => { 'Forbidden: workspace settings can only be changed from localhost', ); } + if (!storageServes('workspace-directory')) { + return sendError( + reply, + 409, + unavailableCapabilityMessage('workspace-directory'), + 'STORAGE_CAPABILITY_UNAVAILABLE', + ); + } const parsed = workspacePathSchema.safeParse(request.body); if (!parsed.success) { return sendError( diff --git a/apps/server/src/modules/workspace.ts b/apps/server/src/modules/workspace.ts index 6d898fda5..c50d4f96f 100644 --- a/apps/server/src/modules/workspace.ts +++ b/apps/server/src/modules/workspace.ts @@ -41,7 +41,10 @@ import path from 'node:path'; import { resetExternalNoteSessions } from './canvas/external-watcher.js'; import { refreshCanvasDirIndex } from './storage/canvas-dirs.js'; -import { adoptWorkspaceDirectory } from './storage/index.js'; +import { + adoptWorkspaceDirectory, + materializesWorkspaces, +} from './storage/index.js'; import { prepareWorkspaceOnDisk } from './workspace-prepare.js'; import { invalidateUserSkill } from '../prompt/index.js'; @@ -130,6 +133,17 @@ export function initWorkspaceFromEnv(): void { `${ENV_KEY} must be an absolute path, got: ${JSON.stringify(fromEnv)}`, ); } + if (!materializesWorkspaces()) { + // `HUABU_WORKSPACE` names a folder, and this backend has none. Refusing + // here rather than half-way through preparation, because the operator's + // next move is a configuration change either way — and because a SQL + // profile is already "locked at startup" without being told a path. + throw new Error( + `${ENV_KEY} names a Workspace folder, which the configured structured ` + + 'backend does not use. Unset it (the backend opens its own ' + + 'Workspace), or select the disk structured backend.', + ); + } const resolvedPath = path.resolve(fromEnv); _managed = true; prepareWorkspaceOnDisk(resolvedPath); @@ -155,6 +169,34 @@ export function getWorkspacePath(): string { return _workspacePath; } +/** + * The active Workspace's directory, or `null` when the backend has none. + * + * The honest form of {@link getWorkspacePath} for code that can cope with a + * Workspace that is a row rather than a folder. Anything that genuinely needs + * a directory should keep calling {@link getWorkspacePath} and let it refuse. + */ +export function getWorkspaceDirectory(): string | null { + return _workspacePath; +} + +/** + * A stable process-local key for the active Workspace. + * + * Leases, admission gates, and scope bindings need to say "the same Workspace + * as before" without needing it to be a place. On Disk that is still the + * resolved path, so nothing about the existing behaviour changes; elsewhere it + * is the Workspace identity. + */ +export function getWorkspaceKey(): string { + if (_workspacePath) return _workspacePath; + if (_workspaceHandle) return `workspace:${_workspaceHandle.workspaceId}`; + throw new Error( + 'Workspace has not been configured. Activate a workspace first ' + + `(PUT /api/workspace) or set ${ENV_KEY} in the environment.`, + ); +} + /** The active immutable Workspace identity, or null before configuration. */ export function getWorkspaceHandle(): WorkspaceHandle | null { return _workspaceHandle; @@ -168,7 +210,7 @@ export function getWorkspaceHandle(): WorkspaceHandle | null { * same path remains allowed. */ export function acquireWorkspaceOperationLease(): WorkspaceOperationLease { - const workspacePath = getWorkspacePath(); + const workspacePath = getWorkspaceKey(); if ( _activatingWorkspacePath !== null && @@ -302,6 +344,32 @@ function commitResolvedWorkspacePath(resolvedPath: string): void { resetExternalNoteSessions(); } +/** + * Activate a Workspace that has no directory. + * + * The counterpart to {@link commitWorkspacePath} for a backend where a + * Workspace is a row: same in-process effects — identity, cache invalidation, + * watcher reset — with nothing to resolve on the filesystem. Kept separate + * rather than making the path optional, so no caller can commit "a Workspace + * somewhere" by accident. + */ +export function commitWorkspaceIdentity(workspace: WorkspaceHandle): void { + assertNoWorkspaceActivationInProgress(); + const key = `workspace:${workspace.workspaceId}`; + if ( + _workspaceOperationLeaseCount > 0 && + _leasedWorkspacePath !== null && + _leasedWorkspacePath !== key + ) { + throw new WorkspaceOperationInProgressError(); + } + _workspaceHandle = workspace; + _workspacePath = null; + refreshCanvasDirIndex(); + invalidateUserSkill(); + resetExternalNoteSessions(); +} + /** Refresh metadata for the active Workspace without switching namespaces. */ export function updateActiveWorkspaceHandle( workspace: WorkspaceHandle, diff --git a/apps/server/src/modules/workspace/paths.ts b/apps/server/src/modules/workspace/paths.ts index 9fd1a22e5..960ca2212 100644 --- a/apps/server/src/modules/workspace/paths.ts +++ b/apps/server/src/modules/workspace/paths.ts @@ -38,7 +38,7 @@ import path from 'node:path'; -import { space } from '../storage/index.js'; +import { materializesWorkspaces, space } from '../storage/index.js'; import { getWorkspacePath } from '../workspace.js'; import type { Namespace } from '@agenetes/protocol'; @@ -60,14 +60,19 @@ const LEGACY_HISTORY_DIR_NAME = '.history'; * these paths exist only where the backend has a tree. */ function spaceRoot(canvasId: string): string { - const tree = space(canvasId).diskTree; - if (!tree) { + const directory = optionalSpaceRoot(canvasId); + if (!directory) { throw new Error( `Per-Space files for "${canvasId}" need a Space directory, which the ` + 'active structured backend does not provide.', ); } - return tree.directory(); + return directory; +} + +/** The Space's directory, or `null` where the backend has no tree. */ +function optionalSpaceRoot(canvasId: string): string | null { + return space(canvasId).diskTree?.directory() ?? null; } function legacyHistoryDir(canvasId: string): string { @@ -85,6 +90,20 @@ export function workspaceMemoryPath(): string { return path.join(settingDir(), 'user.md'); } +/** + * Whether the Workspace-level `setting/` tier exists on this backend at all. + * + * `setting/` is a folder the user edits by hand — the memory document and the + * skills they author. A Workspace that is a row has nowhere to put it, and + * that is declared as the `workspace-user-memory` and `workspace-user-skills` + * capabilities rather than emulated. Callers that merely *read* the tier ask + * here and degrade to absence; callers that write refuse with the declared + * message. + */ +export function hasWorkspaceSettingDirectory(): boolean { + return materializesWorkspaces(); +} + // ─── Workspace-level setting / user skills ───────────────────────────────── /** @@ -130,8 +149,16 @@ export function acpSessionsPath(canvasId: string): string { * empty-canvasId no-op). See docs/proposals/layered-architecture.md §7 M5.0. */ export function canvasAcpNamespace(canvasId: string): Namespace { - return { - name: canvasId, - storage: canvasId ? { root: legacyHistoryDir(canvasId) } : undefined, - }; + if (!canvasId) return { name: canvasId }; + // `storage.root` is a *directory*, so it is present exactly when the Space + // has one. Omitting it is not a degraded namespace: it is how the + // conversation stores learn that this Space keeps its threads somewhere + // other than a folder (`agent/agenetes/conversation-stores.ts`). + const root = optionalSpaceRoot(canvasId); + return root === null + ? { name: canvasId } + : { + name: canvasId, + storage: { root: path.join(root, LEGACY_HISTORY_DIR_NAME) }, + }; } diff --git a/apps/server/src/modules/workspaces.route.test.ts b/apps/server/src/modules/workspaces.route.test.ts index f1af8821b..6d73384fc 100644 --- a/apps/server/src/modules/workspaces.route.test.ts +++ b/apps/server/src/modules/workspaces.route.test.ts @@ -35,6 +35,8 @@ function handleOf({ workspaceId, name }: TestMember): TestHandle { const testState = vi.hoisted(() => ({ managed: false, + /** Whether the configured structured backend files Workspaces as folders. */ + materializes: true, active: null as TestHandle | null, activePath: null as string | null, members: [] as TestMember[], @@ -44,6 +46,11 @@ const testState = vi.hoisted(() => ({ const storageMocks = vi.hoisted(() => ({ resetStorageCache: vi.fn(), + activateWorkspace: vi.fn(async () => {}), + createNamedWorkspace: vi.fn(async (name: string) => ({ + workspaceId: NEW_ID, + name, + })), })); const activationMocks = vi.hoisted(() => ({ @@ -141,9 +148,16 @@ const locatorMocks = vi.hoisted(() => ({ })); vi.mock('./storage/index.js', () => ({ + activateWorkspace: storageMocks.activateWorkspace, getWorkspaceRepository: () => repository, hasWorkspaceRegistry: () => testState.registryInitialized, + // These routes are mostly the directory-shaped Workspace API, so the default + // profile under test is the one that has directories; a case that is about + // the other kind flips `materializes`. + materializesWorkspaces: () => testState.materializes, + createNamedWorkspace: storageMocks.createNamedWorkspace, resetStorageCache: storageMocks.resetStorageCache, + unavailableCapabilityMessage: (id: string) => `capability ${id}`, adoptWorkspaceDirectory: locatorMocks.adoptWorkspaceDirectory, ensureWorkspaceManifestOnDisk: locatorMocks.ensureWorkspaceManifestOnDisk, workspaceAtDirectory: locatorMocks.workspaceAtDirectory, @@ -156,6 +170,7 @@ vi.mock('./workspace.js', () => ({ testState.active = locatorMocks.adoptWorkspaceDirectory(workspacePath); testState.activePath = workspacePath; }, + getWorkspaceDirectory: () => testState.activePath, getWorkspaceHandle: () => testState.active, getWorkspacePath: () => { if (!testState.activePath) throw new Error('No active Workspace path'); @@ -196,6 +211,7 @@ async function buildApp() { beforeEach(() => { testState.managed = false; + testState.materializes = true; testState.registryInitialized = true; testState.members = [ { @@ -542,3 +558,56 @@ describe('plural Workspace management routes', () => { } }); }); + +/** + * A deployment whose Workspaces are rows still holds more than one. + * + * Everything else the collection needs — list, activate, rename, forget — is + * already on the port and backend-neutral. Creation is the one operation the + * folder API could not express, because there is no folder to name. + */ +describe('Workspace collection on a backend with no folders', () => { + beforeEach(() => { + testState.materializes = false; + }); + + it('creates a Workspace from a name alone', async () => { + const app = await buildApp(); + try { + const response = await app.inject({ + method: 'POST', + url: '/workspaces', + payload: { name: 'Second' }, + }); + + expect(response.statusCode).toBe(201); + expect(storageMocks.createNamedWorkspace).toHaveBeenCalledWith('Second'); + expect(response.json()).toEqual({ + workspaceId: NEW_ID, + name: 'Second', + // No folder to report, and not the active one. + path: null, + active: false, + }); + } finally { + await app.close(); + } + }); + + it('asks for the name it can actually use', async () => { + const app = await buildApp(); + try { + const response = await app.inject({ + method: 'POST', + url: '/workspaces', + payload: { path: '/tmp/somewhere' }, + }); + + expect(response.statusCode).toBe(400); + expect(response.json().message).toMatch(/name is required/i); + expect(storageMocks.createNamedWorkspace).not.toHaveBeenCalled(); + } finally { + await app.close(); + } + }); +}); diff --git a/apps/server/src/modules/workspaces.route.ts b/apps/server/src/modules/workspaces.route.ts index 4adcba1bd..77b78dc84 100644 --- a/apps/server/src/modules/workspaces.route.ts +++ b/apps/server/src/modules/workspaces.route.ts @@ -10,10 +10,13 @@ import { workspaceCreateSchema, workspaceRenameSchema } from '@huabu/shared'; import { migrateLegacyDesktopWorkspaceStore } from './legacy-desktop-workspace-store.js'; import { resetPreprocessDispatcher } from './preprocessing/index.js'; import { + activateWorkspace, adoptWorkspaceDirectory, + createNamedWorkspace, ensureWorkspaceManifestOnDisk, getWorkspaceRepository, hasWorkspaceRegistry, + materializesWorkspaces, resetStorageCache, workspaceAtDirectory, workspaceDirectory, @@ -27,6 +30,7 @@ import { } from './workspace-activation.js'; import { commitWorkspacePath, + getWorkspaceDirectory, getWorkspaceHandle, getWorkspacePath, isManagedMode, @@ -84,10 +88,16 @@ function rejectReadOnlyMutation( function descriptor(workspace: WorkspaceHandle): WorkspaceDescriptor { const workspacePath = workspaceDirectory(workspace.workspaceId); const activeHandle = getWorkspaceHandle(); + const activeDirectory = getWorkspaceDirectory(); + // Identity decides which Workspace is active. On Disk the location has to + // agree as well, because a registered id can be pointed at a folder the + // process is not the one serving; where there is no folder, there is + // nothing else to agree. const active = activeHandle?.workspaceId === workspace.workspaceId && - workspacePath !== null && - path.resolve(getWorkspacePath()) === path.resolve(workspacePath); + (workspacePath === null || activeDirectory === null + ? !materializesWorkspaces() + : path.resolve(activeDirectory) === path.resolve(workspacePath)); return { workspaceId: workspace.workspaceId, name: workspace.name, @@ -186,7 +196,14 @@ const workspacesRoutes: FastifyPluginAsync = async (app) => { * behind a preparation fork each. */ function importLegacyDesktopStore(): void { - if (legacyDesktopStoreImported || isManagedMode() || hasWorkspaceRegistry()) + if ( + legacyDesktopStoreImported || + isManagedMode() || + // The deprecated store remembers folders, so there is nothing to import + // where a Workspace is not one. + !materializesWorkspaces() || + hasWorkspaceRegistry() + ) return; const filePath = process.env.HUABU_LEGACY_WORKSPACE_STORE?.trim(); if (!filePath) return; @@ -219,8 +236,31 @@ const workspacesRoutes: FastifyPluginAsync = async (app) => { ); } + // A Workspace that is a row is created by name: there is no folder to + // adopt, prepare, or fork a child process for. The deployment still holds + // as many Workspaces as it likes — this is the only one of the collection + // operations the folder API could not already express. + if (!materializesWorkspaces()) { + const name = parsed.data.name; + if (!name) { + return sendError(reply, 400, 'Workspace name is required'); + } + try { + return reply + .status(201) + .send(descriptor(await createNamedWorkspace(name))); + } catch (error) { + return sendPreparationError(reply, error); + } + } + + const requestedPath = parsed.data.path; + if (!requestedPath) { + return sendError(reply, 400, 'Workspace path is required'); + } + try { - const workspacePath = resolveWorkspacePath(parsed.data.path); + const workspacePath = resolveWorkspacePath(requestedPath); const repository = getWorkspaceRepository(); const existing = workspaceAtDirectory(workspacePath); if (existing) { @@ -284,10 +324,23 @@ const workspacesRoutes: FastifyPluginAsync = async (app) => { if (typeof parsedId !== 'string') return parsedId; const workspace = await getWorkspaceRepository().get(parsedId); - const workspacePath = workspaceDirectory(parsedId); - if (!workspace || !workspacePath) { - return sendError(reply, 404, 'Workspace not found'); + if (!workspace) return sendError(reply, 404, 'Workspace not found'); + + // A Workspace that is a row needs no preparation: activation is + // re-scoping the connection, which is why this profile can switch + // Workspaces without a folder to prepare or a child process to fork. + if (!materializesWorkspaces()) { + try { + await activateWorkspace(workspace); + resetPreprocessDispatcher(); + return reply.send(descriptor(getWorkspaceHandle() ?? workspace)); + } catch (error) { + return sendPreparationError(reply, error); + } } + + const workspacePath = workspaceDirectory(parsedId); + if (!workspacePath) return sendError(reply, 404, 'Workspace not found'); try { await activateWorkspacePath(workspacePath); resetStorageCache(); diff --git a/apps/web/src/api/canvas.test.ts b/apps/web/src/api/canvas.test.ts new file mode 100644 index 000000000..6cb34ecf3 --- /dev/null +++ b/apps/web/src/api/canvas.test.ts @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { exportCanvas } from './canvas'; + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); +}); + +describe('Space export download', () => { + it.each([400, 404, 500])( + 'does not navigate or download on HTTP %s', + async (status) => { + const click = vi + .spyOn(HTMLAnchorElement.prototype, 'click') + .mockImplementation(() => {}); + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + message: 'Export is unavailable', + code: 'STORAGE_CAPABILITY_UNAVAILABLE', + }), + { status }, + ), + ), + ); + + await expect(exportCanvas('space-1')).rejects.toMatchObject({ + status, + message: 'Export is unavailable', + }); + expect(click).not.toHaveBeenCalled(); + expect(document.querySelector('a')).toBeNull(); + }, + ); + + it('preflights eligibility then preserves the native streamed Disk download', async () => { + const fetch = vi + .fn() + .mockResolvedValue(new Response(null, { status: 204 })); + vi.stubGlobal('fetch', fetch); + let download: string | undefined; + vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(function ( + this: HTMLAnchorElement, + ) { + download = this.getAttribute('href') ?? undefined; + }); + + await exportCanvas('space-1'); + + expect(fetch).toHaveBeenCalledTimes(1); + expect(fetch.mock.calls[0][0]).toMatch( + /\/canvas\/space-1\/export\?check=true$/, + ); + expect(download).toMatch(/\/canvas\/space-1\/export$/); + expect(document.querySelector('a')).toBeNull(); + }); +}); diff --git a/apps/web/src/api/canvas.ts b/apps/web/src/api/canvas.ts index 471942b7b..db1c0d83d 100644 --- a/apps/web/src/api/canvas.ts +++ b/apps/web/src/api/canvas.ts @@ -294,9 +294,9 @@ export async function getNodeContent( } /** - * Download the canvas as a self-contained `.huabu.json` export bundle. + * Download the canvas as a self-contained `.huabu.zip` export bundle. * - * Performs a lightweight existence check via getCanvas to catch errors early, + * Preflights export eligibility to surface refusals inside the application, * then triggers a native browser download via a temporary `` link * so the full response body never needs to live in JS memory. * @@ -304,11 +304,7 @@ export async function getNodeContent( * `Content-Disposition` header. */ export async function exportCanvas(canvasId: string): Promise { - // Lightweight pre-check: verify canvas exists without running the export. - const canvas = await getCanvas(canvasId); - if (!canvas) { - throw new Error('Canvas not found'); - } + await apiFetch(`${routes.canvasExport(canvasId)}?check=true`); const url = apiUrl(routes.canvasExport(canvasId)); const a = document.createElement('a'); diff --git a/apps/web/src/components/Panels/Header/CanvasMenu.tsx b/apps/web/src/components/Panels/Header/CanvasMenu.tsx index 505988d29..5bea70317 100644 --- a/apps/web/src/components/Panels/Header/CanvasMenu.tsx +++ b/apps/web/src/components/Panels/Header/CanvasMenu.tsx @@ -6,6 +6,7 @@ import { ChevronDown } from 'lucide-react'; import React, { useCallback, useEffect, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; +import { ApiError } from '../../../api/_client'; import { exportCanvas } from '../../../api/canvas.ts'; import useCanvasStore from '../../../store/canvasStore.ts'; import { useWorkspaceStore } from '../../../store/workspaceStore.ts'; @@ -82,9 +83,14 @@ export const CanvasMenu: React.FC = ({ onOpenShortcuts }) => { await exportCanvas(canvasId); toast(t('canvasList.exportStarted'), { tone: 'success' }); } catch (err) { - toast(err instanceof Error ? err.message : t('canvasList.exportFailed'), { - tone: 'danger', - }); + toast( + err instanceof ApiError && err.code === 'STORAGE_CAPABILITY_UNAVAILABLE' + ? t('canvasList.exportUnavailable') + : err instanceof Error + ? err.message + : t('canvasList.exportFailed'), + { tone: 'danger' }, + ); } }, [canvasId, t]); diff --git a/apps/web/src/hooks/useCanvasActions.test.tsx b/apps/web/src/hooks/useCanvasActions.test.tsx new file mode 100644 index 000000000..9d2090d37 --- /dev/null +++ b/apps/web/src/hooks/useCanvasActions.test.tsx @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { act } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; +import { createMemoryRouter, RouterProvider } from 'react-router-dom'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { useCanvasActions } from './useCanvasActions'; +import { ToastContainer } from '../components/Common/Toast'; + +vi.mock('react-i18next', () => ({ + useTranslation: () => ({ t: (key: string) => key }), +})); +vi.mock('./useInputMode', () => ({ useEffectiveInputMode: () => 'mouse' })); + +( + globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean } +).IS_REACT_ACT_ENVIRONMENT = true; +let container: HTMLDivElement; +let root: Root; +function ImportControl() { + const { onFileChange, isImporting } = useCanvasActions(); + return ( + <> + void onFileChange(e)} + disabled={isImporting} + /> + + + ); +} +beforeEach(() => { + container = document.createElement('div'); + document.body.appendChild(container); + root = createRoot(container); +}); +afterEach(async () => { + await act(async () => { + document + .querySelectorAll('[aria-label="actions.dismiss"]') + .forEach((button) => button.click()); + }); + await act(async () => root.unmount()); + container.remove(); + vi.unstubAllGlobals(); +}); + +async function selectArchive() { + const input = container.querySelector('input'); + if (!input) throw new Error('Import input was not rendered'); + Object.defineProperty(input, 'files', { + value: [new File(['zip'], 'space.huabu.zip')], + configurable: true, + }); + await act(async () => { + input.dispatchEvent(new Event('change', { bubbles: true })); + }); + return input; +} +async function renderImport() { + const router = createMemoryRouter( + [ + { path: '/spaces', element: }, + { path: '/canvas/:id', element:
Imported Space
}, + ], + { initialEntries: ['/spaces'] }, + ); + await act(async () => root.render()); + return router; +} +describe('Space import feedback', () => { + it('shows a dismissible storage refusal, stays in the app, and allows retry', async () => { + const fetch = vi.fn(); + fetch.mockImplementation( + async () => + new Response( + JSON.stringify({ + code: 'STORAGE_CAPABILITY_UNAVAILABLE', + message: 'Technical storage detail', + }), + { status: 400 }, + ), + ); + vi.stubGlobal('fetch', fetch); + const router = await renderImport(); + const input = await selectArchive(); + expect(document.querySelector('[role="status"]')?.textContent).toContain( + 'canvasList.importUnavailable', + ); + expect( + document.querySelector('[aria-label="actions.dismiss"]'), + ).not.toBeNull(); + expect(router.state.location.pathname).toBe('/spaces'); + expect(input.disabled).toBe(false); + expect(input.value).toBe(''); + await selectArchive(); + expect(fetch).toHaveBeenCalledTimes(2); + }); + it('shows other server failures instead of swallowing them', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response(JSON.stringify({ message: 'Invalid archive' }), { + status: 400, + }), + ), + ); + await renderImport(); + await selectArchive(); + expect(document.querySelector('[role="status"]')?.textContent).toContain( + 'Invalid archive', + ); + }); + it('opens a successfully imported Disk Space', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response(JSON.stringify({ canvasId: 'imported-space' }), { + status: 200, + }), + ), + ); + const router = await renderImport(); + await selectArchive(); + expect(router.state.location.pathname).toBe('/canvas/imported-space'); + expect(document.querySelector('[role="status"]')).toBeNull(); + }); +}); diff --git a/apps/web/src/hooks/useCanvasActions.ts b/apps/web/src/hooks/useCanvasActions.ts index 4e9748940..723ac7c84 100644 --- a/apps/web/src/hooks/useCanvasActions.ts +++ b/apps/web/src/hooks/useCanvasActions.ts @@ -2,10 +2,13 @@ // Licensed under the MIT license. import { useCallback, useRef, useState } from 'react'; +import { useTranslation } from 'react-i18next'; import { useNavigate } from 'react-router-dom'; import { useEffectiveInputMode } from './useInputMode'; +import { ApiError } from '../api/_client'; import { createCanvas, importCanvas } from '../api/canvas'; +import { toast } from '../components/Common/Toast'; /** * Shared "create / import canvas" actions. @@ -29,6 +32,7 @@ export interface UseCanvasActionsResult { export function useCanvasActions(): UseCanvasActionsResult { const navigate = useNavigate(); + const { t } = useTranslation(); const inputMode = useEffectiveInputMode(); const fileInputRef = useRef(null); const [isCreating, setIsCreating] = useState(false); @@ -70,12 +74,20 @@ export function useCanvasActions(): UseCanvasActionsResult { const result = await importCanvas(file); navigate(`/canvas/${result.canvasId}`); } catch (err) { - console.error('Failed to import canvas:', err); + toast( + err instanceof ApiError && + err.code === 'STORAGE_CAPABILITY_UNAVAILABLE' + ? t('canvasList.importUnavailable') + : err instanceof Error + ? err.message + : t('canvasList.importFailed'), + { tone: 'danger' }, + ); } finally { setIsImporting(false); } }, - [navigate], + [navigate, t], ); return { diff --git a/apps/web/src/i18n/resources/en/common.json b/apps/web/src/i18n/resources/en/common.json index 5da0d71c2..0a4e93ba0 100644 --- a/apps/web/src/i18n/resources/en/common.json +++ b/apps/web/src/i18n/resources/en/common.json @@ -356,6 +356,9 @@ "nodeCount_other": "{{count}} nodes", "updated": "Updated {{date}}", "exportStarted": "Export started", + "exportUnavailable": "Space export is not available with the current storage setup.", + "importUnavailable": "Space import is not available with the current storage setup.", + "importFailed": "Import failed", "exportFailed": "Export failed", "exporting": "Exporting…", "exportCanvas": "Export Space", diff --git a/apps/web/src/i18n/resources/zh-CN/common.json b/apps/web/src/i18n/resources/zh-CN/common.json index ebd51473c..38ba74bfb 100644 --- a/apps/web/src/i18n/resources/zh-CN/common.json +++ b/apps/web/src/i18n/resources/zh-CN/common.json @@ -356,6 +356,9 @@ "nodeCount_other": "{{count}} 个节点", "updated": "更新于 {{date}}", "exportStarted": "已开始导出", + "exportUnavailable": "当前存储配置不支持导出空间。", + "importUnavailable": "当前存储配置不支持导入空间。", + "importFailed": "导入失败", "exportFailed": "导出失败", "exporting": "正在导出…", "exportCanvas": "导出 Space", diff --git a/apps/web/src/pages/CanvasListPage.test.tsx b/apps/web/src/pages/CanvasListPage.test.tsx index b5c4cf763..af9707f90 100644 --- a/apps/web/src/pages/CanvasListPage.test.tsx +++ b/apps/web/src/pages/CanvasListPage.test.tsx @@ -7,6 +7,9 @@ import { createMemoryRouter, RouterProvider } from 'react-router-dom'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import CanvasListPage from './CanvasListPage'; +import { ApiError } from '../api/_client'; +import { exportCanvas } from '../api/canvas'; +import { toast } from '../components/Common/Toast'; import type { ReactNode } from 'react'; @@ -35,6 +38,8 @@ vi.mock('../api/canvas', () => ({ }), })); +vi.mock('../components/Common/Toast', () => ({ toast: vi.fn() })); + vi.mock('../components/Common/Modal', () => ({ Modal: () => null, })); @@ -129,6 +134,31 @@ async function renderPage() { } describe('CanvasListPage navigation', () => { + it('shows the export refusal without leaving the Spaces list or reporting success', async () => { + vi.mocked(exportCanvas).mockRejectedValueOnce( + new ApiError( + 400, + { + code: 'STORAGE_CAPABILITY_UNAVAILABLE', + message: 'Technical storage detail', + }, + 'Export failed', + ), + ); + const { router } = await renderPage(); + const button = container.querySelector( + 'button[aria-label="canvasList.exportCanvas"]', + ); + if (!button) throw new Error('Export control was not rendered'); + await act(async () => button.click()); + expect(router.state.location.pathname).toBe('/spaces'); + expect(toast).toHaveBeenCalledExactlyOnceWith( + 'canvasList.exportUnavailable', + { tone: 'danger' }, + ); + expect(button.disabled).toBe(false); + }); + it('renders each Space card as a link and uses in-tab routing for a plain click', async () => { const { link, router } = await renderPage(); diff --git a/apps/web/src/pages/CanvasListPage.tsx b/apps/web/src/pages/CanvasListPage.tsx index 86fa70978..46ea3bfe4 100644 --- a/apps/web/src/pages/CanvasListPage.tsx +++ b/apps/web/src/pages/CanvasListPage.tsx @@ -6,6 +6,7 @@ import { useCallback, useEffect, useRef, useState } from 'react'; import { Trans, useTranslation } from 'react-i18next'; import { Link } from 'react-router-dom'; +import { ApiError } from '../api/_client'; import { listCanvases, exportCanvas, deleteCanvasById } from '../api/canvas'; import { Button } from '../components/Common/Button'; import { EmptyState } from '../components/Common/EmptyState'; @@ -84,7 +85,12 @@ export default function CanvasListPage() { toast(t('canvasList.exportStarted'), { tone: 'success' }); } catch (error) { toast( - error instanceof Error ? error.message : t('canvasList.exportFailed'), + error instanceof ApiError && + error.code === 'STORAGE_CAPABILITY_UNAVAILABLE' + ? t('canvasList.exportUnavailable') + : error instanceof Error + ? error.message + : t('canvasList.exportFailed'), { tone: 'danger', }, diff --git a/docs/architecture/canvas-storage.md b/docs/architecture/canvas-storage.md index 2b5afc624..39b35ce23 100644 --- a/docs/architecture/canvas-storage.md +++ b/docs/architecture/canvas-storage.md @@ -1,6 +1,6 @@ # Canvas Storage Architecture -> Last updated: 2026-08-24 +> Last updated: 2026-09-04 ## 1. Overview @@ -59,8 +59,8 @@ Key points: - `space(canvasId)` is the one entry point to a Space. It is a composition-layer facade, not a port type: `StructuredStore` and `BlobStore` never import each other, and they are joined only where the cross-store rules already live — the blob-put precondition and the blob-first delete saga. It composes from its receiver, so substituting one axis on a `Storage` object yields Spaces built on the substitute. - A capability only one backend has hangs off that same handle, named for the backend and typed by its absence rather than stubbed to throw: `diskTree` is the Disk Space directory and is `null` on every other backend. It is not a port and does not live in `ports/`. `module-boundaries.test.ts` holds its exact production consumer census — a list that may shrink and must not grow — and asserts the barrel exposes nothing that reads as a portable path API. - A Space's bytes are reached the same way as its records: `BlobStore.space(id)` returns one member per user-visible area — `artifacts`, `guide`, `memory`, `uploads` — so the Disk paths a user sees are unchanged and retention can diverge later without moving bytes. The `guide` area is bounded by its member names rather than by a directory, because its area is the Space root: a directory scope there would let `list()` claim `space.json` and `deleteAll()` remove the Space. Rename and per-key delete remain unsupported. -- `space(canvasId).extension(namespace)` hands an owner an isolated place to keep its own per-Space state — a reserved directory on Disk — and nothing else. Storage validates the namespace, creates it on demand, and destroys it with the Space, which is the one operation an owner cannot perform itself; it guarantees nothing about the contents, and cannot, because it never sees them. `extension()` returns `null` for a Space that is gone, which is where the per-owner `existsSync` resurrection guards went. Memory-worker bookkeeping and the debug prompt log are its first two owners; ACP session state is assigned here but moves with the Agenetes `Namespace` change. -- Features that are _about_ a filesystem are declared, not emulated. `capabilities.ts` lists bundle export and import, reveal-in-file-manager, the built-in file tools, external-note discovery, and Windows directory-handle coordination as Disk-only; startup logs the ones the selected profile does not offer and each refusal reuses that same wording. An unavailable feature is a stated limitation and startup continues, while a profile naming an unimplemented backend stays a misconfiguration that fails fast. +- `space(canvasId).extension(namespace)` hands an owner an isolated place to keep its own per-Space state — a reserved directory on Disk, a shared connection plus a Space-owned parent row on SQLite — and nothing else. Storage validates the namespace, creates it on demand, and destroys it with the Space, which is the one operation an owner cannot perform itself; it guarantees nothing about the contents, and cannot, because it never sees them. `extension()` returns `null` for a Space that is gone, which is where the per-owner `existsSync` resurrection guards went. Memory-worker bookkeeping, the debug prompt log, and the Agenetes conversation stores are its owners; ACP session state is assigned here but moves with the Agenetes `Namespace` change. Because Agenetes's storage ports are synchronous and `extension()` is not, the composition root also exposes `sqliteTree` — the synchronous form of the same resolution, named for the backend that has it and `null` elsewhere, with its own single-consumer census beside `diskTree`'s. +- Features that are _about_ a filesystem are declared, not emulated. `capabilities.ts` lists Workspace folder selection, bundle export and import, reveal-in-file-manager, the built-in file tools, RFS's file plane, external-note discovery, the Workspace memory document, user-authored skills, and Windows directory-handle coordination as Disk-only; startup logs the ones the selected profile does not offer and each refusal reuses that same wording. An unavailable feature is a stated limitation and startup continues, while a profile naming an unimplemented backend stays a misconfiguration that fails fast. Fewer features is therefore not a reason to make a backend unselectable — an _undeclared_ gap is. - `closeStorage()` closes both connections on graceful Server shutdown and forgets the holder. On Disk it releases nothing a process exit would not, and it exists for the backend that will hold a pool. - `SpaceRepository.ensureWorld()` is the backend-neutral World bootstrap: it returns the established World or mints exactly one version-0 World when the namespace holds none. An _established_ World that is missing or malformed stays the integrity error `worldId()` reports, because regenerating identity there would orphan every reference to it. Disk delegates to the same idempotent primitive Workspace preparation calls, so one file keeps one writer. - An ordinary Space **directory name** is derived from its title via `toSafeFilename(title)`, not from `canvasId`. The stable `canvasId` only lives inside `space.json`; the World is the reserved `.world` exception. @@ -74,15 +74,66 @@ Key points: - Canonical World preview identity is server-owned: non-system commands cannot create, repoint, or delete managed previews. Users may move and resize them. Ordinary Spaces may create and delete their own `spacePreview` nodes through normal UI commands. - Legacy `canvasRef`, `frameRef`, `nodeRef`, `SET_PORTAL_NODE_PINS`, and `GET /api/canvas/:worldCanvasId/references` remain compatibility surfaces for stored World data but are no longer created or exposed by the redesigned World UI. The current model is specified in [space-preview.md](./space-preview.md). - Node filenames are `safe(label).md`; the node's stable id lives in the `id:` frontmatter field. -- The Disk `BlobStore` maps each Space scope to `.artifacts/`, with blobs named `` and no manifest file — the filename is the URL key. Artifacts are one of four blob areas a Space has, resolved as `space(canvasId).artifacts`: `put()` requires an existing Space record, while reads and `deleteAll()` remain available for recovery after a record goes missing. `CanvasStore` owns no artifact methods. Only the Disk blob and structured backends are implemented and selectable today. +- The Disk `BlobStore` maps each Space scope to `.artifacts/`, with blobs named `` and no manifest file — the filename is the URL key. Artifacts are one of four blob areas a Space has, resolved as `space(canvasId).artifacts`: `put()` requires an existing Space record, while reads and `deleteAll()` remain available for recovery after a record goes missing. `CanvasStore` owns no artifact methods. Blobs are always files: `disk` is the only implemented backend and `azure` the settled next one, so no structured backend is ever asked to hold bytes and any structured backend pairs with any blob backend. - Remote PDF preprocessing writes the already-fetched source bytes into the Space BlobStore as `artifact-.pdf` before structured persistence and replaces the node's remote `src` with that key. As with other artifact imports, this blob write precedes the node write operation; a later structured persistence failure may therefore leave an unreferenced blob until Space deletion, while a blob-write failure degrades to retaining the remote URL. - Events are append-only JSONL (`events.jsonl`); each line is `{ ts: number, payload: RecentAction }`. - The memory analyzer reads Space existence and at most 100 recent action events through one `SpaceHandle`. A missing Space skips the pass before reading memory files or calling the model; corrupt part data still fails the pass. Memory body/state files remain materialized workspace paths, while Agenetes-owned chat history is not part of the curator bundle. -- **Chat history is Chat-V2, owned by Agenetes L2 — not `CanvasStore`.** The canonical per-thread conversation is a two-tier append-only log under `chat_v2/`: Tier-1 `.events.jsonl` (`AgentStreamEvent` deltas a running turn appends, written by `FileEventLogStore`) and Tier-2 `.turns.jsonl` (folded `AgentTurn`s, written by `FileTurnStore` — the only tier `history()` reads back). These files sit under the canvas `.history/` only because it is the Agenetes namespace `storage.root` (`canvasAcpNamespace(canvasId)`); `CanvasStore` never touches them. Do **not** confuse `chat_v2/.events.jsonl` (agent stream events) with the sibling `events.jsonl` (canvas action log) — same suffix, unrelated content. +- **Chat history is Chat-V2, owned by Agenetes L2 — not `CanvasStore`.** Which store owns it depends on where the Space lives: a namespace carrying a `storage.root` (Disk) uses the file stores described below, a Space in SQLite uses the `agenetes_*` tables, and an unnamed namespace stays in memory as Agenetes intends. On Disk the canonical per-thread conversation is a two-tier append-only log under `chat_v2/`: Tier-1 `.events.jsonl` (`AgentStreamEvent` deltas a running turn appends, written by `FileEventLogStore`) and Tier-2 `.turns.jsonl` (folded `AgentTurn`s, written by `FileTurnStore` — the only tier `history()` reads back). These files sit under the canvas `.history/` only because it is the Agenetes namespace `storage.root` (`canvasAcpNamespace(canvasId)`); `CanvasStore` never touches them. Do **not** confuse `chat_v2/.events.jsonl` (agent stream events) with the sibling `events.jsonl` (canvas action log) — same suffix, unrelated content. - Durable Agenetes workload records live in `.history/threads.json` (`agenetes-v2` schema, one record per thread; written by `FileThreadStore`). The host-local `namespace.storage.root` is never persisted: reads bind each record to the current Space namespace, so a Home synchronized across computers cannot redirect storage back to another machine's absolute path. -- Canonical Task and Run records live in `.history/tasks.json`, owned by Huabu Server through the async `SpaceTasks` ledger (`read`, `create`, and `runs.create`/`runs.update`). The Disk adapter validates the versioned snapshot and referential integrity on every read, rejects duplicate identifiers and Runs whose Task is absent, serializes read-modify-write operations with an independent per-Canvas process-local mutex, and atomically replaces the file. This mutex is intentionally separate from the Canvas topology write coordinator, so Task metadata does not participate in `space.json` version CAS. +- Canonical Task and Run records live in `.history/tasks.json`, owned by Huabu Server through the async `SpaceTasks` ledger (`read`, `create`, and `runs.create`/`runs.update`/`runs.complete`). The Disk adapter validates the versioned snapshot and referential integrity on every read, rejects duplicate identifiers and Runs whose Task is absent, serializes read-modify-write operations with an independent per-Canvas process-local mutex, and atomically replaces the file. This mutex is intentionally separate from the Canvas topology write coordinator, so Task metadata does not participate in `space.json` version CAS. - Legacy chat files are one-way migrated into `chat_v2/` at workspace activation and retired to `.bak`: the oldest pi-ai `Context` `chat/.json` via `migrate-chat-threads.ts` (hop 1), then the M5.6 `chat/.turns.jsonl` / `.active.json` via `migrate-chat-turns.ts` (hop 2). If hop 1 finds both formats after an interrupted launch, it completes a strict converted prefix atomically or preserves an existing tail when the full conversion is its prefix. Divergent logs are retained rather than guessed or overwritten; hop 2 skips the paired turn log while a valid same-thread legacy Context remains or its JSON cannot be read safely, so a later activation can retry both copies without blocking unrelated migrations. The obsolete `CanvasStore` chat methods and `chatPath()` helper were removed in Phase 2; `chatDir()` remains because change-review and agent-owned files still use that directory. +## 2b. SQLite layout — records in a database, bytes in files + +`HUABU_STRUCTURED_BACKEND=sqlite` selects the second implemented structured backend. The blob axis stays `disk`, because bytes are always files. It needs **no Workspace folder and no Space directories**: every record is a row, and the only directories are the ones a Space's bytes sit in. + +``` +/storage/ + sqlite/ # the SQLite backend's area + huabu.sqlite # every record; override with HUABU_SQLITE_PATH + huabu.sqlite-wal # WAL sidecars, managed by SQLite + huabu.sqlite-shm + disk/ # the Disk backend's area + workspaces.json # Disk *structured* store: Workspace registry (unused here) + blobs/ # Disk *blob* store; override with HUABU_BLOB_ROOT + / + / + skill.md # blob area `guide` + .artifacts/ # blob area `artifacts` + .memory/space.md # blob area `memory` + .upload/ # blob area `uploads` +``` + +Each directory under `storage/` is named for the backend that owns it, and one file per backend decides its layout: `backends/disk/data-dir.ts` and `backends/sqlite/database.ts`. The composition root asks them and builds no path of its own (`module-boundaries.test.ts` enforces that). + +`storage/disk/` has two owners, so they get separate subtrees. `workspaces.json` is the Disk _structured_ store's Workspace registry — present only on a Disk-structured deployment, and never inside `blobs/`, because the blob store deletes whole directories and the registry is not its to delete. `blobs/` is the Disk _blob_ store's, reached only when the structured backend gives a Space no folder; `HUABU_BLOB_ROOT` moves that subtree alone. + +The byte root is Server-owned, not a Workspace folder: nothing in it is a Space record, and the Disk-only capabilities below stay unavailable because they need a real Space tree, not merely a directory. + +The layout beneath `/` is byte-for-byte the one the Disk profile uses inside a Space folder, because it is the same adapter. `blobs=disk` names a _medium_ — bytes are local files — and composition names the place, under one rule: **a Space's bytes live with the Space.** On a Disk-structured profile the Space _is_ a folder, so its bytes stay inside it, which is what keeps a Space folder self-contained for bundle export, reveal-in-file-manager, RFS, and the file tools. On a database-structured profile the Space has no folder to be inside, so the adapter gets the root above. One rule, two outcomes — the Space has two possible homes. + +| Table | Holds | +| ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `workspaces` | Workspace identity and display name; `forgotten_at` is how `remove()` forgets a member without destroying what it owns | +| `spaces` | One row per Space, scoped by `workspace_id`; `is_world` marks the hidden World, and a partial unique index keeps it to one per Workspace | +| `nodes` | Complete node JSON plus its opaque revision token and de-duplicated label key | +| `events` | The Canvas action log, ordered by an autoincrement id | +| `changes` | One coalesced change-review snapshot per `(Space, thread)` | +| `tasks` | The versioned Task/Run snapshot | +| `delta_log` | The executor's private journal, keyed by committed Space version | +| `space_extensions` | One row per extension namespace — the parent an owner's own tables cascade from | + +Owner-created tables hanging off `space_extensions`: `extension_documents` (memory bookkeeping and the debug prompt log) and `agenetes_threads` / `agenetes_events` / `agenetes_turns` (the conversation stores). Storage never reads them; deleting a Space removes them by cascade. + +Notes an operator needs: + +- **A Workspace is a row.** There is no folder to pick, so the Server creates and activates one on first start and reports `path: null` with `canChangeWorkspace: false`; the client shows no picker. Switching Workspaces re-scopes the one connection and reopens nothing. +- **The connection is shared.** The structured store and the Workspace repository use one `node:sqlite` connection, opened in WAL with `synchronous = NORMAL`, a bounded `busy_timeout`, and foreign keys enforced. One process, one connection: nothing here promises a multi-process fence. +- **Bytes are outside the database.** The blob axis is a file system on every profile, so a Space's uploads, artifacts, guide and memory body are ordinary files under the byte root and the database stays the size of its records. Deletion order is the composition layer's saga — sweep every blob area, then drop the record — and, where the record is a row, composition also removes the `//` directory it placed those areas under, because nothing else would. +- **What this profile does not serve** is declared in `capabilities.ts`, logged at startup, and refused in the same words at each call site: choosing/creating/revealing a Workspace folder, `.huabu.zip` export and import, reveal-in-file-manager, the built-in agent file tools, RFS's file plane, external-note discovery, the Workspace `setting/user.md` memory document, and user-authored skills under `setting/skills/`. A Space's _own_ memory body is unaffected — it is a blob. Bundled and Agent Team skills are unaffected. + + These are keyed on the **profile**, both axes. Most need a Space or Workspace to be a real directory (a structured-backend property); bundle export, bundle import, the file tools, and RFS additionally need the Space's _bytes_ to be in that directory, which is the blob backend's. Each row states a requirement with one clause per axis: every clause present must hold (`and` across axes), any listed backend satisfies its own clause (`or` within one), and an absent clause requires nothing — so a blob-agnostic row never needs editing when a blob backend is added. On the hybrid profile the answer is unchanged: a file system for the bytes hands nothing back, because every row still needs the record and node documents to be files. Refusals ask `storageServes(id)`; `module-boundaries.test.ts` checks that every declared row is refused somewhere. + ## 3. Storage composition and ownership `apps/server/src/modules/storage/` has three layers plus its composition root: @@ -103,7 +154,7 @@ Key points: | `index.ts` | Public exports only; application code imports here rather than reaching into an adapter. | | `canvas-store.ts`, `paths.ts`, `canvas-dirs.ts` | Deprecated forwarding shims with no logic, retained only for high-fanout compatibility imports. | -The Disk structured adapter and compatibility facade resolve the same cached legacy object, so migration does not create two in-memory authorities. All portable repository methods are async. `SpaceRepository` owns membership reads, structured create/rename, and an exclusive `beginDelete()` session; composition holds that session across the existing blob-first delete saga and then calls `finish()` or `abort()`. Every Space-record write goes through `SpaceHandle.write`, which is the version-checked replacement with the node and delta batch attached; `SpaceHandle.read` reads only. `SpaceNodes` returns complete records plus revision tokens without exposing filenames. `write` preserves the old node mutations → Space record → optional delta order. When a normal in-process node → record → delta batch rejects, the adapter must restore that batch's prestate before returning the rejection. An explicit title rename remains the preceding ordered, best-effort boundary and is not rolled back with the batch. The port does not promise process-crash or power-loss recovery, a determinate result after an unknown remote outcome, multi-process serialization, idempotent retry, or publication. Disk meets the in-process restoration requirement with its existing before-image rollback; a SQL adapter may use a native transaction. +The Disk structured adapter and compatibility facade resolve the same cached legacy object, so migration does not create two in-memory authorities. The SQLite adapter instead owns one explicit database filename and connection; retained handles stay bound to that connection, and its `init`, `health`, and `close` lifecycle is exercised only by direct tests. All portable repository methods are async. `SpaceRepository` owns membership reads, structured create/rename, and an exclusive `beginDelete()` session; composition holds that session across the existing blob-first delete saga and then calls `finish()` or `abort()`. Every Space-record write goes through `SpaceHandle.write`, which is the version-checked replacement with the node and delta batch attached; `SpaceHandle.read` reads only. `SpaceNodes` returns complete records plus revision tokens without exposing filenames. `write` preserves the old node mutations → Space record → optional delta order. When a normal in-process node → record → delta batch rejects, the adapter must restore that batch's prestate before returning the rejection. An explicit title rename remains the preceding ordered, best-effort boundary and is not rolled back with the batch. The port does not promise process-crash or power-loss recovery, a determinate result after an unknown remote outcome, multi-process serialization, idempotent retry, or publication. Disk meets the in-process restoration requirement with its existing before-image rollback; SQLite uses a native transaction. Canvas persistence DTOs and the write coordinator live under `modules/canvas/`; Workspace identity, durable membership, and Disk locators live under `modules/storage/`, while active-Workspace lifecycle and boot migrations remain under `modules/workspace/`; generic filesystem and Markdown codecs live under `utils/`. `canvasRoot()` validates the identifier and then verifies that the resolved Space directory remains a strict descendant of the active Workspace before any downstream Disk operation receives it. `module-boundaries.test.ts` enforces the storage dependency direction, prevents new consumers of the forwarding shims, and holds the neutrality guard: no production file outside `storage/` may import a Disk layout symbol or a legacy `CanvasStore` symbol. The check is import-level and symbol-level — a local variable that happens to be called `artifactPath` is not a violation, while importing `canvasRoot`, or reaching `getCanvasStore` through the barrel, is. Migrations are exempt because they rewrite frozen historical on-disk shapes; tests are exempt for the same reason they may name an adapter. @@ -133,7 +184,7 @@ The launch path deliberately has no compensation transaction. A launch failure l ### 3.3 Task Run completion -`RunCompletionService.complete()` validates the shared request and delegates the guarded transition to `SpaceTaskRuns.complete()`. The Disk adapter performs lookup, `running → completed`, and persistence under the existing per-Canvas Task mutation mutex, so HTTP and built-in-tool callers share one atomic transition rather than performing a read-then-update race. +`RunCompletionService.complete()` validates the shared request and delegates the guarded transition to `SpaceTaskRuns.complete()`. Both structured adapters perform lookup, `running → completed`, and persistence inside one Task-snapshot mutation boundary: Disk uses the per-Canvas Task mutex and atomic file replacement, while SQLite uses an immediate transaction. HTTP and built-in-tool callers therefore share one atomic transition rather than performing a read-then-update race. A completed Run stores immutable `completion.completedAt` and an optional trimmed caller-owned `completion.message`. The platform treats the message as untrusted text and does not interpret issue, pull-request, or URL semantics. A retry with the same normalized message is idempotent and preserves the original timestamp; a different message conflicts. A `pending` Run cannot complete, and Agent turn termination never implies Run completion. diff --git a/docs/proposals/multi-backend-storage.md b/docs/proposals/multi-backend-storage.md index c6d2371be..3c8288232 100644 --- a/docs/proposals/multi-backend-storage.md +++ b/docs/proposals/multi-backend-storage.md @@ -1,7 +1,7 @@ # Multi-Backend Storage -Status: Phases 1–4.5 and §§12.6–12.8 implemented -Last updated: 2026-08-24 +Status: Phases 1–5 implemented; SQLite is a selectable profile +Last updated: 2026-09-04 > **Scope and decision confidence.** This proposal records the two-port > `StructuredStore` / `BlobStore` split and their target backend families as @@ -48,8 +48,7 @@ Last updated: 2026-08-24 > review are recorded in place, including the CAS race ordering (§12.2.5), > log-family interface segregation (§12.2.6), and retained-handle Workspace > guards (§12.2.4). Remaining Disk-only read and physical capabilities still -> keep non-Disk profiles unselectable. No SQLite, Postgres, or Azure adapter -> exists. +> keep non-Disk profiles unselectable. > > Phase 4.5 moved storage-owned Disk layout behind the storage boundary in > PR #93. What remains between the portable contracts and a second structured @@ -59,6 +58,16 @@ Last updated: 2026-08-24 > **implemented**), and §12.8 (the dispositions and the product-level > harness, **implemented**). §12 is the authoritative plan; > the decision table in §2 marks what each step has actually settled. +> +> Phase 5 is specified in §12.9 and is **implemented by this branch**. +> `HUABU_STRUCTURED_BACKEND=sqlite` is a real profile: Workspaces, Spaces, +> nodes, logs, Tasks, and agent conversations are rows in one database file +> under `/storage/sqlite/`, and the deployment needs no Workspace +> folder and no Space directories. The blob axis stays `disk`, because bytes +> are always files — SQL records beside ordinary files is the profile, not a +> compromise within it. What it does **not** serve is enumerated in §12.9.4 +> and declared in `storage/capabilities.ts`, which is the list an operator +> sees at startup. Postgres and Azure adapters still do not exist. --- @@ -85,24 +94,24 @@ built above these ports, but its form is intentionally unresolved here. ## 2. Decision status -| Topic | Status | Current position | -| ------------------------------------------------------ | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Separate authoritative structured and blob ports | **Accepted** (P1, merged) | Storage is composed from `StructuredStore` and `BlobStore`; there is no single backend interface that mixes both concerns. | -| Structured backend family | **Settled direction** | Support Disk, SQLite, and Postgres implementations. Only Disk exists. | -| Blob backend family | **Settled direction** | Support Disk and Azure Blob implementations. Only Disk exists. | -| Independent composition | **Accepted** (P1, merged) | `StorageProfile` has two env-parsed axes; `validateStorageProfile` fails fast on unimplemented kinds and is the extension point for combination rules. The lazy `getStorage()` path now rejects profiles whose adapters require awaited initialization (§12.1.1). | -| Blob port contract | **Accepted** (P1, merged) | Connection → scope, stream-oriented, no permanent absolute path in the common contract; `materialize()` returns a bounded lease for the one consumer needing a file. Replacement atomicity and post-release lease semantics are contract terms, not adapter accidents (§6.2, §12.1.1). | -| Concrete interface shape and async migration | **Accepted** (P4) | Blob and portable structured repositories are async. `StructuredStore` exposes catalogue/lifecycle and scoped Space handles; the structured mutations enumerated in §12.4 use those ports. Disk-only physical capabilities remain explicit blockers for selecting another profile. | -| Exact structured repositories and aggregate boundaries | **Accepted minimum** (P4) | Catalogue, lifecycle, Space CAS, nodes, four Canvas-log families, Tasks, and the ordered writer have reusable contracts. A rejected in-process node → record → optional-delta batch restores prestate; explicit title rename remains an earlier best-effort boundary. Crash recovery, unknown remote outcomes, idempotency, publication, and multi-process serialization are not promised. | -| Node Markdown ownership | **Accepted** (P4) | Authored node content remains with structured node records because it participates in revision CAS, search, and node mutation. Opaque and large bytes remain in BlobStore. | -| Blob key, staging, deletion, and GC semantics | Proposed / open | Names are the existing `` keys; `deleteAll()` covers Space destruction. Staging, reference counting, and GC remain undesigned. Per-key deletion stays out of the public port, but the absence of any cleanup path is what makes atomic replace mandatory (§6.2). | -| Space-handle identity and caching | **Corrected** (P1) | `space(id)` returning a stable handle is bounded by the LRU behind it, not guaranteed. In-memory tombstones and the filename index are therefore adapter-local caches, never durable state (§12.1.1, §12.2.4). | -| Reaching one Space | **Accepted** (§12.6) | One `space(canvasId)` facade on the composition root joins both ports; the two ports keep their independence and are joined only where the cross-store rules already live. A capability only one backend has hangs off the same handle, named for that backend and typed by its absence — `diskTree`, `null` elsewhere (§6.4.1). | -| Residual per-Space files | **Accepted** (§12.8) | Four dispositions, not one: Disk-only and declared, portable and re-implemented, structured record, or blob (§6.4.2). Every current consumer is assigned in §6.4.3 and the ones that pay for themselves on Disk are built; what a second backend must pay for is named, not deferred silently. | -| Backend selection scope | **Accepted** | Backend selection and its connection/pool are process-global. Workspaces are namespaces inside the configured backend; activating another Workspace re-scopes repository/handle operations without dropping or reconnecting the backend. A SQL profile serves every Workspace through one live connection/pool. | -| Logical filesystem view | Open | A possible `SpaceFileView` above both stores; name and contract are not accepted yet. | -| Real agent workspace | Open | Materialized directory, OS mount, protocol-only access, or a combination remain under evaluation. | -| Agent-authored filesystem write-back | Open | Read-only projection, explicit checkout/commit, and live bidirectional sync are alternatives, not decisions. | +| Topic | Status | Current position | +| ------------------------------------------------------ | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Separate authoritative structured and blob ports | **Accepted** (P1, merged) | Storage is composed from `StructuredStore` and `BlobStore`; there is no single backend interface that mixes both concerns. | +| Structured backend family | **Settled direction** | Support Disk, SQLite, and Postgres implementations. Disk and SQLite are selectable; Postgres has no adapter. | +| Blob backend family | **Settled direction** | Support Disk and Azure Blob implementations — both file systems. Only Disk exists. A structured backend never holds bytes, so the two axes share nothing and any implemented pairing is a valid deployment. | +| Independent composition | **Accepted** (P1, merged) | `StorageProfile` has two env-parsed axes; `validateStorageProfile` fails fast on unimplemented kinds and is the extension point for combination rules. The lazy `getStorage()` path now rejects profiles whose adapters require awaited initialization (§12.1.1). | +| Blob port contract | **Accepted** (P1, merged) | Connection → scope, stream-oriented, no permanent absolute path in the common contract; `materialize()` returns a bounded lease for the one consumer needing a file. Replacement atomicity and post-release lease semantics are contract terms, not adapter accidents (§6.2, §12.1.1). | +| Concrete interface shape and async migration | **Accepted** (P4) | Blob and portable structured repositories are async. `StructuredStore` exposes catalogue/lifecycle and scoped Space handles; the structured mutations enumerated in §12.4 use those ports. Disk-only physical capabilities remain explicit blockers for selecting another profile. | +| Exact structured repositories and aggregate boundaries | **Accepted minimum** (P4) | Catalogue, lifecycle, Space CAS, nodes, four Canvas-log families, Tasks, and the ordered writer have reusable contracts. A rejected in-process node → record → optional-delta batch restores prestate; explicit title rename remains an earlier best-effort boundary. Crash recovery, unknown remote outcomes, idempotency, publication, and multi-process serialization are not promised. | +| Node Markdown ownership | **Accepted** (P4) | Authored node content remains with structured node records because it participates in revision CAS, search, and node mutation. Opaque and large bytes remain in BlobStore. | +| Blob key, staging, deletion, and GC semantics | Proposed / open | Names are the existing `` keys; `deleteAll()` covers Space destruction. Staging, reference counting, and GC remain undesigned. Per-key deletion stays out of the public port, but the absence of any cleanup path is what makes atomic replace mandatory (§6.2). | +| Space-handle identity and caching | **Corrected** (P1) | `space(id)` returning a stable handle is bounded by the LRU behind it, not guaranteed. In-memory tombstones and the filename index are therefore adapter-local caches, never durable state (§12.1.1, §12.2.4). | +| Reaching one Space | **Accepted** (§12.6) | One `space(canvasId)` facade on the composition root joins both ports; the two ports keep their independence and are joined only where the cross-store rules already live. A capability only one backend has hangs off the same handle, named for that backend and typed by its absence — `diskTree`, `null` elsewhere (§6.4.1). | +| Residual per-Space files | **Accepted** (§12.8) | Four dispositions, not one: Disk-only and declared, portable and re-implemented, structured record, or blob (§6.4.2). Every current consumer is assigned in §6.4.3 and the ones that pay for themselves on Disk are built; what a second backend must pay for is named, not deferred silently. | +| Backend selection scope | **Accepted** (§12.9) | Backend selection and its connection/pool are process-global. Workspaces are namespaces inside the configured backend; activating another Workspace re-scopes repository/handle operations without dropping or reconnecting the backend. Implemented for SQLite: one connection, a `workspace_id` on every Space, and a retained handle that refuses after a switch rather than answering for the new namespace. | +| Logical filesystem view | Open | A possible `SpaceFileView` above both stores; name and contract are not accepted yet. | +| Real agent workspace | Open | Materialized directory, OS mount, protocol-only access, or a combination remain under evaluation. | +| Agent-authored filesystem write-back | Open | Read-only projection, explicit checkout/commit, and live bidirectional sync are alternatives, not decisions. | ## 3. Current system @@ -144,8 +153,8 @@ external-note discovery watches `nodes/`, and export archives the entire Space directory. Therefore wrapping `CanvasStore` in a database adapter would not by itself make the application backend-neutral. -Canvas/Space persistence is currently Disk-only. SQLite, Postgres, and Azure -Blob adapters for this data do not yet exist. +Runtime Canvas/Space persistence is Disk by default and SQLite by selection +(§12.9). Postgres and Azure Blob adapters do not yet exist. ## 4. Goals @@ -166,7 +175,9 @@ Blob adapters for this data do not yet exist. ## 5. Non-goals -- Selecting an ORM, SQL query builder, Postgres driver, or SQLite driver. +- Selecting a production ORM, SQL query builder, Postgres driver, or final + SQLite driver. The isolated Phase 5 preview uses built-in `node:sqlite` + without making that production choice. - Defining the final relational schema or migration framework. - Choosing a VFS, FUSE, materialization, cache, or write-back design. - Replacing RFS or the canonical `SpaceQuery` / `CanvasCommand` contracts in @@ -175,9 +186,11 @@ Blob adapters for this data do not yet exist. their product semantics are defined. - Implementing online backend migration, replication, backup, or disaster recovery. -- Shipping any non-Disk adapter. The phases in §12 remove reasons why SQLite, - Postgres, and Azure _cannot_ be implemented; that is not the same as - implementing them. +- Making **every** feature portable. Phase 5 makes a non-Disk adapter + selectable, which is a different claim: a selectable profile may serve fewer + features, as long as each missing one is declared in `capabilities.ts` and + refused in those words where a user reaches for it (§12.9.4). Emulating a + filesystem so that a file-shaped feature _nearly_ works stays out of scope. ## 6. Settled backend split and implemented minimum contracts @@ -301,8 +314,10 @@ into place makes the failed write invisible instead of unremovable. ### 6.3 Composition -Configuration has two axes. The current shape carries only a backend kind per -axis, because no adapter yet needs more: +Configuration has two axes. The runtime-selectable profile carries only a +backend kind per axis; where an adapter needs a location, composition resolves +it (`HUABU_SQLITE_PATH`, `HUABU_BLOB_ROOT`) rather than the profile carrying +it: ```ts interface StorageProfile { @@ -320,14 +335,16 @@ connection/pool. Credential references, config storage, and deployment-level backend migration remain open — a Postgres DSN or Azure container reference will extend these members. -Some combinations require capability validation. For example, Postgres plus a -node-local DiskBlob implementation is unsafe in a multi-replica deployment -unless the path is a deliberately shared and supported filesystem. SQLite on a -network filesystem has different correctness and availability constraints from -local SQLite. `validateStorageProfile()` is where such rules live; today it -rejects kinds that are named but not implemented, so an unsupported profile -fails at startup with an actionable message rather than nondeterministically -while serving data. +The axes share nothing — records go to the structured backend, bytes go to a +file system — so every pairing of implemented backends is a valid deployment +today. Future cross-axis rules are still possible on _deployment_ grounds +rather than storage ones: Postgres plus a node-local DiskBlob root is unsafe +across replicas unless the path is a deliberately shared filesystem, and +SQLite on a network filesystem has different correctness and availability +constraints from local SQLite. `validateStorageProfile()` is where such rules +would live; today it only rejects recognized kinds that have no adapter, so an +unsupported profile fails at startup with an actionable message rather than +nondeterministically while serving data. ### 6.4 One Space handle, four dispositions — revised direction @@ -684,7 +701,7 @@ exceptions: one names what it returns, the other opens a session. ```ts interface StructuredStore { - readonly kind: StructuredBackendKind; // 'disk' — implemented adapters only + readonly kind: StructuredBackendKind; // 'disk' | 'sqlite'; only Disk is selectable init(): Promise; health(): Promise; @@ -746,7 +763,7 @@ interface SpaceChanges { interface SpaceTasks { read(): Promise; // Tasks and Runs in one snapshot create(task: TaskRecord): Promise; - readonly runs: SpaceTaskRuns; // create(run), update(runId, patch) + readonly runs: SpaceTaskRuns; // create, update, and atomic complete } ``` @@ -943,9 +960,10 @@ explicitly: ## 12. Migration plan -Phases 1–4 are implemented and specified below. Phase 5 onward keeps the -provisional character of the original outline: those entries record intended -order, not approved designs. +Phases 1–4.5 are implemented and merged. Phase 5 is implemented by this +isolated contract preview. Phase 6 onward keeps the provisional character of +the original outline: those entries record intended order, not approved +designs. The current on-disk format remains readable throughout port extraction. A database adapter must not require Disk consumers to simulate tables, and the @@ -1803,16 +1821,20 @@ justify. kind now names only kinds that exist. The wider vocabulary a profile may _request_ moved to `profile.ts` as `RequestedStructuredKind`, which is what preserves the actionable "not implemented yet" error for a configured - `sqlite` or `postgres`. `BlobBackendKind` still carries `azure` on the same - footing and was left alone as Phase-1 surface. + `sqlite` or `postgres`. Phase 5 narrowed `BlobBackendKind` the same way, to + `'disk'`, leaving `azure` in `RequestedBlobKind`. Not changed, deliberately: `authoritativeInsert` and the `write-suppressed` -put outcome remain in the portable shapes. Both exist for Disk's in-memory -deletion fence, and neither has a portable meaning a SQL adapter would -produce. They are now documented as adapter-shaped, the way `duplicate-node` -already was, rather than renamed or pushed behind the adapter — the honest -resolution needs a second adapter to say what the shared abstraction is, and -inventing one now would be the same speculative move this trim is undoing. +put outcome remain in the portable shapes. At this phase boundary, both +existed for Disk's in-memory deletion fence and a second adapter was still +needed to establish their shared meaning. + +**Resolved by Phase 5:** the SQLite contract preview supplies that second +adapter and confirms that these outcomes are adapter-shaped. Disk keeps its +process-local anti-resurrection fence and uses `authoritativeInsert` to lift +it. SQLite deletion is final at transaction commit, permits immediate reuse of +the primary key, and issues a fresh opaque revision so a token from the +deleted row cannot win a later compare-and-swap (§12.9.2). The review also asked composition to move default-title allocation ("Untitled", "Untitled (1)", …) into `create`, which would have removed the @@ -1961,10 +1983,11 @@ bearing: change-review records and Tasks are not history, whatever Disk's arrives, the group comes back — and `events` is where it was before, so nothing else has to move. -### 12.5 Phase 4.5 — storage-owned layout moves inside the boundary — **implemented** +### 12.5 Phase 4.5 — storage-owned layout moves inside the boundary — **merged** -Phase 5 adds a second structured backend. Before it does, the layout knowledge -that belongs to the _Disk_ backend has to stop living outside `storage/`. +Phase 5 would introduce a second structured backend. Before that work, the +layout knowledge that belongs to the _Disk_ backend had to stop living outside +`storage/`. Otherwise every later backend inherits a module named `disk` as the ambient description of where Spaces are, and each one pays to migrate the same callers again. @@ -2047,11 +2070,11 @@ substrate-specific but fails the test for the same reason — it exists so Windows can rename a Space _directory_ safely, and under SQLite there is no such rename. -`naming.ts` is misfiled in a different way: pure string logic with no I/O, -already re-exported rather than owned. It passes the test trivially (a second -backend needs the identical rules) but has no business behind a `disk` -segment. Phase 5 extracts it to `utils/naming.ts` as a side effect of needing -it twice; that extraction belongs here, where it is the point. +`naming.ts` was misfiled in a different way: pure string logic with no I/O, +already re-exported rather than owned. It passed the test trivially (a second +backend needs the identical rules) but had no business behind a `disk` +segment. Phase 4.5 extracted it to `utils/naming.ts`, where the shared rule has +a backend-neutral owner. Because the residue that survives the test is three setting helpers and `getWorkspacePath()` itself — none of it filesystem-specific — the target is a @@ -2128,8 +2151,9 @@ boundary test; behavior parity is asserted by the existing Disk suites, which must pass unchanged — a diff that alters a Disk test's expectations is out of scope by definition. -Phase 5 rebases onto this and drops its `utils/naming.ts` extraction, its -`workspace/disk/naming.ts` shim, and the corresponding roadmap edits. +Phase 5 builds on this merged result and carries none of its former +`utils/naming.ts` extraction, `workspace/disk/naming.ts` shim, or parallel +roadmap edits. **Landed for the Workspace-to-storage substrate move.** `modules/workspace/` is flat and holds `paths.ts` plus `migrations/`; the Disk record layout, blob @@ -2477,14 +2501,15 @@ temporary Workspace through the production lifecycle — prepared Workspace, opened connections, `ensureWorld()` — rather than swapping in a stub. A stub proves the application talks to an interface; only a real backend proves one serves the product, which is the half that decides whether a second adapter -works. `product-boundary.test.ts` runs the criterion against every profile in -`PRODUCT_STORAGE_PROFILES`, naming no directory, filename, or `space.json`; -Phase 5 adds one entry to that list and the same behaviours are covered for -SQLite. A guard reads the suite's own source and rejects a directory, a +works. `product-boundary.test.ts` runs the criterion against every selectable +profile in `PRODUCT_STORAGE_PROFILES`, naming no directory, filename, or +`space.json`. A guard reads the suite's own source and rejects a directory, a filename, or a `readFileSync` appearing in it, because the failure mode here is a helpful-looking assertion someone adds later. The records the suite reads back are built through the write engine, because a fixture that skips the -engine asserts nothing about what the product actually stores. +engine asserts nothing about what the product actually stores. The isolated +Phase 5 adapter runs the lower-level portable contracts; it does not enter +this product-profile harness until it becomes selectable. `closeStorage()` arrives with it, registered on graceful Server shutdown and used by the harness between profiles. On Disk it is close to a no-op — which @@ -2502,21 +2527,237 @@ bundle export, external-note claim), RFS's sidecar-to-record mapping (**B**, deferred until a second backend has a file plane at all), and the ACP session path that leaves with the Agenetes `Namespace` change. -Out of scope, unchanged: a SQLite adapter or schema, Disk→SQLite data -migration, SQLite profile registration, Postgres/Azure, the portable +Out of scope, unchanged: SQLite runtime composition and profile selection, +Disk→SQLite data migration, Postgres/Azure, the portable change-notification capability, RFS's backend-neutral path vocabulary, ACP session relocation, the rest of the Agenetes persistence migration, the portable export format, a writable general-purpose virtual filesystem or OS mount, protocol or UI changes, and stronger crash/distributed transaction guarantees. -### 12.9 Later phases — provisional +### 12.9 Phase 5 — SQLite as a selectable profile — **implemented** + +Phase 5 adds a second structured backend and turns it on. The question it +answers is not "does the boundary compile against a database" — §12.8's +harness already asked that — but the harder one behind it: can a deployment +run with **no Workspace folder and no Space directories at all**, and can it +say plainly what it gives up by doing so. + +`HUABU_STRUCTURED_BACKEND=sqlite` is the profile. Every record — Workspaces, +Spaces, nodes, events, changes, Tasks, extension namespaces, and agent +conversations — is a row in one file at +`/storage/sqlite/huabu.sqlite` (override with `HUABU_SQLITE_PATH`). + +The blob axis stays `disk`, and there is no SQLite blob adapter. **Bytes are +always a file system** — a local directory now, Azure Blob later — so no +structured backend is asked to hold them and the two axes genuinely share +nothing. A Space's bytes therefore need a directory even where its record does +not: the Disk blob adapter writes them to +`/storage/disk/blobs///` (override the base +with `HUABU_BLOB_ROOT`), in the same area layout it writes inside a Space +folder. + +`blobs=disk` names a medium, not a directory. Where the bytes go is +composition's, under one rule — **a Space's bytes live with the Space** — and +the Space has two possible homes. On Disk records it is a folder, so the bytes +stay inside it; that is not merely backward compatibility, it is what +`space-bundle-export`, `space-bundle-import`, `reveal-space-folder` and +`builtin-file-tools` are made of, since each of them is the Space folder being +complete. On database records there is no folder to be inside. The corollary +is a cross-axis constraint that does not bite yet: a blob backend that cannot +co-locate — an object store — would put bytes outside the Space folder even on +Disk records, and those four capabilities would then depend on both axes +rather than the structured one alone. `capabilities.ts` records that where the +matrix is defined. That directory is Server-owned and holds nothing but bytes; it is not +a Workspace folder and it is not a Space tree, so none of §12.9.4's Disk-only +capabilities become available because it exists. Postgres and Azure Blob +adapters still do not exist. + +Each directory under `/storage/` is named for the backend that owns +it, and each backend decides its own layout in one file — +`backends/disk/data-dir.ts` and `backends/sqlite/database.ts`. The composition +root asks and builds no path of its own, which `module-boundaries.test.ts` +enforces. `storage/disk/` has two owners and therefore two subtrees: the +structured store's `workspaces.json` Workspace registry, and the blob store's +`blobs/`. Keeping the registry outside `blobs/` is not tidiness — the blob +store deletes whole directories, and the registry is not its to delete. + +#### 12.9.1 Scope and lifecycle + +- Built-in `node:sqlite`. No package, no native addon. That is not a + production driver decision (§5); it is what let this phase be about the + boundary rather than about dependencies. +- One connection per process, shared by the structured store and the Workspace + repository — because they are one file, and two writers to one SQLite file + is a lock error rather than a queue. The connection opens in WAL with + `synchronous = NORMAL`, a bounded `busy_timeout`, and foreign keys + enforced. +- Opening it is _synchronous_, so the composition root can hand out a + Workspace repository before `initStorage()` has been awaited — which managed + mode needs, and which is the reason the profile does not have to weaken the + `requiresExplicitInit` guard for anyone else. +- A Workspace is a **row**. There is no folder to pick, so the first start + creates one and activates it; `activateWorkspace` re-points the connection's + namespace and reopens nothing. Handles bind the Workspace they were resolved + in and refuse afterwards, exactly as the Disk adapters refuse a retained + workspace path. + +#### 12.9.2 Schema and behavior + +Schema versioning uses `PRAGMA user_version`; migrations run transactionally, +reject databases from the future, and create `STRICT` tables with foreign keys +enabled. Version 1 holds Workspaces, Space records and World membership, +complete node JSON with opaque revision tokens, ordered events, coalesced +changes, Task/Run snapshots, extension namespaces, and the private delta +journal. No table holds bytes. + +The two ports are configured independently and their lifecycles are joined +only by the deletion saga in `storage.ts`, which sweeps every blob area +_before_ the structured record goes and can therefore also sweep orphans for a +record that is already missing. Where the record is a row, that saga then +removes the `//` directory composition placed those +areas under, because no structured delete ever will. + +Every ordered Space write applies node mutations, record replacement, and the +optional delta insert in one immediate transaction. Same-baseline writers have +one winner. Space deletion uses the shared process-local admission coordinator +under a database-specific scope: reads remain available, mutations reject, +concurrent deletion sessions queue, and `finish()` removes all owned rows by +foreign-key cascade. No SQL transaction remains open across blob cleanup, and +no multi-process deletion fence is promised. + +SQLite does not emulate Disk's node tombstones. A committed delete immediately +frees the `(canvas_id, node_id)` key. Every successful put receives a new UUID +revision token, including a delete/recreate cycle, so a stale token from the +old row cannot match the replacement. `write-suppressed` and +`authoritativeInsert` remain valid adapter-specific parts of the common shape +for Disk rather than requirements every SQL adapter must reproduce. + +Value encoding follows `JSON.stringify`, because Disk persists through that +same function. An `undefined` own property is dropped rather than rejected; +what is genuinely unrepresentable — a cycle, a non-finite number, a non-plain +object — still rejects. The alternative was a record Disk accepts and SQLite +refuses, which is §13's silent-divergence risk in its most ordinary form: an +optional field spread onto a node. + +`remove()` on the Workspace repository is a **forget**, not a delete. The +port's wording is "forget one member without deleting any Workspace-owned +data", which Disk honours for free because the folder outlives the registry +entry. A database has no second copy, so forgetting is a timestamp and the +rows stay. + +#### 12.9.3 The extension substrate, and one synchronous exception + +The substrate is §6.4.4 as written: the port hands a namespace a connection +plus a Space-owned parent row, owner tables reference it with +`ON DELETE CASCADE`, and storage never sees what is in them. Three owners use +it — memory bookkeeping, the debug prompt log, and the Agenetes conversation +stores. + +The conversation stores forced one addition. Agenetes's three storage ports +(thread table, Tier-1 event log, Tier-2 folded turns) are **synchronous**, and +the port's `extension()` is async. Rather than have that owner keep a cache +warmed from an unrelated code path, the composition root exposes +`Space.sqliteTree` — a synchronous resolver, named for the backend that has +it and `null` everywhere else, exactly like `Space.diskTree`. It has its own +census in `module-boundaries.test.ts` with exactly one production consumer, +because the only justification for it is that one owner's synchronous +interface; a second consumer would mean the reason had drifted. + +The payoff is the thing a user would notice: an agent conversation in a Space +that has no directory survives a restart, and is removed with its Space by the +same cascade as everything else. + +#### 12.9.4 What this profile does not serve + +These capabilities are Disk-only, declared in `storage/capabilities.ts`, +logged at startup, and refused at their own call sites in the same words. + +They are keyed on the **profile**, not on one axis. Most need a Space or a +Workspace to be a real directory, which is a structured-backend property; four +need more than that — they need the Space's _bytes_ to be in that directory +too, and that is the blob backend's. A structured-only matrix would call a +bundle exportable on a profile that archives a Space folder its artifacts had +never been written to. + +Each row states a requirement with one clause per axis. **Every clause present +must hold** (the axes are an `and`) and **within a clause the backends are an +`or`**, so `{ structured: ['disk'], blobs: ['disk'] }` reads "Disk records +_and_ Disk bytes". **An absent clause is not a requirement**, so any backend on +that axis passes — which is the point rather than a shortcut: a feature that +does not touch a Space's bytes must not need editing when a blob backend is +added, and the ones that do are exactly the ones that should be forced to +decide then. + +On the hybrid profile the answer is unchanged — a real file system for the +bytes hands nothing back, because every row still needs the record and node +documents to be files. `detached-blobs.test.ts` pins the fact underneath that: +bytes are files and `Space.diskTree` is still `null`. + +| Capability | What is lost | Why it is not emulated | +| --------------------------------------------- | ----------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `workspace-directory` | Choosing, creating, or revealing a Workspace folder | A Workspace is a row. The Server opens its own on first start; the client is told `canChangeWorkspace: false` and shows no picker. | +| `space-bundle-export` / `space-bundle-import` | `.huabu.zip` round-trip | The bundle _is_ the Space directory, archived. A portable export built from records plus reachable blob references is a separate design. | +| `reveal-space-folder` | Open a Space's `nodes/` folder in the OS file manager | It exists so a user can settle a duplicate-markdown collision by hand. A node is a row here: no folder of documents to open, and no such collision to settle. | +| `builtin-file-tools` | The agent's `read`/`write`/`glob`/`grep` tools | The documents they edit are the node sidecars under `nodes/`, which are rows. The first-party agent uses the Canvas tools instead. | +| `space-file-plane` | RFS, the HTTP file plane external agents mount | Listed apart from the tools above because it is what they were previously said to fall back to. A Space with no file plane has neither. | +| `external-note-discovery` | Adopting Markdown dropped into a Space from outside | It watches for documents that arrived without going through the application. A database has no such arrival path. | +| `workspace-user-memory` | `setting/user.md`, the cross-Space memory document | A file the user edits at the root of a Workspace they chose. The blob port has no Workspace-level scope, so it has none to live in. A Space's _own_ memory body is a blob and is unaffected. | +| `workspace-user-skills` | `setting/skills//SKILL.md` | Same arrival path as external notes. Bundled and Agent Team skills are unaffected. | + +Two further limits are not capability rows, because nothing refuses them and +a row nothing can refuse is a fact about a backend rather than a capability: + +- **Windows directory-handle coordination.** It exists so renaming a Space + directory can succeed against a live `fs.watch` handle. A Space that is a + row is never filed under its title and nothing watches it, so no handle is + ever registered and nothing asks. It was listed as a capability and removed: + reading "unavailable" told an operator they had lost something, when the + profile simply does not have the problem. + +- **Multi-process access.** One process, one connection. WAL and + `busy_timeout` make a second reader survivable, and nothing here promises a + multi-process deletion fence or a distributed transaction. + +Two rules keep the call sites honest, and `module-boundaries.test.ts` checks +the first: a **refusal asks the matrix** through `storageServes(id)` on the +composition root, never a re-derivation such as "is there a `diskTree`", +because a re-derivation is a second copy of the rule that never learns when a +row grows a second axis. A **degradation does not** — code that renders +absence instead of refusing asks the concrete predicate, because it is not +making the profile's promise. + +#### 12.9.5 Proof + +The reusable contracts — structured store, Space repository, nodes, ordered +write, logs, Tasks, extension substrate, and **Workspace repository** — run +against Disk and against real temporary SQLite files. The blob contract runs +once, against the one blob adapter there is. + +`PRODUCT_STORAGE_PROFILES` gains `sqlite/disk`, so the §12.8 product-boundary +suite runs unchanged against it: World bootstrap, Space creation, ordered +writes through every node read shape, version conflict, bytes in every area, +the cross-store put guard, extension isolation and cleanup, the log families, +the Task ledger, deletion, World protection, and — added here — that all of it +is still there after a restart. That suite names no directory and no filename; +`module-boundaries.test.ts` enforces that mechanically. What _is_ about +placement has its own small suite instead (`detached-blobs.test.ts`): bytes +land as real files under the Workspace-scoped root, one Workspace's root is +not another's, deleting a Space leaves no directory behind, and the Workspace +registry sits outside the blob root so no byte sweep can reach it. + +SQLite integration tests additionally cover strict schema creation, WAL and +foreign-key pragmas read back on a second connection, close/reopen +persistence, an immutable v1 fixture, future-version rejection, migration +rollback, SQL fault injection, foreign-key cascades, revision safety across +delete/recreate, `JSON.stringify` encoding parity, incremental streaming and +early abort, batched `readMany`, Workspace scoping and handle invalidation +across a switch, and forget-without-delete. The Agenetes conversation stores +have their own suite against a mounted profile, covering round-trip, +isolation, restart, and destruction with the Space. + +### 12.10 Later phases — provisional -5. Add one new adapter at a time — SQLite, then Postgres, then Azure Blob — - running the same contract suites, migration fixtures, failure injection, - and concurrency tests against each. An adapter may exist for isolated - testing before its backend profile is selectable; profile validation keeps - rejecting it until the required capability matrix is satisfied. 6. Migrate the currently synchronous Agenetes persistence ports without changing their persist-before-notify, sequence, and fencing semantics. 7. Refactor RFS and built-in file tools only after a logical file-view contract @@ -2730,18 +2971,19 @@ Before a new backend is production-ready: persistence ownership, namespace, sequence, and replay invariants. - [Agenetes-Agentlet Gateway Consolidation](./agenetes-agentlet-gateway-consolidation.md) — records removal of the old Agentlet SQLite session store; it must not be - confused with the proposed SQLite structured backend. + confused with the SQLite structured contract-preview backend. ## 17. Code entry points | File/dir | Responsibility | | -------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [`apps/server/src/modules/storage/`](../../apps/server/src/modules/storage/) | Ports, composition, adapters, compatibility, tests, and three forwarding shims — the canonical Phase-1–4 tree (§§12.1–12.4), guarded by `module-boundaries.test.ts`. | +| [`apps/server/src/modules/storage/`](../../apps/server/src/modules/storage/) | Ports, composition, adapters, compatibility, tests, and three forwarding shims — the canonical Phase-1–5 tree (§§12.1–12.9), guarded by `module-boundaries.test.ts`. | | [`apps/server/src/modules/storage/ports/`](../../apps/server/src/modules/storage/ports/) | The two ports; reusable suites live in `ports/contracts/`. `blob.ts` is normative (§7.1); `structured.ts` owns the Space collection and the per-Space handle: record read/write, nodes, changes, Tasks, and history. | | [`apps/server/src/modules/storage/storage.ts`](../../apps/server/src/modules/storage/storage.ts) | Composition root: maps profiles to adapters, guards blob puts, and holds a lifecycle deletion session across the blob-first cleanup saga. | | [`.../storage/backends/disk/legacy/canvas-store-cache.ts`](../../apps/server/src/modules/storage/backends/disk/legacy/canvas-store-cache.ts) | Bounded LRU of legacy Disk Space objects. The single owner both the adapter and the facade resolve through, and the real limit of `space(id)` identity (§12.2.4). | | [`apps/server/src/modules/storage/profile.ts`](../../apps/server/src/modules/storage/profile.ts) | Two-axis backend selection from env, and the fail-fast validation hook for unsupported combinations. | | [`apps/server/src/modules/storage/backends/disk/`](../../apps/server/src/modules/storage/backends/disk/) | Every Disk implementation: blob/structured stores, the Space collection, and the per-Space record, node, log, and Task adapters, in-process batch restoration, and the legacy class under `legacy/`. | +| [`apps/server/src/modules/storage/backends/sqlite/`](../../apps/server/src/modules/storage/backends/sqlite/) | Selectable `node:sqlite` structured adapter: strict schema and migrations, Workspace-scoped Spaces, transaction-backed writes, and real-file contract/integration tests. Records only — bytes stay on the blob axis. | | [`.../storage/compatibility/canvas.ts`](../../apps/server/src/modules/storage/compatibility/canvas.ts) | Residual Disk read surface plus direct-module lifecycle test fixtures; production structured mutations enumerated in §12.4 use the portable ports. | | [`apps/server/src/modules/agent/memory/analyzer.ts`](../../apps/server/src/modules/agent/memory/analyzer.ts) | P3 repository consumer for strict Space existence, bounded action events, and intent episodes; physical chat and memory files remain Disk-specific. | | [`apps/server/src/modules/canvas/write-coordinator.ts`](../../apps/server/src/modules/canvas/write-coordinator.ts) | Canvas mutation coordinator and per-Space write lock, held across asynchronous node read, revision CAS, and put. | diff --git a/packages/shared/src/types/api/canvas.ts b/packages/shared/src/types/api/canvas.ts index 095663137..93fff48c7 100644 --- a/packages/shared/src/types/api/canvas.ts +++ b/packages/shared/src/types/api/canvas.ts @@ -327,6 +327,8 @@ export interface UpdateCanvasStateResult { * omitted, mirroring the pre-schema behaviour. */ export const exportCanvasQuerySchema = z.object({ + /** Validate export eligibility without building or downloading the archive. */ + check: z.enum(['true', 'false']).optional(), includeHistory: z.enum(['true', 'false']).optional(), }); export type ExportCanvasQuery = z.infer; diff --git a/packages/shared/src/types/api/workspace.ts b/packages/shared/src/types/api/workspace.ts index efeb439a4..9ffc9dc5e 100644 --- a/packages/shared/src/types/api/workspace.ts +++ b/packages/shared/src/types/api/workspace.ts @@ -55,8 +55,16 @@ export const workspaceDescriptorSchema = z.object({ export type WorkspaceDescriptor = z.infer; /** Body for `POST /api/workspaces`. */ +/** + * Body for `POST /api/workspaces`. + * + * `path` is the folder to adopt, and is how a Workspace is created where a + * Workspace *is* a folder. Where it is a row in a database there is nothing to + * adopt, so `name` alone creates one. Which form is required is the Server's + * answer, because the configured backend is the only thing that knows. + */ export const workspaceCreateSchema = z.object({ - path: z.string().min(1, 'Workspace path is required'), + path: z.string().min(1, 'Workspace path is required').optional(), name: z.string().trim().min(1, 'Workspace name is required').optional(), }); export type WorkspaceCreateRequest = z.infer;