From f77a80a2df611cdaa13123df04b73f0e23f94077 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Thu, 10 Sep 2026 19:21:15 -0600 Subject: [PATCH 01/14] fix(publication): distinguish native read absence from injected errors Preserve protected retirement evidence through explicit initial-path observation hooks and retain exact v2 client fixtures for the remaining migration work. Read-evidence prerequisite for fixes #53; publication remains gated on the complete journal replacement. --- .../protected-publication-v2/provenance.json | 16 + .../pylon-bounded-file.mjs | 375 ++ .../pylon-consumer-lock.mjs | 3751 +++++++++++++++++ scripts/lib/pylon-bounded-file.mjs | 14 +- scripts/lib/pylon-consumer-lock.mjs | 13 +- scripts/pylon-bounded-file.test.mjs | 96 + scripts/pylon-publication.test.mjs | 46 +- 7 files changed, 4276 insertions(+), 35 deletions(-) create mode 100644 scripts/fixtures/protected-publication-v2/provenance.json create mode 100644 scripts/fixtures/protected-publication-v2/pylon-bounded-file.mjs create mode 100644 scripts/fixtures/protected-publication-v2/pylon-consumer-lock.mjs create mode 100644 scripts/pylon-bounded-file.test.mjs diff --git a/scripts/fixtures/protected-publication-v2/provenance.json b/scripts/fixtures/protected-publication-v2/provenance.json new file mode 100644 index 0000000000..b1f83f9250 --- /dev/null +++ b/scripts/fixtures/protected-publication-v2/provenance.json @@ -0,0 +1,16 @@ +{ + "repository": "pylon-code/prime-agent", + "commit": "68603ed89bb597cd715fd6a77bc1c39d7e110298", + "files": [ + { + "path": "pylon-consumer-lock.mjs", + "sourcePath": "scripts/lib/pylon-consumer-lock.mjs", + "sha256": "b9baea4d051d7276a203f0c8b519ce132c51ca55d450fa5b69aac99a16220d49" + }, + { + "path": "pylon-bounded-file.mjs", + "sourcePath": "scripts/lib/pylon-bounded-file.mjs", + "sha256": "7098a628df2cf840cc5c4c42757f2c5f10d30378e16f81ed46f39c19d6b0d81e" + } + ] +} diff --git a/scripts/fixtures/protected-publication-v2/pylon-bounded-file.mjs b/scripts/fixtures/protected-publication-v2/pylon-bounded-file.mjs new file mode 100644 index 0000000000..f43bc3292d --- /dev/null +++ b/scripts/fixtures/protected-publication-v2/pylon-bounded-file.mjs @@ -0,0 +1,375 @@ +import { createHash } from "node:crypto"; +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readSync, +} from "node:fs"; +import { lstat, open } from "node:fs/promises"; +export const PYLON_PUBLICATION_MANIFEST_MAX_BYTES = 64 * 1024; +export const PYLON_STABLE_HISTORY_MAX_MANIFESTS = 4096; +export const PYLON_STABLE_HISTORY_MAX_BYTES = 32 * 1024 * 1024; + +function statEvidence(stat) { + return Object.freeze({ + dev: stat.dev, + ino: stat.ino, + size: stat.size, + mtimeMs: stat.mtimeMs, + ctimeMs: stat.ctimeMs, + nlink: stat.nlink, + }); +} + +export class BoundedFileUnlinkedDuringReadError extends Error { + constructor(path, description, bytes, expectedSha256, pathEntry, before, after, confirmedHandle = null) { + super(`${description} changed while it was read because the same opened inode was removed.`); + this.name = "BoundedFileUnlinkedDuringReadError"; + this.path = path; + this.description = description; + this.bytes = Buffer.from(bytes); + this.expectedSha256 = expectedSha256; + this.sha256 = createHash("sha256").update(this.bytes).digest("hex"); + this.statTransition = Object.freeze({ + pathEntry: statEvidence(pathEntry), + before: statEvidence(before), + after: statEvidence(after), + confirmedHandle: confirmedHandle === null ? null : statEvidence(confirmedHandle), + }); + } +} + +export class BoundedFileLinkRetiredBeforeReadError extends Error { + constructor(path, description, bytes, expectedSha256, pathEntry, openedHandle) { + super(`${description} changed while it was read because one publication hardlink was retired before the file was opened.`); + this.name = "BoundedFileLinkRetiredBeforeReadError"; + this.path = path; + this.description = description; + this.bytes = Buffer.from(bytes); + this.expectedSha256 = expectedSha256; + this.sha256 = createHash("sha256").update(this.bytes).digest("hex"); + this.statTransition = Object.freeze({ + pathEntry: statEvidence(pathEntry), + openedHandle: statEvidence(openedHandle), + }); + } +} + +export class BoundedFileLinkRetiredDuringReadError extends Error { + constructor(path, description, bytes, expectedSha256, pathEntry, before, after, finalPathEntry) { + super(`${description} changed while it was read because one publication hardlink was retired during the bounded read.`); + this.name = "BoundedFileLinkRetiredDuringReadError"; + this.path = path; + this.description = description; + this.bytes = Buffer.from(bytes); + this.expectedSha256 = expectedSha256; + this.sha256 = createHash("sha256").update(this.bytes).digest("hex"); + this.statTransition = Object.freeze({ + pathEntry: statEvidence(pathEntry), + before: statEvidence(before), + after: statEvidence(after), + finalPathEntry: statEvidence(finalPathEntry), + }); + } +} + +function sameInodeReadBounds(left, right) { + return left.dev === right.dev && left.ino === right.ino && left.size === right.size && + left.mtimeMs === right.mtimeMs; +} + +function sameStat(left, right) { + return sameInodeReadBounds(left, right) && left.ctimeMs === right.ctimeMs && left.nlink === right.nlink; +} + +function exactMonotoneStatCut(observations, fromLinks, toLinks) { + if ( + observations.length < 2 || observations[0].nlink !== fromLinks || + observations.at(-1).nlink !== toLinks || + observations.some((stat) => !sameInodeReadBounds(observations[0], stat)) + ) return null; + let cut = null; + for (let index = 1; index < observations.length; index += 1) { + const previous = observations[index - 1]; + const current = observations[index]; + if (previous.nlink === current.nlink) { + if (previous.ctimeMs !== current.ctimeMs) return null; + continue; + } + if ( + cut !== null || previous.nlink !== fromLinks || current.nlink !== toLinks || + previous.ctimeMs === current.ctimeMs + ) return null; + cut = index; + } + return cut; +} + +function permitsInitialStatTransition(pathEntry, before, expectedSha256) { + return sameStat(pathEntry, before) || ( + expectedSha256 !== null && ( + exactMonotoneStatCut([pathEntry, before], 2, 1) === 1 || + exactMonotoneStatCut([pathEntry, before], 1, 0) === 1 + ) + ); +} + +function isRegularPathEntry(pathEntry) { + return !pathEntry.isSymbolicLink?.() && pathEntry.isFile(); +} + +function exactSha256(bytes, expectedSha256) { + return expectedSha256 !== null && createHash("sha256").update(bytes).digest("hex") === expectedSha256; +} + +export async function readBoundedRegularFile( + path, + { + maxBytes, + minBytes = 1, + description = "Input", + openFile = open, + lstatEntry = lstat, + validateHandle, + hooks, + expectedSha256 = null, + } = {}, +) { + if ( + !Number.isSafeInteger(maxBytes) || maxBytes < 1 || !Number.isSafeInteger(minBytes) || minBytes < 0 || minBytes > maxBytes || + !(expectedSha256 === null || /^[0-9a-f]{64}$/.test(expectedSha256)) + ) { + throw new Error("Bounded file limits are invalid."); + } + let pathEntry; + try { + pathEntry = await lstatEntry(path); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } + if (pathEntry.isSymbolicLink?.() || !pathEntry.isFile()) { + throw new Error(`${description} is not one regular non-symlink file.`); + } + let handle; + try { + handle = await openFile(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + } catch (error) { + if (error?.code === "ENOENT") return null; + if (["ELOOP", "EISDIR"].includes(error?.code)) { + throw new Error(`${description} is not one regular non-symlink file.`); + } + throw error; + } + try { + let before = await handle.stat(); + if (!before.isFile()) throw new Error(`${description} is not one regular non-symlink file.`); + if (validateHandle) before = await validateHandle(handle, before, description); + if (!permitsInitialStatTransition(pathEntry, before, expectedSha256)) { + throw new Error(`${description} changed while it was read.`); + } + if (before.size < minBytes || before.size > maxBytes) throw new Error(`${description} exceeds its format byte limit or is malformed.`); + await hooks?.afterInitialStat?.({ path, handle, stat: before }); + const bytes = Buffer.alloc(before.size); + let offset = 0; + while (offset < bytes.length) { + const { bytesRead } = await handle.read(bytes, offset, bytes.length - offset, offset); + if (bytesRead === 0) throw new Error(`${description} changed while it was read.`); + offset += bytesRead; + } + const extra = Buffer.alloc(1); + const { bytesRead: extraBytes } = await handle.read(extra, 0, 1, bytes.length); + await hooks?.beforeFinalStat?.({ path, handle, bytes }); + const after = await handle.stat(); + await hooks?.afterFinalStat?.({ path, handle, stat: after, bytes }); + let finalPathEntry; + let finalPathMissing = false; + try { + finalPathEntry = await lstatEntry(path); + } catch (error) { + if (error?.code !== "ENOENT") throw error; + finalPathMissing = true; + } + let confirmedHandle = null; + if (finalPathMissing && after.nlink === 1 && sameStat(pathEntry, before) && sameStat(before, after)) { + confirmedHandle = await handle.stat(); + } + if (extraBytes === 0 && exactSha256(bytes, expectedSha256)) { + if (!finalPathMissing && isRegularPathEntry(finalPathEntry)) { + const retirementCut = exactMonotoneStatCut([pathEntry, before, after, finalPathEntry], 2, 1); + if (retirementCut === 1) { + throw new BoundedFileLinkRetiredBeforeReadError( + path, + description, + bytes, + expectedSha256, + pathEntry, + before, + ); + } + if (retirementCut !== null) { + throw new BoundedFileLinkRetiredDuringReadError( + path, + description, + bytes, + expectedSha256, + pathEntry, + before, + after, + finalPathEntry, + ); + } + } + if (finalPathMissing) { + const unlinkStats = [pathEntry, before, after]; + if (confirmedHandle !== null) unlinkStats.push(confirmedHandle); + if (exactMonotoneStatCut(unlinkStats, 1, 0) !== null) { + throw new BoundedFileUnlinkedDuringReadError( + path, + description, + bytes, + expectedSha256, + pathEntry, + before, + after, + confirmedHandle, + ); + } + } + } + if ( + extraBytes !== 0 || finalPathMissing || !isRegularPathEntry(finalPathEntry) || + !sameStat(pathEntry, before) || !sameStat(before, after) || !sameStat(after, finalPathEntry) + ) throw new Error(`${description} changed while it was read.`); + return bytes; + } finally { + await handle.close(); + } +} + + +export function readBoundedRegularFileSync( + path, + { + maxBytes, + minBytes = 1, + description = "Input", + openFile = openSync, + lstatEntry = lstatSync, + statFile = fstatSync, + readFile = readSync, + closeFile = closeSync, + hooks, + expectedSha256 = null, + } = {}, +) { + if ( + !Number.isSafeInteger(maxBytes) || maxBytes < 1 || !Number.isSafeInteger(minBytes) || minBytes < 0 || minBytes > maxBytes || + !(expectedSha256 === null || /^[0-9a-f]{64}$/.test(expectedSha256)) + ) { + throw new Error("Bounded file limits are invalid."); + } + let pathEntry; + try { + pathEntry = lstatEntry(path); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } + if (pathEntry.isSymbolicLink?.() || !pathEntry.isFile()) { + throw new Error(`${description} is not one regular non-symlink file.`); + } + let descriptor; + try { + descriptor = openFile(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + } catch (error) { + if (error?.code === "ENOENT") return null; + if (["ELOOP", "EISDIR"].includes(error?.code)) throw new Error(`${description} is not one regular non-symlink file.`); + throw error; + } + try { + const before = statFile(descriptor); + if (!before.isFile()) throw new Error(`${description} is not one regular non-symlink file.`); + if (!permitsInitialStatTransition(pathEntry, before, expectedSha256)) { + throw new Error(`${description} changed while it was read.`); + } + if (before.size < minBytes || before.size > maxBytes) throw new Error(`${description} exceeds its format byte limit or is malformed.`); + hooks?.afterInitialStat?.({ path, descriptor, stat: before }); + const bytes = Buffer.alloc(before.size); + let offset = 0; + while (offset < bytes.length) { + const bytesRead = readFile(descriptor, bytes, offset, bytes.length - offset, offset); + if (bytesRead === 0) throw new Error(`${description} changed while it was read.`); + offset += bytesRead; + } + const extra = Buffer.alloc(1); + const extraBytes = readFile(descriptor, extra, 0, 1, bytes.length); + hooks?.beforeFinalStat?.({ path, descriptor, bytes }); + const after = statFile(descriptor); + hooks?.afterFinalStat?.({ path, descriptor, stat: after, bytes }); + let finalPathEntry; + let finalPathMissing = false; + try { + finalPathEntry = lstatEntry(path); + } catch (error) { + if (error?.code !== "ENOENT") throw error; + finalPathMissing = true; + } + let confirmedHandle = null; + if (finalPathMissing && after.nlink === 1 && sameStat(pathEntry, before) && sameStat(before, after)) { + confirmedHandle = statFile(descriptor); + } + if (extraBytes === 0 && exactSha256(bytes, expectedSha256)) { + if (!finalPathMissing && isRegularPathEntry(finalPathEntry)) { + const retirementCut = exactMonotoneStatCut([pathEntry, before, after, finalPathEntry], 2, 1); + if (retirementCut === 1) { + throw new BoundedFileLinkRetiredBeforeReadError( + path, + description, + bytes, + expectedSha256, + pathEntry, + before, + ); + } + if (retirementCut !== null) { + throw new BoundedFileLinkRetiredDuringReadError( + path, + description, + bytes, + expectedSha256, + pathEntry, + before, + after, + finalPathEntry, + ); + } + } + if (finalPathMissing) { + const unlinkStats = [pathEntry, before, after]; + if (confirmedHandle !== null) unlinkStats.push(confirmedHandle); + if (exactMonotoneStatCut(unlinkStats, 1, 0) !== null) { + throw new BoundedFileUnlinkedDuringReadError( + path, + description, + bytes, + expectedSha256, + pathEntry, + before, + after, + confirmedHandle, + ); + } + } + } + if ( + extraBytes !== 0 || finalPathMissing || !isRegularPathEntry(finalPathEntry) || + !sameStat(pathEntry, before) || !sameStat(before, after) || !sameStat(after, finalPathEntry) + ) throw new Error(`${description} changed while it was read.`); + return bytes; + } finally { + closeFile(descriptor); + } +} diff --git a/scripts/fixtures/protected-publication-v2/pylon-consumer-lock.mjs b/scripts/fixtures/protected-publication-v2/pylon-consumer-lock.mjs new file mode 100644 index 0000000000..eaf377c474 --- /dev/null +++ b/scripts/fixtures/protected-publication-v2/pylon-consumer-lock.mjs @@ -0,0 +1,3751 @@ +import { createHash, randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { link, lstat, mkdir, open, readdir, rename, rm } from "node:fs/promises"; +import { basename, dirname, join, parse, relative, resolve, sep } from "node:path"; + +import { + BoundedFileLinkRetiredBeforeReadError, + BoundedFileLinkRetiredDuringReadError, + BoundedFileUnlinkedDuringReadError, + readBoundedRegularFile, +} from "./pylon-bounded-file.mjs"; + +class ConsumerEpochAdvancedError extends Error { + constructor() { + super("Consumer high-water journal epoch changed and fenced a paused writer."); + this.name = "ConsumerEpochAdvancedError"; + } +} + +export const PYLON_CONSUMER_LOCK_STALE_MS = 30_000; +export const PYLON_CONSUMER_LOCK_UPDATE_MS = 10_000; +export const PYLON_CONSUMER_ROTATE_CLAIM_TRIGGER = 60_000; +export const PYLON_CONSUMER_ROTATE_TRANSITION_TRIGGER = 3_800; +const LOCK_SCHEMA_VERSION = 2; +const CLAIM_INDEX_SCHEMA_VERSION = 1; +const LEGACY_LOCK_SCHEMA_VERSION = 1; +const TRANSACTION_SCHEMA_VERSION = 1; +const CHECKPOINT_SCHEMA_VERSION = 2; +const ROTATION_INTENT_SCHEMA_VERSION = 2; +const LEGACY_GUARD_SCHEMA_VERSION = 1; +const LEGACY_RETIREMENT_SCHEMA_VERSION = 1; +const GENESIS_DIGEST = "0".repeat(64); +const DEFAULT_STATE_MAX_BYTES = 1024 * 1024; +const MAX_STATE_BYTES = 16 * 1024 * 1024; +const DEFAULT_JOURNAL_MAX_BYTES = 64 * 1024 * 1024; +const MAX_JOURNAL_BYTES = 256 * 1024 * 1024; +const MAX_TRANSACTION_DEPTH = 4096; +const MAX_LOCK_GENERATIONS = 65_536; +const MAX_OPERATION_GENERATIONS = MAX_LOCK_GENERATIONS + 1; +const MAX_JOURNAL_ROOT_ENTRIES = 16; +const MAX_TEMPORARY_ENTRIES = 65_536; +const PROJECTION_RETRY_LIMIT = 32; +const TEMPORARY_DIRECTORY_NAME = ".owned-temporaries-v2"; +const LEGACY_RETIREMENT_MARKER_NAME = ".pylon-consumer-v1-retired.json"; +const uuidSource = "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"; +const uuidPattern = new RegExp(`^${uuidSource}$`); +const claimPattern = /^claim-([0-9]{16})-([0-9a-f]{64})\.json$/; +const claimIndexPattern = /^claim-index-([0-9]{16})\.json$/; +const undigestedClaimPattern = /^claim-([0-9]{16})\.json$/; +const transitionPattern = /^transition-([0-9a-f]{64})\.json$/; +const legacyTransitionPattern = /^([0-9a-f]{64})\.json$/; +const checkpointPattern = new RegExp(`^checkpoint-([0-9]{16})-(${uuidSource})\\.json$`); +const epochPattern = new RegExp(`^epoch-([0-9]{16})-(${uuidSource})$`); +const heartbeatPattern = new RegExp(`^heartbeat-([0-9]{16})-(${uuidSource})\\.json$`); +const terminalPattern = new RegExp(`^terminal-([0-9]{16})-(${uuidSource})\\.json$`); +const appliedPattern = new RegExp(`^applied-([0-9]{16})-(${uuidSource})\\.json$`); +const temporaryPattern = new RegExp( + `^\\.pylon-consumer-tmp-v1-p([1-9][0-9]*)-e(${uuidSource})-g([0-9]{16})-w(${uuidSource})-n([0-9a-f]{12})-k([a-z0-9-]{1,40})-t([0-9a-f]{64})\\.tmp$`, +); + +function exactKeys(value, keys) { + return value !== null && typeof value === "object" && !Array.isArray(value) && + Object.keys(value).sort().join(",") === [...keys].sort().join(","); +} + +function metadataBytes(value) { + return Buffer.from(`${JSON.stringify(value)}\n`); +} + +function digest(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +function generationName(generation) { + if (!Number.isSafeInteger(generation) || generation < 0 || generation > 9_999_999_999_999_999) { + throw new Error("Consumer high-water lock generation is exhausted or malformed."); + } + return String(generation).padStart(16, "0"); +} + +function deterministicUuid(value) { + const hex = digest(Buffer.from(value)); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-4${hex.slice(13, 16)}-8${hex.slice(17, 20)}-${hex.slice(20, 32)}`; +} + +function checkpointName(checkpoint) { + return `checkpoint-${generationName(checkpoint.epoch)}-${checkpoint.epochId}.json`; +} + +function epochName(checkpoint) { + return `epoch-${generationName(checkpoint.epoch)}-${checkpoint.epochId}`; +} + +function claimPath(context, claim) { + return join(context.epochDirectory, `claim-${generationName(claim.generation)}-${digest(metadataBytes(claim))}.json`); +} + +function claimIndexPath(context, generation) { + return join(context.epochDirectory, `claim-index-${generationName(generation)}.json`); +} + +function heartbeatPath(context, claim) { + return join(context.epochDirectory, `heartbeat-${generationName(claim.generation)}-${claim.token}.json`); +} + +function terminalPath(context, claim) { + return join(context.epochDirectory, `terminal-${generationName(claim.generation)}-${claim.token}.json`); +} + +function appliedPath(context, claim) { + return join(context.epochDirectory, `applied-${generationName(claim.generation)}-${claim.token}.json`); +} + +function transitionPath(context, baseDigest) { + return join(context.epochDirectory, `transition-${baseDigest}.json`); +} + +function validateClaim(value, context, stateMaxBytes) { + if ( + !value || value.schemaVersion !== LOCK_SCHEMA_VERSION || !Number.isSafeInteger(value.generation) || + value.generation < 1 || value.generation > MAX_OPERATION_GENERATIONS || !uuidPattern.test(value.token ?? "") || + !["normal", "rotation"].includes(value.type) + ) throw new Error("Consumer high-water operation claim is malformed."); + if (value.type === "normal") { + if ( + !exactKeys(value, ["schemaVersion", "generation", "token", "type", "ownerPid", "createdAtMs"]) || + !Number.isSafeInteger(value.ownerPid) || value.ownerPid < 1 || + !Number.isSafeInteger(value.createdAtMs) || value.createdAtMs < 0 + ) throw new Error("Consumer high-water normal operation claim is malformed."); + return value; + } + if (!exactKeys(value, ["schemaVersion", "generation", "token", "type", "intent"]) || !context) { + throw new Error("Consumer high-water rotation operation claim is malformed."); + } + const intent = validateRotationIntent(value.intent, context, stateMaxBytes); + if (value.token !== intent.checkpoint.epochId) { + throw new Error("Consumer high-water rotation operation claim differs from its deterministic intent."); + } + return value; +} + +function claimIndexFor(claim) { + return { + schemaVersion: CLAIM_INDEX_SCHEMA_VERSION, + generation: claim.generation, + claimSha256: digest(metadataBytes(claim)), + }; +} + +function validateClaimIndex(value, generation) { + if ( + !exactKeys(value, ["schemaVersion", "generation", "claimSha256"]) || + value.schemaVersion !== CLAIM_INDEX_SCHEMA_VERSION || value.generation !== generation || + !/^[0-9a-f]{64}$/.test(value.claimSha256 ?? "") + ) throw new Error("Consumer high-water claim index is malformed."); + return value; +} + +function validateHeartbeat(value, claim) { + if ( + claim.type !== "normal" || + !exactKeys(value, ["schemaVersion", "generation", "token", "refreshedAtMs"]) || + value.schemaVersion !== LOCK_SCHEMA_VERSION || value.generation !== claim.generation || value.token !== claim.token || + !Number.isSafeInteger(value.refreshedAtMs) || value.refreshedAtMs < claim.createdAtMs + ) throw new Error("Consumer high-water lock heartbeat is malformed."); + return value; +} + +function transactionFor(baseDigest, candidateBytes) { + return { + schemaVersion: TRANSACTION_SCHEMA_VERSION, + baseDigest, + candidateDigest: digest(candidateBytes), + candidateBase64: candidateBytes.toString("base64"), + }; +} + +function validateTransaction(value, expectedBaseDigest, stateMaxBytes) { + if ( + !exactKeys(value, ["schemaVersion", "baseDigest", "candidateDigest", "candidateBase64"]) || + value.schemaVersion !== TRANSACTION_SCHEMA_VERSION || value.baseDigest !== expectedBaseDigest || + !/^[0-9a-f]{64}$/.test(value.candidateDigest ?? "") || typeof value.candidateBase64 !== "string" || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value.candidateBase64) + ) throw new Error("Consumer high-water transaction is malformed."); + const candidateBytes = Buffer.from(value.candidateBase64, "base64"); + if ( + candidateBytes.length < 1 || candidateBytes.length > stateMaxBytes || + candidateBytes.toString("base64") !== value.candidateBase64 || digest(candidateBytes) !== value.candidateDigest || + value.candidateDigest === value.baseDigest + ) throw new Error("Consumer high-water transaction payload is malformed."); + return { value, candidateBytes }; +} + +function validateCheckpoint(value, stateMaxBytes) { + if ( + !exactKeys(value, [ + "schemaVersion", "epoch", "epochId", "previousCheckpointSha256", "previousTipSha256", + "historySha256", "anchorDigest", "anchorBase64", "retiredEpochDirectory", "sourceAuthoritySha256", + "sourceAuthorityTipDigest", "sourceAuthorityTipBase64", + ]) || value.schemaVersion !== CHECKPOINT_SCHEMA_VERSION || !Number.isSafeInteger(value.epoch) || value.epoch < 1 || + !uuidPattern.test(value.epochId ?? "") || !/^[0-9a-f]{64}$/.test(value.previousCheckpointSha256 ?? "") || + !/^[0-9a-f]{64}$/.test(value.previousTipSha256 ?? "") || !/^[0-9a-f]{64}$/.test(value.historySha256 ?? "") || + !/^[0-9a-f]{64}$/.test(value.anchorDigest ?? "") || !/^[0-9a-f]{64}$/.test(value.sourceAuthoritySha256 ?? "") || + !/^[0-9a-f]{64}$/.test(value.sourceAuthorityTipDigest ?? "") || + !(value.retiredEpochDirectory === null || epochPattern.test(value.retiredEpochDirectory)) || + !(value.anchorBase64 === null || typeof value.anchorBase64 === "string") || + !(value.sourceAuthorityTipBase64 === null || typeof value.sourceAuthorityTipBase64 === "string") + ) throw new Error("Consumer high-water journal checkpoint is malformed."); + let anchorBytes = null; + if (value.anchorBase64 !== null) { + if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value.anchorBase64)) { + throw new Error("Consumer high-water journal checkpoint is malformed."); + } + anchorBytes = Buffer.from(value.anchorBase64, "base64"); + if ( + anchorBytes.length < 1 || anchorBytes.length > stateMaxBytes || anchorBytes.toString("base64") !== value.anchorBase64 || + digest(anchorBytes) !== value.anchorDigest + ) throw new Error("Consumer high-water journal checkpoint anchor is malformed."); + } else if (value.anchorDigest !== GENESIS_DIGEST) { + throw new Error("Consumer high-water journal checkpoint anchor is malformed."); + } + if (value.sourceAuthorityTipBase64 === null) { + if (value.sourceAuthorityTipDigest !== GENESIS_DIGEST) { + throw new Error("Consumer high-water checkpoint source-authority tip is malformed."); + } + } else { + if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value.sourceAuthorityTipBase64)) { + throw new Error("Consumer high-water checkpoint source-authority tip is malformed."); + } + const sourceTip = Buffer.from(value.sourceAuthorityTipBase64, "base64"); + if ( + sourceTip.length < 1 || sourceTip.length > stateMaxBytes || + sourceTip.toString("base64") !== value.sourceAuthorityTipBase64 || digest(sourceTip) !== value.sourceAuthorityTipDigest + ) throw new Error("Consumer high-water checkpoint source-authority tip is malformed."); + } + if (value.epoch === 1) { + if ( + value.previousCheckpointSha256 !== GENESIS_DIGEST || value.previousTipSha256 !== GENESIS_DIGEST || + value.retiredEpochDirectory !== null + ) throw new Error("Consumer high-water genesis checkpoint is malformed."); + } else if (value.retiredEpochDirectory === null || value.previousTipSha256 !== value.anchorDigest) { + throw new Error("Consumer high-water rotated checkpoint is malformed."); + } + return { value, anchorBytes }; +} + +function validateRotationIntent(value, context, stateMaxBytes) { + if ( + !exactKeys(value, ["schemaVersion", "epoch", "epochId", "checkpointSha256", "tipSha256", "checkpoint"]) || + value.schemaVersion !== ROTATION_INTENT_SCHEMA_VERSION || value.epoch !== context.checkpoint.epoch || + value.epochId !== context.checkpoint.epochId || value.checkpointSha256 !== context.checkpointDigest || + !/^[0-9a-f]{64}$/.test(value.tipSha256 ?? "") + ) throw new Error("Consumer high-water rotation intent is malformed."); + const checkpoint = validateCheckpoint(value.checkpoint, stateMaxBytes).value; + if ( + checkpoint.epoch !== context.checkpoint.epoch + 1 || + checkpoint.previousCheckpointSha256 !== context.checkpointDigest || + checkpoint.previousTipSha256 !== value.tipSha256 || checkpoint.anchorDigest !== value.tipSha256 || + checkpoint.retiredEpochDirectory !== basename(context.epochDirectory) || + checkpoint.sourceAuthoritySha256 !== context.checkpoint.sourceAuthoritySha256 || + checkpoint.sourceAuthorityTipDigest !== context.checkpoint.sourceAuthorityTipDigest || + checkpoint.sourceAuthorityTipBase64 !== context.checkpoint.sourceAuthorityTipBase64 || + checkpoint.historySha256 !== digest(Buffer.from( + `${context.checkpoint.historySha256}:${context.checkpointDigest}:${value.tipSha256}`, + )) + ) throw new Error("Consumer high-water rotation intent does not anchor the exact epoch and tip."); + return value; +} + +function validateTerminal(value, claim, stateMaxBytes) { + const common = ["schemaVersion", "generation", "token", "outcome"]; + if ( + claim.type !== "normal" || !value || value.schemaVersion !== LOCK_SCHEMA_VERSION || value.generation !== claim.generation || + value.token !== claim.token || !["released", "retired", "commit"].includes(value.outcome) + ) throw new Error("Consumer high-water lock terminal marker is malformed."); + if (["released", "retired"].includes(value.outcome)) { + if (!exactKeys(value, common)) throw new Error("Consumer high-water lock terminal marker is malformed."); + return value; + } + if ( + !exactKeys(value, [...common, "transactions"]) || !Array.isArray(value.transactions) || + value.transactions.length < 1 || value.transactions.length > 2 + ) throw new Error("Consumer high-water lock commit marker is malformed."); + let expectedBase = value.transactions[0]?.baseDigest; + if (!/^[0-9a-f]{64}$/.test(expectedBase ?? "")) throw new Error("Consumer high-water lock commit marker is malformed."); + for (const transaction of value.transactions) { + validateTransaction(transaction, expectedBase, stateMaxBytes); + expectedBase = transaction.candidateDigest; + } + return value; +} + +function validateApplied(value, claim, terminal) { + if ( + !exactKeys(value, ["schemaVersion", "generation", "token", "terminalSha256"]) || + value.schemaVersion !== LOCK_SCHEMA_VERSION || value.generation !== claim.generation || value.token !== claim.token || + terminal?.outcome !== "commit" || value.terminalSha256 !== digest(metadataBytes(terminal)) + ) throw new Error("Consumer high-water lock applied marker is malformed."); + return value; +} + +function validateLegacyClaim(value) { + if ( + !exactKeys(value, ["schemaVersion", "generation", "token", "ownerPid", "createdAtMs"]) || + value.schemaVersion !== LEGACY_LOCK_SCHEMA_VERSION || !Number.isSafeInteger(value.generation) || value.generation < 1 || + !uuidPattern.test(value.token ?? "") || !Number.isSafeInteger(value.ownerPid) || value.ownerPid < 1 || + !Number.isSafeInteger(value.createdAtMs) || value.createdAtMs < 0 + ) throw new Error("Legacy consumer high-water lock claim is malformed."); + return value; +} + +function validateLegacyHeartbeat(value, claim) { + if ( + !exactKeys(value, ["schemaVersion", "generation", "token", "refreshedAtMs"]) || + value.schemaVersion !== LEGACY_LOCK_SCHEMA_VERSION || value.generation !== claim.generation || + value.token !== claim.token || !Number.isSafeInteger(value.refreshedAtMs) || value.refreshedAtMs < claim.createdAtMs + ) throw new Error("Legacy consumer high-water heartbeat is malformed."); + return value; +} + +function validateLegacyTerminal(value, claim, stateMaxBytes) { + const common = ["schemaVersion", "generation", "token", "outcome"]; + if ( + !value || value.schemaVersion !== LEGACY_LOCK_SCHEMA_VERSION || value.generation !== claim.generation || + value.token !== claim.token || !["released", "retired", "commit"].includes(value.outcome) + ) throw new Error("Legacy consumer high-water terminal marker is malformed."); + if (value.outcome !== "commit") { + if (!exactKeys(value, common)) throw new Error("Legacy consumer high-water terminal marker is malformed."); + return value; + } + if ( + !exactKeys(value, [...common, "transactions"]) || !Array.isArray(value.transactions) || + value.transactions.length < 1 || value.transactions.length > 2 + ) throw new Error("Legacy consumer high-water commit marker is malformed."); + let expectedBase = value.transactions[0]?.baseDigest; + if (!/^[0-9a-f]{64}$/.test(expectedBase ?? "")) throw new Error("Legacy consumer high-water commit marker is malformed."); + for (const transaction of value.transactions) { + validateTransaction(transaction, expectedBase, stateMaxBytes); + expectedBase = transaction.candidateDigest; + } + return value; +} + +function validateLegacyApplied(value, claim, terminal) { + if ( + !exactKeys(value, ["schemaVersion", "generation", "token", "terminalSha256"]) || + value.schemaVersion !== LEGACY_LOCK_SCHEMA_VERSION || value.generation !== claim.generation || + value.token !== claim.token || terminal?.outcome !== "commit" || + value.terminalSha256 !== digest(metadataBytes(terminal)) + ) throw new Error("Legacy consumer high-water applied marker is malformed."); + return value; +} + +function legacyGuardFor(statePath) { + return { + schemaVersion: LEGACY_GUARD_SCHEMA_VERSION, + kind: "pylon-consumer-legacy-lock-guard", + statePathSha256: digest(Buffer.from(statePath)), + }; +} + +function legacyRetirementMarkerFor(statePath, legacy) { + return { + schemaVersion: LEGACY_RETIREMENT_SCHEMA_VERSION, + kind: "pylon-consumer-v1-retirement", + statePathSha256: digest(Buffer.from(statePath)), + authoritySha256: legacy.authoritySha256, + tipSha256: legacy.tipDigest, + }; +} + +function validateLegacyRetirementMarker(value, statePath) { + if ( + !exactKeys(value, ["schemaVersion", "kind", "statePathSha256", "authoritySha256", "tipSha256"]) || + value.schemaVersion !== LEGACY_RETIREMENT_SCHEMA_VERSION || value.kind !== "pylon-consumer-v1-retirement" || + value.statePathSha256 !== digest(Buffer.from(statePath)) || + !/^[0-9a-f]{64}$/.test(value.authoritySha256 ?? "") || !/^[0-9a-f]{64}$/.test(value.tipSha256 ?? "") + ) throw new Error("Legacy consumer high-water retirement marker is malformed."); + return value; +} + +async function secureHandle(handle, stat, description, type, options) { + if ((type === "file" && !stat.isFile()) || (type === "directory" && !stat.isDirectory())) { + throw new Error(`${description} must be one real ${type}.`); + } + if (stat.uid !== options.currentUid) throw new Error(`${description} must be owned by the current uid.`); + const requiredMode = type === "directory" ? 0o700 : 0o600; + if ((stat.mode & 0o7777) !== requiredMode) { + throw new Error(`${description} must already have exact ${requiredMode.toString(8)} permissions before use.`); + } + return stat; +} + +async function secureDirectory(path, description, options) { + let handle; + try { + handle = await options.openFile( + path, + constants.O_RDONLY | (constants.O_DIRECTORY ?? 0) | (constants.O_NOFOLLOW ?? 0), + ); + } catch (error) { + if (["ELOOP", "ENOTDIR"].includes(error?.code)) throw new Error(`${description} must be one real directory.`); + throw error; + } + try { + await secureHandle(handle, await handle.stat(), description, "directory", options); + } finally { + await handle.close(); + } +} + +export async function syncConsumerStateDirectory(path, { openDirectory = open } = {}) { + let handle; + try { + handle = await openDirectory(path, "r"); + await handle.sync(); + } catch (error) { + if (!["EINVAL", "EPERM", "EISDIR"].includes(error?.code)) throw error; + } finally { + if (handle !== undefined) await handle.close(); + } +} + +export async function ensureDurableConsumerStateDirectory( + directory, + { lstatEntry = lstat, makeDirectory = mkdir, syncDirectory = syncConsumerStateDirectory, create = true } = {}, +) { + const absolute = resolve(directory); + const root = parse(absolute).root; + let parent = root; + const rootEntry = await lstatEntry(root); + if (!rootEntry.isDirectory()) throw new Error("Consumer high-water state directory must be one canonical real directory."); + const remainder = relative(root, absolute); + for (const component of remainder ? remainder.split(sep) : []) { + const current = join(parent, component); + let entry; + try { + entry = await lstatEntry(current); + } catch (error) { + if (error?.code !== "ENOENT" || !create) throw error; + try { + await makeDirectory(current, { mode: 0o700 }); + } catch (mkdirError) { + if (mkdirError?.code !== "EEXIST") throw mkdirError; + } + entry = await lstatEntry(current); + } + if (!entry.isDirectory() || entry.isSymbolicLink?.()) { + throw new Error("Consumer high-water state directory must be one canonical real directory."); + } + await syncDirectory(parent); + parent = current; + } + return absolute; +} + +async function ensureDirectory(path, description, options) { + try { + await options.makeDirectory(path, { mode: 0o700 }); + } catch (error) { + if (error?.code !== "EEXIST") throw error; + } + const entry = await options.lstatEntry(path); + if (!entry.isDirectory() || entry.isSymbolicLink?.()) throw new Error(`${description} must be one real directory.`); + await secureDirectory(path, description, options); + await options.syncDirectory(path); + await options.syncDirectory(dirname(path)); +} + +async function readSecureFile(path, maxBytes, description, options, minBytes = 1, hooks, expectedSha256 = null) { + return readBoundedRegularFile(path, { + maxBytes, + minBytes, + description, + openFile: options.openFile, + lstatEntry: options.lstatEntry, + hooks, + expectedSha256, + validateHandle: (handle, stat) => secureHandle(handle, stat, description, "file", options), + }); +} + +async function readExactMetadata(path, maxBytes, validate, description, options, budget, expectedSha256 = null) { + const bytes = await readSecureFile( + path, + maxBytes, + description, + options, + 1, + options.hooks?.metadataRead, + expectedSha256, + ); + if (bytes === null) return null; + if (budget) { + budget.bytes += bytes.length; + if (budget.bytes > options.maxJournalBytes) throw new Error("Consumer high-water journal exceeds its safe byte bound."); + } + let value; + try { + value = validate(JSON.parse(bytes)); + } catch (error) { + if (error instanceof SyntaxError) throw new Error(`${description} is malformed.`); + throw error; + } + if (!bytes.equals(metadataBytes(value))) throw new Error(`${description} is not canonical.`); + return value; +} + +function temporaryName(targetPath, kind, writer, context) { + if (!/^[a-z0-9-]{1,40}$/.test(kind)) throw new Error("Consumer high-water temporary kind is malformed."); + const attempt = randomUUID().replaceAll("-", "").slice(0, 12); + return `.pylon-consumer-tmp-v1-p${process.pid}-e${context.checkpoint.epochId}-g${generationName(writer.generation)}` + + `-w${writer.token}-n${attempt}-k${kind}-t${digest(Buffer.from(resolve(targetPath)))}.tmp`; +} + +async function inspectTemporary(path, options) { + const match = temporaryPattern.exec(basename(path)); + if (!match) throw new Error("Consumer high-water journal contains an unexpected hidden entry."); + let handle; + try { + handle = await options.openFile(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + } catch (error) { + if (error?.code === "ENOENT") return null; + if (["ELOOP", "EISDIR"].includes(error?.code)) { + throw new Error("Consumer high-water owned temporary is not one regular non-symlink file."); + } + throw error; + } + try { + const stat = await secureHandle( + handle, + await handle.stat(), + "Consumer high-water owned temporary", + "file", + options, + ); + if (stat.size > options.metadataMaxBytes) throw new Error("Consumer high-water owned temporary exceeds its safe byte bound."); + } finally { + await handle.close(); + } + const kind = match[6]; + const allowedKinds = new Set([ + "checkpoint", "projection", "transition", "claim", "claim-index", "initial-heartbeat", "heartbeat", + "terminal-released", "terminal-retired", "terminal-commit", "applied", "legacy-guard", + "legacy-retirement", + ]); + if (!allowedKinds.has(kind)) throw new Error("Consumer high-water owned temporary target metadata is malformed."); + return { + path, + pid: Number(match[1]), + epochId: match[2], + generation: Number(match[3]), + token: match[4], + attempt: match[5], + kind, + targetSha256: match[7], + }; +} + +function isImmediateSuccessorCheckpoint(context, checkpoint) { + return checkpoint.epoch === context.checkpoint.epoch + 1 && + checkpoint.epochId === deterministicUuid( + `pylon-consumer-rotation-v2:${context.checkpointDigest}:${checkpoint.anchorDigest}`, + ) && + checkpoint.previousCheckpointSha256 === context.checkpointDigest && + checkpoint.previousTipSha256 === checkpoint.anchorDigest && + checkpoint.retiredEpochDirectory === basename(context.epochDirectory) && + checkpoint.sourceAuthoritySha256 === context.checkpoint.sourceAuthoritySha256 && + checkpoint.sourceAuthorityTipDigest === context.checkpoint.sourceAuthorityTipDigest && + checkpoint.sourceAuthorityTipBase64 === context.checkpoint.sourceAuthorityTipBase64 && + checkpoint.historySha256 === digest(Buffer.from( + `${context.checkpoint.historySha256}:${context.checkpointDigest}:${checkpoint.anchorDigest}`, + )); +} + +function retainedCheckpointPath(context) { + if (context.checkpoint.retiredEpochDirectory === null) return null; + const match = epochPattern.exec(context.checkpoint.retiredEpochDirectory); + if (!match) throw new Error("Consumer high-water journal checkpoint context is malformed."); + return join(context.journalDirectory, `checkpoint-${match[1]}-${match[2]}.json`); +} + +function contextCheckpointAnchors(context) { + const anchors = new Map([[context.checkpointPath, context.checkpointDigest]]); + const retainedPath = retainedCheckpointPath(context); + if (retainedPath !== null) anchors.set(retainedPath, context.checkpoint.previousCheckpointSha256); + return anchors; +} + +function isContextAnchoredCheckpoint(context, path, checkpoint, expectedSha256) { + if (path === context.checkpointPath) { + return expectedSha256 === context.checkpointDigest && + metadataBytes(checkpoint).equals(metadataBytes(context.checkpoint)); + } + const retainedPath = retainedCheckpointPath(context); + return retainedPath !== null && path === retainedPath && + expectedSha256 === context.checkpoint.previousCheckpointSha256 && + checkpoint.epoch + 1 === context.checkpoint.epoch && + epochName(checkpoint) === context.checkpoint.retiredEpochDirectory && + checkpoint.sourceAuthoritySha256 === context.checkpoint.sourceAuthoritySha256 && + checkpoint.sourceAuthorityTipDigest === context.checkpoint.sourceAuthorityTipDigest && + checkpoint.sourceAuthorityTipBase64 === context.checkpoint.sourceAuthorityTipBase64 && + context.checkpoint.historySha256 === digest(Buffer.from( + `${checkpoint.historySha256}:${expectedSha256}:${context.checkpoint.anchorDigest}`, + )); +} + +function isAuthenticatedCheckpointAnchor(context, path, checkpoint, expectedSha256, additionalAnchor = null) { + return isContextAnchoredCheckpoint(context, path, checkpoint, expectedSha256) || ( + additionalAnchor !== null && path === additionalAnchor.path && expectedSha256 === additionalAnchor.digest && + metadataBytes(checkpoint).equals(metadataBytes(additionalAnchor.checkpoint)) && + isImmediateSuccessorCheckpoint(context, checkpoint) + ); +} + +function canonicalCheckpointNameEpoch(name) { + const match = checkpointPattern.exec(name); + if (!match) return null; + const epoch = Number(match[1]); + return Number.isSafeInteger(epoch) && generationName(epoch) === match[1] ? epoch : null; +} + +function isProvisionallyRemovedContextCurrentCheckpoint(path, context, anchors, rootNames) { + const name = basename(path); + if ( + path !== context.checkpointPath || path !== join(context.journalDirectory, name) || + name !== checkpointName(context.checkpoint) || anchors.get(path) !== context.checkpointDigest || + canonicalCheckpointNameEpoch(name) !== context.checkpoint.epoch + ) return false; + return rootNames.some((candidate) => { + const candidateEpoch = canonicalCheckpointNameEpoch(candidate); + return candidateEpoch !== null && candidateEpoch > context.checkpoint.epoch; + }); +} + +function isVanishedRetainedCheckpoint(path, context, anchors, rootNames) { + const retainedPath = retainedCheckpointPath(context); + if ( + retainedPath === null || path !== retainedPath || + anchors.get(path) !== context.checkpoint.previousCheckpointSha256 + ) return false; + const retainedMatch = checkpointPattern.exec(basename(path)); + if (!retainedMatch || Number(retainedMatch[1]) + 1 !== context.checkpoint.epoch) return false; + return rootNames.some((candidate) => { + const candidateEpoch = canonicalCheckpointNameEpoch(candidate); + return candidateEpoch !== null && candidateEpoch > Number(retainedMatch[1]); + }); +} + +const checkpointStatEvidenceKeys = ["dev", "ino", "size", "mtimeMs", "ctimeMs", "nlink"]; + +function isFrozenRecord(value) { + return value !== null && typeof value === "object" && Object.isFrozen(value); +} + +function isExactCheckpointStatEvidence(value) { + return isFrozenRecord(value) && exactKeys(value, checkpointStatEvidenceKeys) && + checkpointStatEvidenceKeys.every((key) => Number.isFinite(value[key])) && + Number.isSafeInteger(value.size) && value.size >= 0 && Number.isSafeInteger(value.nlink) && value.nlink >= 0; +} + +function exactEvidenceMonotoneCut(observations, fromLinks, toLinks, byteLength) { + if ( + observations.length < 2 || observations.some((stat) => !isExactCheckpointStatEvidence(stat)) || + observations[0].size !== byteLength || observations[0].nlink !== fromLinks || + observations.at(-1).nlink !== toLinks || + observations.some((stat) => ( + stat.dev !== observations[0].dev || stat.ino !== observations[0].ino || + stat.size !== observations[0].size || stat.mtimeMs !== observations[0].mtimeMs + )) + ) return null; + let cut = null; + for (let index = 1; index < observations.length; index += 1) { + const previous = observations[index - 1]; + const current = observations[index]; + if (previous.nlink === current.nlink) { + if (previous.ctimeMs !== current.ctimeMs) return null; + continue; + } + if ( + cut !== null || previous.nlink !== fromLinks || current.nlink !== toLinks || + previous.ctimeMs === current.ctimeMs + ) return null; + cut = index; + } + return cut; +} + +function isExactLinkRetiredBeforeReadEvidence(error) { + const transition = error.statTransition; + return isFrozenRecord(transition) && exactKeys(transition, ["pathEntry", "openedHandle"]) && + exactEvidenceMonotoneCut( + [transition.pathEntry, transition.openedHandle], + 2, + 1, + error.bytes.length, + ) === 1; +} + +function isExactLinkRetiredDuringReadEvidence(error) { + const transition = error.statTransition; + return isFrozenRecord(transition) && exactKeys(transition, ["pathEntry", "before", "after", "finalPathEntry"]) && + [2, 3].includes(exactEvidenceMonotoneCut( + [transition.pathEntry, transition.before, transition.after, transition.finalPathEntry], + 2, + 1, + error.bytes.length, + )); +} + +function isExactUnlinkedDuringReadEvidence(error) { + const transition = error.statTransition; + if ( + !isFrozenRecord(transition) || + !exactKeys(transition, ["pathEntry", "before", "after", "confirmedHandle"]) || + !(transition.confirmedHandle === null || isExactCheckpointStatEvidence(transition.confirmedHandle)) + ) return false; + const observations = [transition.pathEntry, transition.before, transition.after]; + if (transition.confirmedHandle !== null) observations.push(transition.confirmedHandle); + return exactEvidenceMonotoneCut(observations, 1, 0, error.bytes.length) !== null; +} + +function authenticatedChangedCheckpointRead(error, context, options, anchors, rootNames, additionalAnchor = null) { + const linkRetiredBeforeRead = error instanceof BoundedFileLinkRetiredBeforeReadError && + error.constructor === BoundedFileLinkRetiredBeforeReadError && + error.name === "BoundedFileLinkRetiredBeforeReadError"; + const linkRetiredDuringRead = error instanceof BoundedFileLinkRetiredDuringReadError && + error.constructor === BoundedFileLinkRetiredDuringReadError && + error.name === "BoundedFileLinkRetiredDuringReadError"; + const unlinkedDuringRead = error instanceof BoundedFileUnlinkedDuringReadError && + error.constructor === BoundedFileUnlinkedDuringReadError && + error.name === "BoundedFileUnlinkedDuringReadError"; + const linkRetiredDuringOrBeforeRead = linkRetiredBeforeRead || linkRetiredDuringRead; + if ( + (!linkRetiredDuringOrBeforeRead && !unlinkedDuringRead) || + error.description !== "Consumer high-water journal checkpoint" || typeof error.path !== "string" || + !Buffer.isBuffer(error.bytes) || error.bytes.length < 1 || error.bytes.length > options.metadataMaxBytes || + (linkRetiredBeforeRead && !isExactLinkRetiredBeforeReadEvidence(error)) || + (linkRetiredDuringRead && !isExactLinkRetiredDuringReadEvidence(error)) || + (unlinkedDuringRead && !isExactUnlinkedDuringReadEvidence(error)) + ) return null; + const expectedSha256 = anchors.get(error.path); + if ( + expectedSha256 === undefined || error.expectedSha256 !== expectedSha256 || + digest(error.bytes) !== expectedSha256 || error.sha256 !== expectedSha256 || + dirname(error.path) !== context.journalDirectory + ) return null; + const name = basename(error.path); + const match = checkpointPattern.exec(name); + if (!match || error.path !== join(context.journalDirectory, name)) return null; + let checkpoint; + try { + checkpoint = validateCheckpoint(JSON.parse(error.bytes.toString("utf8")), options.stateMaxBytes).value; + } catch { + return null; + } + if ( + !metadataBytes(checkpoint).equals(error.bytes) || checkpointName(checkpoint) !== name || + checkpoint.epoch !== Number(match[1]) || + !isAuthenticatedCheckpointAnchor(context, error.path, checkpoint, expectedSha256, additionalAnchor) + ) return null; + if (unlinkedDuringRead) { + const hasLaterCheckpoint = rootNames.some((candidate) => { + const candidateEpoch = canonicalCheckpointNameEpoch(candidate); + return candidateEpoch !== null && candidateEpoch > checkpoint.epoch; + }); + if (!hasLaterCheckpoint) return null; + } + const linkRetirementStat = linkRetiredBeforeRead + ? error.statTransition.openedHandle + : linkRetiredDuringRead ? error.statTransition.finalPathEntry : null; + return { + checkpoint, + linkRetiredBeforeRead: linkRetiredDuringOrBeforeRead, + linkRetirementStat, + unlinkedDuringRead, + }; +} + +function sameRetiredLinkStat(left, right) { + return left !== null && right !== null && + left.dev === right.dev && left.ino === right.ino && left.size === right.size && + left.mtimeMs === right.mtimeMs && left.ctimeMs === right.ctimeMs && left.nlink === right.nlink; +} + +function checkpointProofOptions(entry, options, invalidRoot) { + if (entry.checkpointStat === null) return options; + let initialPathStat = true; + return { + ...options, + lstatEntry: async (path) => { + const stat = await options.lstatEntry(path); + if (path === entry.path && initialPathStat) { + initialPathStat = false; + if (!sameRetiredLinkStat(stat, entry.checkpointStat)) throw invalidRoot(); + } + return stat; + }, + }; +} + +async function authenticateStableChangedRoot(scan, context, options, anchors, target, invalidRoot) { + const initialNames = new Set(scan.rootNames); + if (initialNames.size !== scan.rootNames.length) throw invalidRoot(); + const targetCheckpointName = basename(target.path); + const targetEpochName = epochName(target.checkpoint); + const proofAnchors = new Map(anchors); + proofAnchors.set(target.path, target.digest); + const optionalCheckpointNames = new Set([ + ...scan.checkpointEntries + .filter((entry) => entry.path !== target.path && entry.checkpoint.epoch < target.checkpoint.epoch) + .map((entry) => entry.name), + ...scan.vanishedRetainedCheckpointNames, + ]); + const optionalEpochNames = new Set( + scan.epochEntries + .filter((entry) => entry.epoch < target.checkpoint.epoch) + .map((entry) => entry.name), + ); + const optionalTemporaryNames = new Set( + scan.temporaries + .filter((temporary) => dirname(temporary.path) === context.journalDirectory) + .map((temporary) => basename(temporary.path)), + ); + const optionalNames = new Set([ + ...optionalCheckpointNames, + ...optionalEpochNames, + ...optionalTemporaryNames, + ]); + const requiredNames = new Set([TEMPORARY_DIRECTORY_NAME, targetCheckpointName, targetEpochName]); + const removedBeforeProof = new Set(scan.removedCheckpointEntries.map((entry) => entry.name)); + const proofNamesArray = await options.readDirectory(context.journalDirectory); + const proofNames = new Set(proofNamesArray); + if ( + proofNamesArray.length > MAX_JOURNAL_ROOT_ENTRIES + MAX_TEMPORARY_ENTRIES || + proofNames.size !== proofNamesArray.length || + [...proofNames].some((name) => !initialNames.has(name)) || + [...removedBeforeProof].some((name) => proofNames.has(name)) || + scan.vanishedRetainedCheckpointNames.some((name) => proofNames.has(name)) || + [...initialNames].some((name) => !optionalNames.has(name) && !proofNames.has(name)) || + [...requiredNames].some((name) => !proofNames.has(name)) + ) throw invalidRoot(); + const removedDuringProof = new Set(); + let targetAuthenticated = false; + for (const entry of scan.checkpointEntries) { + if (!proofNames.has(entry.name)) continue; + const optional = optionalCheckpointNames.has(entry.name); + await options.hooks?.beforeStableCheckpointProofRead?.({ + name: entry.name, + path: entry.path, + target: entry.path === target.path, + }); + const expectedSha256 = proofAnchors.get(entry.path) ?? null; + let checkpoint; + let changedRead = null; + try { + checkpoint = await readExactMetadata( + entry.path, + options.metadataMaxBytes, + (value) => validateCheckpoint(value, options.stateMaxBytes).value, + "Consumer high-water journal checkpoint", + checkpointProofOptions(entry, options, invalidRoot), + undefined, + expectedSha256, + ); + } catch (error) { + changedRead = authenticatedChangedCheckpointRead( + error, + context, + options, + proofAnchors, + proofNamesArray, + target, + ); + if (changedRead === null) throw error; + checkpoint = changedRead.checkpoint; + } + if (checkpoint === null) { + if (!optional) throw invalidRoot(); + removedDuringProof.add(entry.name); + continue; + } + if ( + !metadataBytes(checkpoint).equals(metadataBytes(entry.checkpoint)) || + (expectedSha256 !== null && !isAuthenticatedCheckpointAnchor( + context, + entry.path, + checkpoint, + expectedSha256, + target, + )) + ) throw invalidRoot(); + if (changedRead?.unlinkedDuringRead) { + if (!optional) throw invalidRoot(); + removedDuringProof.add(entry.name); + continue; + } + if (entry.path === target.path) targetAuthenticated = true; + } + if (!targetAuthenticated) throw invalidRoot(); + await secureDirectory( + join(context.journalDirectory, TEMPORARY_DIRECTORY_NAME), + "Consumer high-water temporary directory", + options, + ); + const targetEpoch = scan.epochEntries.find((entry) => entry.name === targetEpochName); + if (targetEpoch === undefined) throw invalidRoot(); + await secureDirectory(targetEpoch.path, "Consumer high-water epoch directory", options); + for (const temporary of scan.temporaries) { + if (dirname(temporary.path) === context.journalDirectory && proofNames.has(basename(temporary.path))) { + if ((await inspectTemporary(temporary.path, options)) === null) throw invalidRoot(); + } + } + const finalNamesArray = await options.readDirectory(context.journalDirectory); + const finalNames = new Set(finalNamesArray); + if ( + finalNamesArray.length > MAX_JOURNAL_ROOT_ENTRIES + MAX_TEMPORARY_ENTRIES || + finalNames.size !== finalNamesArray.length || + [...finalNames].some((name) => !proofNames.has(name)) || + [...proofNames].some((name) => !optionalNames.has(name) && !finalNames.has(name)) || + [...removedBeforeProof].some((name) => finalNames.has(name)) || + [...removedDuringProof].some((name) => finalNames.has(name)) || + [...requiredNames].some((name) => !finalNames.has(name)) + ) throw invalidRoot(); +} + +async function inProgressDirectoryStats(context, nextEpochPath, options, invalidRoot) { + const [temporaryDirectory, currentEpoch, nextEpoch] = await Promise.all([ + options.lstatEntry(context.temporaryDirectory), + options.lstatEntry(context.epochDirectory), + options.lstatEntry(nextEpochPath), + ]); + if ( + !temporaryDirectory.isDirectory() || temporaryDirectory.isSymbolicLink?.() || + !currentEpoch.isDirectory() || currentEpoch.isSymbolicLink?.() || + !nextEpoch.isDirectory() || nextEpoch.isSymbolicLink?.() + ) throw invalidRoot(); + return { temporaryDirectory, currentEpoch, nextEpoch }; +} + +async function authenticateLinkRetiredInProgressRoot( + scan, + context, + options, + anchors, + currentCheckpoint, + nextEpochPath, + kind, + invalidRoot, +) { + if (scan.linkRetiredCheckpointEntries.length === 0) return; + if ( + currentCheckpoint === undefined || scan.linkRetiredCheckpointEntries.length !== 1 || + scan.linkRetiredCheckpointEntries[0] !== currentCheckpoint || currentCheckpoint.path !== context.checkpointPath || + currentCheckpoint.linkRetirementStat === null + ) throw invalidRoot(); + const permittedNames = new Set([ + TEMPORARY_DIRECTORY_NAME, + basename(context.checkpointPath), + basename(context.epochDirectory), + basename(nextEpochPath), + ]); + if ( + scan.rootNames.length !== permittedNames.size || + scan.rootNames.some((name) => !permittedNames.has(name)) + ) throw invalidRoot(); + const beforeDirectories = await inProgressDirectoryStats(context, nextEpochPath, options, invalidRoot); + await options.hooks?.beforeInProgressStableRootProof?.({ kind }); + await authenticateStableChangedRoot(scan, context, options, anchors, currentCheckpoint, invalidRoot); + const afterDirectories = await inProgressDirectoryStats(context, nextEpochPath, options, invalidRoot); + if ( + !sameRetiredLinkStat(beforeDirectories.temporaryDirectory, afterDirectories.temporaryDirectory) || + !sameRetiredLinkStat(beforeDirectories.currentEpoch, afterDirectories.currentEpoch) || + !sameRetiredLinkStat(beforeDirectories.nextEpoch, afterDirectories.nextEpoch) || + (await options.readDirectory(nextEpochPath)).length !== 0 + ) throw invalidRoot(); +} + +async function authenticateChangedRoot( + context, + options, + inProgressCheckpoint = null, + allowInProgressDiscovery = false, +) { + const anchors = contextCheckpointAnchors(context); + const scan = await scanJournalRoot(context.statePath, context.journalDirectory, options, { + checkpointAnchors: anchors, + checkpointContext: context, + }); + const invalidRoot = () => new Error( + "Consumer high-water journal root changed without one exact current or immediate-successor authority.", + ); + const currentCheckpoint = scan.checkpointEntries.find((entry) => entry.path === context.checkpointPath); + if (currentCheckpoint && currentCheckpoint.digest !== context.checkpointDigest) throw invalidRoot(); + + if (inProgressCheckpoint !== null && scan.checkpointEntries.length === 1) { + const checkpoint = validateCheckpoint(inProgressCheckpoint, options.stateMaxBytes).value; + const nextEpochPath = join(context.journalDirectory, epochName(checkpoint)); + await secureDirectory(context.epochDirectory, "Consumer high-water epoch directory", options); + await secureDirectory(nextEpochPath, "Consumer high-water epoch directory", options); + if ( + !isImmediateSuccessorCheckpoint(context, checkpoint) || + scan.checkpointEntries.length !== 1 || scan.head?.path !== context.checkpointPath || scan.missingHeadEpoch || + scan.epochEntries.length !== 2 || + scan.epochEntries.some((entry) => ![context.epochDirectory, nextEpochPath].includes(entry.path)) || + !scan.epochEntries.some((entry) => entry.path === nextEpochPath) || + (await options.readDirectory(nextEpochPath)).length !== 0 + ) throw invalidRoot(); + if ((await options.readDirectory(nextEpochPath)).length !== 0) throw invalidRoot(); + await authenticateLinkRetiredInProgressRoot( + scan, + context, + options, + anchors, + currentCheckpoint, + nextEpochPath, + "known", + invalidRoot, + ); + return false; + } + + const discoveredNextEpoch = scan.epochEntries.find((entry) => entry.path !== context.epochDirectory); + if ( + allowInProgressDiscovery && inProgressCheckpoint === null && + scan.checkpointEntries.length === 1 && scan.head?.path === context.checkpointPath && !scan.missingHeadEpoch && + scan.epochEntries.length === 2 && discoveredNextEpoch?.epoch === context.checkpoint.epoch + 1 && + (await options.readDirectory(discoveredNextEpoch.path)).length === 0 + ) { + await secureDirectory(context.epochDirectory, "Consumer high-water epoch directory", options); + await secureDirectory(discoveredNextEpoch.path, "Consumer high-water epoch directory", options); + if ((await options.readDirectory(discoveredNextEpoch.path)).length !== 0) throw invalidRoot(); + await authenticateLinkRetiredInProgressRoot( + scan, + context, + options, + anchors, + currentCheckpoint, + discoveredNextEpoch.path, + "discovered", + invalidRoot, + ); + return discoveredNextEpoch.path; + } + + const retainedCheckpoint = scan.checkpointEntries.find((entry) => entry.path !== context.checkpointPath); + const retiredEpochPath = context.checkpoint.retiredEpochDirectory === null + ? null + : join(context.journalDirectory, context.checkpoint.retiredEpochDirectory); + if ( + currentCheckpoint && scan.head?.path === context.checkpointPath && !scan.missingHeadEpoch && + scan.checkpointEntries.length <= 2 && scan.epochEntries.length <= 2 && + (!retainedCheckpoint || ( + retainedCheckpoint.digest === context.checkpoint.previousCheckpointSha256 && + epochName(retainedCheckpoint.checkpoint) === context.checkpoint.retiredEpochDirectory + )) && + scan.epochEntries.every((entry) => [context.epochDirectory, retiredEpochPath].includes(entry.path)) + ) { + if ( + scan.removedCheckpointEntries.length > 0 || scan.linkRetiredCheckpointEntries.length > 0 || + scan.vanishedRetainedCheckpointNames.length > 0 + ) { + await authenticateStableChangedRoot(scan, context, options, anchors, currentCheckpoint, invalidRoot); + } else { + await secureDirectory(context.epochDirectory, "Consumer high-water epoch directory", options); + } + return false; + } + + const successor = scan.head; + const expectedCheckpointPath = successor + ? join(context.journalDirectory, checkpointName(successor.checkpoint)) + : null; + const expectedEpochPath = successor + ? join(context.journalDirectory, epochName(successor.checkpoint)) + : null; + if ( + !successor || scan.missingHeadEpoch || successor.path !== expectedCheckpointPath || + !isImmediateSuccessorCheckpoint(context, successor.checkpoint) || + scan.checkpointEntries.length < 1 || scan.checkpointEntries.length > 2 || + scan.epochEntries.length < 1 || scan.epochEntries.length > 2 || + scan.checkpointEntries.some((entry) => ![context.checkpointPath, expectedCheckpointPath].includes(entry.path)) || + scan.epochEntries.some((entry) => ![context.epochDirectory, expectedEpochPath].includes(entry.path)) || + !scan.epochEntries.some((entry) => entry.path === expectedEpochPath) + ) throw invalidRoot(); + await authenticateStableChangedRoot(scan, context, options, anchors, successor, invalidRoot); + return true; +} + +async function revalidateAuthority( + context, + operation, + options, + inProgressCheckpoint = options.inProgressCheckpoint ?? null, + allowInProgressDiscovery = false, +) { + await options.hooks?.beforePathOperation?.({ + operation, + statePath: context.statePath, + lockDirectory: context.journalDirectory, + transactionDirectory: context.epochDirectory, + inProgressCheckpoint: inProgressCheckpoint === null ? null : structuredClone(inProgressCheckpoint), + }); + await ensureDurableConsumerStateDirectory(dirname(context.statePath), { + ...options.directoryOperations, + create: false, + }); + await secureDirectory(dirname(context.statePath), "Consumer high-water state directory", options); + await secureDirectory(context.journalDirectory, "Consumer high-water journal directory", options); + await secureDirectory(context.temporaryDirectory, "Consumer high-water temporary directory", options); + let oldEpochError = null; + try { + await secureDirectory(context.epochDirectory, "Consumer high-water epoch directory", options); + } catch (error) { + if (error?.code !== "ENOENT") throw error; + oldEpochError = error; + } + const entries = await options.readDirectory(context.journalDirectory); + if (entries.length > MAX_JOURNAL_ROOT_ENTRIES + MAX_TEMPORARY_ENTRIES) { + throw new Error("Consumer high-water journal root exceeds its safe allocation bound."); + } + const expectedRootNames = new Set([ + TEMPORARY_DIRECTORY_NAME, + basename(context.checkpointPath), + basename(context.epochDirectory), + ]); + let changedRoot = false; + if (entries.some((name) => !expectedRootNames.has(name))) { + changedRoot = await authenticateChangedRoot( + context, + options, + inProgressCheckpoint, + allowInProgressDiscovery, + ); + if (changedRoot === true) throw new ConsumerEpochAdvancedError(); + } + if (oldEpochError) throw oldEpochError; + const current = await readExactMetadata( + context.checkpointPath, + options.metadataMaxBytes, + (value) => validateCheckpoint(value, options.stateMaxBytes).value, + "Consumer high-water journal checkpoint", + options, + ); + if (digest(metadataBytes(current)) !== context.checkpointDigest) { + throw new Error("Consumer high-water journal checkpoint changed and fenced a paused writer."); + } + return typeof changedRoot === "string" ? changedRoot : null; +} + +async function publishImmutable({ + path, + bytes, + directory, + kind, + context, + writer, + options, + revalidate = true, + beforeLink, + inProgressCheckpoint = null, +}) { + if (revalidate) await revalidateAuthority(context, kind, options, inProgressCheckpoint); + const temporary = join(context.temporaryDirectory, temporaryName(path, kind, writer, context)); + let handle; + let linked = false; + try { + handle = await options.openFile(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); + await handle.chmod?.(0o600); + await handle.writeFile(bytes); + await handle.sync(); + await handle.close(); + handle = undefined; + await options.hooks?.afterFileSync?.({ kind, path, temporary }); + await beforeLink?.(); + if (revalidate) await revalidateAuthority(context, `${kind}-link`, options, inProgressCheckpoint); + try { + await options.linkFile(temporary, path); + linked = true; + } catch (error) { + if (error?.code !== "EEXIST") throw error; + } + if (linked) await options.hooks?.afterMetadataLink?.({ kind, path }); + await options.syncDirectory(directory); + await options.hooks?.afterMetadataDirectorySync?.({ kind, path, linked }); + return linked; + } finally { + if (handle !== undefined) await handle.close(); + await options.removeFile(temporary, { force: true }); + await options.syncDirectory(context.temporaryDirectory); + } +} + +async function publishMetadata(path, value, kind, context, writer, options, inProgressCheckpoint = null) { + const created = await publishImmutable({ + path, + bytes: metadataBytes(value), + directory: dirname(path), + kind, + context, + writer, + options, + inProgressCheckpoint, + }); + if (created) return { value, created: true }; + await revalidateAuthority(context, `${kind}-existing`, options, inProgressCheckpoint); + const existing = await readExactMetadata( + path, + options.metadataMaxBytes, + (candidate) => candidate, + "Consumer high-water lock metadata", + options, + ); + return { value: existing, created: false }; +} + +function genesisCheckpoint(statePath) { + const epochId = deterministicUuid(`pylon-consumer-journal:${statePath}`); + return { + schemaVersion: CHECKPOINT_SCHEMA_VERSION, + epoch: 1, + epochId, + previousCheckpointSha256: GENESIS_DIGEST, + previousTipSha256: GENESIS_DIGEST, + historySha256: digest(Buffer.from(`pylon-consumer-history:${digest(Buffer.from(statePath))}`)), + anchorDigest: GENESIS_DIGEST, + anchorBase64: null, + retiredEpochDirectory: null, + sourceAuthoritySha256: GENESIS_DIGEST, + sourceAuthorityTipDigest: GENESIS_DIGEST, + sourceAuthorityTipBase64: null, + }; +} + +async function scanJournalRoot( + statePath, + journalDirectory, + options, + { checkpointAnchors = null, checkpointContext = null } = {}, +) { + await secureDirectory(journalDirectory, "Consumer high-water journal directory", options); + await options.syncDirectory(journalDirectory); + const names = await options.readDirectory(journalDirectory); + if (names.length > MAX_JOURNAL_ROOT_ENTRIES + MAX_TEMPORARY_ENTRIES) { + throw new Error("Consumer high-water journal root exceeds its safe allocation bound."); + } + const checkpointEntries = []; + const removedCheckpointEntries = []; + const linkRetiredCheckpointEntries = []; + const vanishedRetainedCheckpointNames = []; + const epochEntries = []; + const temporaries = []; + let temporaryDirectorySeen = false; + for (const name of names) { + const path = join(journalDirectory, name); + if (name === TEMPORARY_DIRECTORY_NAME) { + if (temporaryDirectorySeen) throw new Error("Consumer high-water temporary namespace is duplicated."); + temporaryDirectorySeen = true; + const entry = await options.lstatEntry(path); + if (!entry.isDirectory() || entry.isSymbolicLink?.()) { + throw new Error("Consumer high-water temporary namespace must be one real directory."); + } + await secureDirectory(path, "Consumer high-water temporary directory", options); + const temporaryNames = await options.readDirectory(path); + if (temporaryNames.length > MAX_TEMPORARY_ENTRIES) { + throw new Error("Consumer high-water temporary namespace exceeds its safe allocation bound."); + } + for (const temporaryName of temporaryNames) { + const temporary = await inspectTemporary(join(path, temporaryName), options); + if (temporary) temporaries.push(temporary); + } + continue; + } + const checkpointMatch = checkpointPattern.exec(name); + if (checkpointMatch) { + const expectedSha256 = checkpointAnchors?.get(path) ?? null; + let checkpoint; + let removedDuringRead = false; + let linkRetiredBeforeRead = false; + let linkRetirementStat = null; + try { + checkpoint = await readExactMetadata( + path, + options.metadataMaxBytes, + (value) => validateCheckpoint(value, options.stateMaxBytes).value, + "Consumer high-water journal checkpoint", + options, + undefined, + expectedSha256, + ); + } catch (error) { + const authenticated = checkpointContext === null || checkpointAnchors === null + ? null + : authenticatedChangedCheckpointRead(error, checkpointContext, options, checkpointAnchors, names); + if (authenticated === null) throw error; + checkpoint = authenticated.checkpoint; + linkRetiredBeforeRead = authenticated.linkRetiredBeforeRead; + linkRetirementStat = authenticated.linkRetirementStat; + removedDuringRead = !linkRetiredBeforeRead; + } + if (checkpoint === null) { + if (checkpointContext === null || checkpointAnchors === null) { + throw new Error("Consumer high-water journal lost its current checkpoint during an authenticated scan."); + } + if (isProvisionallyRemovedContextCurrentCheckpoint(path, checkpointContext, checkpointAnchors, names)) { + checkpoint = checkpointContext.checkpoint; + removedDuringRead = true; + } else { + if (!isVanishedRetainedCheckpoint(path, checkpointContext, checkpointAnchors, names)) { + throw new Error("Consumer high-water journal lost its current checkpoint during an authenticated scan."); + } + vanishedRetainedCheckpointNames.push(name); + continue; + } + } + let checkpointStat = linkRetirementStat; + if (!removedDuringRead && checkpointStat === null) { + await options.hooks?.beforeCheckpointIdentityStat?.({ name, path }); + try { + checkpointStat = await options.lstatEntry(path); + } catch (error) { + const allowedRemoval = checkpointContext !== null && checkpointAnchors !== null && ( + isProvisionallyRemovedContextCurrentCheckpoint(path, checkpointContext, checkpointAnchors, names) || + isVanishedRetainedCheckpoint(path, checkpointContext, checkpointAnchors, names) + ); + if (error?.code !== "ENOENT" || !allowedRemoval) throw error; + removedDuringRead = true; + } + if ( + checkpointStat !== null && + (checkpointStat.isSymbolicLink?.() || !checkpointStat.isFile()) + ) throw new Error("Consumer high-water journal checkpoint must remain one regular non-symlink file."); + } + if (checkpointName(checkpoint) !== name || checkpoint.epoch !== Number(checkpointMatch[1])) { + throw new Error("Consumer high-water journal checkpoint name is malformed."); + } + const entry = { + name, + path, + checkpoint, + digest: digest(metadataBytes(checkpoint)), + removedDuringRead, + linkRetiredBeforeRead, + linkRetirementStat, + checkpointStat, + }; + checkpointEntries.push(entry); + if (removedDuringRead) removedCheckpointEntries.push(entry); + if (linkRetiredBeforeRead) linkRetiredCheckpointEntries.push(entry); + continue; + } + const epochMatch = epochPattern.exec(name); + if (epochMatch) { + epochEntries.push({ name, path, epoch: Number(epochMatch[1]), epochId: epochMatch[2] }); + continue; + } + if (name.startsWith(".")) { + const temporary = await inspectTemporary(path, options); + if (temporary?.kind !== "checkpoint") { + throw new Error("Consumer high-water journal root contains an unexpected owned temporary."); + } + if (temporary) temporaries.push(temporary); + continue; + } + throw new Error("Consumer high-water journal root contains an unexpected entry."); + } + if (!temporaryDirectorySeen) throw new Error("Consumer high-water journal lacks its exact temporary namespace."); + const checkpointNameCount = names.filter((name) => checkpointPattern.test(name)).length; + const authoritativeEntries = checkpointNameCount + epochEntries.length + 1; + if (authoritativeEntries > MAX_JOURNAL_ROOT_ENTRIES) { + throw new Error("Consumer high-water journal root exceeds its safe entry bound."); + } + checkpointEntries.sort((left, right) => left.checkpoint.epoch - right.checkpoint.epoch); + epochEntries.sort((left, right) => left.epoch - right.epoch); + if (checkpointNameCount > 2 || epochEntries.length > 2) { + throw new Error("Consumer high-water journal root contains unbounded checkpoint metadata."); + } + for (let index = 1; index < checkpointEntries.length; index += 1) { + if (checkpointEntries[index - 1].checkpoint.epoch === checkpointEntries[index].checkpoint.epoch) { + throw new Error("Consumer high-water journal contains competing checkpoints for one parent epoch."); + } + if (checkpointEntries[index - 1].checkpoint.epoch + 1 !== checkpointEntries[index].checkpoint.epoch) { + throw new Error("Consumer high-water journal checkpoints are not contiguous."); + } + } + for (let index = 1; index < epochEntries.length; index += 1) { + if (epochEntries[index - 1].epoch === epochEntries[index].epoch) { + throw new Error("Consumer high-water journal contains competing epoch directories for one parent epoch."); + } + } + const head = checkpointEntries.at(-1) ?? null; + if (head) { + const previous = checkpointEntries.at(-2); + if (previous && ( + head.checkpoint.previousCheckpointSha256 !== previous.digest || + head.checkpoint.retiredEpochDirectory !== epochName(previous.checkpoint) || + head.checkpoint.sourceAuthoritySha256 !== previous.checkpoint.sourceAuthoritySha256 || + head.checkpoint.sourceAuthorityTipDigest !== previous.checkpoint.sourceAuthorityTipDigest || + head.checkpoint.sourceAuthorityTipBase64 !== previous.checkpoint.sourceAuthorityTipBase64 || + head.checkpoint.historySha256 !== digest(Buffer.from( + `${previous.checkpoint.historySha256}:${previous.digest}:${head.checkpoint.anchorDigest}`, + )) + )) throw new Error("Consumer high-water journal checkpoint does not anchor its exact predecessor."); + } + const missingHeadEpoch = head ? !epochEntries.some((entry) => entry.name === epochName(head.checkpoint)) : false; + if (checkpointContext === null) { + for (const entry of epochEntries) { + const pathEntry = await options.lstatEntry(entry.path); + if (!pathEntry.isDirectory() || pathEntry.isSymbolicLink?.()) { + throw new Error("Consumer high-water epoch entry must be one real directory."); + } + await secureDirectory(entry.path, "Consumer high-water epoch directory", options); + } + } + return { + checkpointEntries, + removedCheckpointEntries, + linkRetiredCheckpointEntries, + vanishedRetainedCheckpointNames, + epochEntries, + temporaries, + head, + missingHeadEpoch, + rootNames: names, + }; +} + +function classifyContextCheckpointAuthority(scan, context, anchors, invalidRoot) { + const current = scan.checkpointEntries.find((entry) => entry.path === context.checkpointPath); + if (current && !isContextAnchoredCheckpoint(context, current.path, current.checkpoint, current.digest)) { + throw invalidRoot(); + } + const successors = scan.checkpointEntries.filter((entry) => entry.checkpoint.epoch > context.checkpoint.epoch); + if (successors.length === 0) { + if ( + !current || scan.head?.path !== current.path || + scan.checkpointEntries.some((entry) => { + const expectedSha256 = anchors.get(entry.path); + return expectedSha256 === undefined || + !isContextAnchoredCheckpoint(context, entry.path, entry.checkpoint, expectedSha256); + }) + ) throw invalidRoot(); + return { kind: "current", entry: current }; + } + const successor = successors[0]; + const successorEpochPath = join(context.journalDirectory, epochName(successor.checkpoint)); + if ( + successors.length !== 1 || scan.head?.path !== successor.path || + !isImmediateSuccessorCheckpoint(context, successor.checkpoint) || + scan.checkpointEntries.some((entry) => ![context.checkpointPath, successor.path].includes(entry.path)) || + scan.epochEntries.some((entry) => ![context.epochDirectory, successorEpochPath].includes(entry.path)) || + !scan.epochEntries.some((entry) => entry.path === successorEpochPath) + ) throw invalidRoot(); + return { kind: "successor", entry: successor }; +} + +async function scanAuthenticatedContextRoot(context, options) { + const anchors = contextCheckpointAnchors(context); + const scan = await scanJournalRoot(context.statePath, context.journalDirectory, options, { + checkpointAnchors: anchors, + checkpointContext: context, + }); + const invalidRoot = () => new Error( + "Consumer high-water journal root has neither its byte-exact current checkpoint nor one exact immediate successor.", + ); + const authority = classifyContextCheckpointAuthority(scan, context, anchors, invalidRoot); + await authenticateStableChangedRoot(scan, context, options, anchors, authority.entry, invalidRoot); + return { scan, authority }; +} + +async function initializeJournal( + statePath, + journalDirectory, + options, + bootstrapCheckpoint = genesisCheckpoint(statePath), + beforeCheckpointLink, +) { + let scan = await scanJournalRoot(statePath, journalDirectory, options); + if (scan.head) { + if (!scan.missingHeadEpoch) return scan; + if ( + scan.head.checkpoint.epoch !== 1 || !metadataBytes(scan.head.checkpoint).equals(metadataBytes(bootstrapCheckpoint)) || + scan.epochEntries.length !== 0 + ) throw new Error("Consumer high-water journal checkpoint lacks its exact epoch directory."); + await ensureDirectory( + join(journalDirectory, epochName(scan.head.checkpoint)), + "Consumer high-water epoch directory", + options, + ); + return scanJournalRoot(statePath, journalDirectory, options); + } + if (scan.epochEntries.length > 0) throw new Error("Consumer high-water journal contains an orphan epoch directory."); + const checkpoint = bootstrapCheckpoint; + const bootstrap = { generation: 0, token: checkpoint.epochId }; + const bootstrapContext = { + statePath, + journalDirectory, + checkpoint, + checkpointPath: join(journalDirectory, checkpointName(checkpoint)), + checkpointDigest: digest(metadataBytes(checkpoint)), + epochDirectory: join(journalDirectory, epochName(checkpoint)), + temporaryDirectory: join(journalDirectory, TEMPORARY_DIRECTORY_NAME), + }; + await publishImmutable({ + path: bootstrapContext.checkpointPath, + bytes: metadataBytes(checkpoint), + directory: journalDirectory, + kind: "checkpoint", + context: bootstrapContext, + writer: bootstrap, + options, + revalidate: false, + beforeLink: beforeCheckpointLink, + }); + await ensureDirectory(bootstrapContext.epochDirectory, "Consumer high-water epoch directory", options); + scan = await scanJournalRoot(statePath, journalDirectory, options); + if (!scan.head) throw new Error("Consumer high-water journal initialization did not publish a checkpoint."); + return scan; +} + +function contextFromHead(statePath, guardPath, journalDirectory, head) { + return { + statePath, + guardPath, + journalDirectory, + checkpoint: head.checkpoint, + checkpointPath: head.path, + checkpointDigest: head.digest, + epochDirectory: join(journalDirectory, epochName(head.checkpoint)), + temporaryDirectory: join(journalDirectory, TEMPORARY_DIRECTORY_NAME), + }; +} + +async function readProjection(context, operation, options) { + await revalidateAuthority(context, operation, options); + const bytes = await readSecureFile( + context.statePath, + options.stateMaxBytes, + "Consumer high-water state", + options, + 0, + options.hooks?.projectionRead, + ); + if (bytes === null) return { exists: false, bytes: null, sha256: null, malformed: false }; + if (bytes.length < 1) return { exists: true, bytes: null, sha256: null, malformed: true }; + return { exists: true, bytes, sha256: digest(bytes), malformed: false }; +} + +async function walkTransactions(context, options) { + await revalidateAuthority(context, "walk-transactions", options); + await options.syncDirectory(context.epochDirectory); + const entries = await options.readDirectory(context.epochDirectory); + if (entries.length > options.maxJournalEntries + MAX_TEMPORARY_ENTRIES) { + throw new Error("Consumer high-water epoch exceeds its safe allocation bound."); + } + const named = new Map(); + for (const name of entries) { + const match = transitionPattern.exec(name); + if (match) { + if (named.has(match[1])) throw new Error("Consumer high-water journal contains a duplicate transition."); + named.set(match[1], name); + if (named.size > options.maxTransactionDepth) { + throw new Error("Consumer high-water transaction chain exceeds its safe entry bound."); + } + } + } + const visited = new Set(); + let tipDigest = context.checkpoint.anchorDigest; + let tipBytes = validateCheckpoint(context.checkpoint, options.stateMaxBytes).anchorBytes; + const budget = { bytes: 0 }; + for (let depth = 0; named.has(tipDigest); depth += 1) { + if (depth >= options.maxTransactionDepth || visited.has(tipDigest)) { + throw new Error("Consumer high-water transaction chain is cyclic or exceeds its safe bound."); + } + visited.add(tipDigest); + const path = transitionPath(context, tipDigest); + await revalidateAuthority(context, "read-transition", options); + const value = await readExactMetadata( + path, + options.metadataMaxBytes, + (candidate) => validateTransaction(candidate, tipDigest, options.stateMaxBytes).value, + "Consumer high-water transaction", + options, + budget, + ); + const validated = validateTransaction(value, tipDigest, options.stateMaxBytes); + tipDigest = value.candidateDigest; + tipBytes = validated.candidateBytes; + } + if (visited.size !== named.size) throw new Error("Consumer high-water transaction chain contains an unreachable transition."); + return { tipDigest, tipBytes, length: visited.size }; +} + +function isProjectionReplacementTransient(error) { + return error?.code === "ENOENT" || error?.message === "Consumer high-water state changed while it was read."; +} + +function isCommitHelperReplacementTransient(error) { + return isProjectionReplacementTransient(error) || error instanceof ConsumerEpochAdvancedError; +} + +async function repairProjection(context, initialTip, options, writer = options.activeWriter) { + let tip = initialTip; + for (let attempt = 0; attempt < PROJECTION_RETRY_LIMIT; attempt += 1) { + if (tip.tipBytes === null) return tip; + try { + const projection = await readProjection(context, "projection-read", options); + if (projection.sha256 !== tip.tipDigest) { + await options.hooks?.beforeProjectionWrite?.({ tipDigest: tip.tipDigest }); + await revalidateAuthority(context, "projection-write", options); + const temporary = join(context.temporaryDirectory, temporaryName(context.statePath, "projection", writer, context)); + let handle; + try { + handle = await options.openFile(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); + await handle.chmod?.(0o600); + await handle.writeFile(tip.tipBytes); + await handle.sync(); + await handle.close(); + handle = undefined; + await options.hooks?.afterProjectionFileSync?.({ tipDigest: tip.tipDigest, temporary }); + await revalidateAuthority(context, "projection-rename", options); + await options.renameFile(temporary, context.statePath); + await options.hooks?.afterProjectionRename?.({ tipDigest: tip.tipDigest }); + await options.syncDirectory(context.temporaryDirectory); + await options.syncDirectory(dirname(context.statePath)); + await options.hooks?.afterProjectionDirectorySync?.({ tipDigest: tip.tipDigest }); + } finally { + if (handle !== undefined) await handle.close(); + await options.removeFile(temporary, { force: true }); + await options.syncDirectory(context.temporaryDirectory); + } + } + } catch (error) { + if (!isProjectionReplacementTransient(error)) throw error; + await revalidateAuthority(context, "projection-retry-authentication", options); + tip = await walkTransactions(context, options); + continue; + } + const latest = await walkTransactions(context, options); + if (latest.tipDigest === tip.tipDigest) return latest; + tip = latest; + } + throw new Error("Consumer high-water projection could not catch up with its immutable transaction tip."); +} + +async function publishTransition(context, transaction, claim, options) { + validateTransaction(transaction, transaction.baseDigest, options.stateMaxBytes); + const path = transitionPath(context, transaction.baseDigest); + const result = await publishMetadata(path, transaction, "transition", context, claim, options); + const existing = validateTransaction(result.value, transaction.baseDigest, options.stateMaxBytes).value; + if (!metadataBytes(existing).equals(metadataBytes(transaction))) { + throw new Error("Consumer high-water transaction lost its immutable base-digest compare-and-set."); + } +} + +async function scanEpoch(context, options) { + const discoveredNextEpoch = await revalidateAuthority( + context, + "scan-claims", + options, + options.inProgressCheckpoint ?? null, + true, + ); + await options.syncDirectory(context.epochDirectory); + const names = await options.readDirectory(context.epochDirectory); + if (names.length > options.maxJournalEntries + MAX_TEMPORARY_ENTRIES) { + throw new Error("Consumer high-water epoch exceeds its safe allocation bound."); + } + const claimContentNames = new Map(); + const claimIndexNames = new Map(); + const legacyClaimNames = new Map(); + const heartbeatNames = new Map(); + const terminalNames = new Map(); + const appliedNames = new Map(); + const temporaries = []; + let authoritativeEntryCount = 0; + for (const name of names) { + let match; + if ((match = claimPattern.exec(name))) { + const generation = Number(match[1]); + const contents = claimContentNames.get(generation) ?? new Map(); + contents.set(match[2], name); + claimContentNames.set(generation, contents); + authoritativeEntryCount += 1; + } else if ((match = claimIndexPattern.exec(name))) { + const generation = Number(match[1]); + if (claimIndexNames.has(generation)) throw new Error("Consumer high-water lock contains a duplicate claim index."); + claimIndexNames.set(generation, name); + authoritativeEntryCount += 1; + } else if ((match = undigestedClaimPattern.exec(name))) { + const generation = Number(match[1]); + if (legacyClaimNames.has(generation)) throw new Error("Consumer high-water lock contains a duplicate legacy claim."); + legacyClaimNames.set(generation, name); + authoritativeEntryCount += 1; + } else if ((match = heartbeatPattern.exec(name))) { + heartbeatNames.set(`${Number(match[1])}:${match[2]}`, name); + authoritativeEntryCount += 1; + } else if ((match = terminalPattern.exec(name))) { + terminalNames.set(`${Number(match[1])}:${match[2]}`, name); + authoritativeEntryCount += 1; + } else if ((match = appliedPattern.exec(name))) { + appliedNames.set(`${Number(match[1])}:${match[2]}`, name); + authoritativeEntryCount += 1; + } else if (transitionPattern.test(name)) { + // Validated by the transaction walk before any state decision. + authoritativeEntryCount += 1; + } else if (name.startsWith(".")) { + const temporary = await inspectTemporary(join(context.epochDirectory, name), options); + if (temporary && ["checkpoint", "projection", "legacy-guard"].includes(temporary.kind)) { + throw new Error("Consumer high-water epoch contains an unexpected owned temporary."); + } + if (temporary) temporaries.push(temporary); + } else { + throw new Error("Consumer high-water epoch contains a malformed or unexpected entry."); + } + } + if (authoritativeEntryCount > options.maxJournalEntries) { + throw new Error("Consumer high-water epoch exceeds its safe entry bound."); + } + const budget = { bytes: 0 }; + const claims = []; + const byKey = new Map(); + const referencedClaimContents = new Set(); + const generations = new Set([...claimIndexNames.keys(), ...legacyClaimNames.keys()]); + for (const generation of [...generations].sort((left, right) => left - right)) { + if (claimIndexNames.has(generation) && legacyClaimNames.has(generation)) { + throw new Error("Consumer high-water lock contains competing indexed and legacy claims."); + } + let claim; + if (claimIndexNames.has(generation)) { + const index = await readExactMetadata( + join(context.epochDirectory, claimIndexNames.get(generation)), + options.metadataMaxBytes, + (value) => validateClaimIndex(value, generation), + "Consumer high-water claim index", + options, + budget, + ); + const name = claimContentNames.get(generation)?.get(index.claimSha256); + if (!name) throw new Error("Consumer high-water claim index lacks its exact digest-bound claim bytes."); + referencedClaimContents.add(name); + claim = await readExactMetadata( + join(context.epochDirectory, name), + options.metadataMaxBytes, + (value) => validateClaim(value, context, options.stateMaxBytes), + "Consumer high-water operation claim", + options, + budget, + index.claimSha256, + ); + if ( + claim.generation !== generation || digest(metadataBytes(claim)) !== index.claimSha256 || + name !== basename(claimPath(context, claim)) + ) throw new Error("Consumer high-water claim index differs from its exact canonical claim bytes."); + } else { + const name = legacyClaimNames.get(generation); + claim = await readExactMetadata( + join(context.epochDirectory, name), + options.metadataMaxBytes, + (value) => validateClaim(value, context, options.stateMaxBytes), + "Consumer high-water legacy operation claim", + options, + budget, + ); + if (claim.generation !== generation || name !== `claim-${generationName(generation)}.json`) { + throw new Error("Consumer high-water legacy claim name differs from its exact generation."); + } + } + claims.push(claim); + byKey.set(`${generation}:${claim.token}`, claim); + } + for (const [generation, contents] of [...claimContentNames].sort((left, right) => left[0] - right[0])) { + for (const [claimSha256, name] of [...contents].sort((left, right) => left[0].localeCompare(right[0]))) { + if (referencedClaimContents.has(name)) continue; + const claim = await readExactMetadata( + join(context.epochDirectory, name), + options.metadataMaxBytes, + (value) => validateClaim(value, context, options.stateMaxBytes), + "Consumer high-water unindexed claim content", + options, + budget, + ); + if ( + claim.generation !== generation || digest(metadataBytes(claim)) !== claimSha256 || + name !== basename(claimPath(context, claim)) + ) throw new Error("Consumer high-water unindexed claim content differs from its exact canonical bytes."); + } + } + if (claims.length > MAX_OPERATION_GENERATIONS) throw new Error("Consumer high-water operation generation bound is exhausted."); + for (let index = 0; index < claims.length; index += 1) { + if (claims[index].generation !== index + 1) throw new Error("Consumer high-water lock generations are not contiguous."); + } + for (const [key, name] of heartbeatNames) { + const claim = byKey.get(key); + if (!claim) throw new Error("Consumer high-water epoch contains an orphan heartbeat entry."); + await readExactMetadata( + join(context.epochDirectory, name), + options.metadataMaxBytes, + (value) => validateHeartbeat(value, claim), + "Consumer high-water lock heartbeat", + options, + budget, + ); + } + const terminals = new Map(); + for (const [key, name] of terminalNames) { + const claim = byKey.get(key); + if (!claim) throw new Error("Consumer high-water epoch contains an orphan terminal entry."); + terminals.set(key, await readExactMetadata( + join(context.epochDirectory, name), + options.metadataMaxBytes, + (value) => validateTerminal(value, claim, options.stateMaxBytes), + "Consumer high-water lock terminal marker", + options, + budget, + )); + } + const appliedClaims = new Set(); + for (const [key, name] of appliedNames) { + const claim = byKey.get(key); + const terminal = terminals.get(key); + if (!claim || !terminal) throw new Error("Consumer high-water epoch contains an orphan applied entry."); + await readExactMetadata( + join(context.epochDirectory, name), + options.metadataMaxBytes, + (value) => validateApplied(value, claim, terminal), + "Consumer high-water lock applied marker", + options, + budget, + ); + appliedClaims.add(key); + } + for (const claim of claims.slice(0, -1)) { + const key = `${claim.generation}:${claim.token}`; + const terminal = terminals.get(key); + if (claim.type === "rotation" || !terminal || (terminal.outcome === "commit" && !appliedClaims.has(key))) { + throw new Error("Consumer high-water operation generations crossed an unresolved earlier slot."); + } + } + if (discoveredNextEpoch !== null) { + const latest = claims.at(-1); + if (latest?.type !== "rotation") { + throw new Error("Consumer high-water in-progress next epoch lacks its exact published rotation intent."); + } + const intent = validateRotationIntent(latest.intent, context, options.stateMaxBytes); + if ( + !isImmediateSuccessorCheckpoint(context, intent.checkpoint) || + discoveredNextEpoch !== join(context.journalDirectory, epochName(intent.checkpoint)) + ) throw new Error("Consumer high-water in-progress next epoch differs from its exact published rotation intent."); + if (await authenticateChangedRoot(context, options, intent.checkpoint)) { + throw new ConsumerEpochAdvancedError(); + } + } + return { claims, terminals, temporaries }; +} + +async function readTerminal(context, claim, options) { + await revalidateAuthority(context, "read-terminal", options); + return readExactMetadata( + terminalPath(context, claim), + options.metadataMaxBytes, + (value) => validateTerminal(value, claim, options.stateMaxBytes), + "Consumer high-water lock terminal marker", + options, + ); +} + +async function readHeartbeat(context, claim, options) { + await revalidateAuthority(context, "read-heartbeat", options); + const heartbeat = await readExactMetadata( + heartbeatPath(context, claim), + options.metadataMaxBytes, + (value) => validateHeartbeat(value, claim), + "Consumer high-water lock heartbeat", + options, + ); + return heartbeat ?? { ...claim, refreshedAtMs: claim.createdAtMs }; +} + +async function publishTerminal(context, claim, wanted, options) { + const result = await publishMetadata( + terminalPath(context, claim), + wanted, + `terminal-${wanted.outcome}`, + context, + claim, + options, + ); + return validateTerminal(result.value, claim, options.stateMaxBytes); +} + +async function refreshHeartbeat(context, claim, options) { + if (await readTerminal(context, claim, options) !== null) return false; + const value = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation: claim.generation, + token: claim.token, + refreshedAtMs: options.now(), + }; + const path = heartbeatPath(context, claim); + await revalidateAuthority(context, "heartbeat", options); + const temporary = join(context.temporaryDirectory, temporaryName(path, "heartbeat", claim, context)); + let handle; + try { + handle = await options.openFile(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); + await handle.chmod?.(0o600); + await handle.writeFile(metadataBytes(value)); + await handle.sync(); + await handle.close(); + handle = undefined; + if (await readTerminal(context, claim, options) !== null) return false; + await revalidateAuthority(context, "heartbeat-rename", options); + await options.renameFile(temporary, path); + await options.syncDirectory(context.temporaryDirectory); + await options.syncDirectory(context.epochDirectory); + return true; + } finally { + if (handle !== undefined) await handle.close(); + await options.removeFile(temporary, { force: true }); + await options.syncDirectory(context.temporaryDirectory); + } +} + +function defaultHeartbeatScheduler({ interval, beat }) { + let stopped = false; + let timer; + let pending = Promise.resolve(); + const arm = () => { + if (stopped) return; + timer = setTimeout(() => { + pending = beat().catch(() => false).finally(arm); + }, interval); + timer.unref?.(); + }; + arm(); + return async () => { + stopped = true; + clearTimeout(timer); + await pending; + }; +} + +async function publishApplied(context, claim, terminal, options) { + const value = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation: claim.generation, + token: claim.token, + terminalSha256: digest(metadataBytes(terminal)), + }; + const result = await publishMetadata(appliedPath(context, claim), value, "applied", context, claim, options); + validateApplied(result.value, claim, terminal); + await options.hooks?.afterApplied?.({ claim, terminal }); +} + +async function readApplied(context, claim, terminal, options) { + await revalidateAuthority(context, "read-applied", options); + return readExactMetadata( + appliedPath(context, claim), + options.metadataMaxBytes, + (value) => validateApplied(value, claim, terminal), + "Consumer high-water lock applied marker", + options, + ); +} + +async function authenticateAppliedCommit(context, claim, terminal, options) { + for (const transaction of terminal.transactions) { + await revalidateAuthority(context, "read-applied-transition", options); + const existing = await readExactMetadata( + transitionPath(context, transaction.baseDigest), + options.metadataMaxBytes, + (value) => validateTransaction(value, transaction.baseDigest, options.stateMaxBytes).value, + "Consumer high-water applied transaction", + options, + ); + if (existing === null || !metadataBytes(existing).equals(metadataBytes(transaction))) { + throw new Error("Consumer high-water applied marker does not authenticate its exact immutable transaction chain."); + } + } + const tip = await walkTransactions(context, options); + const terminalDigest = terminal.transactions.at(-1).candidateDigest; + if (tip.tipDigest !== terminalDigest) { + await revalidateAuthority(context, "read-applied-continuation", options); + const continuation = await readExactMetadata( + transitionPath(context, terminalDigest), + options.metadataMaxBytes, + (value) => validateTransaction(value, terminalDigest, options.stateMaxBytes).value, + "Consumer high-water applied transaction continuation", + options, + ); + if (continuation === null) { + throw new Error("Consumer high-water applied marker does not authenticate its exact terminal digest."); + } + } + const repairedTip = await repairProjection(context, tip, options, claim); + const projection = await readProjection(context, "read-applied-projection", options); + if (projection.malformed || projection.sha256 !== repairedTip.tipDigest) { + throw new Error("Consumer high-water applied marker does not authenticate the current immutable tip and projection."); + } +} + +async function finishCommitAtAuthenticatedDescendant(context, options) { + const { authority } = await scanAuthenticatedContextRoot(context, options); + if (authority.kind !== "successor") return false; + const descendant = contextFromHead(context.statePath, context.guardPath, context.journalDirectory, authority.entry); + const tip = await walkTransactions(descendant, options); + await repairProjection(descendant, tip, options, { + generation: 0, + token: descendant.checkpoint.epochId, + type: "rotation", + }); + return true; +} + +async function finishCommit(context, claim, terminal, options) { + for (let attempt = 0; attempt < PROJECTION_RETRY_LIMIT; attempt += 1) { + try { + const applied = await readApplied(context, claim, terminal, options); + if (applied !== null) { + await authenticateAppliedCommit(context, claim, terminal, options); + return; + } + for (const transaction of terminal.transactions) await publishTransition(context, transaction, claim, options); + const tip = await walkTransactions(context, options); + await repairProjection(context, tip, options, claim); + await publishApplied(context, claim, terminal, options); + return; + } catch (error) { + if (!isCommitHelperReplacementTransient(error)) throw error; + try { + await revalidateAuthority(context, "finish-commit-retry-authentication", options); + await walkTransactions(context, options); + } catch (authenticationError) { + if (!isCommitHelperReplacementTransient(authenticationError)) throw authenticationError; + if (await finishCommitAtAuthenticatedDescendant(context, options)) return; + throw authenticationError; + } + } + } + throw new Error("Consumer high-water commit helper could not converge after bounded projection replacement retries."); +} + +function rotationCheckpoint(context, tip) { + const epochId = deterministicUuid( + `pylon-consumer-rotation-v2:${context.checkpointDigest}:${tip.tipDigest}`, + ); + const checkpoint = { + schemaVersion: CHECKPOINT_SCHEMA_VERSION, + epoch: context.checkpoint.epoch + 1, + epochId, + previousCheckpointSha256: context.checkpointDigest, + previousTipSha256: tip.tipDigest, + historySha256: digest(Buffer.from( + `${context.checkpoint.historySha256}:${context.checkpointDigest}:${tip.tipDigest}`, + )), + anchorDigest: tip.tipDigest, + anchorBase64: tip.tipBytes === null ? null : tip.tipBytes.toString("base64"), + retiredEpochDirectory: basename(context.epochDirectory), + sourceAuthoritySha256: context.checkpoint.sourceAuthoritySha256, + sourceAuthorityTipDigest: context.checkpoint.sourceAuthorityTipDigest, + sourceAuthorityTipBase64: context.checkpoint.sourceAuthorityTipBase64, + }; + validateCheckpoint(checkpoint, Number.MAX_SAFE_INTEGER); + return checkpoint; +} + +function rotationIntentFor(context, tip) { + return { + schemaVersion: ROTATION_INTENT_SCHEMA_VERSION, + epoch: context.checkpoint.epoch, + epochId: context.checkpoint.epochId, + checkpointSha256: context.checkpointDigest, + tipSha256: tip.tipDigest, + checkpoint: rotationCheckpoint(context, tip), + }; +} + +function rotationClaimFor(context, generation, tip) { + const intent = rotationIntentFor(context, tip); + return { + schemaVersion: LOCK_SCHEMA_VERSION, + generation, + token: intent.checkpoint.epochId, + type: "rotation", + intent, + }; +} + +async function effectiveTip(context, options) { + const chain = await walkTransactions(context, options); + if (chain.tipBytes !== null) return chain; + const projection = await readProjection(context, "rotation-legacy-state-read", options); + if (projection.malformed) throw new Error("Consumer high-water rotation cannot authenticate its legacy projection anchor."); + if (projection.bytes === null) return chain; + return { tipDigest: digest(projection.bytes), tipBytes: projection.bytes, length: chain.length }; +} + +async function scanRotationPublicationSet(context, checkpoint, options, requirePublished) { + const { scan, authority } = await scanAuthenticatedContextRoot(context, options); + const nextCheckpointPath = join(context.journalDirectory, checkpointName(checkpoint)); + const nextEpochPath = join(context.journalDirectory, epochName(checkpoint)); + const allowedCheckpointPaths = new Set([context.checkpointPath, nextCheckpointPath]); + const allowedEpochPaths = new Set([context.epochDirectory, nextEpochPath]); + const currentCheckpoint = scan.checkpointEntries.find((entry) => entry.path === context.checkpointPath); + const currentEpoch = scan.epochEntries.find((entry) => entry.path === context.epochDirectory); + const published = scan.checkpointEntries.find((entry) => entry.path === nextCheckpointPath); + if ( + scan.checkpointEntries.some((entry) => !allowedCheckpointPaths.has(entry.path)) || + scan.epochEntries.some((entry) => !allowedEpochPaths.has(entry.path)) || + (currentCheckpoint && currentCheckpoint.digest !== context.checkpointDigest) || + !scan.epochEntries.some((entry) => entry.path === nextEpochPath) || + (!requirePublished && (!currentCheckpoint || !currentEpoch)) || + (published === undefined + ? authority.kind !== "current" + : authority.kind !== "successor" || authority.entry.path !== published.path) + ) throw new Error("Consumer high-water rotation found a competing root or epoch publication."); + if (published && !metadataBytes(published.checkpoint).equals(metadataBytes(checkpoint))) { + throw new Error("Consumer high-water rotation found a competing checkpoint for the same epoch."); + } + if (requirePublished && (!published || scan.head?.path !== nextCheckpointPath || scan.missingHeadEpoch)) { + throw new Error("Consumer high-water rotation checkpoint did not become the unique complete journal head."); + } + return scan; +} + +async function finishRotationCheckpoint(context, checkpoint, writer, options) { + validateCheckpoint(checkpoint, options.stateMaxBytes); + if ( + checkpoint.epoch !== context.checkpoint.epoch + 1 || + checkpoint.previousCheckpointSha256 !== context.checkpointDigest || + checkpoint.retiredEpochDirectory !== basename(context.epochDirectory) || + checkpoint.sourceAuthoritySha256 !== context.checkpoint.sourceAuthoritySha256 || + checkpoint.sourceAuthorityTipDigest !== context.checkpoint.sourceAuthorityTipDigest || + checkpoint.sourceAuthorityTipBase64 !== context.checkpoint.sourceAuthorityTipBase64 || + checkpoint.historySha256 !== digest(Buffer.from( + `${context.checkpoint.historySha256}:${context.checkpointDigest}:${checkpoint.anchorDigest}`, + )) + ) throw new Error("Consumer high-water rotation does not anchor the exact current epoch."); + const tip = await effectiveTip(context, options); + const anchorBytes = validateCheckpoint(checkpoint, options.stateMaxBytes).anchorBytes; + if ( + checkpoint.previousTipSha256 !== tip.tipDigest || checkpoint.anchorDigest !== tip.tipDigest || + (anchorBytes === null ? tip.tipBytes !== null : !anchorBytes.equals(tip.tipBytes)) + ) throw new Error("Consumer high-water rotation does not anchor the exact immutable tip."); + const nextEpoch = join(context.journalDirectory, epochName(checkpoint)); + await ensureDirectory(nextEpoch, "Consumer high-water epoch directory", options); + await options.hooks?.afterRotationEpochSync?.({ checkpoint: structuredClone(checkpoint), nextEpoch }); + await secureDirectory(nextEpoch, "Consumer high-water next epoch directory", options); + await options.syncDirectory(nextEpoch); + if ((await options.readDirectory(nextEpoch)).length !== 0) { + throw new Error("Consumer high-water rotation found a competing next-epoch directory for the same parent."); + } + const nextPath = join(context.journalDirectory, checkpointName(checkpoint)); + await publishImmutable({ + path: nextPath, + bytes: metadataBytes(checkpoint), + directory: context.journalDirectory, + kind: "checkpoint", + context, + writer, + options, + inProgressCheckpoint: checkpoint, + beforeLink: () => scanRotationPublicationSet(context, checkpoint, options, false), + }); + await options.hooks?.afterRotationCheckpoint?.({ checkpoint: structuredClone(checkpoint), nextPath }); + await scanRotationPublicationSet(context, checkpoint, options, true); +} + +async function resolveLatestOperation(context, claim, options) { + if (claim.type === "rotation") { + await helpRotationOperation(context, claim, options); + return "rotated"; + } + const terminal = await readTerminal(context, claim, options); + if (terminal?.outcome === "commit") { + await finishCommit(context, claim, terminal, options); + return "resolved"; + } + if (terminal !== null) return "resolved"; + const heartbeat = await readHeartbeat(context, claim, options); + if (options.now() - heartbeat.refreshedAtMs < options.stale) return "active"; + await options.hooks?.afterObserveStale?.({ claim, heartbeat }); + const retired = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation: claim.generation, + token: claim.token, + outcome: "retired", + }; + const decision = await publishTerminal(context, claim, retired, options); + await options.hooks?.afterRetire?.({ claim, decision }); + if (decision.outcome === "commit") await finishCommit(context, claim, decision, options); + return "resolved"; +} + +function operationIdentity(claim) { + return claim ? `${claim.generation}:${claim.token}:${claim.type}` : null; +} + +function sameOperationClaim(left, right) { + return left === null + ? right === null + : right !== null && operationIdentity(left) === operationIdentity(right) && metadataBytes(left).equals(metadataBytes(right)); +} + +async function resolveOperationFrontier(context, options) { + const initial = await scanEpoch(context, options); + const latest = initial.claims.at(-1) ?? null; + if (latest) { + const outcome = await resolveLatestOperation(context, latest, options); + if (outcome === "rotated") return { rotated: true }; + if (outcome === "active") return { active: true }; + } + const scan = await scanEpoch(context, options); + if (!sameOperationClaim(scan.claims.at(-1) ?? null, latest)) return { retry: true }; + return { scan, frontier: latest, rotated: false, active: false }; +} + +function inProgressCheckpointForClaim(context, claim, options) { + if (claim.type !== "rotation") return null; + const validatedClaim = validateClaim(claim, context, options.stateMaxBytes); + const intent = validateRotationIntent(validatedClaim.intent, context, options.stateMaxBytes); + const { anchorBytes } = validateCheckpoint(intent.checkpoint, options.stateMaxBytes); + const expectedClaim = rotationClaimFor(context, validatedClaim.generation, { + tipDigest: intent.tipSha256, + tipBytes: anchorBytes, + }); + if (!metadataBytes(validatedClaim).equals(metadataBytes(expectedClaim))) { + throw new Error("Consumer high-water rotation claim does not match its exact authenticated intent."); + } + return intent.checkpoint; +} + +async function tryPublishClaim(context, claim, options) { + const inProgressCheckpoint = inProgressCheckpointForClaim(context, claim, options); + const contentPath = claimPath(context, claim); + const contentResult = await publishMetadata( + contentPath, + claim, + "claim", + context, + claim, + options, + inProgressCheckpoint, + ); + const existingClaim = validateClaim(contentResult.value, context, options.stateMaxBytes); + if (!metadataBytes(existingClaim).equals(metadataBytes(claim))) { + throw new Error("Consumer high-water claim content lost its exact digest-bound publication."); + } + const index = claimIndexFor(claim); + const indexResult = await publishMetadata( + claimIndexPath(context, claim.generation), + index, + "claim-index", + context, + claim, + options, + inProgressCheckpoint, + ); + const existingIndex = validateClaimIndex(indexResult.value, claim.generation); + if (!metadataBytes(existingIndex).equals(metadataBytes(index))) return false; + return indexResult.created; +} + +async function tryCreateNormalClaim(context, generation, options) { + const claim = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation, + token: randomUUID(), + type: "normal", + ownerPid: process.pid, + createdAtMs: options.now(), + }; + if (!(await tryPublishClaim(context, claim, options))) return null; + const heartbeat = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation, + token: claim.token, + refreshedAtMs: claim.createdAtMs, + }; + await publishMetadata(heartbeatPath(context, claim), heartbeat, "initial-heartbeat", context, claim, options); + await options.hooks?.afterClaim?.({ claim }); + return claim; +} + +async function tryCreateRotationClaim(context, generation, tip, options) { + const claim = rotationClaimFor(context, generation, tip); + await options.hooks?.beforeRotationDecision?.({ intent: structuredClone(claim.intent), claim: structuredClone(claim) }); + if (!(await tryPublishClaim(context, claim, options))) return null; + await options.hooks?.afterRotationIntent?.({ intent: structuredClone(claim.intent), claim: structuredClone(claim) }); + return claim; +} + +async function acquireNormalOperation(context, options) { + for (;;) { + const frontier = await resolveOperationFrontier(context, options); + if (frontier.rotated) return { rotated: true }; + if (frontier.active) throw new Error(`Consumer high-water state is actively locked: ${context.journalDirectory}`); + if (frontier.retry) continue; + const nextGeneration = (frontier.scan.claims.at(-1)?.generation ?? 0) + 1; + if (nextGeneration > options.maxLockGenerations) { + throw new Error("Consumer high-water claim epoch is exhausted; run the consumer journal rotation command."); + } + const confirmation = await scanEpoch(context, options); + if (!sameOperationClaim(confirmation.claims.at(-1) ?? null, frontier.frontier)) continue; + const claim = await tryCreateNormalClaim(context, nextGeneration, options); + if (!claim) continue; + const afterClaim = await scanEpoch(context, options); + if (!sameOperationClaim(afterClaim.claims.at(-1) ?? null, claim)) { + throw new Error("Consumer high-water normal operation did not remain the unique latest slot."); + } + return { claim, temporaries: afterClaim.temporaries, rotated: false }; + } +} + +function temporaryIsFenced(temporary, context, writer) { + if (temporary.epochId !== context.checkpoint.epochId) return true; + if (writer.generation === 0) { + return temporary.generation !== 0 || temporary.token !== writer.token; + } + if (temporary.generation === 0 || temporary.generation < writer.generation) return true; + return temporary.generation === writer.generation && temporary.token !== writer.token; +} + +function temporaryBelongsToRetiredClaim(temporary, context, epochAuthority) { + if (!epochAuthority || temporary.epochId !== context.checkpoint.epochId || temporary.generation === 0) return false; + const claim = epochAuthority.claims.find((candidate) => ( + candidate.generation === temporary.generation && candidate.token === temporary.token + )); + return claim !== undefined && epochAuthority.terminals.has(`${claim.generation}:${claim.token}`); +} + +function temporaryProcessIsAlive(temporary, options) { + try { + options.processKill(temporary.pid, 0); + return true; + } catch (error) { + if (error?.code === "ESRCH") return false; + if (error?.code === "EPERM") return true; + throw error; + } +} + +async function cleanupAuthority( + context, + writer, + rootScan, + epochTemporaries, + options, + requireQuiescent, + epochAuthority = null, + allowedNextEpoch = null, +) { + await revalidateAuthority(context, "cleanup", options); + const candidatesByPath = new Map( + [...rootScan.temporaries, ...epochTemporaries].map((temporary) => [temporary.path, temporary]), + ); + const parentNames = await options.readDirectory(dirname(context.statePath)); + const targetDigests = new Set([digest(Buffer.from(resolve(context.statePath))), digest(Buffer.from(resolve(context.guardPath)))]); + for (const name of parentNames) { + if (!name.startsWith(".pylon-consumer-tmp-v1-")) continue; + const temporary = await inspectTemporary(join(dirname(context.statePath), name), options); + if (!temporary || !targetDigests.has(temporary.targetSha256)) continue; + const expectedKind = temporary.targetSha256 === digest(Buffer.from(resolve(context.statePath))) + ? "projection" + : "legacy-guard"; + if (temporary.kind !== expectedKind) { + throw new Error("Consumer high-water state directory contains an unexpected owned temporary."); + } + candidatesByPath.set(temporary.path, temporary); + } + for (const temporary of candidatesByPath.values()) { + const fenced = temporaryIsFenced(temporary, context, writer); + if (!fenced && temporary.token !== writer.token) { + throw new Error("Consumer high-water journal contains a live or future owned temporary."); + } + if (!fenced) { + if (!requireQuiescent) continue; + if (temporaryProcessIsAlive(temporary, options)) { + if (writer.type === "rotation" && temporary.generation === writer.generation && temporary.token === writer.token) { + continue; + } + throw new Error("Consumer high-water journal rotation operation is pending until every prior owned temporary writer quiesces."); + } + await options.removeFile(temporary.path, { force: true }); + await options.syncDirectory(dirname(temporary.path)); + continue; + } + const retiredClaimTemporary = writer.generation === 0 && + temporaryBelongsToRetiredClaim(temporary, context, epochAuthority); + if (!retiredClaimTemporary && temporaryProcessIsAlive(temporary, options)) { + if (requireQuiescent) { + throw new Error("Consumer high-water journal rotation operation is pending until every prior owned temporary writer quiesces."); + } + continue; + } + await options.removeFile(temporary.path, { force: true }); + await options.syncDirectory(dirname(temporary.path)); + } + let retiredEpochDeferred = false; + for (const epoch of rootScan.epochEntries) { + if (epoch.name === basename(context.epochDirectory)) continue; + if (allowedNextEpoch !== null && epoch.name === allowedNextEpoch) continue; + if (epoch.name !== context.checkpoint.retiredEpochDirectory) { + throw new Error("Consumer high-water journal contains an orphan epoch directory."); + } + let retiredNames; + try { + retiredNames = await options.readDirectory(epoch.path); + } catch (error) { + if (error?.code === "ENOENT") continue; + throw error; + } + if (retiredNames.length > options.maxJournalEntries + MAX_TEMPORARY_ENTRIES) { + throw new Error("Consumer high-water retired epoch exceeds its safe allocation bound."); + } + const retiredTemporaries = []; + for (const name of retiredNames) { + const path = join(epoch.path, name); + let entry; + try { + entry = await options.lstatEntry(path); + } catch (error) { + if (error?.code === "ENOENT") continue; + throw error; + } + if (entry.isSymbolicLink?.() || (!entry.isFile() && !entry.isDirectory())) { + throw new Error("Consumer high-water retired epoch contains an unsafe entry."); + } + if (name.startsWith(".")) { + const temporary = await inspectTemporary(path, options); + if (temporary) retiredTemporaries.push(temporary); + } else if ( + !claimPattern.test(name) && !claimIndexPattern.test(name) && !undigestedClaimPattern.test(name) && + !heartbeatPattern.test(name) && + !terminalPattern.test(name) && + !appliedPattern.test(name) && !transitionPattern.test(name) + ) { + throw new Error("Consumer high-water retired epoch contains an unexpected entry."); + } else if (!entry.isFile()) { + throw new Error("Consumer high-water retired epoch metadata must be regular files."); + } + } + if (retiredTemporaries.some((temporary) => temporaryProcessIsAlive(temporary, options))) { + if (requireQuiescent) { + throw new Error("Consumer high-water journal rotation operation is pending until every retired temporary writer quiesces."); + } + retiredEpochDeferred = true; + continue; + } + await options.removeFile(epoch.path, { recursive: true, force: true }); + await options.syncDirectory(context.journalDirectory); + } + for (const entry of rootScan.checkpointEntries) { + if (entry.path === context.checkpointPath) continue; + if ( + entry.digest !== context.checkpoint.previousCheckpointSha256 || + epochName(entry.checkpoint) !== context.checkpoint.retiredEpochDirectory + ) throw new Error("Consumer high-water journal contains an orphan checkpoint entry."); + if (retiredEpochDeferred) continue; + await options.removeFile(entry.path, { force: true }); + await options.syncDirectory(context.journalDirectory); + } + const { scan: final, authority: finalAuthority } = await scanAuthenticatedContextRoot(context, options); + const allowedCheckpoints = retiredEpochDeferred ? 2 : 1; + const expectedEpochs = new Set([context.epochDirectory]); + if (retiredEpochDeferred) { + expectedEpochs.add(join(context.journalDirectory, context.checkpoint.retiredEpochDirectory)); + } + if ( + allowedNextEpoch !== null && + final.epochEntries.some((entry) => entry.name === allowedNextEpoch) + ) expectedEpochs.add(join(context.journalDirectory, allowedNextEpoch)); + if ( + finalAuthority.kind !== "current" || + final.checkpointEntries.length !== allowedCheckpoints || final.epochEntries.length !== expectedEpochs.size || + final.temporaries.some((temporary) => !temporaryProcessIsAlive(temporary, options)) || + final.head?.path !== context.checkpointPath || + final.epochEntries.some((entry) => !expectedEpochs.has(entry.path)) + ) throw new Error("Consumer high-water journal did not converge to one bounded current epoch."); +} + +async function helpRotationOperation(context, claim, options) { + if (claim.type !== "rotation") throw new Error("Consumer high-water rotation helper requires one rotation operation slot."); + const intent = validateRotationIntent(claim.intent, context, options.stateMaxBytes); + const helperOptions = { ...options, inProgressCheckpoint: intent.checkpoint }; + const completedBeforeHelp = await completedRotationResult(context, intent, helperOptions).catch(() => null); + if (completedBeforeHelp) return true; + try { + const scan = await scanEpoch(context, helperOptions); + const latest = scan.claims.at(-1); + if (operationIdentity(latest) !== operationIdentity(claim) || !metadataBytes(latest).equals(metadataBytes(claim))) { + throw new Error("Consumer high-water rotation operation is not the unique latest slot."); + } + const tip = await effectiveTip(context, helperOptions); + if (tip.tipDigest !== intent.tipSha256) { + throw new Error("Consumer high-water rotation operation no longer matches its exact authoritative tip."); + } + const { scan: rootScan, authority: rootAuthority } = await scanAuthenticatedContextRoot(context, helperOptions); + if (rootAuthority.kind !== "current") { + throw new Error("Consumer high-water rotation helper lost its exact current checkpoint authority."); + } + const nextEpochName = epochName(intent.checkpoint); + await cleanupAuthority( + context, + claim, + rootScan, + scan.temporaries, + helperOptions, + true, + scan, + nextEpochName, + ); + await finishRotationCheckpoint(context, intent.checkpoint, claim, helperOptions); + return true; + } catch (error) { + const completed = await completedRotationResult(context, intent, helperOptions).catch(() => null); + if (completed) return true; + throw error; + } +} + +async function inspectLegacyGuard(context, options) { + let entry; + try { + entry = await options.lstatEntry(context.guardPath); + } catch (error) { + if (error?.code === "ENOENT") return "absent"; + throw error; + } + if (entry.isDirectory() && !entry.isSymbolicLink?.()) { + if (await lstatOrNull(join(context.guardPath, LEGACY_RETIREMENT_MARKER_NAME), options) === null) { + throw new Error( + `Legacy consumer lock directory exists at ${context.guardPath}. Stop every legacy proper-lockfile client, ` + + "confirm that no owner remains, remove that directory manually, and retry.", + ); + } + await secureDirectory(context.guardPath, "Legacy consumer high-water lock directory", options); + const marker = await readExactMetadata( + join(context.guardPath, LEGACY_RETIREMENT_MARKER_NAME), + options.metadataMaxBytes, + (value) => validateLegacyRetirementMarker(value, context.statePath), + "Legacy consumer high-water retirement marker", + options, + ); + await options.syncDirectory(context.guardPath); + await options.syncDirectory(dirname(context.guardPath)); + return "retirement-marker"; + } + if (!entry.isFile() || entry.isSymbolicLink?.()) { + throw new Error("Legacy consumer lock guard is not one exact regular non-symlink file."); + } + const expected = legacyGuardFor(context.statePath); + const actual = await readExactMetadata( + context.guardPath, + options.metadataMaxBytes, + (value) => value, + "Legacy consumer lock guard", + options, + ); + if (!metadataBytes(actual).equals(metadataBytes(expected))) { + throw new Error("Legacy consumer lock guard differs from the exact durable handoff guard."); + } + await options.syncDirectory(dirname(context.guardPath)); + return "guard"; +} + +async function ensureLegacyGuard(context, claim, options) { + if (["guard", "retirement-marker"].includes(await inspectLegacyGuard(context, options))) return; + const expected = legacyGuardFor(context.statePath); + await publishImmutable({ + path: context.guardPath, + bytes: metadataBytes(expected), + directory: dirname(context.guardPath), + kind: "legacy-guard", + context, + writer: claim, + options, + }); + if (await inspectLegacyGuard(context, options) !== "guard") { + throw new Error("Legacy consumer lock handoff did not publish the exact durable guard."); + } +} + +function normalizeOptions({ + stale = PYLON_CONSUMER_LOCK_STALE_MS, + update = PYLON_CONSUMER_LOCK_UPDATE_MS, + stateMaxBytes = DEFAULT_STATE_MAX_BYTES, + maxTransactionDepth = MAX_TRANSACTION_DEPTH, + maxLockGenerations = MAX_LOCK_GENERATIONS, + maxJournalBytes = DEFAULT_JOURNAL_MAX_BYTES, + now = Date.now, + startHeartbeat = defaultHeartbeatScheduler, + hooks, + directoryOperations = {}, + lstatEntry = lstat, + makeDirectory = mkdir, + syncDirectory = syncConsumerStateDirectory, + openFile = open, + linkFile = link, + readDirectory = readdir, + renameFile = rename, + removeFile = rm, + processKill = process.kill.bind(process), + currentUid = typeof process.getuid === "function" ? process.getuid() : null, +} = {}) { + if ( + !Number.isSafeInteger(stale) || !Number.isSafeInteger(update) || update < 1 || stale <= update || + !Number.isSafeInteger(stateMaxBytes) || stateMaxBytes < 1 || stateMaxBytes > MAX_STATE_BYTES || + !Number.isSafeInteger(maxTransactionDepth) || maxTransactionDepth < 1 || maxTransactionDepth > MAX_TRANSACTION_DEPTH || + !Number.isSafeInteger(maxLockGenerations) || maxLockGenerations < 2 || maxLockGenerations > MAX_LOCK_GENERATIONS || + !Number.isSafeInteger(maxJournalBytes) || maxJournalBytes < stateMaxBytes || maxJournalBytes > MAX_JOURNAL_BYTES || + !Number.isSafeInteger(currentUid) || currentUid < 0 + ) throw new Error("Consumer high-water lock timing, state-size, journal, or transaction bound is invalid."); + return { + stale, + update, + stateMaxBytes, + maxTransactionDepth, + maxLockGenerations, + maxJournalBytes, + maxJournalEntries: MAX_OPERATION_GENERATIONS * 5 + MAX_TRANSACTION_DEPTH + 32, + metadataMaxBytes: stateMaxBytes * 3 + 8192, + now, + startHeartbeat, + hooks, + directoryOperations, + lstatEntry, + makeDirectory, + syncDirectory, + openFile, + linkFile, + readDirectory, + renameFile, + removeFile, + processKill, + currentUid, + activeWriter: null, + }; +} + +function normalizeRotationOptions(rawOptions) { + const options = normalizeOptions(rawOptions); + options.stateMaxBytes = MAX_STATE_BYTES; + options.maxTransactionDepth = MAX_TRANSACTION_DEPTH; + options.maxLockGenerations = MAX_LOCK_GENERATIONS; + options.maxJournalBytes = MAX_JOURNAL_BYTES; + options.maxJournalEntries = MAX_OPERATION_GENERATIONS * 5 + MAX_TRANSACTION_DEPTH + 32; + options.metadataMaxBytes = MAX_STATE_BYTES * 3 + 8192; + return options; +} + +async function lstatOrNull(path, options) { + try { + return await options.lstatEntry(path); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } +} + +function legacyTerminalFileName(claim) { + return `terminal-${generationName(claim.generation)}-${claim.token}.json`; +} + +function legacyHeartbeatFileName(claim) { + return `heartbeat-${generationName(claim.generation)}-${claim.token}.json`; +} + +function legacyAppliedFileName(claim) { + return `applied-${generationName(claim.generation)}-${claim.token}.json`; +} + +function authorityDigest(entries, tipDigest, tipBytes) { + const hash = createHash("sha256"); + hash.update("pylon-consumer-v1-authority\0"); + const sorted = [...entries].sort((left, right) => { + if (left[0] < right[0]) return -1; + if (left[0] > right[0]) return 1; + return 0; + }); + for (const [name, bytes] of sorted) { + const nameBytes = Buffer.from(name); + const header = Buffer.alloc(12); + header.writeUInt32BE(nameBytes.length, 0); + header.writeBigUInt64BE(BigInt(bytes.length), 4); + hash.update(header); + hash.update(nameBytes); + hash.update(bytes); + } + hash.update(Buffer.from(`tip:${tipDigest}:`)); + if (tipBytes !== null) hash.update(tipBytes); + return hash.digest("hex"); +} + +async function readLegacyAuthority(statePath, lockDirectory, transactionDirectory, options) { + await secureDirectory(lockDirectory, "Legacy consumer high-water lock directory", options); + await secureDirectory(transactionDirectory, "Legacy consumer high-water transaction directory", options); + await options.syncDirectory(lockDirectory); + await options.syncDirectory(transactionDirectory); + const transactionNames = await options.readDirectory(transactionDirectory); + if (transactionNames.length > options.maxTransactionDepth) { + throw new Error("Legacy consumer high-water transaction directory exceeds its safe entry bound."); + } + const budget = { bytes: 0 }; + const authorityEntries = []; + const actualTransactions = new Map(); + for (const name of transactionNames) { + const match = legacyTransitionPattern.exec(name); + if (!match || actualTransactions.has(match[1])) { + throw new Error("Legacy consumer high-water transaction directory contains a malformed or extra entry."); + } + const value = await readExactMetadata( + join(transactionDirectory, name), + options.metadataMaxBytes, + (candidate) => validateTransaction(candidate, match[1], options.stateMaxBytes).value, + "Legacy consumer high-water transaction", + options, + budget, + ); + actualTransactions.set(match[1], value); + authorityEntries.push([`transactions/${name}`, metadataBytes(value)]); + } + const lockNames = await options.readDirectory(lockDirectory); + if (lockNames.length > MAX_OPERATION_GENERATIONS * 4 + 1) { + throw new Error("Legacy consumer high-water lock directory exceeds its safe entry bound."); + } + const claimNames = new Map(); + const heartbeatNames = new Map(); + const terminalNames = new Map(); + const appliedNames = new Map(); + let retirementMarker = null; + for (const name of lockNames) { + let match; + if (name === LEGACY_RETIREMENT_MARKER_NAME) { + if (retirementMarker !== null) throw new Error("Legacy consumer high-water retirement marker is duplicated."); + retirementMarker = await readExactMetadata( + join(lockDirectory, name), + options.metadataMaxBytes, + (value) => validateLegacyRetirementMarker(value, statePath), + "Legacy consumer high-water retirement marker", + options, + budget, + ); + continue; + } + if ((match = undigestedClaimPattern.exec(name))) claimNames.set(Number(match[1]), name); + else if ((match = heartbeatPattern.exec(name))) heartbeatNames.set(`${Number(match[1])}:${match[2]}`, name); + else if ((match = terminalPattern.exec(name))) terminalNames.set(`${Number(match[1])}:${match[2]}`, name); + else if ((match = appliedPattern.exec(name))) appliedNames.set(`${Number(match[1])}:${match[2]}`, name); + else throw new Error("Legacy consumer high-water lock directory contains a malformed or extra entry."); + } + const claims = []; + const byKey = new Map(); + for (const [generation, name] of [...claimNames].sort((left, right) => left[0] - right[0])) { + const claim = await readExactMetadata( + join(lockDirectory, name), + options.metadataMaxBytes, + validateLegacyClaim, + "Legacy consumer high-water lock claim", + options, + budget, + ); + if (claim.generation !== generation || name !== `claim-${generationName(generation)}.json`) { + throw new Error("Legacy consumer high-water claim name differs from its exact generation."); + } + claims.push(claim); + byKey.set(`${claim.generation}:${claim.token}`, claim); + authorityEntries.push([`lock/${name}`, metadataBytes(claim)]); + } + if (claims.length > options.maxLockGenerations) throw new Error("Legacy consumer high-water claim bound is exhausted."); + for (let index = 0; index < claims.length; index += 1) { + if (claims[index].generation !== index + 1) throw new Error("Legacy consumer high-water claims are not contiguous."); + } + for (const [key, name] of heartbeatNames) { + const claim = byKey.get(key); + if (!claim || name !== legacyHeartbeatFileName(claim)) { + throw new Error("Legacy consumer high-water lock contains an orphan heartbeat."); + } + const heartbeat = await readExactMetadata( + join(lockDirectory, name), + options.metadataMaxBytes, + (value) => validateLegacyHeartbeat(value, claim), + "Legacy consumer high-water heartbeat", + options, + budget, + ); + authorityEntries.push([`lock/${name}`, metadataBytes(heartbeat)]); + } + const terminals = new Map(); + for (const [key, name] of terminalNames) { + const claim = byKey.get(key); + if (!claim || name !== legacyTerminalFileName(claim)) { + throw new Error("Legacy consumer high-water lock contains an orphan terminal marker."); + } + const terminal = await readExactMetadata( + join(lockDirectory, name), + options.metadataMaxBytes, + (value) => validateLegacyTerminal(value, claim, options.stateMaxBytes), + "Legacy consumer high-water terminal marker", + options, + budget, + ); + terminals.set(key, terminal); + authorityEntries.push([`lock/${name}`, metadataBytes(terminal)]); + } + for (const claim of claims) { + const key = `${claim.generation}:${claim.token}`; + if (!heartbeatNames.has(key)) throw new Error("Legacy consumer high-water claim lacks its exact heartbeat."); + } + const appliedTerminals = new Set(); + for (const [key, name] of appliedNames) { + const claim = byKey.get(key); + const terminal = terminals.get(key); + if (!claim || !terminal || name !== legacyAppliedFileName(claim)) { + throw new Error("Legacy consumer high-water lock contains an orphan applied marker."); + } + const applied = await readExactMetadata( + join(lockDirectory, name), + options.metadataMaxBytes, + (value) => validateLegacyApplied(value, claim, terminal), + "Legacy consumer high-water applied marker", + options, + budget, + ); + authorityEntries.push([`lock/${name}`, metadataBytes(applied)]); + appliedTerminals.add(key); + } + const decidedTransactions = new Map(); + const decidedDigests = new Set([GENESIS_DIGEST]); + let tipDigest = GENESIS_DIGEST; + let tipBytes = null; + let decidedLength = 0; + for (const claim of claims) { + const terminal = terminals.get(`${claim.generation}:${claim.token}`); + if (!terminal || terminal.outcome !== "commit") continue; + for (const transaction of terminal.transactions) { + if (transaction.baseDigest !== tipDigest) { + throw new Error("Legacy consumer high-water commit decisions do not form one exact authoritative chain."); + } + const prior = decidedTransactions.get(transaction.baseDigest); + if (prior && !metadataBytes(prior).equals(metadataBytes(transaction))) { + throw new Error("Legacy consumer high-water commit decisions equivocate at one base digest."); + } + decidedTransactions.set(transaction.baseDigest, transaction); + const validated = validateTransaction(transaction, tipDigest, options.stateMaxBytes); + tipDigest = transaction.candidateDigest; + tipBytes = validated.candidateBytes; + decidedDigests.add(tipDigest); + decidedLength += 1; + if (decidedLength > options.maxTransactionDepth) { + throw new Error("Legacy consumer high-water decisions exceed their safe transaction bound."); + } + } + } + let actualDigest = GENESIS_DIGEST; + let actualCount = 0; + while (actualTransactions.has(actualDigest)) { + const actual = actualTransactions.get(actualDigest); + const decided = decidedTransactions.get(actualDigest); + if (!decided || !metadataBytes(actual).equals(metadataBytes(decided))) { + throw new Error("Legacy consumer high-water transition lacks its exact immutable commit decision."); + } + actualDigest = actual.candidateDigest; + actualCount += 1; + if (actualCount > options.maxTransactionDepth) { + throw new Error("Legacy consumer high-water transition chain exceeds its safe bound."); + } + } + if (actualCount !== actualTransactions.size) { + throw new Error("Legacy consumer high-water transaction chain contains a corrupt, unreachable, or extra transition."); + } + for (const [baseDigest, actual] of actualTransactions) { + const decided = decidedTransactions.get(baseDigest); + if (!decided || !metadataBytes(actual).equals(metadataBytes(decided))) { + throw new Error("Legacy consumer high-water transition differs from its exact commit decision."); + } + } + for (const key of appliedTerminals) { + const terminal = terminals.get(key); + for (const transaction of terminal.transactions) { + const actual = actualTransactions.get(transaction.baseDigest); + if (!actual || !metadataBytes(actual).equals(metadataBytes(transaction))) { + throw new Error("Legacy consumer high-water applied marker is missing its completed transition."); + } + } + } + const recoveries = []; + for (const claim of claims) { + const key = `${claim.generation}:${claim.token}`; + const terminal = terminals.get(key); + if (!terminal) { + recoveries.push({ kind: "retire", claim }); + continue; + } + if (terminal.outcome === "commit" && !appliedTerminals.has(key)) { + recoveries.push({ + kind: "commit", + claim, + terminal, + missingTransactions: terminal.transactions.filter((transaction) => !actualTransactions.has(transaction.baseDigest)), + }); + } + } + const projection = await readSecureFile( + statePath, + options.stateMaxBytes, + "Legacy consumer high-water projection", + options, + 0, + ); + if (projection !== null && projection.length < 1) throw new Error("Legacy consumer high-water projection is malformed."); + if (decidedLength === 0 && projection !== null) { + tipBytes = projection; + tipDigest = digest(projection); + authorityEntries.push(["explicit-quiescent-projection", projection]); + } else if (projection !== null && !decidedDigests.has(digest(projection))) { + throw new Error("Legacy consumer high-water projection is not an authenticated prefix of its immutable authority."); + } + + if (budget.bytes > options.maxJournalBytes) { + throw new Error("Legacy consumer high-water authority exceeds its safe byte bound."); + } + const authoritySha256 = authorityDigest(authorityEntries, tipDigest, tipBytes); + if (retirementMarker !== null) { + const expectedMarker = legacyRetirementMarkerFor(statePath, { authoritySha256, tipDigest }); + if (!metadataBytes(retirementMarker).equals(metadataBytes(expectedMarker))) { + throw new Error("Legacy consumer high-water retirement marker conflicts with the exact pre-marker authority or tip."); + } + if (recoveries.length !== 0) { + throw new Error("Legacy consumer high-water retirement marker was published before its authority became quiescent."); + } + } + return { + tipDigest, + tipBytes, + length: decidedLength, + authoritySha256, + authorityEntries, + recoveries, + retirementMarker, + }; +} + +function migrationCheckpoint(statePath, legacy) { + const checkpoint = { + schemaVersion: CHECKPOINT_SCHEMA_VERSION, + epoch: 1, + epochId: deterministicUuid(`pylon-consumer-v1-migration:${statePath}:${legacy.authoritySha256}:${legacy.tipDigest}`), + previousCheckpointSha256: GENESIS_DIGEST, + previousTipSha256: GENESIS_DIGEST, + historySha256: digest(Buffer.from( + `pylon-consumer-history:${digest(Buffer.from(statePath))}:v1:${legacy.authoritySha256}:${legacy.tipDigest}`, + )), + anchorDigest: legacy.tipDigest, + anchorBase64: legacy.tipBytes === null ? null : legacy.tipBytes.toString("base64"), + retiredEpochDirectory: null, + sourceAuthoritySha256: legacy.authoritySha256, + sourceAuthorityTipDigest: legacy.tipDigest, + sourceAuthorityTipBase64: legacy.tipBytes === null ? null : legacy.tipBytes.toString("base64"), + }; + validateCheckpoint(checkpoint, Number.MAX_SAFE_INTEGER); + return checkpoint; +} + +function sameLegacyAuthority(left, right) { + return left.authoritySha256 === right.authoritySha256 && left.tipDigest === right.tipDigest && + (left.tipBytes === null ? right.tipBytes === null : right.tipBytes !== null && left.tipBytes.equals(right.tipBytes)); +} + +function legacyOwnerIsDefinitivelyDead(claim, options) { + try { + options.processKill(claim.ownerPid, 0); + return false; + } catch (error) { + if (error?.code === "ESRCH") return true; + return false; + } +} + +function requireRecoverableLegacyOwners(legacy, options) { + for (const recovery of legacy.recoveries) { + if (!legacyOwnerIsDefinitivelyDead(recovery.claim, options)) { + throw new Error( + "Legacy consumer high-water migration is blocked by a live or uncertain incomplete v1 commit owner.", + ); + } + } +} + +function recoveredLegacyAuthorityEntries(legacy) { + const entries = []; + for (const recovery of legacy.recoveries) { + if (recovery.kind === "retire") { + const terminal = { + schemaVersion: LEGACY_LOCK_SCHEMA_VERSION, + generation: recovery.claim.generation, + token: recovery.claim.token, + outcome: "retired", + }; + entries.push([`lock/${legacyTerminalFileName(recovery.claim)}`, metadataBytes(terminal)]); + continue; + } + for (const transaction of recovery.missingTransactions) { + entries.push([`transactions/${transaction.baseDigest}.json`, metadataBytes(transaction)]); + } + const applied = { + schemaVersion: LEGACY_LOCK_SCHEMA_VERSION, + generation: recovery.claim.generation, + token: recovery.claim.token, + terminalSha256: digest(metadataBytes(recovery.terminal)), + }; + entries.push([`lock/${legacyAppliedFileName(recovery.claim)}`, metadataBytes(applied)]); + } + return entries; +} + +function expectedRecoveredLegacyAuthoritySha256(legacy) { + return authorityDigest( + [...legacy.authorityEntries, ...recoveredLegacyAuthorityEntries(legacy)], + legacy.tipDigest, + legacy.tipBytes, + ); +} + +function legacyAuthorityIsExactRecoveryProgress(previous, current) { + if ( + previous.tipDigest !== current.tipDigest || + (previous.tipBytes === null + ? current.tipBytes !== null + : current.tipBytes === null || !previous.tipBytes.equals(current.tipBytes)) + ) return false; + const required = new Map(previous.authorityEntries); + const allowed = new Map(recoveredLegacyAuthorityEntries(previous)); + const actual = new Map(current.authorityEntries); + if (required.size !== previous.authorityEntries.length || actual.size !== current.authorityEntries.length) return false; + for (const [name, bytes] of required) { + if (!actual.get(name)?.equals(bytes)) return false; + } + for (const [name, bytes] of actual) { + if (required.has(name)) continue; + if (!allowed.get(name)?.equals(bytes)) return false; + } + return true; +} +async function publishExactLegacyMetadata(path, value, validate, description, directory, context, writer, kind, options) { + await publishImmutable({ + path, + bytes: metadataBytes(value), + directory, + kind, + context, + writer, + options, + revalidate: false, + }); + const actual = await readExactMetadata(path, options.metadataMaxBytes, validate, description, options); + if (!metadataBytes(actual).equals(metadataBytes(value))) { + throw new Error(`${description} lost its immutable exact-value publication.`); + } +} + +async function helpLegacyAuthority(retiredLockDirectory, transactionDirectory, legacy, context, options) { + for (const recovery of legacy.recoveries) { + await secureDirectory(retiredLockDirectory, "Legacy consumer high-water lock directory", options); + if (await lstatOrNull(join(retiredLockDirectory, LEGACY_RETIREMENT_MARKER_NAME), options) !== null) { + throw new Error("Legacy consumer authority recovery cannot cross its immutable retirement marker."); + } + if (recovery.kind === "retire") { + const terminal = { + schemaVersion: LEGACY_LOCK_SCHEMA_VERSION, + generation: recovery.claim.generation, + token: recovery.claim.token, + outcome: "retired", + }; + await publishExactLegacyMetadata( + join(retiredLockDirectory, legacyTerminalFileName(recovery.claim)), + terminal, + (value) => validateLegacyTerminal(value, recovery.claim, options.stateMaxBytes), + "Legacy consumer high-water recovered terminal marker", + retiredLockDirectory, + context, + recovery.claim, + "terminal-retired", + options, + ); + continue; + } + for (const transaction of recovery.missingTransactions) { + await publishExactLegacyMetadata( + join(transactionDirectory, `${transaction.baseDigest}.json`), + transaction, + (value) => validateTransaction(value, transaction.baseDigest, options.stateMaxBytes).value, + "Legacy consumer high-water recovered transition", + transactionDirectory, + context, + recovery.claim, + "transition", + options, + ); + } + const applied = { + schemaVersion: LEGACY_LOCK_SCHEMA_VERSION, + generation: recovery.claim.generation, + token: recovery.claim.token, + terminalSha256: digest(metadataBytes(recovery.terminal)), + }; + await publishExactLegacyMetadata( + join(retiredLockDirectory, legacyAppliedFileName(recovery.claim)), + applied, + (value) => validateLegacyApplied(value, recovery.claim, recovery.terminal), + "Legacy consumer high-water recovered applied marker", + retiredLockDirectory, + context, + recovery.claim, + "applied", + options, + ); + } + await options.syncDirectory(retiredLockDirectory); + await options.syncDirectory(transactionDirectory); +} + +async function legacyMigrationSource(statePath, options) { + const guardPath = `${statePath}.lock`; + const retiredLockDirectory = `${statePath}.lock.v1-retired`; + const guardEntry = await lstatOrNull(guardPath, options); + const retiredEntry = await lstatOrNull(retiredLockDirectory, options); + if (retiredEntry && (!retiredEntry.isDirectory() || retiredEntry.isSymbolicLink?.())) { + throw new Error("Prior retired v1 consumer lock authority must be one real directory and is never replaced."); + } + if (guardEntry?.isDirectory() && !guardEntry.isSymbolicLink?.() && retiredEntry) { + throw new Error("Live and retired v1 consumer lock authority both exist; migration fails closed."); + } + if (retiredEntry) { + if (guardEntry !== null && (!guardEntry.isFile() || guardEntry.isSymbolicLink?.())) { + throw new Error("Prior retired v1 consumer authority has an unsafe or ambiguous live lock path."); + } + return { guardPath, sourceLockDirectory: retiredLockDirectory, layout: "prior-retired", guardEntry }; + } + if (guardEntry?.isDirectory() && !guardEntry.isSymbolicLink?.()) { + return { guardPath, sourceLockDirectory: guardPath, layout: "in-place", guardEntry }; + } + throw new Error("Prior v1 consumer lock authority is absent, unsafe, or ambiguous."); +} + +async function publishLegacyRetirementMarker(source, legacy, context, options) { + if (legacy.retirementMarker !== null) return legacy; + if (legacy.recoveries.length !== 0) { + throw new Error("Legacy consumer high-water authority must be quiescent before retirement marker publication."); + } + const markerPath = join(source.sourceLockDirectory, LEGACY_RETIREMENT_MARKER_NAME); + const marker = legacyRetirementMarkerFor(context.statePath, legacy); + const authenticateBeforeMarkerLink = async () => { + const currentSource = await legacyMigrationSource(context.statePath, options); + if (currentSource.layout !== "in-place" || currentSource.sourceLockDirectory !== source.sourceLockDirectory) { + throw new Error("Legacy consumer high-water source changed before retirement marker publication."); + } + const current = await readLegacyAuthority( + context.statePath, + source.sourceLockDirectory, + `${context.statePath}.transactions`, + options, + ); + if (current.retirementMarker !== null) { + if (sameLegacyAuthority(legacy, current)) { + throw Object.assign(new Error("Concurrent migration already published the exact retirement marker."), { + code: "PYLON_EXACT_RETIREMENT_JOIN", + }); + } + throw new Error("Legacy consumer high-water authority changed before retirement marker publication."); + } + if (!sameLegacyAuthority(legacy, current) || current.recoveries.length !== 0) { + throw new Error("Legacy consumer high-water authority changed before retirement marker publication."); + } + requireRecoverableLegacyOwners(current, options); + }; + try { + await publishImmutable({ + path: markerPath, + bytes: metadataBytes(marker), + directory: source.sourceLockDirectory, + kind: "legacy-retirement", + context, + writer: { generation: 0, token: context.checkpoint.epochId, type: "rotation" }, + options, + revalidate: false, + beforeLink: authenticateBeforeMarkerLink, + }); + } catch (error) { + if (error?.code !== "PYLON_EXACT_RETIREMENT_JOIN") throw error; + const joined = await readLegacyAuthority( + context.statePath, + source.sourceLockDirectory, + `${context.statePath}.transactions`, + options, + ); + if (joined.retirementMarker === null || !sameLegacyAuthority(legacy, joined)) throw error; + } + await options.syncDirectory(source.sourceLockDirectory); + await options.syncDirectory(dirname(source.sourceLockDirectory)); + const guarded = await readLegacyAuthority( + context.statePath, + source.sourceLockDirectory, + `${context.statePath}.transactions`, + options, + ); + if (guarded.retirementMarker === null || !sameLegacyAuthority(legacy, guarded)) { + throw new Error("Legacy consumer high-water retirement marker does not authenticate its exact pre-marker authority."); + } + await options.hooks?.afterMigrationRetirementMarker?.({ markerPath, marker: structuredClone(marker) }); + await options.hooks?.afterMigrationGuard?.({ guardPath: source.guardPath, markerPath }); + return guarded; +} + +async function publishPriorLayoutGuard(source, legacy, context, options) { + if (source.guardEntry === null) { + await publishImmutable({ + path: source.guardPath, + bytes: metadataBytes(legacyGuardFor(context.statePath)), + directory: dirname(source.guardPath), + kind: "legacy-guard", + context, + writer: { generation: 0, token: context.checkpoint.epochId, type: "rotation" }, + options, + revalidate: false, + beforeLink: async () => { + const current = await readLegacyAuthority( + context.statePath, + source.sourceLockDirectory, + `${context.statePath}.transactions`, + options, + ); + if (!sameLegacyAuthority(legacy, current) || current.recoveries.length !== 0) { + throw new Error("Prior retired v1 authority changed before downgrade guard publication."); + } + }, + }); + } + if (await inspectLegacyGuard(context, options) !== "guard") { + throw new Error("Prior retired v1 authority lacks its exact permanent downgrade guard."); + } + await options.syncDirectory(dirname(source.guardPath)); + await options.hooks?.afterMigrationGuard?.({ guardPath: source.guardPath }); +} + +async function validateMigratedAuthority(context, options) { + if (context.checkpoint.epoch < 1 || context.checkpoint.sourceAuthoritySha256 === GENESIS_DIGEST) { + throw new Error("Prior v1 consumer authority exists but the v2 journal lacks an authenticated migration checkpoint."); + } + const source = await legacyMigrationSource(context.statePath, options); + const sourceTipBytes = context.checkpoint.sourceAuthorityTipBase64 === null + ? null + : Buffer.from(context.checkpoint.sourceAuthorityTipBase64, "base64"); + const legacy = await readLegacyAuthority( + context.statePath, + source.sourceLockDirectory, + `${context.statePath}.transactions`, + options, + ); + if ( + legacy.authoritySha256 !== context.checkpoint.sourceAuthoritySha256 || + legacy.tipDigest !== context.checkpoint.sourceAuthorityTipDigest || legacy.recoveries.length !== 0 || + (sourceTipBytes === null ? legacy.tipBytes !== null : !sourceTipBytes.equals(legacy.tipBytes)) + ) throw new Error("The v2 migration checkpoint does not authenticate the complete prior v1 authority and tip."); + const guardKind = await inspectLegacyGuard(context, options); + if ( + (source.layout === "in-place" && (guardKind !== "retirement-marker" || legacy.retirementMarker === null)) || + (source.layout === "prior-retired" && guardKind !== "guard") + ) throw new Error("Prior v1 consumer authority is not fenced by its exact permanent downgrade guard."); + return { source, legacy }; +} + +export async function migrateConsumerStateJournal(statePath, rawOptions = {}) { + if (typeof statePath !== "string" || !statePath) throw new Error("A consumer-local state path is required for v1 journal migration."); + const options = normalizeOptions(rawOptions); + const absoluteStatePath = resolve(statePath); + const directory = dirname(absoluteStatePath); + await ensureDurableConsumerStateDirectory(directory, options.directoryOperations); + await secureDirectory(directory, "Consumer high-water state directory", options); + const transactionDirectory = `${absoluteStatePath}.transactions`; + const transactionEntry = await lstatOrNull(transactionDirectory, options); + if (!transactionEntry) throw new Error("No prior v1 consumer transaction authority exists to migrate."); + if (!transactionEntry.isDirectory() || transactionEntry.isSymbolicLink?.()) { + throw new Error("Prior v1 consumer transaction authority must be one real directory."); + } + let source = await legacyMigrationSource(absoluteStatePath, options); + let legacy = await readLegacyAuthority(absoluteStatePath, source.sourceLockDirectory, transactionDirectory, options); + const initialCheckpoint = migrationCheckpoint(absoluteStatePath, legacy); + await options.hooks?.afterMigrationAuthorityRead?.({ + checkpoint: structuredClone(initialCheckpoint), + legacy: structuredClone(legacy), + }); + + const journalDirectory = `${absoluteStatePath}.journal`; + await ensureDirectory(journalDirectory, "Consumer high-water journal directory", options); + const temporaryDirectory = join(journalDirectory, TEMPORARY_DIRECTORY_NAME); + await ensureDirectory(temporaryDirectory, "Consumer high-water temporary directory", options); + + source = await legacyMigrationSource(absoluteStatePath, options); + const currentLegacy = await readLegacyAuthority(absoluteStatePath, source.sourceLockDirectory, transactionDirectory, options); + if (!sameLegacyAuthority(legacy, currentLegacy) && !legacyAuthorityIsExactRecoveryProgress(legacy, currentLegacy)) { + throw new Error("Concurrent v1 migration changed the exact authenticated legacy authority or tip."); + } + legacy = currentLegacy; + if (source.layout === "in-place" && legacy.retirementMarker === null) { + requireRecoverableLegacyOwners(legacy, options); + const expectedRecoveredAuthoritySha256 = expectedRecoveredLegacyAuthoritySha256(legacy); + const recoveryCheckpoint = migrationCheckpoint(absoluteStatePath, legacy); + try { + await helpLegacyAuthority(source.sourceLockDirectory, transactionDirectory, legacy, { + statePath: absoluteStatePath, + guardPath: source.guardPath, + journalDirectory, + checkpoint: recoveryCheckpoint, + checkpointPath: join(journalDirectory, checkpointName(recoveryCheckpoint)), + checkpointDigest: digest(metadataBytes(recoveryCheckpoint)), + epochDirectory: join(journalDirectory, epochName(recoveryCheckpoint)), + temporaryDirectory, + }, options); + } catch (error) { + const joined = await readLegacyAuthority(absoluteStatePath, source.sourceLockDirectory, transactionDirectory, options); + if (joined.retirementMarker === null) throw error; + legacy = joined; + } + if (legacy.retirementMarker === null) { + const recovered = await readLegacyAuthority(absoluteStatePath, source.sourceLockDirectory, transactionDirectory, options); + if ( + recovered.recoveries.length !== 0 || recovered.authoritySha256 !== expectedRecoveredAuthoritySha256 || + recovered.tipDigest !== legacy.tipDigest || + (legacy.tipBytes === null ? recovered.tipBytes !== null : recovered.tipBytes === null || !legacy.tipBytes.equals(recovered.tipBytes)) + ) throw new Error("V1 authority recovery did not produce only the exact authenticated dead-owner completion."); + legacy = recovered; + } + } + if (source.layout === "prior-retired" && legacy.recoveries.length !== 0) { + throw new Error("Interrupted prior-layout v1 migration authority is supported read-only and still requires recovery."); + } + if (legacy.recoveries.length !== 0) { + throw new Error("Legacy consumer high-water authority is not quiescent after recovery."); + } + + let checkpoint = migrationCheckpoint(absoluteStatePath, legacy); + let bootstrapContext = { + statePath: absoluteStatePath, + guardPath: source.guardPath, + journalDirectory, + checkpoint, + checkpointPath: join(journalDirectory, checkpointName(checkpoint)), + checkpointDigest: digest(metadataBytes(checkpoint)), + epochDirectory: join(journalDirectory, epochName(checkpoint)), + temporaryDirectory, + }; + if (source.layout === "in-place") { + legacy = await publishLegacyRetirementMarker(source, legacy, bootstrapContext, options); + } else { + await publishPriorLayoutGuard(source, legacy, bootstrapContext, options); + } + + const guardedLegacy = await readLegacyAuthority( + absoluteStatePath, + source.sourceLockDirectory, + transactionDirectory, + options, + ); + if (!sameLegacyAuthority(legacy, guardedLegacy) || guardedLegacy.recoveries.length !== 0) { + throw new Error("V1 authority mutated across its exact durable retirement handoff."); + } + checkpoint = migrationCheckpoint(absoluteStatePath, guardedLegacy); + bootstrapContext = { + ...bootstrapContext, + checkpoint, + checkpointPath: join(journalDirectory, checkpointName(checkpoint)), + checkpointDigest: digest(metadataBytes(checkpoint)), + epochDirectory: join(journalDirectory, epochName(checkpoint)), + }; + const authenticateBeforeCheckpointLink = async () => { + const currentSource = await legacyMigrationSource(absoluteStatePath, options); + if (currentSource.layout !== source.layout || currentSource.sourceLockDirectory !== source.sourceLockDirectory) { + throw new Error("V1 authority source changed immediately before migration checkpoint publication."); + } + const current = await readLegacyAuthority( + absoluteStatePath, + source.sourceLockDirectory, + transactionDirectory, + options, + ); + if (!sameLegacyAuthority(guardedLegacy, current) || current.recoveries.length !== 0) { + throw new Error("V1 authority mutated immediately before migration checkpoint publication."); + } + }; + const scan = await initializeJournal( + absoluteStatePath, + journalDirectory, + options, + checkpoint, + authenticateBeforeCheckpointLink, + ); + if (!scan.head || !metadataBytes(scan.head.checkpoint).equals(metadataBytes(checkpoint))) { + throw new Error("V1 migration encountered a different existing v2 journal checkpoint."); + } + const context = contextFromHead(absoluteStatePath, source.guardPath, journalDirectory, scan.head); + await validateMigratedAuthority(context, options); + await repairProjection(context, await walkTransactions(context, options), options, { + generation: 0, + token: checkpoint.epochId, + type: "rotation", + }); + await validateMigratedAuthority(context, options); + await options.hooks?.afterMigrationComplete?.({ checkpoint: structuredClone(checkpoint) }); + await validateMigratedAuthority(context, options); + return { epoch: 1, tipSha256: checkpoint.anchorDigest, sourceAuthoritySha256: checkpoint.sourceAuthoritySha256 }; +} + +async function prepareContext(statePath, options) { + const absoluteStatePath = resolve(statePath); + const directory = dirname(absoluteStatePath); + await ensureDurableConsumerStateDirectory(directory, options.directoryOperations); + await secureDirectory(directory, "Consumer high-water state directory", options); + const guardPath = `${absoluteStatePath}.lock`; + const retiredLockDirectory = `${absoluteStatePath}.lock.v1-retired`; + const journalDirectory = `${absoluteStatePath}.journal`; + const legacyTransactionDirectory = `${absoluteStatePath}.transactions`; + + // Detect every old-authority signal before creating a guard or a genesis journal. + const legacyEntry = await lstatOrNull(legacyTransactionDirectory, options); + const guardEntry = await lstatOrNull(guardPath, options); + const retiredEntry = await lstatOrNull(retiredLockDirectory, options); + const journalEntry = await lstatOrNull(journalDirectory, options); + if (legacyEntry && (!legacyEntry.isDirectory() || legacyEntry.isSymbolicLink?.())) { + throw new Error("Prior v1 consumer transaction authority must be one real directory."); + } + if (retiredEntry && (!retiredEntry.isDirectory() || retiredEntry.isSymbolicLink?.())) { + throw new Error("Prior retired v1 consumer lock authority must be one real directory and is never replaced."); + } + if (guardEntry && ( + guardEntry.isSymbolicLink?.() || (!guardEntry.isFile() && !guardEntry.isDirectory()) + )) throw new Error("Legacy consumer lock guard is not one exact regular non-symlink file."); + + let inPlaceMarkerEntry = null; + if (guardEntry?.isDirectory() && !guardEntry.isSymbolicLink?.()) { + const markerPath = join(guardPath, LEGACY_RETIREMENT_MARKER_NAME); + inPlaceMarkerEntry = await lstatOrNull(markerPath, options); + if (inPlaceMarkerEntry) { + if (!inPlaceMarkerEntry.isFile() || inPlaceMarkerEntry.isSymbolicLink?.()) { + throw new Error("Legacy consumer high-water retirement marker must be one real file."); + } + await readExactMetadata( + markerPath, + options.metadataMaxBytes, + (value) => validateLegacyRetirementMarker(value, absoluteStatePath), + "Legacy consumer high-water retirement marker", + options, + ); + } + } + + let scan = null; + if (journalEntry) { + if (!journalEntry.isDirectory() || journalEntry.isSymbolicLink?.()) { + throw new Error("Consumer high-water journal directory must be one real directory."); + } + await secureDirectory(journalDirectory, "Consumer high-water journal directory", options); + try { + scan = await scanJournalRoot(absoluteStatePath, journalDirectory, options); + } catch (error) { + if (error?.message !== "Consumer high-water journal lacks its exact temporary namespace.") throw error; + const names = await options.readDirectory(journalDirectory); + if (names.length === 1 && names[0] === TEMPORARY_DIRECTORY_NAME) { + scan = await scanJournalRoot(absoluteStatePath, journalDirectory, options); + } else if (names.length !== 0) { + throw error; + } + } + } + const hasInPlaceLegacyDirectory = guardEntry?.isDirectory() && !guardEntry.isSymbolicLink?.(); + const hasMigratedV2Head = scan?.head?.checkpoint.sourceAuthoritySha256 !== undefined && + scan.head.checkpoint.sourceAuthoritySha256 !== GENESIS_DIGEST; + const hasLegacySignal = legacyEntry !== null || retiredEntry !== null || hasInPlaceLegacyDirectory || hasMigratedV2Head; + if (hasLegacySignal) { + if (!legacyEntry) { + if (hasInPlaceLegacyDirectory && !inPlaceMarkerEntry && !retiredEntry && !hasMigratedV2Head) { + throw new Error( + `Legacy consumer lock directory exists at ${guardPath}. Stop every legacy proper-lockfile client, ` + + "confirm that no owner remains, remove that directory manually, and retry.", + ); + } + throw new Error("Prior v1 consumer authority is incomplete because its transaction namespace is missing."); + } + // This independently validates the selected live/in-place or prior-retired lock namespace. + await legacyMigrationSource(absoluteStatePath, options); + if (!journalEntry) { + throw new Error( + "Prior v1 consumer authority exists. Stop every old client and run the explicit quiescent consumer journal migration command.", + ); + } + if (!scan?.head || scan.missingHeadEpoch) { + throw new Error("Prior v1 authority has no complete authenticated v2 migration checkpoint."); + } + const context = contextFromHead(absoluteStatePath, guardPath, journalDirectory, scan.head); + await validateMigratedAuthority(context, options); + return { context, scan }; + } + + if (scan?.head) { + if (scan.missingHeadEpoch) scan = await initializeJournal(absoluteStatePath, journalDirectory, options); + return { context: contextFromHead(absoluteStatePath, guardPath, journalDirectory, scan.head), scan }; + } + await ensureDirectory(journalDirectory, "Consumer high-water journal directory", options); + await ensureDirectory(join(journalDirectory, TEMPORARY_DIRECTORY_NAME), "Consumer high-water temporary directory", options); + scan = await initializeJournal(absoluteStatePath, journalDirectory, options); + return { context: contextFromHead(absoluteStatePath, guardPath, journalDirectory, scan.head), scan }; +} + +async function runNormalLocked(statePath, action, rawOptions) { + const options = normalizeOptions(rawOptions); + for (;;) { + const prepared = await prepareContext(statePath, options); + const acquired = await acquireNormalOperation(prepared.context, options); + if (acquired.rotated) continue; + const { context } = prepared; + const { claim, temporaries } = acquired; + options.activeWriter = claim; + let terminal = null; + let heartbeatStopped = false; + const stopHeartbeat = options.startHeartbeat({ + interval: options.update, + beat: () => refreshHeartbeat(context, claim, options), + }); + const stopHeartbeatOnce = async () => { + if (heartbeatStopped) return; + heartbeatStopped = true; + await stopHeartbeat(); + }; + const release = async (cause) => { + if (terminal !== null) return; + const wanted = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation: claim.generation, + token: claim.token, + outcome: "released", + }; + terminal = await publishTerminal(context, claim, wanted, options); + if (terminal.outcome !== "released") { + throw new Error("Consumer high-water lock ownership was retired before release.", { cause }); + } + }; + try { + const { scan: rootScan, authority: rootAuthority } = await scanAuthenticatedContextRoot(context, options); + if (rootAuthority.kind !== "current") { + throw new Error("Consumer high-water operation lost its exact current checkpoint authority."); + } + await cleanupAuthority(context, claim, rootScan, temporaries, options, false); + await ensureLegacyGuard(context, claim, options); + let chain = await walkTransactions(context, options); + let legacyBytes = null; + if (chain.tipBytes === null) { + const legacy = await readProjection(context, "legacy-state-read", options); + if (legacy.malformed) throw new Error("Consumer high-water state is malformed."); + legacyBytes = legacy.bytes; + } else { + chain = await repairProjection(context, chain, options); + } + const baseBytes = chain.tipBytes ?? legacyBytes; + const baseDigest = baseBytes === null ? GENESIS_DIGEST : digest(baseBytes); + let stagedCandidate = null; + let candidateWasStaged = false; + const commitTransactions = async (candidateBytes) => { + const transactions = []; + if (chain.tipBytes === null && legacyBytes !== null) { + transactions.push(transactionFor(GENESIS_DIGEST, legacyBytes)); + } + if (candidateBytes !== null && digest(candidateBytes) !== baseDigest) { + transactions.push(transactionFor(baseDigest, candidateBytes)); + } + if (transactions.length === 0) return false; + if (chain.length + transactions.length > options.maxTransactionDepth) { + throw new Error("Consumer high-water transaction epoch reached its safe bound; run the consumer journal rotation command."); + } + const wanted = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation: claim.generation, + token: claim.token, + outcome: "commit", + transactions, + }; + await options.hooks?.beforeCommitDecision?.({ claim, transactions }); + terminal = await publishTerminal(context, claim, wanted, options); + if (terminal.outcome !== "commit" || !metadataBytes(terminal).equals(metadataBytes(wanted))) { + throw new Error("Consumer high-water transaction lost ownership before its commit decision."); + } + await options.hooks?.afterCommitDecision?.({ claim, terminal }); + await finishCommit(context, claim, terminal, options); + return true; + }; + const transaction = Object.freeze({ + readStateBytes: () => baseBytes === null ? null : Buffer.from(baseBytes), + commitState: async (value) => { + if (terminal !== null || candidateWasStaged) { + throw new Error("Consumer high-water transaction already staged a candidate or has a terminal decision."); + } + const bytes = Buffer.isBuffer(value) ? Buffer.from(value) : Buffer.from(value); + if (bytes.length < 1 || bytes.length > options.stateMaxBytes) throw new Error("Consumer high-water state is malformed."); + stagedCandidate = bytes; + candidateWasStaged = true; + }, + }); + let result; + let actionError; + try { + result = await action(context.statePath, transaction); + } catch (error) { + actionError = error; + } + if (actionError === undefined && (candidateWasStaged || legacyBytes !== null)) { + await commitTransactions(candidateWasStaged ? stagedCandidate : null); + } + await stopHeartbeatOnce(); + await release(actionError); + if (actionError !== undefined) throw actionError; + return result; + } catch (error) { + await stopHeartbeatOnce(); + await release(error); + throw error; + } + } +} + +function isExpectedRemovedClaimRead(error, context, options) { + if ( + !(error instanceof BoundedFileUnlinkedDuringReadError) || + error.constructor !== BoundedFileUnlinkedDuringReadError || error.name !== "BoundedFileUnlinkedDuringReadError" || + error.description !== "Consumer high-water operation claim" || typeof error.path !== "string" || + !Buffer.isBuffer(error.bytes) || error.bytes.length < 1 || error.bytes.length > options.metadataMaxBytes || + !isExactUnlinkedDuringReadEvidence(error) + ) return false; + const name = basename(error.path); + const match = claimPattern.exec(name); + if ( + !match || error.expectedSha256 !== match[2] || error.sha256 !== match[2] || digest(error.bytes) !== match[2] || + error.path !== join(context.epochDirectory, name) || dirname(error.path) !== context.epochDirectory + ) return false; + let claim; + try { + claim = validateClaim(JSON.parse(error.bytes), context, options.stateMaxBytes); + } catch { + return false; + } + return claim.generation === Number(match[1]) && metadataBytes(claim).equals(error.bytes) && + error.path === claimPath(context, claim); +} + +async function completedRotationResult(context, intent, options) { + const { authority } = await scanAuthenticatedContextRoot(context, options); + if ( + authority.kind !== "successor" || + !metadataBytes(authority.entry.checkpoint).equals(metadataBytes(intent.checkpoint)) + ) return null; + await scanRotationPublicationSet(context, intent.checkpoint, options, true); + return { epoch: intent.checkpoint.epoch, tipSha256: intent.checkpoint.anchorDigest }; +} + +async function recoverCompletedCurrentRotation(context, scan, options) { + if (context.checkpoint.epoch === 1 || scan.claims.length !== 0) return null; + const tip = await effectiveTip(context, options); + if (tip.length !== 0 || tip.tipDigest !== context.checkpoint.anchorDigest) return null; + const writer = { generation: 0, token: context.checkpoint.epochId, type: "rotation" }; + await repairProjection(context, tip, options, writer); + const { scan: rootScan, authority: rootAuthority } = await scanAuthenticatedContextRoot(context, options); + if (rootAuthority.kind !== "current") { + throw new Error("Consumer high-water recovery lost its exact current checkpoint authority."); + } + await cleanupAuthority(context, writer, rootScan, scan.temporaries, options, false, scan); + return { epoch: context.checkpoint.epoch, tipSha256: context.checkpoint.anchorDigest }; +} + +async function runRotation(statePath, rawOptions) { + const options = normalizeRotationOptions(rawOptions); + for (;;) { + let context; + let expectedIntent; + try { + ({ context } = await prepareContext(statePath, options)); + await ensureLegacyGuard(context, { generation: 0, token: context.checkpoint.epochId, type: "rotation" }, options); + const initialScan = await scanEpoch(context, options); + const latest = initialScan.claims.at(-1); + const preparationOptions = latest?.type === "rotation" + ? { ...options, inProgressCheckpoint: validateRotationIntent(latest.intent, context, options.stateMaxBytes).checkpoint } + : options; + const completed = await recoverCompletedCurrentRotation(context, initialScan, preparationOptions); + if (completed) return completed; + expectedIntent = rotationIntentFor(context, await effectiveTip(context, preparationOptions)); + } catch (error) { + if (error instanceof ConsumerEpochAdvancedError) continue; + throw error; + } + let frontier; + try { + frontier = await resolveOperationFrontier(context, options); + } catch (error) { + if (!(error instanceof ConsumerEpochAdvancedError) && !isExpectedRemovedClaimRead(error, context, options)) throw error; + const completedResult = await completedRotationResult(context, expectedIntent, options); + if (completedResult) return completedResult; + throw error; + } + if (frontier.rotated) continue; + if (frontier.active) { + throw new Error("Consumer high-water state is actively locked; rotation will retry after the claim quiesces."); + } + if (frontier.retry) continue; + const nextGeneration = (frontier.scan.claims.at(-1)?.generation ?? 0) + 1; + if (nextGeneration > MAX_OPERATION_GENERATIONS) { + throw new Error("Consumer high-water operation epoch is exhausted and cannot publish its cap-exempt rotation slot."); + } + const tip = await effectiveTip(context, options); + const wanted = rotationClaimFor(context, nextGeneration, tip); + const confirmation = await scanEpoch(context, options); + if (!sameOperationClaim(confirmation.claims.at(-1) ?? null, frontier.frontier)) continue; + const confirmedTip = await effectiveTip(context, options); + const confirmed = rotationClaimFor(context, nextGeneration, confirmedTip); + if (!metadataBytes(confirmed).equals(metadataBytes(wanted))) continue; + let claim; + try { + claim = await tryCreateRotationClaim(context, nextGeneration, confirmedTip, options); + } catch (error) { + const completedResult = await completedRotationResult(context, confirmed.intent, options).catch(() => null); + if (completedResult) return completedResult; + throw error; + } + if (!claim) continue; + try { + await helpRotationOperation(context, claim, options); + } catch (error) { + const completedResult = await completedRotationResult(context, claim.intent, options).catch(() => null); + if (completedResult) return completedResult; + throw error; + } + await scanRotationPublicationSet(context, claim.intent.checkpoint, options, true); + return { epoch: claim.intent.checkpoint.epoch, tipSha256: claim.intent.checkpoint.anchorDigest }; + } +} + +export async function withConsumerStateLock(statePath, action, rawOptions = {}) { + if (typeof action !== "function") throw new Error("Consumer high-water lock action must be a function."); + return runNormalLocked(statePath, action, rawOptions); +} + +export async function rotateConsumerStateJournal(statePath, rawOptions = {}) { + if (typeof statePath !== "string" || !statePath) throw new Error("A consumer-local state path is required for journal rotation."); + return runRotation(statePath, rawOptions); +} diff --git a/scripts/lib/pylon-bounded-file.mjs b/scripts/lib/pylon-bounded-file.mjs index f43bc3292d..66081a3a98 100644 --- a/scripts/lib/pylon-bounded-file.mjs +++ b/scripts/lib/pylon-bounded-file.mjs @@ -147,9 +147,10 @@ export async function readBoundedRegularFile( try { pathEntry = await lstatEntry(path); } catch (error) { - if (error?.code === "ENOENT") return null; + if (lstatEntry === lstat && error?.code === "ENOENT") return null; throw error; } + await hooks?.afterInitialPathStat?.({ path, stat: pathEntry }); if (pathEntry.isSymbolicLink?.() || !pathEntry.isFile()) { throw new Error(`${description} is not one regular non-symlink file.`); } @@ -157,7 +158,7 @@ export async function readBoundedRegularFile( try { handle = await openFile(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); } catch (error) { - if (error?.code === "ENOENT") return null; + if (openFile === open && error?.code === "ENOENT") return null; if (["ELOOP", "EISDIR"].includes(error?.code)) { throw new Error(`${description} is not one regular non-symlink file.`); } @@ -189,7 +190,7 @@ export async function readBoundedRegularFile( try { finalPathEntry = await lstatEntry(path); } catch (error) { - if (error?.code !== "ENOENT") throw error; + if (lstatEntry !== lstat || error?.code !== "ENOENT") throw error; finalPathMissing = true; } let confirmedHandle = null; @@ -275,9 +276,10 @@ export function readBoundedRegularFileSync( try { pathEntry = lstatEntry(path); } catch (error) { - if (error?.code === "ENOENT") return null; + if (lstatEntry === lstatSync && error?.code === "ENOENT") return null; throw error; } + hooks?.afterInitialPathStat?.({ path, stat: pathEntry }); if (pathEntry.isSymbolicLink?.() || !pathEntry.isFile()) { throw new Error(`${description} is not one regular non-symlink file.`); } @@ -285,7 +287,7 @@ export function readBoundedRegularFileSync( try { descriptor = openFile(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); } catch (error) { - if (error?.code === "ENOENT") return null; + if (openFile === openSync && error?.code === "ENOENT") return null; if (["ELOOP", "EISDIR"].includes(error?.code)) throw new Error(`${description} is not one regular non-symlink file.`); throw error; } @@ -314,7 +316,7 @@ export function readBoundedRegularFileSync( try { finalPathEntry = lstatEntry(path); } catch (error) { - if (error?.code !== "ENOENT") throw error; + if (lstatEntry !== lstatSync || error?.code !== "ENOENT") throw error; finalPathMissing = true; } let confirmedHandle = null; diff --git a/scripts/lib/pylon-consumer-lock.mjs b/scripts/lib/pylon-consumer-lock.mjs index eaf377c474..31910bbf96 100644 --- a/scripts/lib/pylon-consumer-lock.mjs +++ b/scripts/lib/pylon-consumer-lock.mjs @@ -474,7 +474,13 @@ async function readSecureFile(path, maxBytes, description, options, minBytes = 1 description, openFile: options.openFile, lstatEntry: options.lstatEntry, - hooks, + hooks: { + ...hooks, + afterInitialPathStat: async (observation) => { + await options.afterInitialPathStat?.(observation); + await hooks?.afterInitialPathStat?.(observation); + }, + }, expectedSha256, validateHandle: (handle, stat) => secureHandle(handle, stat, description, "file", options), }); @@ -788,13 +794,12 @@ function checkpointProofOptions(entry, options, invalidRoot) { let initialPathStat = true; return { ...options, - lstatEntry: async (path) => { - const stat = await options.lstatEntry(path); + afterInitialPathStat: async ({ path, stat }) => { + await options.afterInitialPathStat?.({ path, stat }); if (path === entry.path && initialPathStat) { initialPathStat = false; if (!sameRetiredLinkStat(stat, entry.checkpointStat)) throw invalidRoot(); } - return stat; }, }; } diff --git a/scripts/pylon-bounded-file.test.mjs b/scripts/pylon-bounded-file.test.mjs new file mode 100644 index 0000000000..1c8f5e3128 --- /dev/null +++ b/scripts/pylon-bounded-file.test.mjs @@ -0,0 +1,96 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { closeSync, lstatSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { lstat, open } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { readBoundedRegularFile, readBoundedRegularFileSync } from "./lib/pylon-bounded-file.mjs"; + +const readers = [ + { name: "async", read: readBoundedRegularFile, lstat }, + { name: "sync", read: readBoundedRegularFileSync, lstat: lstatSync }, +]; + +for (const reader of readers) { + for (const code of ["ENOENT", "EIO", "EPERM"]) { + for (const stage of ["initial lstat", "open", "final lstat", "afterInitialPathStat", "afterInitialStat", "beforeFinalStat", "afterFinalStat", "stat", "read", "close"]) { + test(`bounded ${reader.name} retains injected ${code} identity at ${stage}`, async () => { + const fixture = mkdtempSync(join(tmpdir(), "pylon-bounded-identity-")); + const path = join(fixture, "input"); + const bytes = Buffer.from("exact pinned input"); + const failure = Object.assign(new Error(`injected ${stage}`), { code }); + writeFileSync(path, bytes); + const fail = () => { throw failure; }; + let calls = 0; + const options = { maxBytes: 1024, expectedSha256: createHash("sha256").update(bytes).digest("hex") }; + if (stage === "initial lstat") options.lstatEntry = fail; + else if (stage === "open") options.openFile = fail; + else if (stage === "final lstat") { + options.lstatEntry = (target) => { + if (++calls === 2) { + // A real namespace handoff cannot convert an injected error into native evidence. + renameSync(target, `${target}.retired`); + throw failure; + } + return reader.lstat(target); + }; + } else if (["stat", "read", "close"].includes(stage)) { + if (reader.name === "sync") { + if (stage === "stat") options.statFile = fail; + if (stage === "read") options.readFile = fail; + if (stage === "close") options.closeFile = (descriptor) => { closeSync(descriptor); throw failure; }; + } else { + options.openFile = async (target, flags) => { + const handle = await open(target, flags); + return { + stat: stage === "stat" ? fail : handle.stat.bind(handle), + read: stage === "read" ? fail : handle.read.bind(handle), + close: async () => { await handle.close(); if (stage === "close") throw failure; }, + }; + }; + } + } else options.hooks = { [stage]: () => { renameSync(path, `${path}.retired`); throw failure; } }; + try { + await assert.rejects(async () => reader.read(path, options), (error) => error === failure); + } finally { + rmSync(fixture, { recursive: true, force: true }); + } + }); + } + } + test(`bounded ${reader.name} classifies only native initial and open absence`, async () => { + const fixture = mkdtempSync(join(tmpdir(), "pylon-bounded-native-")); + const path = join(fixture, "input"); + try { + assert.equal(await reader.read(path, { maxBytes: 1024 }), null); + writeFileSync(path, "input"); + assert.equal(await reader.read(path, { + maxBytes: 1024, + lstatEntry: reader.name === "async" ? async (target) => { + const stat = await lstat(target); + rmSync(target); + return stat; + } : (target) => { + const stat = lstatSync(target); + rmSync(target); + return stat; + }, + }), null); + } finally { + rmSync(fixture, { recursive: true, force: true }); + } + }); +} + +test("protected publication reader and consumer fixtures retain exact commit provenance", () => { + const root = new URL("./fixtures/protected-publication-v2/", import.meta.url); + const provenance = JSON.parse(readFileSync(new URL("provenance.json", root), "utf8")); + assert.equal(provenance.commit, "68603ed89bb597cd715fd6a77bc1c39d7e110298"); + assert.equal(provenance.repository, "pylon-code/prime-agent"); + assert.deepEqual(provenance.files.map((file) => file.path), ["pylon-consumer-lock.mjs", "pylon-bounded-file.mjs"]); + for (const file of provenance.files) { + assert.equal(file.sourcePath, `scripts/lib/${file.path}`); + assert.equal(createHash("sha256").update(readFileSync(new URL(file.path, root))).digest("hex"), file.sha256); + } +}); diff --git a/scripts/pylon-publication.test.mjs b/scripts/pylon-publication.test.mjs index 2e977dadb9..1821aed17a 100644 --- a/scripts/pylon-publication.test.mjs +++ b/scripts/pylon-publication.test.mjs @@ -1,3 +1,4 @@ +import "./pylon-bounded-file.test.mjs"; import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; @@ -1521,23 +1522,18 @@ test("bounded reads authenticate every exact monotone retirement cut and confirm writeFileSync(path, exactBytes); const base = lstatSync(path); const pathEntry = preciseStat(base, transition === "link" ? 2 : 1, 10); - const before = preciseStat(base, transition === "link" ? 2 : 1, 10); + const before = transition === "unlink" ? base : preciseStat(base, 2, 10); const after = preciseStat(base, transition === "link" ? 1 : 0, 20); const finalPathEntry = preciseStat(base, 1, 20); let lstats = 0; await rejectGenericAsync(() => readBoundedRegularFile(path, { maxBytes: 1024, expectedSha256: exactDigest, - lstatEntry: async () => { + lstatEntry: transition === "unlink" ? lstatFile : async () => { lstats += 1; - if (lstats === 1) return pathEntry; - if (transition === "unlink") { - const missing = new Error("precise final path is absent"); - missing.code = "ENOENT"; - throw missing; - } - return finalPathEntry; + return lstats === 1 ? pathEntry : finalPathEntry; }, + hooks: transition === "unlink" ? { afterFinalStat: () => rmSync(path) } : {}, openFile: async (openedPath, flags) => { const handle = await openFileHandle(openedPath, flags); let stats = 0; @@ -1551,7 +1547,7 @@ test("bounded reads authenticate every exact monotone retirement cut and confirm }; }, })); - assert.equal(lstats, 2); + assert.equal(lstats, transition === "unlink" ? 0 : 2); }; await asyncExtraTransition("eligible-link", "link"); await asyncExtraTransition("eligible-unlink", "unlink"); @@ -2611,6 +2607,13 @@ test("stable checkpoint proof consumes exact retirements and fences namespace ch await releaseProof.promise; }, metadataRead: { + afterInitialPathStat: ({ path }) => { + if (removeBeforeScanOpen && path === original.checkpoint) { + assert.equal(scanRootCaptured, true); + removeBeforeScanOpen = false; + rmSync(path); + } + }, afterInitialStat: async ({ path }) => { if (!phaseArmed || phase !== "afterInitialStat" || path !== proofPath) return; phaseArmed = false; @@ -2633,14 +2636,6 @@ test("stable checkpoint proof consumes exact retirements and fences namespace ch } return names; }, - openFile: async (path, flags, mode) => { - if (removeBeforeScanOpen && path === original.checkpoint) { - assert.equal(scanRootCaptured, true); - removeBeforeScanOpen = false; - rmSync(path); - } - return openFileHandle(path, flags, mode); - }, })); const readerOutcome = outcome(reader); await readerPrepared.promise; @@ -3025,13 +3020,14 @@ test("stable checkpoint proof consumes exact retirements and fences namespace ch const noHigherEpochBefore = directoryBytes(noHigherJournal.epoch); let removeNoHigher = true; await assert.rejects( - () => rotateConsumerStateJournal(noHigherPath, runtime({}, { - openFile: async (path, flags, mode) => { - if (removeNoHigher && path === noHigherJournal.checkpoint) { - removeNoHigher = false; - rmSync(path); - } - return openFileHandle(path, flags, mode); + () => rotateConsumerStateJournal(noHigherPath, runtime({ + metadataRead: { + afterInitialPathStat: ({ path }) => { + if (removeNoHigher && path === noHigherJournal.checkpoint) { + removeNoHigher = false; + rmSync(path); + } + }, }, })), /lost its current checkpoint/, From 59579d1c008c05b70a79ed79c16f6ef8b62552e8 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Thu, 10 Sep 2026 19:45:28 -0600 Subject: [PATCH 02/14] fix(publication): add authenticated generation primitives Implement the bounded generation-format and publication group of the journal replacement. Existing public v2 entrypoints remain active; rotation integration, migration, and final safety evidence remain pending. Fixes #53 --- scripts/lib/pylon-consumer-lock.mjs | 380 +++++++++++++++++++++++- scripts/lib/pylon-generation-format.mjs | 147 +++++++++ scripts/pylon-generation.test.mjs | 300 +++++++++++++++++++ scripts/pylon-publication.test.mjs | 1 + 4 files changed, 826 insertions(+), 2 deletions(-) create mode 100644 scripts/lib/pylon-generation-format.mjs create mode 100644 scripts/pylon-generation.test.mjs diff --git a/scripts/lib/pylon-consumer-lock.mjs b/scripts/lib/pylon-consumer-lock.mjs index 31910bbf96..dc81b917ae 100644 --- a/scripts/lib/pylon-consumer-lock.mjs +++ b/scripts/lib/pylon-consumer-lock.mjs @@ -10,6 +10,22 @@ import { readBoundedRegularFile, } from "./pylon-bounded-file.mjs"; +import { + consumerGenerationGenesisCheckpoint, + consumerGenerationName, + consumerGenerationRotationClaim, + consumerGenerationSuccessorCheckpoint, + generationRecordMaxBytes, + GENERATION_EPOCH_MAX_ENTRIES, + GENERATION_JOURNAL_MAX_BYTES, + GENERATION_RECEIPT_MAX_ENTRIES, + GENERATION_ROOT_MAX_ENTRIES, + GENERATION_STATE_MAX_BYTES, + validateGenerationCheckpoint, +} from "./pylon-generation-format.mjs"; + +export { consumerGenerationGenesisCheckpoint, consumerGenerationName, consumerGenerationRotationClaim, consumerGenerationSuccessorCheckpoint }; + class ConsumerEpochAdvancedError extends Error { constructor() { super("Consumer high-water journal epoch changed and fenced a paused writer."); @@ -267,7 +283,7 @@ function validateRotationIntent(value, context, stateMaxBytes) { return value; } -function validateTerminal(value, claim, stateMaxBytes) { +function validateTerminal(value, claim, stateMaxBytes, validatePayload = validateTransaction) { const common = ["schemaVersion", "generation", "token", "outcome"]; if ( claim.type !== "normal" || !value || value.schemaVersion !== LOCK_SCHEMA_VERSION || value.generation !== claim.generation || @@ -284,7 +300,7 @@ function validateTerminal(value, claim, stateMaxBytes) { let expectedBase = value.transactions[0]?.baseDigest; if (!/^[0-9a-f]{64}$/.test(expectedBase ?? "")) throw new Error("Consumer high-water lock commit marker is malformed."); for (const transaction of value.transactions) { - validateTransaction(transaction, expectedBase, stateMaxBytes); + validatePayload(transaction, expectedBase, stateMaxBytes); expectedBase = transaction.candidateDigest; } return value; @@ -3754,3 +3770,363 @@ export async function rotateConsumerStateJournal(statePath, rawOptions = {}) { if (typeof statePath !== "string" || !statePath) throw new Error("A consumer-local state path is required for journal rotation."); return runRotation(statePath, rawOptions); } + +// V3 primitives remain separate from the public v2 preparation/rotation entrypoints. +const generationBuilders = new WeakMap(); +const generationReceiptPattern = /^receipt-([0-9a-f]{64})\.json$/; +const generationReceiptTemporaryPattern = new RegExp(`^\\.receipt-p([1-9][0-9]*)-w(${uuidSource})-t([0-9a-f]{64})\\.tmp$`); +function generationOptions(raw = {}) { + const options = { + stateMaxBytes: GENERATION_STATE_MAX_BYTES, + maxJournalBytes: GENERATION_JOURNAL_MAX_BYTES, + currentUid: process.getuid(), lstatEntry: lstat, openFile: open, readDirectory: readdir, + makeDirectory: mkdir, linkFile: link, renameFile: rename, ...raw, + }; + options.metadataMaxBytes = generationRecordMaxBytes(options.stateMaxBytes); + if (!Number.isSafeInteger(options.maxJournalBytes) || options.maxJournalBytes < 1 || options.maxJournalBytes > GENERATION_JOURNAL_MAX_BYTES || + !Number.isSafeInteger(options.currentUid) || options.currentUid < 0) throw new Error("Generation byte bound or uid is invalid."); + return options; +} +function generationCanonical(bytes, maxBytes) { + if (!Buffer.isBuffer(bytes) || bytes.length < 1 || bytes.length > maxBytes) throw new Error("Generation metadata exceeds its byte bound."); + const value = JSON.parse(bytes); + if (!metadataBytes(value).equals(bytes)) throw new Error("Generation metadata is not canonical."); + return value; +} +function validateGenerationTransaction(value, expectedBaseDigest, stateMaxBytes) { + if (!exactKeys(value, ["schemaVersion", "baseDigest", "candidateDigest", "candidateBase64"]) || value.schemaVersion !== 1 || + value.baseDigest !== expectedBaseDigest || !/^[0-9a-f]{64}$/.test(value.candidateDigest ?? "") || + typeof value.candidateBase64 !== "string" || value.candidateBase64.length > 4 * Math.ceil(stateMaxBytes / 3)) throw new Error("Generation transaction is malformed or exceeds its byte bound."); + const candidateBytes = Buffer.from(value.candidateBase64, "base64"); + if (candidateBytes.length < 1 || candidateBytes.length > stateMaxBytes || candidateBytes.toString("base64") !== value.candidateBase64 || + digest(candidateBytes) !== value.candidateDigest || value.candidateDigest === value.baseDigest) throw new Error("Generation transaction payload is malformed."); + return { value, candidateBytes }; +} +function generationEpochAuthority(snapshot, options) { + const checkpoint = validateGenerationCheckpoint(snapshot.checkpoint, options.stateMaxBytes).checkpoint; + if (!Buffer.isBuffer(snapshot.checkpointBytes) || !metadataBytes(checkpoint).equals(snapshot.checkpointBytes) || snapshot.name !== consumerGenerationName(checkpoint)) throw new Error("Generation predecessor checkpoint authority is not exact."); + const records = snapshot.epochRecords; + if (!(records instanceof Map) || records.size > GENERATION_EPOCH_MAX_ENTRIES) throw new Error("Generation epoch entry bound is invalid."); + let totalBytes = snapshot.checkpointBytes.length * 2; + for (const bytes of records.values()) { + if (!Buffer.isBuffer(bytes) || bytes.length < 1 || bytes.length > options.metadataMaxBytes) throw new Error("Generation epoch metadata byte bound is invalid."); + totalBytes += bytes.length * 2; + if (totalBytes > options.maxJournalBytes) throw new Error("Generation epoch exceeds its byte bound."); + } + const contents = new Map(); + const indexes = new Map(); + const heartbeats = new Map(); + const terminals = new Map(); + const applied = new Map(); + const transitions = new Map(); + for (const [name, bytes] of records) { + const value = generationCanonical(bytes, options.metadataMaxBytes); + let match; + if ((match = claimPattern.exec(name))) { + if (value.generation !== Number(match[1]) || generationName(value.generation) !== match[1] || digest(bytes) !== match[2]) throw new Error("Generation claim name is not exact."); + if (value.type === "rotation") { + const successor = validateGenerationCheckpoint(value.intent?.checkpoint, options.stateMaxBytes); + const expected = consumerGenerationRotationClaim(checkpoint, value.generation, { + tipDigest: successor.checkpoint.anchorDigest, tipBytes: successor.anchorBytes, + }, options.stateMaxBytes); + if (!bytes.equals(metadataBytes(expected))) throw new Error("Generation rotation intent is not exact."); + } else validateClaim(value, null, options.stateMaxBytes); + contents.set(match[2], value); + } else if ((match = claimIndexPattern.exec(name))) { + validateClaimIndex(value, Number(match[1])); + if (generationName(value.generation) !== match[1]) throw new Error("Generation claim CAS name is malformed."); + indexes.set(value.generation, value); + } else if ((match = heartbeatPattern.exec(name))) heartbeats.set(`${Number(match[1])}:${match[2]}`, value); + else if ((match = terminalPattern.exec(name))) terminals.set(`${Number(match[1])}:${match[2]}`, value); + else if ((match = appliedPattern.exec(name))) applied.set(`${Number(match[1])}:${match[2]}`, value); + else if ((match = transitionPattern.exec(name))) { + validateGenerationTransaction(value, match[1], options.stateMaxBytes); + transitions.set(match[1], value); + } else throw new Error("Generation epoch contains an unexpected entry."); + } + const claims = []; + const byKey = new Map(); + for (const [slot, index] of [...indexes].sort(([a], [b]) => a - b)) { + const claim = contents.get(index.claimSha256); + if (!claim || claim.generation !== slot || slot !== claims.length + 1) throw new Error("Generation rotation authority has a missing claim CAS or noncontiguous slot."); + claims.push(claim); + byKey.set(`${claim.generation}:${claim.token}`, claim); + } + for (const claim of contents.values()) { + if (claim.generation > claims.length + 1) throw new Error("Generation epoch contains a future unindexed claim."); + } + for (const [key, value] of heartbeats) { + const claim = byKey.get(key); + if (!claim) throw new Error("Generation epoch contains an orphan heartbeat."); + validateHeartbeat(value, claim); + } + for (const [key, value] of terminals) { + const claim = byKey.get(key); + if (!claim) throw new Error("Generation epoch contains an orphan terminal."); + validateTerminal(value, claim, options.stateMaxBytes, validateGenerationTransaction); + } + for (const [key, value] of applied) { + const claim = byKey.get(key); + if (!claim) throw new Error("Generation epoch contains an orphan applied marker."); + validateApplied(value, claim, terminals.get(key)); + } + let tipDigest = checkpoint.anchorDigest; + let tipBytes = validateGenerationCheckpoint(checkpoint, options.stateMaxBytes).anchorBytes; + const decided = new Map(); + let depth = 0; + for (const claim of claims) { + const key = `${claim.generation}:${claim.token}`; + const terminal = terminals.get(key); + if (claim !== claims.at(-1) && (claim.type === "rotation" || !terminal || (terminal.outcome === "commit" && !applied.has(key)))) throw new Error("Generation epoch crossed an unresolved earlier slot."); + if (terminal?.outcome !== "commit") continue; + for (const transaction of terminal.transactions) { + if (transaction.baseDigest !== tipDigest || decided.has(tipDigest) || ++depth > MAX_TRANSACTION_DEPTH) throw new Error("Generation commit decisions do not form one bounded exact chain."); + decided.set(tipDigest, transaction); + tipDigest = transaction.candidateDigest; + } + } + let actualDigest = checkpoint.anchorDigest; + const visited = new Set(); + while (transitions.has(actualDigest)) { + const actual = transitions.get(actualDigest); + if (visited.has(actualDigest) || !decided.has(actualDigest) || !metadataBytes(actual).equals(metadataBytes(decided.get(actualDigest)))) throw new Error("Generation transition lacks its exact commit decision."); + visited.add(actualDigest); + tipBytes = validateGenerationTransaction(actual, actualDigest, options.stateMaxBytes).candidateBytes; + actualDigest = actual.candidateDigest; + } + if (visited.size !== transitions.size) throw new Error("Generation epoch has an unreachable transition."); + for (const key of applied.keys()) { + for (const transaction of terminals.get(key).transactions) { + if (!transitions.has(transaction.baseDigest) || !metadataBytes(transitions.get(transaction.baseDigest)).equals(metadataBytes(transaction))) throw new Error("Generation applied marker lacks its exact transition."); + } + } + return { claims, terminals, applied, tip: { tipDigest: actualDigest, tipBytes }, decidedTipDigest: tipDigest }; +} + +// The caller must read/revalidate the predecessor from its pinned directory immediately +// before a handoff. Snapshots are evidence inputs, never a permission to reuse stale authority. +export function assertConsumerGenerationSuccessor(predecessor, name, bytes, rawOptions = {}) { + const options = generationOptions(rawOptions); + const scan = generationEpochAuthority(predecessor, options); + const latest = scan.claims.at(-1); + if (latest?.type !== "rotation" || scan.decidedTipDigest !== scan.tip.tipDigest) throw new Error("Generation successor lacks exact latest rotation authority."); + const expected = consumerGenerationRotationClaim(predecessor.checkpoint, latest.generation, scan.tip, options.stateMaxBytes); + if (!metadataBytes(latest).equals(metadataBytes(expected)) || name !== consumerGenerationName(expected.intent.checkpoint) || !Buffer.isBuffer(bytes) || !bytes.equals(metadataBytes(expected.intent.checkpoint))) throw new Error("Generation successor differs from its exact latest rotation authority and immutable tip."); + return expected.intent.checkpoint; +} +function expectedConsumerGeneration(authority, options) { + if (exactKeys(authority, ["genesis"])) return consumerGenerationGenesisCheckpoint(authority.genesis, options.stateMaxBytes); + if (!exactKeys(authority, ["predecessor"])) throw new Error("Generation construction requires exact genesis or predecessor authority."); + const scan = generationEpochAuthority(authority.predecessor, options); + const candidate = scan.claims.at(-1)?.intent?.checkpoint; + if (!candidate) throw new Error("Generation successor lacks latest rotation authority."); + return assertConsumerGenerationSuccessor(authority.predecessor, consumerGenerationName(candidate), metadataBytes(candidate), options); +} +function generationSameInode(a, b) { return a.dev === b.dev && a.ino === b.ino; } +function generationEntryStat(stat, type, options) { + if (stat.isSymbolicLink?.() || (type === "directory" ? !stat.isDirectory() : !stat.isFile()) || stat.uid !== options.currentUid || (stat.mode & 0o7777) !== (type === "directory" ? 0o700 : 0o600)) throw new Error("Generation entry has unsafe type, owner or exact permissions."); + return stat; +} +async function generationDirectory(path, options, sync = false) { + const before = generationEntryStat(await options.lstatEntry(path), "directory", options); + await options.hooks?.afterInitialPathStat?.({ path, stat: before }); + const handle = await options.openFile(path, constants.O_RDONLY | (constants.O_DIRECTORY ?? 0) | (constants.O_NOFOLLOW ?? 0)); + try { + const opened = generationEntryStat(await handle.stat(), "directory", options); + if (!generationSameInode(before, opened)) throw new Error("Generation directory identity changed."); + if (sync) await handle.sync(); + const final = generationEntryStat(await options.lstatEntry(path), "directory", options); + if (!generationSameInode(opened, final)) throw new Error("Generation directory identity changed."); + return Object.freeze({ dev: opened.dev, ino: opened.ino }); + } finally { await handle.close(); } +} +async function generationBoundary(options, phase, operation, path) { + await options.hooks?.generationBoundary?.({ phase, operation, path }); +} +async function generationSync(path, options) { + await generationBoundary(options, "before", "sync", path); + await generationDirectory(path, options, true); + await generationBoundary(options, "after", "sync", path); +} +async function generationNames(path, limit, options) { + const names = await options.readDirectory(path); + if (!Array.isArray(names) || names.length > limit || new Set(names).size !== names.length || names.some((name) => typeof name !== "string" || basename(name) !== name || [".", ".."].includes(name))) throw new Error("Generation directory exceeds its entry bound or closed namespace."); + return names; +} +async function generationRootPreflight(root, options) { + const rootNames = await generationNames(root, GENERATION_ROOT_MAX_ENTRIES, options); + const finalPattern = /^generation-[0-9]{16}-[0-9a-f]{64}$/; + const hiddenPattern = new RegExp(`^\\.building-${uuidSource}$`); + const retiredPattern = /^\.((retired)|(deleting))-generation-[0-9]{16}-[0-9a-f]{64}$/; + if (rootNames.filter((name) => finalPattern.test(name)).length > 2 || rootNames.some((name) => !finalPattern.test(name) && !hiddenPattern.test(name) && !retiredPattern.test(name))) throw new Error("Generation root contains an unexpected entry or competing finals."); + let totalBytes = 0; + const charge = async (path) => { + const stat = generationEntryStat(await options.lstatEntry(path), "file", options); + if (stat.size < 1 || stat.size > options.metadataMaxBytes) throw new Error("Generation root metadata exceeds its byte bound."); + totalBytes += stat.size; + if (totalBytes > options.maxJournalBytes) throw new Error("Generation root exceeds its aggregate byte bound."); + }; + for (const name of rootNames) { + const path = join(root, name); + generationEntryStat(await options.lstatEntry(path), "directory", options); + const entries = await generationNames(path, 3, options); + if (entries.some((entry) => !["checkpoint.json", "epoch", "receipts"].includes(entry))) throw new Error("Generation root contains an unexpected nested entry."); + for (const entry of entries) { + if (entry === "checkpoint.json") { await charge(join(path, entry)); continue; } + const directory = join(path, entry); + generationEntryStat(await options.lstatEntry(directory), "directory", options); + const names = await generationNames(directory, entry === "epoch" ? GENERATION_EPOCH_MAX_ENTRIES : GENERATION_RECEIPT_MAX_ENTRIES, options); + for (const child of names) await charge(join(directory, child)); + } + } + return totalBytes; +} +async function readGenerationSnapshot(path, checkpoint, options, expectedIdentity = null, requireEmpty = false) { + const root = dirname(path); + await generationRootPreflight(root, options); + const names = await generationNames(path, 3, options); + if (names.slice().sort().join() !== "checkpoint.json,epoch,receipts") throw new Error("Generation has an incomplete or unexpected closed namespace."); + const epochNames = await generationNames(join(path, "epoch"), GENERATION_EPOCH_MAX_ENTRIES, options); + const receiptNames = await generationNames(join(path, "receipts"), GENERATION_RECEIPT_MAX_ENTRIES, options); + if (requireEmpty && (epochNames.length !== 0 || receiptNames.length !== 1)) throw new Error("Generation publication requires an exactly empty epoch and checkpoint-only receipts."); + // Stat and charge every name, including both hardlinks, before opening nested metadata. + const canonical = new Map(); + const byInode = new Map(); + const receiptEntries = []; + let totalBytes = 0; + for (const name of ["checkpoint.json", ...epochNames.map((name) => `epoch/${name}`)]) { + const stat = generationEntryStat(await options.lstatEntry(join(path, name)), "file", options); + if (stat.size < 1 || stat.size > options.metadataMaxBytes) throw new Error("Generation metadata exceeds its byte bound."); + totalBytes += stat.size; + if (totalBytes > options.maxJournalBytes) throw new Error("Generation exceeds its aggregate byte bound."); + const key = `${stat.dev}:${stat.ino}`; + if (byInode.has(key)) throw new Error("Generation canonical entries alias the same inode."); + const entry = { name, stat, receipts: [] }; + canonical.set(name, entry); byInode.set(key, entry); + } + for (const name of receiptNames) { + const fixed = generationReceiptPattern.exec(name); + const temporary = generationReceiptTemporaryPattern.exec(name); + if (!fixed && !temporary) throw new Error("Generation receipt name is malformed."); + const stat = generationEntryStat(await options.lstatEntry(join(path, "receipts", name)), "file", options); + if (stat.size < 1 || stat.size > options.metadataMaxBytes) throw new Error("Generation receipt exceeds its byte bound."); + totalBytes += stat.size; + if (totalBytes > options.maxJournalBytes) throw new Error("Generation exceeds its aggregate byte bound."); + const target = byInode.get(`${stat.dev}:${stat.ino}`); + if (!target || (fixed?.[1] ?? temporary?.[3]) !== digest(Buffer.from(target.name)) || stat.size !== target.stat.size) throw new Error("Generation receipt lacks its exact canonical inode and target."); + target.receipts.push({ name, stat, temporary: !!temporary }); + receiptEntries.push({ name, stat, target: target.name, temporary: !!temporary }); + } + for (const entry of canonical.values()) { + if (entry.receipts.length !== 1 || entry.stat.nlink !== 2 || entry.receipts[0].stat.nlink !== 2 || (requireEmpty && entry.receipts[0].temporary)) throw new Error("Generation canonical metadata lacks its exact durable receipt inode."); + } + await generationDirectory(root, options); + const identity = await generationDirectory(path, options); + if (expectedIdentity !== null && !generationSameInode(identity, expectedIdentity)) throw new Error("Generation directory inode differs from the observed builder identity."); + const epochIdentity = await generationDirectory(join(path, "epoch"), options); + const receiptsIdentity = await generationDirectory(join(path, "receipts"), options); + const epochRecords = new Map(); + let checkpointBytes; + for (const entry of canonical.values()) { + const bytes = await readBoundedRegularFile(join(path, entry.name), { + maxBytes: options.metadataMaxBytes, openFile: options.openFile, lstatEntry: options.lstatEntry, + hooks: { ...options.hooks?.metadataRead, afterInitialPathStat: async (observation) => { + if (!sameRetiredLinkStat(observation.stat, entry.stat)) throw new Error("Generation metadata changed after allocation preflight."); + await options.hooks?.metadataRead?.afterInitialPathStat?.(observation); + } }, + validateHandle: async (_handle, stat) => generationEntryStat(stat, "file", options), + }); + if (bytes === null) throw new Error("Generation required metadata disappeared."); + if (entry.name === "checkpoint.json") checkpointBytes = bytes; + else epochRecords.set(entry.name.slice(6), bytes); + } + if (!checkpointBytes.equals(metadataBytes(checkpoint))) throw new Error("Generation checkpoint differs from exact expected authority."); + const snapshot = { path, name: consumerGenerationName(checkpoint), checkpoint, checkpointBytes, epochRecords, identity, totalBytes, receiptEntries }; + generationEpochAuthority(snapshot, options); + for (const entry of receiptEntries) { + if (!sameRetiredLinkStat(entry.stat, await options.lstatEntry(join(path, "receipts", entry.name)))) throw new Error("Generation receipt inode or stat changed."); + } + for (const [directory, observed, expectedNames, limit] of [[path, identity, names, 3], [join(path, "epoch"), epochIdentity, epochNames, GENERATION_EPOCH_MAX_ENTRIES], [join(path, "receipts"), receiptsIdentity, receiptNames, GENERATION_RECEIPT_MAX_ENTRIES]]) { + if (!generationSameInode(observed, await generationDirectory(directory, options)) || (await generationNames(directory, limit, options)).sort().join() !== expectedNames.slice().sort().join()) throw new Error("Generation namespace changed during validation."); + } + return snapshot; +} +export async function readConsumerGeneration(path, authority, rawOptions = {}) { + const options = generationOptions(rawOptions); + const checkpoint = expectedConsumerGeneration(authority, options); + if (basename(path) !== consumerGenerationName(checkpoint)) throw new Error("Generation final name differs from its exact authority."); + return readGenerationSnapshot(path, checkpoint, options); +} +async function publishGenerationCheckpoint(path, checkpoint, options) { + const receipts = join(path, "receipts"); + const target = join(path, "checkpoint.json"); + const targetHash = digest(Buffer.from("checkpoint.json")); + const temporary = join(receipts, `.receipt-p${process.pid}-w${randomUUID()}-t${targetHash}.tmp`); + const fixed = join(receipts, `receipt-${targetHash}.json`); + const handle = await options.openFile(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | (constants.O_NOFOLLOW ?? 0), 0o600); + try { + await handle.writeFile(metadataBytes(checkpoint)); + await generationBoundary(options, "before", "file-sync", temporary); + await handle.sync(); + await generationBoundary(options, "after", "file-sync", temporary); + } finally { await handle.close(); } + await generationSync(receipts, options); + await generationBoundary(options, "before", "link", target); + await options.linkFile(temporary, target); + await generationBoundary(options, "after", "link", target); + await generationSync(path, options); + await generationBoundary(options, "before", "rename", fixed); + await options.renameFile(temporary, fixed); + await generationBoundary(options, "after", "rename", fixed); + await generationSync(receipts, options); +} +export async function buildConsumerGeneration(root, authority, rawOptions = {}) { + const options = generationOptions(rawOptions); + const checkpoint = expectedConsumerGeneration(authority, options); + if (2 * metadataBytes(checkpoint).length > options.maxJournalBytes) throw new Error("Generation checkpoint and receipt exceed aggregate byte bound."); + await generationNames(root, GENERATION_ROOT_MAX_ENTRIES - 1, options); + const rootBytes = await generationRootPreflight(root, options); + if (rootBytes + 2 * metadataBytes(checkpoint).length > options.maxJournalBytes) throw new Error("Generation build exceeds the root aggregate byte bound."); + const rootIdentity = await generationDirectory(root, options); + const path = join(root, `.building-${randomUUID()}`); + await options.makeDirectory(path, { mode: 0o700 }); + const identity = await generationDirectory(path, options); + await options.makeDirectory(join(path, "epoch"), { mode: 0o700 }); + await options.makeDirectory(join(path, "receipts"), { mode: 0o700 }); + await generationSync(join(path, "epoch"), options); + await generationSync(join(path, "receipts"), options); + await publishGenerationCheckpoint(path, checkpoint, options); + await generationSync(join(path, "epoch"), options); + await generationSync(path, options); + await generationSync(root, options); + await readGenerationSnapshot(path, checkpoint, options, identity, true); + const builder = Object.freeze({ path, identity }); + generationBuilders.set(builder, { path, identity, rootIdentity, checkpoint }); + return builder; +} +export async function publishConsumerGeneration(builder, rawOptions = {}) { + const evidence = generationBuilders.get(builder); + if (!evidence) throw new Error("Generation publication requires an observed builder identity."); + const options = generationOptions(rawOptions); + const { path: source, identity, checkpoint, rootIdentity } = evidence; + const root = dirname(source); + const destination = join(root, consumerGenerationName(checkpoint)); + await readGenerationSnapshot(source, checkpoint, options, identity, true); + if (!generationSameInode(rootIdentity, await generationDirectory(root, options))) throw new Error("Generation root inode changed before publication."); + const observation = { source, destination, identity }; + await options.hooks?.beforeGenerationRename?.(observation); + await generationBoundary(options, "before", "rename", destination); + try { + await options.renameFile(source, destination); + } catch (error) { + if (options.renameFile !== rename || error?.code !== "ENOENT") throw error; + // Only native source loss can join this rename; a byte-identical winner is not ours. + await readGenerationSnapshot(destination, checkpoint, options, identity, true); + } + await generationBoundary(options, "after", "rename", destination); + await options.hooks?.afterGenerationRename?.(observation); + const result = await readGenerationSnapshot(destination, checkpoint, options, identity, true); + if (!generationSameInode(rootIdentity, await generationDirectory(root, options))) throw new Error("Generation root inode changed after publication."); + await generationSync(root, options); + return result; +} diff --git a/scripts/lib/pylon-generation-format.mjs b/scripts/lib/pylon-generation-format.mjs new file mode 100644 index 0000000000..67a4a12d49 --- /dev/null +++ b/scripts/lib/pylon-generation-format.mjs @@ -0,0 +1,147 @@ +import { createHash } from "node:crypto"; +import { resolve } from "node:path"; + +export const GENERATION_STATE_MAX_BYTES = 16 * 1024 * 1024; +export const GENERATION_ZERO = "0".repeat(64); +export const GENERATION_ROOT_MAX_ENTRIES = 16; +export const GENERATION_EPOCH_MAX_ENTRIES = 65_537 * 5 + 4096 + 32; +export const GENERATION_RECEIPT_MAX_ENTRIES = GENERATION_EPOCH_MAX_ENTRIES * 2 + 2; +export const GENERATION_JOURNAL_MAX_BYTES = 512 * 1024 * 1024; +const hex = /^[0-9a-f]{64}$/; +const fields = [ + "schemaVersion", "epoch", "epochId", "statePathSha256", "previousCheckpointSha256", "previousGeneration", + "previousTipSha256", "historySha256", "anchorDigest", "anchorBase64", "sourceKind", "sourceAuthoritySha256", + "sourceTipDigest", "sourceTipBase64", "migrationKind", "migrationAuthoritySha256", "migrationTipDigest", "migrationTipBase64", +]; +export const generationBytes = (value) => Buffer.from(`${JSON.stringify(value)}\n`); +export const generationDigest = (value) => createHash("sha256").update(value).digest("hex"); +export function generationCheckpointMaxBytes(stateMaxBytes = GENERATION_STATE_MAX_BYTES) { + if (!Number.isSafeInteger(stateMaxBytes) || stateMaxBytes < 1 || stateMaxBytes > GENERATION_STATE_MAX_BYTES) throw new Error("Generation state bound is invalid."); + return 3 * 4 * Math.ceil(stateMaxBytes / 3) + checkpointEnvelopeBytes; +} +const checkpointEnvelopeBytes = generationBytes({ + schemaVersion: 3, epoch: Number.MAX_SAFE_INTEGER, epochId: GENERATION_ZERO, statePathSha256: GENERATION_ZERO, + previousCheckpointSha256: GENERATION_ZERO, previousGeneration: `generation-9007199254740991-${GENERATION_ZERO}`, + previousTipSha256: GENERATION_ZERO, historySha256: GENERATION_ZERO, anchorDigest: GENERATION_ZERO, anchorBase64: "", + sourceKind: "v2", sourceAuthoritySha256: GENERATION_ZERO, sourceTipDigest: GENERATION_ZERO, sourceTipBase64: "", + migrationKind: "v1", migrationAuthoritySha256: GENERATION_ZERO, migrationTipDigest: GENERATION_ZERO, migrationTipBase64: "", +}).length; +// Rotation claims wrap the checkpoint; normal terminal records contain at most two state fields. +export function generationRecordMaxBytes(stateMaxBytes = GENERATION_STATE_MAX_BYTES) { + const rotationEnvelopeBytes = generationBytes({ + schemaVersion: 3, generation: 65_537, token: GENERATION_ZERO, type: "rotation", intent: { + schemaVersion: 3, predecessorGeneration: `generation-9007199254740991-${GENERATION_ZERO}`, + checkpointSha256: GENERATION_ZERO, tipSha256: GENERATION_ZERO, checkpoint: null, + }, + }).length - 5; + return generationCheckpointMaxBytes(stateMaxBytes) + rotationEnvelopeBytes; +} +function commitment(domain, values) { + const hash = createHash("sha256").update(`${domain}\0`); + for (const value of values) { + const data = Buffer.from(value); + const length = Buffer.alloc(8); + length.writeBigUInt64BE(BigInt(data.length)); + hash.update(length).update(data); + } + return hash.digest("hex"); +} +function stateFields(data, stateMaxBytes) { + if (data === null) return { digest: GENERATION_ZERO, base64: null }; + if (!Buffer.isBuffer(data) || data.length < 1 || data.length > stateMaxBytes) throw new Error("Generation state exceeds its byte bound."); + return { digest: generationDigest(data), base64: data.toString("base64") }; +} +function decodeState(encoded, digest, stateMaxBytes) { + if (!hex.test(digest)) throw new Error("Generation state digest is malformed."); + if (encoded === null) { + if (digest !== GENERATION_ZERO) throw new Error("Generation null state digest is malformed."); + return null; + } + if (typeof encoded !== "string" || encoded.length < 4 || encoded.length > 4 * Math.ceil(stateMaxBytes / 3)) throw new Error("Generation state exceeds its byte bound."); + const data = Buffer.from(encoded, "base64"); + if (data.length < 1 || data.length > stateMaxBytes || data.toString("base64") !== encoded || generationDigest(data) !== digest) throw new Error("Generation state bytes are malformed."); + return data; +} +function identity(value) { + const { epochId: _epochId, ...payload } = value; + return commitment("pylon-generation-identity-v3", [generationBytes(payload)]); +} +export function consumerGenerationName(value) { + if (!Number.isSafeInteger(value.epoch) || value.epoch < 1 || typeof value.epochId !== "string" || !hex.test(value.epochId)) throw new Error("Generation name is malformed."); + return `generation-${String(value.epoch).padStart(16, "0")}-${value.epochId}`; +} +function provenance(value, stateMaxBytes, migration = false) { + if (value === null) return { kind: null, authoritySha256: GENERATION_ZERO, digest: GENERATION_ZERO, base64: null }; + if (!value || Object.keys(value).sort().join() !== "authoritySha256,kind,tipBytes" || + !(migration ? value.kind === "v1" : ["v1", "v2"].includes(value.kind)) || typeof value.authoritySha256 !== "string" || !hex.test(value.authoritySha256) || value.authoritySha256 === GENERATION_ZERO) throw new Error("Generation provenance is malformed."); + return { kind: value.kind, authoritySha256: value.authoritySha256, ...stateFields(value.tipBytes, stateMaxBytes) }; +} +export function consumerGenerationGenesisCheckpoint({ statePath, stateBytes = null, source = null, migration = null }, stateMaxBytes = GENERATION_STATE_MAX_BYTES) { + generationCheckpointMaxBytes(stateMaxBytes); + if (typeof statePath !== "string" || statePath.length === 0) throw new Error("Generation state path is required."); + const anchor = stateFields(stateBytes, stateMaxBytes); + const src = provenance(source, stateMaxBytes); + const old = provenance(migration, stateMaxBytes, true); + if (old.kind !== null && src.kind !== "v2") throw new Error("Generation migration provenance requires a v2 source."); + if (src.kind !== null && (src.digest !== anchor.digest || src.base64 !== anchor.base64)) throw new Error("Generation source must bind the exact genesis state."); + const value = { + schemaVersion: 3, epoch: 1, epochId: "", statePathSha256: generationDigest(Buffer.from(resolve(statePath))), + previousCheckpointSha256: GENERATION_ZERO, previousGeneration: null, previousTipSha256: GENERATION_ZERO, + historySha256: "", anchorDigest: anchor.digest, anchorBase64: anchor.base64, + sourceKind: src.kind, sourceAuthoritySha256: src.authoritySha256, sourceTipDigest: src.digest, sourceTipBase64: src.base64, + migrationKind: old.kind, migrationAuthoritySha256: old.authoritySha256, migrationTipDigest: old.digest, migrationTipBase64: old.base64, + }; + value.historySha256 = genesisHistory(value); + value.epochId = identity(value); + return value; +} +function genesisHistory(value) { + return commitment("pylon-generation-genesis-history-v3", [ + value.statePathSha256, value.anchorDigest, value.sourceKind ?? "none", value.sourceAuthoritySha256, + value.sourceTipDigest, value.migrationKind ?? "none", value.migrationAuthoritySha256, value.migrationTipDigest, + ]); +} +export function validateGenerationCheckpoint(input, stateMaxBytes = GENERATION_STATE_MAX_BYTES) { + generationCheckpointMaxBytes(stateMaxBytes); + if (!input || Object.keys(input).sort().join() !== [...fields].sort().join()) throw new Error("Generation checkpoint closed format is malformed."); + const value = Object.fromEntries(fields.map((key) => [key, input[key]])); + if (value.schemaVersion !== 3 || !Number.isSafeInteger(value.epoch) || value.epoch < 1 || + ![value.epochId, value.statePathSha256, value.previousCheckpointSha256, value.previousTipSha256, value.historySha256, + value.sourceAuthoritySha256, value.migrationAuthoritySha256].every((v) => typeof v === "string" && hex.test(v))) throw new Error("Generation checkpoint is malformed."); + const anchorBytes = decodeState(value.anchorBase64, value.anchorDigest, stateMaxBytes); + decodeState(value.sourceTipBase64, value.sourceTipDigest, stateMaxBytes); + decodeState(value.migrationTipBase64, value.migrationTipDigest, stateMaxBytes); + for (const prefix of ["source", "migration"]) { + if (value[`${prefix}Kind`] === null) { + if (value[`${prefix}AuthoritySha256`] !== GENERATION_ZERO || value[`${prefix}TipBase64`] !== null) throw new Error("Generation absent provenance is malformed."); + } else if (!(prefix === "source" ? ["v1", "v2"] : ["v1"]).includes(value[`${prefix}Kind`]) || value[`${prefix}AuthoritySha256`] === GENERATION_ZERO) throw new Error("Generation provenance is malformed."); + } + if (value.migrationKind !== null && value.sourceKind !== "v2") throw new Error("Generation migration provenance is malformed."); + if (value.epoch === 1) { + if (value.previousCheckpointSha256 !== GENERATION_ZERO || value.previousTipSha256 !== GENERATION_ZERO || value.previousGeneration !== null || value.historySha256 !== genesisHistory(value) || + (value.sourceKind !== null && (value.anchorBase64 !== value.sourceTipBase64 || value.anchorDigest !== value.sourceTipDigest))) throw new Error("Generation genesis is not exact."); + } else if (typeof value.previousGeneration !== "string" || !/^generation-[0-9]{16}-[0-9a-f]{64}$/.test(value.previousGeneration) || value.previousTipSha256 !== value.anchorDigest) throw new Error("Generation successor is malformed."); + if (generationBytes(value).length > generationCheckpointMaxBytes(stateMaxBytes)) throw new Error("Generation checkpoint exceeds its exact envelope byte bound."); + if (value.epochId !== identity(value)) throw new Error("Generation deterministic identity is not exact."); + return { checkpoint: value, anchorBytes }; +} +export function consumerGenerationSuccessorCheckpoint(predecessor, tip, stateMaxBytes = GENERATION_STATE_MAX_BYTES) { + const { checkpoint } = validateGenerationCheckpoint(predecessor, stateMaxBytes); + const anchor = stateFields(tip.tipBytes, stateMaxBytes); + if (anchor.digest !== tip.tipDigest || checkpoint.epoch === Number.MAX_SAFE_INTEGER) throw new Error("Generation immutable tip or epoch is malformed."); + const previousDigest = generationDigest(generationBytes(checkpoint)); + const next = { ...checkpoint, epoch: checkpoint.epoch + 1, epochId: "", previousCheckpointSha256: previousDigest, + previousGeneration: consumerGenerationName(checkpoint), previousTipSha256: anchor.digest, + historySha256: commitment("pylon-generation-rotation-history-v3", [checkpoint.historySha256, previousDigest, anchor.digest]), + anchorDigest: anchor.digest, anchorBase64: anchor.base64 }; + next.epochId = identity(next); + return next; +} +export function consumerGenerationRotationClaim(checkpoint, generation, tip, stateMaxBytes = GENERATION_STATE_MAX_BYTES) { + if (!Number.isSafeInteger(generation) || generation < 1 || generation > 65_537) throw new Error("Generation rotation slot is malformed."); + const successor = consumerGenerationSuccessorCheckpoint(checkpoint, tip, stateMaxBytes); + return { schemaVersion: 3, generation, token: successor.epochId, type: "rotation", intent: { + schemaVersion: 3, predecessorGeneration: consumerGenerationName(checkpoint), checkpointSha256: generationDigest(generationBytes(checkpoint)), + tipSha256: tip.tipDigest, checkpoint: successor, + } }; +} diff --git a/scripts/pylon-generation.test.mjs b/scripts/pylon-generation.test.mjs new file mode 100644 index 0000000000..f8bc823567 --- /dev/null +++ b/scripts/pylon-generation.test.mjs @@ -0,0 +1,300 @@ +import assert from "node:assert/strict"; +import { createHash, randomUUID } from "node:crypto"; +import { chmod, cp, link, lstat, mkdtemp, open, readFile, readdir, rename, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { test } from "node:test"; +import * as journal from "./lib/pylon-consumer-lock.mjs"; +import { generationCheckpointMaxBytes } from "./lib/pylon-generation-format.mjs"; + +const hash = (bytes) => createHash("sha256").update(bytes).digest("hex"); +const bytes = (value) => Buffer.from(`${JSON.stringify(value)}\n`); +const statePath = "/consumer/state.json"; +const genesis = (stateBytes = Buffer.from("initial"), rest = {}) => ({ genesis: { statePath, stateBytes, ...rest } }); +async function fixture(t) { + const root = await mkdtemp(join(tmpdir(), "pylon-generation-")); + await chmod(root, 0o700); + t.after(() => rm(root, { recursive: true, force: true })); + return root; +} +async function installed(t, authority = genesis()) { + const root = await fixture(t); + const builder = await journal.buildConsumerGeneration(root, authority); + const result = await journal.publishConsumerGeneration(builder); + return { root, authority, ...result }; +} +async function record(path, name, value) { + await writeFile(join(path, "epoch", name), bytes(value), { mode: 0o600 }); + await link(join(path, "epoch", name), join(path, "receipts", `receipt-${hash(Buffer.from(`epoch/${name}`))}.json`)); +} +async function predecessor(t) { + const result = await installed(t); + const checkpoint = result.checkpoint; + const claim = journal.consumerGenerationRotationClaim(checkpoint, 1, { tipBytes: Buffer.from("initial"), tipDigest: hash(Buffer.from("initial")) }); + await record(result.path, `claim-0000000000000001-${hash(bytes(claim))}.json`, claim); + await record(result.path, "claim-index-0000000000000001.json", { schemaVersion: 1, generation: 1, claimSha256: hash(bytes(claim)) }); + const snapshot = await journal.readConsumerGeneration(result.path, result.authority); + return { ...result, claim, snapshot }; +} + +test("v3 generation genesis reconstructs exact state, identity, history and provenance", async (t) => { + const authority = genesis(); + const first = await installed(t, authority); + const expected = journal.consumerGenerationGenesisCheckpoint(authority.genesis); + assert.deepEqual(first.checkpoint, expected); + assert.equal(expected.anchorBase64, Buffer.from("initial").toString("base64")); + assert.notEqual(expected.epochId, journal.consumerGenerationGenesisCheckpoint(genesis(Buffer.from("other")).genesis).epochId); + await assert.rejects(journal.readConsumerGeneration(first.path, genesis(Buffer.from("other"))), /exact|authority/); + for (const stateBytes of [Buffer.alloc(0), Buffer.alloc(16 * 1024 * 1024 + 1)]) { + assert.throws(() => journal.consumerGenerationGenesisCheckpoint(genesis(stateBytes).genesis), /bound|state/); + } +}); + +test("v3 generation requires exact latest rotation claim CAS and byte-bound successor", async (t) => { + const p = await predecessor(t); + const candidate = bytes(p.claim.intent.checkpoint); + const name = journal.consumerGenerationName(p.claim.intent.checkpoint); + assert.doesNotThrow(() => journal.assertConsumerGenerationSuccessor(p.snapshot, name, candidate)); + for (const field of Object.keys(p.claim.intent.checkpoint)) { + const malformed = structuredClone(p.claim.intent.checkpoint); + malformed[field] = typeof malformed[field] === "number" ? malformed[field] + 1 : "wrong"; + assert.throws(() => journal.assertConsumerGenerationSuccessor(p.snapshot, name, bytes(malformed)), undefined, field); + } + assert.throws(() => journal.assertConsumerGenerationSuccessor({ ...p.snapshot, epochRecords: new Map() }, name, candidate), /rotation|authority/); + const absentIndex = new Map(p.snapshot.epochRecords); + absentIndex.delete("claim-index-0000000000000001.json"); + assert.throws(() => journal.assertConsumerGenerationSuccessor({ ...p.snapshot, epochRecords: absentIndex }, name, candidate), /rotation|authority/); + const unknown = new Map(p.snapshot.epochRecords).set("unexpected.json", bytes({})); + assert.throws(() => journal.assertConsumerGenerationSuccessor({ ...p.snapshot, epochRecords: unknown }, name, candidate), /epoch|entry/); + const next = await journal.buildConsumerGeneration(p.root, { predecessor: p.snapshot }); + const published = await journal.publishConsumerGeneration(next); + assert.equal(published.checkpoint.epoch, 2); +}); + +test("v3 generation receipt and publication durability order", async (t) => { + const root = await fixture(t); + const events = []; + const hooks = { generationBoundary: ({ phase, operation, path }) => { events.push([phase, operation, basename(path)]); } }; + const builder = await journal.buildConsumerGeneration(root, genesis(), { hooks }); + assert.deepEqual((await readdir(root)), [basename(builder.path)]); + assert.deepEqual(await readdir(join(builder.path, "epoch")), []); + assert.equal((await readdir(join(builder.path, "receipts"))).length, 1); + const published = await journal.publishConsumerGeneration(builder, { hooks }); + const operations = events.filter(([phase]) => phase === "after").map(([, operation, path]) => `${operation}:${path}`); + const fileSync = operations.findIndex((value) => value.startsWith("file-sync:.receipt-")); + const receiptSync = operations.indexOf("sync:receipts", fileSync + 1); + const canonicalLink = operations.indexOf("link:checkpoint.json"); + const receiptRename = operations.findIndex((value) => value.startsWith("rename:receipt-")); + const finalRename = operations.indexOf(`rename:${basename(published.path)}`); + assert.ok(fileSync >= 0 && fileSync < receiptSync); + assert.ok(receiptSync < canonicalLink && canonicalLink < receiptRename && receiptRename < finalRename); + assert.ok(operations.slice(canonicalLink + 1, receiptRename).includes(`sync:${basename(builder.path)}`)); + assert.ok(operations.slice(receiptRename + 1, finalRename).includes("sync:receipts")); + assert.ok(operations.slice(receiptRename + 1, finalRename).includes("sync:epoch")); + assert.equal(operations.at(-1), `sync:${basename(root)}`); +}); + +test("v3 generation rejects incomplete epoch, orphan receipts and identical receipt replacements", async (t) => { + for (const mutation of [ + async (p) => writeFile(join(p, "epoch", "unknown.json"), "{}", { mode: 0o600 }), + async (p) => writeFile(join(p, "receipts", `receipt-${"0".repeat(64)}.json`), "{}", { mode: 0o600 }), + async (p) => { const r = join(p, "receipts", (await readdir(join(p, "receipts")))[0]); const b = await readFile(r); await rm(r); await writeFile(r, b, { mode: 0o600 }); }, + async (p) => { await rm(join(p, "epoch"), { recursive: true }); }, + async (p) => chmod(join(p, "receipts"), 0o755), + async (p) => { await rm(join(p, "epoch"), { recursive: true }); await symlink("receipts", join(p, "epoch")); }, + ]) { + const g = await installed(t); + await mutation(g.path); + await assert.rejects(journal.readConsumerGeneration(g.path, g.authority)); + } +}); + +test("v3 generation native rename loss joins only the observed builder inode", async (t) => { + const root = await fixture(t); + const builder = await journal.buildConsumerGeneration(root, genesis()); + let moved = false; + const result = await journal.publishConsumerGeneration(builder, { hooks: { beforeGenerationRename: async ({ source, destination }) => { + moved = true; + await rename(source, destination); + } } }); + assert.ok(moved); + assert.equal((await lstat(result.path)).ino, builder.identity.ino); + const otherRoot = await fixture(t); + const other = await journal.buildConsumerGeneration(otherRoot, genesis()); + await assert.rejects(journal.publishConsumerGeneration(other, { hooks: { beforeGenerationRename: async ({ source, destination }) => { + await cp(source, destination, { recursive: true }); + const receipt = join(destination, "receipts", (await readdir(join(destination, "receipts")))[0]); + await rm(receipt); await link(join(destination, "checkpoint.json"), receipt); + await rm(source, { recursive: true }); + } } }), /inode|identity/); +}); + +for (const code of ["ENOENT", "EIO", "EPERM"]) { + test(`v3 generation preserves injected ${code} identity even with concurrent rename`, async (t) => { + for (const stage of ["beforeGenerationRename", "afterGenerationRename", "renameFile"]) { + const root = await fixture(t); + const builder = await journal.buildConsumerGeneration(root, genesis()); + const error = Object.assign(new Error(`injected ${stage}`), { code }); + const options = stage === "renameFile" ? { renameFile: async (source, destination) => { await rename(source, destination); throw error; } } : { hooks: { [stage]: async ({ source, destination }) => { + if (stage === "beforeGenerationRename") await rename(source, destination); + throw error; + } } }; + await assert.rejects(journal.publishConsumerGeneration(builder, options), (actual) => actual === error); + } + }); +} + +test("v3 generation bounds all three actual 16 MiB base64 fields and receipt duplication", async (t) => { + const state = Buffer.alloc(16 * 1024 * 1024, 0x61); + const provenance = { kind: "v2", authoritySha256: "1".repeat(64), tipBytes: state }; + const migration = { kind: "v1", authoritySha256: "2".repeat(64), tipBytes: state }; + const authority = genesis(state, { source: provenance, migration }); + const value = journal.consumerGenerationGenesisCheckpoint(authority.genesis); + const encoded = bytes(value); + assert.ok(encoded.length > 67_108_872); + assert.ok(encoded.length <= generationCheckpointMaxBytes()); + const root = await fixture(t); + const builder = await journal.buildConsumerGeneration(root, authority); + const g = await journal.publishConsumerGeneration(builder); + assert.equal(g.totalBytes, 2 * encoded.length); + await assert.rejects(journal.readConsumerGeneration(g.path, authority, { maxJournalBytes: 2 * encoded.length - 1 }), /byte bound/); + const claim = journal.consumerGenerationRotationClaim(g.checkpoint, 1, { tipDigest: hash(state), tipBytes: state }); + await record(g.path, `claim-0000000000000001-${hash(bytes(claim))}.json`, claim); + await record(g.path, "claim-index-0000000000000001.json", { schemaVersion: 1, generation: 1, claimSha256: hash(bytes(claim)) }); + const snapshot = await journal.readConsumerGeneration(g.path, authority); + const successor = await journal.buildConsumerGeneration(root, { predecessor: snapshot }); + const rotated = await journal.publishConsumerGeneration(successor); + assert.equal(rotated.checkpoint.epoch, 2); + assert.ok(snapshot.totalBytes + rotated.totalBytes > 384 * 1024 * 1024); +}); + +test("v3 generation enforces root, epoch and receipt counts before nested reads", async (t) => { + const g = await installed(t); + for (const target of [g.root, join(g.path, "epoch"), join(g.path, "receipts")]) { + let nested = false; + await assert.rejects(journal.readConsumerGeneration(g.path, g.authority, { + readDirectory: async (path) => path === target ? Array.from({ length: 700_001 }, (_, i) => `entry-${i}`) : readdir(path), + openFile: async () => { nested = true; throw new Error("nested open"); }, + }), /entry bound/); + assert.equal(nested, false); + } +}); + +test("v3 generation epoch authenticates commits, indexed losers, latest intent and immutable tip", async (t) => { + const g = await installed(t); + const token = randomUUID(); + const normal = { schemaVersion: 2, generation: 1, token, type: "normal", ownerPid: process.pid, createdAtMs: 1 }; + const tx = { schemaVersion: 1, baseDigest: g.checkpoint.anchorDigest, candidateDigest: hash(Buffer.from("updated")), candidateBase64: Buffer.from("updated").toString("base64") }; + const terminal = { schemaVersion: 2, generation: 1, token, outcome: "commit", transactions: [tx] }; + await record(g.path, `claim-0000000000000001-${hash(bytes(normal))}.json`, normal); + await record(g.path, "claim-index-0000000000000001.json", { schemaVersion: 1, generation: 1, claimSha256: hash(bytes(normal)) }); + await record(g.path, `terminal-0000000000000001-${token}.json`, terminal); + // A latest decided but unapplied commit is a valid helpable epoch, never rotation authority. + const incomplete = await journal.readConsumerGeneration(g.path, g.authority); + const intended = journal.consumerGenerationRotationClaim(g.checkpoint, 2, { tipDigest: tx.candidateDigest, tipBytes: Buffer.from("updated") }); + assert.throws(() => journal.assertConsumerGenerationSuccessor(incomplete, journal.consumerGenerationName(intended.intent.checkpoint), bytes(intended.intent.checkpoint)), /rotation authority/); + await record(g.path, `transition-${tx.baseDigest}.json`, tx); + await record(g.path, `applied-0000000000000001-${token}.json`, { schemaVersion: 2, generation: 1, token, terminalSha256: hash(bytes(terminal)) }); + await record(g.path, `claim-0000000000000002-${hash(bytes(intended))}.json`, intended); + await record(g.path, "claim-index-0000000000000002.json", { schemaVersion: 1, generation: 2, claimSha256: hash(bytes(intended)) }); + const loser = { ...normal, generation: 2, token: randomUUID() }; + await record(g.path, `claim-0000000000000002-${hash(bytes(loser))}.json`, loser); + const snapshot = await journal.readConsumerGeneration(g.path, g.authority); + const name = journal.consumerGenerationName(intended.intent.checkpoint); + assert.doesNotThrow(() => journal.assertConsumerGenerationSuccessor(snapshot, name, bytes(intended.intent.checkpoint))); + for (const mutation of [ + (records) => records.set("claim-index-0000000000000002.json", bytes({ schemaVersion: 1, generation: 2, claimSha256: hash(bytes(loser)) })), + (records) => records.delete(`transition-${tx.baseDigest}.json`), + (records) => records.delete(`applied-0000000000000001-${token}.json`), + (records) => { const future = { ...normal, generation: 3, token: randomUUID() }; records.set(`claim-0000000000000003-${hash(bytes(future))}.json`, bytes(future)); records.set("claim-index-0000000000000003.json", bytes({ schemaVersion: 1, generation: 3, claimSha256: hash(bytes(future)) })); }, + ]) { + const epochRecords = new Map(snapshot.epochRecords); mutation(epochRecords); + assert.throws(() => journal.assertConsumerGenerationSuccessor({ ...snapshot, epochRecords }, name, bytes(intended.intent.checkpoint))); + } +}); + +test("v3 generation rejects mutated hidden builder and malformed destination after native loss", async (t) => { + for (const duringRename of [false, true]) { + const root = await fixture(t); + const builder = await journal.buildConsumerGeneration(root, genesis()); + const corrupt = (path) => writeFile(join(path, "epoch", "extra.json"), "{}", { mode: 0o600 }); + if (!duringRename) await corrupt(builder.path); + await assert.rejects(journal.publishConsumerGeneration(builder, { hooks: { beforeGenerationRename: async ({ source, destination }) => { + await rename(source, destination); await corrupt(destination); + } } })); + } +}); + +test("v3 generation preserves metadata read hook errors through concurrent ancestor rename", async (t) => { + for (const code of ["ENOENT", "EIO", "EPERM"]) { + const g = await installed(t); + const error = Object.assign(new Error("metadata fault"), { code }); + await assert.rejects(journal.readConsumerGeneration(g.path, g.authority, { hooks: { metadataRead: { afterInitialPathStat: async () => { + await rename(g.path, join(g.root, `.retired-${basename(g.path)}`)); throw error; + } } } }), (actual) => actual === error); + } +}); + +test("v3 generation charges the entire root including hidden builders before file reads", async (t) => { + const g = await installed(t); + await assert.rejects(journal.buildConsumerGeneration(g.root, genesis(), { maxJournalBytes: g.totalBytes }), /aggregate byte bound/); + assert.equal((await readdir(g.root)).length, 1); + await journal.buildConsumerGeneration(g.root, genesis()); + let fileOpened = false; + await assert.rejects(journal.readConsumerGeneration(g.path, g.authority, { maxJournalBytes: g.totalBytes, hooks: { metadataRead: { afterInitialStat: () => { fileOpened = true; } } } }), /aggregate byte bound/); + assert.equal(fileOpened, false); + await writeFile(join(g.root, "unexpected"), "data", { mode: 0o600 }); + await assert.rejects(journal.readConsumerGeneration(g.path, g.authority), /unexpected entry/); +}); + +test("v3 generation injected filesystem failures remain terminal during a concurrent rename", async (t) => { + for (const code of ["ENOENT", "EIO", "EPERM"]) { + for (const operation of ["lstatEntry", "openFile", "readDirectory"]) { + const g = await installed(t); + const error = Object.assign(new Error(`${operation} fault`), { code }); + const original = { lstatEntry: lstat, openFile: open, readDirectory: readdir }[operation]; + let fired = false; + await assert.rejects(journal.readConsumerGeneration(g.path, g.authority, { [operation]: async (path, ...args) => { + if (!fired && path.startsWith(g.path)) { + fired = true; await rename(g.path, join(g.root, `.retired-${basename(g.path)}`)); throw error; + } + return original(path, ...args); + } }), (actual) => actual === error); + assert.ok(fired); + } + } +}); + +test("v3 generation receipt crash cuts retain the durable temporary proof", async (t) => { + for (const cut of ["file-sync", "link", "rename"]) { + const root = await fixture(t); + const error = new Error(`cut after receipt ${cut}`); + await assert.rejects(journal.buildConsumerGeneration(root, genesis(), { hooks: { generationBoundary: ({ phase, operation }) => { + if (phase === "after" && operation === cut) throw error; + } } }), (actual) => actual === error); + const [building] = await readdir(root); + assert.ok(building.startsWith(".building-")); + const path = join(root, building); + const [receipt] = await readdir(join(path, "receipts")); + assert.ok(receipt); + const receiptStat = await lstat(join(path, "receipts", receipt)); + assert.equal(receiptStat.nlink, cut === "file-sync" ? 1 : 2); + if (cut !== "file-sync") assert.equal((await lstat(join(path, "checkpoint.json"))).ino, receiptStat.ino); + assert.equal(receipt.startsWith("receipt-"), cut === "rename"); + } +}); + +test("v3 generation validates an actual 16 MiB immutable transaction without recursive base64 matching", async (t) => { + const g = await installed(t); + const state = Buffer.alloc(16 * 1024 * 1024, 0x62); + const token = randomUUID(); + const claim = { schemaVersion: 2, generation: 1, token, type: "normal", ownerPid: process.pid, createdAtMs: 1 }; + const transaction = { schemaVersion: 1, baseDigest: g.checkpoint.anchorDigest, candidateDigest: hash(state), candidateBase64: state.toString("base64") }; + const terminal = { schemaVersion: 2, generation: 1, token, outcome: "commit", transactions: [transaction] }; + await record(g.path, `claim-0000000000000001-${hash(bytes(claim))}.json`, claim); + await record(g.path, "claim-index-0000000000000001.json", { schemaVersion: 1, generation: 1, claimSha256: hash(bytes(claim)) }); + await record(g.path, `terminal-0000000000000001-${token}.json`, terminal); + await record(g.path, `transition-${transaction.baseDigest}.json`, transaction); + const snapshot = await journal.readConsumerGeneration(g.path, g.authority); + assert.ok(snapshot.epochRecords.get(`transition-${transaction.baseDigest}.json`).equals(bytes(transaction))); +}); diff --git a/scripts/pylon-publication.test.mjs b/scripts/pylon-publication.test.mjs index 1821aed17a..df835a42b5 100644 --- a/scripts/pylon-publication.test.mjs +++ b/scripts/pylon-publication.test.mjs @@ -1,3 +1,4 @@ +import "./pylon-generation.test.mjs"; import "./pylon-bounded-file.test.mjs"; import assert from "node:assert/strict"; import { spawn } from "node:child_process"; From 1b9ebfbc777b6409ae2881087570cfc3a1e5dea9 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Thu, 10 Sep 2026 20:28:51 -0600 Subject: [PATCH 03/14] fix(publication): checkpoint generation rotation and cleanup Partial group 3 implementation toward fixes #53. Public v3 entrypoints remain disabled; projection temporary ownership and crash cleanup still require followup before group 3 is complete. --- scripts/lib/pylon-consumer-lock.mjs | 692 ++++++++++++++++++- scripts/lib/pylon-generation-format.mjs | 14 +- scripts/pylon-generation-operations.test.mjs | 211 ++++++ scripts/pylon-publication.test.mjs | 1 + 4 files changed, 892 insertions(+), 26 deletions(-) create mode 100644 scripts/pylon-generation-operations.test.mjs diff --git a/scripts/lib/pylon-consumer-lock.mjs b/scripts/lib/pylon-consumer-lock.mjs index dc81b917ae..2534dc2563 100644 --- a/scripts/lib/pylon-consumer-lock.mjs +++ b/scripts/lib/pylon-consumer-lock.mjs @@ -3773,6 +3773,7 @@ export async function rotateConsumerStateJournal(statePath, rawOptions = {}) { // V3 primitives remain separate from the public v2 preparation/rotation entrypoints. const generationBuilders = new WeakMap(); +const generationHeartbeatPattern = new RegExp(`^heartbeat-([0-9]{16})-(${uuidSource})-([0-9]{16})\\.json$`); const generationReceiptPattern = /^receipt-([0-9a-f]{64})\.json$/; const generationReceiptTemporaryPattern = new RegExp(`^\\.receipt-p([1-9][0-9]*)-w(${uuidSource})-t([0-9a-f]{64})\\.tmp$`); function generationOptions(raw = {}) { @@ -3780,7 +3781,8 @@ function generationOptions(raw = {}) { stateMaxBytes: GENERATION_STATE_MAX_BYTES, maxJournalBytes: GENERATION_JOURNAL_MAX_BYTES, currentUid: process.getuid(), lstatEntry: lstat, openFile: open, readDirectory: readdir, - makeDirectory: mkdir, linkFile: link, renameFile: rename, ...raw, + makeDirectory: mkdir, linkFile: link, renameFile: rename, removeFile: rm, now: Date.now, + processKill: process.kill.bind(process), stale: PYLON_CONSUMER_LOCK_STALE_MS, ...raw, }; options.metadataMaxBytes = generationRecordMaxBytes(options.stateMaxBytes); if (!Number.isSafeInteger(options.maxJournalBytes) || options.maxJournalBytes < 1 || options.maxJournalBytes > GENERATION_JOURNAL_MAX_BYTES || @@ -3816,6 +3818,7 @@ function generationEpochAuthority(snapshot, options) { const contents = new Map(); const indexes = new Map(); const heartbeats = new Map(); + const heartbeatRecords = []; const terminals = new Map(); const applied = new Map(); const transitions = new Map(); @@ -3827,7 +3830,7 @@ function generationEpochAuthority(snapshot, options) { if (value.type === "rotation") { const successor = validateGenerationCheckpoint(value.intent?.checkpoint, options.stateMaxBytes); const expected = consumerGenerationRotationClaim(checkpoint, value.generation, { - tipDigest: successor.checkpoint.anchorDigest, tipBytes: successor.anchorBytes, + tipDigest: successor.checkpoint.anchorDigest, tipBytes: successor.anchorBytes, previousGenerationIdentity: successor.checkpoint.previousGenerationIdentity, retirementAuthoritySha256: successor.checkpoint.retirementAuthoritySha256, }, options.stateMaxBytes); if (!bytes.equals(metadataBytes(expected))) throw new Error("Generation rotation intent is not exact."); } else validateClaim(value, null, options.stateMaxBytes); @@ -3836,7 +3839,16 @@ function generationEpochAuthority(snapshot, options) { validateClaimIndex(value, Number(match[1])); if (generationName(value.generation) !== match[1]) throw new Error("Generation claim CAS name is malformed."); indexes.set(value.generation, value); - } else if ((match = heartbeatPattern.exec(name))) heartbeats.set(`${Number(match[1])}:${match[2]}`, value); + } else if ((match = heartbeatPattern.exec(name))) { + heartbeatRecords.push([`${Number(match[1])}:${match[2]}`, value]); + const key = `${Number(match[1])}:${match[2]}`; + if (!heartbeats.has(key) || heartbeats.get(key).refreshedAtMs < value.refreshedAtMs) heartbeats.set(key, value); + } else if ((match = generationHeartbeatPattern.exec(name))) { + heartbeatRecords.push([`${Number(match[1])}:${match[2]}`, value]); + if (value.refreshedAtMs !== Number(match[3]) || value.generation !== Number(match[1]) || value.token !== match[2]) throw new Error("Generation immutable heartbeat name is not exact."); + const key = `${Number(match[1])}:${match[2]}`; + if (!heartbeats.has(key) || heartbeats.get(key).refreshedAtMs < value.refreshedAtMs) heartbeats.set(key, value); + } else if ((match = terminalPattern.exec(name))) terminals.set(`${Number(match[1])}:${match[2]}`, value); else if ((match = appliedPattern.exec(name))) applied.set(`${Number(match[1])}:${match[2]}`, value); else if ((match = transitionPattern.exec(name))) { @@ -3855,7 +3867,7 @@ function generationEpochAuthority(snapshot, options) { for (const claim of contents.values()) { if (claim.generation > claims.length + 1) throw new Error("Generation epoch contains a future unindexed claim."); } - for (const [key, value] of heartbeats) { + for (const [key, value] of heartbeatRecords) { const claim = byKey.get(key); if (!claim) throw new Error("Generation epoch contains an orphan heartbeat."); validateHeartbeat(value, claim); @@ -3900,7 +3912,7 @@ function generationEpochAuthority(snapshot, options) { if (!transitions.has(transaction.baseDigest) || !metadataBytes(transitions.get(transaction.baseDigest)).equals(metadataBytes(transaction))) throw new Error("Generation applied marker lacks its exact transition."); } } - return { claims, terminals, applied, tip: { tipDigest: actualDigest, tipBytes }, decidedTipDigest: tipDigest }; + return { claims, contents, indexes, heartbeats, terminals, applied, transitions, tip: { tipDigest: actualDigest, tipBytes, length: visited.size }, decidedTipDigest: tipDigest }; } // The caller must read/revalidate the predecessor from its pinned directory immediately @@ -3910,7 +3922,8 @@ export function assertConsumerGenerationSuccessor(predecessor, name, bytes, rawO const scan = generationEpochAuthority(predecessor, options); const latest = scan.claims.at(-1); if (latest?.type !== "rotation" || scan.decidedTipDigest !== scan.tip.tipDigest) throw new Error("Generation successor lacks exact latest rotation authority."); - const expected = consumerGenerationRotationClaim(predecessor.checkpoint, latest.generation, scan.tip, options.stateMaxBytes); + const expected = consumerGenerationRotationClaim(predecessor.checkpoint, latest.generation, { ...scan.tip, previousGenerationIdentity: latest.intent.checkpoint.previousGenerationIdentity, retirementAuthoritySha256: latest.intent.checkpoint.retirementAuthoritySha256 }, options.stateMaxBytes); + if (expected.intent.checkpoint.retirementAuthoritySha256 !== GENESIS_DIGEST) validateGenerationRetirementCertificate(predecessor, expected.intent.checkpoint, options); if (!metadataBytes(latest).equals(metadataBytes(expected)) || name !== consumerGenerationName(expected.intent.checkpoint) || !Buffer.isBuffer(bytes) || !bytes.equals(metadataBytes(expected.intent.checkpoint))) throw new Error("Generation successor differs from its exact latest rotation authority and immutable tip."); return expected.intent.checkpoint; } @@ -3953,26 +3966,33 @@ async function generationNames(path, limit, options) { if (!Array.isArray(names) || names.length > limit || new Set(names).size !== names.length || names.some((name) => typeof name !== "string" || basename(name) !== name || [".", ".."].includes(name))) throw new Error("Generation directory exceeds its entry bound or closed namespace."); return names; } +class GenerationDiscoveryLost extends Error {} +const generationDiscovery = Symbol("generation discovery"); async function generationRootPreflight(root, options) { const rootNames = await generationNames(root, GENERATION_ROOT_MAX_ENTRIES, options); const finalPattern = /^generation-[0-9]{16}-[0-9a-f]{64}$/; - const hiddenPattern = new RegExp(`^\\.building-${uuidSource}$`); + const hiddenPattern = new RegExp(`^\\.building-(?:p[1-9][0-9]*-)?${uuidSource}$`); const retiredPattern = /^\.((retired)|(deleting))-generation-[0-9]{16}-[0-9a-f]{64}$/; if (rootNames.filter((name) => finalPattern.test(name)).length > 2 || rootNames.some((name) => !finalPattern.test(name) && !hiddenPattern.test(name) && !retiredPattern.test(name))) throw new Error("Generation root contains an unexpected entry or competing finals."); let totalBytes = 0; const charge = async (path) => { const stat = generationEntryStat(await options.lstatEntry(path), "file", options); - if (stat.size < 1 || stat.size > options.metadataMaxBytes) throw new Error("Generation root metadata exceeds its byte bound."); + if (stat.size < (generationReceiptTemporaryPattern.test(basename(path)) ? 0 : 1) || stat.size > options.metadataMaxBytes) throw new Error("Generation root metadata exceeds its byte bound."); totalBytes += stat.size; if (totalBytes > options.maxJournalBytes) throw new Error("Generation root exceeds its aggregate byte bound."); }; for (const name of rootNames) { const path = join(root, name); - generationEntryStat(await options.lstatEntry(path), "directory", options); - const entries = await generationNames(path, 3, options); - if (entries.some((entry) => !["checkpoint.json", "epoch", "receipts"].includes(entry))) throw new Error("Generation root contains an unexpected nested entry."); + try { + generationEntryStat(await options.lstatEntry(path), "directory", options); + } catch (error) { + if (options[generationDiscovery] && options.lstatEntry === lstat && error?.code === "ENOENT") throw new GenerationDiscoveryLost(); + throw error; + } + const entries = await generationNames(path, 4, options); + if (entries.some((entry) => !["checkpoint.json", "retirement.json", "epoch", "receipts"].includes(entry))) throw new Error("Generation root contains an unexpected nested entry."); for (const entry of entries) { - if (entry === "checkpoint.json") { await charge(join(path, entry)); continue; } + if (["checkpoint.json", "retirement.json"].includes(entry)) { await charge(join(path, entry)); continue; } const directory = join(path, entry); generationEntryStat(await options.lstatEntry(directory), "directory", options); const names = await generationNames(directory, entry === "epoch" ? GENERATION_EPOCH_MAX_ENTRIES : GENERATION_RECEIPT_MAX_ENTRIES, options); @@ -3984,8 +4004,8 @@ async function generationRootPreflight(root, options) { async function readGenerationSnapshot(path, checkpoint, options, expectedIdentity = null, requireEmpty = false) { const root = dirname(path); await generationRootPreflight(root, options); - const names = await generationNames(path, 3, options); - if (names.slice().sort().join() !== "checkpoint.json,epoch,receipts") throw new Error("Generation has an incomplete or unexpected closed namespace."); + const names = await generationNames(path, 4, options); + if (!["checkpoint.json,epoch,receipts", "checkpoint.json,epoch,receipts,retirement.json"].includes(names.slice().sort().join())) throw new Error("Generation has an incomplete or unexpected closed namespace."); const epochNames = await generationNames(join(path, "epoch"), GENERATION_EPOCH_MAX_ENTRIES, options); const receiptNames = await generationNames(join(path, "receipts"), GENERATION_RECEIPT_MAX_ENTRIES, options); if (requireEmpty && (epochNames.length !== 0 || receiptNames.length !== 1)) throw new Error("Generation publication requires an exactly empty epoch and checkpoint-only receipts."); @@ -3994,7 +4014,7 @@ async function readGenerationSnapshot(path, checkpoint, options, expectedIdentit const byInode = new Map(); const receiptEntries = []; let totalBytes = 0; - for (const name of ["checkpoint.json", ...epochNames.map((name) => `epoch/${name}`)]) { + for (const name of ["checkpoint.json", ...(names.includes("retirement.json") ? ["retirement.json"] : []), ...epochNames.map((name) => `epoch/${name}`)]) { const stat = generationEntryStat(await options.lstatEntry(join(path, name)), "file", options); if (stat.size < 1 || stat.size > options.metadataMaxBytes) throw new Error("Generation metadata exceeds its byte bound."); totalBytes += stat.size; @@ -4009,13 +4029,17 @@ async function readGenerationSnapshot(path, checkpoint, options, expectedIdentit const temporary = generationReceiptTemporaryPattern.exec(name); if (!fixed && !temporary) throw new Error("Generation receipt name is malformed."); const stat = generationEntryStat(await options.lstatEntry(join(path, "receipts", name)), "file", options); - if (stat.size < 1 || stat.size > options.metadataMaxBytes) throw new Error("Generation receipt exceeds its byte bound."); + if (stat.size < (temporary ? 0 : 1) || stat.size > options.metadataMaxBytes) throw new Error("Generation receipt exceeds its byte bound."); totalBytes += stat.size; if (totalBytes > options.maxJournalBytes) throw new Error("Generation exceeds its aggregate byte bound."); const target = byInode.get(`${stat.dev}:${stat.ino}`); + if (!target && temporary && stat.nlink === 1) { + receiptEntries.push({ name, stat, target: null, temporary: true, pid: Number(temporary[1]) }); + continue; + } if (!target || (fixed?.[1] ?? temporary?.[3]) !== digest(Buffer.from(target.name)) || stat.size !== target.stat.size) throw new Error("Generation receipt lacks its exact canonical inode and target."); target.receipts.push({ name, stat, temporary: !!temporary }); - receiptEntries.push({ name, stat, target: target.name, temporary: !!temporary }); + receiptEntries.push({ name, stat, target: target.name, temporary: !!temporary, pid: temporary ? Number(temporary[1]) : null }); } for (const entry of canonical.values()) { if (entry.receipts.length !== 1 || entry.stat.nlink !== 2 || entry.receipts[0].stat.nlink !== 2 || (requireEmpty && entry.receipts[0].temporary)) throw new Error("Generation canonical metadata lacks its exact durable receipt inode."); @@ -4027,6 +4051,7 @@ async function readGenerationSnapshot(path, checkpoint, options, expectedIdentit const receiptsIdentity = await generationDirectory(join(path, "receipts"), options); const epochRecords = new Map(); let checkpointBytes; + let retirementCertificate = null; for (const entry of canonical.values()) { const bytes = await readBoundedRegularFile(join(path, entry.name), { maxBytes: options.metadataMaxBytes, openFile: options.openFile, lstatEntry: options.lstatEntry, @@ -4038,15 +4063,16 @@ async function readGenerationSnapshot(path, checkpoint, options, expectedIdentit }); if (bytes === null) throw new Error("Generation required metadata disappeared."); if (entry.name === "checkpoint.json") checkpointBytes = bytes; + else if (entry.name === "retirement.json") retirementCertificate = bytes; else epochRecords.set(entry.name.slice(6), bytes); } if (!checkpointBytes.equals(metadataBytes(checkpoint))) throw new Error("Generation checkpoint differs from exact expected authority."); - const snapshot = { path, name: consumerGenerationName(checkpoint), checkpoint, checkpointBytes, epochRecords, identity, totalBytes, receiptEntries }; + const snapshot = { path, name: consumerGenerationName(checkpoint), checkpoint, checkpointBytes, retirementCertificate, epochRecords, identity, epochIdentity, receiptsIdentity, totalBytes, receiptEntries, canonicalEntries: [...canonical.values()].map(({ name, stat }) => ({ name, stat })) }; generationEpochAuthority(snapshot, options); for (const entry of receiptEntries) { if (!sameRetiredLinkStat(entry.stat, await options.lstatEntry(join(path, "receipts", entry.name)))) throw new Error("Generation receipt inode or stat changed."); } - for (const [directory, observed, expectedNames, limit] of [[path, identity, names, 3], [join(path, "epoch"), epochIdentity, epochNames, GENERATION_EPOCH_MAX_ENTRIES], [join(path, "receipts"), receiptsIdentity, receiptNames, GENERATION_RECEIPT_MAX_ENTRIES]]) { + for (const [directory, observed, expectedNames, limit] of [[path, identity, names, 4], [join(path, "epoch"), epochIdentity, epochNames, GENERATION_EPOCH_MAX_ENTRIES], [join(path, "receipts"), receiptsIdentity, receiptNames, GENERATION_RECEIPT_MAX_ENTRIES]]) { if (!generationSameInode(observed, await generationDirectory(directory, options)) || (await generationNames(directory, limit, options)).sort().join() !== expectedNames.slice().sort().join()) throw new Error("Generation namespace changed during validation."); } return snapshot; @@ -4088,7 +4114,7 @@ export async function buildConsumerGeneration(root, authority, rawOptions = {}) const rootBytes = await generationRootPreflight(root, options); if (rootBytes + 2 * metadataBytes(checkpoint).length > options.maxJournalBytes) throw new Error("Generation build exceeds the root aggregate byte bound."); const rootIdentity = await generationDirectory(root, options); - const path = join(root, `.building-${randomUUID()}`); + const path = join(root, `.building-p${process.pid}-${randomUUID()}`); await options.makeDirectory(path, { mode: 0o700 }); const identity = await generationDirectory(path, options); await options.makeDirectory(join(path, "epoch"), { mode: 0o700 }); @@ -4101,7 +4127,7 @@ export async function buildConsumerGeneration(root, authority, rawOptions = {}) await generationSync(root, options); await readGenerationSnapshot(path, checkpoint, options, identity, true); const builder = Object.freeze({ path, identity }); - generationBuilders.set(builder, { path, identity, rootIdentity, checkpoint }); + generationBuilders.set(builder, { path, identity, rootIdentity, checkpoint, predecessor: authority.predecessor ?? null }); return builder; } export async function publishConsumerGeneration(builder, rawOptions = {}) { @@ -4116,6 +4142,7 @@ export async function publishConsumerGeneration(builder, rawOptions = {}) { const observation = { source, destination, identity }; await options.hooks?.beforeGenerationRename?.(observation); await generationBoundary(options, "before", "rename", destination); + if (evidence.predecessor !== null) assertConsumerGenerationSuccessor(await generationReadPinned(evidence.predecessor, options), basename(destination), metadataBytes(checkpoint), options); try { await options.renameFile(source, destination); } catch (error) { @@ -4126,7 +4153,630 @@ export async function publishConsumerGeneration(builder, rawOptions = {}) { await generationBoundary(options, "after", "rename", destination); await options.hooks?.afterGenerationRename?.(observation); const result = await readGenerationSnapshot(destination, checkpoint, options, identity, true); + if (evidence.predecessor !== null) assertConsumerGenerationSuccessor(await generationReadPinned(evidence.predecessor, options), result.name, result.checkpointBytes, options); if (!generationSameInode(rootIdentity, await generationDirectory(root, options))) throw new Error("Generation root inode changed after publication."); await generationSync(root, options); return result; } + +function generationCertificateEntry(snapshot, entry) { + const bytes = entry.name === "checkpoint.json" ? snapshot.checkpointBytes : snapshot.epochRecords.get(entry.name.slice(6)); + return { name: entry.name, dev: entry.stat.dev, ino: entry.stat.ino, size: bytes.length, sha256: digest(bytes) }; +} +function generationRetirementCertificate(snapshot, slot) { + return { schemaVersion: 1, predecessorGeneration: snapshot.name, predecessorIdentity: snapshot.identity, + epochIdentity: snapshot.epochIdentity, receiptsIdentity: snapshot.receiptsIdentity, slot, + entries: snapshot.canonicalEntries.filter((entry) => entry.name !== "retirement.json").map((entry) => generationCertificateEntry(snapshot, entry)).sort((a, b) => a.name.localeCompare(b.name)) }; +} +function validateGenerationRetirementCertificate(snapshot, successor, options) { + const bytes = snapshot.retirementCertificate; + if (!Buffer.isBuffer(bytes) || digest(bytes) !== successor.retirementAuthoritySha256 || !generationSameInode(snapshot.identity, successor.previousGenerationIdentity)) throw new Error("Generation retirement certificate does not bind the exact predecessor inode."); + const certificate = generationCanonical(bytes, options.metadataMaxBytes); + if (!exactKeys(certificate, ["schemaVersion", "predecessorGeneration", "predecessorIdentity", "epochIdentity", "receiptsIdentity", "slot", "entries"]) || certificate.schemaVersion !== 1 || certificate.predecessorGeneration !== snapshot.name || !generationSameInode(certificate.predecessorIdentity, snapshot.identity) || !Number.isSafeInteger(certificate.slot) || certificate.slot < 1 || certificate.slot > MAX_OPERATION_GENERATIONS || !Array.isArray(certificate.entries) || certificate.entries.length > GENERATION_EPOCH_MAX_ENTRIES + 1) throw new Error("Generation retirement certificate is malformed."); + const records = new Map(snapshot.epochRecords); + const latest = generationEpochAuthority(snapshot, options).claims.at(-1); + if (latest?.type !== "rotation" || latest.generation !== certificate.slot) throw new Error("Generation retirement certificate lacks its exact rotation slot."); + records.delete(basename(claimPath({ epochDirectory: "" }, latest))); + records.delete(`claim-index-${generationName(latest.generation)}.json`); + const preRotation = { ...snapshot, epochRecords: records, canonicalEntries: snapshot.canonicalEntries.filter((entry) => entry.name === "checkpoint.json" || records.has(entry.name.slice(6))) }; + const expected = generationRetirementCertificate(preRotation, certificate.slot); + if (!metadataBytes(expected).equals(bytes)) throw new Error("Generation retirement certificate differs from the full exact predecessor authority."); + return certificate; +} +async function generationNativeStatOrNull(path, options) { + try { return await options.lstatEntry(path); } catch (error) { + if (options.lstatEntry === lstat && error?.code === "ENOENT") return null; + throw error; + } +} +async function generationReadPinned(snapshot, options) { + return readGenerationSnapshot(snapshot.path, snapshot.checkpoint, options, snapshot.identity); +} +async function generationWriteReceipt(snapshot, targetName, bytes, options, beforeLink) { + if (targetName !== "retirement.json" && !/^epoch\/[a-z0-9-]+\.json$/.test(targetName)) throw new Error("Generation publication target is invalid."); + if (bytes.length < 1 || bytes.length > options.metadataMaxBytes) throw new Error("Generation publication exceeds its byte bound."); + const rootBytes = await generationRootPreflight(dirname(snapshot.path), options); + if (rootBytes + bytes.length * 2 > options.maxJournalBytes) throw new Error("Generation publication exceeds aggregate byte bound."); + await generationReadPinned(snapshot, options); + const target = join(snapshot.path, targetName); + const receipts = join(snapshot.path, "receipts"); + const targetHash = digest(Buffer.from(targetName)); + const temporary = join(receipts, `.receipt-p${process.pid}-w${randomUUID()}-t${targetHash}.tmp`); + const fixed = join(receipts, `receipt-${targetHash}.json`); + const handle = await options.openFile(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | (constants.O_NOFOLLOW ?? 0), 0o600); + try { + await handle.writeFile(bytes); + await generationBoundary(options, "before", "file-sync", temporary); + await handle.sync(); + await generationBoundary(options, "after", "file-sync", temporary); + } finally { await handle.close(); } + await generationSync(receipts, options); + await beforeLink?.(temporary); + if (!generationSameInode(snapshot.identity, await generationDirectory(snapshot.path, options))) throw new Error("Generation publication was fenced by a different container inode."); + await generationBoundary(options, "before", "link", target); + let created = true; + try { await options.linkFile(temporary, target); } catch (error) { + if (options.linkFile !== link || error?.code !== "EEXIST") throw error; + created = false; + } + await generationBoundary(options, "after", "link", target); + await generationSync(dirname(target), options); + if (!created) { + const existing = await readSecureFile(target, options.metadataMaxBytes, "Generation existing metadata", options); + await options.removeFile(temporary); + await generationSync(receipts, options); + return { created, bytes: existing }; + } + await generationBoundary(options, "before", "rename", fixed); + await options.renameFile(temporary, fixed); + await generationBoundary(options, "after", "rename", fixed); + await generationSync(receipts, options); + return { created, bytes }; +} + +// Cold root discovery may discard only a direct native missing root entry, before +// pinning checkpoint bytes. All other operation and hook failures are terminal. +export async function discoverConsumerGenerations(root, authority, rawOptions = {}) { + const options = generationOptions(rawOptions); + const expectedGenesis = expectedConsumerGeneration(authority, options); + if (expectedGenesis.epoch !== 1) throw new Error("Generation discovery requires exact genesis source authority."); + const rootIdentity = await generationDirectory(root, options); + for (let attempt = 0; attempt < PROJECTION_RETRY_LIMIT; attempt += 1) { + try { + await generationRootPreflight(root, { ...options, [generationDiscovery]: true }); + } catch (error) { + if (!(error instanceof GenerationDiscoveryLost)) throw error; + if (!generationSameInode(rootIdentity, await generationDirectory(root, options))) throw new Error("Generation discovery root inode changed."); + continue; + } + const names = await generationNames(root, GENERATION_ROOT_MAX_ENTRIES, options); + const finals = names.filter((name) => /^generation-[0-9]{16}-[0-9a-f]{64}$/.test(name)).sort(); + const snapshots = []; + let lost = false; + for (const name of finals) { + const path = join(root, name); + const bytes = await readSecureFile(join(path, "checkpoint.json"), options.metadataMaxBytes, "Generation discovery checkpoint", options, 1, options.hooks?.metadataRead); + if (bytes === null) { lost = true; break; } + const checkpoint = validateGenerationCheckpoint(generationCanonical(bytes, options.metadataMaxBytes), options.stateMaxBytes).checkpoint; + if (name !== consumerGenerationName(checkpoint) || checkpoint.statePathSha256 !== expectedGenesis.statePathSha256) throw new Error("Generation discovery checkpoint has a conflicting source."); + for (const field of ["sourceKind", "sourceAuthoritySha256", "sourceTipDigest", "sourceTipBase64", "migrationKind", "migrationAuthoritySha256", "migrationTipDigest", "migrationTipBase64"]) { + if (checkpoint[field] !== expectedGenesis[field]) throw new Error("Generation discovery provenance changed."); + } + if (checkpoint.epoch === 1 && !bytes.equals(metadataBytes(expectedGenesis))) throw new Error("Generation genesis differs from exact supplied source state."); + const identity = await generationDirectory(path, options); + const snapshot = await readGenerationSnapshot(path, checkpoint, options, identity); + if (snapshots.length !== 0) assertConsumerGenerationSuccessor(await generationReadPinned(snapshots[0], options), name, bytes, options); + snapshots.push(snapshot); + } + if (lost) { + if (snapshots.length !== 0) throw new Error("Generation required pinned successor disappeared."); + continue; + } + if (!generationSameInode(rootIdentity, await generationDirectory(root, options))) throw new Error("Generation discovery root inode changed."); + return { root, rootIdentity, names, generations: snapshots }; + } + throw new Error("Generation discovery exceeded its bounded native-loss handoff limit."); +} + +export async function recoverConsumerGenerationBuilder(path, authority, rawOptions = {}) { + const options = generationOptions(rawOptions); + if (!new RegExp(`^\\.building-(?:p[1-9][0-9]*-)?${uuidSource}$`).test(basename(path))) throw new Error("Generation recovery requires an exact builder name."); + const checkpoint = expectedConsumerGeneration(authority, options); + const identity = await generationDirectory(path, options); + const rootIdentity = await generationDirectory(dirname(path), options); + await generationRootPreflight(dirname(path), options); + let names = await generationNames(path, 3, options); + const ownerPid = /^\.building-p([1-9][0-9]*)-/.exec(basename(path))?.[1]; + if (ownerPid && !temporaryProcessIsAlive({ pid: Number(ownerPid) }, options)) { + if (names.some((name) => !["epoch", "receipts", "checkpoint.json"].includes(name))) throw new Error("Generation dead builder contains an unexpected entry."); + for (const name of ["epoch", "receipts"]) { + if (!names.includes(name)) { + try { await options.makeDirectory(join(path, name), { mode: 0o700 }); } catch (error) { if (options.makeDirectory !== mkdir || error?.code !== "EEXIST") throw error; } + await generationSync(path, options); + } + } + if (!(await generationNames(join(path, "receipts"), 1, options)).length && !names.includes("checkpoint.json")) { + await publishGenerationCheckpoint(path, checkpoint, options); + } + names = await generationNames(path, 3, options); + } + if (names.some((name) => !["epoch", "receipts", "checkpoint.json"].includes(name)) || !names.includes("receipts") || !names.includes("epoch") || (await generationNames(join(path, "epoch"), 0, options)).length !== 0) throw new Error("Generation incomplete builder cannot prove its owned checkpoint writer."); + const receipts = join(path, "receipts"); + const receiptNames = await generationNames(receipts, 1, options); + if (receiptNames.length !== 1) throw new Error("Generation builder lacks its exact durable receipt proof."); + const receiptName = receiptNames[0]; + const targetHash = digest(Buffer.from("checkpoint.json")); + if (receiptName !== `receipt-${targetHash}.json` && generationReceiptTemporaryPattern.exec(receiptName)?.[3] !== targetHash) throw new Error("Generation builder receipt has a conflicting target."); + const receiptPath = join(receipts, receiptName); + const receiptStat = generationEntryStat(await options.lstatEntry(receiptPath), "file", options); + const bytes = await readSecureFile(receiptPath, options.metadataMaxBytes, "Generation builder receipt", options); + if (bytes === null || !bytes.equals(metadataBytes(checkpoint))) throw new Error("Generation builder receipt differs from exact construction authority."); + const target = join(path, "checkpoint.json"); + const canonical = await generationNativeStatOrNull(target, options); + if (canonical === null) { + if (receiptStat.nlink !== 1 || receiptName.startsWith("receipt-")) throw new Error("Generation builder has malformed pre-link receipt evidence."); + await generationSync(receipts, options); + await generationBoundary(options, "before", "link", target); + try { await options.linkFile(receiptPath, target); } catch (error) { + if (options.linkFile !== link || error?.code !== "EEXIST") throw error; + } + await generationBoundary(options, "after", "link", target); + } + if (!generationSameInode(receiptStat, generationEntryStat(await options.lstatEntry(target), "file", options))) throw new Error("Generation builder checkpoint is a different inode from its receipt."); + await generationSync(path, options); + if (!receiptName.startsWith("receipt-")) { + const fixed = join(receipts, `receipt-${targetHash}.json`); + await generationBoundary(options, "before", "rename", fixed); + await options.renameFile(receiptPath, fixed); + await generationBoundary(options, "after", "rename", fixed); + } + await generationSync(receipts, options); + await generationSync(path, options); + await generationSync(dirname(path), options); + await readGenerationSnapshot(path, checkpoint, options, identity, true); + const builder = Object.freeze({ path, identity }); + generationBuilders.set(builder, { path, identity, rootIdentity, checkpoint, predecessor: authority.predecessor ?? null }); + return builder; +} + +async function generationQuiesce(snapshot, options, ownTemporary = null) { + const scan = generationEpochAuthority(snapshot, options); + for (const receipt of snapshot.receiptEntries.filter((entry) => entry.temporary)) { + const path = join(snapshot.path, "receipts", receipt.name); + if (path === ownTemporary) continue; + let decided = false; + if (receipt.target === null) { + const bytes = await readSecureFile(path, options.metadataMaxBytes, "Generation owned receipt temporary", options, 0); + if (bytes === null) throw new Error("Generation receipt temporary disappeared after pinning."); + let value; + try { value = JSON.parse(bytes); } catch (error) { if (!(error instanceof SyntaxError)) throw error; } + if (value && metadataBytes(value).equals(bytes)) { + let claim = value.type === "normal" ? validateClaim(value, null, options.stateMaxBytes) : scan.claims.find((claim) => claim.generation === value.generation && claim.token === value.token); + let target = null; + if (value.type === "normal") target = `epoch/${basename(claimPath({ epochDirectory: "" }, value))}`; + else if (value.claimSha256 && value.schemaVersion === 1) { + claim = scan.contents.get(value.claimSha256); + if (claim && value.generation === claim.generation) target = `epoch/claim-index-${generationName(value.generation)}.json`; + } else if (claim && value.outcome) { + validateTerminal(value, claim, options.stateMaxBytes, validateGenerationTransaction); + target = `epoch/terminal-${generationName(claim.generation)}-${claim.token}.json`; + } else if (claim && value.refreshedAtMs !== undefined) { + validateHeartbeat(value, claim); + target = `epoch/heartbeat-${generationName(claim.generation)}-${claim.token}-${generationName(value.refreshedAtMs)}.json`; + } else if (claim && value.terminalSha256) { + validateApplied(value, claim, scan.terminals.get(`${claim.generation}:${claim.token}`)); + target = `epoch/applied-${generationName(claim.generation)}-${claim.token}.json`; + } + if (target !== null && generationReceiptTemporaryPattern.exec(receipt.name)[3] !== digest(Buffer.from(target))) throw new Error("Generation owned receipt temporary has a conflicting target hash."); + if (claim && target !== null) { + const winner = scan.claims.find((candidate) => candidate.generation === claim.generation); + decided = !!winner && (!metadataBytes(winner).equals(metadataBytes(claim)) || scan.terminals.has(`${claim.generation}:${claim.token}`)); + } + } + } + if (receipt.target === null && !decided && temporaryProcessIsAlive({ pid: receipt.pid }, options)) throw new Error("Generation rotation is pending until its live unresolved receipt writer quiesces."); + if (!sameRetiredLinkStat(receipt.stat, await options.lstatEntry(path))) throw new Error("Generation receipt temporary inode changed before recovery."); + if (receipt.target === null) { + await options.removeFile(path); + } else { + const fixed = join(snapshot.path, "receipts", `receipt-${digest(Buffer.from(receipt.target))}.json`); + await generationBoundary(options, "before", "rename", fixed); + await options.renameFile(path, fixed); + await generationBoundary(options, "after", "rename", fixed); + } + await generationSync(join(snapshot.path, "receipts"), options); + } +} + +async function generationOwnsClaim(snapshot, claim, options, ownTemporary = null) { + const current = await generationReadPinned(snapshot, options); + const scan = generationEpochAuthority(current, options); + const latest = scan.claims.at(-1); + if (!latest || !metadataBytes(latest).equals(metadataBytes(claim)) || scan.terminals.has(`${claim.generation}:${claim.token}`)) throw new Error("Generation operation lost its exact latest claim ownership."); + await generationQuiesce(current, options, ownTemporary); + return { snapshot: current, scan }; +} + +async function generationPublishClaim(snapshot, claim, options) { + const revalidate = async (temporary) => { + const current = await generationReadPinned(snapshot, options); + const scan = generationEpochAuthority(current, options); + if (claim.type === "normal" && current.retirementCertificate !== null) throw new Error("Generation normal claim is fenced by retirement preparation."); + const latest = scan.claims.at(-1); + if (latest?.type === "rotation" && !metadataBytes(latest).equals(metadataBytes(claim))) throw new Error("Generation claim was fenced by its rotation CAS."); + if (claim.generation !== (latest?.generation ?? 0) + 1 && claim.generation !== latest?.generation) throw new Error("Generation claim frontier changed before publication."); + await generationQuiesce(current, options, temporary); + }; + const claimName = `epoch/${basename(claimPath({ epochDirectory: "" }, claim))}`; + const result = await generationWriteReceipt(snapshot, claimName, metadataBytes(claim), options, revalidate); + if (!result.bytes?.equals(metadataBytes(claim))) throw new Error("Generation claim publication lost its digest-bound bytes."); + const current = await generationReadPinned(snapshot, options); + const cas = claimIndexFor(claim); + const published = await generationWriteReceipt(current, `epoch/claim-index-${generationName(claim.generation)}.json`, metadataBytes(cas), options, revalidate); + return published.bytes?.equals(metadataBytes(cas)) ?? false; +} + +async function generationRepairProjection(root, authority, statePath, options) { + for (let attempt = 0; attempt < PROJECTION_RETRY_LIMIT; attempt += 1) { + const discovered = await discoverConsumerGenerations(root, authority, options); + if (discovered.generations.length !== 1) throw new Error("Generation projection requires a converged unique final."); + const snapshot = discovered.generations[0]; + const tip = generationEpochAuthority(snapshot, options).tip; + const current = await readSecureFile(statePath, options.stateMaxBytes, "Generation projection", options, 0, options.hooks?.projectionRead); + if (tip.tipBytes !== null && (current === null || !current.equals(tip.tipBytes))) { + await options.hooks?.beforeProjectionWrite?.({ tipDigest: tip.tipDigest }); + const temporary = join(dirname(statePath), `.pylon-generation-projection-${randomUUID()}.tmp`); + const handle = await options.openFile(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | (constants.O_NOFOLLOW ?? 0), 0o600); + try { await handle.writeFile(tip.tipBytes); await handle.sync(); } finally { await handle.close(); } + const identity = await options.lstatEntry(temporary); + await options.hooks?.afterProjectionFileSync?.({ tipDigest: tip.tipDigest, temporary }); + await generationReadPinned(snapshot, options); + await options.renameFile(temporary, statePath); + if (!generationSameInode(identity, await options.lstatEntry(statePath))) throw new Error("Generation projection rename has a different destination inode."); + await options.hooks?.afterProjectionRename?.({ tipDigest: tip.tipDigest }); + await generationSync(dirname(statePath), options); + // A writer may have paused after its final authority read and overwritten + // a newer projection. Only a fresh immutable-tip scan can permit success. + const afterRename = await discoverConsumerGenerations(root, authority, options); + if (afterRename.generations.length !== 1) continue; + const newer = generationEpochAuthority(afterRename.generations[0], options).tip; + if (newer.tipDigest !== tip.tipDigest) continue; + } + const readback = await readSecureFile(statePath, options.stateMaxBytes, "Generation projection", options, 0, options.hooks?.projectionRead); + const afterRead = await discoverConsumerGenerations(root, authority, options); + if (afterRead.generations.length !== 1) continue; + const latest = generationEpochAuthority(afterRead.generations[0], options).tip; + if (latest.tipDigest !== tip.tipDigest) continue; + if (latest.tipBytes === null ? readback !== null : readback === null || !readback.equals(latest.tipBytes)) continue; + return latest; + } + throw new Error("Generation projection could not catch up to its immutable tip."); +} + +async function generationFinishCommit(snapshot, claim, terminal, root, authority, statePath, options) { + for (const transaction of terminal.transactions) { + const current = await generationReadPinned(snapshot, options); + const result = await generationWriteReceipt(current, `epoch/transition-${transaction.baseDigest}.json`, metadataBytes(transaction), options); + if (!result.bytes?.equals(metadataBytes(transaction))) throw new Error("Generation transition lost its immutable compare-and-set."); + } + await generationRepairProjection(root, authority, statePath, options); + const applied = { schemaVersion: 2, generation: claim.generation, token: claim.token, terminalSha256: digest(metadataBytes(terminal)) }; + const current = await generationReadPinned(snapshot, options); + const result = await generationWriteReceipt(current, `epoch/applied-${generationName(claim.generation)}-${claim.token}.json`, metadataBytes(applied), options); + if (!result.bytes?.equals(metadataBytes(applied))) throw new Error("Generation applied marker differs from its exact terminal."); + await generationRepairProjection(root, authority, statePath, options); +} + +async function generationMove(source, destination, identity, options, validate) { + await options.hooks?.beforeGenerationMove?.({ source, destination, identity }); + await validate(source); + await generationBoundary(options, "before", "rename", destination); + const remainingSource = await generationNativeStatOrNull(source, options); + if (remainingSource === null) { + if (!generationSameInode(identity, await generationDirectory(destination, options))) throw new Error("Generation rename join has a different destination inode."); + await validate(destination); + } else { + if (!generationSameInode(identity, remainingSource)) throw new Error("Generation rename source was replaced."); + await validate(source); + } + try { await options.renameFile(source, destination); } catch (error) { + if (options.renameFile !== rename || error?.code !== "ENOENT") throw error; + if (!generationSameInode(identity, await generationDirectory(destination, options))) throw error; + } + await generationBoundary(options, "after", "rename", destination); + await options.hooks?.afterGenerationMove?.({ source, destination, identity }); + if (!generationSameInode(identity, await generationDirectory(destination, options))) throw new Error("Generation rename destination is a different inode."); + await validate(destination); + await generationSync(dirname(source), options); +} + +async function generationDeleteRetired(path, successor, options) { + const expectedIdentity = successor.checkpoint.previousGenerationIdentity; + if (!expectedIdentity || !generationSameInode(expectedIdentity, await generationDirectory(path, options))) throw new Error("Generation deletion container is a different predecessor inode."); + const certificatePath = join(path, "retirement.json"); + const receiptName = `receipt-${digest(Buffer.from("retirement.json"))}.json`; + const certificateReceiptPath = join(path, "receipts", receiptName); + let certificateBytes = await readSecureFile(certificatePath, options.metadataMaxBytes, "Generation deletion certificate", options); + if (certificateBytes === null) certificateBytes = await readSecureFile(certificateReceiptPath, options.metadataMaxBytes, "Generation deletion certificate receipt", options); + const names = await generationNames(path, 4, options); + if (certificateBytes === null) { + // The final proof links are deleted only after every authority file. The + // successor still commits this exact container inode through the empty cut. + if (names.some((name) => !["epoch", "receipts"].includes(name))) throw new Error("Generation deletion lost its certificate before authority cleanup."); + for (const name of names) { + await generationDirectory(join(path, name), options); + await generationNames(join(path, name), 0, options); + } + } else { + if (digest(certificateBytes) !== successor.checkpoint.retirementAuthoritySha256) throw new Error("Generation deletion certificate differs from successor commitment."); + const certificate = generationCanonical(certificateBytes, options.metadataMaxBytes); + if (certificate.predecessorGeneration !== successor.checkpoint.previousGeneration || !generationSameInode(certificate.predecessorIdentity, expectedIdentity) || !Array.isArray(certificate.entries) || certificate.entries.length > GENERATION_EPOCH_MAX_ENTRIES + 1) throw new Error("Generation deletion certificate is not bound to its successor."); + const expected = new Map(certificate.entries.map((entry) => [entry.name, entry])); + if (expected.size !== certificate.entries.length || expected.get("checkpoint.json")?.sha256 !== successor.checkpoint.previousCheckpointSha256) throw new Error("Generation deletion certificate lacks exact predecessor checkpoint."); + const claim = { schemaVersion: 3, generation: certificate.slot, token: successor.checkpoint.epochId, type: "rotation", intent: { + schemaVersion: 3, predecessorGeneration: successor.checkpoint.previousGeneration, + checkpointSha256: successor.checkpoint.previousCheckpointSha256, tipSha256: successor.checkpoint.anchorDigest, checkpoint: successor.checkpoint, + } }; + for (const [name, value] of [[`epoch/${basename(claimPath({ epochDirectory: "" }, claim))}`, claim], [`epoch/claim-index-${generationName(certificate.slot)}.json`, claimIndexFor(claim)]]) { + const bytes = metadataBytes(value); + expected.set(name, { name, size: bytes.length, sha256: digest(bytes), dev: null, ino: null }); + } + expected.set("retirement.json", { name: "retirement.json", size: certificateBytes.length, sha256: digest(certificateBytes), dev: null, ino: null }); + const allowed = new Map(); + for (const [name, entry] of expected) { + allowed.set(name, entry); + allowed.set(`receipts/receipt-${digest(Buffer.from(name))}.json`, entry); + } + const actual = []; + for (const name of names) { + if (["epoch", "receipts"].includes(name)) { + const directory = join(path, name); + const identity = await generationDirectory(directory, options); + if (!generationSameInode(identity, certificate[name === "epoch" ? "epochIdentity" : "receiptsIdentity"])) throw new Error("Generation deletion child directory changed inode."); + for (const child of await generationNames(directory, name === "epoch" ? GENERATION_EPOCH_MAX_ENTRIES : GENERATION_RECEIPT_MAX_ENTRIES, options)) actual.push(`${name}/${child}`); + } else actual.push(name); + } + const observations = new Map(); + for (const name of actual) { + const descriptor = allowed.get(name); + if (!descriptor) throw new Error("Generation deleting container contains an unauthorized entry."); + const stat = generationEntryStat(await options.lstatEntry(join(path, name)), "file", options); + if (stat.size !== descriptor.size || (descriptor.dev !== null && !generationSameInode(stat, descriptor)) || ![1, 2].includes(stat.nlink)) throw new Error("Generation deleting entry differs from its committed inode or size."); + const bytes = await readSecureFile(join(path, name), options.metadataMaxBytes, "Generation deleting authority", options); + if (bytes === null || digest(bytes) !== descriptor.sha256) throw new Error("Generation deleting authority differs from its committed bytes."); + observations.set(name, stat); + } + for (const [name] of expected) { + const canonical = observations.get(name); + const receipt = observations.get(`receipts/receipt-${digest(Buffer.from(name))}.json`); + if (canonical && receipt ? !generationSameInode(canonical, receipt) || canonical.nlink !== 2 || receipt.nlink !== 2 : (canonical ?? receipt)?.nlink !== undefined && (canonical ?? receipt).nlink !== 1) throw new Error("Generation deletion receipt is not its exact remaining canonical inode."); + } + const proofNames = new Set(["retirement.json", `receipts/${receiptName}`]); + for (const name of [...actual.filter((name) => !proofNames.has(name)), ...actual.filter((name) => proofNames.has(name))]) { + await generationReadPinned(successor, options); + if (!generationSameInode(expectedIdentity, await generationDirectory(path, options))) throw new Error("Generation deletion container was replaced."); + const entryPath = join(path, name); + const current = generationEntryStat(await options.lstatEntry(entryPath), "file", options); + if (!generationSameInode(current, observations.get(name))) throw new Error("Generation deleting entry was replaced before unlink."); + await generationBoundary(options, "before", "unlink", entryPath); + await options.removeFile(entryPath); + await generationBoundary(options, "after", "unlink", entryPath); + await generationSync(dirname(entryPath), options); + } + } + for (const name of await generationNames(path, 2, options)) { + if (!["epoch", "receipts"].includes(name)) throw new Error("Generation deletion is not empty."); + await generationNames(join(path, name), 0, options); + await generationBoundary(options, "before", "remove-directory", join(path, name)); + await options.removeFile(join(path, name), { recursive: true }); + await generationBoundary(options, "after", "remove-directory", join(path, name)); + await generationSync(path, options); + } + await generationReadPinned(successor, options); + if (!generationSameInode(expectedIdentity, await generationDirectory(path, options))) throw new Error("Generation deleting container was replaced at final removal."); + await generationNames(path, 0, options); + await generationBoundary(options, "before", "remove-directory", path); + await options.removeFile(path, { recursive: true }); + await generationBoundary(options, "after", "remove-directory", path); + await generationSync(dirname(path), options); +} + +async function generationConverge(root, authority, discovered, options) { + let successor = discovered.generations.at(-1); + if (!successor) return null; + if (discovered.generations.length === 2) { + const predecessor = discovered.generations[0]; + const validate = async (path) => { + const pinned = await readGenerationSnapshot(path, predecessor.checkpoint, options, predecessor.identity); + const currentSuccessor = await generationReadPinned(successor, options); + assertConsumerGenerationSuccessor(pinned, currentSuccessor.name, currentSuccessor.checkpointBytes, options); + await generationQuiesce(pinned, options); + }; + await validate(predecessor.path); + const retired = join(root, `.retired-${predecessor.name}`); + await generationMove(predecessor.path, retired, predecessor.identity, options, validate); + } + successor = await generationReadPinned(successor, options); + for (const name of await generationNames(root, GENERATION_ROOT_MAX_ENTRIES, options)) { + if (!name.startsWith(".retired-") && !name.startsWith(".deleting-")) continue; + const expected = successor.checkpoint.previousGeneration; + if (![`.retired-${expected}`, `.deleting-${expected}`].includes(name)) throw new Error("Generation cleanup contains an orphan predecessor."); + let path = join(root, name); + if (name.startsWith(".retired-")) { + const checkpointBytes = await readSecureFile(join(path, "checkpoint.json"), options.metadataMaxBytes, "Generation retired checkpoint", options); + if (checkpointBytes === null) throw new Error("Generation retired checkpoint disappeared."); + const checkpoint = validateGenerationCheckpoint(generationCanonical(checkpointBytes, options.metadataMaxBytes), options.stateMaxBytes).checkpoint; + const validate = async (currentPath) => { + const pinned = await readGenerationSnapshot(currentPath, checkpoint, options, successor.checkpoint.previousGenerationIdentity); + const currentSuccessor = await generationReadPinned(successor, options); + assertConsumerGenerationSuccessor(pinned, currentSuccessor.name, currentSuccessor.checkpointBytes, options); + await generationQuiesce(pinned, options); + }; + const deleting = join(root, `.deleting-${expected}`); + await generationMove(path, deleting, successor.checkpoint.previousGenerationIdentity, options, validate); + path = deleting; + } + await generationDeleteRetired(path, successor, options); + } + const final = await discoverConsumerGenerations(root, authority, options); + if (final.generations.length !== 1 || final.names.some((name) => name.startsWith(".retired-") || name.startsWith(".deleting-"))) throw new Error("Generation preparation did not converge its predecessor cleanup."); + return final.generations[0]; +} + +export async function prepareConsumerGeneration(root, authority, rawOptions = {}) { + const options = generationOptions(rawOptions); + let discovered = await discoverConsumerGenerations(root, authority, options); + if (discovered.generations.length === 0) { + const builders = discovered.names.filter((name) => name.startsWith(".building-")); + const builder = builders.length === 0 ? await buildConsumerGeneration(root, authority, options) : await recoverConsumerGenerationBuilder(join(root, builders[0]), authority, options); + await publishConsumerGeneration(builder, options); + discovered = await discoverConsumerGenerations(root, authority, options); + } + return generationConverge(root, authority, discovered, options); +} + +export async function rotateConsumerGeneration(root, authority, rawOptions = {}) { + const options = generationOptions(rawOptions); + let snapshot = await prepareConsumerGeneration(root, authority, options); + await generationQuiesce(snapshot, options); + snapshot = await generationReadPinned(snapshot, options); + let scan = generationEpochAuthority(snapshot, options); + let latest = scan.claims.at(-1); + if (latest?.type === "normal" && (!scan.terminals.has(`${latest.generation}:${latest.token}`) || (scan.terminals.get(`${latest.generation}:${latest.token}`).outcome === "commit" && !scan.applied.has(`${latest.generation}:${latest.token}`)))) throw new Error("Generation rotation requires a resolved normal operation frontier."); + if (latest?.type !== "rotation") { + const slot = (latest?.generation ?? 0) + 1; + const certificateBytes = metadataBytes(generationRetirementCertificate(snapshot, slot)); + const wanted = consumerGenerationRotationClaim(snapshot.checkpoint, slot, { ...scan.tip, previousGenerationIdentity: snapshot.identity, retirementAuthoritySha256: digest(certificateBytes) }, options.stateMaxBytes); + const headroom = 2 * certificateBytes.length + 2 * metadataBytes(wanted).length + 2 * metadataBytes(claimIndexFor(wanted)).length + 2 * metadataBytes(wanted.intent.checkpoint).length; + if (await generationRootPreflight(root, options) + headroom > options.maxJournalBytes) throw new Error("Generation lacks reserved rotation headroom."); + await options.hooks?.beforeRotationDecision?.({ claim: wanted, intent: wanted.intent }); + const result = await generationWriteReceipt(snapshot, "retirement.json", certificateBytes, options); + if (!result.bytes?.equals(certificateBytes)) throw new Error("Generation retirement certificate lost its immutable publication."); + snapshot = await generationReadPinned(snapshot, options); + if (!(await generationPublishClaim(snapshot, wanted, options))) throw new Error("Generation rotation lost its exact winning CAS."); + latest = wanted; + await options.hooks?.afterRotationIntent?.({ claim: latest, intent: latest.intent }); + } + snapshot = await generationReadPinned(snapshot, options); + assertConsumerGenerationSuccessor(snapshot, consumerGenerationName(latest.intent.checkpoint), metadataBytes(latest.intent.checkpoint), options); + const builders = (await generationNames(root, GENERATION_ROOT_MAX_ENTRIES, options)).filter((name) => name.startsWith(".building-")); + const builder = builders.length === 0 ? await buildConsumerGeneration(root, { predecessor: snapshot }, options) : await recoverConsumerGenerationBuilder(join(root, builders[0]), { predecessor: snapshot }, options); + await publishConsumerGeneration(builder, { ...options, hooks: { ...options.hooks, beforeGenerationRename: async (observation) => { + await options.hooks?.beforeGenerationRename?.(observation); + const latestPredecessor = await generationReadPinned(snapshot, options); + assertConsumerGenerationSuccessor(latestPredecessor, basename(observation.destination), metadataBytes(latest.intent.checkpoint), options); + } } }); + const final = await prepareConsumerGeneration(root, authority, options); + return { epoch: final.checkpoint.epoch, tipSha256: generationEpochAuthority(final, options).tip.tipDigest }; +} + +export async function withConsumerGenerationLock(root, authority, action, rawOptions = {}) { + if (typeof action !== "function" || !authority?.genesis?.statePath) throw new Error("Generation operation requires an action and exact genesis authority."); + const options = generationOptions(rawOptions); + const statePath = resolve(authority.genesis.statePath); + await generationDirectory(dirname(statePath), options); + let snapshot; + let claim; + let acquired = false; + for (let attempt = 0; attempt < PROJECTION_RETRY_LIMIT; attempt += 1) { + snapshot = await prepareConsumerGeneration(root, authority, options); + await generationQuiesce(snapshot, options); + snapshot = await generationReadPinned(snapshot, options); + let scan = generationEpochAuthority(snapshot, options); + const latest = scan.claims.at(-1); + if (latest?.type === "rotation" || snapshot.retirementCertificate !== null) { await rotateConsumerGeneration(root, authority, options); continue; } + if (latest) { + const key = `${latest.generation}:${latest.token}`; + let terminal = scan.terminals.get(key); + if (!terminal) { + const heartbeat = scan.heartbeats.get(key)?.refreshedAtMs ?? latest.createdAtMs; + if (options.now() - heartbeat < options.stale) throw new Error("Generation state is actively locked."); + await options.hooks?.afterObserveStale?.({ claim: latest, heartbeat }); + const wanted = { schemaVersion: 2, generation: latest.generation, token: latest.token, outcome: "retired" }; + const result = await generationWriteReceipt(snapshot, `epoch/terminal-${generationName(latest.generation)}-${latest.token}.json`, metadataBytes(wanted), options); + terminal = validateTerminal(generationCanonical(result.bytes, options.metadataMaxBytes), latest, options.stateMaxBytes, validateGenerationTransaction); + } + if (terminal.outcome === "commit" && !scan.applied.has(key)) await generationFinishCommit(snapshot, latest, terminal, root, authority, statePath, options); + } + snapshot = await generationReadPinned(snapshot, options); + scan = generationEpochAuthority(snapshot, options); + const slot = (scan.claims.at(-1)?.generation ?? 0) + 1; + // Reserve the full next checkpoint/claim, duplicate receipt links, a + // maximum staged transaction and its terminal, plus certificate growth. + const certificateBytes = metadataBytes(generationRetirementCertificate(snapshot, slot)).length; + const reserve = 4 * options.metadataMaxBytes + 4 * (4 * Math.ceil(options.stateMaxBytes / 3) + 1024) + 2 * (certificateBytes + 16_384); + if (certificateBytes + 2048 > options.metadataMaxBytes || slot > (options.maxLockGenerations ?? PYLON_CONSUMER_ROTATE_CLAIM_TRIGGER) || scan.tip.length >= (options.maxTransactionDepth ?? PYLON_CONSUMER_ROTATE_TRANSITION_TRIGGER) || await generationRootPreflight(root, options) + reserve > options.maxJournalBytes) { + if (scan.claims.length === 0) throw new Error("Generation byte budget cannot reserve one maximum operation and rotation."); + await rotateConsumerGeneration(root, authority, options); + continue; + } + claim = { schemaVersion: 2, generation: slot, token: randomUUID(), type: "normal", ownerPid: process.pid, createdAtMs: options.now() }; + if (!(await generationPublishClaim(snapshot, claim, options))) continue; + await generationOwnsClaim(snapshot, claim, options); + acquired = true; + break; + } + if (!acquired) throw new Error("Generation operation could not acquire its bounded claim frontier."); + let terminal = null; + let active = true; + let staged = null; + let stagedOnce = false; + let heartbeatFailure = null; + const beat = async () => { + if (!active) return false; + try { + const owned = await generationOwnsClaim(snapshot, claim, options); + const refreshedAtMs = options.now(); + const value = { schemaVersion: 2, generation: claim.generation, token: claim.token, refreshedAtMs }; + await generationWriteReceipt(owned.snapshot, `epoch/heartbeat-${generationName(claim.generation)}-${claim.token}-${generationName(refreshedAtMs)}.json`, metadataBytes(value), options, + async (temporary) => generationOwnsClaim(snapshot, claim, options, temporary)); + return true; + } catch (error) { heartbeatFailure = error; throw error; } + }; + await beat(); + const stopHeartbeat = (options.startHeartbeat ?? defaultHeartbeatScheduler)({ interval: options.update ?? PYLON_CONSUMER_LOCK_UPDATE_MS, beat }); + let stopped = false; + const stop = async () => { if (!stopped) { stopped = true; await stopHeartbeat(); } }; + const publishDecision = async (wanted) => { + const current = await generationReadPinned(snapshot, options); + const result = await generationWriteReceipt(current, `epoch/terminal-${generationName(claim.generation)}-${claim.token}.json`, metadataBytes(wanted), options); + terminal = validateTerminal(generationCanonical(result.bytes, options.metadataMaxBytes), claim, options.stateMaxBytes, validateGenerationTransaction); + if (!result.bytes.equals(metadataBytes(wanted))) throw new Error("Generation operation lost ownership before its terminal decision."); + }; + try { + await options.hooks?.afterClaim?.({ claim }); + const base = await generationRepairProjection(root, authority, statePath, options); + const transaction = Object.freeze({ + readStateBytes: () => base.tipBytes === null ? null : Buffer.from(base.tipBytes), + commitState: async (value) => { + if (!active || stagedOnce || terminal !== null) throw new Error("Generation transaction is no longer live or already staged."); + const bytes = Buffer.from(value); + if (bytes.length < 1 || bytes.length > options.stateMaxBytes) throw new Error("Generation state exceeds its byte bound."); + staged = bytes; stagedOnce = true; + }, + }); + let result; + try { result = await action(statePath, transaction); } finally { active = false; await stop(); } + if (heartbeatFailure !== null) throw heartbeatFailure; + await generationOwnsClaim(snapshot, claim, options); + if (staged !== null && digest(staged) !== base.tipDigest) { + const wanted = { schemaVersion: 2, generation: claim.generation, token: claim.token, outcome: "commit", transactions: [transactionFor(base.tipDigest, staged)] }; + await options.hooks?.beforeCommitDecision?.({ claim, transactions: wanted.transactions }); + await publishDecision(wanted); + await options.hooks?.afterCommitDecision?.({ claim, terminal }); + await generationFinishCommit(snapshot, claim, terminal, root, authority, statePath, options); + } else await publishDecision({ schemaVersion: 2, generation: claim.generation, token: claim.token, outcome: "released" }); + await generationRepairProjection(root, authority, statePath, options); + await prepareConsumerGeneration(root, authority, options); + return result; + } catch (error) { + active = false; + await stop(); + // Preserve the original action/I/O failure, including its object identity. + // A failed commit decision remains helpable; callbacks are never replayed. + if (terminal === null) { + try { await publishDecision({ schemaVersion: 2, generation: claim.generation, token: claim.token, outcome: "released" }); } catch { /* Recovery uses the durable slot on the next entry. */ } + } + throw error; + } +} diff --git a/scripts/lib/pylon-generation-format.mjs b/scripts/lib/pylon-generation-format.mjs index 67a4a12d49..bb28186648 100644 --- a/scripts/lib/pylon-generation-format.mjs +++ b/scripts/lib/pylon-generation-format.mjs @@ -9,7 +9,7 @@ export const GENERATION_RECEIPT_MAX_ENTRIES = GENERATION_EPOCH_MAX_ENTRIES * 2 + export const GENERATION_JOURNAL_MAX_BYTES = 512 * 1024 * 1024; const hex = /^[0-9a-f]{64}$/; const fields = [ - "schemaVersion", "epoch", "epochId", "statePathSha256", "previousCheckpointSha256", "previousGeneration", + "schemaVersion", "epoch", "epochId", "statePathSha256", "previousCheckpointSha256", "previousGeneration", "previousGenerationIdentity", "retirementAuthoritySha256", "previousTipSha256", "historySha256", "anchorDigest", "anchorBase64", "sourceKind", "sourceAuthoritySha256", "sourceTipDigest", "sourceTipBase64", "migrationKind", "migrationAuthoritySha256", "migrationTipDigest", "migrationTipBase64", ]; @@ -22,6 +22,7 @@ export function generationCheckpointMaxBytes(stateMaxBytes = GENERATION_STATE_MA const checkpointEnvelopeBytes = generationBytes({ schemaVersion: 3, epoch: Number.MAX_SAFE_INTEGER, epochId: GENERATION_ZERO, statePathSha256: GENERATION_ZERO, previousCheckpointSha256: GENERATION_ZERO, previousGeneration: `generation-9007199254740991-${GENERATION_ZERO}`, + previousGenerationIdentity: { dev: Number.MAX_SAFE_INTEGER, ino: Number.MAX_SAFE_INTEGER }, retirementAuthoritySha256: GENERATION_ZERO, previousTipSha256: GENERATION_ZERO, historySha256: GENERATION_ZERO, anchorDigest: GENERATION_ZERO, anchorBase64: "", sourceKind: "v2", sourceAuthoritySha256: GENERATION_ZERO, sourceTipDigest: GENERATION_ZERO, sourceTipBase64: "", migrationKind: "v1", migrationAuthoritySha256: GENERATION_ZERO, migrationTipDigest: GENERATION_ZERO, migrationTipBase64: "", @@ -86,7 +87,7 @@ export function consumerGenerationGenesisCheckpoint({ statePath, stateBytes = nu if (src.kind !== null && (src.digest !== anchor.digest || src.base64 !== anchor.base64)) throw new Error("Generation source must bind the exact genesis state."); const value = { schemaVersion: 3, epoch: 1, epochId: "", statePathSha256: generationDigest(Buffer.from(resolve(statePath))), - previousCheckpointSha256: GENERATION_ZERO, previousGeneration: null, previousTipSha256: GENERATION_ZERO, + previousCheckpointSha256: GENERATION_ZERO, previousGeneration: null, previousGenerationIdentity: null, retirementAuthoritySha256: GENERATION_ZERO, previousTipSha256: GENERATION_ZERO, historySha256: "", anchorDigest: anchor.digest, anchorBase64: anchor.base64, sourceKind: src.kind, sourceAuthoritySha256: src.authoritySha256, sourceTipDigest: src.digest, sourceTipBase64: src.base64, migrationKind: old.kind, migrationAuthoritySha256: old.authoritySha256, migrationTipDigest: old.digest, migrationTipBase64: old.base64, @@ -107,7 +108,7 @@ export function validateGenerationCheckpoint(input, stateMaxBytes = GENERATION_S const value = Object.fromEntries(fields.map((key) => [key, input[key]])); if (value.schemaVersion !== 3 || !Number.isSafeInteger(value.epoch) || value.epoch < 1 || ![value.epochId, value.statePathSha256, value.previousCheckpointSha256, value.previousTipSha256, value.historySha256, - value.sourceAuthoritySha256, value.migrationAuthoritySha256].every((v) => typeof v === "string" && hex.test(v))) throw new Error("Generation checkpoint is malformed."); + value.sourceAuthoritySha256, value.migrationAuthoritySha256, value.retirementAuthoritySha256].every((v) => typeof v === "string" && hex.test(v))) throw new Error("Generation checkpoint is malformed."); const anchorBytes = decodeState(value.anchorBase64, value.anchorDigest, stateMaxBytes); decodeState(value.sourceTipBase64, value.sourceTipDigest, stateMaxBytes); decodeState(value.migrationTipBase64, value.migrationTipDigest, stateMaxBytes); @@ -117,8 +118,10 @@ export function validateGenerationCheckpoint(input, stateMaxBytes = GENERATION_S } else if (!(prefix === "source" ? ["v1", "v2"] : ["v1"]).includes(value[`${prefix}Kind`]) || value[`${prefix}AuthoritySha256`] === GENERATION_ZERO) throw new Error("Generation provenance is malformed."); } if (value.migrationKind !== null && value.sourceKind !== "v2") throw new Error("Generation migration provenance is malformed."); + if (value.previousGenerationIdentity !== null && (!value.previousGenerationIdentity || Object.keys(value.previousGenerationIdentity).sort().join() !== "dev,ino" || !Object.values(value.previousGenerationIdentity).every((v) => Number.isSafeInteger(v) && v >= 0))) throw new Error("Generation predecessor inode is malformed."); + if ((value.previousGenerationIdentity === null) !== (value.retirementAuthoritySha256 === GENERATION_ZERO)) throw new Error("Generation retirement commitment and inode must be paired."); if (value.epoch === 1) { - if (value.previousCheckpointSha256 !== GENERATION_ZERO || value.previousTipSha256 !== GENERATION_ZERO || value.previousGeneration !== null || value.historySha256 !== genesisHistory(value) || + if (value.previousGenerationIdentity !== null || value.previousCheckpointSha256 !== GENERATION_ZERO || value.previousTipSha256 !== GENERATION_ZERO || value.previousGeneration !== null || value.historySha256 !== genesisHistory(value) || (value.sourceKind !== null && (value.anchorBase64 !== value.sourceTipBase64 || value.anchorDigest !== value.sourceTipDigest))) throw new Error("Generation genesis is not exact."); } else if (typeof value.previousGeneration !== "string" || !/^generation-[0-9]{16}-[0-9a-f]{64}$/.test(value.previousGeneration) || value.previousTipSha256 !== value.anchorDigest) throw new Error("Generation successor is malformed."); if (generationBytes(value).length > generationCheckpointMaxBytes(stateMaxBytes)) throw new Error("Generation checkpoint exceeds its exact envelope byte bound."); @@ -131,7 +134,8 @@ export function consumerGenerationSuccessorCheckpoint(predecessor, tip, stateMax if (anchor.digest !== tip.tipDigest || checkpoint.epoch === Number.MAX_SAFE_INTEGER) throw new Error("Generation immutable tip or epoch is malformed."); const previousDigest = generationDigest(generationBytes(checkpoint)); const next = { ...checkpoint, epoch: checkpoint.epoch + 1, epochId: "", previousCheckpointSha256: previousDigest, - previousGeneration: consumerGenerationName(checkpoint), previousTipSha256: anchor.digest, + previousGeneration: consumerGenerationName(checkpoint), previousGenerationIdentity: tip.previousGenerationIdentity ?? null, + retirementAuthoritySha256: tip.retirementAuthoritySha256 ?? GENERATION_ZERO, previousTipSha256: anchor.digest, historySha256: commitment("pylon-generation-rotation-history-v3", [checkpoint.historySha256, previousDigest, anchor.digest]), anchorDigest: anchor.digest, anchorBase64: anchor.base64 }; next.epochId = identity(next); diff --git a/scripts/pylon-generation-operations.test.mjs b/scripts/pylon-generation-operations.test.mjs new file mode 100644 index 0000000000..625b1efdf1 --- /dev/null +++ b/scripts/pylon-generation-operations.test.mjs @@ -0,0 +1,211 @@ +import assert from "node:assert/strict"; +import { createHash, randomUUID } from "node:crypto"; +import { chmod, cp, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { test } from "node:test"; +import { buildConsumerGeneration, discoverConsumerGenerations, prepareConsumerGeneration, recoverConsumerGenerationBuilder, rotateConsumerGeneration, withConsumerGenerationLock } from "./lib/pylon-consumer-lock.mjs"; + +async function fixture(t) { + const directory = await mkdtemp(join(tmpdir(), "pylon-generation-operations-")); + await chmod(directory, 0o700); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, "journal"); + await mkdir(root, { mode: 0o700 }); + // mkdir is intentionally separate from journal preparation: migration owns installation. + return { directory, root, authority: { genesis: { statePath: join(directory, "state.json"), stateBytes: null } } }; +} +const options = { stateMaxBytes: 1024, startHeartbeat: () => async () => {} }; + +test("v3 operation stages callbacks and converges rotation before another callback", async (t) => { + const f = await fixture(t); + await withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => { + assert.equal(tx.readStateBytes(), null); + await tx.commitState(Buffer.from("first")); + assert.equal(await readFile(f.authority.genesis.statePath).catch(() => null), null); + }, options); + assert.equal((await readFile(f.authority.genesis.statePath)).toString(), "first"); + const result = await rotateConsumerGeneration(f.root, f.authority, options); + assert.equal(result.epoch, 2); + assert.equal((await readdir(f.root)).length, 1); + await withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => assert.equal(tx.readStateBytes().toString(), "first"), options); +}); + +test("v3 operation callback throw preserves identity and never commits staged bytes or replays", async (t) => { + const f = await fixture(t); + const error = Object.assign(new Error("callback"), { code: "ENOENT" }); + let calls = 0; + let escaped; + await assert.rejects(withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => { + calls++; escaped = tx; await tx.commitState(Buffer.from("not committed")); throw error; + }, options), (actual) => actual === error); + assert.equal(calls, 1); + await assert.rejects(escaped.commitState(Buffer.from("late")), /live|staged/); + await withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => assert.equal(tx.readStateBytes(), null), options); +}); + +test("v3 operation recovers durable two-final and every partial deletion cut before callback", async (t) => { + for (const stage of ["two-finals", "retired", "deleting", "first-unlink", "last-proof", "empty-container"]) { + const f = await fixture(t); + await withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => tx.commitState(Buffer.from(stage)), options); + const error = new Error(stage); + let fired = false; + await assert.rejects(rotateConsumerGeneration(f.root, f.authority, { ...options, hooks: { + afterGenerationRename: () => { if (stage === "two-finals") { fired = true; throw error; } }, + generationBoundary: ({ phase, operation, path }) => { + if (fired || phase !== "after") return; + const hit = stage === "retired" && operation === "rename" && basename(path).startsWith(".retired-") || + stage === "deleting" && operation === "rename" && basename(path).startsWith(".deleting-") || + stage === "first-unlink" && operation === "unlink" || + stage === "last-proof" && operation === "unlink" && basename(path) === `receipt-${createHash("sha256").update("retirement.json").digest("hex")}.json` || + stage === "empty-container" && operation === "remove-directory" && basename(path) === "receipts"; + if (hit) { fired = true; throw error; } + }, + } }), (actual) => actual === error); + assert.ok(fired, stage); + let called = false; + await withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => { + called = true; + assert.equal((await readdir(f.root)).length, 1, stage); + assert.equal(tx.readStateBytes().toString(), stage); + }, options); + assert.ok(called, stage); + } +}); + +test("v3 operation cold root-readdir handoff restarts only direct native lost discovery", async (t) => { + const f = await fixture(t); + await withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => tx.commitState(Buffer.from("initial")), options); + let handedOff = false; + let called = false; + await withConsumerGenerationLock(f.root, f.authority, async () => { called = true; }, { ...options, readDirectory: async (path) => { + const names = await readdir(path); + if (path === f.root && !handedOff) { + handedOff = true; + await rotateConsumerGeneration(f.root, f.authority, options); + await rotateConsumerGeneration(f.root, f.authority, options); + const currentNames = await readdir(path); + assert.ok(names.some((name) => name.startsWith("generation-") && !currentNames.includes(name))); + } + return names; + } }); + assert.ok(called); +}); + +for (const code of ["ENOENT", "EIO", "EPERM"]) { + test(`v3 operation preserves injected ${code} through final-retired and retired-deleting response loss`, async (t) => { + for (const prefix of [".retired-", ".deleting-"]) { + const f = await fixture(t); + await prepareConsumerGeneration(f.root, f.authority, options); + const error = Object.assign(new Error("response loss"), { code }); + let moved = false; + await assert.rejects(rotateConsumerGeneration(f.root, f.authority, { ...options, renameFile: async (source, destination) => { + await rename(source, destination); + if (basename(destination).startsWith(prefix)) { moved = true; throw error; } + } }), (actual) => actual === error); + assert.ok(moved); + await prepareConsumerGeneration(f.root, f.authority, options); + assert.equal((await readdir(f.root)).length, 1); + } + }); +} + +test("v3 operation rejects byte-identical retired container replacement", async (t) => { + const f = await fixture(t); + await prepareConsumerGeneration(f.root, f.authority, options); + const cut = new Error("retired"); + let retired; + await assert.rejects(rotateConsumerGeneration(f.root, f.authority, { ...options, hooks: { afterGenerationMove: ({ destination }) => { + if (basename(destination).startsWith(".retired-")) { retired = destination; throw cut; } + } } }), (actual) => actual === cut); + const replacement = join(f.directory, "replacement"); + await cp(retired, replacement, { recursive: true }); + const originalInode = (await lstat(retired)).ino; + await rm(retired, { recursive: true }); + await rename(replacement, retired); + assert.notEqual((await lstat(retired)).ino, originalInode); + await assert.rejects(prepareConsumerGeneration(f.root, f.authority, options), /inode|receipt/); +}); + +test("v3 operation rescans after projection rename and readback and repairs a delayed writer forward", async (t) => { + const f = await fixture(t); + let advanced = false; + let ownClock = 1; + await withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => tx.commitState(Buffer.from("first")), { ...options, now: () => ownClock, hooks: { + afterProjectionFileSync: async () => { + if (advanced) return; + advanced = true; + ownClock = 100_000; + await withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => tx.commitState(Buffer.from("second")), { ...options, now: () => ownClock }); + }, + } }); + assert.ok(advanced); + assert.equal((await readFile(f.authority.genesis.statePath)).toString(), "second"); +}); + +test("v3 operation revalidates latest rotation CAS after the final pre-rename barrier", async (t) => { + const f = await fixture(t); + const initial = await prepareConsumerGeneration(f.root, f.authority, options); + let removed = false; + await assert.rejects(rotateConsumerGeneration(f.root, f.authority, { ...options, hooks: { generationBoundary: async ({ phase, operation, path }) => { + if (!removed && phase === "before" && operation === "rename" && basename(path).startsWith("generation-") && path !== initial.path) { + removed = true; + const index = join(initial.path, "epoch", "claim-index-0000000000000001.json"); + await rm(index); + } + } } })); + assert.ok(removed); + assert.equal((await readdir(f.root)).filter((name) => name.startsWith("generation-")).length, 1); +}); + +test("v3 operation resumes an observed builder from a durable pre-link or linked receipt", async (t) => { + for (const cut of ["file-sync", "link", "rename"]) { + const f = await fixture(t); + const error = new Error(cut); + await assert.rejects(buildConsumerGeneration(f.root, f.authority, { ...options, hooks: { generationBoundary: ({ phase, operation }) => { + if (phase === "after" && operation === cut) throw error; + } } }), (actual) => actual === error); + const path = join(f.root, (await readdir(f.root))[0]); + const inode = (await lstat(path)).ino; + const builder = await recoverConsumerGenerationBuilder(path, f.authority, options); + assert.equal(builder.identity.ino, inode); + await prepareConsumerGeneration(f.root, f.authority, options); + assert.equal((await readdir(f.root)).length, 1); + } +}); + +test("v3 operation cleans exact decided receipt losers despite PID reuse and blocks unresolved live writers", async (t) => { + const f = await fixture(t); + let winner; + await withConsumerGenerationLock(f.root, f.authority, async () => {}, { ...options, hooks: { afterClaim: ({ claim }) => { winner = claim; } } }); + let snapshot = await prepareConsumerGeneration(f.root, f.authority, options); + const loser = { ...winner, token: randomUUID() }; + const loserBytes = Buffer.from(`${JSON.stringify(loser)}\n`); + const contentHash = createHash("sha256").update(loserBytes).digest("hex"); + const target = `epoch/claim-0000000000000001-${contentHash}.json`; + const temporary = join(snapshot.path, "receipts", `.receipt-p${process.pid}-w${randomUUID()}-t${createHash("sha256").update(target).digest("hex")}.tmp`); + await writeFile(temporary, loserBytes, { mode: 0o600 }); + await rotateConsumerGeneration(f.root, f.authority, options); + snapshot = await prepareConsumerGeneration(f.root, f.authority, options); + const live = { ...loser, token: randomUUID() }; + const liveBytes = Buffer.from(`${JSON.stringify(live)}\n`); + const liveTarget = `epoch/claim-0000000000000001-${createHash("sha256").update(liveBytes).digest("hex")}.json`; + await writeFile(join(snapshot.path, "receipts", `.receipt-p${process.pid}-w${randomUUID()}-t${createHash("sha256").update(liveTarget).digest("hex")}.tmp`), liveBytes, { mode: 0o600 }); + await assert.rejects(rotateConsumerGeneration(f.root, f.authority, options), /live unresolved receipt/); +}); + +test("v3 operation cold discovery preserves injected ENOENT even when a successor exists", async (t) => { + const f = await fixture(t); + const initial = await prepareConsumerGeneration(f.root, f.authority, options); + const error = Object.assign(new Error("injected native-looking stat"), { code: "ENOENT" }); + let fired = false; + await assert.rejects(discoverConsumerGenerations(f.root, f.authority, { ...options, lstatEntry: async (path) => { + if (!fired && path === initial.path) { + fired = true; + await rotateConsumerGeneration(f.root, f.authority, options); + throw error; + } + return lstat(path); + } }), (actual) => actual === error); + assert.ok(fired); +}); diff --git a/scripts/pylon-publication.test.mjs b/scripts/pylon-publication.test.mjs index df835a42b5..459cc5ead3 100644 --- a/scripts/pylon-publication.test.mjs +++ b/scripts/pylon-publication.test.mjs @@ -1,3 +1,4 @@ +import "./pylon-generation-operations.test.mjs"; import "./pylon-generation.test.mjs"; import "./pylon-bounded-file.test.mjs"; import assert from "node:assert/strict"; From 676a9652f0e7a789a915af351f81cdc5d0b5ed7f Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Thu, 10 Sep 2026 21:16:47 -0600 Subject: [PATCH 04/14] fix(publication): bind generation temporary recovery and reserve rotation capacity --- package.json | 1 + .../fixtures/generation-operation/worker.mjs | 40 +++ scripts/lib/pylon-consumer-lock.mjs | 294 +++++++++++++++--- scripts/pylon-generation-maximum.test.mjs | 54 ++++ scripts/pylon-generation-operations.test.mjs | 254 ++++++++++++++- 5 files changed, 596 insertions(+), 47 deletions(-) create mode 100644 scripts/fixtures/generation-operation/worker.mjs create mode 100644 scripts/pylon-generation-maximum.test.mjs diff --git a/package.json b/package.json index 53b7ffc2c6..8f8f0ebafe 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "release:pylon:smoke": "node scripts/smoke-pylon-prime-agent-release.mjs", "test:pylon-release": "node --test scripts/pylon-prime-agent-release.test.mjs", "test:pylon-publication": "node --test scripts/pylon-publication.test.mjs", + "test:pylon-publication-maximum": "node --test scripts/pylon-generation-maximum.test.mjs", "test:pylon-ruleset-auditor-app": "node --test scripts/pylon-ruleset-auditor-acceptance.test.mjs", "release:patch": "node scripts/release.mjs patch", "release:minor": "node scripts/release.mjs minor", diff --git a/scripts/fixtures/generation-operation/worker.mjs b/scripts/fixtures/generation-operation/worker.mjs new file mode 100644 index 0000000000..c1ccee605b --- /dev/null +++ b/scripts/fixtures/generation-operation/worker.mjs @@ -0,0 +1,40 @@ +import { basename, join } from "node:path"; +import { buildConsumerGeneration, prepareConsumerGeneration, publishConsumerGeneration, recoverConsumerGenerationBuilder, rotateConsumerGeneration, withConsumerGenerationLock } from "../../lib/pylon-consumer-lock.mjs"; + +const [root, mode, cut = "", suppliedBuilder = ""] = process.argv.slice(2); +const authority = { genesis: { statePath: join(root, "..", "state.json"), stateBytes: null } }; +let stopped = false; +const boundary = async ({ phase, operation, path }) => { + const projection = basename(path).startsWith(".projection-"); + const hit = cut === "projection-created" && phase === "after" && operation === "create-projection" || + cut === "projection-synced" && projection && phase === "after" && operation === "file-sync" || + cut === "projection-before-rename" && phase === "before" && operation === "projection-rename" || + cut === "projection-after-rename" && phase === "after" && operation === "projection-rename" || + cut === "builder-before-publish" && phase === "before" && operation === "rename" && basename(path).startsWith("generation-") || + cut === "retire-before-rename" && phase === "before" && operation === "rename" && basename(path).startsWith(".retired-") || + cut === "delete-before-rename" && phase === "before" && operation === "rename" && basename(path).startsWith(".deleting-") || + cut === "retired" && phase === "after" && operation === "rename" && basename(path).startsWith(".retired-") || + cut === "deleting" && phase === "after" && operation === "rename" && basename(path).startsWith(".deleting-"); + if (!stopped && hit) { + stopped = true; + process.send?.({ type: "cut", phase, operation, path, pid: process.pid }); + await new Promise((resolve) => process.once("message", resolve)); + } +}; +const options = { stateMaxBytes: 1024, startHeartbeat: () => async () => {}, hooks: { generationBoundary: boundary } }; +try { + let result; + if (mode === "commit") result = await withConsumerGenerationLock(root, authority, async (_path, tx) => tx.commitState(Buffer.from("worker-state")), options); + else if (mode === "rotate") result = await rotateConsumerGeneration(root, authority, options); + else if (mode === "prepare") result = await prepareConsumerGeneration(root, authority, options); + else if (mode === "builder") { + const builder = suppliedBuilder ? await recoverConsumerGenerationBuilder(suppliedBuilder, authority, options) : await buildConsumerGeneration(root, authority, options); + result = await publishConsumerGeneration(builder, options); + } else throw new Error("Unknown generation worker mode."); + process.send?.({ type: "done", epoch: result?.checkpoint?.epoch ?? result?.epoch }); + process.disconnect?.(); +} catch (error) { + process.send?.({ type: "error", message: error.message, code: error.code }); + process.disconnect?.(); + process.exitCode = 1; +} diff --git a/scripts/lib/pylon-consumer-lock.mjs b/scripts/lib/pylon-consumer-lock.mjs index 2534dc2563..0b958bd5ea 100644 --- a/scripts/lib/pylon-consumer-lock.mjs +++ b/scripts/lib/pylon-consumer-lock.mjs @@ -3774,8 +3774,9 @@ export async function rotateConsumerStateJournal(statePath, rawOptions = {}) { // V3 primitives remain separate from the public v2 preparation/rotation entrypoints. const generationBuilders = new WeakMap(); const generationHeartbeatPattern = new RegExp(`^heartbeat-([0-9]{16})-(${uuidSource})-([0-9]{16})\\.json$`); +const generationProjectionTemporaryPattern = new RegExp(`^\\.projection-p([1-9][0-9]*)-g([0-9]{16})-c([0-9a-f]{64})-t([0-9a-f]{64})-a(${uuidSource})\\.tmp$`); const generationReceiptPattern = /^receipt-([0-9a-f]{64})\.json$/; -const generationReceiptTemporaryPattern = new RegExp(`^\\.receipt-p([1-9][0-9]*)-w(${uuidSource})-t([0-9a-f]{64})\\.tmp$`); +const generationReceiptTemporaryPattern = new RegExp(`^\\.receipt-p([1-9][0-9]*)-w(${uuidSource})-t([0-9a-f]{64})(?:-g([0-9]{16})-c([0-9a-f]{64}))?\\.tmp$`); function generationOptions(raw = {}) { const options = { stateMaxBytes: GENERATION_STATE_MAX_BYTES, @@ -3962,29 +3963,39 @@ async function generationSync(path, options) { await generationBoundary(options, "after", "sync", path); } async function generationNames(path, limit, options) { - const names = await options.readDirectory(path); + let names; + try { names = await options.readDirectory(path); } catch (error) { + if (options[generationDiscovery] && options.readDirectory === readdir && error?.code === "ENOENT") throw new GenerationDiscoveryLost(); + throw error; + } if (!Array.isArray(names) || names.length > limit || new Set(names).size !== names.length || names.some((name) => typeof name !== "string" || basename(name) !== name || [".", ".."].includes(name))) throw new Error("Generation directory exceeds its entry bound or closed namespace."); return names; } class GenerationDiscoveryLost extends Error {} const generationDiscovery = Symbol("generation discovery"); +async function generationDiscoveryStat(path, options) { + try { return await options.lstatEntry(path); } catch (error) { + if (options[generationDiscovery] && options.lstatEntry === lstat && error?.code === "ENOENT") throw new GenerationDiscoveryLost(); + throw error; + } +} async function generationRootPreflight(root, options) { const rootNames = await generationNames(root, GENERATION_ROOT_MAX_ENTRIES, options); const finalPattern = /^generation-[0-9]{16}-[0-9a-f]{64}$/; - const hiddenPattern = new RegExp(`^\\.building-(?:p[1-9][0-9]*-)?${uuidSource}$`); + const hiddenPattern = new RegExp(`^\\.building-(?:p[1-9][0-9]*-(?:g[0-9a-f]{64}-)?)?${uuidSource}$`); const retiredPattern = /^\.((retired)|(deleting))-generation-[0-9]{16}-[0-9a-f]{64}$/; if (rootNames.filter((name) => finalPattern.test(name)).length > 2 || rootNames.some((name) => !finalPattern.test(name) && !hiddenPattern.test(name) && !retiredPattern.test(name))) throw new Error("Generation root contains an unexpected entry or competing finals."); let totalBytes = 0; const charge = async (path) => { - const stat = generationEntryStat(await options.lstatEntry(path), "file", options); - if (stat.size < (generationReceiptTemporaryPattern.test(basename(path)) ? 0 : 1) || stat.size > options.metadataMaxBytes) throw new Error("Generation root metadata exceeds its byte bound."); + const stat = generationEntryStat(await generationDiscoveryStat(path, options), "file", options); + if (stat.size < ((generationReceiptTemporaryPattern.test(basename(path)) || generationProjectionTemporaryPattern.test(basename(path))) ? 0 : 1) || stat.size > options.metadataMaxBytes) throw new Error("Generation root metadata exceeds its byte bound."); totalBytes += stat.size; if (totalBytes > options.maxJournalBytes) throw new Error("Generation root exceeds its aggregate byte bound."); }; for (const name of rootNames) { const path = join(root, name); try { - generationEntryStat(await options.lstatEntry(path), "directory", options); + generationEntryStat(await generationDiscoveryStat(path, options), "directory", options); } catch (error) { if (options[generationDiscovery] && options.lstatEntry === lstat && error?.code === "ENOENT") throw new GenerationDiscoveryLost(); throw error; @@ -3994,7 +4005,7 @@ async function generationRootPreflight(root, options) { for (const entry of entries) { if (["checkpoint.json", "retirement.json"].includes(entry)) { await charge(join(path, entry)); continue; } const directory = join(path, entry); - generationEntryStat(await options.lstatEntry(directory), "directory", options); + generationEntryStat(await generationDiscoveryStat(directory, options), "directory", options); const names = await generationNames(directory, entry === "epoch" ? GENERATION_EPOCH_MAX_ENTRIES : GENERATION_RECEIPT_MAX_ENTRIES, options); for (const child of names) await charge(join(directory, child)); } @@ -4027,24 +4038,30 @@ async function readGenerationSnapshot(path, checkpoint, options, expectedIdentit for (const name of receiptNames) { const fixed = generationReceiptPattern.exec(name); const temporary = generationReceiptTemporaryPattern.exec(name); - if (!fixed && !temporary) throw new Error("Generation receipt name is malformed."); + const projection = generationProjectionTemporaryPattern.exec(name); + if (!fixed && !temporary && !projection) throw new Error("Generation receipt name is malformed."); const stat = generationEntryStat(await options.lstatEntry(join(path, "receipts", name)), "file", options); - if (stat.size < (temporary ? 0 : 1) || stat.size > options.metadataMaxBytes) throw new Error("Generation receipt exceeds its byte bound."); + if (stat.size < (temporary || projection ? 0 : 1) || stat.size > options.metadataMaxBytes) throw new Error("Generation receipt exceeds its byte bound."); totalBytes += stat.size; if (totalBytes > options.maxJournalBytes) throw new Error("Generation exceeds its aggregate byte bound."); const target = byInode.get(`${stat.dev}:${stat.ino}`); + if (projection) { + if (target || stat.nlink !== 1 || stat.size > options.stateMaxBytes || projection[4] !== checkpoint.statePathSha256 || Number(projection[2]) < 1 || Number(projection[2]) > MAX_LOCK_GENERATIONS) throw new Error("Generation projection temporary has unsafe ownership, target or byte bounds."); + receiptEntries.push({ name, stat, target: null, temporary: true, projection: { generation: Number(projection[2]), claimSha256: projection[3] }, pid: Number(projection[1]) }); + continue; + } if (!target && temporary && stat.nlink === 1) { - receiptEntries.push({ name, stat, target: null, temporary: true, pid: Number(temporary[1]) }); + receiptEntries.push({ name, stat, target: null, temporary: true, pid: Number(temporary[1]), owner: temporary[4] ? { generation: Number(temporary[4]), claimSha256: temporary[5] } : null }); continue; } if (!target || (fixed?.[1] ?? temporary?.[3]) !== digest(Buffer.from(target.name)) || stat.size !== target.stat.size) throw new Error("Generation receipt lacks its exact canonical inode and target."); target.receipts.push({ name, stat, temporary: !!temporary }); - receiptEntries.push({ name, stat, target: target.name, temporary: !!temporary, pid: temporary ? Number(temporary[1]) : null }); + receiptEntries.push({ name, stat, target: target.name, temporary: !!temporary, pid: temporary ? Number(temporary[1]) : null, owner: temporary?.[4] ? { generation: Number(temporary[4]), claimSha256: temporary[5] } : null }); } for (const entry of canonical.values()) { if (entry.receipts.length !== 1 || entry.stat.nlink !== 2 || entry.receipts[0].stat.nlink !== 2 || (requireEmpty && entry.receipts[0].temporary)) throw new Error("Generation canonical metadata lacks its exact durable receipt inode."); } - await generationDirectory(root, options); + const rootIdentity = await generationDirectory(root, options); const identity = await generationDirectory(path, options); if (expectedIdentity !== null && !generationSameInode(identity, expectedIdentity)) throw new Error("Generation directory inode differs from the observed builder identity."); const epochIdentity = await generationDirectory(join(path, "epoch"), options); @@ -4067,8 +4084,22 @@ async function readGenerationSnapshot(path, checkpoint, options, expectedIdentit else epochRecords.set(entry.name.slice(6), bytes); } if (!checkpointBytes.equals(metadataBytes(checkpoint))) throw new Error("Generation checkpoint differs from exact expected authority."); - const snapshot = { path, name: consumerGenerationName(checkpoint), checkpoint, checkpointBytes, retirementCertificate, epochRecords, identity, epochIdentity, receiptsIdentity, totalBytes, receiptEntries, canonicalEntries: [...canonical.values()].map(({ name, stat }) => ({ name, stat })) }; - generationEpochAuthority(snapshot, options); + const snapshot = { path, name: consumerGenerationName(checkpoint), checkpoint, checkpointBytes, retirementCertificate, epochRecords, identity, rootIdentity, epochIdentity, receiptsIdentity, totalBytes, receiptEntries, canonicalEntries: [...canonical.values()].map(({ name, stat }) => ({ name, stat })) }; + const epochAuthority = generationEpochAuthority(snapshot, options); + for (const entry of receiptEntries) { + if (entry.temporary && (!Number.isSafeInteger(entry.pid) || entry.pid < 1)) throw new Error("Generation temporary owner PID is malformed."); + const ownerReference = entry.projection ?? entry.owner; + if (ownerReference) { + if (!Number.isSafeInteger(ownerReference.generation) || ownerReference.generation < 1 || ownerReference.generation > MAX_OPERATION_GENERATIONS) throw new Error("Generation temporary owner slot is malformed."); + const claim = epochAuthority.contents.get(ownerReference.claimSha256); + if (claim && claim.generation !== ownerReference.generation) throw new Error("Generation temporary differs from its exact owner slot."); + if (entry.projection && claim?.type !== "normal") throw new Error("Generation projection temporary lacks its exact normal claim."); + if (!claim && entry.owner) { + const expectedTarget = `epoch/claim-${generationName(ownerReference.generation)}-${ownerReference.claimSha256}.json`; + if (entry.target !== null || generationReceiptTemporaryPattern.exec(entry.name)[3] !== digest(Buffer.from(expectedTarget))) throw new Error("Generation unlinked receipt lacks exact pre-claim ownership."); + } + } + } for (const entry of receiptEntries) { if (!sameRetiredLinkStat(entry.stat, await options.lstatEntry(join(path, "receipts", entry.name)))) throw new Error("Generation receipt inode or stat changed."); } @@ -4114,7 +4145,7 @@ export async function buildConsumerGeneration(root, authority, rawOptions = {}) const rootBytes = await generationRootPreflight(root, options); if (rootBytes + 2 * metadataBytes(checkpoint).length > options.maxJournalBytes) throw new Error("Generation build exceeds the root aggregate byte bound."); const rootIdentity = await generationDirectory(root, options); - const path = join(root, `.building-p${process.pid}-${randomUUID()}`); + const path = join(root, `.building-p${process.pid}-g${checkpoint.epochId}-${randomUUID()}`); await options.makeDirectory(path, { mode: 0o700 }); const identity = await generationDirectory(path, options); await options.makeDirectory(join(path, "epoch"), { mode: 0o700 }); @@ -4163,6 +4194,9 @@ function generationCertificateEntry(snapshot, entry) { const bytes = entry.name === "checkpoint.json" ? snapshot.checkpointBytes : snapshot.epochRecords.get(entry.name.slice(6)); return { name: entry.name, dev: entry.stat.dev, ino: entry.stat.ino, size: bytes.length, sha256: digest(bytes) }; } +function generationRetirementDigest(bytes) { + return createHash("sha256").update("pylon-generation-retirement-v3\0").update(bytes).digest("hex"); +} function generationRetirementCertificate(snapshot, slot) { return { schemaVersion: 1, predecessorGeneration: snapshot.name, predecessorIdentity: snapshot.identity, epochIdentity: snapshot.epochIdentity, receiptsIdentity: snapshot.receiptsIdentity, slot, @@ -4170,7 +4204,7 @@ function generationRetirementCertificate(snapshot, slot) { } function validateGenerationRetirementCertificate(snapshot, successor, options) { const bytes = snapshot.retirementCertificate; - if (!Buffer.isBuffer(bytes) || digest(bytes) !== successor.retirementAuthoritySha256 || !generationSameInode(snapshot.identity, successor.previousGenerationIdentity)) throw new Error("Generation retirement certificate does not bind the exact predecessor inode."); + if (!Buffer.isBuffer(bytes) || generationRetirementDigest(bytes) !== successor.retirementAuthoritySha256 || !generationSameInode(snapshot.identity, successor.previousGenerationIdentity)) throw new Error("Generation retirement certificate does not bind the exact predecessor inode."); const certificate = generationCanonical(bytes, options.metadataMaxBytes); if (!exactKeys(certificate, ["schemaVersion", "predecessorGeneration", "predecessorIdentity", "epochIdentity", "receiptsIdentity", "slot", "entries"]) || certificate.schemaVersion !== 1 || certificate.predecessorGeneration !== snapshot.name || !generationSameInode(certificate.predecessorIdentity, snapshot.identity) || !Number.isSafeInteger(certificate.slot) || certificate.slot < 1 || certificate.slot > MAX_OPERATION_GENERATIONS || !Array.isArray(certificate.entries) || certificate.entries.length > GENERATION_EPOCH_MAX_ENTRIES + 1) throw new Error("Generation retirement certificate is malformed."); const records = new Map(snapshot.epochRecords); @@ -4184,14 +4218,36 @@ function validateGenerationRetirementCertificate(snapshot, successor, options) { return certificate; } async function generationNativeStatOrNull(path, options) { - try { return await options.lstatEntry(path); } catch (error) { + try { + const stat = await options.lstatEntry(path); + if (stat === null || stat === undefined) throw new Error("Generation stat operation returned invalid evidence."); + return stat; + } catch (error) { if (options.lstatEntry === lstat && error?.code === "ENOENT") return null; throw error; } } async function generationReadPinned(snapshot, options) { - return readGenerationSnapshot(snapshot.path, snapshot.checkpoint, options, snapshot.identity); + if (snapshot.rootIdentity && !generationSameInode(snapshot.rootIdentity, await generationDirectory(dirname(snapshot.path), options))) throw new Error("Generation pinned root inode changed."); + const current = await readGenerationSnapshot(snapshot.path, snapshot.checkpoint, options, snapshot.identity); + if (snapshot.rootIdentity && !generationSameInode(snapshot.rootIdentity, current.rootIdentity)) throw new Error("Generation pinned root inode changed during validation."); + return current; +} +function generationReceiptOwner(snapshot, targetName, bytes, options) { + if (targetName === "retirement.json") return null; + const value = generationCanonical(bytes, options.metadataMaxBytes); + const scan = generationEpochAuthority(snapshot, options); + let claim; + if (claimPattern.test(basename(targetName))) claim = value; + else if (claimIndexPattern.test(basename(targetName))) claim = scan.contents.get(value.claimSha256); + else if (transitionPattern.test(basename(targetName))) { + const entry = [...scan.terminals].find(([, terminal]) => terminal.outcome === "commit" && terminal.transactions.some((transaction) => metadataBytes(transaction).equals(bytes))); + claim = entry && scan.claims.find((candidate) => `${candidate.generation}:${candidate.token}` === entry[0]); + } else claim = scan.claims.find((candidate) => candidate.generation === value.generation && candidate.token === value.token); + if (!claim) throw new Error("Generation receipt publication lacks an exact claim owner."); + return { generation: claim.generation, claimSha256: digest(metadataBytes(claim)) }; } + async function generationWriteReceipt(snapshot, targetName, bytes, options, beforeLink) { if (targetName !== "retirement.json" && !/^epoch\/[a-z0-9-]+\.json$/.test(targetName)) throw new Error("Generation publication target is invalid."); if (bytes.length < 1 || bytes.length > options.metadataMaxBytes) throw new Error("Generation publication exceeds its byte bound."); @@ -4200,8 +4256,12 @@ async function generationWriteReceipt(snapshot, targetName, bytes, options, befo await generationReadPinned(snapshot, options); const target = join(snapshot.path, targetName); const receipts = join(snapshot.path, "receipts"); + await generationNames(receipts, GENERATION_RECEIPT_MAX_ENTRIES - 1, options); + if (targetName.startsWith("epoch/")) await generationNames(join(snapshot.path, "epoch"), GENERATION_EPOCH_MAX_ENTRIES - 1, options); const targetHash = digest(Buffer.from(targetName)); - const temporary = join(receipts, `.receipt-p${process.pid}-w${randomUUID()}-t${targetHash}.tmp`); + const owner = generationReceiptOwner(snapshot, targetName, bytes, options); + const ownerSuffix = owner === null ? "" : `-g${generationName(owner.generation)}-c${owner.claimSha256}`; + const temporary = join(receipts, `.receipt-p${process.pid}-w${randomUUID()}-t${targetHash}${ownerSuffix}.tmp`); const fixed = join(receipts, `receipt-${targetHash}.json`); const handle = await options.openFile(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | (constants.O_NOFOLLOW ?? 0), 0o600); try { @@ -4214,6 +4274,16 @@ async function generationWriteReceipt(snapshot, targetName, bytes, options, befo await beforeLink?.(temporary); if (!generationSameInode(snapshot.identity, await generationDirectory(snapshot.path, options))) throw new Error("Generation publication was fenced by a different container inode."); await generationBoundary(options, "before", "link", target); + await beforeLink?.(temporary); + const beforePublication = await generationReadPinned(snapshot, options); + if (beforePublication.retirementCertificate !== null && targetName !== "retirement.json") { + const existing = beforePublication.epochRecords.get(targetName.slice(6)); + if (!existing?.equals(bytes)) { + const value = generationCanonical(bytes, options.metadataMaxBytes); + const claim = value.type === "rotation" ? value : generationEpochAuthority(beforePublication, options).contents.get(value.claimSha256); + if (claim?.type !== "rotation" || claim.intent.checkpoint.retirementAuthoritySha256 !== generationRetirementDigest(beforePublication.retirementCertificate)) throw new Error("Generation metadata publication is fenced by retirement preparation."); + } + } let created = true; try { await options.linkFile(temporary, target); } catch (error) { if (options.linkFile !== link || error?.code !== "EEXIST") throw error; @@ -4280,7 +4350,7 @@ export async function discoverConsumerGenerations(root, authority, rawOptions = export async function recoverConsumerGenerationBuilder(path, authority, rawOptions = {}) { const options = generationOptions(rawOptions); - if (!new RegExp(`^\\.building-(?:p[1-9][0-9]*-)?${uuidSource}$`).test(basename(path))) throw new Error("Generation recovery requires an exact builder name."); + if (!new RegExp(`^\\.building-(?:p[1-9][0-9]*-(?:g[0-9a-f]{64}-)?)?${uuidSource}$`).test(basename(path))) throw new Error("Generation recovery requires an exact builder name."); const checkpoint = expectedConsumerGeneration(authority, options); const identity = await generationDirectory(path, options); const rootIdentity = await generationDirectory(dirname(path), options); @@ -4339,13 +4409,36 @@ export async function recoverConsumerGenerationBuilder(path, authority, rawOptio return builder; } -async function generationQuiesce(snapshot, options, ownTemporary = null) { +async function generationQuiesce(snapshot, options, ownTemporary = null, requireQuiescent = true) { const scan = generationEpochAuthority(snapshot, options); for (const receipt of snapshot.receiptEntries.filter((entry) => entry.temporary)) { const path = join(snapshot.path, "receipts", receipt.name); if (path === ownTemporary) continue; let decided = false; - if (receipt.target === null) { + if (receipt.owner) { + const owner = scan.contents.get(receipt.owner.claimSha256); + if (owner && owner.generation !== receipt.owner.generation) throw new Error("Generation receipt temporary owner slot differs from its claim."); + if (owner) { + const winner = scan.claims.find((claim) => claim.generation === owner.generation); + decided = !!winner && (!metadataBytes(winner).equals(metadataBytes(owner)) || scan.terminals.has(`${owner.generation}:${owner.token}`)); + } + } + if (receipt.projection) { + const owner = scan.contents.get(receipt.projection.claimSha256); + if (!owner || owner.type !== "normal" || owner.generation !== receipt.projection.generation) throw new Error("Generation projection temporary lacks its exact claim owner."); + const winner = scan.claims.find((claim) => claim.generation === owner.generation); + decided = !!winner && (!metadataBytes(winner).equals(metadataBytes(owner)) || scan.terminals.has(`${owner.generation}:${owner.token}`)); + const alive = temporaryProcessIsAlive({ pid: receipt.pid }, options); + if (alive && !requireQuiescent) continue; + if (alive && !decided) throw new Error("Generation rotation is pending until its live unresolved projection writer quiesces."); + if (!sameRetiredLinkStat(receipt.stat, await options.lstatEntry(path))) throw new Error("Generation projection temporary inode changed before cleanup."); + await generationBoundary(options, "before", "unlink", path); + await options.removeFile(path); + await generationBoundary(options, "after", "unlink", path); + await generationSync(join(snapshot.path, "receipts"), options); + continue; + } + if (receipt.target === null && !decided) { const bytes = await readSecureFile(path, options.metadataMaxBytes, "Generation owned receipt temporary", options, 0); if (bytes === null) throw new Error("Generation receipt temporary disappeared after pinning."); let value; @@ -4357,6 +4450,11 @@ async function generationQuiesce(snapshot, options, ownTemporary = null) { else if (value.claimSha256 && value.schemaVersion === 1) { claim = scan.contents.get(value.claimSha256); if (claim && value.generation === claim.generation) target = `epoch/claim-index-${generationName(value.generation)}.json`; + } else if (value.baseDigest && value.candidateBase64) { + validateGenerationTransaction(value, value.baseDigest, options.stateMaxBytes); + const entry = [...scan.terminals].find(([, terminal]) => terminal.outcome === "commit" && terminal.transactions.some((transaction) => metadataBytes(transaction).equals(bytes))); + claim = entry && scan.claims.find((candidate) => `${candidate.generation}:${candidate.token}` === entry[0]); + if (claim) target = `epoch/transition-${value.baseDigest}.json`; } else if (claim && value.outcome) { validateTerminal(value, claim, options.stateMaxBytes, validateGenerationTransaction); target = `epoch/terminal-${generationName(claim.generation)}-${claim.token}.json`; @@ -4393,7 +4491,7 @@ async function generationOwnsClaim(snapshot, claim, options, ownTemporary = null const scan = generationEpochAuthority(current, options); const latest = scan.claims.at(-1); if (!latest || !metadataBytes(latest).equals(metadataBytes(claim)) || scan.terminals.has(`${claim.generation}:${claim.token}`)) throw new Error("Generation operation lost its exact latest claim ownership."); - await generationQuiesce(current, options, ownTemporary); + await generationQuiesce(current, options, ownTemporary, false); return { snapshot: current, scan }; } @@ -4405,7 +4503,7 @@ async function generationPublishClaim(snapshot, claim, options) { const latest = scan.claims.at(-1); if (latest?.type === "rotation" && !metadataBytes(latest).equals(metadataBytes(claim))) throw new Error("Generation claim was fenced by its rotation CAS."); if (claim.generation !== (latest?.generation ?? 0) + 1 && claim.generation !== latest?.generation) throw new Error("Generation claim frontier changed before publication."); - await generationQuiesce(current, options, temporary); + await generationQuiesce(current, options, temporary, claim.type === "rotation"); }; const claimName = `epoch/${basename(claimPath({ epochDirectory: "" }, claim))}`; const result = await generationWriteReceipt(snapshot, claimName, metadataBytes(claim), options, revalidate); @@ -4421,19 +4519,42 @@ async function generationRepairProjection(root, authority, statePath, options) { const discovered = await discoverConsumerGenerations(root, authority, options); if (discovered.generations.length !== 1) throw new Error("Generation projection requires a converged unique final."); const snapshot = discovered.generations[0]; - const tip = generationEpochAuthority(snapshot, options).tip; + const scan = generationEpochAuthority(snapshot, options); + const tip = scan.tip; const current = await readSecureFile(statePath, options.stateMaxBytes, "Generation projection", options, 0, options.hooks?.projectionRead); if (tip.tipBytes !== null && (current === null || !current.equals(tip.tipBytes))) { await options.hooks?.beforeProjectionWrite?.({ tipDigest: tip.tipDigest }); - const temporary = join(dirname(statePath), `.pylon-generation-projection-${randomUUID()}.tmp`); + const owner = scan.claims.at(-1); + if (owner?.type !== "normal") throw new Error("Generation projection requires an exact normal claim owner."); + const receipts = join(snapshot.path, "receipts"); + await generationNames(receipts, GENERATION_RECEIPT_MAX_ENTRIES - 1, options); + if (await generationRootPreflight(root, options) + tip.tipBytes.length > options.maxJournalBytes) throw new Error("Generation projection temporary exceeds aggregate byte bound."); + const temporary = join(receipts, `.projection-p${process.pid}-g${generationName(owner.generation)}-c${digest(metadataBytes(owner))}-t${snapshot.checkpoint.statePathSha256}-a${randomUUID()}.tmp`); const handle = await options.openFile(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | (constants.O_NOFOLLOW ?? 0), 0o600); - try { await handle.writeFile(tip.tipBytes); await handle.sync(); } finally { await handle.close(); } - const identity = await options.lstatEntry(temporary); + try { + await generationBoundary(options, "after", "create-projection", temporary); + await handle.writeFile(tip.tipBytes); + await generationBoundary(options, "before", "file-sync", temporary); + await handle.sync(); + await generationBoundary(options, "after", "file-sync", temporary); + } finally { await handle.close(); } + await generationSync(receipts, options); + const identity = generationEntryStat(await options.lstatEntry(temporary), "file", options); await options.hooks?.afterProjectionFileSync?.({ tipDigest: tip.tipDigest, temporary }); await generationReadPinned(snapshot, options); - await options.renameFile(temporary, statePath); + if (!sameRetiredLinkStat(identity, await options.lstatEntry(temporary))) throw new Error("Generation projection temporary changed before rename."); + await options.hooks?.beforeProjectionRename?.({ source: temporary, destination: statePath, identity }); + await generationBoundary(options, "before", "projection-rename", statePath); + try { await options.renameFile(temporary, statePath); } catch (error) { + if (options.renameFile !== rename || error?.code !== "ENOENT") throw error; + if (!generationSameInode(identity, await options.lstatEntry(statePath))) throw error; + const completed = await readSecureFile(statePath, options.stateMaxBytes, "Generation projection rename join", options); + if (completed === null || !completed.equals(tip.tipBytes)) throw error; + } if (!generationSameInode(identity, await options.lstatEntry(statePath))) throw new Error("Generation projection rename has a different destination inode."); + await generationBoundary(options, "after", "projection-rename", statePath); await options.hooks?.afterProjectionRename?.({ tipDigest: tip.tipDigest }); + await generationSync(receipts, options); await generationSync(dirname(statePath), options); // A writer may have paused after its final authority read and overwritten // a newer projection. Only a fresh immutable-tip scan can permit success. @@ -4467,12 +4588,25 @@ async function generationFinishCommit(snapshot, claim, terminal, root, authority await generationRepairProjection(root, authority, statePath, options); } -async function generationMove(source, destination, identity, options, validate) { +async function generationCleanupCompleted(root, successor, predecessorIdentity, options) { + const names = await generationNames(root, GENERATION_ROOT_MAX_ENTRIES, options); + if (names.length !== 1 || names[0] !== successor.name) return false; + const current = await generationReadPinned(successor, options); + if (current.checkpoint.previousGenerationIdentity === null || !generationSameInode(current.checkpoint.previousGenerationIdentity, predecessorIdentity) || current.checkpoint.retirementAuthoritySha256 === GENESIS_DIGEST) throw new Error("Generation cleanup join lacks the committed predecessor identity."); + await generationSync(root, options); + return true; +} + +async function generationMove(source, destination, identity, options, validate, completed) { + const rootIdentity = await generationDirectory(dirname(source), options); await options.hooks?.beforeGenerationMove?.({ source, destination, identity }); + if (await generationNativeStatOrNull(source, options) === null && await generationNativeStatOrNull(destination, options) === null && await completed?.()) return true; await validate(source); await generationBoundary(options, "before", "rename", destination); + if (!generationSameInode(rootIdentity, await generationDirectory(dirname(source), options))) throw new Error("Generation cleanup root inode changed."); const remainingSource = await generationNativeStatOrNull(source, options); if (remainingSource === null) { + if (await generationNativeStatOrNull(destination, options) === null && await completed?.()) return true; if (!generationSameInode(identity, await generationDirectory(destination, options))) throw new Error("Generation rename join has a different destination inode."); await validate(destination); } else { @@ -4508,7 +4642,7 @@ async function generationDeleteRetired(path, successor, options) { await generationNames(join(path, name), 0, options); } } else { - if (digest(certificateBytes) !== successor.checkpoint.retirementAuthoritySha256) throw new Error("Generation deletion certificate differs from successor commitment."); + if (generationRetirementDigest(certificateBytes) !== successor.checkpoint.retirementAuthoritySha256) throw new Error("Generation deletion certificate differs from successor commitment."); const certificate = generationCanonical(certificateBytes, options.metadataMaxBytes); if (certificate.predecessorGeneration !== successor.checkpoint.previousGeneration || !generationSameInode(certificate.predecessorIdentity, expectedIdentity) || !Array.isArray(certificate.entries) || certificate.entries.length > GENERATION_EPOCH_MAX_ENTRIES + 1) throw new Error("Generation deletion certificate is not bound to its successor."); const expected = new Map(certificate.entries.map((entry) => [entry.name, entry])); @@ -4594,7 +4728,7 @@ async function generationConverge(root, authority, discovered, options) { }; await validate(predecessor.path); const retired = join(root, `.retired-${predecessor.name}`); - await generationMove(predecessor.path, retired, predecessor.identity, options, validate); + await generationMove(predecessor.path, retired, predecessor.identity, options, validate, () => generationCleanupCompleted(root, successor, predecessor.identity, options)); } successor = await generationReadPinned(successor, options); for (const name of await generationNames(root, GENERATION_ROOT_MAX_ENTRIES, options)) { @@ -4613,7 +4747,7 @@ async function generationConverge(root, authority, discovered, options) { await generationQuiesce(pinned, options); }; const deleting = join(root, `.deleting-${expected}`); - await generationMove(path, deleting, successor.checkpoint.previousGenerationIdentity, options, validate); + if (await generationMove(path, deleting, successor.checkpoint.previousGenerationIdentity, options, validate, () => generationCleanupCompleted(root, successor, successor.checkpoint.previousGenerationIdentity, options))) continue; path = deleting; } await generationDeleteRetired(path, successor, options); @@ -4623,6 +4757,62 @@ async function generationConverge(root, authority, discovered, options) { return final.generations[0]; } +async function generationCleanupInstalledBuilders(root, current, options) { + for (const name of await generationNames(root, GENERATION_ROOT_MAX_ENTRIES, options)) { + if (!name.startsWith(".building-")) continue; + const path = join(root, name); + const identity = await generationDirectory(path, options); + const namedGoal = /^\.building-p[1-9][0-9]*-g([0-9a-f]{64})-/.exec(name)?.[1]; + const predecessorGoal = current.checkpoint.previousGeneration?.slice(-64); + const expectedDigest = namedGoal === predecessorGoal ? current.checkpoint.previousCheckpointSha256 : digest(current.checkpointBytes); + if (namedGoal && ![current.checkpoint.epochId, predecessorGoal].includes(namedGoal)) { + const latest = generationEpochAuthority(current, options).claims.at(-1); + if (latest?.type === "rotation" && namedGoal === latest.intent.checkpoint.epochId) continue; + throw new Error("Generation root contains a builder without an installed or latest rotation goal."); + } + const names = await generationNames(path, 3, options); + if (names.some((entry) => !["checkpoint.json", "epoch", "receipts"].includes(entry))) throw new Error("Generation losing builder has an unexpected namespace."); + if (names.includes("epoch")) { await generationDirectory(join(path, "epoch"), options); await generationNames(join(path, "epoch"), 0, options); } + let receiptNames = []; + if (names.includes("receipts")) { await generationDirectory(join(path, "receipts"), options); receiptNames = await generationNames(join(path, "receipts"), 1, options); } + const targetHash = digest(Buffer.from("checkpoint.json")); + if (receiptNames.some((entry) => entry !== `receipt-${targetHash}.json` && generationReceiptTemporaryPattern.exec(entry)?.[3] !== targetHash)) throw new Error("Generation losing builder lacks exact checkpoint receipt ownership."); + const files = [...(names.includes("checkpoint.json") ? ["checkpoint.json"] : []), ...receiptNames.map((entry) => `receipts/${entry}`)]; + const observed = new Map(); + let proved = !!namedGoal && [current.checkpoint.epochId, predecessorGoal].includes(namedGoal); + for (const file of files) { + const stat = generationEntryStat(await options.lstatEntry(join(path, file)), "file", options); + if (stat.size < 0 || stat.size > options.metadataMaxBytes || ![1, 2].includes(stat.nlink)) throw new Error("Generation losing builder file exceeds exact bounds."); + if (file === "checkpoint.json" || file.startsWith("receipts/receipt-") || stat.nlink === 2 || !proved) { + const bytes = await readSecureFile(join(path, file), options.metadataMaxBytes, "Generation losing builder checkpoint", options); + if (bytes === null || digest(bytes) !== expectedDigest) throw new Error("Generation builder conflicts with the independently installed winner."); + proved = true; + } + observed.set(file, stat); + } + if (!proved) throw new Error("Generation incomplete builder lacks an exact installed-winner commitment."); + if (observed.size === 2 && (!generationSameInode(...observed.values()) || [...observed.values()].some((stat) => stat.nlink !== 2))) throw new Error("Generation losing builder receipt is a different inode."); + for (const file of files) { + await generationReadPinned(current, options); + await generationBoundary(options, "before", "unlink", join(path, file)); + if (!generationSameInode(identity, await generationDirectory(path, options)) || !generationSameInode(observed.get(file), await options.lstatEntry(join(path, file)))) throw new Error("Generation losing builder inode changed before cleanup."); + await options.removeFile(join(path, file)); + await generationBoundary(options, "after", "unlink", join(path, file)); + await generationSync(dirname(join(path, file)), options); + } + for (const directory of ["epoch", "receipts"]) { + if (!names.includes(directory)) continue; + await generationNames(join(path, directory), 0, options); + await options.removeFile(join(path, directory), { recursive: true }); + await generationSync(path, options); + } + if (!generationSameInode(identity, await generationDirectory(path, options))) throw new Error("Generation losing builder container was replaced."); + await generationNames(path, 0, options); + await options.removeFile(path, { recursive: true }); + await generationSync(root, options); + } +} + export async function prepareConsumerGeneration(root, authority, rawOptions = {}) { const options = generationOptions(rawOptions); let discovered = await discoverConsumerGenerations(root, authority, options); @@ -4632,7 +4822,9 @@ export async function prepareConsumerGeneration(root, authority, rawOptions = {} await publishConsumerGeneration(builder, options); discovered = await discoverConsumerGenerations(root, authority, options); } - return generationConverge(root, authority, discovered, options); + const current = await generationConverge(root, authority, discovered, options); + await generationCleanupInstalledBuilders(root, current, options); + return current; } export async function rotateConsumerGeneration(root, authority, rawOptions = {}) { @@ -4646,11 +4838,14 @@ export async function rotateConsumerGeneration(root, authority, rawOptions = {}) if (latest?.type !== "rotation") { const slot = (latest?.generation ?? 0) + 1; const certificateBytes = metadataBytes(generationRetirementCertificate(snapshot, slot)); - const wanted = consumerGenerationRotationClaim(snapshot.checkpoint, slot, { ...scan.tip, previousGenerationIdentity: snapshot.identity, retirementAuthoritySha256: digest(certificateBytes) }, options.stateMaxBytes); + const wanted = consumerGenerationRotationClaim(snapshot.checkpoint, slot, { ...scan.tip, previousGenerationIdentity: snapshot.identity, retirementAuthoritySha256: generationRetirementDigest(certificateBytes) }, options.stateMaxBytes); const headroom = 2 * certificateBytes.length + 2 * metadataBytes(wanted).length + 2 * metadataBytes(claimIndexFor(wanted)).length + 2 * metadataBytes(wanted.intent.checkpoint).length; if (await generationRootPreflight(root, options) + headroom > options.maxJournalBytes) throw new Error("Generation lacks reserved rotation headroom."); await options.hooks?.beforeRotationDecision?.({ claim: wanted, intent: wanted.intent }); - const result = await generationWriteReceipt(snapshot, "retirement.json", certificateBytes, options); + const result = await generationWriteReceipt(snapshot, "retirement.json", certificateBytes, options, async () => { + const current = await generationReadPinned(snapshot, options); + if (!metadataBytes(generationRetirementCertificate(current, slot)).equals(certificateBytes)) throw new Error("Generation retirement authority changed before certificate publication."); + }); if (!result.bytes?.equals(certificateBytes)) throw new Error("Generation retirement certificate lost its immutable publication."); snapshot = await generationReadPinned(snapshot, options); if (!(await generationPublishClaim(snapshot, wanted, options))) throw new Error("Generation rotation lost its exact winning CAS."); @@ -4670,6 +4865,14 @@ export async function rotateConsumerGeneration(root, authority, rawOptions = {}) return { epoch: final.checkpoint.epoch, tipSha256: generationEpochAuthority(final, options).tip.tipDigest }; } +async function generationHasOperationCapacity(snapshot, options) { + const certificateBytes = metadataBytes(generationRetirementCertificate(snapshot, 1)).length; + // Maximum state staging, terminal/transition pairs, the next checkpoint and + // rotation claim pairs, and certificate growth remain reserved until quiescence. + const reserve = options.stateMaxBytes + 4 * options.metadataMaxBytes + 4 * (4 * Math.ceil(options.stateMaxBytes / 3) + 1024) + 2 * (certificateBytes + 16_384); + return certificateBytes + 2048 <= options.metadataMaxBytes && snapshot.epochRecords.size + 16 < GENERATION_EPOCH_MAX_ENTRIES && snapshot.receiptEntries.length + 32 < GENERATION_RECEIPT_MAX_ENTRIES && await generationRootPreflight(dirname(snapshot.path), options) + reserve <= options.maxJournalBytes; +} + export async function withConsumerGenerationLock(root, authority, action, rawOptions = {}) { if (typeof action !== "function" || !authority?.genesis?.statePath) throw new Error("Generation operation requires an action and exact genesis authority."); const options = generationOptions(rawOptions); @@ -4680,7 +4883,7 @@ export async function withConsumerGenerationLock(root, authority, action, rawOpt let acquired = false; for (let attempt = 0; attempt < PROJECTION_RETRY_LIMIT; attempt += 1) { snapshot = await prepareConsumerGeneration(root, authority, options); - await generationQuiesce(snapshot, options); + await generationQuiesce(snapshot, options, null, false); snapshot = await generationReadPinned(snapshot, options); let scan = generationEpochAuthority(snapshot, options); const latest = scan.claims.at(-1); @@ -4701,11 +4904,7 @@ export async function withConsumerGenerationLock(root, authority, action, rawOpt snapshot = await generationReadPinned(snapshot, options); scan = generationEpochAuthority(snapshot, options); const slot = (scan.claims.at(-1)?.generation ?? 0) + 1; - // Reserve the full next checkpoint/claim, duplicate receipt links, a - // maximum staged transaction and its terminal, plus certificate growth. - const certificateBytes = metadataBytes(generationRetirementCertificate(snapshot, slot)).length; - const reserve = 4 * options.metadataMaxBytes + 4 * (4 * Math.ceil(options.stateMaxBytes / 3) + 1024) + 2 * (certificateBytes + 16_384); - if (certificateBytes + 2048 > options.metadataMaxBytes || slot > (options.maxLockGenerations ?? PYLON_CONSUMER_ROTATE_CLAIM_TRIGGER) || scan.tip.length >= (options.maxTransactionDepth ?? PYLON_CONSUMER_ROTATE_TRANSITION_TRIGGER) || await generationRootPreflight(root, options) + reserve > options.maxJournalBytes) { + if (slot > (options.maxLockGenerations ?? PYLON_CONSUMER_ROTATE_CLAIM_TRIGGER) || scan.tip.length >= (options.maxTransactionDepth ?? PYLON_CONSUMER_ROTATE_TRANSITION_TRIGGER) || !(await generationHasOperationCapacity(snapshot, options))) { if (scan.claims.length === 0) throw new Error("Generation byte budget cannot reserve one maximum operation and rotation."); await rotateConsumerGeneration(root, authority, options); continue; @@ -4726,6 +4925,7 @@ export async function withConsumerGenerationLock(root, authority, action, rawOpt if (!active) return false; try { const owned = await generationOwnsClaim(snapshot, claim, options); + if (!(await generationHasOperationCapacity(owned.snapshot, options))) throw new Error("Generation heartbeat must quiesce to preserve rotation headroom."); const refreshedAtMs = options.now(); const value = { schemaVersion: 2, generation: claim.generation, token: claim.token, refreshedAtMs }; await generationWriteReceipt(owned.snapshot, `epoch/heartbeat-${generationName(claim.generation)}-${claim.token}-${generationName(refreshedAtMs)}.json`, metadataBytes(value), options, @@ -4733,10 +4933,8 @@ export async function withConsumerGenerationLock(root, authority, action, rawOpt return true; } catch (error) { heartbeatFailure = error; throw error; } }; - await beat(); - const stopHeartbeat = (options.startHeartbeat ?? defaultHeartbeatScheduler)({ interval: options.update ?? PYLON_CONSUMER_LOCK_UPDATE_MS, beat }); - let stopped = false; - const stop = async () => { if (!stopped) { stopped = true; await stopHeartbeat(); } }; + let stopHeartbeat = null; + const stop = async () => { if (stopHeartbeat !== null) { const finish = stopHeartbeat; stopHeartbeat = null; await finish(); } }; const publishDecision = async (wanted) => { const current = await generationReadPinned(snapshot, options); const result = await generationWriteReceipt(current, `epoch/terminal-${generationName(claim.generation)}-${claim.token}.json`, metadataBytes(wanted), options); @@ -4744,8 +4942,12 @@ export async function withConsumerGenerationLock(root, authority, action, rawOpt if (!result.bytes.equals(metadataBytes(wanted))) throw new Error("Generation operation lost ownership before its terminal decision."); }; try { + await beat(); await options.hooks?.afterClaim?.({ claim }); const base = await generationRepairProjection(root, authority, statePath, options); + // Serialize preparation writes; only a live callback needs concurrent heartbeats. + await beat(); + stopHeartbeat = (options.startHeartbeat ?? defaultHeartbeatScheduler)({ interval: options.update ?? PYLON_CONSUMER_LOCK_UPDATE_MS, beat }); const transaction = Object.freeze({ readStateBytes: () => base.tipBytes === null ? null : Buffer.from(base.tipBytes), commitState: async (value) => { diff --git a/scripts/pylon-generation-maximum.test.mjs b/scripts/pylon-generation-maximum.test.mjs new file mode 100644 index 0000000000..96cf641b44 --- /dev/null +++ b/scripts/pylon-generation-maximum.test.mjs @@ -0,0 +1,54 @@ +import assert from "node:assert/strict"; +import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { rotateConsumerGeneration, withConsumerGenerationLock } from "./lib/pylon-consumer-lock.mjs"; + +// Required publication gate, separated from smoke because full authority revalidation +// with three actual 16 MiB checkpoint fields takes several minutes. +async function fixture(t) { + const directory = await mkdtemp(join(tmpdir(), "pylon-generation-maximum-")); + await chmod(directory, 0o700); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, "journal"); + await mkdir(root, { mode: 0o700 }); + return { root, authority: { genesis: { statePath: join(directory, "state.json"), stateBytes: null } } }; +} + +test("v3 operation integrated actual 16 MiB provenance and transaction retain rotation headroom", async (t) => { + const f = await fixture(t); + const initial = Buffer.alloc(16 * 1024 * 1024, 0x61); + const candidate = Buffer.alloc(16 * 1024 * 1024, 0x62); + f.authority.genesis.stateBytes = initial; + f.authority.genesis.source = { kind: "v2", authoritySha256: "1".repeat(64), tipBytes: initial }; + f.authority.genesis.migration = { kind: "v1", authoritySha256: "2".repeat(64), tipBytes: initial }; + const maximum = { startHeartbeat: () => async () => {} }; + let projectionCount = 0; + await withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => { + assert.ok(tx.readStateBytes().equals(initial)); + await tx.commitState(candidate); + }, { ...maximum, hooks: { afterProjectionFileSync: async ({ temporary }) => { + projectionCount++; + assert.equal((await lstat(temporary)).size, candidate.length); + assert.ok(temporary.includes("/receipts/.projection-")); + } } }); + assert.equal(projectionCount, 2); + let peak = 0; + const result = await rotateConsumerGeneration(f.root, f.authority, { ...maximum, hooks: { beforeGenerationRename: async () => { + for (const name of await readdir(f.root)) { + const path = join(f.root, name); + for (const entry of await readdir(path)) { + if (["epoch", "receipts"].includes(entry)) { + for (const file of await readdir(join(path, entry))) peak += (await lstat(join(path, entry, file))).size; + } else peak += (await lstat(join(path, entry))).size; + } + } + } } }); + assert.equal(result.epoch, 2); + assert.ok(peak > 400 * 1024 * 1024 && peak < 512 * 1024 * 1024, String(peak)); + assert.equal((await readdir(f.root)).length, 1); + assert.ok((await readFile(f.authority.genesis.statePath)).equals(candidate)); + t.diagnostic(`Actual integrated rotation root bytes, charging receipt duplicates: ${peak}`); +}); + diff --git a/scripts/pylon-generation-operations.test.mjs b/scripts/pylon-generation-operations.test.mjs index 625b1efdf1..fc69f20d58 100644 --- a/scripts/pylon-generation-operations.test.mjs +++ b/scripts/pylon-generation-operations.test.mjs @@ -1,10 +1,11 @@ import assert from "node:assert/strict"; +import { fork } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; import { chmod, cp, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { basename, join } from "node:path"; import { test } from "node:test"; -import { buildConsumerGeneration, discoverConsumerGenerations, prepareConsumerGeneration, recoverConsumerGenerationBuilder, rotateConsumerGeneration, withConsumerGenerationLock } from "./lib/pylon-consumer-lock.mjs"; +import { buildConsumerGeneration, discoverConsumerGenerations, prepareConsumerGeneration, publishConsumerGeneration, readConsumerGeneration, recoverConsumerGenerationBuilder, rotateConsumerGeneration, withConsumerGenerationLock } from "./lib/pylon-consumer-lock.mjs"; async function fixture(t) { const directory = await mkdtemp(join(tmpdir(), "pylon-generation-operations-")); @@ -209,3 +210,254 @@ test("v3 operation cold discovery preserves injected ENOENT even when a successo } }), (actual) => actual === error); assert.ok(fired); }); + +function generationWorker(t, root, mode, cut = "", builder = "") { + const child = fork(new URL("./fixtures/generation-operation/worker.mjs", import.meta.url), [root, mode, cut, builder], { stdio: ["ignore", "ignore", "pipe", "ipc"] }); + const events = []; + let stderr = ""; + child.stderr.on("data", (data) => { stderr += data.toString(); }); + let cutResolve; + let cutReject; + const atCut = new Promise((resolve, reject) => { cutResolve = resolve; cutReject = reject; }); + const exited = new Promise((resolve) => child.once("exit", (code, signal) => { + if (cut && !events.some((event) => event.type === "cut")) cutReject(new Error(`Worker exited before cut: ${JSON.stringify(events)} ${stderr}`)); + resolve({ code, signal }); + })); + child.on("message", (event) => { events.push(event); if (event.type === "cut") cutResolve(event); }); + child.on("error", cutReject); + t.after(async () => { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await exited; + }); + return { child, atCut, exited, events }; +} + +test("v3 operation SIGKILL projection cuts recover owned staging without callback replay", async (t) => { + for (const cut of ["projection-created", "projection-synced", "projection-before-rename", "projection-after-rename"]) { + const f = await fixture(t); + const worker = generationWorker(t, f.root, "commit", cut); + const observation = await worker.atCut; + assert.equal(observation.pid, worker.child.pid); + worker.child.kill("SIGKILL"); + assert.equal((await worker.exited).signal, "SIGKILL"); + let callbacks = 0; + await withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => { callbacks++; assert.equal(tx.readStateBytes().toString(), "worker-state"); }, options); + assert.equal(callbacks, 1); + const current = await prepareConsumerGeneration(f.root, f.authority, options); + assert.equal(current.receiptEntries.filter((entry) => entry.temporary).length, 0); + assert.equal((await readFile(f.authority.genesis.statePath)).toString(), "worker-state"); + assert.deepEqual((await readdir(f.directory)).sort(), ["journal", "state.json"]); + } +}); + +test("v3 operation projection rename joins only native loss of its own observed inode", async (t) => { + for (const replacement of [false, true]) { + const f = await fixture(t); + const run = withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => tx.commitState(Buffer.from("candidate")), { ...options, hooks: { + beforeProjectionRename: async ({ source, destination }) => { + if (replacement) { await writeFile(destination, await readFile(source), { mode: 0o600 }); await rm(source); } + else await rename(source, destination); + }, + } }); + if (replacement) await assert.rejects(run, (error) => error.code === "ENOENT"); + else await run; + } +}); + +for (const code of ["ENOENT", "EIO", "EPERM"]) { + test(`v3 operation projection preserves injected ${code} after real rename`, async (t) => { + for (const injection of ["renameFile", "beforeProjectionRename", "afterProjectionRename"]) { + const f = await fixture(t); + const error = Object.assign(new Error(injection), { code }); + const raw = injection === "renameFile" ? { renameFile: async (source, destination) => { + await rename(source, destination); if (destination === f.authority.genesis.statePath) throw error; + } } : { hooks: { [injection]: async (observation) => { + if (observation.source) await rename(observation.source, observation.destination); + throw error; + } } }; + await assert.rejects(withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => tx.commitState(Buffer.from("candidate")), { ...options, ...raw }), (actual) => actual === error); + await withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => assert.equal(tx.readStateBytes().toString(), "candidate"), options); + } + }); +} + +test("v3 operation rotation cleans exact decided projection writers despite PID reuse and fences their delayed rename", async (t) => { + const f = await fixture(t); + let paused = false; + await assert.rejects(withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => tx.commitState(Buffer.from("first")), { ...options, hooks: { + afterProjectionFileSync: async () => { + if (paused) return; + paused = true; + await withConsumerGenerationLock(f.root, f.authority, async () => {}, options); + await rotateConsumerGeneration(f.root, f.authority, options); + }, + } })); + assert.ok(paused); + const current = await prepareConsumerGeneration(f.root, f.authority, options); + assert.equal(current.checkpoint.epoch, 2); + assert.equal(current.receiptEntries.filter((entry) => entry.temporary).length, 0); + assert.equal((await readFile(f.authority.genesis.statePath)).toString(), "first"); +}); + +test("v3 operation heartbeat growth stops before consuming rotation certificate headroom", async (t) => { + const f = await fixture(t); + let beat; + let now = 1; + let beats = 0; + await assert.rejects(withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => { + await tx.commitState(Buffer.from("must remain staged")); + for (let index = 0; index < 100; index++) { + now++; + try { await beat(); beats++; } catch (error) { assert.match(error.message, /headroom/); break; } + } + }, { ...options, now: () => now, startHeartbeat: ({ beat: callback }) => { beat = callback; return async () => {}; } }), /headroom/); + assert.ok(beats > 0 && beats < 100); + const rotated = await rotateConsumerGeneration(f.root, f.authority, options); + assert.equal(rotated.epoch, 2); + await withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => assert.equal(tx.readStateBytes(), null), options); +}); + +test("v3 operation two processes join one observed builder inode after native rename loss", async (t) => { + const f = await fixture(t); + const builder = await buildConsumerGeneration(f.root, f.authority, options); + const first = generationWorker(t, f.root, "builder", "builder-before-publish", builder.path); + await first.atCut; + const second = generationWorker(t, f.root, "builder", "builder-before-publish", builder.path); + await second.atCut; + first.child.send("continue"); + assert.equal((await first.exited).code, 0, JSON.stringify(first.events)); + second.child.send("continue"); + assert.equal((await second.exited).code, 0, JSON.stringify(second.events)); + const current = await prepareConsumerGeneration(f.root, f.authority, options); + assert.equal(current.identity.ino, builder.identity.ino); +}); + +test("v3 operation two preparation helpers join committed retirement completion at rename cuts", async (t) => { + for (const cut of ["retire-before-rename", "delete-before-rename"]) { + const f = await fixture(t); + await prepareConsumerGeneration(f.root, f.authority, options); + const stopped = new Error("two finals"); + await assert.rejects(rotateConsumerGeneration(f.root, f.authority, { ...options, hooks: { afterGenerationRename: () => { throw stopped; } } }), (error) => error === stopped); + const first = generationWorker(t, f.root, "prepare", cut); + await first.atCut; + const second = generationWorker(t, f.root, "prepare", cut); + await second.atCut; + first.child.send("continue"); + assert.equal((await first.exited).code, 0, JSON.stringify(first.events)); + second.child.send("continue"); + assert.equal((await second.exited).code, 0, JSON.stringify(second.events)); + assert.equal((await readdir(f.root)).length, 1); + } +}); + + +test("v3 operation exact decided owners clean partial receipts of every operation kind despite PID reuse", async (t) => { + const f = await fixture(t); + let owner; + await withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => tx.commitState(Buffer.from("committed")), { ...options, hooks: { afterClaim: ({ claim }) => { owner = claim; } } }); + const snapshot = await prepareConsumerGeneration(f.root, f.authority, options); + const sha = createHash("sha256").update(`${JSON.stringify(owner)}\n`).digest("hex"); + const targets = [`claim-0000000000000001-${sha}.json`, "claim-index-0000000000000001.json", `heartbeat-0000000000000001-${owner.token}-0000000000000001.json`, `terminal-0000000000000001-${owner.token}.json`, `applied-0000000000000001-${owner.token}.json`, `transition-${"0".repeat(64)}.json`]; + for (const name of targets) { + const targetHash = createHash("sha256").update(`epoch/${name}`).digest("hex"); + await writeFile(join(snapshot.path, "receipts", `.receipt-p${process.pid}-w${randomUUID()}-t${targetHash}-g0000000000000001-c${sha}.tmp`), Buffer.alloc(0), { mode: 0o600 }); + } + const result = await rotateConsumerGeneration(f.root, f.authority, options); + assert.equal(result.epoch, 2); + assert.equal((await readdir(f.root)).length, 1); +}); + +test("v3 operation live unresolved projection writers block rotation until their exact claim is decided", async (t) => { + const f = await fixture(t); + let owner; + let temporary; + await withConsumerGenerationLock(f.root, f.authority, async () => { + const snapshot = await prepareConsumerGeneration(f.root, f.authority, options); + const sha = createHash("sha256").update(`${JSON.stringify(owner)}\n`).digest("hex"); + temporary = join(snapshot.path, "receipts", `.projection-p${process.pid}-g0000000000000001-c${sha}-t${snapshot.checkpoint.statePathSha256}-a${randomUUID()}.tmp`); + await writeFile(temporary, Buffer.alloc(0), { mode: 0o600 }); + await assert.rejects(rotateConsumerGeneration(f.root, f.authority, options), /live unresolved projection/); + assert.equal((await lstat(temporary)).size, 0); + }, { ...options, hooks: { afterClaim: ({ claim }) => { owner = claim; } } }); + await rotateConsumerGeneration(f.root, f.authority, options); + await assert.rejects(lstat(temporary), (error) => error.code === "ENOENT"); +}); + +test("v3 operation independently installed winner permits bounded losing-builder cleanup across its last proof", async (t) => { + const f = await fixture(t); + const first = await buildConsumerGeneration(f.root, f.authority, options); + const loser = await buildConsumerGeneration(f.root, f.authority, options); + await publishConsumerGeneration(first, options); + const cut = new Error("loser last proof"); + await assert.rejects(prepareConsumerGeneration(f.root, f.authority, { ...options, hooks: { generationBoundary: ({ phase, operation, path }) => { + if (phase === "after" && operation === "unlink" && path.startsWith(loser.path) && basename(path).startsWith("receipt-")) throw cut; + } } }), (error) => error === cut); + assert.ok((await readdir(f.root)).includes(basename(loser.path))); + await prepareConsumerGeneration(f.root, f.authority, options); + assert.equal((await readdir(f.root)).length, 1); + await assert.rejects(publishConsumerGeneration(loser, options)); +}); + +test("v3 operation every pinned canonical metadata read preserves hook errors across ancestor retirement", async (t) => { + for (const kind of ["checkpoint.json", "retirement.json", "claim-", "claim-index-", "heartbeat-", "terminal-", "transition-", "applied-"]) { + for (const code of ["ENOENT", "EIO", "EPERM", "native-loss"]) { + const f = await fixture(t); + await withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => tx.commitState(Buffer.from("committed")), options); + const stopped = new Error("two finals"); + await assert.rejects(rotateConsumerGeneration(f.root, f.authority, { ...options, hooks: { afterGenerationRename: () => { throw stopped; } } }), (error) => error === stopped); + const names = (await readdir(f.root)).filter((name) => name.startsWith("generation-")).sort(); + const predecessorPath = join(f.root, names[0]); + const injected = Object.assign(new Error(`${kind} ${code}`), { code }); + let fired = false; + await assert.rejects(readConsumerGeneration(predecessorPath, f.authority, { ...options, hooks: { metadataRead: { afterInitialStat: async ({ path }) => { + const name = basename(path); + if (!fired && path.startsWith(predecessorPath) && (kind === "claim-" ? name.startsWith("claim-") && !name.startsWith("claim-index-") : name.startsWith(kind))) { + fired = true; + await rename(predecessorPath, join(f.root, `.retired-${names[0]}`)); + if (code !== "native-loss") throw injected; + } + } } } }), code === "native-loss" ? /changed|disappeared|ENOENT/ : (error) => error === injected); + assert.ok(fired, `${kind} ${code}`); + } + } +}); + +test("v3 operation schedules callback heartbeats only after projection preparation", async (t) => { + const f = await fixture(t); + f.authority.genesis.stateBytes = Buffer.from("base"); + let prepared = false; + let started = false; + let stopped = false; + await withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => { + assert.ok(started); + assert.equal(tx.readStateBytes().toString(), "base"); + }, { ...options, hooks: { afterProjectionRename: () => { assert.equal(started, false); prepared = true; } }, startHeartbeat: () => { + assert.ok(prepared); + started = true; + return async () => { stopped = true; }; + } }); + assert.ok(stopped); +}); + +test("v3 operation slow preparation loses ownership before callback without replay", async (t) => { + const f = await fixture(t); + f.authority.genesis.stateBytes = Buffer.from("base"); + let now = 1; + let handedOff = false; + let callbacks = 0; + let schedules = 0; + await assert.rejects(withConsumerGenerationLock(f.root, f.authority, async () => { callbacks++; }, { + ...options, now: () => now, stale: 100, + startHeartbeat: () => { schedules++; return async () => {}; }, + hooks: { afterProjectionFileSync: async () => { + if (handedOff) return; + handedOff = true; + now = 1000; + await withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => tx.commitState(Buffer.from("successor")), { ...options, now: () => now, stale: 100 }); + } }, + }), /ownership|claim|terminal|decided/); + assert.ok(handedOff); + assert.equal(callbacks, 0); + assert.equal(schedules, 0); + assert.equal((await readFile(f.authority.genesis.statePath)).toString(), "successor"); +}); From 72dd169abd6bc9dea404d7bd082fe498c76b8c52 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Thu, 10 Sep 2026 23:55:05 -0600 Subject: [PATCH 05/14] fix(publication): validate historical sources before migration Checkpoint the read-only historical authority parser for fixes #53; public migration remains protected v2 until the acknowledged blocker and installation contract is complete. --- scripts/lib/pylon-consumer-lock.mjs | 55 +++- scripts/lib/pylon-consumer-migration.mjs | 289 ++++++++++++++++++++ scripts/pylon-generation-migration.test.mjs | 130 +++++++++ 3 files changed, 462 insertions(+), 12 deletions(-) create mode 100644 scripts/lib/pylon-consumer-migration.mjs create mode 100644 scripts/pylon-generation-migration.test.mjs diff --git a/scripts/lib/pylon-consumer-lock.mjs b/scripts/lib/pylon-consumer-lock.mjs index 0b958bd5ea..b7f651de64 100644 --- a/scripts/lib/pylon-consumer-lock.mjs +++ b/scripts/lib/pylon-consumer-lock.mjs @@ -1,4 +1,5 @@ import { createHash, randomUUID } from "node:crypto"; +import { createConsumerMigrationApi } from "./pylon-consumer-migration.mjs"; import { constants } from "node:fs"; import { link, lstat, mkdir, open, readdir, rename, rm } from "node:fs/promises"; import { basename, dirname, join, parse, relative, resolve, sep } from "node:path"; @@ -196,7 +197,7 @@ function validateTransaction(value, expectedBaseDigest, stateMaxBytes) { !exactKeys(value, ["schemaVersion", "baseDigest", "candidateDigest", "candidateBase64"]) || value.schemaVersion !== TRANSACTION_SCHEMA_VERSION || value.baseDigest !== expectedBaseDigest || !/^[0-9a-f]{64}$/.test(value.candidateDigest ?? "") || typeof value.candidateBase64 !== "string" || - !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value.candidateBase64) + value.candidateBase64.length > 4 * Math.ceil(stateMaxBytes / 3) ) throw new Error("Consumer high-water transaction is malformed."); const candidateBytes = Buffer.from(value.candidateBase64, "base64"); if ( @@ -224,7 +225,7 @@ function validateCheckpoint(value, stateMaxBytes) { ) throw new Error("Consumer high-water journal checkpoint is malformed."); let anchorBytes = null; if (value.anchorBase64 !== null) { - if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value.anchorBase64)) { + if (value.anchorBase64.length > 4 * Math.ceil(stateMaxBytes / 3)) { throw new Error("Consumer high-water journal checkpoint is malformed."); } anchorBytes = Buffer.from(value.anchorBase64, "base64"); @@ -240,7 +241,7 @@ function validateCheckpoint(value, stateMaxBytes) { throw new Error("Consumer high-water checkpoint source-authority tip is malformed."); } } else { - if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value.sourceAuthorityTipBase64)) { + if (value.sourceAuthorityTipBase64.length > 4 * Math.ceil(stateMaxBytes / 3)) { throw new Error("Consumer high-water checkpoint source-authority tip is malformed."); } const sourceTip = Buffer.from(value.sourceAuthorityTipBase64, "base64"); @@ -3805,19 +3806,20 @@ function validateGenerationTransaction(value, expectedBaseDigest, stateMaxBytes) digest(candidateBytes) !== value.candidateDigest || value.candidateDigest === value.baseDigest) throw new Error("Generation transaction payload is malformed."); return { value, candidateBytes }; } -function generationEpochAuthority(snapshot, options) { - const checkpoint = validateGenerationCheckpoint(snapshot.checkpoint, options.stateMaxBytes).checkpoint; - if (!Buffer.isBuffer(snapshot.checkpointBytes) || !metadataBytes(checkpoint).equals(snapshot.checkpointBytes) || snapshot.name !== consumerGenerationName(checkpoint)) throw new Error("Generation predecessor checkpoint authority is not exact."); +function generationEpochAuthority(snapshot, options, historical = false) { + const checkpoint = historical ? validateCheckpoint(snapshot.checkpoint, options.stateMaxBytes).value : validateGenerationCheckpoint(snapshot.checkpoint, options.stateMaxBytes).checkpoint; + if (!Buffer.isBuffer(snapshot.checkpointBytes) || !metadataBytes(checkpoint).equals(snapshot.checkpointBytes) || snapshot.name !== (historical ? epochName(checkpoint) : consumerGenerationName(checkpoint))) throw new Error("Generation predecessor checkpoint authority is not exact."); const records = snapshot.epochRecords; if (!(records instanceof Map) || records.size > GENERATION_EPOCH_MAX_ENTRIES) throw new Error("Generation epoch entry bound is invalid."); - let totalBytes = snapshot.checkpointBytes.length * 2; + let totalBytes = snapshot.checkpointBytes.length * (historical ? 1 : 2); for (const bytes of records.values()) { if (!Buffer.isBuffer(bytes) || bytes.length < 1 || bytes.length > options.metadataMaxBytes) throw new Error("Generation epoch metadata byte bound is invalid."); - totalBytes += bytes.length * 2; + totalBytes += bytes.length * (historical ? 1 : 2); if (totalBytes > options.maxJournalBytes) throw new Error("Generation epoch exceeds its byte bound."); } const contents = new Map(); const indexes = new Map(); + const undigested = new Map(); const heartbeats = new Map(); const heartbeatRecords = []; const terminals = new Map(); @@ -3828,7 +3830,14 @@ function generationEpochAuthority(snapshot, options) { let match; if ((match = claimPattern.exec(name))) { if (value.generation !== Number(match[1]) || generationName(value.generation) !== match[1] || digest(bytes) !== match[2]) throw new Error("Generation claim name is not exact."); - if (value.type === "rotation") { + if (historical) { + validateClaim(value, { checkpoint, checkpointDigest: digest(snapshot.checkpointBytes), epochDirectory: snapshot.name }, options.stateMaxBytes); + if (value.type === "rotation") { + const context = { checkpoint, checkpointDigest: digest(snapshot.checkpointBytes), epochDirectory: snapshot.name }; + const tip = { tipDigest: value.intent.tipSha256, tipBytes: validateCheckpoint(value.intent.checkpoint, options.stateMaxBytes).anchorBytes }; + if (!bytes.equals(metadataBytes(rotationClaimFor(context, value.generation, tip)))) throw new Error("Historical rotation content is not exact."); + } + } else if (value.type === "rotation") { const successor = validateGenerationCheckpoint(value.intent?.checkpoint, options.stateMaxBytes); const expected = consumerGenerationRotationClaim(checkpoint, value.generation, { tipDigest: successor.checkpoint.anchorDigest, tipBytes: successor.anchorBytes, previousGenerationIdentity: successor.checkpoint.previousGenerationIdentity, retirementAuthoritySha256: successor.checkpoint.retirementAuthoritySha256, @@ -3836,6 +3845,11 @@ function generationEpochAuthority(snapshot, options) { if (!bytes.equals(metadataBytes(expected))) throw new Error("Generation rotation intent is not exact."); } else validateClaim(value, null, options.stateMaxBytes); contents.set(match[2], value); + } else if (historical && (match = undigestedClaimPattern.exec(name))) { + validateClaim(value, { checkpoint, checkpointDigest: digest(snapshot.checkpointBytes), epochDirectory: snapshot.name }, options.stateMaxBytes); + if (value.generation !== Number(match[1]) || generationName(value.generation) !== match[1]) throw new Error("Historical claim name is not exact."); + undigested.set(value.generation, value); + contents.set(digest(bytes), value); } else if ((match = claimIndexPattern.exec(name))) { validateClaimIndex(value, Number(match[1])); if (generationName(value.generation) !== match[1]) throw new Error("Generation claim CAS name is malformed."); @@ -3844,7 +3858,7 @@ function generationEpochAuthority(snapshot, options) { heartbeatRecords.push([`${Number(match[1])}:${match[2]}`, value]); const key = `${Number(match[1])}:${match[2]}`; if (!heartbeats.has(key) || heartbeats.get(key).refreshedAtMs < value.refreshedAtMs) heartbeats.set(key, value); - } else if ((match = generationHeartbeatPattern.exec(name))) { + } else if (!historical && (match = generationHeartbeatPattern.exec(name))) { heartbeatRecords.push([`${Number(match[1])}:${match[2]}`, value]); if (value.refreshedAtMs !== Number(match[3]) || value.generation !== Number(match[1]) || value.token !== match[2]) throw new Error("Generation immutable heartbeat name is not exact."); const key = `${Number(match[1])}:${match[2]}`; @@ -3859,6 +3873,10 @@ function generationEpochAuthority(snapshot, options) { } const claims = []; const byKey = new Map(); + for (const [slot, claim] of undigested) { + if (indexes.has(slot)) throw new Error("Historical epoch has competing indexed and undigested claims."); + indexes.set(slot, claimIndexFor(claim)); + } for (const [slot, index] of [...indexes].sort(([a], [b]) => a - b)) { const claim = contents.get(index.claimSha256); if (!claim || claim.generation !== slot || slot !== claims.length + 1) throw new Error("Generation rotation authority has a missing claim CAS or noncontiguous slot."); @@ -3866,7 +3884,7 @@ function generationEpochAuthority(snapshot, options) { byKey.set(`${claim.generation}:${claim.token}`, claim); } for (const claim of contents.values()) { - if (claim.generation > claims.length + 1) throw new Error("Generation epoch contains a future unindexed claim."); + if (!historical && claim.generation > claims.length + 1) throw new Error("Generation epoch contains a future unindexed claim."); } for (const [key, value] of heartbeatRecords) { const claim = byKey.get(key); @@ -3884,7 +3902,7 @@ function generationEpochAuthority(snapshot, options) { validateApplied(value, claim, terminals.get(key)); } let tipDigest = checkpoint.anchorDigest; - let tipBytes = validateGenerationCheckpoint(checkpoint, options.stateMaxBytes).anchorBytes; + let tipBytes = historical ? validateCheckpoint(checkpoint, options.stateMaxBytes).anchorBytes : validateGenerationCheckpoint(checkpoint, options.stateMaxBytes).anchorBytes; const decided = new Map(); let depth = 0; for (const claim of claims) { @@ -4982,3 +5000,16 @@ export async function withConsumerGenerationLock(root, authority, action, rawOpt throw error; } } + +const migrationApi = createConsumerMigrationApi({ + validateLegacyClaim, validateLegacyHeartbeat, validateLegacyTerminal, validateLegacyApplied, + validateLegacyRetirementMarker, legacyRetirementMarkerFor, validateGenerationTransaction, + authorityDigest, validateCheckpoint, checkpointName, epochName, genesisCheckpoint, + migrationCheckpoint, deterministicUuid, generationEpochAuthority, rotationClaimFor, +}); + +// Read-only historical inventory; public migration stays on v2 until the blocker +// and installation contract has passed the protected-client matrix. +export async function inspectConsumerMigrationSource(statePath, rawOptions = {}) { + return migrationApi.inspect(statePath, rawOptions); +} diff --git a/scripts/lib/pylon-consumer-migration.mjs b/scripts/lib/pylon-consumer-migration.mjs new file mode 100644 index 0000000000..0f2143e308 --- /dev/null +++ b/scripts/lib/pylon-consumer-migration.mjs @@ -0,0 +1,289 @@ +import { createHash, randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { link, lstat, mkdir, open, readdir, rename, rm } from "node:fs/promises"; +import { basename, dirname, join, resolve } from "node:path"; +import { readBoundedRegularFile } from "./pylon-bounded-file.mjs"; +import { generationBytes as bytes, generationDigest as digest, GENERATION_ZERO as ZERO, generationRecordMaxBytes, GENERATION_STATE_MAX_BYTES } from "./pylon-generation-format.mjs"; + +const BLOCKER = "claim-9999999999999999.json"; +const MARKER = ".pylon-consumer-v1-retired.json"; +const UUID = "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"; +const checkpointPattern = new RegExp(`^checkpoint-([0-9]{16})-(${UUID})\\.json$`); +const epochPattern = new RegExp(`^epoch-([0-9]{16})-(${UUID})$`); +const temporaryPattern = new RegExp(`^\\.pylon-consumer-tmp-v1-p([1-9][0-9]*)-e(${UUID})-g([0-9]{16})-w(${UUID})-n([0-9a-f]{12})-k([a-z0-9-]{1,40})-t([0-9a-f]{64})\\.tmp$`); +const MAX_ENTRIES = 65_537 * 5 + 4096 + 32; +const MAX_BYTES = 256 * 1024 * 1024; +const same = (a, b) => a !== null && b !== null && a.dev === b.dev && a.ino === b.ino; +const identity = ({ dev, ino }) => ({ dev, ino }); +const slot = (value) => String(value).padStart(16, "0"); +const key = (claim) => `${claim.generation}:${claim.token}`; +const sameBytes = (a, b) => a === null ? b === null : b !== null && a.equals(b); +const closed = (value, keys) => value !== null && typeof value === "object" && !Array.isArray(value) && Object.keys(value).sort().join() === keys.slice().sort().join(); + +function commitment(domain, entries) { + const hash = createHash("sha256").update(`${domain}\0`); + for (const [name, value] of entries) { + for (const data of [Buffer.from(name), value]) { + const size = Buffer.alloc(8); size.writeBigUInt64BE(BigInt(data.length)); hash.update(size).update(data); + } + } + return hash.digest("hex"); +} +function optionsFor(raw) { + const options = { stateMaxBytes: GENERATION_STATE_MAX_BYTES, currentUid: process.getuid(), lstatEntry: lstat, openFile: open, readDirectory: readdir, + makeDirectory: mkdir, linkFile: link, renameFile: rename, removeFile: rm, processKill: process.kill.bind(process), ...raw }; + options.metadataMaxBytes = generationRecordMaxBytes(options.stateMaxBytes); + if (!Number.isSafeInteger(options.currentUid) || options.currentUid < 0) throw new Error("Migration uid is invalid."); + return options; +} +async function boundary(options, phase, operation, path) { await options.hooks?.migrationBoundary?.({ phase, operation, path }); } +async function absent(path, options) { + try { const stat = await options.lstatEntry(path); if (!stat) throw new Error("Migration stat operation returned invalid evidence."); return stat; } + catch (error) { if (options.lstatEntry === lstat && error?.code === "ENOENT") return null; throw error; } +} +function safeStat(stat, type, options, frozen = false) { + const modes = type === "file" ? [0o600] : frozen ? [0o700, 0o500] : [0o700]; + if (stat.isSymbolicLink() || !(type === "file" ? stat.isFile() : stat.isDirectory()) || stat.uid !== options.currentUid || !modes.includes(stat.mode & 0o7777)) throw new Error("Migration entry has unsafe type, owner or exact permissions."); + return stat; +} +async function directory(path, options, frozen = false, synchronize = false) { + const initial = safeStat(await options.lstatEntry(path), "directory", options, frozen); + await options.hooks?.migrationDirectoryObserved?.({ path, identity: identity(initial) }); + const handle = await options.openFile(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + try { + const opened = safeStat(await handle.stat(), "directory", options, frozen); + if (!same(initial, opened)) throw new Error("Migration directory changed inode while opening."); + if (synchronize) { await boundary(options, "before", "directory-sync", path); await handle.sync(); await boundary(options, "after", "directory-sync", path); } + if (!same(opened, safeStat(await options.lstatEntry(path), "directory", options, frozen))) throw new Error("Migration directory changed inode after opening."); + return identity(opened); + } finally { await handle.close(); } +} +async function names(path, max, options) { + const result = await options.readDirectory(path); + if (!Array.isArray(result) || result.length > max || new Set(result).size !== result.length || result.some((name) => typeof name !== "string" || basename(name) !== name || [".", ".."].includes(name))) throw new Error("Migration namespace exceeds its closed entry bound."); + return result.sort(); +} +async function file(path, options, max = options.metadataMaxBytes, required = true, minBytes = 1) { + const data = await readBoundedRegularFile(path, { maxBytes: max, minBytes, openFile: options.openFile, lstatEntry: options.lstatEntry, + hooks: options.hooks?.metadataRead, validateHandle: async (_handle, stat) => safeStat(stat, "file", options) }); + if (data === null && required) throw new Error("Migration required pinned file disappeared."); + return data; +} +function canonical(data) { + const value = JSON.parse(data); + if (!bytes(value).equals(data)) throw new Error("Migration metadata is not canonical."); + return value; +} +async function inventory(path, options, { frozen = false, excludeBlocker = false, maximum = MAX_ENTRIES, budget = { bytes: 0 } } = {}) { + const id = await directory(path, options, frozen); + const entries = await names(path, maximum, options); + const records = new Map(); const stats = new Map(); + for (const name of entries) { + const stat = safeStat(await options.lstatEntry(join(path, name)), "file", options); + if (stat.size < (temporaryPattern.test(name) ? 0 : 1) || stat.size > options.metadataMaxBytes) throw new Error("Historical metadata exceeds its byte bound."); + budget.bytes += stat.size; + if (budget.bytes > MAX_BYTES) throw new Error("Historical source exceeds its aggregate byte bound."); + stats.set(name, stat); + } + for (const name of entries) { + const data = await file(join(path, name), options, options.metadataMaxBytes, true, temporaryPattern.test(name) ? 0 : 1); + const stat = await options.lstatEntry(join(path, name)); + if (!same(stats.get(name), stat) || stats.get(name).size !== stat.size || stats.get(name).mtimeMs !== stat.mtimeMs || stats.get(name).ctimeMs !== stat.ctimeMs) throw new Error("Historical file changed after inventory."); + if (!excludeBlocker || name !== BLOCKER) records.set(name, data); + } + if (!same(id, await directory(path, options, frozen)) || entries.join() !== (await names(path, maximum, options)).join()) throw new Error("Historical namespace changed during inventory."); + return { path, identity: id, records, stats, names: entries }; +} +function dead(pid, options) { + try { options.processKill(pid, 0); return false; } catch (error) { if (error?.code === "ESRCH") return true; if (error?.code === "EPERM") return false; throw error; } +} + +// The factory shares the protected closed value validators and generation engine; +// filesystem migration never calls an old operation/rotation helper. +export function createConsumerMigrationApi(format) { + async function readV1(statePath, lockPath, projection, options, frozen = false) { + const budget = { bytes: 0 }; + const lock = await inventory(lockPath, options, { frozen, maximum: 65_537 * 4 + 2, budget }); + const transactions = await inventory(`${statePath}.transactions`, options, { frozen, maximum: 4096, budget }); + const claims = []; const heartbeats = new Map(); const terminals = new Map(); const applied = new Map(); + let marker = null; const records = []; + for (const [name, data] of lock.records) { + const value = canonical(data); let match; + if (name === MARKER) { marker = format.validateLegacyRetirementMarker(value, statePath); continue; } + if ((match = /^claim-([0-9]{16})\.json$/.exec(name))) { + format.validateLegacyClaim(value); + if (value.generation !== Number(match[1]) || slot(value.generation) !== match[1]) throw new Error("Historical v1 claim name is malformed."); + claims.push(value); + } else if ((match = new RegExp(`^(heartbeat|terminal|applied)-([0-9]{16})-(${UUID})\\.json$`).exec(name))) { + ({ heartbeat: heartbeats, terminal: terminals, applied })[match[1]].set(`${Number(match[2])}:${match[3]}`, value); + } else throw new Error("Historical v1 authority contains an unexpected entry."); + records.push([`lock/${name}`, data]); + } + claims.sort((a, b) => a.generation - b.generation); + if (claims.length > 65_536 || claims.some((claim, index) => claim.generation !== index + 1)) throw new Error("Historical v1 claims are not bounded and contiguous."); + const byKey = new Map(claims.map((claim) => [key(claim), claim])); + for (const [k, value] of heartbeats) { if (!byKey.has(k)) throw new Error("Historical v1 orphan heartbeat."); format.validateLegacyHeartbeat(value, byKey.get(k)); } + for (const claim of claims) if (!heartbeats.has(key(claim))) throw new Error("Historical v1 claim lacks heartbeat."); + for (const [k, value] of terminals) { if (!byKey.has(k)) throw new Error("Historical v1 orphan terminal."); format.validateLegacyTerminal(value, byKey.get(k), options.stateMaxBytes); } + for (const [k, value] of applied) { if (!byKey.has(k)) throw new Error("Historical v1 orphan applied marker."); format.validateLegacyApplied(value, byKey.get(k), terminals.get(k)); } + let tipDigest = ZERO; let tipBytes = null; const decided = new Map(); const prefixes = new Set([ZERO]); + for (const claim of claims) { + const terminal = terminals.get(key(claim)); + for (const transaction of terminal?.outcome === "commit" ? terminal.transactions : []) { + if (transaction.baseDigest !== tipDigest || decided.has(tipDigest) || decided.size >= 4096) throw new Error("Historical v1 decisions are not one bounded exact chain."); + decided.set(tipDigest, transaction); tipBytes = format.validateGenerationTransaction(transaction, tipDigest, options.stateMaxBytes).candidateBytes; + tipDigest = transaction.candidateDigest; prefixes.add(tipDigest); + } + } + const actual = new Map(); + for (const [name, data] of transactions.records) { + const match = /^([0-9a-f]{64})\.json$/.exec(name); if (!match) throw new Error("Historical v1 unexpected transaction."); + const value = canonical(data); format.validateGenerationTransaction(value, match[1], options.stateMaxBytes); + if (!decided.has(match[1]) || !bytes(decided.get(match[1])).equals(data)) throw new Error("Historical v1 transition lacks exact commit decision."); + actual.set(match[1], value); records.push([`transactions/${name}`, data]); + } + let reached = ZERO; const visited = new Set(); + while (actual.has(reached)) { if (visited.has(reached)) throw new Error("Historical v1 transaction cycle."); visited.add(reached); reached = actual.get(reached).candidateDigest; } + if (visited.size !== actual.size) throw new Error("Historical v1 unreachable transaction."); + for (const k of applied.keys()) for (const transaction of terminals.get(k).transactions) if (!actual.has(transaction.baseDigest)) throw new Error("Historical v1 applied transition is absent."); + if (decided.size === 0 && projection !== null) { tipBytes = projection; tipDigest = digest(projection); records.push(["explicit-quiescent-projection", projection]); } + else if (projection !== null && !prefixes.has(digest(projection))) throw new Error("Historical v1 projection is not an authenticated prefix."); + const recoveries = []; + for (const claim of claims) { + const terminal = terminals.get(key(claim)); + if (!terminal) recoveries.push({ target: join(lockPath, `terminal-${slot(claim.generation)}-${claim.token}.json`), value: { schemaVersion: 1, generation: claim.generation, token: claim.token, outcome: "retired" }, owner: claim }); + else if (terminal.outcome === "commit" && !applied.has(key(claim))) { + for (const transaction of terminal.transactions) if (!actual.has(transaction.baseDigest)) recoveries.push({ target: join(transactions.path, `${transaction.baseDigest}.json`), value: transaction, owner: claim }); + recoveries.push({ target: join(lockPath, `applied-${slot(claim.generation)}-${claim.token}.json`), value: { schemaVersion: 1, generation: claim.generation, token: claim.token, terminalSha256: digest(bytes(terminal)) }, owner: claim }); + } + } + const authoritySha256 = format.authorityDigest(records, tipDigest, tipBytes); + if (marker !== null && (!bytes(marker).equals(bytes(format.legacyRetirementMarkerFor(statePath, { authoritySha256, tipDigest }))) || recoveries.length)) throw new Error("Historical v1 retirement marker differs from exact quiescent authority."); + return { kind: "v1", lock, transactions, marker, records, tipDigest, tipBytes, authoritySha256, recoveries }; + } + + async function readV2(statePath, root, projection, options, legacy = null) { + const rootIdentity = await directory(root, options); + const rootNames = await names(root, 16 + 65_536, options); + if (rootNames.filter((name) => checkpointPattern.test(name)).length > 2 || rootNames.filter((name) => epochPattern.test(name)).length > 2 || rootNames.filter((name) => !temporaryPattern.test(name)).length > 16) throw new Error("Historical v2 root exceeds its pre-allocation authority bound."); + const checkpoints = []; const epochs = new Map(); const raw = []; const temporaries = []; const budget = { bytes: 0 }; + const stats = new Map(); + for (const name of rootNames) { + const path = join(root, name); const stat = await options.lstatEntry(path); stats.set(name, stat); + if (checkpointPattern.test(name) || temporaryPattern.test(name)) { + safeStat(stat, "file", options); budget.bytes += stat.size; + if (stat.size < 1 || stat.size > options.metadataMaxBytes || budget.bytes > MAX_BYTES) throw new Error("Historical v2 root exceeds byte bounds."); + } else if (epochPattern.test(name) || name === ".owned-temporaries-v2") safeStat(stat, "directory", options); + else throw new Error("Historical v2 root contains unexpected authority."); + } + for (const name of rootNames) { + const path = join(root, name); + if (checkpointPattern.test(name)) { + const data = await file(path, options); const checkpoint = format.validateCheckpoint(canonical(data), options.stateMaxBytes).value; + if (format.checkpointName(checkpoint) !== name) throw new Error("Historical v2 checkpoint name is not exact."); + checkpoints.push({ checkpoint, data, name }); raw.push([name, data]); + } else if (epochPattern.test(name) || name === ".owned-temporaries-v2") { + const scanned = await inventory(path, options, { maximum: MAX_ENTRIES + 65_536, budget }); + if (name === ".owned-temporaries-v2") temporaries.push(...[...scanned.records].map(([child, data]) => ({ name: child, data, path: join(path, child) }))); + else epochs.set(name, scanned); + for (const [child, data] of scanned.records) raw.push([`${name}/${child}`, data]); + } else { const data = await file(path, options); temporaries.push({ name, data, path }); raw.push([name, data]); } + } + if (!rootNames.includes(".owned-temporaries-v2") || checkpoints.length < 1 || checkpoints.length > 2 || epochs.size > 2) throw new Error("Historical v2 root lacks bounded exact checkpoint authority."); + checkpoints.sort((a, b) => a.checkpoint.epoch - b.checkpoint.epoch); + const scans = new Map(); + for (const entry of checkpoints) { + const checkpoint = entry.checkpoint; + if (checkpoint.epoch === 1) { + const expected = legacy ? format.migrationCheckpoint(statePath, legacy) : format.genesisCheckpoint(statePath); + if (!bytes(expected).equals(entry.data)) throw new Error("Historical v2 genesis differs from exact state/source authority."); + } else if (checkpoint.epochId !== format.deterministicUuid(`pylon-consumer-rotation-v2:${checkpoint.previousCheckpointSha256}:${checkpoint.anchorDigest}`)) throw new Error("Historical v2 rotated identity is not deterministic."); + if (legacy ? checkpoint.sourceAuthoritySha256 !== legacy.authoritySha256 || checkpoint.sourceAuthorityTipDigest !== legacy.tipDigest || checkpoint.sourceAuthorityTipBase64 !== (legacy.tipBytes?.toString("base64") ?? null) : checkpoint.sourceAuthoritySha256 !== ZERO || checkpoint.sourceAuthorityTipDigest !== ZERO || checkpoint.sourceAuthorityTipBase64 !== null) throw new Error("Historical v2 underlying v1 provenance differs."); + const epoch = epochs.get(format.epochName(checkpoint)); + if (!epoch) { + if (entry !== checkpoints.at(-1) || checkpoint.epoch !== 1 || epochs.size !== 0) throw new Error("Historical v2 required epoch is missing."); + continue; + } + const records = new Map(); + for (const [name, data] of epoch.records) { if (temporaryPattern.test(name)) temporaries.push({ name, data, path: join(epoch.path, name) }); else records.set(name, data); } + const scan = format.generationEpochAuthority({ checkpoint, checkpointBytes: entry.data, name: format.epochName(checkpoint), epochRecords: records }, options, true); + scans.set(entry.name, scan); + } + const head = checkpoints.at(-1); const previous = checkpoints.at(-2); + if (previous) { + const previousScan = scans.get(previous.name); const latest = previousScan?.claims.at(-1); + const context = { checkpoint: previous.checkpoint, checkpointDigest: digest(previous.data), epochDirectory: format.epochName(previous.checkpoint) }; + if (latest?.type !== "rotation" || previousScan.decidedTipDigest !== previousScan.tip.tipDigest || !bytes(format.rotationClaimFor(context, latest.generation, previousScan.tip)).equals(bytes(latest)) || !bytes(latest.intent.checkpoint).equals(head.data)) throw new Error("Historical retained predecessor lacks exact latest rotation claim/CAS and tip."); + } + const headEpoch = format.epochName(head.checkpoint); + const headScan = scans.get(head.name) ?? format.generationEpochAuthority({ checkpoint: head.checkpoint, checkpointBytes: head.data, name: headEpoch, epochRecords: new Map() }, options, true); + const latest = headScan.claims.at(-1); + for (const [name, epoch] of epochs) { + if (name === headEpoch || previous && name === format.epochName(previous.checkpoint)) continue; + if (latest?.type !== "rotation" || name !== format.epochName(latest.intent.checkpoint) || epoch.records.size !== 0) throw new Error("Historical v2 extra epoch lacks exact pending rotation intent."); + } + let tipDigest = head.checkpoint.anchorDigest; let tipBytes = format.validateCheckpoint(head.checkpoint, options.stateMaxBytes).anchorBytes; + const prefixes = new Set([tipDigest]); const recoveries = []; + for (const claim of headScan.claims) { + const terminal = headScan.terminals.get(key(claim)); + if (claim.type === "normal" && !terminal) recoveries.push({ target: join(root, headEpoch, `terminal-${slot(claim.generation)}-${claim.token}.json`), value: { schemaVersion: 2, generation: claim.generation, token: claim.token, outcome: "retired" }, owner: claim }); + for (const transaction of terminal?.outcome === "commit" ? terminal.transactions : []) { + tipBytes = format.validateGenerationTransaction(transaction, tipDigest, options.stateMaxBytes).candidateBytes; tipDigest = transaction.candidateDigest; prefixes.add(tipDigest); + if (!headScan.transitions.has(transaction.baseDigest)) recoveries.push({ target: join(root, headEpoch, `transition-${transaction.baseDigest}.json`), value: transaction, owner: claim }); + } + if (terminal?.outcome === "commit" && !headScan.applied.has(key(claim))) recoveries.push({ target: join(root, headEpoch, `applied-${slot(claim.generation)}-${claim.token}.json`), value: { schemaVersion: 2, generation: claim.generation, token: claim.token, terminalSha256: digest(bytes(terminal)) }, owner: claim }); + } + if (tipBytes === null && projection !== null) { tipBytes = projection; tipDigest = digest(projection); } + else if (projection !== null && !prefixes.has(digest(projection))) throw new Error("Historical v2 projection is not an authenticated prefix."); + if (latest?.type === "rotation") { + const context = { checkpoint: head.checkpoint, checkpointDigest: digest(head.data), epochDirectory: headEpoch }; + if (!bytes(format.rotationClaimFor(context, latest.generation, { tipBytes, tipDigest })).equals(bytes(latest))) throw new Error("Historical v2 latest rotation differs from exact immutable tip."); + } + for (const temporary of temporaries) { + const match = temporaryPattern.exec(temporary.name); + if (!match || !["checkpoint", "projection", "transition", "claim", "claim-index", "initial-heartbeat", "heartbeat", "terminal-released", "terminal-retired", "terminal-commit", "applied", "legacy-guard", "legacy-retirement"].includes(match[6])) throw new Error("Historical temporary grammar is invalid."); + if (!dead(Number(match[1]), options)) throw new Error("Historical migration has a live unresolved temporary writer."); + } + if (!same(rootIdentity, await directory(root, options)) || rootNames.join() !== (await names(root, 16 + 65_536, options)).join()) throw new Error("Historical v2 root changed during read."); + for (const [name, stat] of stats) if (!same(stat, await options.lstatEntry(join(root, name)))) throw new Error("Historical v2 root entry changed inode."); + return { kind: "v2", root, identity: rootIdentity, checkpoints, epochs, head, headEpoch, records: raw, tipDigest, tipBytes, recoveries, temporaries }; + } + + + async function inspect(statePath, rawOptions = {}) { + if (typeof statePath !== "string" || !statePath) throw new Error("Migration state path is required."); + const options = optionsFor(rawOptions); + const state = resolve(statePath); + await directory(dirname(state), options); + const projection = await file(state, options, options.stateMaxBytes, false); + const journal = await absent(`${state}.journal`, options); + let legacyProjection = projection; + if (journal) { + await directory(`${state}.journal`, options); + const checkpointNames = (await names(`${state}.journal`, 16 + 65_536, options)).filter((name) => checkpointPattern.test(name)); + if (checkpointNames.length > 2) throw new Error("Historical v2 checkpoint bound is exceeded."); + if (checkpointNames.length) { + const checkpoint = format.validateCheckpoint(canonical(await file(join(`${state}.journal`, checkpointNames.at(-1)), options)), options.stateMaxBytes).value; + if (checkpoint.sourceAuthoritySha256 !== ZERO) legacyProjection = checkpoint.sourceAuthorityTipBase64 === null ? null : Buffer.from(checkpoint.sourceAuthorityTipBase64, "base64"); + } + } + const lock = await absent(`${state}.lock`, options); + const retired = await absent(`${state}.lock.v1-retired`, options); + const transactions = await absent(`${state}.transactions`, options); + if (lock?.isDirectory() && retired) throw new Error("Historical authority has competing original and retired v1 locks."); + let legacy = null; + if (transactions || retired || lock?.isDirectory()) { + if (!transactions || !(retired || lock?.isDirectory())) throw new Error("Historical v1 authority is incomplete."); + legacy = await readV1(state, retired ? `${state}.lock.v1-retired` : `${state}.lock`, legacyProjection, options); + if (retired && lock) { + const expected = { schemaVersion: 1, kind: "pylon-consumer-legacy-lock-guard", statePathSha256: digest(Buffer.from(state)) }; + if (!bytes(expected).equals(await file(`${state}.lock`, options))) throw new Error("Historical prior-retired guard is not exact."); + } + } + const source = journal ? await readV2(state, `${state}.journal`, projection, { ...options, maxJournalBytes: MAX_BYTES }, legacy) : legacy; + if (!source) throw new Error("No historical authority exists."); + return { source, legacy, projection }; + } + return { inspect, readV1, readV2 }; + +} diff --git a/scripts/pylon-generation-migration.test.mjs b/scripts/pylon-generation-migration.test.mjs new file mode 100644 index 0000000000..22d53509a5 --- /dev/null +++ b/scripts/pylon-generation-migration.test.mjs @@ -0,0 +1,130 @@ +import assert from "node:assert/strict"; +import { randomUUID } from "node:crypto"; +import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { inspectConsumerMigrationSource } from "./lib/pylon-consumer-lock.mjs"; +import { generationBytes as bytes, generationDigest as digest, GENERATION_ZERO as ZERO } from "./lib/pylon-generation-format.mjs"; +import * as protectedV2 from "./fixtures/protected-publication-v2/pylon-consumer-lock.mjs"; + +const runtime = { startHeartbeat: () => async () => {}, stateMaxBytes: 1024 * 1024 }; +async function fixture(t) { + const directory = await realpath(await mkdtemp(join(tmpdir(), "pylon-migration-"))); + await chmod(directory, 0o700); + t.after(() => rm(directory, { recursive: true, force: true })); + return join(directory, "state.json"); +} +async function put(path, value) { await writeFile(path, bytes(value), { mode: 0o600 }); } +async function v1(t, incomplete = false) { + const state = await fixture(t); + await mkdir(`${state}.lock`, { mode: 0o700 }); await mkdir(`${state}.transactions`, { mode: 0o700 }); + const claim = { schemaVersion: 1, generation: 1, token: randomUUID(), ownerPid: 2_000_000_000, createdAtMs: 0 }; + const value = Buffer.from("committed"); + const transaction = { schemaVersion: 1, baseDigest: ZERO, candidateDigest: digest(value), candidateBase64: value.toString("base64") }; + const terminal = { schemaVersion: 1, generation: 1, token: claim.token, outcome: "commit", transactions: [transaction] }; + await put(`${state}.lock/claim-0000000000000001.json`, claim); + await put(`${state}.lock/heartbeat-0000000000000001-${claim.token}.json`, { schemaVersion: 1, generation: 1, token: claim.token, refreshedAtMs: 0 }); + await put(`${state}.lock/terminal-0000000000000001-${claim.token}.json`, terminal); + if (!incomplete) { + await put(`${state}.transactions/${ZERO}.json`, transaction); + await put(`${state}.lock/applied-0000000000000001-${claim.token}.json`, { schemaVersion: 1, generation: 1, token: claim.token, terminalSha256: digest(bytes(terminal)) }); + await writeFile(state, value, { mode: 0o600 }); + } + return { state, value, claim, transaction }; +} + +test("migration historical reader preserves exact direct and prior-retired v1 authority", async (t) => { + for (const retired of [false, true]) { + const { state, value } = await v1(t); + if (retired) { + await rename(`${state}.lock`, `${state}.lock.v1-retired`); + await put(`${state}.lock`, { schemaVersion: 1, kind: "pylon-consumer-legacy-lock-guard", statePathSha256: digest(Buffer.from(state)) }); + } + const observed = await inspectConsumerMigrationSource(state); + assert.equal(observed.source.kind, "v1"); assert.deepEqual(observed.source.tipBytes, value); + assert.equal(observed.source.records.length, 5); assert.equal(observed.source.recoveries.length, 0); + } +}); + +test("migration historical reader identifies helpable v1 records without helping or repairing", async (t) => { + const { state, value } = await v1(t, true); + const original = await readdir(`${state}.lock`); + const observed = await inspectConsumerMigrationSource(state); + assert.deepEqual(observed.source.tipBytes, value); assert.equal(observed.source.recoveries.length, 2); + assert.deepEqual(await readdir(`${state}.lock`), original); assert.deepEqual(await readdir(`${state}.transactions`), []); + await assert.rejects(lstat(state), { code: "ENOENT" }); +}); + +test("migration historical reader validates native v2 completed and committed incomplete decisions", async (t) => { + for (const incomplete of [false, true]) { + const state = await fixture(t); const failure = new Error("stop after durable decision"); + const operation = protectedV2.withConsumerStateLock(state, async (_path, tx) => tx.commitState("new state"), { ...runtime, hooks: incomplete ? { afterCommitDecision: () => { throw failure; } } : {} }); + if (incomplete) await assert.rejects(operation, (error) => error === failure); else await operation; + const observed = await inspectConsumerMigrationSource(state); + assert.equal(observed.source.kind, "v2"); assert.equal(observed.source.tipBytes.toString(), "new state"); + assert.equal(observed.source.recoveries.length, incomplete ? 2 : 0); + if (incomplete) await assert.rejects(lstat(state), { code: "ENOENT" }); + } +}); + +test("migration historical reader authenticates retained v2 latest rotation and every epoch", async (t) => { + const state = await fixture(t); + await protectedV2.withConsumerStateLock(state, async (_path, tx) => tx.commitState("one"), runtime); + await protectedV2.rotateConsumerStateJournal(state, runtime); + const observed = await inspectConsumerMigrationSource(state); + assert.equal(observed.source.checkpoints.length, 2); + assert.equal(observed.source.epochs.size, 2); + const predecessor = observed.source.checkpoints[0]; + const epoch = observed.source.epochs.get(predecessor.name.replace("checkpoint-", "epoch-").replace(".json", "")); + const claimIndex = [...epoch.records].filter(([name]) => name.startsWith("claim-index-")).at(-1)[0]; + await rm(join(epoch.path, claimIndex)); + await assert.rejects(inspectConsumerMigrationSource(state), /rotation|unresolved|claim|CAS/); +}); + +test("migration historical reader rejects malformed namespace, nonprefix projection, and unproved frozen modes", async (t) => { + const { state } = await v1(t); + await put(`${state}.lock/extra.json`, {}); + await assert.rejects(inspectConsumerMigrationSource(state), /unexpected/); + await rm(`${state}.lock/extra.json`); + await writeFile(state, "forged", { mode: 0o600 }); + await assert.rejects(inspectConsumerMigrationSource(state), /prefix/); + await writeFile(state, "committed", { mode: 0o600 }); + await chmod(`${state}.lock`, 0o500); + await assert.rejects(inspectConsumerMigrationSource(state), /permissions/); + await chmod(`${state}.lock`, 0o700); +}); + +test("migration historical reader preserves injected native-looking error identity", async (t) => { + const { state } = await v1(t); + for (const code of ["ENOENT", "EIO", "EPERM"]) { + const failure = Object.assign(new Error(`injected ${code}`), { code }); + await assert.rejects(inspectConsumerMigrationSource(state, { lstatEntry: async (path) => { + if (path === `${state}.journal`) throw failure; + return lstat(path); + } }), (error) => error === failure); + } + assert.equal((await readFile(state)).toString(), "committed"); +}); + + +test("migration historical reader keeps underlying v1 provenance after v2 advances projection", async (t) => { + const { state, value } = await v1(t); + await protectedV2.migrateConsumerStateJournal(state, runtime); + await protectedV2.withConsumerStateLock(state, async (_path, tx) => tx.commitState("v2 advanced"), runtime); + const observed = await inspectConsumerMigrationSource(state); + assert.deepEqual(observed.legacy.tipBytes, value); + assert.equal(observed.source.tipBytes.toString(), "v2 advanced"); + assert.equal(observed.projection.toString(), "v2 advanced"); +}); + +test("migration historical reader never silently excludes an unauthenticated blocker", async (t) => { + const { state } = await v1(t); + await put(`${state}.lock/claim-9999999999999999.json`, { schemaVersion: 3, kind: "forged" }); + await assert.rejects(inspectConsumerMigrationSource(state), /claim.*malformed/); + const native = await fixture(t); + await protectedV2.withConsumerStateLock(native, async (_path, tx) => tx.commitState("state"), runtime); + const clean = await inspectConsumerMigrationSource(native); + await put(join(clean.source.root, clean.source.headEpoch, "claim-9999999999999999.json"), { schemaVersion: 3, kind: "forged" }); + await assert.rejects(inspectConsumerMigrationSource(native), /claim.*malformed/); +}); From 0d17544342395f60f4a681875e157c5a068fdf38 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Fri, 11 Sep 2026 01:13:31 -0600 Subject: [PATCH 06/14] fix(publication): migrate preserved consumer authority to v3 Fixes #53 --- .../retained-publication-v2/provenance.json | 22 + .../pylon-consumer-lock.mjs | 3757 +++++++++++++++++ .../verify-pylon-preview-history.mjs | 140 + .../verify-pylon-stable-history.mjs | 162 + scripts/lib/pylon-consumer-lock.mjs | 47 +- scripts/lib/pylon-consumer-migration.mjs | 631 ++- scripts/migrate-pylon-consumer-journal.mjs | 13 +- scripts/pylon-generation-migration.test.mjs | 369 +- scripts/pylon-public-state.test.mjs | 253 ++ scripts/pylon-publication.test.mjs | 14 +- 10 files changed, 5373 insertions(+), 35 deletions(-) create mode 100644 scripts/fixtures/retained-publication-v2/provenance.json create mode 100644 scripts/fixtures/retained-publication-v2/pylon-consumer-lock.mjs create mode 100644 scripts/fixtures/retained-publication-v2/verify-pylon-preview-history.mjs create mode 100644 scripts/fixtures/retained-publication-v2/verify-pylon-stable-history.mjs create mode 100644 scripts/pylon-public-state.test.mjs diff --git a/scripts/fixtures/retained-publication-v2/provenance.json b/scripts/fixtures/retained-publication-v2/provenance.json new file mode 100644 index 0000000000..2ff590429b --- /dev/null +++ b/scripts/fixtures/retained-publication-v2/provenance.json @@ -0,0 +1,22 @@ +{ + "repository": "pylon-code/prime-agent", + "commit": "c4f3cf669961d85718f767b1e6f0869304c03f14", + "transform": "Retained v2 implementation before public v3 dispatch; omit v3-only imports/engine and rewrite local imports. Shared current bounded-file evidence classes are intentional. No fixture is shipped as a public entrypoint.", + "files": [ + { + "path": "pylon-consumer-lock.mjs", + "sourcePath": "scripts/lib/pylon-consumer-lock.mjs", + "sha256": "0288682cee693f286a840514744e15142ee82d18a9ddfb0083af31fcb87e85a8" + }, + { + "path": "verify-pylon-preview-history.mjs", + "sourcePath": "scripts/verify-pylon-preview-history.mjs", + "sha256": "3b58766a65d0ab2c497c8b79328f28b05d1c1d1f3c6748b6a5f3f4ca956f9de6" + }, + { + "path": "verify-pylon-stable-history.mjs", + "sourcePath": "scripts/verify-pylon-stable-history.mjs", + "sha256": "88a779785eb35dcfbd858c0f17c593826ff4501aabc8d16654ff2bd72123219d" + } + ] +} diff --git a/scripts/fixtures/retained-publication-v2/pylon-consumer-lock.mjs b/scripts/fixtures/retained-publication-v2/pylon-consumer-lock.mjs new file mode 100644 index 0000000000..7715a419d1 --- /dev/null +++ b/scripts/fixtures/retained-publication-v2/pylon-consumer-lock.mjs @@ -0,0 +1,3757 @@ +import { createHash, randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { link, lstat, mkdir, open, readdir, rename, rm } from "node:fs/promises"; +import { basename, dirname, join, parse, relative, resolve, sep } from "node:path"; + +import { + BoundedFileLinkRetiredBeforeReadError, + BoundedFileLinkRetiredDuringReadError, + BoundedFileUnlinkedDuringReadError, + readBoundedRegularFile, +} from "../../lib/pylon-bounded-file.mjs"; + +class ConsumerEpochAdvancedError extends Error { + constructor() { + super("Consumer high-water journal epoch changed and fenced a paused writer."); + this.name = "ConsumerEpochAdvancedError"; + } +} + +export const PYLON_CONSUMER_LOCK_STALE_MS = 30_000; +export const PYLON_CONSUMER_LOCK_UPDATE_MS = 10_000; +export const PYLON_CONSUMER_ROTATE_CLAIM_TRIGGER = 60_000; +export const PYLON_CONSUMER_ROTATE_TRANSITION_TRIGGER = 3_800; +const LOCK_SCHEMA_VERSION = 2; +const CLAIM_INDEX_SCHEMA_VERSION = 1; +const LEGACY_LOCK_SCHEMA_VERSION = 1; +const TRANSACTION_SCHEMA_VERSION = 1; +const CHECKPOINT_SCHEMA_VERSION = 2; +const ROTATION_INTENT_SCHEMA_VERSION = 2; +const LEGACY_GUARD_SCHEMA_VERSION = 1; +const LEGACY_RETIREMENT_SCHEMA_VERSION = 1; +const GENESIS_DIGEST = "0".repeat(64); +const DEFAULT_STATE_MAX_BYTES = 1024 * 1024; +const MAX_STATE_BYTES = 16 * 1024 * 1024; +const DEFAULT_JOURNAL_MAX_BYTES = 64 * 1024 * 1024; +const MAX_JOURNAL_BYTES = 256 * 1024 * 1024; +const MAX_TRANSACTION_DEPTH = 4096; +const MAX_LOCK_GENERATIONS = 65_536; +const MAX_OPERATION_GENERATIONS = MAX_LOCK_GENERATIONS + 1; +const MAX_JOURNAL_ROOT_ENTRIES = 16; +const MAX_TEMPORARY_ENTRIES = 65_536; +const PROJECTION_RETRY_LIMIT = 32; +const TEMPORARY_DIRECTORY_NAME = ".owned-temporaries-v2"; +const LEGACY_RETIREMENT_MARKER_NAME = ".pylon-consumer-v1-retired.json"; +const uuidSource = "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"; +const uuidPattern = new RegExp(`^${uuidSource}$`); +const claimPattern = /^claim-([0-9]{16})-([0-9a-f]{64})\.json$/; +const claimIndexPattern = /^claim-index-([0-9]{16})\.json$/; +const undigestedClaimPattern = /^claim-([0-9]{16})\.json$/; +const transitionPattern = /^transition-([0-9a-f]{64})\.json$/; +const legacyTransitionPattern = /^([0-9a-f]{64})\.json$/; +const checkpointPattern = new RegExp(`^checkpoint-([0-9]{16})-(${uuidSource})\\.json$`); +const epochPattern = new RegExp(`^epoch-([0-9]{16})-(${uuidSource})$`); +const heartbeatPattern = new RegExp(`^heartbeat-([0-9]{16})-(${uuidSource})\\.json$`); +const terminalPattern = new RegExp(`^terminal-([0-9]{16})-(${uuidSource})\\.json$`); +const appliedPattern = new RegExp(`^applied-([0-9]{16})-(${uuidSource})\\.json$`); +const temporaryPattern = new RegExp( + `^\\.pylon-consumer-tmp-v1-p([1-9][0-9]*)-e(${uuidSource})-g([0-9]{16})-w(${uuidSource})-n([0-9a-f]{12})-k([a-z0-9-]{1,40})-t([0-9a-f]{64})\\.tmp$`, +); + +function exactKeys(value, keys) { + return value !== null && typeof value === "object" && !Array.isArray(value) && + Object.keys(value).sort().join(",") === [...keys].sort().join(","); +} + +function metadataBytes(value) { + return Buffer.from(`${JSON.stringify(value)}\n`); +} + +function digest(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +function generationName(generation) { + if (!Number.isSafeInteger(generation) || generation < 0 || generation > 9_999_999_999_999_999) { + throw new Error("Consumer high-water lock generation is exhausted or malformed."); + } + return String(generation).padStart(16, "0"); +} + +function deterministicUuid(value) { + const hex = digest(Buffer.from(value)); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-4${hex.slice(13, 16)}-8${hex.slice(17, 20)}-${hex.slice(20, 32)}`; +} + +function checkpointName(checkpoint) { + return `checkpoint-${generationName(checkpoint.epoch)}-${checkpoint.epochId}.json`; +} + +function epochName(checkpoint) { + return `epoch-${generationName(checkpoint.epoch)}-${checkpoint.epochId}`; +} + +function claimPath(context, claim) { + return join(context.epochDirectory, `claim-${generationName(claim.generation)}-${digest(metadataBytes(claim))}.json`); +} + +function claimIndexPath(context, generation) { + return join(context.epochDirectory, `claim-index-${generationName(generation)}.json`); +} + +function heartbeatPath(context, claim) { + return join(context.epochDirectory, `heartbeat-${generationName(claim.generation)}-${claim.token}.json`); +} + +function terminalPath(context, claim) { + return join(context.epochDirectory, `terminal-${generationName(claim.generation)}-${claim.token}.json`); +} + +function appliedPath(context, claim) { + return join(context.epochDirectory, `applied-${generationName(claim.generation)}-${claim.token}.json`); +} + +function transitionPath(context, baseDigest) { + return join(context.epochDirectory, `transition-${baseDigest}.json`); +} + +function validateClaim(value, context, stateMaxBytes) { + if ( + !value || value.schemaVersion !== LOCK_SCHEMA_VERSION || !Number.isSafeInteger(value.generation) || + value.generation < 1 || value.generation > MAX_OPERATION_GENERATIONS || !uuidPattern.test(value.token ?? "") || + !["normal", "rotation"].includes(value.type) + ) throw new Error("Consumer high-water operation claim is malformed."); + if (value.type === "normal") { + if ( + !exactKeys(value, ["schemaVersion", "generation", "token", "type", "ownerPid", "createdAtMs"]) || + !Number.isSafeInteger(value.ownerPid) || value.ownerPid < 1 || + !Number.isSafeInteger(value.createdAtMs) || value.createdAtMs < 0 + ) throw new Error("Consumer high-water normal operation claim is malformed."); + return value; + } + if (!exactKeys(value, ["schemaVersion", "generation", "token", "type", "intent"]) || !context) { + throw new Error("Consumer high-water rotation operation claim is malformed."); + } + const intent = validateRotationIntent(value.intent, context, stateMaxBytes); + if (value.token !== intent.checkpoint.epochId) { + throw new Error("Consumer high-water rotation operation claim differs from its deterministic intent."); + } + return value; +} + +function claimIndexFor(claim) { + return { + schemaVersion: CLAIM_INDEX_SCHEMA_VERSION, + generation: claim.generation, + claimSha256: digest(metadataBytes(claim)), + }; +} + +function validateClaimIndex(value, generation) { + if ( + !exactKeys(value, ["schemaVersion", "generation", "claimSha256"]) || + value.schemaVersion !== CLAIM_INDEX_SCHEMA_VERSION || value.generation !== generation || + !/^[0-9a-f]{64}$/.test(value.claimSha256 ?? "") + ) throw new Error("Consumer high-water claim index is malformed."); + return value; +} + +function validateHeartbeat(value, claim) { + if ( + claim.type !== "normal" || + !exactKeys(value, ["schemaVersion", "generation", "token", "refreshedAtMs"]) || + value.schemaVersion !== LOCK_SCHEMA_VERSION || value.generation !== claim.generation || value.token !== claim.token || + !Number.isSafeInteger(value.refreshedAtMs) || value.refreshedAtMs < claim.createdAtMs + ) throw new Error("Consumer high-water lock heartbeat is malformed."); + return value; +} + +function transactionFor(baseDigest, candidateBytes) { + return { + schemaVersion: TRANSACTION_SCHEMA_VERSION, + baseDigest, + candidateDigest: digest(candidateBytes), + candidateBase64: candidateBytes.toString("base64"), + }; +} + +function validateTransaction(value, expectedBaseDigest, stateMaxBytes) { + if ( + !exactKeys(value, ["schemaVersion", "baseDigest", "candidateDigest", "candidateBase64"]) || + value.schemaVersion !== TRANSACTION_SCHEMA_VERSION || value.baseDigest !== expectedBaseDigest || + !/^[0-9a-f]{64}$/.test(value.candidateDigest ?? "") || typeof value.candidateBase64 !== "string" || + value.candidateBase64.length > 4 * Math.ceil(stateMaxBytes / 3) + ) throw new Error("Consumer high-water transaction is malformed."); + const candidateBytes = Buffer.from(value.candidateBase64, "base64"); + if ( + candidateBytes.length < 1 || candidateBytes.length > stateMaxBytes || + candidateBytes.toString("base64") !== value.candidateBase64 || digest(candidateBytes) !== value.candidateDigest || + value.candidateDigest === value.baseDigest + ) throw new Error("Consumer high-water transaction payload is malformed."); + return { value, candidateBytes }; +} + +function validateCheckpoint(value, stateMaxBytes) { + if ( + !exactKeys(value, [ + "schemaVersion", "epoch", "epochId", "previousCheckpointSha256", "previousTipSha256", + "historySha256", "anchorDigest", "anchorBase64", "retiredEpochDirectory", "sourceAuthoritySha256", + "sourceAuthorityTipDigest", "sourceAuthorityTipBase64", + ]) || value.schemaVersion !== CHECKPOINT_SCHEMA_VERSION || !Number.isSafeInteger(value.epoch) || value.epoch < 1 || + !uuidPattern.test(value.epochId ?? "") || !/^[0-9a-f]{64}$/.test(value.previousCheckpointSha256 ?? "") || + !/^[0-9a-f]{64}$/.test(value.previousTipSha256 ?? "") || !/^[0-9a-f]{64}$/.test(value.historySha256 ?? "") || + !/^[0-9a-f]{64}$/.test(value.anchorDigest ?? "") || !/^[0-9a-f]{64}$/.test(value.sourceAuthoritySha256 ?? "") || + !/^[0-9a-f]{64}$/.test(value.sourceAuthorityTipDigest ?? "") || + !(value.retiredEpochDirectory === null || epochPattern.test(value.retiredEpochDirectory)) || + !(value.anchorBase64 === null || typeof value.anchorBase64 === "string") || + !(value.sourceAuthorityTipBase64 === null || typeof value.sourceAuthorityTipBase64 === "string") + ) throw new Error("Consumer high-water journal checkpoint is malformed."); + let anchorBytes = null; + if (value.anchorBase64 !== null) { + if (value.anchorBase64.length > 4 * Math.ceil(stateMaxBytes / 3)) { + throw new Error("Consumer high-water journal checkpoint is malformed."); + } + anchorBytes = Buffer.from(value.anchorBase64, "base64"); + if ( + anchorBytes.length < 1 || anchorBytes.length > stateMaxBytes || anchorBytes.toString("base64") !== value.anchorBase64 || + digest(anchorBytes) !== value.anchorDigest + ) throw new Error("Consumer high-water journal checkpoint anchor is malformed."); + } else if (value.anchorDigest !== GENESIS_DIGEST) { + throw new Error("Consumer high-water journal checkpoint anchor is malformed."); + } + if (value.sourceAuthorityTipBase64 === null) { + if (value.sourceAuthorityTipDigest !== GENESIS_DIGEST) { + throw new Error("Consumer high-water checkpoint source-authority tip is malformed."); + } + } else { + if (value.sourceAuthorityTipBase64.length > 4 * Math.ceil(stateMaxBytes / 3)) { + throw new Error("Consumer high-water checkpoint source-authority tip is malformed."); + } + const sourceTip = Buffer.from(value.sourceAuthorityTipBase64, "base64"); + if ( + sourceTip.length < 1 || sourceTip.length > stateMaxBytes || + sourceTip.toString("base64") !== value.sourceAuthorityTipBase64 || digest(sourceTip) !== value.sourceAuthorityTipDigest + ) throw new Error("Consumer high-water checkpoint source-authority tip is malformed."); + } + if (value.epoch === 1) { + if ( + value.previousCheckpointSha256 !== GENESIS_DIGEST || value.previousTipSha256 !== GENESIS_DIGEST || + value.retiredEpochDirectory !== null + ) throw new Error("Consumer high-water genesis checkpoint is malformed."); + } else if (value.retiredEpochDirectory === null || value.previousTipSha256 !== value.anchorDigest) { + throw new Error("Consumer high-water rotated checkpoint is malformed."); + } + return { value, anchorBytes }; +} + +function validateRotationIntent(value, context, stateMaxBytes) { + if ( + !exactKeys(value, ["schemaVersion", "epoch", "epochId", "checkpointSha256", "tipSha256", "checkpoint"]) || + value.schemaVersion !== ROTATION_INTENT_SCHEMA_VERSION || value.epoch !== context.checkpoint.epoch || + value.epochId !== context.checkpoint.epochId || value.checkpointSha256 !== context.checkpointDigest || + !/^[0-9a-f]{64}$/.test(value.tipSha256 ?? "") + ) throw new Error("Consumer high-water rotation intent is malformed."); + const checkpoint = validateCheckpoint(value.checkpoint, stateMaxBytes).value; + if ( + checkpoint.epoch !== context.checkpoint.epoch + 1 || + checkpoint.previousCheckpointSha256 !== context.checkpointDigest || + checkpoint.previousTipSha256 !== value.tipSha256 || checkpoint.anchorDigest !== value.tipSha256 || + checkpoint.retiredEpochDirectory !== basename(context.epochDirectory) || + checkpoint.sourceAuthoritySha256 !== context.checkpoint.sourceAuthoritySha256 || + checkpoint.sourceAuthorityTipDigest !== context.checkpoint.sourceAuthorityTipDigest || + checkpoint.sourceAuthorityTipBase64 !== context.checkpoint.sourceAuthorityTipBase64 || + checkpoint.historySha256 !== digest(Buffer.from( + `${context.checkpoint.historySha256}:${context.checkpointDigest}:${value.tipSha256}`, + )) + ) throw new Error("Consumer high-water rotation intent does not anchor the exact epoch and tip."); + return value; +} + +function validateTerminal(value, claim, stateMaxBytes, validatePayload = validateTransaction) { + const common = ["schemaVersion", "generation", "token", "outcome"]; + if ( + claim.type !== "normal" || !value || value.schemaVersion !== LOCK_SCHEMA_VERSION || value.generation !== claim.generation || + value.token !== claim.token || !["released", "retired", "commit"].includes(value.outcome) + ) throw new Error("Consumer high-water lock terminal marker is malformed."); + if (["released", "retired"].includes(value.outcome)) { + if (!exactKeys(value, common)) throw new Error("Consumer high-water lock terminal marker is malformed."); + return value; + } + if ( + !exactKeys(value, [...common, "transactions"]) || !Array.isArray(value.transactions) || + value.transactions.length < 1 || value.transactions.length > 2 + ) throw new Error("Consumer high-water lock commit marker is malformed."); + let expectedBase = value.transactions[0]?.baseDigest; + if (!/^[0-9a-f]{64}$/.test(expectedBase ?? "")) throw new Error("Consumer high-water lock commit marker is malformed."); + for (const transaction of value.transactions) { + validatePayload(transaction, expectedBase, stateMaxBytes); + expectedBase = transaction.candidateDigest; + } + return value; +} + +function validateApplied(value, claim, terminal) { + if ( + !exactKeys(value, ["schemaVersion", "generation", "token", "terminalSha256"]) || + value.schemaVersion !== LOCK_SCHEMA_VERSION || value.generation !== claim.generation || value.token !== claim.token || + terminal?.outcome !== "commit" || value.terminalSha256 !== digest(metadataBytes(terminal)) + ) throw new Error("Consumer high-water lock applied marker is malformed."); + return value; +} + +function validateLegacyClaim(value) { + if ( + !exactKeys(value, ["schemaVersion", "generation", "token", "ownerPid", "createdAtMs"]) || + value.schemaVersion !== LEGACY_LOCK_SCHEMA_VERSION || !Number.isSafeInteger(value.generation) || value.generation < 1 || + !uuidPattern.test(value.token ?? "") || !Number.isSafeInteger(value.ownerPid) || value.ownerPid < 1 || + !Number.isSafeInteger(value.createdAtMs) || value.createdAtMs < 0 + ) throw new Error("Legacy consumer high-water lock claim is malformed."); + return value; +} + +function validateLegacyHeartbeat(value, claim) { + if ( + !exactKeys(value, ["schemaVersion", "generation", "token", "refreshedAtMs"]) || + value.schemaVersion !== LEGACY_LOCK_SCHEMA_VERSION || value.generation !== claim.generation || + value.token !== claim.token || !Number.isSafeInteger(value.refreshedAtMs) || value.refreshedAtMs < claim.createdAtMs + ) throw new Error("Legacy consumer high-water heartbeat is malformed."); + return value; +} + +function validateLegacyTerminal(value, claim, stateMaxBytes) { + const common = ["schemaVersion", "generation", "token", "outcome"]; + if ( + !value || value.schemaVersion !== LEGACY_LOCK_SCHEMA_VERSION || value.generation !== claim.generation || + value.token !== claim.token || !["released", "retired", "commit"].includes(value.outcome) + ) throw new Error("Legacy consumer high-water terminal marker is malformed."); + if (value.outcome !== "commit") { + if (!exactKeys(value, common)) throw new Error("Legacy consumer high-water terminal marker is malformed."); + return value; + } + if ( + !exactKeys(value, [...common, "transactions"]) || !Array.isArray(value.transactions) || + value.transactions.length < 1 || value.transactions.length > 2 + ) throw new Error("Legacy consumer high-water commit marker is malformed."); + let expectedBase = value.transactions[0]?.baseDigest; + if (!/^[0-9a-f]{64}$/.test(expectedBase ?? "")) throw new Error("Legacy consumer high-water commit marker is malformed."); + for (const transaction of value.transactions) { + validateTransaction(transaction, expectedBase, stateMaxBytes); + expectedBase = transaction.candidateDigest; + } + return value; +} + +function validateLegacyApplied(value, claim, terminal) { + if ( + !exactKeys(value, ["schemaVersion", "generation", "token", "terminalSha256"]) || + value.schemaVersion !== LEGACY_LOCK_SCHEMA_VERSION || value.generation !== claim.generation || + value.token !== claim.token || terminal?.outcome !== "commit" || + value.terminalSha256 !== digest(metadataBytes(terminal)) + ) throw new Error("Legacy consumer high-water applied marker is malformed."); + return value; +} + +function legacyGuardFor(statePath) { + return { + schemaVersion: LEGACY_GUARD_SCHEMA_VERSION, + kind: "pylon-consumer-legacy-lock-guard", + statePathSha256: digest(Buffer.from(statePath)), + }; +} + +function legacyRetirementMarkerFor(statePath, legacy) { + return { + schemaVersion: LEGACY_RETIREMENT_SCHEMA_VERSION, + kind: "pylon-consumer-v1-retirement", + statePathSha256: digest(Buffer.from(statePath)), + authoritySha256: legacy.authoritySha256, + tipSha256: legacy.tipDigest, + }; +} + +function validateLegacyRetirementMarker(value, statePath) { + if ( + !exactKeys(value, ["schemaVersion", "kind", "statePathSha256", "authoritySha256", "tipSha256"]) || + value.schemaVersion !== LEGACY_RETIREMENT_SCHEMA_VERSION || value.kind !== "pylon-consumer-v1-retirement" || + value.statePathSha256 !== digest(Buffer.from(statePath)) || + !/^[0-9a-f]{64}$/.test(value.authoritySha256 ?? "") || !/^[0-9a-f]{64}$/.test(value.tipSha256 ?? "") + ) throw new Error("Legacy consumer high-water retirement marker is malformed."); + return value; +} + +async function secureHandle(handle, stat, description, type, options) { + if ((type === "file" && !stat.isFile()) || (type === "directory" && !stat.isDirectory())) { + throw new Error(`${description} must be one real ${type}.`); + } + if (stat.uid !== options.currentUid) throw new Error(`${description} must be owned by the current uid.`); + const requiredMode = type === "directory" ? 0o700 : 0o600; + if ((stat.mode & 0o7777) !== requiredMode) { + throw new Error(`${description} must already have exact ${requiredMode.toString(8)} permissions before use.`); + } + return stat; +} + +async function secureDirectory(path, description, options) { + let handle; + try { + handle = await options.openFile( + path, + constants.O_RDONLY | (constants.O_DIRECTORY ?? 0) | (constants.O_NOFOLLOW ?? 0), + ); + } catch (error) { + if (["ELOOP", "ENOTDIR"].includes(error?.code)) throw new Error(`${description} must be one real directory.`); + throw error; + } + try { + await secureHandle(handle, await handle.stat(), description, "directory", options); + } finally { + await handle.close(); + } +} + +export async function syncConsumerStateDirectory(path, { openDirectory = open } = {}) { + let handle; + try { + handle = await openDirectory(path, "r"); + await handle.sync(); + } catch (error) { + if (!["EINVAL", "EPERM", "EISDIR"].includes(error?.code)) throw error; + } finally { + if (handle !== undefined) await handle.close(); + } +} + +export async function ensureDurableConsumerStateDirectory( + directory, + { lstatEntry = lstat, makeDirectory = mkdir, syncDirectory = syncConsumerStateDirectory, create = true } = {}, +) { + const absolute = resolve(directory); + const root = parse(absolute).root; + let parent = root; + const rootEntry = await lstatEntry(root); + if (!rootEntry.isDirectory()) throw new Error("Consumer high-water state directory must be one canonical real directory."); + const remainder = relative(root, absolute); + for (const component of remainder ? remainder.split(sep) : []) { + const current = join(parent, component); + let entry; + try { + entry = await lstatEntry(current); + } catch (error) { + if (error?.code !== "ENOENT" || !create) throw error; + try { + await makeDirectory(current, { mode: 0o700 }); + } catch (mkdirError) { + if (mkdirError?.code !== "EEXIST") throw mkdirError; + } + entry = await lstatEntry(current); + } + if (!entry.isDirectory() || entry.isSymbolicLink?.()) { + throw new Error("Consumer high-water state directory must be one canonical real directory."); + } + await syncDirectory(parent); + parent = current; + } + return absolute; +} + +async function ensureDirectory(path, description, options) { + try { + await options.makeDirectory(path, { mode: 0o700 }); + } catch (error) { + if (error?.code !== "EEXIST") throw error; + } + const entry = await options.lstatEntry(path); + if (!entry.isDirectory() || entry.isSymbolicLink?.()) throw new Error(`${description} must be one real directory.`); + await secureDirectory(path, description, options); + await options.syncDirectory(path); + await options.syncDirectory(dirname(path)); +} + +async function readSecureFile(path, maxBytes, description, options, minBytes = 1, hooks, expectedSha256 = null) { + return readBoundedRegularFile(path, { + maxBytes, + minBytes, + description, + openFile: options.openFile, + lstatEntry: options.lstatEntry, + hooks: { + ...hooks, + afterInitialPathStat: async (observation) => { + await options.afterInitialPathStat?.(observation); + await hooks?.afterInitialPathStat?.(observation); + }, + }, + expectedSha256, + validateHandle: (handle, stat) => secureHandle(handle, stat, description, "file", options), + }); +} + +async function readExactMetadata(path, maxBytes, validate, description, options, budget, expectedSha256 = null) { + const bytes = await readSecureFile( + path, + maxBytes, + description, + options, + 1, + options.hooks?.metadataRead, + expectedSha256, + ); + if (bytes === null) return null; + if (budget) { + budget.bytes += bytes.length; + if (budget.bytes > options.maxJournalBytes) throw new Error("Consumer high-water journal exceeds its safe byte bound."); + } + let value; + try { + value = validate(JSON.parse(bytes)); + } catch (error) { + if (error instanceof SyntaxError) throw new Error(`${description} is malformed.`); + throw error; + } + if (!bytes.equals(metadataBytes(value))) throw new Error(`${description} is not canonical.`); + return value; +} + +function temporaryName(targetPath, kind, writer, context) { + if (!/^[a-z0-9-]{1,40}$/.test(kind)) throw new Error("Consumer high-water temporary kind is malformed."); + const attempt = randomUUID().replaceAll("-", "").slice(0, 12); + return `.pylon-consumer-tmp-v1-p${process.pid}-e${context.checkpoint.epochId}-g${generationName(writer.generation)}` + + `-w${writer.token}-n${attempt}-k${kind}-t${digest(Buffer.from(resolve(targetPath)))}.tmp`; +} + +async function inspectTemporary(path, options) { + const match = temporaryPattern.exec(basename(path)); + if (!match) throw new Error("Consumer high-water journal contains an unexpected hidden entry."); + let handle; + try { + handle = await options.openFile(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + } catch (error) { + if (error?.code === "ENOENT") return null; + if (["ELOOP", "EISDIR"].includes(error?.code)) { + throw new Error("Consumer high-water owned temporary is not one regular non-symlink file."); + } + throw error; + } + try { + const stat = await secureHandle( + handle, + await handle.stat(), + "Consumer high-water owned temporary", + "file", + options, + ); + if (stat.size > options.metadataMaxBytes) throw new Error("Consumer high-water owned temporary exceeds its safe byte bound."); + } finally { + await handle.close(); + } + const kind = match[6]; + const allowedKinds = new Set([ + "checkpoint", "projection", "transition", "claim", "claim-index", "initial-heartbeat", "heartbeat", + "terminal-released", "terminal-retired", "terminal-commit", "applied", "legacy-guard", + "legacy-retirement", + ]); + if (!allowedKinds.has(kind)) throw new Error("Consumer high-water owned temporary target metadata is malformed."); + return { + path, + pid: Number(match[1]), + epochId: match[2], + generation: Number(match[3]), + token: match[4], + attempt: match[5], + kind, + targetSha256: match[7], + }; +} + +function isImmediateSuccessorCheckpoint(context, checkpoint) { + return checkpoint.epoch === context.checkpoint.epoch + 1 && + checkpoint.epochId === deterministicUuid( + `pylon-consumer-rotation-v2:${context.checkpointDigest}:${checkpoint.anchorDigest}`, + ) && + checkpoint.previousCheckpointSha256 === context.checkpointDigest && + checkpoint.previousTipSha256 === checkpoint.anchorDigest && + checkpoint.retiredEpochDirectory === basename(context.epochDirectory) && + checkpoint.sourceAuthoritySha256 === context.checkpoint.sourceAuthoritySha256 && + checkpoint.sourceAuthorityTipDigest === context.checkpoint.sourceAuthorityTipDigest && + checkpoint.sourceAuthorityTipBase64 === context.checkpoint.sourceAuthorityTipBase64 && + checkpoint.historySha256 === digest(Buffer.from( + `${context.checkpoint.historySha256}:${context.checkpointDigest}:${checkpoint.anchorDigest}`, + )); +} + +function retainedCheckpointPath(context) { + if (context.checkpoint.retiredEpochDirectory === null) return null; + const match = epochPattern.exec(context.checkpoint.retiredEpochDirectory); + if (!match) throw new Error("Consumer high-water journal checkpoint context is malformed."); + return join(context.journalDirectory, `checkpoint-${match[1]}-${match[2]}.json`); +} + +function contextCheckpointAnchors(context) { + const anchors = new Map([[context.checkpointPath, context.checkpointDigest]]); + const retainedPath = retainedCheckpointPath(context); + if (retainedPath !== null) anchors.set(retainedPath, context.checkpoint.previousCheckpointSha256); + return anchors; +} + +function isContextAnchoredCheckpoint(context, path, checkpoint, expectedSha256) { + if (path === context.checkpointPath) { + return expectedSha256 === context.checkpointDigest && + metadataBytes(checkpoint).equals(metadataBytes(context.checkpoint)); + } + const retainedPath = retainedCheckpointPath(context); + return retainedPath !== null && path === retainedPath && + expectedSha256 === context.checkpoint.previousCheckpointSha256 && + checkpoint.epoch + 1 === context.checkpoint.epoch && + epochName(checkpoint) === context.checkpoint.retiredEpochDirectory && + checkpoint.sourceAuthoritySha256 === context.checkpoint.sourceAuthoritySha256 && + checkpoint.sourceAuthorityTipDigest === context.checkpoint.sourceAuthorityTipDigest && + checkpoint.sourceAuthorityTipBase64 === context.checkpoint.sourceAuthorityTipBase64 && + context.checkpoint.historySha256 === digest(Buffer.from( + `${checkpoint.historySha256}:${expectedSha256}:${context.checkpoint.anchorDigest}`, + )); +} + +function isAuthenticatedCheckpointAnchor(context, path, checkpoint, expectedSha256, additionalAnchor = null) { + return isContextAnchoredCheckpoint(context, path, checkpoint, expectedSha256) || ( + additionalAnchor !== null && path === additionalAnchor.path && expectedSha256 === additionalAnchor.digest && + metadataBytes(checkpoint).equals(metadataBytes(additionalAnchor.checkpoint)) && + isImmediateSuccessorCheckpoint(context, checkpoint) + ); +} + +function canonicalCheckpointNameEpoch(name) { + const match = checkpointPattern.exec(name); + if (!match) return null; + const epoch = Number(match[1]); + return Number.isSafeInteger(epoch) && generationName(epoch) === match[1] ? epoch : null; +} + +function isProvisionallyRemovedContextCurrentCheckpoint(path, context, anchors, rootNames) { + const name = basename(path); + if ( + path !== context.checkpointPath || path !== join(context.journalDirectory, name) || + name !== checkpointName(context.checkpoint) || anchors.get(path) !== context.checkpointDigest || + canonicalCheckpointNameEpoch(name) !== context.checkpoint.epoch + ) return false; + return rootNames.some((candidate) => { + const candidateEpoch = canonicalCheckpointNameEpoch(candidate); + return candidateEpoch !== null && candidateEpoch > context.checkpoint.epoch; + }); +} + +function isVanishedRetainedCheckpoint(path, context, anchors, rootNames) { + const retainedPath = retainedCheckpointPath(context); + if ( + retainedPath === null || path !== retainedPath || + anchors.get(path) !== context.checkpoint.previousCheckpointSha256 + ) return false; + const retainedMatch = checkpointPattern.exec(basename(path)); + if (!retainedMatch || Number(retainedMatch[1]) + 1 !== context.checkpoint.epoch) return false; + return rootNames.some((candidate) => { + const candidateEpoch = canonicalCheckpointNameEpoch(candidate); + return candidateEpoch !== null && candidateEpoch > Number(retainedMatch[1]); + }); +} + +const checkpointStatEvidenceKeys = ["dev", "ino", "size", "mtimeMs", "ctimeMs", "nlink"]; + +function isFrozenRecord(value) { + return value !== null && typeof value === "object" && Object.isFrozen(value); +} + +function isExactCheckpointStatEvidence(value) { + return isFrozenRecord(value) && exactKeys(value, checkpointStatEvidenceKeys) && + checkpointStatEvidenceKeys.every((key) => Number.isFinite(value[key])) && + Number.isSafeInteger(value.size) && value.size >= 0 && Number.isSafeInteger(value.nlink) && value.nlink >= 0; +} + +function exactEvidenceMonotoneCut(observations, fromLinks, toLinks, byteLength) { + if ( + observations.length < 2 || observations.some((stat) => !isExactCheckpointStatEvidence(stat)) || + observations[0].size !== byteLength || observations[0].nlink !== fromLinks || + observations.at(-1).nlink !== toLinks || + observations.some((stat) => ( + stat.dev !== observations[0].dev || stat.ino !== observations[0].ino || + stat.size !== observations[0].size || stat.mtimeMs !== observations[0].mtimeMs + )) + ) return null; + let cut = null; + for (let index = 1; index < observations.length; index += 1) { + const previous = observations[index - 1]; + const current = observations[index]; + if (previous.nlink === current.nlink) { + if (previous.ctimeMs !== current.ctimeMs) return null; + continue; + } + if ( + cut !== null || previous.nlink !== fromLinks || current.nlink !== toLinks || + previous.ctimeMs === current.ctimeMs + ) return null; + cut = index; + } + return cut; +} + +function isExactLinkRetiredBeforeReadEvidence(error) { + const transition = error.statTransition; + return isFrozenRecord(transition) && exactKeys(transition, ["pathEntry", "openedHandle"]) && + exactEvidenceMonotoneCut( + [transition.pathEntry, transition.openedHandle], + 2, + 1, + error.bytes.length, + ) === 1; +} + +function isExactLinkRetiredDuringReadEvidence(error) { + const transition = error.statTransition; + return isFrozenRecord(transition) && exactKeys(transition, ["pathEntry", "before", "after", "finalPathEntry"]) && + [2, 3].includes(exactEvidenceMonotoneCut( + [transition.pathEntry, transition.before, transition.after, transition.finalPathEntry], + 2, + 1, + error.bytes.length, + )); +} + +function isExactUnlinkedDuringReadEvidence(error) { + const transition = error.statTransition; + if ( + !isFrozenRecord(transition) || + !exactKeys(transition, ["pathEntry", "before", "after", "confirmedHandle"]) || + !(transition.confirmedHandle === null || isExactCheckpointStatEvidence(transition.confirmedHandle)) + ) return false; + const observations = [transition.pathEntry, transition.before, transition.after]; + if (transition.confirmedHandle !== null) observations.push(transition.confirmedHandle); + return exactEvidenceMonotoneCut(observations, 1, 0, error.bytes.length) !== null; +} + +function authenticatedChangedCheckpointRead(error, context, options, anchors, rootNames, additionalAnchor = null) { + const linkRetiredBeforeRead = error instanceof BoundedFileLinkRetiredBeforeReadError && + error.constructor === BoundedFileLinkRetiredBeforeReadError && + error.name === "BoundedFileLinkRetiredBeforeReadError"; + const linkRetiredDuringRead = error instanceof BoundedFileLinkRetiredDuringReadError && + error.constructor === BoundedFileLinkRetiredDuringReadError && + error.name === "BoundedFileLinkRetiredDuringReadError"; + const unlinkedDuringRead = error instanceof BoundedFileUnlinkedDuringReadError && + error.constructor === BoundedFileUnlinkedDuringReadError && + error.name === "BoundedFileUnlinkedDuringReadError"; + const linkRetiredDuringOrBeforeRead = linkRetiredBeforeRead || linkRetiredDuringRead; + if ( + (!linkRetiredDuringOrBeforeRead && !unlinkedDuringRead) || + error.description !== "Consumer high-water journal checkpoint" || typeof error.path !== "string" || + !Buffer.isBuffer(error.bytes) || error.bytes.length < 1 || error.bytes.length > options.metadataMaxBytes || + (linkRetiredBeforeRead && !isExactLinkRetiredBeforeReadEvidence(error)) || + (linkRetiredDuringRead && !isExactLinkRetiredDuringReadEvidence(error)) || + (unlinkedDuringRead && !isExactUnlinkedDuringReadEvidence(error)) + ) return null; + const expectedSha256 = anchors.get(error.path); + if ( + expectedSha256 === undefined || error.expectedSha256 !== expectedSha256 || + digest(error.bytes) !== expectedSha256 || error.sha256 !== expectedSha256 || + dirname(error.path) !== context.journalDirectory + ) return null; + const name = basename(error.path); + const match = checkpointPattern.exec(name); + if (!match || error.path !== join(context.journalDirectory, name)) return null; + let checkpoint; + try { + checkpoint = validateCheckpoint(JSON.parse(error.bytes.toString("utf8")), options.stateMaxBytes).value; + } catch { + return null; + } + if ( + !metadataBytes(checkpoint).equals(error.bytes) || checkpointName(checkpoint) !== name || + checkpoint.epoch !== Number(match[1]) || + !isAuthenticatedCheckpointAnchor(context, error.path, checkpoint, expectedSha256, additionalAnchor) + ) return null; + if (unlinkedDuringRead) { + const hasLaterCheckpoint = rootNames.some((candidate) => { + const candidateEpoch = canonicalCheckpointNameEpoch(candidate); + return candidateEpoch !== null && candidateEpoch > checkpoint.epoch; + }); + if (!hasLaterCheckpoint) return null; + } + const linkRetirementStat = linkRetiredBeforeRead + ? error.statTransition.openedHandle + : linkRetiredDuringRead ? error.statTransition.finalPathEntry : null; + return { + checkpoint, + linkRetiredBeforeRead: linkRetiredDuringOrBeforeRead, + linkRetirementStat, + unlinkedDuringRead, + }; +} + +function sameRetiredLinkStat(left, right) { + return left !== null && right !== null && + left.dev === right.dev && left.ino === right.ino && left.size === right.size && + left.mtimeMs === right.mtimeMs && left.ctimeMs === right.ctimeMs && left.nlink === right.nlink; +} + +function checkpointProofOptions(entry, options, invalidRoot) { + if (entry.checkpointStat === null) return options; + let initialPathStat = true; + return { + ...options, + afterInitialPathStat: async ({ path, stat }) => { + await options.afterInitialPathStat?.({ path, stat }); + if (path === entry.path && initialPathStat) { + initialPathStat = false; + if (!sameRetiredLinkStat(stat, entry.checkpointStat)) throw invalidRoot(); + } + }, + }; +} + +async function authenticateStableChangedRoot(scan, context, options, anchors, target, invalidRoot) { + const initialNames = new Set(scan.rootNames); + if (initialNames.size !== scan.rootNames.length) throw invalidRoot(); + const targetCheckpointName = basename(target.path); + const targetEpochName = epochName(target.checkpoint); + const proofAnchors = new Map(anchors); + proofAnchors.set(target.path, target.digest); + const optionalCheckpointNames = new Set([ + ...scan.checkpointEntries + .filter((entry) => entry.path !== target.path && entry.checkpoint.epoch < target.checkpoint.epoch) + .map((entry) => entry.name), + ...scan.vanishedRetainedCheckpointNames, + ]); + const optionalEpochNames = new Set( + scan.epochEntries + .filter((entry) => entry.epoch < target.checkpoint.epoch) + .map((entry) => entry.name), + ); + const optionalTemporaryNames = new Set( + scan.temporaries + .filter((temporary) => dirname(temporary.path) === context.journalDirectory) + .map((temporary) => basename(temporary.path)), + ); + const optionalNames = new Set([ + ...optionalCheckpointNames, + ...optionalEpochNames, + ...optionalTemporaryNames, + ]); + const requiredNames = new Set([TEMPORARY_DIRECTORY_NAME, targetCheckpointName, targetEpochName]); + const removedBeforeProof = new Set(scan.removedCheckpointEntries.map((entry) => entry.name)); + const proofNamesArray = await options.readDirectory(context.journalDirectory); + const proofNames = new Set(proofNamesArray); + if ( + proofNamesArray.length > MAX_JOURNAL_ROOT_ENTRIES + MAX_TEMPORARY_ENTRIES || + proofNames.size !== proofNamesArray.length || + [...proofNames].some((name) => !initialNames.has(name)) || + [...removedBeforeProof].some((name) => proofNames.has(name)) || + scan.vanishedRetainedCheckpointNames.some((name) => proofNames.has(name)) || + [...initialNames].some((name) => !optionalNames.has(name) && !proofNames.has(name)) || + [...requiredNames].some((name) => !proofNames.has(name)) + ) throw invalidRoot(); + const removedDuringProof = new Set(); + let targetAuthenticated = false; + for (const entry of scan.checkpointEntries) { + if (!proofNames.has(entry.name)) continue; + const optional = optionalCheckpointNames.has(entry.name); + await options.hooks?.beforeStableCheckpointProofRead?.({ + name: entry.name, + path: entry.path, + target: entry.path === target.path, + }); + const expectedSha256 = proofAnchors.get(entry.path) ?? null; + let checkpoint; + let changedRead = null; + try { + checkpoint = await readExactMetadata( + entry.path, + options.metadataMaxBytes, + (value) => validateCheckpoint(value, options.stateMaxBytes).value, + "Consumer high-water journal checkpoint", + checkpointProofOptions(entry, options, invalidRoot), + undefined, + expectedSha256, + ); + } catch (error) { + changedRead = authenticatedChangedCheckpointRead( + error, + context, + options, + proofAnchors, + proofNamesArray, + target, + ); + if (changedRead === null) throw error; + checkpoint = changedRead.checkpoint; + } + if (checkpoint === null) { + if (!optional) throw invalidRoot(); + removedDuringProof.add(entry.name); + continue; + } + if ( + !metadataBytes(checkpoint).equals(metadataBytes(entry.checkpoint)) || + (expectedSha256 !== null && !isAuthenticatedCheckpointAnchor( + context, + entry.path, + checkpoint, + expectedSha256, + target, + )) + ) throw invalidRoot(); + if (changedRead?.unlinkedDuringRead) { + if (!optional) throw invalidRoot(); + removedDuringProof.add(entry.name); + continue; + } + if (entry.path === target.path) targetAuthenticated = true; + } + if (!targetAuthenticated) throw invalidRoot(); + await secureDirectory( + join(context.journalDirectory, TEMPORARY_DIRECTORY_NAME), + "Consumer high-water temporary directory", + options, + ); + const targetEpoch = scan.epochEntries.find((entry) => entry.name === targetEpochName); + if (targetEpoch === undefined) throw invalidRoot(); + await secureDirectory(targetEpoch.path, "Consumer high-water epoch directory", options); + for (const temporary of scan.temporaries) { + if (dirname(temporary.path) === context.journalDirectory && proofNames.has(basename(temporary.path))) { + if ((await inspectTemporary(temporary.path, options)) === null) throw invalidRoot(); + } + } + const finalNamesArray = await options.readDirectory(context.journalDirectory); + const finalNames = new Set(finalNamesArray); + if ( + finalNamesArray.length > MAX_JOURNAL_ROOT_ENTRIES + MAX_TEMPORARY_ENTRIES || + finalNames.size !== finalNamesArray.length || + [...finalNames].some((name) => !proofNames.has(name)) || + [...proofNames].some((name) => !optionalNames.has(name) && !finalNames.has(name)) || + [...removedBeforeProof].some((name) => finalNames.has(name)) || + [...removedDuringProof].some((name) => finalNames.has(name)) || + [...requiredNames].some((name) => !finalNames.has(name)) + ) throw invalidRoot(); +} + +async function inProgressDirectoryStats(context, nextEpochPath, options, invalidRoot) { + const [temporaryDirectory, currentEpoch, nextEpoch] = await Promise.all([ + options.lstatEntry(context.temporaryDirectory), + options.lstatEntry(context.epochDirectory), + options.lstatEntry(nextEpochPath), + ]); + if ( + !temporaryDirectory.isDirectory() || temporaryDirectory.isSymbolicLink?.() || + !currentEpoch.isDirectory() || currentEpoch.isSymbolicLink?.() || + !nextEpoch.isDirectory() || nextEpoch.isSymbolicLink?.() + ) throw invalidRoot(); + return { temporaryDirectory, currentEpoch, nextEpoch }; +} + +async function authenticateLinkRetiredInProgressRoot( + scan, + context, + options, + anchors, + currentCheckpoint, + nextEpochPath, + kind, + invalidRoot, +) { + if (scan.linkRetiredCheckpointEntries.length === 0) return; + if ( + currentCheckpoint === undefined || scan.linkRetiredCheckpointEntries.length !== 1 || + scan.linkRetiredCheckpointEntries[0] !== currentCheckpoint || currentCheckpoint.path !== context.checkpointPath || + currentCheckpoint.linkRetirementStat === null + ) throw invalidRoot(); + const permittedNames = new Set([ + TEMPORARY_DIRECTORY_NAME, + basename(context.checkpointPath), + basename(context.epochDirectory), + basename(nextEpochPath), + ]); + if ( + scan.rootNames.length !== permittedNames.size || + scan.rootNames.some((name) => !permittedNames.has(name)) + ) throw invalidRoot(); + const beforeDirectories = await inProgressDirectoryStats(context, nextEpochPath, options, invalidRoot); + await options.hooks?.beforeInProgressStableRootProof?.({ kind }); + await authenticateStableChangedRoot(scan, context, options, anchors, currentCheckpoint, invalidRoot); + const afterDirectories = await inProgressDirectoryStats(context, nextEpochPath, options, invalidRoot); + if ( + !sameRetiredLinkStat(beforeDirectories.temporaryDirectory, afterDirectories.temporaryDirectory) || + !sameRetiredLinkStat(beforeDirectories.currentEpoch, afterDirectories.currentEpoch) || + !sameRetiredLinkStat(beforeDirectories.nextEpoch, afterDirectories.nextEpoch) || + (await options.readDirectory(nextEpochPath)).length !== 0 + ) throw invalidRoot(); +} + +async function authenticateChangedRoot( + context, + options, + inProgressCheckpoint = null, + allowInProgressDiscovery = false, +) { + const anchors = contextCheckpointAnchors(context); + const scan = await scanJournalRoot(context.statePath, context.journalDirectory, options, { + checkpointAnchors: anchors, + checkpointContext: context, + }); + const invalidRoot = () => new Error( + "Consumer high-water journal root changed without one exact current or immediate-successor authority.", + ); + const currentCheckpoint = scan.checkpointEntries.find((entry) => entry.path === context.checkpointPath); + if (currentCheckpoint && currentCheckpoint.digest !== context.checkpointDigest) throw invalidRoot(); + + if (inProgressCheckpoint !== null && scan.checkpointEntries.length === 1) { + const checkpoint = validateCheckpoint(inProgressCheckpoint, options.stateMaxBytes).value; + const nextEpochPath = join(context.journalDirectory, epochName(checkpoint)); + await secureDirectory(context.epochDirectory, "Consumer high-water epoch directory", options); + await secureDirectory(nextEpochPath, "Consumer high-water epoch directory", options); + if ( + !isImmediateSuccessorCheckpoint(context, checkpoint) || + scan.checkpointEntries.length !== 1 || scan.head?.path !== context.checkpointPath || scan.missingHeadEpoch || + scan.epochEntries.length !== 2 || + scan.epochEntries.some((entry) => ![context.epochDirectory, nextEpochPath].includes(entry.path)) || + !scan.epochEntries.some((entry) => entry.path === nextEpochPath) || + (await options.readDirectory(nextEpochPath)).length !== 0 + ) throw invalidRoot(); + if ((await options.readDirectory(nextEpochPath)).length !== 0) throw invalidRoot(); + await authenticateLinkRetiredInProgressRoot( + scan, + context, + options, + anchors, + currentCheckpoint, + nextEpochPath, + "known", + invalidRoot, + ); + return false; + } + + const discoveredNextEpoch = scan.epochEntries.find((entry) => entry.path !== context.epochDirectory); + if ( + allowInProgressDiscovery && inProgressCheckpoint === null && + scan.checkpointEntries.length === 1 && scan.head?.path === context.checkpointPath && !scan.missingHeadEpoch && + scan.epochEntries.length === 2 && discoveredNextEpoch?.epoch === context.checkpoint.epoch + 1 && + (await options.readDirectory(discoveredNextEpoch.path)).length === 0 + ) { + await secureDirectory(context.epochDirectory, "Consumer high-water epoch directory", options); + await secureDirectory(discoveredNextEpoch.path, "Consumer high-water epoch directory", options); + if ((await options.readDirectory(discoveredNextEpoch.path)).length !== 0) throw invalidRoot(); + await authenticateLinkRetiredInProgressRoot( + scan, + context, + options, + anchors, + currentCheckpoint, + discoveredNextEpoch.path, + "discovered", + invalidRoot, + ); + return discoveredNextEpoch.path; + } + + const retainedCheckpoint = scan.checkpointEntries.find((entry) => entry.path !== context.checkpointPath); + const retiredEpochPath = context.checkpoint.retiredEpochDirectory === null + ? null + : join(context.journalDirectory, context.checkpoint.retiredEpochDirectory); + if ( + currentCheckpoint && scan.head?.path === context.checkpointPath && !scan.missingHeadEpoch && + scan.checkpointEntries.length <= 2 && scan.epochEntries.length <= 2 && + (!retainedCheckpoint || ( + retainedCheckpoint.digest === context.checkpoint.previousCheckpointSha256 && + epochName(retainedCheckpoint.checkpoint) === context.checkpoint.retiredEpochDirectory + )) && + scan.epochEntries.every((entry) => [context.epochDirectory, retiredEpochPath].includes(entry.path)) + ) { + if ( + scan.removedCheckpointEntries.length > 0 || scan.linkRetiredCheckpointEntries.length > 0 || + scan.vanishedRetainedCheckpointNames.length > 0 + ) { + await authenticateStableChangedRoot(scan, context, options, anchors, currentCheckpoint, invalidRoot); + } else { + await secureDirectory(context.epochDirectory, "Consumer high-water epoch directory", options); + } + return false; + } + + const successor = scan.head; + const expectedCheckpointPath = successor + ? join(context.journalDirectory, checkpointName(successor.checkpoint)) + : null; + const expectedEpochPath = successor + ? join(context.journalDirectory, epochName(successor.checkpoint)) + : null; + if ( + !successor || scan.missingHeadEpoch || successor.path !== expectedCheckpointPath || + !isImmediateSuccessorCheckpoint(context, successor.checkpoint) || + scan.checkpointEntries.length < 1 || scan.checkpointEntries.length > 2 || + scan.epochEntries.length < 1 || scan.epochEntries.length > 2 || + scan.checkpointEntries.some((entry) => ![context.checkpointPath, expectedCheckpointPath].includes(entry.path)) || + scan.epochEntries.some((entry) => ![context.epochDirectory, expectedEpochPath].includes(entry.path)) || + !scan.epochEntries.some((entry) => entry.path === expectedEpochPath) + ) throw invalidRoot(); + await authenticateStableChangedRoot(scan, context, options, anchors, successor, invalidRoot); + return true; +} + +async function revalidateAuthority( + context, + operation, + options, + inProgressCheckpoint = options.inProgressCheckpoint ?? null, + allowInProgressDiscovery = false, +) { + await options.hooks?.beforePathOperation?.({ + operation, + statePath: context.statePath, + lockDirectory: context.journalDirectory, + transactionDirectory: context.epochDirectory, + inProgressCheckpoint: inProgressCheckpoint === null ? null : structuredClone(inProgressCheckpoint), + }); + await ensureDurableConsumerStateDirectory(dirname(context.statePath), { + ...options.directoryOperations, + create: false, + }); + await secureDirectory(dirname(context.statePath), "Consumer high-water state directory", options); + await secureDirectory(context.journalDirectory, "Consumer high-water journal directory", options); + await secureDirectory(context.temporaryDirectory, "Consumer high-water temporary directory", options); + let oldEpochError = null; + try { + await secureDirectory(context.epochDirectory, "Consumer high-water epoch directory", options); + } catch (error) { + if (error?.code !== "ENOENT") throw error; + oldEpochError = error; + } + const entries = await options.readDirectory(context.journalDirectory); + if (entries.length > MAX_JOURNAL_ROOT_ENTRIES + MAX_TEMPORARY_ENTRIES) { + throw new Error("Consumer high-water journal root exceeds its safe allocation bound."); + } + const expectedRootNames = new Set([ + TEMPORARY_DIRECTORY_NAME, + basename(context.checkpointPath), + basename(context.epochDirectory), + ]); + let changedRoot = false; + if (entries.some((name) => !expectedRootNames.has(name))) { + changedRoot = await authenticateChangedRoot( + context, + options, + inProgressCheckpoint, + allowInProgressDiscovery, + ); + if (changedRoot === true) throw new ConsumerEpochAdvancedError(); + } + if (oldEpochError) throw oldEpochError; + const current = await readExactMetadata( + context.checkpointPath, + options.metadataMaxBytes, + (value) => validateCheckpoint(value, options.stateMaxBytes).value, + "Consumer high-water journal checkpoint", + options, + ); + if (digest(metadataBytes(current)) !== context.checkpointDigest) { + throw new Error("Consumer high-water journal checkpoint changed and fenced a paused writer."); + } + return typeof changedRoot === "string" ? changedRoot : null; +} + +async function publishImmutable({ + path, + bytes, + directory, + kind, + context, + writer, + options, + revalidate = true, + beforeLink, + inProgressCheckpoint = null, +}) { + if (revalidate) await revalidateAuthority(context, kind, options, inProgressCheckpoint); + const temporary = join(context.temporaryDirectory, temporaryName(path, kind, writer, context)); + let handle; + let linked = false; + try { + handle = await options.openFile(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); + await handle.chmod?.(0o600); + await handle.writeFile(bytes); + await handle.sync(); + await handle.close(); + handle = undefined; + await options.hooks?.afterFileSync?.({ kind, path, temporary }); + await beforeLink?.(); + if (revalidate) await revalidateAuthority(context, `${kind}-link`, options, inProgressCheckpoint); + try { + await options.linkFile(temporary, path); + linked = true; + } catch (error) { + if (error?.code !== "EEXIST") throw error; + } + if (linked) await options.hooks?.afterMetadataLink?.({ kind, path }); + await options.syncDirectory(directory); + await options.hooks?.afterMetadataDirectorySync?.({ kind, path, linked }); + return linked; + } finally { + if (handle !== undefined) await handle.close(); + await options.removeFile(temporary, { force: true }); + await options.syncDirectory(context.temporaryDirectory); + } +} + +async function publishMetadata(path, value, kind, context, writer, options, inProgressCheckpoint = null) { + const created = await publishImmutable({ + path, + bytes: metadataBytes(value), + directory: dirname(path), + kind, + context, + writer, + options, + inProgressCheckpoint, + }); + if (created) return { value, created: true }; + await revalidateAuthority(context, `${kind}-existing`, options, inProgressCheckpoint); + const existing = await readExactMetadata( + path, + options.metadataMaxBytes, + (candidate) => candidate, + "Consumer high-water lock metadata", + options, + ); + return { value: existing, created: false }; +} + +function genesisCheckpoint(statePath) { + const epochId = deterministicUuid(`pylon-consumer-journal:${statePath}`); + return { + schemaVersion: CHECKPOINT_SCHEMA_VERSION, + epoch: 1, + epochId, + previousCheckpointSha256: GENESIS_DIGEST, + previousTipSha256: GENESIS_DIGEST, + historySha256: digest(Buffer.from(`pylon-consumer-history:${digest(Buffer.from(statePath))}`)), + anchorDigest: GENESIS_DIGEST, + anchorBase64: null, + retiredEpochDirectory: null, + sourceAuthoritySha256: GENESIS_DIGEST, + sourceAuthorityTipDigest: GENESIS_DIGEST, + sourceAuthorityTipBase64: null, + }; +} + +async function scanJournalRoot( + statePath, + journalDirectory, + options, + { checkpointAnchors = null, checkpointContext = null } = {}, +) { + await secureDirectory(journalDirectory, "Consumer high-water journal directory", options); + await options.syncDirectory(journalDirectory); + const names = await options.readDirectory(journalDirectory); + if (names.length > MAX_JOURNAL_ROOT_ENTRIES + MAX_TEMPORARY_ENTRIES) { + throw new Error("Consumer high-water journal root exceeds its safe allocation bound."); + } + const checkpointEntries = []; + const removedCheckpointEntries = []; + const linkRetiredCheckpointEntries = []; + const vanishedRetainedCheckpointNames = []; + const epochEntries = []; + const temporaries = []; + let temporaryDirectorySeen = false; + for (const name of names) { + const path = join(journalDirectory, name); + if (name === TEMPORARY_DIRECTORY_NAME) { + if (temporaryDirectorySeen) throw new Error("Consumer high-water temporary namespace is duplicated."); + temporaryDirectorySeen = true; + const entry = await options.lstatEntry(path); + if (!entry.isDirectory() || entry.isSymbolicLink?.()) { + throw new Error("Consumer high-water temporary namespace must be one real directory."); + } + await secureDirectory(path, "Consumer high-water temporary directory", options); + const temporaryNames = await options.readDirectory(path); + if (temporaryNames.length > MAX_TEMPORARY_ENTRIES) { + throw new Error("Consumer high-water temporary namespace exceeds its safe allocation bound."); + } + for (const temporaryName of temporaryNames) { + const temporary = await inspectTemporary(join(path, temporaryName), options); + if (temporary) temporaries.push(temporary); + } + continue; + } + const checkpointMatch = checkpointPattern.exec(name); + if (checkpointMatch) { + const expectedSha256 = checkpointAnchors?.get(path) ?? null; + let checkpoint; + let removedDuringRead = false; + let linkRetiredBeforeRead = false; + let linkRetirementStat = null; + try { + checkpoint = await readExactMetadata( + path, + options.metadataMaxBytes, + (value) => validateCheckpoint(value, options.stateMaxBytes).value, + "Consumer high-water journal checkpoint", + options, + undefined, + expectedSha256, + ); + } catch (error) { + const authenticated = checkpointContext === null || checkpointAnchors === null + ? null + : authenticatedChangedCheckpointRead(error, checkpointContext, options, checkpointAnchors, names); + if (authenticated === null) throw error; + checkpoint = authenticated.checkpoint; + linkRetiredBeforeRead = authenticated.linkRetiredBeforeRead; + linkRetirementStat = authenticated.linkRetirementStat; + removedDuringRead = !linkRetiredBeforeRead; + } + if (checkpoint === null) { + if (checkpointContext === null || checkpointAnchors === null) { + throw new Error("Consumer high-water journal lost its current checkpoint during an authenticated scan."); + } + if (isProvisionallyRemovedContextCurrentCheckpoint(path, checkpointContext, checkpointAnchors, names)) { + checkpoint = checkpointContext.checkpoint; + removedDuringRead = true; + } else { + if (!isVanishedRetainedCheckpoint(path, checkpointContext, checkpointAnchors, names)) { + throw new Error("Consumer high-water journal lost its current checkpoint during an authenticated scan."); + } + vanishedRetainedCheckpointNames.push(name); + continue; + } + } + let checkpointStat = linkRetirementStat; + if (!removedDuringRead && checkpointStat === null) { + await options.hooks?.beforeCheckpointIdentityStat?.({ name, path }); + try { + checkpointStat = await options.lstatEntry(path); + } catch (error) { + const allowedRemoval = checkpointContext !== null && checkpointAnchors !== null && ( + isProvisionallyRemovedContextCurrentCheckpoint(path, checkpointContext, checkpointAnchors, names) || + isVanishedRetainedCheckpoint(path, checkpointContext, checkpointAnchors, names) + ); + if (error?.code !== "ENOENT" || !allowedRemoval) throw error; + removedDuringRead = true; + } + if ( + checkpointStat !== null && + (checkpointStat.isSymbolicLink?.() || !checkpointStat.isFile()) + ) throw new Error("Consumer high-water journal checkpoint must remain one regular non-symlink file."); + } + if (checkpointName(checkpoint) !== name || checkpoint.epoch !== Number(checkpointMatch[1])) { + throw new Error("Consumer high-water journal checkpoint name is malformed."); + } + const entry = { + name, + path, + checkpoint, + digest: digest(metadataBytes(checkpoint)), + removedDuringRead, + linkRetiredBeforeRead, + linkRetirementStat, + checkpointStat, + }; + checkpointEntries.push(entry); + if (removedDuringRead) removedCheckpointEntries.push(entry); + if (linkRetiredBeforeRead) linkRetiredCheckpointEntries.push(entry); + continue; + } + const epochMatch = epochPattern.exec(name); + if (epochMatch) { + epochEntries.push({ name, path, epoch: Number(epochMatch[1]), epochId: epochMatch[2] }); + continue; + } + if (name.startsWith(".")) { + const temporary = await inspectTemporary(path, options); + if (temporary?.kind !== "checkpoint") { + throw new Error("Consumer high-water journal root contains an unexpected owned temporary."); + } + if (temporary) temporaries.push(temporary); + continue; + } + throw new Error("Consumer high-water journal root contains an unexpected entry."); + } + if (!temporaryDirectorySeen) throw new Error("Consumer high-water journal lacks its exact temporary namespace."); + const checkpointNameCount = names.filter((name) => checkpointPattern.test(name)).length; + const authoritativeEntries = checkpointNameCount + epochEntries.length + 1; + if (authoritativeEntries > MAX_JOURNAL_ROOT_ENTRIES) { + throw new Error("Consumer high-water journal root exceeds its safe entry bound."); + } + checkpointEntries.sort((left, right) => left.checkpoint.epoch - right.checkpoint.epoch); + epochEntries.sort((left, right) => left.epoch - right.epoch); + if (checkpointNameCount > 2 || epochEntries.length > 2) { + throw new Error("Consumer high-water journal root contains unbounded checkpoint metadata."); + } + for (let index = 1; index < checkpointEntries.length; index += 1) { + if (checkpointEntries[index - 1].checkpoint.epoch === checkpointEntries[index].checkpoint.epoch) { + throw new Error("Consumer high-water journal contains competing checkpoints for one parent epoch."); + } + if (checkpointEntries[index - 1].checkpoint.epoch + 1 !== checkpointEntries[index].checkpoint.epoch) { + throw new Error("Consumer high-water journal checkpoints are not contiguous."); + } + } + for (let index = 1; index < epochEntries.length; index += 1) { + if (epochEntries[index - 1].epoch === epochEntries[index].epoch) { + throw new Error("Consumer high-water journal contains competing epoch directories for one parent epoch."); + } + } + const head = checkpointEntries.at(-1) ?? null; + if (head) { + const previous = checkpointEntries.at(-2); + if (previous && ( + head.checkpoint.previousCheckpointSha256 !== previous.digest || + head.checkpoint.retiredEpochDirectory !== epochName(previous.checkpoint) || + head.checkpoint.sourceAuthoritySha256 !== previous.checkpoint.sourceAuthoritySha256 || + head.checkpoint.sourceAuthorityTipDigest !== previous.checkpoint.sourceAuthorityTipDigest || + head.checkpoint.sourceAuthorityTipBase64 !== previous.checkpoint.sourceAuthorityTipBase64 || + head.checkpoint.historySha256 !== digest(Buffer.from( + `${previous.checkpoint.historySha256}:${previous.digest}:${head.checkpoint.anchorDigest}`, + )) + )) throw new Error("Consumer high-water journal checkpoint does not anchor its exact predecessor."); + } + const missingHeadEpoch = head ? !epochEntries.some((entry) => entry.name === epochName(head.checkpoint)) : false; + if (checkpointContext === null) { + for (const entry of epochEntries) { + const pathEntry = await options.lstatEntry(entry.path); + if (!pathEntry.isDirectory() || pathEntry.isSymbolicLink?.()) { + throw new Error("Consumer high-water epoch entry must be one real directory."); + } + await secureDirectory(entry.path, "Consumer high-water epoch directory", options); + } + } + return { + checkpointEntries, + removedCheckpointEntries, + linkRetiredCheckpointEntries, + vanishedRetainedCheckpointNames, + epochEntries, + temporaries, + head, + missingHeadEpoch, + rootNames: names, + }; +} + +function classifyContextCheckpointAuthority(scan, context, anchors, invalidRoot) { + const current = scan.checkpointEntries.find((entry) => entry.path === context.checkpointPath); + if (current && !isContextAnchoredCheckpoint(context, current.path, current.checkpoint, current.digest)) { + throw invalidRoot(); + } + const successors = scan.checkpointEntries.filter((entry) => entry.checkpoint.epoch > context.checkpoint.epoch); + if (successors.length === 0) { + if ( + !current || scan.head?.path !== current.path || + scan.checkpointEntries.some((entry) => { + const expectedSha256 = anchors.get(entry.path); + return expectedSha256 === undefined || + !isContextAnchoredCheckpoint(context, entry.path, entry.checkpoint, expectedSha256); + }) + ) throw invalidRoot(); + return { kind: "current", entry: current }; + } + const successor = successors[0]; + const successorEpochPath = join(context.journalDirectory, epochName(successor.checkpoint)); + if ( + successors.length !== 1 || scan.head?.path !== successor.path || + !isImmediateSuccessorCheckpoint(context, successor.checkpoint) || + scan.checkpointEntries.some((entry) => ![context.checkpointPath, successor.path].includes(entry.path)) || + scan.epochEntries.some((entry) => ![context.epochDirectory, successorEpochPath].includes(entry.path)) || + !scan.epochEntries.some((entry) => entry.path === successorEpochPath) + ) throw invalidRoot(); + return { kind: "successor", entry: successor }; +} + +async function scanAuthenticatedContextRoot(context, options) { + const anchors = contextCheckpointAnchors(context); + const scan = await scanJournalRoot(context.statePath, context.journalDirectory, options, { + checkpointAnchors: anchors, + checkpointContext: context, + }); + const invalidRoot = () => new Error( + "Consumer high-water journal root has neither its byte-exact current checkpoint nor one exact immediate successor.", + ); + const authority = classifyContextCheckpointAuthority(scan, context, anchors, invalidRoot); + await authenticateStableChangedRoot(scan, context, options, anchors, authority.entry, invalidRoot); + return { scan, authority }; +} + +async function initializeJournal( + statePath, + journalDirectory, + options, + bootstrapCheckpoint = genesisCheckpoint(statePath), + beforeCheckpointLink, +) { + let scan = await scanJournalRoot(statePath, journalDirectory, options); + if (scan.head) { + if (!scan.missingHeadEpoch) return scan; + if ( + scan.head.checkpoint.epoch !== 1 || !metadataBytes(scan.head.checkpoint).equals(metadataBytes(bootstrapCheckpoint)) || + scan.epochEntries.length !== 0 + ) throw new Error("Consumer high-water journal checkpoint lacks its exact epoch directory."); + await ensureDirectory( + join(journalDirectory, epochName(scan.head.checkpoint)), + "Consumer high-water epoch directory", + options, + ); + return scanJournalRoot(statePath, journalDirectory, options); + } + if (scan.epochEntries.length > 0) throw new Error("Consumer high-water journal contains an orphan epoch directory."); + const checkpoint = bootstrapCheckpoint; + const bootstrap = { generation: 0, token: checkpoint.epochId }; + const bootstrapContext = { + statePath, + journalDirectory, + checkpoint, + checkpointPath: join(journalDirectory, checkpointName(checkpoint)), + checkpointDigest: digest(metadataBytes(checkpoint)), + epochDirectory: join(journalDirectory, epochName(checkpoint)), + temporaryDirectory: join(journalDirectory, TEMPORARY_DIRECTORY_NAME), + }; + await publishImmutable({ + path: bootstrapContext.checkpointPath, + bytes: metadataBytes(checkpoint), + directory: journalDirectory, + kind: "checkpoint", + context: bootstrapContext, + writer: bootstrap, + options, + revalidate: false, + beforeLink: beforeCheckpointLink, + }); + await ensureDirectory(bootstrapContext.epochDirectory, "Consumer high-water epoch directory", options); + scan = await scanJournalRoot(statePath, journalDirectory, options); + if (!scan.head) throw new Error("Consumer high-water journal initialization did not publish a checkpoint."); + return scan; +} + +function contextFromHead(statePath, guardPath, journalDirectory, head) { + return { + statePath, + guardPath, + journalDirectory, + checkpoint: head.checkpoint, + checkpointPath: head.path, + checkpointDigest: head.digest, + epochDirectory: join(journalDirectory, epochName(head.checkpoint)), + temporaryDirectory: join(journalDirectory, TEMPORARY_DIRECTORY_NAME), + }; +} + +async function readProjection(context, operation, options) { + await revalidateAuthority(context, operation, options); + const bytes = await readSecureFile( + context.statePath, + options.stateMaxBytes, + "Consumer high-water state", + options, + 0, + options.hooks?.projectionRead, + ); + if (bytes === null) return { exists: false, bytes: null, sha256: null, malformed: false }; + if (bytes.length < 1) return { exists: true, bytes: null, sha256: null, malformed: true }; + return { exists: true, bytes, sha256: digest(bytes), malformed: false }; +} + +async function walkTransactions(context, options) { + await revalidateAuthority(context, "walk-transactions", options); + await options.syncDirectory(context.epochDirectory); + const entries = await options.readDirectory(context.epochDirectory); + if (entries.length > options.maxJournalEntries + MAX_TEMPORARY_ENTRIES) { + throw new Error("Consumer high-water epoch exceeds its safe allocation bound."); + } + const named = new Map(); + for (const name of entries) { + const match = transitionPattern.exec(name); + if (match) { + if (named.has(match[1])) throw new Error("Consumer high-water journal contains a duplicate transition."); + named.set(match[1], name); + if (named.size > options.maxTransactionDepth) { + throw new Error("Consumer high-water transaction chain exceeds its safe entry bound."); + } + } + } + const visited = new Set(); + let tipDigest = context.checkpoint.anchorDigest; + let tipBytes = validateCheckpoint(context.checkpoint, options.stateMaxBytes).anchorBytes; + const budget = { bytes: 0 }; + for (let depth = 0; named.has(tipDigest); depth += 1) { + if (depth >= options.maxTransactionDepth || visited.has(tipDigest)) { + throw new Error("Consumer high-water transaction chain is cyclic or exceeds its safe bound."); + } + visited.add(tipDigest); + const path = transitionPath(context, tipDigest); + await revalidateAuthority(context, "read-transition", options); + const value = await readExactMetadata( + path, + options.metadataMaxBytes, + (candidate) => validateTransaction(candidate, tipDigest, options.stateMaxBytes).value, + "Consumer high-water transaction", + options, + budget, + ); + const validated = validateTransaction(value, tipDigest, options.stateMaxBytes); + tipDigest = value.candidateDigest; + tipBytes = validated.candidateBytes; + } + if (visited.size !== named.size) throw new Error("Consumer high-water transaction chain contains an unreachable transition."); + return { tipDigest, tipBytes, length: visited.size }; +} + +function isProjectionReplacementTransient(error) { + return error?.code === "ENOENT" || error?.message === "Consumer high-water state changed while it was read."; +} + +function isCommitHelperReplacementTransient(error) { + return isProjectionReplacementTransient(error) || error instanceof ConsumerEpochAdvancedError; +} + +async function repairProjection(context, initialTip, options, writer = options.activeWriter) { + let tip = initialTip; + for (let attempt = 0; attempt < PROJECTION_RETRY_LIMIT; attempt += 1) { + if (tip.tipBytes === null) return tip; + try { + const projection = await readProjection(context, "projection-read", options); + if (projection.sha256 !== tip.tipDigest) { + await options.hooks?.beforeProjectionWrite?.({ tipDigest: tip.tipDigest }); + await revalidateAuthority(context, "projection-write", options); + const temporary = join(context.temporaryDirectory, temporaryName(context.statePath, "projection", writer, context)); + let handle; + try { + handle = await options.openFile(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); + await handle.chmod?.(0o600); + await handle.writeFile(tip.tipBytes); + await handle.sync(); + await handle.close(); + handle = undefined; + await options.hooks?.afterProjectionFileSync?.({ tipDigest: tip.tipDigest, temporary }); + await revalidateAuthority(context, "projection-rename", options); + await options.renameFile(temporary, context.statePath); + await options.hooks?.afterProjectionRename?.({ tipDigest: tip.tipDigest }); + await options.syncDirectory(context.temporaryDirectory); + await options.syncDirectory(dirname(context.statePath)); + await options.hooks?.afterProjectionDirectorySync?.({ tipDigest: tip.tipDigest }); + } finally { + if (handle !== undefined) await handle.close(); + await options.removeFile(temporary, { force: true }); + await options.syncDirectory(context.temporaryDirectory); + } + } + } catch (error) { + if (!isProjectionReplacementTransient(error)) throw error; + await revalidateAuthority(context, "projection-retry-authentication", options); + tip = await walkTransactions(context, options); + continue; + } + const latest = await walkTransactions(context, options); + if (latest.tipDigest === tip.tipDigest) return latest; + tip = latest; + } + throw new Error("Consumer high-water projection could not catch up with its immutable transaction tip."); +} + +async function publishTransition(context, transaction, claim, options) { + validateTransaction(transaction, transaction.baseDigest, options.stateMaxBytes); + const path = transitionPath(context, transaction.baseDigest); + const result = await publishMetadata(path, transaction, "transition", context, claim, options); + const existing = validateTransaction(result.value, transaction.baseDigest, options.stateMaxBytes).value; + if (!metadataBytes(existing).equals(metadataBytes(transaction))) { + throw new Error("Consumer high-water transaction lost its immutable base-digest compare-and-set."); + } +} + +async function scanEpoch(context, options) { + const discoveredNextEpoch = await revalidateAuthority( + context, + "scan-claims", + options, + options.inProgressCheckpoint ?? null, + true, + ); + await options.syncDirectory(context.epochDirectory); + const names = await options.readDirectory(context.epochDirectory); + if (names.length > options.maxJournalEntries + MAX_TEMPORARY_ENTRIES) { + throw new Error("Consumer high-water epoch exceeds its safe allocation bound."); + } + const claimContentNames = new Map(); + const claimIndexNames = new Map(); + const legacyClaimNames = new Map(); + const heartbeatNames = new Map(); + const terminalNames = new Map(); + const appliedNames = new Map(); + const temporaries = []; + let authoritativeEntryCount = 0; + for (const name of names) { + let match; + if ((match = claimPattern.exec(name))) { + const generation = Number(match[1]); + const contents = claimContentNames.get(generation) ?? new Map(); + contents.set(match[2], name); + claimContentNames.set(generation, contents); + authoritativeEntryCount += 1; + } else if ((match = claimIndexPattern.exec(name))) { + const generation = Number(match[1]); + if (claimIndexNames.has(generation)) throw new Error("Consumer high-water lock contains a duplicate claim index."); + claimIndexNames.set(generation, name); + authoritativeEntryCount += 1; + } else if ((match = undigestedClaimPattern.exec(name))) { + const generation = Number(match[1]); + if (legacyClaimNames.has(generation)) throw new Error("Consumer high-water lock contains a duplicate legacy claim."); + legacyClaimNames.set(generation, name); + authoritativeEntryCount += 1; + } else if ((match = heartbeatPattern.exec(name))) { + heartbeatNames.set(`${Number(match[1])}:${match[2]}`, name); + authoritativeEntryCount += 1; + } else if ((match = terminalPattern.exec(name))) { + terminalNames.set(`${Number(match[1])}:${match[2]}`, name); + authoritativeEntryCount += 1; + } else if ((match = appliedPattern.exec(name))) { + appliedNames.set(`${Number(match[1])}:${match[2]}`, name); + authoritativeEntryCount += 1; + } else if (transitionPattern.test(name)) { + // Validated by the transaction walk before any state decision. + authoritativeEntryCount += 1; + } else if (name.startsWith(".")) { + const temporary = await inspectTemporary(join(context.epochDirectory, name), options); + if (temporary && ["checkpoint", "projection", "legacy-guard"].includes(temporary.kind)) { + throw new Error("Consumer high-water epoch contains an unexpected owned temporary."); + } + if (temporary) temporaries.push(temporary); + } else { + throw new Error("Consumer high-water epoch contains a malformed or unexpected entry."); + } + } + if (authoritativeEntryCount > options.maxJournalEntries) { + throw new Error("Consumer high-water epoch exceeds its safe entry bound."); + } + const budget = { bytes: 0 }; + const claims = []; + const byKey = new Map(); + const referencedClaimContents = new Set(); + const generations = new Set([...claimIndexNames.keys(), ...legacyClaimNames.keys()]); + for (const generation of [...generations].sort((left, right) => left - right)) { + if (claimIndexNames.has(generation) && legacyClaimNames.has(generation)) { + throw new Error("Consumer high-water lock contains competing indexed and legacy claims."); + } + let claim; + if (claimIndexNames.has(generation)) { + const index = await readExactMetadata( + join(context.epochDirectory, claimIndexNames.get(generation)), + options.metadataMaxBytes, + (value) => validateClaimIndex(value, generation), + "Consumer high-water claim index", + options, + budget, + ); + const name = claimContentNames.get(generation)?.get(index.claimSha256); + if (!name) throw new Error("Consumer high-water claim index lacks its exact digest-bound claim bytes."); + referencedClaimContents.add(name); + claim = await readExactMetadata( + join(context.epochDirectory, name), + options.metadataMaxBytes, + (value) => validateClaim(value, context, options.stateMaxBytes), + "Consumer high-water operation claim", + options, + budget, + index.claimSha256, + ); + if ( + claim.generation !== generation || digest(metadataBytes(claim)) !== index.claimSha256 || + name !== basename(claimPath(context, claim)) + ) throw new Error("Consumer high-water claim index differs from its exact canonical claim bytes."); + } else { + const name = legacyClaimNames.get(generation); + claim = await readExactMetadata( + join(context.epochDirectory, name), + options.metadataMaxBytes, + (value) => validateClaim(value, context, options.stateMaxBytes), + "Consumer high-water legacy operation claim", + options, + budget, + ); + if (claim.generation !== generation || name !== `claim-${generationName(generation)}.json`) { + throw new Error("Consumer high-water legacy claim name differs from its exact generation."); + } + } + claims.push(claim); + byKey.set(`${generation}:${claim.token}`, claim); + } + for (const [generation, contents] of [...claimContentNames].sort((left, right) => left[0] - right[0])) { + for (const [claimSha256, name] of [...contents].sort((left, right) => left[0].localeCompare(right[0]))) { + if (referencedClaimContents.has(name)) continue; + const claim = await readExactMetadata( + join(context.epochDirectory, name), + options.metadataMaxBytes, + (value) => validateClaim(value, context, options.stateMaxBytes), + "Consumer high-water unindexed claim content", + options, + budget, + ); + if ( + claim.generation !== generation || digest(metadataBytes(claim)) !== claimSha256 || + name !== basename(claimPath(context, claim)) + ) throw new Error("Consumer high-water unindexed claim content differs from its exact canonical bytes."); + } + } + if (claims.length > MAX_OPERATION_GENERATIONS) throw new Error("Consumer high-water operation generation bound is exhausted."); + for (let index = 0; index < claims.length; index += 1) { + if (claims[index].generation !== index + 1) throw new Error("Consumer high-water lock generations are not contiguous."); + } + for (const [key, name] of heartbeatNames) { + const claim = byKey.get(key); + if (!claim) throw new Error("Consumer high-water epoch contains an orphan heartbeat entry."); + await readExactMetadata( + join(context.epochDirectory, name), + options.metadataMaxBytes, + (value) => validateHeartbeat(value, claim), + "Consumer high-water lock heartbeat", + options, + budget, + ); + } + const terminals = new Map(); + for (const [key, name] of terminalNames) { + const claim = byKey.get(key); + if (!claim) throw new Error("Consumer high-water epoch contains an orphan terminal entry."); + terminals.set(key, await readExactMetadata( + join(context.epochDirectory, name), + options.metadataMaxBytes, + (value) => validateTerminal(value, claim, options.stateMaxBytes), + "Consumer high-water lock terminal marker", + options, + budget, + )); + } + const appliedClaims = new Set(); + for (const [key, name] of appliedNames) { + const claim = byKey.get(key); + const terminal = terminals.get(key); + if (!claim || !terminal) throw new Error("Consumer high-water epoch contains an orphan applied entry."); + await readExactMetadata( + join(context.epochDirectory, name), + options.metadataMaxBytes, + (value) => validateApplied(value, claim, terminal), + "Consumer high-water lock applied marker", + options, + budget, + ); + appliedClaims.add(key); + } + for (const claim of claims.slice(0, -1)) { + const key = `${claim.generation}:${claim.token}`; + const terminal = terminals.get(key); + if (claim.type === "rotation" || !terminal || (terminal.outcome === "commit" && !appliedClaims.has(key))) { + throw new Error("Consumer high-water operation generations crossed an unresolved earlier slot."); + } + } + if (discoveredNextEpoch !== null) { + const latest = claims.at(-1); + if (latest?.type !== "rotation") { + throw new Error("Consumer high-water in-progress next epoch lacks its exact published rotation intent."); + } + const intent = validateRotationIntent(latest.intent, context, options.stateMaxBytes); + if ( + !isImmediateSuccessorCheckpoint(context, intent.checkpoint) || + discoveredNextEpoch !== join(context.journalDirectory, epochName(intent.checkpoint)) + ) throw new Error("Consumer high-water in-progress next epoch differs from its exact published rotation intent."); + if (await authenticateChangedRoot(context, options, intent.checkpoint)) { + throw new ConsumerEpochAdvancedError(); + } + } + return { claims, terminals, temporaries }; +} + +async function readTerminal(context, claim, options) { + await revalidateAuthority(context, "read-terminal", options); + return readExactMetadata( + terminalPath(context, claim), + options.metadataMaxBytes, + (value) => validateTerminal(value, claim, options.stateMaxBytes), + "Consumer high-water lock terminal marker", + options, + ); +} + +async function readHeartbeat(context, claim, options) { + await revalidateAuthority(context, "read-heartbeat", options); + const heartbeat = await readExactMetadata( + heartbeatPath(context, claim), + options.metadataMaxBytes, + (value) => validateHeartbeat(value, claim), + "Consumer high-water lock heartbeat", + options, + ); + return heartbeat ?? { ...claim, refreshedAtMs: claim.createdAtMs }; +} + +async function publishTerminal(context, claim, wanted, options) { + const result = await publishMetadata( + terminalPath(context, claim), + wanted, + `terminal-${wanted.outcome}`, + context, + claim, + options, + ); + return validateTerminal(result.value, claim, options.stateMaxBytes); +} + +async function refreshHeartbeat(context, claim, options) { + if (await readTerminal(context, claim, options) !== null) return false; + const value = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation: claim.generation, + token: claim.token, + refreshedAtMs: options.now(), + }; + const path = heartbeatPath(context, claim); + await revalidateAuthority(context, "heartbeat", options); + const temporary = join(context.temporaryDirectory, temporaryName(path, "heartbeat", claim, context)); + let handle; + try { + handle = await options.openFile(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); + await handle.chmod?.(0o600); + await handle.writeFile(metadataBytes(value)); + await handle.sync(); + await handle.close(); + handle = undefined; + if (await readTerminal(context, claim, options) !== null) return false; + await revalidateAuthority(context, "heartbeat-rename", options); + await options.renameFile(temporary, path); + await options.syncDirectory(context.temporaryDirectory); + await options.syncDirectory(context.epochDirectory); + return true; + } finally { + if (handle !== undefined) await handle.close(); + await options.removeFile(temporary, { force: true }); + await options.syncDirectory(context.temporaryDirectory); + } +} + +function defaultHeartbeatScheduler({ interval, beat }) { + let stopped = false; + let timer; + let pending = Promise.resolve(); + const arm = () => { + if (stopped) return; + timer = setTimeout(() => { + pending = beat().catch(() => false).finally(arm); + }, interval); + timer.unref?.(); + }; + arm(); + return async () => { + stopped = true; + clearTimeout(timer); + await pending; + }; +} + +async function publishApplied(context, claim, terminal, options) { + const value = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation: claim.generation, + token: claim.token, + terminalSha256: digest(metadataBytes(terminal)), + }; + const result = await publishMetadata(appliedPath(context, claim), value, "applied", context, claim, options); + validateApplied(result.value, claim, terminal); + await options.hooks?.afterApplied?.({ claim, terminal }); +} + +async function readApplied(context, claim, terminal, options) { + await revalidateAuthority(context, "read-applied", options); + return readExactMetadata( + appliedPath(context, claim), + options.metadataMaxBytes, + (value) => validateApplied(value, claim, terminal), + "Consumer high-water lock applied marker", + options, + ); +} + +async function authenticateAppliedCommit(context, claim, terminal, options) { + for (const transaction of terminal.transactions) { + await revalidateAuthority(context, "read-applied-transition", options); + const existing = await readExactMetadata( + transitionPath(context, transaction.baseDigest), + options.metadataMaxBytes, + (value) => validateTransaction(value, transaction.baseDigest, options.stateMaxBytes).value, + "Consumer high-water applied transaction", + options, + ); + if (existing === null || !metadataBytes(existing).equals(metadataBytes(transaction))) { + throw new Error("Consumer high-water applied marker does not authenticate its exact immutable transaction chain."); + } + } + const tip = await walkTransactions(context, options); + const terminalDigest = terminal.transactions.at(-1).candidateDigest; + if (tip.tipDigest !== terminalDigest) { + await revalidateAuthority(context, "read-applied-continuation", options); + const continuation = await readExactMetadata( + transitionPath(context, terminalDigest), + options.metadataMaxBytes, + (value) => validateTransaction(value, terminalDigest, options.stateMaxBytes).value, + "Consumer high-water applied transaction continuation", + options, + ); + if (continuation === null) { + throw new Error("Consumer high-water applied marker does not authenticate its exact terminal digest."); + } + } + const repairedTip = await repairProjection(context, tip, options, claim); + const projection = await readProjection(context, "read-applied-projection", options); + if (projection.malformed || projection.sha256 !== repairedTip.tipDigest) { + throw new Error("Consumer high-water applied marker does not authenticate the current immutable tip and projection."); + } +} + +async function finishCommitAtAuthenticatedDescendant(context, options) { + const { authority } = await scanAuthenticatedContextRoot(context, options); + if (authority.kind !== "successor") return false; + const descendant = contextFromHead(context.statePath, context.guardPath, context.journalDirectory, authority.entry); + const tip = await walkTransactions(descendant, options); + await repairProjection(descendant, tip, options, { + generation: 0, + token: descendant.checkpoint.epochId, + type: "rotation", + }); + return true; +} + +async function finishCommit(context, claim, terminal, options) { + for (let attempt = 0; attempt < PROJECTION_RETRY_LIMIT; attempt += 1) { + try { + const applied = await readApplied(context, claim, terminal, options); + if (applied !== null) { + await authenticateAppliedCommit(context, claim, terminal, options); + return; + } + for (const transaction of terminal.transactions) await publishTransition(context, transaction, claim, options); + const tip = await walkTransactions(context, options); + await repairProjection(context, tip, options, claim); + await publishApplied(context, claim, terminal, options); + return; + } catch (error) { + if (!isCommitHelperReplacementTransient(error)) throw error; + try { + await revalidateAuthority(context, "finish-commit-retry-authentication", options); + await walkTransactions(context, options); + } catch (authenticationError) { + if (!isCommitHelperReplacementTransient(authenticationError)) throw authenticationError; + if (await finishCommitAtAuthenticatedDescendant(context, options)) return; + throw authenticationError; + } + } + } + throw new Error("Consumer high-water commit helper could not converge after bounded projection replacement retries."); +} + +function rotationCheckpoint(context, tip) { + const epochId = deterministicUuid( + `pylon-consumer-rotation-v2:${context.checkpointDigest}:${tip.tipDigest}`, + ); + const checkpoint = { + schemaVersion: CHECKPOINT_SCHEMA_VERSION, + epoch: context.checkpoint.epoch + 1, + epochId, + previousCheckpointSha256: context.checkpointDigest, + previousTipSha256: tip.tipDigest, + historySha256: digest(Buffer.from( + `${context.checkpoint.historySha256}:${context.checkpointDigest}:${tip.tipDigest}`, + )), + anchorDigest: tip.tipDigest, + anchorBase64: tip.tipBytes === null ? null : tip.tipBytes.toString("base64"), + retiredEpochDirectory: basename(context.epochDirectory), + sourceAuthoritySha256: context.checkpoint.sourceAuthoritySha256, + sourceAuthorityTipDigest: context.checkpoint.sourceAuthorityTipDigest, + sourceAuthorityTipBase64: context.checkpoint.sourceAuthorityTipBase64, + }; + validateCheckpoint(checkpoint, Number.MAX_SAFE_INTEGER); + return checkpoint; +} + +function rotationIntentFor(context, tip) { + return { + schemaVersion: ROTATION_INTENT_SCHEMA_VERSION, + epoch: context.checkpoint.epoch, + epochId: context.checkpoint.epochId, + checkpointSha256: context.checkpointDigest, + tipSha256: tip.tipDigest, + checkpoint: rotationCheckpoint(context, tip), + }; +} + +function rotationClaimFor(context, generation, tip) { + const intent = rotationIntentFor(context, tip); + return { + schemaVersion: LOCK_SCHEMA_VERSION, + generation, + token: intent.checkpoint.epochId, + type: "rotation", + intent, + }; +} + +async function effectiveTip(context, options) { + const chain = await walkTransactions(context, options); + if (chain.tipBytes !== null) return chain; + const projection = await readProjection(context, "rotation-legacy-state-read", options); + if (projection.malformed) throw new Error("Consumer high-water rotation cannot authenticate its legacy projection anchor."); + if (projection.bytes === null) return chain; + return { tipDigest: digest(projection.bytes), tipBytes: projection.bytes, length: chain.length }; +} + +async function scanRotationPublicationSet(context, checkpoint, options, requirePublished) { + const { scan, authority } = await scanAuthenticatedContextRoot(context, options); + const nextCheckpointPath = join(context.journalDirectory, checkpointName(checkpoint)); + const nextEpochPath = join(context.journalDirectory, epochName(checkpoint)); + const allowedCheckpointPaths = new Set([context.checkpointPath, nextCheckpointPath]); + const allowedEpochPaths = new Set([context.epochDirectory, nextEpochPath]); + const currentCheckpoint = scan.checkpointEntries.find((entry) => entry.path === context.checkpointPath); + const currentEpoch = scan.epochEntries.find((entry) => entry.path === context.epochDirectory); + const published = scan.checkpointEntries.find((entry) => entry.path === nextCheckpointPath); + if ( + scan.checkpointEntries.some((entry) => !allowedCheckpointPaths.has(entry.path)) || + scan.epochEntries.some((entry) => !allowedEpochPaths.has(entry.path)) || + (currentCheckpoint && currentCheckpoint.digest !== context.checkpointDigest) || + !scan.epochEntries.some((entry) => entry.path === nextEpochPath) || + (!requirePublished && (!currentCheckpoint || !currentEpoch)) || + (published === undefined + ? authority.kind !== "current" + : authority.kind !== "successor" || authority.entry.path !== published.path) + ) throw new Error("Consumer high-water rotation found a competing root or epoch publication."); + if (published && !metadataBytes(published.checkpoint).equals(metadataBytes(checkpoint))) { + throw new Error("Consumer high-water rotation found a competing checkpoint for the same epoch."); + } + if (requirePublished && (!published || scan.head?.path !== nextCheckpointPath || scan.missingHeadEpoch)) { + throw new Error("Consumer high-water rotation checkpoint did not become the unique complete journal head."); + } + return scan; +} + +async function finishRotationCheckpoint(context, checkpoint, writer, options) { + validateCheckpoint(checkpoint, options.stateMaxBytes); + if ( + checkpoint.epoch !== context.checkpoint.epoch + 1 || + checkpoint.previousCheckpointSha256 !== context.checkpointDigest || + checkpoint.retiredEpochDirectory !== basename(context.epochDirectory) || + checkpoint.sourceAuthoritySha256 !== context.checkpoint.sourceAuthoritySha256 || + checkpoint.sourceAuthorityTipDigest !== context.checkpoint.sourceAuthorityTipDigest || + checkpoint.sourceAuthorityTipBase64 !== context.checkpoint.sourceAuthorityTipBase64 || + checkpoint.historySha256 !== digest(Buffer.from( + `${context.checkpoint.historySha256}:${context.checkpointDigest}:${checkpoint.anchorDigest}`, + )) + ) throw new Error("Consumer high-water rotation does not anchor the exact current epoch."); + const tip = await effectiveTip(context, options); + const anchorBytes = validateCheckpoint(checkpoint, options.stateMaxBytes).anchorBytes; + if ( + checkpoint.previousTipSha256 !== tip.tipDigest || checkpoint.anchorDigest !== tip.tipDigest || + (anchorBytes === null ? tip.tipBytes !== null : !anchorBytes.equals(tip.tipBytes)) + ) throw new Error("Consumer high-water rotation does not anchor the exact immutable tip."); + const nextEpoch = join(context.journalDirectory, epochName(checkpoint)); + await ensureDirectory(nextEpoch, "Consumer high-water epoch directory", options); + await options.hooks?.afterRotationEpochSync?.({ checkpoint: structuredClone(checkpoint), nextEpoch }); + await secureDirectory(nextEpoch, "Consumer high-water next epoch directory", options); + await options.syncDirectory(nextEpoch); + if ((await options.readDirectory(nextEpoch)).length !== 0) { + throw new Error("Consumer high-water rotation found a competing next-epoch directory for the same parent."); + } + const nextPath = join(context.journalDirectory, checkpointName(checkpoint)); + await publishImmutable({ + path: nextPath, + bytes: metadataBytes(checkpoint), + directory: context.journalDirectory, + kind: "checkpoint", + context, + writer, + options, + inProgressCheckpoint: checkpoint, + beforeLink: () => scanRotationPublicationSet(context, checkpoint, options, false), + }); + await options.hooks?.afterRotationCheckpoint?.({ checkpoint: structuredClone(checkpoint), nextPath }); + await scanRotationPublicationSet(context, checkpoint, options, true); +} + +async function resolveLatestOperation(context, claim, options) { + if (claim.type === "rotation") { + await helpRotationOperation(context, claim, options); + return "rotated"; + } + const terminal = await readTerminal(context, claim, options); + if (terminal?.outcome === "commit") { + await finishCommit(context, claim, terminal, options); + return "resolved"; + } + if (terminal !== null) return "resolved"; + const heartbeat = await readHeartbeat(context, claim, options); + if (options.now() - heartbeat.refreshedAtMs < options.stale) return "active"; + await options.hooks?.afterObserveStale?.({ claim, heartbeat }); + const retired = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation: claim.generation, + token: claim.token, + outcome: "retired", + }; + const decision = await publishTerminal(context, claim, retired, options); + await options.hooks?.afterRetire?.({ claim, decision }); + if (decision.outcome === "commit") await finishCommit(context, claim, decision, options); + return "resolved"; +} + +function operationIdentity(claim) { + return claim ? `${claim.generation}:${claim.token}:${claim.type}` : null; +} + +function sameOperationClaim(left, right) { + return left === null + ? right === null + : right !== null && operationIdentity(left) === operationIdentity(right) && metadataBytes(left).equals(metadataBytes(right)); +} + +async function resolveOperationFrontier(context, options) { + const initial = await scanEpoch(context, options); + const latest = initial.claims.at(-1) ?? null; + if (latest) { + const outcome = await resolveLatestOperation(context, latest, options); + if (outcome === "rotated") return { rotated: true }; + if (outcome === "active") return { active: true }; + } + const scan = await scanEpoch(context, options); + if (!sameOperationClaim(scan.claims.at(-1) ?? null, latest)) return { retry: true }; + return { scan, frontier: latest, rotated: false, active: false }; +} + +function inProgressCheckpointForClaim(context, claim, options) { + if (claim.type !== "rotation") return null; + const validatedClaim = validateClaim(claim, context, options.stateMaxBytes); + const intent = validateRotationIntent(validatedClaim.intent, context, options.stateMaxBytes); + const { anchorBytes } = validateCheckpoint(intent.checkpoint, options.stateMaxBytes); + const expectedClaim = rotationClaimFor(context, validatedClaim.generation, { + tipDigest: intent.tipSha256, + tipBytes: anchorBytes, + }); + if (!metadataBytes(validatedClaim).equals(metadataBytes(expectedClaim))) { + throw new Error("Consumer high-water rotation claim does not match its exact authenticated intent."); + } + return intent.checkpoint; +} + +async function tryPublishClaim(context, claim, options) { + const inProgressCheckpoint = inProgressCheckpointForClaim(context, claim, options); + const contentPath = claimPath(context, claim); + const contentResult = await publishMetadata( + contentPath, + claim, + "claim", + context, + claim, + options, + inProgressCheckpoint, + ); + const existingClaim = validateClaim(contentResult.value, context, options.stateMaxBytes); + if (!metadataBytes(existingClaim).equals(metadataBytes(claim))) { + throw new Error("Consumer high-water claim content lost its exact digest-bound publication."); + } + const index = claimIndexFor(claim); + const indexResult = await publishMetadata( + claimIndexPath(context, claim.generation), + index, + "claim-index", + context, + claim, + options, + inProgressCheckpoint, + ); + const existingIndex = validateClaimIndex(indexResult.value, claim.generation); + if (!metadataBytes(existingIndex).equals(metadataBytes(index))) return false; + return indexResult.created; +} + +async function tryCreateNormalClaim(context, generation, options) { + const claim = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation, + token: randomUUID(), + type: "normal", + ownerPid: process.pid, + createdAtMs: options.now(), + }; + if (!(await tryPublishClaim(context, claim, options))) return null; + const heartbeat = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation, + token: claim.token, + refreshedAtMs: claim.createdAtMs, + }; + await publishMetadata(heartbeatPath(context, claim), heartbeat, "initial-heartbeat", context, claim, options); + await options.hooks?.afterClaim?.({ claim }); + return claim; +} + +async function tryCreateRotationClaim(context, generation, tip, options) { + const claim = rotationClaimFor(context, generation, tip); + await options.hooks?.beforeRotationDecision?.({ intent: structuredClone(claim.intent), claim: structuredClone(claim) }); + if (!(await tryPublishClaim(context, claim, options))) return null; + await options.hooks?.afterRotationIntent?.({ intent: structuredClone(claim.intent), claim: structuredClone(claim) }); + return claim; +} + +async function acquireNormalOperation(context, options) { + for (;;) { + const frontier = await resolveOperationFrontier(context, options); + if (frontier.rotated) return { rotated: true }; + if (frontier.active) throw new Error(`Consumer high-water state is actively locked: ${context.journalDirectory}`); + if (frontier.retry) continue; + const nextGeneration = (frontier.scan.claims.at(-1)?.generation ?? 0) + 1; + if (nextGeneration > options.maxLockGenerations) { + throw new Error("Consumer high-water claim epoch is exhausted; run the consumer journal rotation command."); + } + const confirmation = await scanEpoch(context, options); + if (!sameOperationClaim(confirmation.claims.at(-1) ?? null, frontier.frontier)) continue; + const claim = await tryCreateNormalClaim(context, nextGeneration, options); + if (!claim) continue; + const afterClaim = await scanEpoch(context, options); + if (!sameOperationClaim(afterClaim.claims.at(-1) ?? null, claim)) { + throw new Error("Consumer high-water normal operation did not remain the unique latest slot."); + } + return { claim, temporaries: afterClaim.temporaries, rotated: false }; + } +} + +function temporaryIsFenced(temporary, context, writer) { + if (temporary.epochId !== context.checkpoint.epochId) return true; + if (writer.generation === 0) { + return temporary.generation !== 0 || temporary.token !== writer.token; + } + if (temporary.generation === 0 || temporary.generation < writer.generation) return true; + return temporary.generation === writer.generation && temporary.token !== writer.token; +} + +function temporaryBelongsToRetiredClaim(temporary, context, epochAuthority) { + if (!epochAuthority || temporary.epochId !== context.checkpoint.epochId || temporary.generation === 0) return false; + const claim = epochAuthority.claims.find((candidate) => ( + candidate.generation === temporary.generation && candidate.token === temporary.token + )); + return claim !== undefined && epochAuthority.terminals.has(`${claim.generation}:${claim.token}`); +} + +function temporaryProcessIsAlive(temporary, options) { + try { + options.processKill(temporary.pid, 0); + return true; + } catch (error) { + if (error?.code === "ESRCH") return false; + if (error?.code === "EPERM") return true; + throw error; + } +} + +async function cleanupAuthority( + context, + writer, + rootScan, + epochTemporaries, + options, + requireQuiescent, + epochAuthority = null, + allowedNextEpoch = null, +) { + await revalidateAuthority(context, "cleanup", options); + const candidatesByPath = new Map( + [...rootScan.temporaries, ...epochTemporaries].map((temporary) => [temporary.path, temporary]), + ); + const parentNames = await options.readDirectory(dirname(context.statePath)); + const targetDigests = new Set([digest(Buffer.from(resolve(context.statePath))), digest(Buffer.from(resolve(context.guardPath)))]); + for (const name of parentNames) { + if (!name.startsWith(".pylon-consumer-tmp-v1-")) continue; + const temporary = await inspectTemporary(join(dirname(context.statePath), name), options); + if (!temporary || !targetDigests.has(temporary.targetSha256)) continue; + const expectedKind = temporary.targetSha256 === digest(Buffer.from(resolve(context.statePath))) + ? "projection" + : "legacy-guard"; + if (temporary.kind !== expectedKind) { + throw new Error("Consumer high-water state directory contains an unexpected owned temporary."); + } + candidatesByPath.set(temporary.path, temporary); + } + for (const temporary of candidatesByPath.values()) { + const fenced = temporaryIsFenced(temporary, context, writer); + if (!fenced && temporary.token !== writer.token) { + throw new Error("Consumer high-water journal contains a live or future owned temporary."); + } + if (!fenced) { + if (!requireQuiescent) continue; + if (temporaryProcessIsAlive(temporary, options)) { + if (writer.type === "rotation" && temporary.generation === writer.generation && temporary.token === writer.token) { + continue; + } + throw new Error("Consumer high-water journal rotation operation is pending until every prior owned temporary writer quiesces."); + } + await options.removeFile(temporary.path, { force: true }); + await options.syncDirectory(dirname(temporary.path)); + continue; + } + const retiredClaimTemporary = writer.generation === 0 && + temporaryBelongsToRetiredClaim(temporary, context, epochAuthority); + if (!retiredClaimTemporary && temporaryProcessIsAlive(temporary, options)) { + if (requireQuiescent) { + throw new Error("Consumer high-water journal rotation operation is pending until every prior owned temporary writer quiesces."); + } + continue; + } + await options.removeFile(temporary.path, { force: true }); + await options.syncDirectory(dirname(temporary.path)); + } + let retiredEpochDeferred = false; + for (const epoch of rootScan.epochEntries) { + if (epoch.name === basename(context.epochDirectory)) continue; + if (allowedNextEpoch !== null && epoch.name === allowedNextEpoch) continue; + if (epoch.name !== context.checkpoint.retiredEpochDirectory) { + throw new Error("Consumer high-water journal contains an orphan epoch directory."); + } + let retiredNames; + try { + retiredNames = await options.readDirectory(epoch.path); + } catch (error) { + if (error?.code === "ENOENT") continue; + throw error; + } + if (retiredNames.length > options.maxJournalEntries + MAX_TEMPORARY_ENTRIES) { + throw new Error("Consumer high-water retired epoch exceeds its safe allocation bound."); + } + const retiredTemporaries = []; + for (const name of retiredNames) { + const path = join(epoch.path, name); + let entry; + try { + entry = await options.lstatEntry(path); + } catch (error) { + if (error?.code === "ENOENT") continue; + throw error; + } + if (entry.isSymbolicLink?.() || (!entry.isFile() && !entry.isDirectory())) { + throw new Error("Consumer high-water retired epoch contains an unsafe entry."); + } + if (name.startsWith(".")) { + const temporary = await inspectTemporary(path, options); + if (temporary) retiredTemporaries.push(temporary); + } else if ( + !claimPattern.test(name) && !claimIndexPattern.test(name) && !undigestedClaimPattern.test(name) && + !heartbeatPattern.test(name) && + !terminalPattern.test(name) && + !appliedPattern.test(name) && !transitionPattern.test(name) + ) { + throw new Error("Consumer high-water retired epoch contains an unexpected entry."); + } else if (!entry.isFile()) { + throw new Error("Consumer high-water retired epoch metadata must be regular files."); + } + } + if (retiredTemporaries.some((temporary) => temporaryProcessIsAlive(temporary, options))) { + if (requireQuiescent) { + throw new Error("Consumer high-water journal rotation operation is pending until every retired temporary writer quiesces."); + } + retiredEpochDeferred = true; + continue; + } + await options.removeFile(epoch.path, { recursive: true, force: true }); + await options.syncDirectory(context.journalDirectory); + } + for (const entry of rootScan.checkpointEntries) { + if (entry.path === context.checkpointPath) continue; + if ( + entry.digest !== context.checkpoint.previousCheckpointSha256 || + epochName(entry.checkpoint) !== context.checkpoint.retiredEpochDirectory + ) throw new Error("Consumer high-water journal contains an orphan checkpoint entry."); + if (retiredEpochDeferred) continue; + await options.removeFile(entry.path, { force: true }); + await options.syncDirectory(context.journalDirectory); + } + const { scan: final, authority: finalAuthority } = await scanAuthenticatedContextRoot(context, options); + const allowedCheckpoints = retiredEpochDeferred ? 2 : 1; + const expectedEpochs = new Set([context.epochDirectory]); + if (retiredEpochDeferred) { + expectedEpochs.add(join(context.journalDirectory, context.checkpoint.retiredEpochDirectory)); + } + if ( + allowedNextEpoch !== null && + final.epochEntries.some((entry) => entry.name === allowedNextEpoch) + ) expectedEpochs.add(join(context.journalDirectory, allowedNextEpoch)); + if ( + finalAuthority.kind !== "current" || + final.checkpointEntries.length !== allowedCheckpoints || final.epochEntries.length !== expectedEpochs.size || + final.temporaries.some((temporary) => !temporaryProcessIsAlive(temporary, options)) || + final.head?.path !== context.checkpointPath || + final.epochEntries.some((entry) => !expectedEpochs.has(entry.path)) + ) throw new Error("Consumer high-water journal did not converge to one bounded current epoch."); +} + +async function helpRotationOperation(context, claim, options) { + if (claim.type !== "rotation") throw new Error("Consumer high-water rotation helper requires one rotation operation slot."); + const intent = validateRotationIntent(claim.intent, context, options.stateMaxBytes); + const helperOptions = { ...options, inProgressCheckpoint: intent.checkpoint }; + const completedBeforeHelp = await completedRotationResult(context, intent, helperOptions).catch(() => null); + if (completedBeforeHelp) return true; + try { + const scan = await scanEpoch(context, helperOptions); + const latest = scan.claims.at(-1); + if (operationIdentity(latest) !== operationIdentity(claim) || !metadataBytes(latest).equals(metadataBytes(claim))) { + throw new Error("Consumer high-water rotation operation is not the unique latest slot."); + } + const tip = await effectiveTip(context, helperOptions); + if (tip.tipDigest !== intent.tipSha256) { + throw new Error("Consumer high-water rotation operation no longer matches its exact authoritative tip."); + } + const { scan: rootScan, authority: rootAuthority } = await scanAuthenticatedContextRoot(context, helperOptions); + if (rootAuthority.kind !== "current") { + throw new Error("Consumer high-water rotation helper lost its exact current checkpoint authority."); + } + const nextEpochName = epochName(intent.checkpoint); + await cleanupAuthority( + context, + claim, + rootScan, + scan.temporaries, + helperOptions, + true, + scan, + nextEpochName, + ); + await finishRotationCheckpoint(context, intent.checkpoint, claim, helperOptions); + return true; + } catch (error) { + const completed = await completedRotationResult(context, intent, helperOptions).catch(() => null); + if (completed) return true; + throw error; + } +} + +async function inspectLegacyGuard(context, options) { + let entry; + try { + entry = await options.lstatEntry(context.guardPath); + } catch (error) { + if (error?.code === "ENOENT") return "absent"; + throw error; + } + if (entry.isDirectory() && !entry.isSymbolicLink?.()) { + if (await lstatOrNull(join(context.guardPath, LEGACY_RETIREMENT_MARKER_NAME), options) === null) { + throw new Error( + `Legacy consumer lock directory exists at ${context.guardPath}. Stop every legacy proper-lockfile client, ` + + "confirm that no owner remains, remove that directory manually, and retry.", + ); + } + await secureDirectory(context.guardPath, "Legacy consumer high-water lock directory", options); + const marker = await readExactMetadata( + join(context.guardPath, LEGACY_RETIREMENT_MARKER_NAME), + options.metadataMaxBytes, + (value) => validateLegacyRetirementMarker(value, context.statePath), + "Legacy consumer high-water retirement marker", + options, + ); + await options.syncDirectory(context.guardPath); + await options.syncDirectory(dirname(context.guardPath)); + return "retirement-marker"; + } + if (!entry.isFile() || entry.isSymbolicLink?.()) { + throw new Error("Legacy consumer lock guard is not one exact regular non-symlink file."); + } + const expected = legacyGuardFor(context.statePath); + const actual = await readExactMetadata( + context.guardPath, + options.metadataMaxBytes, + (value) => value, + "Legacy consumer lock guard", + options, + ); + if (!metadataBytes(actual).equals(metadataBytes(expected))) { + throw new Error("Legacy consumer lock guard differs from the exact durable handoff guard."); + } + await options.syncDirectory(dirname(context.guardPath)); + return "guard"; +} + +async function ensureLegacyGuard(context, claim, options) { + if (["guard", "retirement-marker"].includes(await inspectLegacyGuard(context, options))) return; + const expected = legacyGuardFor(context.statePath); + await publishImmutable({ + path: context.guardPath, + bytes: metadataBytes(expected), + directory: dirname(context.guardPath), + kind: "legacy-guard", + context, + writer: claim, + options, + }); + if (await inspectLegacyGuard(context, options) !== "guard") { + throw new Error("Legacy consumer lock handoff did not publish the exact durable guard."); + } +} + +function normalizeOptions({ + stale = PYLON_CONSUMER_LOCK_STALE_MS, + update = PYLON_CONSUMER_LOCK_UPDATE_MS, + stateMaxBytes = DEFAULT_STATE_MAX_BYTES, + maxTransactionDepth = MAX_TRANSACTION_DEPTH, + maxLockGenerations = MAX_LOCK_GENERATIONS, + maxJournalBytes = DEFAULT_JOURNAL_MAX_BYTES, + now = Date.now, + startHeartbeat = defaultHeartbeatScheduler, + hooks, + directoryOperations = {}, + lstatEntry = lstat, + makeDirectory = mkdir, + syncDirectory = syncConsumerStateDirectory, + openFile = open, + linkFile = link, + readDirectory = readdir, + renameFile = rename, + removeFile = rm, + processKill = process.kill.bind(process), + currentUid = typeof process.getuid === "function" ? process.getuid() : null, +} = {}) { + if ( + !Number.isSafeInteger(stale) || !Number.isSafeInteger(update) || update < 1 || stale <= update || + !Number.isSafeInteger(stateMaxBytes) || stateMaxBytes < 1 || stateMaxBytes > MAX_STATE_BYTES || + !Number.isSafeInteger(maxTransactionDepth) || maxTransactionDepth < 1 || maxTransactionDepth > MAX_TRANSACTION_DEPTH || + !Number.isSafeInteger(maxLockGenerations) || maxLockGenerations < 2 || maxLockGenerations > MAX_LOCK_GENERATIONS || + !Number.isSafeInteger(maxJournalBytes) || maxJournalBytes < stateMaxBytes || maxJournalBytes > MAX_JOURNAL_BYTES || + !Number.isSafeInteger(currentUid) || currentUid < 0 + ) throw new Error("Consumer high-water lock timing, state-size, journal, or transaction bound is invalid."); + return { + stale, + update, + stateMaxBytes, + maxTransactionDepth, + maxLockGenerations, + maxJournalBytes, + maxJournalEntries: MAX_OPERATION_GENERATIONS * 5 + MAX_TRANSACTION_DEPTH + 32, + metadataMaxBytes: stateMaxBytes * 3 + 8192, + now, + startHeartbeat, + hooks, + directoryOperations, + lstatEntry, + makeDirectory, + syncDirectory, + openFile, + linkFile, + readDirectory, + renameFile, + removeFile, + processKill, + currentUid, + activeWriter: null, + }; +} + +function normalizeRotationOptions(rawOptions) { + const options = normalizeOptions(rawOptions); + options.stateMaxBytes = MAX_STATE_BYTES; + options.maxTransactionDepth = MAX_TRANSACTION_DEPTH; + options.maxLockGenerations = MAX_LOCK_GENERATIONS; + options.maxJournalBytes = MAX_JOURNAL_BYTES; + options.maxJournalEntries = MAX_OPERATION_GENERATIONS * 5 + MAX_TRANSACTION_DEPTH + 32; + options.metadataMaxBytes = MAX_STATE_BYTES * 3 + 8192; + return options; +} + +async function lstatOrNull(path, options) { + try { + return await options.lstatEntry(path); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } +} + +function legacyTerminalFileName(claim) { + return `terminal-${generationName(claim.generation)}-${claim.token}.json`; +} + +function legacyHeartbeatFileName(claim) { + return `heartbeat-${generationName(claim.generation)}-${claim.token}.json`; +} + +function legacyAppliedFileName(claim) { + return `applied-${generationName(claim.generation)}-${claim.token}.json`; +} + +function authorityDigest(entries, tipDigest, tipBytes) { + const hash = createHash("sha256"); + hash.update("pylon-consumer-v1-authority\0"); + const sorted = [...entries].sort((left, right) => { + if (left[0] < right[0]) return -1; + if (left[0] > right[0]) return 1; + return 0; + }); + for (const [name, bytes] of sorted) { + const nameBytes = Buffer.from(name); + const header = Buffer.alloc(12); + header.writeUInt32BE(nameBytes.length, 0); + header.writeBigUInt64BE(BigInt(bytes.length), 4); + hash.update(header); + hash.update(nameBytes); + hash.update(bytes); + } + hash.update(Buffer.from(`tip:${tipDigest}:`)); + if (tipBytes !== null) hash.update(tipBytes); + return hash.digest("hex"); +} + +async function readLegacyAuthority(statePath, lockDirectory, transactionDirectory, options) { + await secureDirectory(lockDirectory, "Legacy consumer high-water lock directory", options); + await secureDirectory(transactionDirectory, "Legacy consumer high-water transaction directory", options); + await options.syncDirectory(lockDirectory); + await options.syncDirectory(transactionDirectory); + const transactionNames = await options.readDirectory(transactionDirectory); + if (transactionNames.length > options.maxTransactionDepth) { + throw new Error("Legacy consumer high-water transaction directory exceeds its safe entry bound."); + } + const budget = { bytes: 0 }; + const authorityEntries = []; + const actualTransactions = new Map(); + for (const name of transactionNames) { + const match = legacyTransitionPattern.exec(name); + if (!match || actualTransactions.has(match[1])) { + throw new Error("Legacy consumer high-water transaction directory contains a malformed or extra entry."); + } + const value = await readExactMetadata( + join(transactionDirectory, name), + options.metadataMaxBytes, + (candidate) => validateTransaction(candidate, match[1], options.stateMaxBytes).value, + "Legacy consumer high-water transaction", + options, + budget, + ); + actualTransactions.set(match[1], value); + authorityEntries.push([`transactions/${name}`, metadataBytes(value)]); + } + const lockNames = await options.readDirectory(lockDirectory); + if (lockNames.length > MAX_OPERATION_GENERATIONS * 4 + 1) { + throw new Error("Legacy consumer high-water lock directory exceeds its safe entry bound."); + } + const claimNames = new Map(); + const heartbeatNames = new Map(); + const terminalNames = new Map(); + const appliedNames = new Map(); + let retirementMarker = null; + for (const name of lockNames) { + let match; + if (name === LEGACY_RETIREMENT_MARKER_NAME) { + if (retirementMarker !== null) throw new Error("Legacy consumer high-water retirement marker is duplicated."); + retirementMarker = await readExactMetadata( + join(lockDirectory, name), + options.metadataMaxBytes, + (value) => validateLegacyRetirementMarker(value, statePath), + "Legacy consumer high-water retirement marker", + options, + budget, + ); + continue; + } + if ((match = undigestedClaimPattern.exec(name))) claimNames.set(Number(match[1]), name); + else if ((match = heartbeatPattern.exec(name))) heartbeatNames.set(`${Number(match[1])}:${match[2]}`, name); + else if ((match = terminalPattern.exec(name))) terminalNames.set(`${Number(match[1])}:${match[2]}`, name); + else if ((match = appliedPattern.exec(name))) appliedNames.set(`${Number(match[1])}:${match[2]}`, name); + else throw new Error("Legacy consumer high-water lock directory contains a malformed or extra entry."); + } + const claims = []; + const byKey = new Map(); + for (const [generation, name] of [...claimNames].sort((left, right) => left[0] - right[0])) { + const claim = await readExactMetadata( + join(lockDirectory, name), + options.metadataMaxBytes, + validateLegacyClaim, + "Legacy consumer high-water lock claim", + options, + budget, + ); + if (claim.generation !== generation || name !== `claim-${generationName(generation)}.json`) { + throw new Error("Legacy consumer high-water claim name differs from its exact generation."); + } + claims.push(claim); + byKey.set(`${claim.generation}:${claim.token}`, claim); + authorityEntries.push([`lock/${name}`, metadataBytes(claim)]); + } + if (claims.length > options.maxLockGenerations) throw new Error("Legacy consumer high-water claim bound is exhausted."); + for (let index = 0; index < claims.length; index += 1) { + if (claims[index].generation !== index + 1) throw new Error("Legacy consumer high-water claims are not contiguous."); + } + for (const [key, name] of heartbeatNames) { + const claim = byKey.get(key); + if (!claim || name !== legacyHeartbeatFileName(claim)) { + throw new Error("Legacy consumer high-water lock contains an orphan heartbeat."); + } + const heartbeat = await readExactMetadata( + join(lockDirectory, name), + options.metadataMaxBytes, + (value) => validateLegacyHeartbeat(value, claim), + "Legacy consumer high-water heartbeat", + options, + budget, + ); + authorityEntries.push([`lock/${name}`, metadataBytes(heartbeat)]); + } + const terminals = new Map(); + for (const [key, name] of terminalNames) { + const claim = byKey.get(key); + if (!claim || name !== legacyTerminalFileName(claim)) { + throw new Error("Legacy consumer high-water lock contains an orphan terminal marker."); + } + const terminal = await readExactMetadata( + join(lockDirectory, name), + options.metadataMaxBytes, + (value) => validateLegacyTerminal(value, claim, options.stateMaxBytes), + "Legacy consumer high-water terminal marker", + options, + budget, + ); + terminals.set(key, terminal); + authorityEntries.push([`lock/${name}`, metadataBytes(terminal)]); + } + for (const claim of claims) { + const key = `${claim.generation}:${claim.token}`; + if (!heartbeatNames.has(key)) throw new Error("Legacy consumer high-water claim lacks its exact heartbeat."); + } + const appliedTerminals = new Set(); + for (const [key, name] of appliedNames) { + const claim = byKey.get(key); + const terminal = terminals.get(key); + if (!claim || !terminal || name !== legacyAppliedFileName(claim)) { + throw new Error("Legacy consumer high-water lock contains an orphan applied marker."); + } + const applied = await readExactMetadata( + join(lockDirectory, name), + options.metadataMaxBytes, + (value) => validateLegacyApplied(value, claim, terminal), + "Legacy consumer high-water applied marker", + options, + budget, + ); + authorityEntries.push([`lock/${name}`, metadataBytes(applied)]); + appliedTerminals.add(key); + } + const decidedTransactions = new Map(); + const decidedDigests = new Set([GENESIS_DIGEST]); + let tipDigest = GENESIS_DIGEST; + let tipBytes = null; + let decidedLength = 0; + for (const claim of claims) { + const terminal = terminals.get(`${claim.generation}:${claim.token}`); + if (!terminal || terminal.outcome !== "commit") continue; + for (const transaction of terminal.transactions) { + if (transaction.baseDigest !== tipDigest) { + throw new Error("Legacy consumer high-water commit decisions do not form one exact authoritative chain."); + } + const prior = decidedTransactions.get(transaction.baseDigest); + if (prior && !metadataBytes(prior).equals(metadataBytes(transaction))) { + throw new Error("Legacy consumer high-water commit decisions equivocate at one base digest."); + } + decidedTransactions.set(transaction.baseDigest, transaction); + const validated = validateTransaction(transaction, tipDigest, options.stateMaxBytes); + tipDigest = transaction.candidateDigest; + tipBytes = validated.candidateBytes; + decidedDigests.add(tipDigest); + decidedLength += 1; + if (decidedLength > options.maxTransactionDepth) { + throw new Error("Legacy consumer high-water decisions exceed their safe transaction bound."); + } + } + } + let actualDigest = GENESIS_DIGEST; + let actualCount = 0; + while (actualTransactions.has(actualDigest)) { + const actual = actualTransactions.get(actualDigest); + const decided = decidedTransactions.get(actualDigest); + if (!decided || !metadataBytes(actual).equals(metadataBytes(decided))) { + throw new Error("Legacy consumer high-water transition lacks its exact immutable commit decision."); + } + actualDigest = actual.candidateDigest; + actualCount += 1; + if (actualCount > options.maxTransactionDepth) { + throw new Error("Legacy consumer high-water transition chain exceeds its safe bound."); + } + } + if (actualCount !== actualTransactions.size) { + throw new Error("Legacy consumer high-water transaction chain contains a corrupt, unreachable, or extra transition."); + } + for (const [baseDigest, actual] of actualTransactions) { + const decided = decidedTransactions.get(baseDigest); + if (!decided || !metadataBytes(actual).equals(metadataBytes(decided))) { + throw new Error("Legacy consumer high-water transition differs from its exact commit decision."); + } + } + for (const key of appliedTerminals) { + const terminal = terminals.get(key); + for (const transaction of terminal.transactions) { + const actual = actualTransactions.get(transaction.baseDigest); + if (!actual || !metadataBytes(actual).equals(metadataBytes(transaction))) { + throw new Error("Legacy consumer high-water applied marker is missing its completed transition."); + } + } + } + const recoveries = []; + for (const claim of claims) { + const key = `${claim.generation}:${claim.token}`; + const terminal = terminals.get(key); + if (!terminal) { + recoveries.push({ kind: "retire", claim }); + continue; + } + if (terminal.outcome === "commit" && !appliedTerminals.has(key)) { + recoveries.push({ + kind: "commit", + claim, + terminal, + missingTransactions: terminal.transactions.filter((transaction) => !actualTransactions.has(transaction.baseDigest)), + }); + } + } + const projection = await readSecureFile( + statePath, + options.stateMaxBytes, + "Legacy consumer high-water projection", + options, + 0, + ); + if (projection !== null && projection.length < 1) throw new Error("Legacy consumer high-water projection is malformed."); + if (decidedLength === 0 && projection !== null) { + tipBytes = projection; + tipDigest = digest(projection); + authorityEntries.push(["explicit-quiescent-projection", projection]); + } else if (projection !== null && !decidedDigests.has(digest(projection))) { + throw new Error("Legacy consumer high-water projection is not an authenticated prefix of its immutable authority."); + } + + if (budget.bytes > options.maxJournalBytes) { + throw new Error("Legacy consumer high-water authority exceeds its safe byte bound."); + } + const authoritySha256 = authorityDigest(authorityEntries, tipDigest, tipBytes); + if (retirementMarker !== null) { + const expectedMarker = legacyRetirementMarkerFor(statePath, { authoritySha256, tipDigest }); + if (!metadataBytes(retirementMarker).equals(metadataBytes(expectedMarker))) { + throw new Error("Legacy consumer high-water retirement marker conflicts with the exact pre-marker authority or tip."); + } + if (recoveries.length !== 0) { + throw new Error("Legacy consumer high-water retirement marker was published before its authority became quiescent."); + } + } + return { + tipDigest, + tipBytes, + length: decidedLength, + authoritySha256, + authorityEntries, + recoveries, + retirementMarker, + }; +} + +function migrationCheckpoint(statePath, legacy) { + const checkpoint = { + schemaVersion: CHECKPOINT_SCHEMA_VERSION, + epoch: 1, + epochId: deterministicUuid(`pylon-consumer-v1-migration:${statePath}:${legacy.authoritySha256}:${legacy.tipDigest}`), + previousCheckpointSha256: GENESIS_DIGEST, + previousTipSha256: GENESIS_DIGEST, + historySha256: digest(Buffer.from( + `pylon-consumer-history:${digest(Buffer.from(statePath))}:v1:${legacy.authoritySha256}:${legacy.tipDigest}`, + )), + anchorDigest: legacy.tipDigest, + anchorBase64: legacy.tipBytes === null ? null : legacy.tipBytes.toString("base64"), + retiredEpochDirectory: null, + sourceAuthoritySha256: legacy.authoritySha256, + sourceAuthorityTipDigest: legacy.tipDigest, + sourceAuthorityTipBase64: legacy.tipBytes === null ? null : legacy.tipBytes.toString("base64"), + }; + validateCheckpoint(checkpoint, Number.MAX_SAFE_INTEGER); + return checkpoint; +} + +function sameLegacyAuthority(left, right) { + return left.authoritySha256 === right.authoritySha256 && left.tipDigest === right.tipDigest && + (left.tipBytes === null ? right.tipBytes === null : right.tipBytes !== null && left.tipBytes.equals(right.tipBytes)); +} + +function legacyOwnerIsDefinitivelyDead(claim, options) { + try { + options.processKill(claim.ownerPid, 0); + return false; + } catch (error) { + if (error?.code === "ESRCH") return true; + return false; + } +} + +function requireRecoverableLegacyOwners(legacy, options) { + for (const recovery of legacy.recoveries) { + if (!legacyOwnerIsDefinitivelyDead(recovery.claim, options)) { + throw new Error( + "Legacy consumer high-water migration is blocked by a live or uncertain incomplete v1 commit owner.", + ); + } + } +} + +function recoveredLegacyAuthorityEntries(legacy) { + const entries = []; + for (const recovery of legacy.recoveries) { + if (recovery.kind === "retire") { + const terminal = { + schemaVersion: LEGACY_LOCK_SCHEMA_VERSION, + generation: recovery.claim.generation, + token: recovery.claim.token, + outcome: "retired", + }; + entries.push([`lock/${legacyTerminalFileName(recovery.claim)}`, metadataBytes(terminal)]); + continue; + } + for (const transaction of recovery.missingTransactions) { + entries.push([`transactions/${transaction.baseDigest}.json`, metadataBytes(transaction)]); + } + const applied = { + schemaVersion: LEGACY_LOCK_SCHEMA_VERSION, + generation: recovery.claim.generation, + token: recovery.claim.token, + terminalSha256: digest(metadataBytes(recovery.terminal)), + }; + entries.push([`lock/${legacyAppliedFileName(recovery.claim)}`, metadataBytes(applied)]); + } + return entries; +} + +function expectedRecoveredLegacyAuthoritySha256(legacy) { + return authorityDigest( + [...legacy.authorityEntries, ...recoveredLegacyAuthorityEntries(legacy)], + legacy.tipDigest, + legacy.tipBytes, + ); +} + +function legacyAuthorityIsExactRecoveryProgress(previous, current) { + if ( + previous.tipDigest !== current.tipDigest || + (previous.tipBytes === null + ? current.tipBytes !== null + : current.tipBytes === null || !previous.tipBytes.equals(current.tipBytes)) + ) return false; + const required = new Map(previous.authorityEntries); + const allowed = new Map(recoveredLegacyAuthorityEntries(previous)); + const actual = new Map(current.authorityEntries); + if (required.size !== previous.authorityEntries.length || actual.size !== current.authorityEntries.length) return false; + for (const [name, bytes] of required) { + if (!actual.get(name)?.equals(bytes)) return false; + } + for (const [name, bytes] of actual) { + if (required.has(name)) continue; + if (!allowed.get(name)?.equals(bytes)) return false; + } + return true; +} +async function publishExactLegacyMetadata(path, value, validate, description, directory, context, writer, kind, options) { + await publishImmutable({ + path, + bytes: metadataBytes(value), + directory, + kind, + context, + writer, + options, + revalidate: false, + }); + const actual = await readExactMetadata(path, options.metadataMaxBytes, validate, description, options); + if (!metadataBytes(actual).equals(metadataBytes(value))) { + throw new Error(`${description} lost its immutable exact-value publication.`); + } +} + +async function helpLegacyAuthority(retiredLockDirectory, transactionDirectory, legacy, context, options) { + for (const recovery of legacy.recoveries) { + await secureDirectory(retiredLockDirectory, "Legacy consumer high-water lock directory", options); + if (await lstatOrNull(join(retiredLockDirectory, LEGACY_RETIREMENT_MARKER_NAME), options) !== null) { + throw new Error("Legacy consumer authority recovery cannot cross its immutable retirement marker."); + } + if (recovery.kind === "retire") { + const terminal = { + schemaVersion: LEGACY_LOCK_SCHEMA_VERSION, + generation: recovery.claim.generation, + token: recovery.claim.token, + outcome: "retired", + }; + await publishExactLegacyMetadata( + join(retiredLockDirectory, legacyTerminalFileName(recovery.claim)), + terminal, + (value) => validateLegacyTerminal(value, recovery.claim, options.stateMaxBytes), + "Legacy consumer high-water recovered terminal marker", + retiredLockDirectory, + context, + recovery.claim, + "terminal-retired", + options, + ); + continue; + } + for (const transaction of recovery.missingTransactions) { + await publishExactLegacyMetadata( + join(transactionDirectory, `${transaction.baseDigest}.json`), + transaction, + (value) => validateTransaction(value, transaction.baseDigest, options.stateMaxBytes).value, + "Legacy consumer high-water recovered transition", + transactionDirectory, + context, + recovery.claim, + "transition", + options, + ); + } + const applied = { + schemaVersion: LEGACY_LOCK_SCHEMA_VERSION, + generation: recovery.claim.generation, + token: recovery.claim.token, + terminalSha256: digest(metadataBytes(recovery.terminal)), + }; + await publishExactLegacyMetadata( + join(retiredLockDirectory, legacyAppliedFileName(recovery.claim)), + applied, + (value) => validateLegacyApplied(value, recovery.claim, recovery.terminal), + "Legacy consumer high-water recovered applied marker", + retiredLockDirectory, + context, + recovery.claim, + "applied", + options, + ); + } + await options.syncDirectory(retiredLockDirectory); + await options.syncDirectory(transactionDirectory); +} + +async function legacyMigrationSource(statePath, options) { + const guardPath = `${statePath}.lock`; + const retiredLockDirectory = `${statePath}.lock.v1-retired`; + const guardEntry = await lstatOrNull(guardPath, options); + const retiredEntry = await lstatOrNull(retiredLockDirectory, options); + if (retiredEntry && (!retiredEntry.isDirectory() || retiredEntry.isSymbolicLink?.())) { + throw new Error("Prior retired v1 consumer lock authority must be one real directory and is never replaced."); + } + if (guardEntry?.isDirectory() && !guardEntry.isSymbolicLink?.() && retiredEntry) { + throw new Error("Live and retired v1 consumer lock authority both exist; migration fails closed."); + } + if (retiredEntry) { + if (guardEntry !== null && (!guardEntry.isFile() || guardEntry.isSymbolicLink?.())) { + throw new Error("Prior retired v1 consumer authority has an unsafe or ambiguous live lock path."); + } + return { guardPath, sourceLockDirectory: retiredLockDirectory, layout: "prior-retired", guardEntry }; + } + if (guardEntry?.isDirectory() && !guardEntry.isSymbolicLink?.()) { + return { guardPath, sourceLockDirectory: guardPath, layout: "in-place", guardEntry }; + } + throw new Error("Prior v1 consumer lock authority is absent, unsafe, or ambiguous."); +} + +async function publishLegacyRetirementMarker(source, legacy, context, options) { + if (legacy.retirementMarker !== null) return legacy; + if (legacy.recoveries.length !== 0) { + throw new Error("Legacy consumer high-water authority must be quiescent before retirement marker publication."); + } + const markerPath = join(source.sourceLockDirectory, LEGACY_RETIREMENT_MARKER_NAME); + const marker = legacyRetirementMarkerFor(context.statePath, legacy); + const authenticateBeforeMarkerLink = async () => { + const currentSource = await legacyMigrationSource(context.statePath, options); + if (currentSource.layout !== "in-place" || currentSource.sourceLockDirectory !== source.sourceLockDirectory) { + throw new Error("Legacy consumer high-water source changed before retirement marker publication."); + } + const current = await readLegacyAuthority( + context.statePath, + source.sourceLockDirectory, + `${context.statePath}.transactions`, + options, + ); + if (current.retirementMarker !== null) { + if (sameLegacyAuthority(legacy, current)) { + throw Object.assign(new Error("Concurrent migration already published the exact retirement marker."), { + code: "PYLON_EXACT_RETIREMENT_JOIN", + }); + } + throw new Error("Legacy consumer high-water authority changed before retirement marker publication."); + } + if (!sameLegacyAuthority(legacy, current) || current.recoveries.length !== 0) { + throw new Error("Legacy consumer high-water authority changed before retirement marker publication."); + } + requireRecoverableLegacyOwners(current, options); + }; + try { + await publishImmutable({ + path: markerPath, + bytes: metadataBytes(marker), + directory: source.sourceLockDirectory, + kind: "legacy-retirement", + context, + writer: { generation: 0, token: context.checkpoint.epochId, type: "rotation" }, + options, + revalidate: false, + beforeLink: authenticateBeforeMarkerLink, + }); + } catch (error) { + if (error?.code !== "PYLON_EXACT_RETIREMENT_JOIN") throw error; + const joined = await readLegacyAuthority( + context.statePath, + source.sourceLockDirectory, + `${context.statePath}.transactions`, + options, + ); + if (joined.retirementMarker === null || !sameLegacyAuthority(legacy, joined)) throw error; + } + await options.syncDirectory(source.sourceLockDirectory); + await options.syncDirectory(dirname(source.sourceLockDirectory)); + const guarded = await readLegacyAuthority( + context.statePath, + source.sourceLockDirectory, + `${context.statePath}.transactions`, + options, + ); + if (guarded.retirementMarker === null || !sameLegacyAuthority(legacy, guarded)) { + throw new Error("Legacy consumer high-water retirement marker does not authenticate its exact pre-marker authority."); + } + await options.hooks?.afterMigrationRetirementMarker?.({ markerPath, marker: structuredClone(marker) }); + await options.hooks?.afterMigrationGuard?.({ guardPath: source.guardPath, markerPath }); + return guarded; +} + +async function publishPriorLayoutGuard(source, legacy, context, options) { + if (source.guardEntry === null) { + await publishImmutable({ + path: source.guardPath, + bytes: metadataBytes(legacyGuardFor(context.statePath)), + directory: dirname(source.guardPath), + kind: "legacy-guard", + context, + writer: { generation: 0, token: context.checkpoint.epochId, type: "rotation" }, + options, + revalidate: false, + beforeLink: async () => { + const current = await readLegacyAuthority( + context.statePath, + source.sourceLockDirectory, + `${context.statePath}.transactions`, + options, + ); + if (!sameLegacyAuthority(legacy, current) || current.recoveries.length !== 0) { + throw new Error("Prior retired v1 authority changed before downgrade guard publication."); + } + }, + }); + } + if (await inspectLegacyGuard(context, options) !== "guard") { + throw new Error("Prior retired v1 authority lacks its exact permanent downgrade guard."); + } + await options.syncDirectory(dirname(source.guardPath)); + await options.hooks?.afterMigrationGuard?.({ guardPath: source.guardPath }); +} + +async function validateMigratedAuthority(context, options) { + if (context.checkpoint.epoch < 1 || context.checkpoint.sourceAuthoritySha256 === GENESIS_DIGEST) { + throw new Error("Prior v1 consumer authority exists but the v2 journal lacks an authenticated migration checkpoint."); + } + const source = await legacyMigrationSource(context.statePath, options); + const sourceTipBytes = context.checkpoint.sourceAuthorityTipBase64 === null + ? null + : Buffer.from(context.checkpoint.sourceAuthorityTipBase64, "base64"); + const legacy = await readLegacyAuthority( + context.statePath, + source.sourceLockDirectory, + `${context.statePath}.transactions`, + options, + ); + if ( + legacy.authoritySha256 !== context.checkpoint.sourceAuthoritySha256 || + legacy.tipDigest !== context.checkpoint.sourceAuthorityTipDigest || legacy.recoveries.length !== 0 || + (sourceTipBytes === null ? legacy.tipBytes !== null : !sourceTipBytes.equals(legacy.tipBytes)) + ) throw new Error("The v2 migration checkpoint does not authenticate the complete prior v1 authority and tip."); + const guardKind = await inspectLegacyGuard(context, options); + if ( + (source.layout === "in-place" && (guardKind !== "retirement-marker" || legacy.retirementMarker === null)) || + (source.layout === "prior-retired" && guardKind !== "guard") + ) throw new Error("Prior v1 consumer authority is not fenced by its exact permanent downgrade guard."); + return { source, legacy }; +} + +export async function migrateConsumerStateJournal(statePath, rawOptions = {}) { + if (typeof statePath !== "string" || !statePath) throw new Error("A consumer-local state path is required for v1 journal migration."); + const options = normalizeOptions(rawOptions); + const absoluteStatePath = resolve(statePath); + const directory = dirname(absoluteStatePath); + await ensureDurableConsumerStateDirectory(directory, options.directoryOperations); + await secureDirectory(directory, "Consumer high-water state directory", options); + const transactionDirectory = `${absoluteStatePath}.transactions`; + const transactionEntry = await lstatOrNull(transactionDirectory, options); + if (!transactionEntry) throw new Error("No prior v1 consumer transaction authority exists to migrate."); + if (!transactionEntry.isDirectory() || transactionEntry.isSymbolicLink?.()) { + throw new Error("Prior v1 consumer transaction authority must be one real directory."); + } + let source = await legacyMigrationSource(absoluteStatePath, options); + let legacy = await readLegacyAuthority(absoluteStatePath, source.sourceLockDirectory, transactionDirectory, options); + const initialCheckpoint = migrationCheckpoint(absoluteStatePath, legacy); + await options.hooks?.afterMigrationAuthorityRead?.({ + checkpoint: structuredClone(initialCheckpoint), + legacy: structuredClone(legacy), + }); + + const journalDirectory = `${absoluteStatePath}.journal`; + await ensureDirectory(journalDirectory, "Consumer high-water journal directory", options); + const temporaryDirectory = join(journalDirectory, TEMPORARY_DIRECTORY_NAME); + await ensureDirectory(temporaryDirectory, "Consumer high-water temporary directory", options); + + source = await legacyMigrationSource(absoluteStatePath, options); + const currentLegacy = await readLegacyAuthority(absoluteStatePath, source.sourceLockDirectory, transactionDirectory, options); + if (!sameLegacyAuthority(legacy, currentLegacy) && !legacyAuthorityIsExactRecoveryProgress(legacy, currentLegacy)) { + throw new Error("Concurrent v1 migration changed the exact authenticated legacy authority or tip."); + } + legacy = currentLegacy; + if (source.layout === "in-place" && legacy.retirementMarker === null) { + requireRecoverableLegacyOwners(legacy, options); + const expectedRecoveredAuthoritySha256 = expectedRecoveredLegacyAuthoritySha256(legacy); + const recoveryCheckpoint = migrationCheckpoint(absoluteStatePath, legacy); + try { + await helpLegacyAuthority(source.sourceLockDirectory, transactionDirectory, legacy, { + statePath: absoluteStatePath, + guardPath: source.guardPath, + journalDirectory, + checkpoint: recoveryCheckpoint, + checkpointPath: join(journalDirectory, checkpointName(recoveryCheckpoint)), + checkpointDigest: digest(metadataBytes(recoveryCheckpoint)), + epochDirectory: join(journalDirectory, epochName(recoveryCheckpoint)), + temporaryDirectory, + }, options); + } catch (error) { + const joined = await readLegacyAuthority(absoluteStatePath, source.sourceLockDirectory, transactionDirectory, options); + if (joined.retirementMarker === null) throw error; + legacy = joined; + } + if (legacy.retirementMarker === null) { + const recovered = await readLegacyAuthority(absoluteStatePath, source.sourceLockDirectory, transactionDirectory, options); + if ( + recovered.recoveries.length !== 0 || recovered.authoritySha256 !== expectedRecoveredAuthoritySha256 || + recovered.tipDigest !== legacy.tipDigest || + (legacy.tipBytes === null ? recovered.tipBytes !== null : recovered.tipBytes === null || !legacy.tipBytes.equals(recovered.tipBytes)) + ) throw new Error("V1 authority recovery did not produce only the exact authenticated dead-owner completion."); + legacy = recovered; + } + } + if (source.layout === "prior-retired" && legacy.recoveries.length !== 0) { + throw new Error("Interrupted prior-layout v1 migration authority is supported read-only and still requires recovery."); + } + if (legacy.recoveries.length !== 0) { + throw new Error("Legacy consumer high-water authority is not quiescent after recovery."); + } + + let checkpoint = migrationCheckpoint(absoluteStatePath, legacy); + let bootstrapContext = { + statePath: absoluteStatePath, + guardPath: source.guardPath, + journalDirectory, + checkpoint, + checkpointPath: join(journalDirectory, checkpointName(checkpoint)), + checkpointDigest: digest(metadataBytes(checkpoint)), + epochDirectory: join(journalDirectory, epochName(checkpoint)), + temporaryDirectory, + }; + if (source.layout === "in-place") { + legacy = await publishLegacyRetirementMarker(source, legacy, bootstrapContext, options); + } else { + await publishPriorLayoutGuard(source, legacy, bootstrapContext, options); + } + + const guardedLegacy = await readLegacyAuthority( + absoluteStatePath, + source.sourceLockDirectory, + transactionDirectory, + options, + ); + if (!sameLegacyAuthority(legacy, guardedLegacy) || guardedLegacy.recoveries.length !== 0) { + throw new Error("V1 authority mutated across its exact durable retirement handoff."); + } + checkpoint = migrationCheckpoint(absoluteStatePath, guardedLegacy); + bootstrapContext = { + ...bootstrapContext, + checkpoint, + checkpointPath: join(journalDirectory, checkpointName(checkpoint)), + checkpointDigest: digest(metadataBytes(checkpoint)), + epochDirectory: join(journalDirectory, epochName(checkpoint)), + }; + const authenticateBeforeCheckpointLink = async () => { + const currentSource = await legacyMigrationSource(absoluteStatePath, options); + if (currentSource.layout !== source.layout || currentSource.sourceLockDirectory !== source.sourceLockDirectory) { + throw new Error("V1 authority source changed immediately before migration checkpoint publication."); + } + const current = await readLegacyAuthority( + absoluteStatePath, + source.sourceLockDirectory, + transactionDirectory, + options, + ); + if (!sameLegacyAuthority(guardedLegacy, current) || current.recoveries.length !== 0) { + throw new Error("V1 authority mutated immediately before migration checkpoint publication."); + } + }; + const scan = await initializeJournal( + absoluteStatePath, + journalDirectory, + options, + checkpoint, + authenticateBeforeCheckpointLink, + ); + if (!scan.head || !metadataBytes(scan.head.checkpoint).equals(metadataBytes(checkpoint))) { + throw new Error("V1 migration encountered a different existing v2 journal checkpoint."); + } + const context = contextFromHead(absoluteStatePath, source.guardPath, journalDirectory, scan.head); + await validateMigratedAuthority(context, options); + await repairProjection(context, await walkTransactions(context, options), options, { + generation: 0, + token: checkpoint.epochId, + type: "rotation", + }); + await validateMigratedAuthority(context, options); + await options.hooks?.afterMigrationComplete?.({ checkpoint: structuredClone(checkpoint) }); + await validateMigratedAuthority(context, options); + return { epoch: 1, tipSha256: checkpoint.anchorDigest, sourceAuthoritySha256: checkpoint.sourceAuthoritySha256 }; +} + +async function prepareContext(statePath, options) { + const absoluteStatePath = resolve(statePath); + const directory = dirname(absoluteStatePath); + await ensureDurableConsumerStateDirectory(directory, options.directoryOperations); + await secureDirectory(directory, "Consumer high-water state directory", options); + const guardPath = `${absoluteStatePath}.lock`; + const retiredLockDirectory = `${absoluteStatePath}.lock.v1-retired`; + const journalDirectory = `${absoluteStatePath}.journal`; + const legacyTransactionDirectory = `${absoluteStatePath}.transactions`; + + // Detect every old-authority signal before creating a guard or a genesis journal. + const legacyEntry = await lstatOrNull(legacyTransactionDirectory, options); + const guardEntry = await lstatOrNull(guardPath, options); + const retiredEntry = await lstatOrNull(retiredLockDirectory, options); + const journalEntry = await lstatOrNull(journalDirectory, options); + if (legacyEntry && (!legacyEntry.isDirectory() || legacyEntry.isSymbolicLink?.())) { + throw new Error("Prior v1 consumer transaction authority must be one real directory."); + } + if (retiredEntry && (!retiredEntry.isDirectory() || retiredEntry.isSymbolicLink?.())) { + throw new Error("Prior retired v1 consumer lock authority must be one real directory and is never replaced."); + } + if (guardEntry && ( + guardEntry.isSymbolicLink?.() || (!guardEntry.isFile() && !guardEntry.isDirectory()) + )) throw new Error("Legacy consumer lock guard is not one exact regular non-symlink file."); + + let inPlaceMarkerEntry = null; + if (guardEntry?.isDirectory() && !guardEntry.isSymbolicLink?.()) { + const markerPath = join(guardPath, LEGACY_RETIREMENT_MARKER_NAME); + inPlaceMarkerEntry = await lstatOrNull(markerPath, options); + if (inPlaceMarkerEntry) { + if (!inPlaceMarkerEntry.isFile() || inPlaceMarkerEntry.isSymbolicLink?.()) { + throw new Error("Legacy consumer high-water retirement marker must be one real file."); + } + await readExactMetadata( + markerPath, + options.metadataMaxBytes, + (value) => validateLegacyRetirementMarker(value, absoluteStatePath), + "Legacy consumer high-water retirement marker", + options, + ); + } + } + + let scan = null; + if (journalEntry) { + if (!journalEntry.isDirectory() || journalEntry.isSymbolicLink?.()) { + throw new Error("Consumer high-water journal directory must be one real directory."); + } + await secureDirectory(journalDirectory, "Consumer high-water journal directory", options); + try { + scan = await scanJournalRoot(absoluteStatePath, journalDirectory, options); + } catch (error) { + if (error?.message !== "Consumer high-water journal lacks its exact temporary namespace.") throw error; + const names = await options.readDirectory(journalDirectory); + if (names.length === 1 && names[0] === TEMPORARY_DIRECTORY_NAME) { + scan = await scanJournalRoot(absoluteStatePath, journalDirectory, options); + } else if (names.length !== 0) { + throw error; + } + } + } + const hasInPlaceLegacyDirectory = guardEntry?.isDirectory() && !guardEntry.isSymbolicLink?.(); + const hasMigratedV2Head = scan?.head?.checkpoint.sourceAuthoritySha256 !== undefined && + scan.head.checkpoint.sourceAuthoritySha256 !== GENESIS_DIGEST; + const hasLegacySignal = legacyEntry !== null || retiredEntry !== null || hasInPlaceLegacyDirectory || hasMigratedV2Head; + if (hasLegacySignal) { + if (!legacyEntry) { + if (hasInPlaceLegacyDirectory && !inPlaceMarkerEntry && !retiredEntry && !hasMigratedV2Head) { + throw new Error( + `Legacy consumer lock directory exists at ${guardPath}. Stop every legacy proper-lockfile client, ` + + "confirm that no owner remains, remove that directory manually, and retry.", + ); + } + throw new Error("Prior v1 consumer authority is incomplete because its transaction namespace is missing."); + } + // This independently validates the selected live/in-place or prior-retired lock namespace. + await legacyMigrationSource(absoluteStatePath, options); + if (!journalEntry) { + throw new Error( + "Prior v1 consumer authority exists. Stop every old client and run the explicit quiescent consumer journal migration command.", + ); + } + if (!scan?.head || scan.missingHeadEpoch) { + throw new Error("Prior v1 authority has no complete authenticated v2 migration checkpoint."); + } + const context = contextFromHead(absoluteStatePath, guardPath, journalDirectory, scan.head); + await validateMigratedAuthority(context, options); + return { context, scan }; + } + + if (scan?.head) { + if (scan.missingHeadEpoch) scan = await initializeJournal(absoluteStatePath, journalDirectory, options); + return { context: contextFromHead(absoluteStatePath, guardPath, journalDirectory, scan.head), scan }; + } + await ensureDirectory(journalDirectory, "Consumer high-water journal directory", options); + await ensureDirectory(join(journalDirectory, TEMPORARY_DIRECTORY_NAME), "Consumer high-water temporary directory", options); + scan = await initializeJournal(absoluteStatePath, journalDirectory, options); + return { context: contextFromHead(absoluteStatePath, guardPath, journalDirectory, scan.head), scan }; +} + +async function runNormalLocked(statePath, action, rawOptions) { + const options = normalizeOptions(rawOptions); + for (;;) { + const prepared = await prepareContext(statePath, options); + const acquired = await acquireNormalOperation(prepared.context, options); + if (acquired.rotated) continue; + const { context } = prepared; + const { claim, temporaries } = acquired; + options.activeWriter = claim; + let terminal = null; + let heartbeatStopped = false; + const stopHeartbeat = options.startHeartbeat({ + interval: options.update, + beat: () => refreshHeartbeat(context, claim, options), + }); + const stopHeartbeatOnce = async () => { + if (heartbeatStopped) return; + heartbeatStopped = true; + await stopHeartbeat(); + }; + const release = async (cause) => { + if (terminal !== null) return; + const wanted = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation: claim.generation, + token: claim.token, + outcome: "released", + }; + terminal = await publishTerminal(context, claim, wanted, options); + if (terminal.outcome !== "released") { + throw new Error("Consumer high-water lock ownership was retired before release.", { cause }); + } + }; + try { + const { scan: rootScan, authority: rootAuthority } = await scanAuthenticatedContextRoot(context, options); + if (rootAuthority.kind !== "current") { + throw new Error("Consumer high-water operation lost its exact current checkpoint authority."); + } + await cleanupAuthority(context, claim, rootScan, temporaries, options, false); + await ensureLegacyGuard(context, claim, options); + let chain = await walkTransactions(context, options); + let legacyBytes = null; + if (chain.tipBytes === null) { + const legacy = await readProjection(context, "legacy-state-read", options); + if (legacy.malformed) throw new Error("Consumer high-water state is malformed."); + legacyBytes = legacy.bytes; + } else { + chain = await repairProjection(context, chain, options); + } + const baseBytes = chain.tipBytes ?? legacyBytes; + const baseDigest = baseBytes === null ? GENESIS_DIGEST : digest(baseBytes); + let stagedCandidate = null; + let candidateWasStaged = false; + const commitTransactions = async (candidateBytes) => { + const transactions = []; + if (chain.tipBytes === null && legacyBytes !== null) { + transactions.push(transactionFor(GENESIS_DIGEST, legacyBytes)); + } + if (candidateBytes !== null && digest(candidateBytes) !== baseDigest) { + transactions.push(transactionFor(baseDigest, candidateBytes)); + } + if (transactions.length === 0) return false; + if (chain.length + transactions.length > options.maxTransactionDepth) { + throw new Error("Consumer high-water transaction epoch reached its safe bound; run the consumer journal rotation command."); + } + const wanted = { + schemaVersion: LOCK_SCHEMA_VERSION, + generation: claim.generation, + token: claim.token, + outcome: "commit", + transactions, + }; + await options.hooks?.beforeCommitDecision?.({ claim, transactions }); + terminal = await publishTerminal(context, claim, wanted, options); + if (terminal.outcome !== "commit" || !metadataBytes(terminal).equals(metadataBytes(wanted))) { + throw new Error("Consumer high-water transaction lost ownership before its commit decision."); + } + await options.hooks?.afterCommitDecision?.({ claim, terminal }); + await finishCommit(context, claim, terminal, options); + return true; + }; + const transaction = Object.freeze({ + readStateBytes: () => baseBytes === null ? null : Buffer.from(baseBytes), + commitState: async (value) => { + if (terminal !== null || candidateWasStaged) { + throw new Error("Consumer high-water transaction already staged a candidate or has a terminal decision."); + } + const bytes = Buffer.isBuffer(value) ? Buffer.from(value) : Buffer.from(value); + if (bytes.length < 1 || bytes.length > options.stateMaxBytes) throw new Error("Consumer high-water state is malformed."); + stagedCandidate = bytes; + candidateWasStaged = true; + }, + }); + let result; + let actionError; + try { + result = await action(context.statePath, transaction); + } catch (error) { + actionError = error; + } + if (actionError === undefined && (candidateWasStaged || legacyBytes !== null)) { + await commitTransactions(candidateWasStaged ? stagedCandidate : null); + } + await stopHeartbeatOnce(); + await release(actionError); + if (actionError !== undefined) throw actionError; + return result; + } catch (error) { + await stopHeartbeatOnce(); + await release(error); + throw error; + } + } +} + +function isExpectedRemovedClaimRead(error, context, options) { + if ( + !(error instanceof BoundedFileUnlinkedDuringReadError) || + error.constructor !== BoundedFileUnlinkedDuringReadError || error.name !== "BoundedFileUnlinkedDuringReadError" || + error.description !== "Consumer high-water operation claim" || typeof error.path !== "string" || + !Buffer.isBuffer(error.bytes) || error.bytes.length < 1 || error.bytes.length > options.metadataMaxBytes || + !isExactUnlinkedDuringReadEvidence(error) + ) return false; + const name = basename(error.path); + const match = claimPattern.exec(name); + if ( + !match || error.expectedSha256 !== match[2] || error.sha256 !== match[2] || digest(error.bytes) !== match[2] || + error.path !== join(context.epochDirectory, name) || dirname(error.path) !== context.epochDirectory + ) return false; + let claim; + try { + claim = validateClaim(JSON.parse(error.bytes), context, options.stateMaxBytes); + } catch { + return false; + } + return claim.generation === Number(match[1]) && metadataBytes(claim).equals(error.bytes) && + error.path === claimPath(context, claim); +} + +async function completedRotationResult(context, intent, options) { + const { authority } = await scanAuthenticatedContextRoot(context, options); + if ( + authority.kind !== "successor" || + !metadataBytes(authority.entry.checkpoint).equals(metadataBytes(intent.checkpoint)) + ) return null; + await scanRotationPublicationSet(context, intent.checkpoint, options, true); + return { epoch: intent.checkpoint.epoch, tipSha256: intent.checkpoint.anchorDigest }; +} + +async function recoverCompletedCurrentRotation(context, scan, options) { + if (context.checkpoint.epoch === 1 || scan.claims.length !== 0) return null; + const tip = await effectiveTip(context, options); + if (tip.length !== 0 || tip.tipDigest !== context.checkpoint.anchorDigest) return null; + const writer = { generation: 0, token: context.checkpoint.epochId, type: "rotation" }; + await repairProjection(context, tip, options, writer); + const { scan: rootScan, authority: rootAuthority } = await scanAuthenticatedContextRoot(context, options); + if (rootAuthority.kind !== "current") { + throw new Error("Consumer high-water recovery lost its exact current checkpoint authority."); + } + await cleanupAuthority(context, writer, rootScan, scan.temporaries, options, false, scan); + return { epoch: context.checkpoint.epoch, tipSha256: context.checkpoint.anchorDigest }; +} + +async function runRotation(statePath, rawOptions) { + const options = normalizeRotationOptions(rawOptions); + for (;;) { + let context; + let expectedIntent; + try { + ({ context } = await prepareContext(statePath, options)); + await ensureLegacyGuard(context, { generation: 0, token: context.checkpoint.epochId, type: "rotation" }, options); + const initialScan = await scanEpoch(context, options); + const latest = initialScan.claims.at(-1); + const preparationOptions = latest?.type === "rotation" + ? { ...options, inProgressCheckpoint: validateRotationIntent(latest.intent, context, options.stateMaxBytes).checkpoint } + : options; + const completed = await recoverCompletedCurrentRotation(context, initialScan, preparationOptions); + if (completed) return completed; + expectedIntent = rotationIntentFor(context, await effectiveTip(context, preparationOptions)); + } catch (error) { + if (error instanceof ConsumerEpochAdvancedError) continue; + throw error; + } + let frontier; + try { + frontier = await resolveOperationFrontier(context, options); + } catch (error) { + if (!(error instanceof ConsumerEpochAdvancedError) && !isExpectedRemovedClaimRead(error, context, options)) throw error; + const completedResult = await completedRotationResult(context, expectedIntent, options); + if (completedResult) return completedResult; + throw error; + } + if (frontier.rotated) continue; + if (frontier.active) { + throw new Error("Consumer high-water state is actively locked; rotation will retry after the claim quiesces."); + } + if (frontier.retry) continue; + const nextGeneration = (frontier.scan.claims.at(-1)?.generation ?? 0) + 1; + if (nextGeneration > MAX_OPERATION_GENERATIONS) { + throw new Error("Consumer high-water operation epoch is exhausted and cannot publish its cap-exempt rotation slot."); + } + const tip = await effectiveTip(context, options); + const wanted = rotationClaimFor(context, nextGeneration, tip); + const confirmation = await scanEpoch(context, options); + if (!sameOperationClaim(confirmation.claims.at(-1) ?? null, frontier.frontier)) continue; + const confirmedTip = await effectiveTip(context, options); + const confirmed = rotationClaimFor(context, nextGeneration, confirmedTip); + if (!metadataBytes(confirmed).equals(metadataBytes(wanted))) continue; + let claim; + try { + claim = await tryCreateRotationClaim(context, nextGeneration, confirmedTip, options); + } catch (error) { + const completedResult = await completedRotationResult(context, confirmed.intent, options).catch(() => null); + if (completedResult) return completedResult; + throw error; + } + if (!claim) continue; + try { + await helpRotationOperation(context, claim, options); + } catch (error) { + const completedResult = await completedRotationResult(context, claim.intent, options).catch(() => null); + if (completedResult) return completedResult; + throw error; + } + await scanRotationPublicationSet(context, claim.intent.checkpoint, options, true); + return { epoch: claim.intent.checkpoint.epoch, tipSha256: claim.intent.checkpoint.anchorDigest }; + } +} + +export async function withConsumerStateLock(statePath, action, rawOptions = {}) { + if (typeof action !== "function") throw new Error("Consumer high-water lock action must be a function."); + return runNormalLocked(statePath, action, rawOptions); +} + +export async function rotateConsumerStateJournal(statePath, rawOptions = {}) { + if (typeof statePath !== "string" || !statePath) throw new Error("A consumer-local state path is required for journal rotation."); + return runRotation(statePath, rawOptions); +} + diff --git a/scripts/fixtures/retained-publication-v2/verify-pylon-preview-history.mjs b/scripts/fixtures/retained-publication-v2/verify-pylon-preview-history.mjs new file mode 100644 index 0000000000..6b6d3c5fac --- /dev/null +++ b/scripts/fixtures/retained-publication-v2/verify-pylon-preview-history.mjs @@ -0,0 +1,140 @@ +#!/usr/bin/env node + +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + PYLON_PUBLICATION_MANIFEST_MAX_BYTES, + readBoundedRegularFile, +} from "../../lib/pylon-bounded-file.mjs"; +import { + canonicalJson, + parsePreviewTag, + PYLON_PREVIEW_MANIFEST, + sha256Bytes, +} from "../../lib/pylon-publication.mjs"; +import { PYLON_RELEASE_REPOSITORY } from "../../lib/pylon-release.mjs"; +import { withConsumerStateLock } from "./pylon-consumer-lock.mjs"; +import { verifyPreviewAttestations } from "../../verify-pylon-publication-attestations.mjs"; + +const STATE_SCHEMA_VERSION = 1; +const STATE_MAX_BYTES = 4 * 1024; +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +function exactKeys(value, keys) { + return value !== null && typeof value === "object" && !Array.isArray(value) && + Object.keys(value).sort().join(",") === [...keys].sort().join(","); +} + +function validateState(state) { + if ( + !exactKeys(state, ["schemaVersion", "repository", "channel", "sequenceEpoch", "highWater"]) || + state.schemaVersion !== STATE_SCHEMA_VERSION || state.repository !== PYLON_RELEASE_REPOSITORY || + state.channel !== "preview" || state.sequenceEpoch !== 1 || + !exactKeys(state.highWater, ["sequence", "tag", "sha256", "workflowRunId"]) || + !Number.isSafeInteger(state.highWater.sequence) || state.highWater.sequence < 1 || + parsePreviewTag(state.highWater.tag).recipeRevision < 1 || !/^[0-9a-f]{64}$/.test(state.highWater.sha256 ?? "") || + !/^[1-9][0-9]*$/.test(state.highWater.workflowRunId ?? "") + ) throw new Error("Consumer preview high-water state is malformed."); + return state; +} + +function readState(bytes) { + if (!Buffer.isBuffer(bytes) || bytes.length < 1 || bytes.length > STATE_MAX_BYTES) { + throw new Error("Consumer preview high-water state is malformed."); + } + const state = validateState(JSON.parse(bytes)); + if (bytes.toString("utf8") !== canonicalJson(state)) throw new Error("Consumer preview high-water state is not canonical JSON."); + return state; +} + +export async function recordPreviewHighWater(previewManifest, previewBytes, { statePath, initialize = false }) { + if (typeof statePath !== "string" || !statePath) throw new Error("A consumer-local --state path is required."); + if (!Buffer.isBuffer(previewBytes) || previewBytes.toString("utf8") !== canonicalJson(previewManifest)) { + throw new Error("Preview high-water requires exact canonical verified manifest bytes."); + } + if ( + previewManifest.sequenceEpoch !== 1 || !Number.isSafeInteger(previewManifest.sequence) || previewManifest.sequence < 1 || + !/^[1-9][0-9]*$/.test(previewManifest.workflowRunId ?? "") + ) throw new Error("Verified preview has a malformed monotonic sequence identity."); + const path = resolve(statePath); + const highWater = { + sequence: previewManifest.sequence, + tag: previewManifest.build.tag, + sha256: sha256Bytes(previewBytes), + workflowRunId: previewManifest.workflowRunId, + }; + return withConsumerStateLock(path, async (_lockedPath, transaction) => { + const priorBytes = transaction.readStateBytes(); + if (priorBytes === null && !initialize) throw new Error("No consumer preview high-water exists. Verify the release, then use --initialize once."); + if (priorBytes !== null && initialize) throw new Error("Consumer preview high-water already exists; --initialize cannot reset it."); + const prior = priorBytes === null ? null : readState(priorBytes); + if (prior) { + if (prior.sequenceEpoch !== previewManifest.sequenceEpoch) throw new Error("Preview sequence epoch changed without a new signed state schema."); + if (highWater.sequence < prior.highWater.sequence) throw new Error("Verified preview is older than the consumer high-water sequence."); + if (highWater.sequence === prior.highWater.sequence) { + if (canonicalJson(highWater) !== canonicalJson(prior.highWater)) { + throw new Error("Verified preview equivocates at the consumer high-water sequence."); + } + return { state: prior, advanced: false }; + } + } + const state = { + schemaVersion: STATE_SCHEMA_VERSION, + repository: PYLON_RELEASE_REPOSITORY, + channel: "preview", + sequenceEpoch: previewManifest.sequenceEpoch, + highWater, + }; + await transaction.commitState(Buffer.from(canonicalJson(state))); + return { state, advanced: true }; + }, { stateMaxBytes: STATE_MAX_BYTES }); +} + +function parseArgs(args) { + const remaining = [...args]; + const flag = (name) => { + const index = remaining.indexOf(name); + if (index === -1) return false; + remaining.splice(index, 1); + return true; + }; + const initialize = flag("--initialize"); + const historical = flag("--historical"); + const value = (name, fallback) => { + const index = remaining.indexOf(name); + if (index === -1) return fallback; + const result = remaining[index + 1]; + if (!result || result.startsWith("--")) throw new Error(`Missing value for ${name}.`); + remaining.splice(index, 2); + return result; + }; + const statePath = value("--state", ""); + const artifactDir = resolve(root, value("--artifact-dir", ".npm/pylon-release/artifacts")); + if (remaining.length > 0 || !statePath) throw new Error("Usage: verify-pylon-preview-history --state [--initialize] [--historical] [--artifact-dir path]"); + return { statePath, artifactDir, initialize, historical }; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + try { + const args = parseArgs(process.argv.slice(2)); + const previewPath = join(args.artifactDir, PYLON_PREVIEW_MANIFEST); + const previewBytes = await readBoundedRegularFile(previewPath, { + maxBytes: PYLON_PUBLICATION_MANIFEST_MAX_BYTES, + description: "Preview manifest", + }); + if (previewBytes === null) throw new Error("Preview manifest does not exist."); + const untrusted = JSON.parse(previewBytes); + const verified = verifyPreviewAttestations({ + artifactDir: args.artifactDir, + sourceSha: untrusted.build?.source?.commit ?? "", + sourceTree: untrusted.build?.source?.tree ?? "", + historical: args.historical, + }); + const result = await recordPreviewHighWater(verified.previewManifest, previewBytes, args); + console.log(JSON.stringify({ highWater: result.state.highWater, advanced: result.advanced })); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/scripts/fixtures/retained-publication-v2/verify-pylon-stable-history.mjs b/scripts/fixtures/retained-publication-v2/verify-pylon-stable-history.mjs new file mode 100644 index 0000000000..611dfcd320 --- /dev/null +++ b/scripts/fixtures/retained-publication-v2/verify-pylon-stable-history.mjs @@ -0,0 +1,162 @@ +#!/usr/bin/env node + +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + PYLON_PUBLICATION_MANIFEST_MAX_BYTES, + PYLON_STABLE_HISTORY_MAX_BYTES, + PYLON_STABLE_HISTORY_MAX_MANIFESTS, + readBoundedRegularFile, +} from "../../lib/pylon-bounded-file.mjs"; +import { PYLON_RELEASE_REPOSITORY } from "../../lib/pylon-release.mjs"; +import { withConsumerStateLock } from "./pylon-consumer-lock.mjs"; +import { + canonicalJson, + parseStableTag, + sha256Bytes, + validateStableHistory, + validateStableManifest, +} from "../../lib/pylon-publication.mjs"; + +const STATE_SCHEMA_VERSION = 1; +const STATE_MAX_BYTES = 4 * 1024; + +function exactKeys(value, keys) { + return ( + value !== null && + typeof value === "object" && + !Array.isArray(value) && + Object.keys(value).sort().join(",") === [...keys].sort().join(",") + ); +} + +function validateConsumerState(state) { + if ( + !exactKeys(state, ["schemaVersion", "repository", "channel", "highWater"]) || + state.schemaVersion !== STATE_SCHEMA_VERSION || + state.repository !== PYLON_RELEASE_REPOSITORY || + state.channel !== "stable" || + !exactKeys(state.highWater, ["sequence", "tag", "sha256"]) || + !Number.isSafeInteger(state.highWater.sequence) || + parseStableTag(state.highWater.tag).sequence !== state.highWater.sequence || + !/^[0-9a-f]{64}$/.test(state.highWater.sha256 ?? "") + ) { + throw new Error("Consumer stable high-water state is malformed."); + } + return state; +} + +function readCanonicalState(bytes) { + if (!Buffer.isBuffer(bytes) || bytes.length < 1 || bytes.length > STATE_MAX_BYTES) { + throw new Error("Consumer stable high-water state is malformed."); + } + const state = validateConsumerState(JSON.parse(bytes)); + if (bytes.toString("utf8") !== canonicalJson(state)) { + throw new Error("Consumer stable high-water state is not canonical JSON."); + } + return state; +} + +async function verifiedManifestFiles(paths, fileOptions = {}) { + if (!Array.isArray(paths) || paths.length === 0) throw new Error("Provide every stable manifest from sequence 1 through current high-water."); + if (paths.length > PYLON_STABLE_HISTORY_MAX_MANIFESTS) { + throw new Error(`Stable history exceeds its ${PYLON_STABLE_HISTORY_MAX_MANIFESTS}-manifest work bound.`); + } + let totalBytes = 0; + const manifests = []; + for (const input of paths) { + const path = resolve(input); + const bytes = await readBoundedRegularFile(path, { + maxBytes: PYLON_PUBLICATION_MANIFEST_MAX_BYTES, + description: `Stable manifest ${path}`, + ...fileOptions, + }); + if (bytes === null) throw new Error(`Stable manifest does not exist: ${path}`); + totalBytes += bytes.length; + if (totalBytes > PYLON_STABLE_HISTORY_MAX_BYTES) { + throw new Error("Stable history exceeds its total manifest byte bound."); + } + const manifest = validateStableManifest(JSON.parse(bytes)); + if (bytes.toString("utf8") !== canonicalJson(manifest)) throw new Error(`Stable manifest is not canonical: ${path}`); + manifests.push(manifest); + } + return manifests; +} + +export async function verifyStableHistoryWithState(paths, { statePath, initialize = false, fileOptions = {} }) { + if (typeof statePath !== "string" || !statePath) throw new Error("A consumer-local --state path is required."); + const absoluteStatePath = resolve(statePath); + const history = validateStableHistory(await verifiedManifestFiles(paths, fileOptions)); + const witnessed = new Map(history.map((manifest) => [manifest.sequence, { + tag: manifest.tag, + sha256: sha256Bytes(Buffer.from(canonicalJson(manifest))), + }])); + const latest = history.at(-1); + const highWater = { + sequence: latest.sequence, + tag: latest.tag, + sha256: witnessed.get(latest.sequence).sha256, + }; + return withConsumerStateLock(absoluteStatePath, async (_lockedPath, transaction) => { + const priorBytes = transaction.readStateBytes(); + const stateExists = priorBytes !== null; + if (!stateExists && !initialize) { + throw new Error("No consumer high-water state exists. Inspect the full history, then use --initialize once to accept its witnessed high-water."); + } + if (stateExists && initialize) throw new Error("Consumer high-water state already exists; --initialize cannot reset it."); + const priorState = stateExists ? readCanonicalState(priorBytes) : null; + if (priorState) { + if (latest.sequence < priorState.highWater.sequence) { + throw new Error("Verified stable history is older than the persisted consumer high-water mark."); + } + const priorWitness = witnessed.get(priorState.highWater.sequence); + if ( + !priorWitness || + priorWitness.tag !== priorState.highWater.tag || + priorWitness.sha256 !== priorState.highWater.sha256 + ) { + throw new Error("Verified stable history rewrites the consumer's persisted high-water sequence."); + } + } + const state = { + schemaVersion: STATE_SCHEMA_VERSION, + repository: PYLON_RELEASE_REPOSITORY, + channel: "stable", + highWater, + }; + const advanced = !priorState || highWater.sequence > priorState.highWater.sequence; + if (advanced) await transaction.commitState(Buffer.from(canonicalJson(state))); + return { history, state: advanced ? state : priorState, advanced }; + }, { stateMaxBytes: STATE_MAX_BYTES }); +} + +function parseArgs(args) { + const remaining = [...args]; + const stateIndex = remaining.indexOf("--state"); + if (stateIndex === -1 || !remaining[stateIndex + 1] || remaining[stateIndex + 1].startsWith("--")) { + throw new Error("Usage: verify-pylon-stable-history --state [--initialize] "); + } + const statePath = remaining[stateIndex + 1]; + remaining.splice(stateIndex, 2); + const initializeIndex = remaining.indexOf("--initialize"); + const initialize = initializeIndex !== -1; + if (initialize) remaining.splice(initializeIndex, 1); + if (remaining.some((value) => value.startsWith("--"))) throw new Error("Unknown stable history verifier option."); + return { statePath, initialize, paths: remaining }; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + try { + const args = parseArgs(process.argv.slice(2)); + const verified = await verifyStableHistoryWithState(args.paths, args); + console.log(JSON.stringify({ + sequences: verified.history.length, + highWater: verified.state.highWater, + advanced: verified.advanced, + })); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/scripts/lib/pylon-consumer-lock.mjs b/scripts/lib/pylon-consumer-lock.mjs index b7f651de64..f256391812 100644 --- a/scripts/lib/pylon-consumer-lock.mjs +++ b/scripts/lib/pylon-consumer-lock.mjs @@ -27,6 +27,8 @@ import { export { consumerGenerationGenesisCheckpoint, consumerGenerationName, consumerGenerationRotationClaim, consumerGenerationSuccessorCheckpoint }; +class GenerationOperationBusyError extends Error {} + class ConsumerEpochAdvancedError extends Error { constructor() { super("Consumer high-water journal epoch changed and fenced a paused writer."); @@ -3292,7 +3294,7 @@ async function validateMigratedAuthority(context, options) { return { source, legacy }; } -export async function migrateConsumerStateJournal(statePath, rawOptions = {}) { +async function migrateHistoricalV1ToV2(statePath, rawOptions = {}) { if (typeof statePath !== "string" || !statePath) throw new Error("A consumer-local state path is required for v1 journal migration."); const options = normalizeOptions(rawOptions); const absoluteStatePath = resolve(statePath); @@ -3764,15 +3766,15 @@ async function runRotation(statePath, rawOptions) { export async function withConsumerStateLock(statePath, action, rawOptions = {}) { if (typeof action !== "function") throw new Error("Consumer high-water lock action must be a function."); - return runNormalLocked(statePath, action, rawOptions); + return migrationApi.withState(statePath, action, rawOptions); } export async function rotateConsumerStateJournal(statePath, rawOptions = {}) { if (typeof statePath !== "string" || !statePath) throw new Error("A consumer-local state path is required for journal rotation."); - return runRotation(statePath, rawOptions); + return migrationApi.rotate(statePath, rawOptions); } -// V3 primitives remain separate from the public v2 preparation/rotation entrypoints. +// Generation primitives share exact validators with the migration reader. const generationBuilders = new WeakMap(); const generationHeartbeatPattern = new RegExp(`^heartbeat-([0-9]{16})-(${uuidSource})-([0-9]{16})\\.json$`); const generationProjectionTemporaryPattern = new RegExp(`^\\.projection-p([1-9][0-9]*)-g([0-9]{16})-c([0-9a-f]{64})-t([0-9a-f]{64})-a(${uuidSource})\\.tmp$`); @@ -4195,6 +4197,15 @@ export async function publishConsumerGeneration(builder, rawOptions = {}) { try { await options.renameFile(source, destination); } catch (error) { + if (options.renameFile === rename && ["EEXIST", "ENOTEMPTY"].includes(error?.code)) { + // An independently published exact winner is not completion of this + // builder's rename. Preparation separately retires the unused builder. + const winner = await readGenerationSnapshot(destination, checkpoint, options); + await readGenerationSnapshot(source, checkpoint, options, identity, true); + if (!generationSameInode(rootIdentity, await generationDirectory(root, options))) throw error; + await generationSync(root, options); + return winner; + } if (options.renameFile !== rename || error?.code !== "ENOENT") throw error; // Only native source loss can join this rename; a byte-identical winner is not ours. await readGenerationSnapshot(destination, checkpoint, options, identity, true); @@ -4836,7 +4847,11 @@ export async function prepareConsumerGeneration(root, authority, rawOptions = {} let discovered = await discoverConsumerGenerations(root, authority, options); if (discovered.generations.length === 0) { const builders = discovered.names.filter((name) => name.startsWith(".building-")); - const builder = builders.length === 0 ? await buildConsumerGeneration(root, authority, options) : await recoverConsumerGenerationBuilder(join(root, builders[0]), authority, options); + const recoverable = builders.find((name) => { + const pid = /^\.building-p([1-9][0-9]*)-/.exec(name)?.[1]; + return !pid || !temporaryProcessIsAlive({ pid: Number(pid) }, options); + }); + const builder = recoverable === undefined ? await buildConsumerGeneration(root, authority, options) : await recoverConsumerGenerationBuilder(join(root, recoverable), authority, options); await publishConsumerGeneration(builder, options); discovered = await discoverConsumerGenerations(root, authority, options); } @@ -4911,7 +4926,7 @@ export async function withConsumerGenerationLock(root, authority, action, rawOpt let terminal = scan.terminals.get(key); if (!terminal) { const heartbeat = scan.heartbeats.get(key)?.refreshedAtMs ?? latest.createdAtMs; - if (options.now() - heartbeat < options.stale) throw new Error("Generation state is actively locked."); + if (options.now() - heartbeat < options.stale) throw new GenerationOperationBusyError("Generation state is actively locked."); await options.hooks?.afterObserveStale?.({ claim: latest, heartbeat }); const wanted = { schemaVersion: 2, generation: latest.generation, token: latest.token, outcome: "retired" }; const result = await generationWriteReceipt(snapshot, `epoch/terminal-${generationName(latest.generation)}-${latest.token}.json`, metadataBytes(wanted), options); @@ -5006,10 +5021,26 @@ const migrationApi = createConsumerMigrationApi({ validateLegacyRetirementMarker, legacyRetirementMarkerFor, validateGenerationTransaction, authorityDigest, validateCheckpoint, checkpointName, epochName, genesisCheckpoint, migrationCheckpoint, deterministicUuid, generationEpochAuthority, rotationClaimFor, + consumerGenerationGenesisCheckpoint, prepareConsumerGeneration, withConsumerGenerationLock, rotateConsumerGeneration, + isGenerationBusy: (error) => error instanceof GenerationOperationBusyError, }); -// Read-only historical inventory; public migration stays on v2 until the blocker -// and installation contract has passed the protected-client matrix. +// Historical inspection is read-only; public entry requires explicit acknowledged migration. export async function inspectConsumerMigrationSource(statePath, rawOptions = {}) { return migrationApi.inspect(statePath, rawOptions); } + +export async function migrateConsumerGenerationJournal(statePath, rawOptions = {}) { + return migrationApi.migrate(statePath, rawOptions); +} + +export async function withConsumerGenerationStateLock(statePath, action, rawOptions = {}) { + return migrationApi.withState(statePath, action, rawOptions); +} +export async function rotateConsumerGenerationStateJournal(statePath, rawOptions = {}) { + return migrationApi.rotate(statePath, rawOptions); +} + +export async function migrateConsumerStateJournal(statePath, rawOptions = {}) { + return migrationApi.migrate(statePath, rawOptions); +} diff --git a/scripts/lib/pylon-consumer-migration.mjs b/scripts/lib/pylon-consumer-migration.mjs index 0f2143e308..76bf123c5c 100644 --- a/scripts/lib/pylon-consumer-migration.mjs +++ b/scripts/lib/pylon-consumer-migration.mjs @@ -13,6 +13,7 @@ const epochPattern = new RegExp(`^epoch-([0-9]{16})-(${UUID})$`); const temporaryPattern = new RegExp(`^\\.pylon-consumer-tmp-v1-p([1-9][0-9]*)-e(${UUID})-g([0-9]{16})-w(${UUID})-n([0-9a-f]{12})-k([a-z0-9-]{1,40})-t([0-9a-f]{64})\\.tmp$`); const MAX_ENTRIES = 65_537 * 5 + 4096 + 32; const MAX_BYTES = 256 * 1024 * 1024; +const MAX_CONSTRUCTION_ROOTS = 64; const same = (a, b) => a !== null && b !== null && a.dev === b.dev && a.ino === b.ino; const identity = ({ dev, ino }) => ({ dev, ino }); const slot = (value) => String(value).padStart(16, "0"); @@ -98,12 +99,120 @@ function dead(pid, options) { try { options.processKill(pid, 0); return false; } catch (error) { if (error?.code === "ESRCH") return true; if (error?.code === "EPERM") return false; throw error; } } +async function sync(path, options, frozen = false) { return directory(path, options, frozen, true); } +async function make(path, options) { + await boundary(options, "before", "mkdir", path); + try { await options.makeDirectory(path, { mode: 0o700 }); } + catch (error) { if (options.makeDirectory !== mkdir || error?.code !== "EEXIST") throw error; } + await boundary(options, "after", "mkdir", path); + const observed = await directory(path, options); + await sync(path, options); await sync(dirname(path), options); + return observed; +} +async function canonicalAncestors(state, options, create = false) { + const components = dirname(resolve(state)).split("/").filter(Boolean); + let path = "/"; + for (const component of components) { + path = join(path, component); + let stat = await absent(path, options); + if (stat === null && create) { + await boundary(options, "before", "mkdir", path); + try { await options.makeDirectory(path, { mode: 0o700 }); } + catch (error) { if (options.makeDirectory !== mkdir || error?.code !== "EEXIST") throw error; } + await boundary(options, "after", "mkdir", path); + await sync(path, options); + const parentHandle = await options.openFile(dirname(path), constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + try { await parentHandle.sync(); } finally { await parentHandle.close(); } + stat = await options.lstatEntry(path); + } + if (!stat?.isDirectory() || stat.isSymbolicLink()) throw new Error("Migration ancestor must be a canonical real directory."); + } + await directory(dirname(state), options); +} +function receiptName(logical) { return `receipt-${digest(Buffer.from(logical))}.json`; } +const writingPattern = new RegExp(`^\\.writing-p([1-9][0-9]*)-(${UUID})-([0-9a-f]{64})\\.tmp$`); +async function receiptFor(meta, logical, target, expected, options, { publish = false, repair = true } = {}) { + await directory(meta, options); + const receipts = join(meta, "receipts"); + await directory(receipts, options); + await directory(dirname(target), options, true); + const fixed = join(receipts, receiptName(logical)); + const wanted = await absent(target, options); + if (wanted !== null) { + if (!sameBytes(await file(target, options), expected)) throw new Error("Migration immutable target has conflicting exact bytes."); + let proof = await absent(fixed, options); + if (proof === null) { + if (!repair) throw new Error("Interrupted legacy migration receipt requires explicitly acknowledged migration."); + for (const name of await names(receipts, MAX_ENTRIES, options)) { + const match = writingPattern.exec(name); + if (!match || match[3] !== digest(Buffer.from(logical))) continue; + const candidate = await options.lstatEntry(join(receipts, name)); + if (!same(candidate, wanted)) continue; + await boundary(options, "before", "receipt-rename", fixed); + try { await options.renameFile(join(receipts, name), fixed); } + catch (error) { if (options.renameFile !== rename || error?.code !== "ENOENT" || !same(candidate, await absent(fixed, options))) throw error; } + await boundary(options, "after", "receipt-rename", fixed); + await sync(receipts, options); proof = await options.lstatEntry(fixed); break; + } + } + if (proof === null || !same(proof, wanted) || ![2, ...(logical === "guard.json" ? [3, 4] : [])].includes(wanted.nlink) || proof.nlink !== wanted.nlink) throw new Error("Migration immutable record lacks its exact durable receipt inode."); + if (!sameBytes(await file(fixed, options), expected)) throw new Error("Migration receipt bytes changed."); + return identity(wanted); + } + if (!publish) throw new Error("Migration required immutable record is absent."); + await names(receipts, MAX_ENTRIES - 1, options); + const temporary = join(receipts, `.writing-p${process.pid}-${randomUUID()}-${digest(Buffer.from(logical))}.tmp`); + const handle = await options.openFile(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 0o600); + let observed; + try { + await boundary(options, "after", "receipt-create", temporary); + await handle.writeFile(expected); + observed = safeStat(await handle.stat(), "file", options); + await boundary(options, "before", "file-sync", temporary); await handle.sync(); await boundary(options, "after", "file-sync", temporary); + } finally { await handle.close(); } + await sync(receipts, options); + await boundary(options, "before", "immutable-link", target); + try { await options.linkFile(temporary, target); } + catch (error) { + if (options.linkFile !== link || error?.code !== "EEXIST") throw error; + // An independent winner is validated separately; it is not this attempt's link. + await receiptFor(meta, logical, target, expected, options); + if (!same(observed, await options.lstatEntry(temporary))) throw new Error("Migration losing receipt was replaced."); + await options.removeFile(temporary); await sync(receipts, options); + return identity(await options.lstatEntry(target)); + } + await boundary(options, "after", "immutable-link", target); + await sync(dirname(target), options); + await boundary(options, "before", "receipt-rename", fixed); + try { await options.renameFile(temporary, fixed); } + catch (error) { if (options.renameFile !== rename || error?.code !== "ENOENT" || !same(observed, await absent(fixed, options))) throw error; } + await boundary(options, "after", "receipt-rename", fixed); + await sync(receipts, options); + return receiptFor(meta, logical, target, expected, options); +} +async function immutable(meta, name, value, options) { + const data = bytes(value); + if (data.length > options.metadataMaxBytes) throw new Error("Migration metadata exceeds its exact byte bound."); + return receiptFor(meta, name, join(meta, name), data, options, { publish: true }); +} +function projectionFields(projection) { return { projectionSha256: projection === null ? ZERO : digest(projection), projectionBase64: projection?.toString("base64") ?? null }; } +function decodeProjection(value, options) { + if (value.projectionBase64 === null) { if (value.projectionSha256 !== ZERO) throw new Error("Migration absent projection digest is invalid."); return null; } + if (typeof value.projectionBase64 !== "string" || value.projectionBase64.length > 4 * Math.ceil(options.stateMaxBytes / 3)) throw new Error("Migration projection exceeds its encoded bound."); + const data = Buffer.from(value.projectionBase64, "base64"); + if (data.length > options.stateMaxBytes || data.toString("base64") !== value.projectionBase64 || digest(data) !== value.projectionSha256) throw new Error("Migration projection does not match exact bytes and digest."); + return data; +} + // The factory shares the protected closed value validators and generation engine; // filesystem migration never calls an old operation/rotation helper. export function createConsumerMigrationApi(format) { async function readV1(statePath, lockPath, projection, options, frozen = false) { const budget = { bytes: 0 }; - const lock = await inventory(lockPath, options, { frozen, maximum: 65_537 * 4 + 2, budget }); + if (options.blockers?.has(lockPath)) { + if (!sameBytes(await file(join(lockPath, BLOCKER), options), options.blockers.get(lockPath))) throw new Error("Historical v1 blocker differs from exact migration proof."); + } else if (frozen) throw new Error("Frozen v1 authority requires its exact migration blocker proof."); + const lock = await inventory(lockPath, options, { frozen, excludeBlocker: options.blockers?.has(lockPath) ?? false, maximum: 65_537 * 4 + 2, budget }); const transactions = await inventory(`${statePath}.transactions`, options, { frozen, maximum: 4096, budget }); const claims = []; const heartbeats = new Map(); const terminals = new Map(); const applied = new Map(); let marker = null; const records = []; @@ -146,8 +255,8 @@ export function createConsumerMigrationApi(format) { while (actual.has(reached)) { if (visited.has(reached)) throw new Error("Historical v1 transaction cycle."); visited.add(reached); reached = actual.get(reached).candidateDigest; } if (visited.size !== actual.size) throw new Error("Historical v1 unreachable transaction."); for (const k of applied.keys()) for (const transaction of terminals.get(k).transactions) if (!actual.has(transaction.baseDigest)) throw new Error("Historical v1 applied transition is absent."); - if (decided.size === 0 && projection !== null) { tipBytes = projection; tipDigest = digest(projection); records.push(["explicit-quiescent-projection", projection]); } - else if (projection !== null && !prefixes.has(digest(projection))) throw new Error("Historical v1 projection is not an authenticated prefix."); + if (decided.size === 0 && projection !== null && projection.length > 0) { tipBytes = projection; tipDigest = digest(projection); records.push(["explicit-quiescent-projection", projection]); } + else if (projection !== null && projection.length > 0 && !prefixes.has(digest(projection))) throw new Error("Historical v1 projection is not an authenticated prefix."); const recoveries = []; for (const claim of claims) { const terminal = terminals.get(key(claim)); @@ -166,13 +275,13 @@ export function createConsumerMigrationApi(format) { const rootIdentity = await directory(root, options); const rootNames = await names(root, 16 + 65_536, options); if (rootNames.filter((name) => checkpointPattern.test(name)).length > 2 || rootNames.filter((name) => epochPattern.test(name)).length > 2 || rootNames.filter((name) => !temporaryPattern.test(name)).length > 16) throw new Error("Historical v2 root exceeds its pre-allocation authority bound."); - const checkpoints = []; const epochs = new Map(); const raw = []; const temporaries = []; const budget = { bytes: 0 }; + const checkpoints = []; const epochs = new Map(); const directories = new Map(); const raw = []; const temporaries = []; const budget = { bytes: 0 }; const stats = new Map(); for (const name of rootNames) { const path = join(root, name); const stat = await options.lstatEntry(path); stats.set(name, stat); if (checkpointPattern.test(name) || temporaryPattern.test(name)) { safeStat(stat, "file", options); budget.bytes += stat.size; - if (stat.size < 1 || stat.size > options.metadataMaxBytes || budget.bytes > MAX_BYTES) throw new Error("Historical v2 root exceeds byte bounds."); + if (stat.size < (temporaryPattern.test(name) ? 0 : 1) || stat.size > options.metadataMaxBytes || budget.bytes > MAX_BYTES) throw new Error("Historical v2 root exceeds byte bounds."); } else if (epochPattern.test(name) || name === ".owned-temporaries-v2") safeStat(stat, "directory", options); else throw new Error("Historical v2 root contains unexpected authority."); } @@ -183,11 +292,13 @@ export function createConsumerMigrationApi(format) { if (format.checkpointName(checkpoint) !== name) throw new Error("Historical v2 checkpoint name is not exact."); checkpoints.push({ checkpoint, data, name }); raw.push([name, data]); } else if (epochPattern.test(name) || name === ".owned-temporaries-v2") { - const scanned = await inventory(path, options, { maximum: MAX_ENTRIES + 65_536, budget }); - if (name === ".owned-temporaries-v2") temporaries.push(...[...scanned.records].map(([child, data]) => ({ name: child, data, path: join(path, child) }))); + if (options.blockers?.has(path) && !sameBytes(await file(join(path, BLOCKER), options), options.blockers.get(path))) throw new Error("Historical v2 blocker differs from exact migration proof."); + const scanned = await inventory(path, options, { excludeBlocker: options.blockers?.has(path) ?? false, maximum: name === ".owned-temporaries-v2" ? 65_536 : MAX_ENTRIES + 65_536, budget }); + directories.set(name, scanned); + if (name === ".owned-temporaries-v2") temporaries.push(...[...scanned.records].map(([child, data]) => ({ name: child, data, path: join(path, child) }))); else epochs.set(name, scanned); for (const [child, data] of scanned.records) raw.push([`${name}/${child}`, data]); - } else { const data = await file(path, options); temporaries.push({ name, data, path }); raw.push([name, data]); } + } else { const data = await file(path, options, options.metadataMaxBytes, true, 0); temporaries.push({ name, data, path }); raw.push([name, data]); } } if (!rootNames.includes(".owned-temporaries-v2") || checkpoints.length < 1 || checkpoints.length > 2 || epochs.size > 2) throw new Error("Historical v2 root lacks bounded exact checkpoint authority."); checkpoints.sort((a, b) => a.checkpoint.epoch - b.checkpoint.epoch); @@ -233,29 +344,516 @@ export function createConsumerMigrationApi(format) { } if (terminal?.outcome === "commit" && !headScan.applied.has(key(claim))) recoveries.push({ target: join(root, headEpoch, `applied-${slot(claim.generation)}-${claim.token}.json`), value: { schemaVersion: 2, generation: claim.generation, token: claim.token, terminalSha256: digest(bytes(terminal)) }, owner: claim }); } - if (tipBytes === null && projection !== null) { tipBytes = projection; tipDigest = digest(projection); } - else if (projection !== null && !prefixes.has(digest(projection))) throw new Error("Historical v2 projection is not an authenticated prefix."); + if (tipBytes === null && projection !== null && projection.length > 0) { tipBytes = projection; tipDigest = digest(projection); } + else if (projection !== null && projection.length > 0 && !prefixes.has(digest(projection))) throw new Error("Historical v2 projection is not an authenticated prefix."); if (latest?.type === "rotation") { const context = { checkpoint: head.checkpoint, checkpointDigest: digest(head.data), epochDirectory: headEpoch }; if (!bytes(format.rotationClaimFor(context, latest.generation, { tipBytes, tipDigest })).equals(bytes(latest))) throw new Error("Historical v2 latest rotation differs from exact immutable tip."); } + if (temporaries.length > 65_536) throw new Error("Historical temporary namespace exceeds its exact aggregate entry bound."); for (const temporary of temporaries) { const match = temporaryPattern.exec(temporary.name); + if (match && (!Number.isSafeInteger(Number(match[1])) || Number(match[1]) < 1 || !Number.isSafeInteger(Number(match[3])) || Number(match[3]) > 65_537)) throw new Error("Historical temporary writer or generation is outside its exact bound."); if (!match || !["checkpoint", "projection", "transition", "claim", "claim-index", "initial-heartbeat", "heartbeat", "terminal-released", "terminal-retired", "terminal-commit", "applied", "legacy-guard", "legacy-retirement"].includes(match[6])) throw new Error("Historical temporary grammar is invalid."); - if (!dead(Number(match[1]), options)) throw new Error("Historical migration has a live unresolved temporary writer."); + const epoch = checkpoints.find((entry) => entry.checkpoint.epochId === match[2]); + const scan = epoch && scans.get(epoch.name); + const winner = scan?.claims.find((claim) => claim.generation === Number(match[3])); + const decided = winner && (winner.token !== match[4] || scan.terminals.has(key(winner))); + if (!decided && !dead(Number(match[1]), options)) throw new Error("Historical migration has a live unresolved temporary writer."); + if (dirname(temporary.path) === root && match[6] !== "checkpoint") throw new Error("Historical root temporary has a forbidden target kind."); + if (epochPattern.test(basename(dirname(temporary.path))) && ["checkpoint", "projection", "legacy-guard"].includes(match[6])) throw new Error("Historical epoch temporary has a forbidden target kind."); } if (!same(rootIdentity, await directory(root, options)) || rootNames.join() !== (await names(root, 16 + 65_536, options)).join()) throw new Error("Historical v2 root changed during read."); for (const [name, stat] of stats) if (!same(stat, await options.lstatEntry(join(root, name)))) throw new Error("Historical v2 root entry changed inode."); - return { kind: "v2", root, identity: rootIdentity, checkpoints, epochs, head, headEpoch, records: raw, tipDigest, tipBytes, recoveries, temporaries }; + return { kind: "v2", root, identity: rootIdentity, stats, directories, checkpoints, epochs, head, headEpoch, records: raw, tipDigest, tipBytes, recoveries, temporaries }; } + function sourceDescription(observed) { + const { source, legacy, projection } = observed; + const records = [["projection", projection ?? Buffer.alloc(0)]]; + const addInventory = (role, entry, selected = null) => { + records.push([`${role}/identity`, bytes(entry.identity)]); + for (const [name, data] of entry.records) { + if (selected && !selected.has(name)) continue; + records.push([`${role}/${name}/identity`, bytes(identity(entry.stats.get(name)))]); + records.push([`${role}/${name}/bytes`, data]); + } + }; + if (legacy) { + addInventory("v1-lock", legacy.lock); addInventory("v1-transactions", legacy.transactions); + } + if (source?.kind === "v2") { + records.push(["v2-root/identity", bytes(source.identity)]); + for (const checkpoint of source.checkpoints) { + records.push([`v2-root/${checkpoint.name}/identity`, bytes(identity(source.stats.get(checkpoint.name)))]); + records.push([`v2-root/${checkpoint.name}/bytes`, checkpoint.data]); + } + for (const [name, entry] of source.directories) addInventory(`v2/${name}`, entry); + for (const [name, data] of source.records) if (!name.includes("/") && !checkpointPattern.test(name)) { + records.push([`v2-root/${name}/identity`, bytes(identity(source.stats.get(name)))]); records.push([`v2-root/${name}/bytes`, data]); + } + } + records.sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0); + return { + kind: source?.kind ?? "fresh", + legacyLayout: legacy === null ? null : legacy.lock.path.endsWith(".lock.v1-retired") ? "prior-retired" : "in-place", + sourceIdentity: source?.kind === "v2" ? source.identity : null, + legacyLockIdentity: legacy?.lock.identity ?? null, + legacyTransactionsIdentity: legacy?.transactions.identity ?? null, + authoritySha256: commitment("pylon-migration-source-v3", records), + tipSha256: source?.tipDigest ?? (projection === null ? ZERO : digest(projection)), + legacyAuthoritySha256: legacy?.authoritySha256 ?? ZERO, + legacyMarkerSha256: legacy?.marker ? digest(bytes(legacy.marker)) : ZERO, + }; + } + function intentFor(state, observed) { + return { schemaVersion: 3, kind: "pylon-consumer-migration-intent", statePathSha256: digest(Buffer.from(state)), + source: sourceDescription(observed), ...projectionFields(observed.projection), + legacyProjection: projectionFields(observed.legacy?.tipBytes ?? null) }; + } + function validateIntent(value, state, options) { + if (!closed(value, ["schemaVersion", "kind", "statePathSha256", "source", "projectionSha256", "projectionBase64", "legacyProjection"]) || value.schemaVersion !== 3 || value.kind !== "pylon-consumer-migration-intent" || value.statePathSha256 !== digest(Buffer.from(state))) throw new Error("Migration intent closed format or state path is invalid."); + if (!closed(value.source, ["kind", "legacyLayout", "sourceIdentity", "legacyLockIdentity", "legacyTransactionsIdentity", "authoritySha256", "tipSha256", "legacyAuthoritySha256", "legacyMarkerSha256"]) || !["fresh", "v1", "v2"].includes(value.source.kind) || ![null, "prior-retired", "in-place"].includes(value.source.legacyLayout)) throw new Error("Migration intent source is malformed."); + for (const field of ["sourceIdentity", "legacyLockIdentity", "legacyTransactionsIdentity"]) { + const id = value.source[field]; + if (id !== null && (!closed(id, ["dev", "ino"]) || !Object.values(id).every((part) => Number.isSafeInteger(part) && part >= 0))) throw new Error("Migration source inode is malformed."); + } + for (const field of ["authoritySha256", "tipSha256", "legacyAuthoritySha256", "legacyMarkerSha256"]) if (!/^[0-9a-f]{64}$/.test(value.source[field] ?? "")) throw new Error("Migration source commitment is malformed."); + if (!closed(value.legacyProjection, ["projectionSha256", "projectionBase64"])) throw new Error("Migration legacy projection is malformed."); + decodeProjection(value, options); decodeProjection(value.legacyProjection, options); + return value; + } + function blockerFor(intent) { + return { schemaVersion: 3, kind: "pylon-consumer-impossible-generation-blocker", generation: "9999999999999999", + statePathSha256: intent.statePathSha256, migrationIntentSha256: digest(bytes(intent)), source: intent.source }; + } + function guardFor(intent) { return { schemaVersion: 3, kind: "pylon-consumer-v3-guard", statePathSha256: intent.statePathSha256, migrationIntentSha256: digest(bytes(intent)) }; } + function lockPathFor(state, intent) { return intent.source.legacyLayout === "prior-retired" ? `${state}.lock.v1-retired` : `${state}.lock`; } + async function readIntent(meta, state, options, allowHistoricalRepair = false) { + await directory(meta, options); + const data = await file(join(meta, "intent.json"), options, options.metadataMaxBytes, false); + if (data === null) return null; + const intent = validateIntent(canonical(data), state, options); + await receiptFor(meta, "intent.json", join(meta, "intent.json"), data, options, { repair: allowHistoricalRepair || intent.source.kind === "fresh" }); + return intent; + } + async function historicalFromIntent(state, intent, options, { final = false, blocked = false } = {}) { + const blockers = new Map(); + const blocker = bytes(blockerFor(intent)); + const expectedLegacyPath = lockPathFor(state, intent); + let legacy = null; + if (intent.source.legacyLayout !== null) { + const found = await absent(join(expectedLegacyPath, BLOCKER), options); + if (found) blockers.set(expectedLegacyPath, blocker); + else if (blocked || final) throw new Error("Migration required v1 blocker disappeared."); + const frozen = ((await options.lstatEntry(expectedLegacyPath)).mode & 0o7777) === 0o500 || ((await options.lstatEntry(`${state}.transactions`)).mode & 0o7777) === 0o500; + legacy = await readV1(state, expectedLegacyPath, decodeProjection(intent.legacyProjection, options), { ...options, blockers }, frozen); + } + let source = legacy; + if (intent.source.kind === "fresh") { + for (const path of [`${state}.journal`, `${state}.journal.v2-retired`, `${state}.transactions`, `${state}.lock.v1-retired`]) if (await absent(path, options) !== null) throw new Error("Fresh v3 authority conflicts with a historical source; explicit migration is required."); + const guard = await absent(`${state}.lock`, options); + if (guard?.isDirectory()) throw new Error("Fresh v3 authority conflicts with a legacy lock directory."); + } + if (intent.source.kind === "v2") { + const original = await absent(`${state}.journal`, options); + const retired = await absent(`${state}.journal.v2-retired`, options); + if (retired !== null && original !== null) throw new Error("Migration source retirement conflicts with a foreign historical journal inode."); + const root = retired !== null ? `${state}.journal.v2-retired` : `${state}.journal`; + if (final && retired === null) throw new Error("Migration final source is not path fenced."); + if (!same(intent.source.sourceIdentity, retired ?? original)) throw new Error("Migration retained source has a different inode."); + const rootNames = await names(root, 16 + 65_536, options); + for (const name of rootNames.filter((entry) => epochPattern.test(entry))) { + if (await absent(join(root, name, BLOCKER), options)) blockers.set(join(root, name), blocker); + else if (blocked || final || retired !== null) throw new Error("Migration required v2 blocker disappeared."); + } + source = await readV2(state, root, decodeProjection(intent, options), { ...options, blockers, maxJournalBytes: MAX_BYTES }, legacy); + } + const observed = { source, legacy, projection: decodeProjection(intent, options) }; + if (!bytes(sourceDescription(observed)).equals(bytes(intent.source))) throw new Error("Migration historical source differs from its complete pre-block authority, inode, tip or projection commitment."); + if (final && legacy && ([(await options.lstatEntry(legacy.lock.path)).mode, (await options.lstatEntry(legacy.transactions.path)).mode].some((mode) => (mode & 0o7777) !== 0o500))) throw new Error("Migration final v1 authority is not frozen at exact 0500."); + return observed; + } + async function freeze(path, expected, revalidate, options) { + await revalidate(); + const initial = safeStat(await options.lstatEntry(path), "directory", options, true); + if (!same(expected, initial)) throw new Error("Migration freeze target differs from its pinned original inode."); + const handle = await options.openFile(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + try { + const opened = safeStat(await handle.stat(), "directory", options, true); + if (!same(initial, opened)) throw new Error("Migration freeze handle changed inode."); + await boundary(options, "before", "freeze", path); await revalidate(); + if (!same(opened, await options.lstatEntry(path))) throw new Error("Migration freeze pathname was replaced."); + await handle.chmod(0o500); + await boundary(options, "after", "freeze", path); + const after = safeStat(await handle.stat(), "directory", options, true); + if ((after.mode & 0o7777) !== 0o500 || !same(opened, after) || !same(after, await options.lstatEntry(path))) throw new Error("Migration freeze did not preserve the exact original directory."); + await boundary(options, "before", "directory-sync", path); await handle.sync(); await boundary(options, "after", "directory-sync", path); + } finally { await handle.close(); } + await sync(dirname(path), options); await revalidate(); + } + async function installGuard(state, meta, intent, options) { + if (intent.source.legacyLayout === "in-place") return; + const guard = bytes(guardFor(intent)); + const target = `${state}.lock`; + const current = await file(target, options, options.metadataMaxBytes, false); + const proofPath = join(meta, "guard.json"); + await immutable(meta, "guard.json", guardFor(intent), options); + const sourceIdentity = identity(await options.lstatEntry(proofPath)); + // guard.json and its receipt are immutable; the third link is the downgrade fence. + if (sameBytes(current, guard)) { + if (!same(sourceIdentity, await options.lstatEntry(target))) throw new Error("Migration v3 guard is a different inode from its proof."); + return; + } + const oldGuard = { schemaVersion: 1, kind: "pylon-consumer-legacy-lock-guard", statePathSha256: intent.statePathSha256 }; + if (current !== null && !current.equals(bytes(oldGuard))) throw new Error("Migration prior guard is not exact."); + const previousIdentity = await absent(target, options); + const temporary = join(meta, `guard-install-${digest(bytes(intent))}.tmp`); + try { await options.linkFile(proofPath, temporary); } + catch (error) { if (options.linkFile !== link || error?.code !== "EEXIST" || !same(sourceIdentity, await options.lstatEntry(temporary))) throw error; } + await sync(meta, options); + await boundary(options, "before", "guard-rename", target); + const before = await absent(target, options); + if (same(sourceIdentity, before)) { + const pending = await absent(temporary, options); + if (pending !== null) { if (!same(sourceIdentity, pending)) throw new Error("Migration guard link was replaced."); await options.removeFile(temporary); await sync(meta, options); } + return; + } + if (previousIdentity === null ? before !== null : !same(previousIdentity, before)) throw new Error("Migration guard changed inode before replacement."); + if (!same(sourceIdentity, await options.lstatEntry(temporary))) throw new Error("Migration guard temporary changed inode."); + try { await options.renameFile(temporary, target); } + catch (error) { if (options.renameFile !== rename || error?.code !== "ENOENT" || !same(sourceIdentity, await absent(target, options))) throw error; } + await boundary(options, "after", "guard-rename", target); + if (!same(sourceIdentity, await options.lstatEntry(target)) || !sameBytes(await file(target, options), guard)) throw new Error("Migration guard replacement differs from its exact receipt-backed inode."); + await sync(meta, options); await sync(dirname(target), options); + } + async function finalAuthority(state, meta, intent, options) { + const observed = await historicalFromIntent(state, intent, options, { final: true }); + if (intent.source.legacyLayout !== "in-place") { + if (!sameBytes(await file(`${state}.lock`, options), bytes(guardFor(intent))) || !same(await options.lstatEntry(`${state}.lock`), await options.lstatEntry(join(meta, "guard.json")))) throw new Error("Migration final source lacks its exact v3 guard inode."); + } + const source = intent.source.kind === "fresh" ? null : { kind: intent.source.kind, + authoritySha256: commitment("pylon-migration-final-source-v3", [["state", Buffer.from(state)], ["intent", bytes(intent)], ["final-path", Buffer.from(intent.source.kind === "v2" ? `${state}.journal.v2-retired` : lockPathFor(state, intent))], ["source", bytes(sourceDescription(observed))]]), + tipBytes: observed.source.tipBytes }; + const migration = intent.source.kind === "v2" && observed.legacy ? { kind: "v1", authoritySha256: observed.legacy.authoritySha256, tipBytes: observed.legacy.tipBytes } : null; + const authority = { genesis: { statePath: state, stateBytes: observed.source ? observed.source.tipBytes : decodeProjection(intent, options), source, migration } }; + const checkpoint = format.consumerGenerationGenesisCheckpoint(authority.genesis, options.stateMaxBytes); + return { authority, checkpoint, observed }; + } + async function validateMeta(state, meta, intent, observed, options) { + const present = await names(meta, MAX_CONSTRUCTION_ROOTS + 7, options); + const allowed = new Set(["receipts", "intent.json", "guard.json", "final.json", "root.json", "complete.json"]); + if (intent) { + allowed.add(`guard-install-${digest(bytes(intent))}.tmp`); + const rootPattern = new RegExp(`^journal-${digest(bytes(intent))}-${UUID}$`); + const roots = present.filter((name) => rootPattern.test(name)); + if (roots.length > MAX_CONSTRUCTION_ROOTS) throw new Error("Migration construction root bound is exhausted."); + const selectedBytes = await file(join(meta, "root.json"), options, options.metadataMaxBytes, false); + const selected = selectedBytes === null ? null : canonical(selectedBytes).goal; + for (const name of roots) { + allowed.add(name); await directory(join(meta, name), options); + if (name !== selected && (await names(join(meta, name), 0, options)).length !== 0) throw new Error("Unselected migration construction root contains foreign authority."); + } + } + if (!present.includes("receipts") || present.some((name) => !allowed.has(name))) throw new Error("Migration authority sidecar contains an unexpected closed entry."); + const targets = new Map(); + for (const name of ["intent.json", "guard.json", "final.json", "root.json", "complete.json"]) targets.set(receiptName(name), join(meta, name)); + if (observed?.legacy) { + targets.set(receiptName(`v1-lock/${MARKER}`), join(observed.legacy.lock.path, MARKER)); + for (const name of observed.legacy.lock.records.keys()) targets.set(receiptName(`v1-lock/${name}`), join(observed.legacy.lock.path, name)); + for (const name of observed.legacy.transactions.records.keys()) targets.set(receiptName(`v1-transactions/${name}`), join(observed.legacy.transactions.path, name)); + for (const recovery of observed.legacy.recoveries) { + const logical = `v1-${recovery.target.startsWith(`${state}.transactions/`) ? "transactions" : "lock"}/${basename(recovery.target)}`; + targets.set(receiptName(logical), recovery.target); + } + if (intent) targets.set(receiptName("blocker-v1"), join(observed.legacy.lock.path, BLOCKER)); + } + if (intent && observed?.source?.kind === "v2") for (const epoch of observed.source.epochs.values()) targets.set(receiptName(`blocker-v2/${basename(epoch.path)}`), join(epoch.path, BLOCKER)); + const receipts = join(meta, "receipts"); await directory(receipts, options); + const entries = await names(receipts, MAX_ENTRIES, options); let charged = 0; + for (const path of [...present.filter((name) => name !== "receipts" && !name.startsWith("journal-")).map((name) => join(meta, name)), ...entries.map((name) => join(receipts, name))]) { + const stat = safeStat(await options.lstatEntry(path), "file", options); + if (stat.size < (writingPattern.test(basename(path)) ? 0 : 1) || stat.size > options.metadataMaxBytes || (charged += stat.size) > MAX_BYTES * 2) throw new Error("Migration authority sidecar exceeds its aggregate byte bound."); + } + for (const name of entries) { + const path = join(receipts, name); const stat = await options.lstatEntry(path); + const temporary = writingPattern.exec(name); + if (temporary) { + if (!targets.has(`receipt-${temporary[3]}.json`)) throw new Error("Migration temporary has an unknown immutable target."); + if (![1, 2].includes(stat.nlink)) throw new Error("Migration receipt temporary has unexpected links."); + if (stat.nlink === 2) { + const target = targets.get(`receipt-${temporary[3]}.json`); + if (!target || !same(stat, await options.lstatEntry(target))) throw new Error("Migration linked temporary lacks its exact canonical target inode."); + } + continue; + } + const target = targets.get(name); + if (!target || !same(stat, await options.lstatEntry(target)) || ![2, ...(target === join(meta, "guard.json") ? [3, 4] : [])].includes(stat.nlink)) throw new Error("Migration fixed receipt lacks exact canonical authority."); + if (!sameBytes(await file(path, options), await file(target, options))) throw new Error("Migration receipt differs from its canonical target bytes."); + } + if (intent && present.includes("guard.json")) { + const guardStat = await options.lstatEntry(join(meta, "guard.json")); + const target = await absent(`${state}.lock`, options); + const temporary = await absent(join(meta, `guard-install-${digest(bytes(intent))}.tmp`), options); + if (temporary && !same(guardStat, temporary)) throw new Error("Migration guard staging link changed inode."); + const expectedLinks = 2 + (same(guardStat, target) ? 1 : 0) + (temporary ? 1 : 0); + if (guardStat.nlink !== expectedLinks) throw new Error("Migration guard has an unaccounted authority link."); + } + return { charged, targets }; + } + async function cleanupReceipts(state, meta, intent, observed, options) { + const { targets } = await validateMeta(state, meta, intent, observed, options); + const receipts = join(meta, "receipts"); const directoryIdentity = await directory(receipts, options); + for (const name of await names(receipts, MAX_ENTRIES, options)) { + const match = writingPattern.exec(name); if (!match) continue; + const path = join(receipts, name); const stat = safeStat(await options.lstatEntry(path), "file", options); + const fixed = join(receipts, `receipt-${match[3]}.json`); const target = targets.get(`receipt-${match[3]}.json`); + const canonicalStat = await absent(target, options); + await file(path, options, options.metadataMaxBytes, true, 0); + if (stat.nlink === 2) { + if (!same(stat, canonicalStat)) throw new Error("Migration linked temporary lost its canonical inode."); + await boundary(options, "before", "receipt-rename", fixed); + if (!same(stat, await options.lstatEntry(path)) || !same(directoryIdentity, await directory(receipts, options))) throw new Error("Migration receipt cleanup was fenced by inode replacement."); + try { await options.renameFile(path, fixed); } + catch (error) { if (options.renameFile !== rename || error?.code !== "ENOENT" || !same(stat, await absent(fixed, options))) throw error; } + await boundary(options, "after", "receipt-rename", fixed); await sync(receipts, options); + } else { + const decided = canonicalStat !== null && same(canonicalStat, await absent(fixed, options)); + if (!decided && !dead(Number(match[1]), options)) throw new Error("Migration has a live unresolved receipt writer."); + await boundary(options, "before", "receipt-unlink", path); + if (!same(stat, await options.lstatEntry(path)) || !same(directoryIdentity, await directory(receipts, options))) throw new Error("Migration losing receipt changed inode before cleanup."); + await options.removeFile(path); await boundary(options, "after", "receipt-unlink", path); await sync(receipts, options); + } + } + } + + async function prepareInstalled(state, meta, intent, options) { + const { authority, checkpoint } = await finalAuthority(state, meta, intent, options); + const wanted = { schemaVersion: 3, kind: "pylon-consumer-final-source", intentSha256: digest(bytes(intent)), sourceAuthoritySha256: checkpoint.sourceAuthoritySha256, genesisSha256: digest(bytes(checkpoint)) }; + await immutable(meta, "final.json", wanted, options); + let rootRecord = await file(join(meta, "root.json"), options, options.metadataMaxBytes, false); + if (rootRecord === null) { + await validateMeta(state, meta, intent, (await finalAuthority(state, meta, intent, options)).observed, options); + const roots = (await names(meta, MAX_CONSTRUCTION_ROOTS + 7, options)).filter((name) => name.startsWith("journal-")); + if (roots.length >= MAX_CONSTRUCTION_ROOTS) throw new Error("Migration construction root bound is exhausted."); + const goal = `journal-${digest(bytes(intent))}-${randomUUID()}`; + const root = join(meta, goal); + await boundary(options, "before", "mkdir", root); + // A collision is never an ownership join. An unrecorded empty directory is + // inert after a crash; resume creates another exclusive construction root. + await options.makeDirectory(root, { mode: 0o700 }); + const created = await directory(root, options); + await boundary(options, "after", "mkdir", root); + if (!same(created, await directory(root, options))) throw new Error("Exclusive migration root changed inode after creation."); + await sync(root, options); await sync(meta, options); + const candidate = { schemaVersion: 3, kind: "pylon-consumer-root", intentSha256: digest(bytes(intent)), goal, identity: created }; + await immutable(meta, "root.json", candidate, options); + rootRecord = await file(join(meta, "root.json"), options); + } + const expected = canonical(rootRecord); + if (!closed(expected, ["schemaVersion", "kind", "intentSha256", "goal", "identity"]) || expected.schemaVersion !== 3 || expected.kind !== "pylon-consumer-root" || expected.intentSha256 !== digest(bytes(intent)) || !new RegExp(`^journal-${digest(bytes(intent))}-${UUID}$`).test(expected.goal ?? "") || !closed(expected.identity, ["dev", "ino"]) || !Object.values(expected.identity).every((value) => Number.isSafeInteger(value) && value >= 0)) throw new Error("Migration canonical root proof is malformed."); + const root = join(meta, expected.goal); + await receiptFor(meta, "root.json", join(meta, "root.json"), rootRecord, options); + if (!same(expected.identity, await directory(root, options))) throw new Error("Migration canonical root is a different inode from its durable creation proof."); + await cleanupReceipts(state, meta, intent, (await finalAuthority(state, meta, intent, options)).observed, options); + await format.prepareConsumerGeneration(root, authority, options); + if (!same(expected.identity, await directory(root, options))) throw new Error("Migration canonical root was replaced during preparation."); + await validateMeta(state, meta, intent, (await finalAuthority(state, meta, intent, options)).observed, options); + return { root, authority, checkpoint }; + } + async function recoverDirectV1(state, meta, observed, options) { + if (observed.source.kind !== "v1") return observed; + const initial = observed; + const expectedLock = new Map(initial.legacy.lock.records); + const expectedTransactions = new Map(initial.legacy.transactions.records); + for (const recovery of initial.legacy.recoveries) { + (recovery.target.startsWith(`${state}.transactions/`) ? expectedTransactions : expectedLock).set(basename(recovery.target), bytes(recovery.value)); + } + const validateProgress = (current) => { + if (!same(initial.legacy.lock.identity, current.legacy.lock.identity) || !same(initial.legacy.transactions.identity, current.legacy.transactions.identity) || current.legacy.tipDigest !== initial.legacy.tipDigest || !sameBytes(current.legacy.tipBytes, initial.legacy.tipBytes)) throw new Error("Migration v1 recovery changed original source identity or tip."); + for (const [original, actual, expected] of [[initial.legacy.lock, current.legacy.lock, expectedLock], [initial.legacy.transactions, current.legacy.transactions, expectedTransactions]]) { + if (actual.records.size !== expected.size) throw new Error("Migration v1 recovery contains unapproved authority changes."); + for (const [name, data] of expected) if (!sameBytes(actual.records.get(name) ?? null, data) || original.stats.has(name) && !same(original.stats.get(name), actual.stats.get(name))) throw new Error("Migration v1 recovery differs from its exact allowed bytes and original inodes."); + } + }; + for (const recovery of observed.legacy.recoveries) { + if (!dead(recovery.owner.ownerPid, options)) throw new Error("Migration has a live or uncertain unresolved v1 owner."); + const logical = `v1-${recovery.target.startsWith(`${state}.transactions/`) ? "transactions" : "lock"}/${basename(recovery.target)}`; + await receiptFor(meta, logical, recovery.target, bytes(recovery.value), options, { publish: true }); + } + observed = await inspect(state, options); + validateProgress(observed); + if (observed.legacy.recoveries.length) throw new Error("Migration v1 recovery did not reach exact permitted decisions."); + if (observed.legacy.marker === null) { + const marker = format.legacyRetirementMarkerFor(state, observed.legacy); + expectedLock.set(MARKER, bytes(marker)); + await receiptFor(meta, `v1-lock/${MARKER}`, join(observed.legacy.lock.path, MARKER), bytes(marker), options, { publish: true }); + observed = await inspect(state, options); + validateProgress(observed); + } + return observed; + } + async function migrate(statePath, rawOptions = {}) { + if (rawOptions.acknowledgeLegacyProcessesStopped !== true) throw new Error("Explicit acknowledgement that all legacy processes are stopped is required before any migration mutation."); + if (typeof statePath !== "string" || !statePath) throw new Error("Migration state path is required."); + const options = optionsFor(rawOptions); const state = resolve(statePath); const meta = `${state}.journal-v3`; + await canonicalAncestors(state, options); + let intent = await absent(meta, options) === null ? null : await readIntent(meta, state, options, true); + if (intent === null) { + let observed = await inspect(state, options); + await make(meta, options); await make(join(meta, "receipts"), options); + await validateMeta(state, meta, null, observed, options); + observed = await recoverDirectV1(state, meta, observed, options); + if (observed.source.kind === "v2" && !observed.source.epochs.has(observed.source.headEpoch)) { + const original = observed; + await make(join(original.source.root, original.source.headEpoch), options); + observed = await inspect(state, options); + if (!same(original.source.identity, observed.source.identity) || observed.source.epochs.size !== 1 || observed.source.epochs.get(original.source.headEpoch)?.records.size !== 0 || observed.source.tipDigest !== original.source.tipDigest || !sameBytes(observed.projection, original.projection)) throw new Error("Initial v2 epoch completion changed historical authority."); + for (const [name, stat] of original.source.stats) if (!same(stat, observed.source.stats.get(name))) throw new Error("Initial v2 epoch completion replaced an original source inode."); + if (!bytes(original.source.records).equals(bytes(observed.source.records))) throw new Error("Initial v2 epoch completion changed original source bytes."); + } + if (observed.source.kind === "v2") { + for (const recovery of observed.source.recoveries) if (recovery.value.outcome === "retired" && !dead(recovery.owner.ownerPid, options)) throw new Error("Migration has a live unresolved v2 owner."); + } + intent = intentFor(state, observed); + await immutable(meta, "intent.json", intent, options); + } + await historicalFromIntent(state, intent, options); + const blocker = bytes(blockerFor(intent)); + let observed = await historicalFromIntent(state, intent, options); + if (observed.legacy) await receiptFor(meta, "blocker-v1", join(observed.legacy.lock.path, BLOCKER), blocker, options, { publish: true }); + if (observed.source.kind === "v2") { + // Every retained epoch is fenced: the protected reader can validate and help + // a retained operation before inspecting the replacement regular guard. + for (const epoch of observed.source.epochs.values()) await receiptFor(meta, `blocker-v2/${basename(epoch.path)}`, join(epoch.path, BLOCKER), blocker, options, { publish: true }); + } + await options.hooks?.afterMigrationBlocker?.({ statePath: state, meta }); + observed = await historicalFromIntent(state, intent, options, { blocked: true }); + if (observed.legacy) { + const revalidate = () => historicalFromIntent(state, intent, options, { blocked: true }); + await freeze(observed.legacy.lock.path, intent.source.legacyLockIdentity, revalidate, options); + await freeze(observed.legacy.transactions.path, intent.source.legacyTransactionsIdentity, revalidate, options); + } + await historicalFromIntent(state, intent, options, { blocked: true }); + await installGuard(state, meta, intent, options); + await options.hooks?.afterMigrationGuard?.({ statePath: state, meta }); + if (intent.source.kind === "v2" && await absent(`${state}.journal.v2-retired`, options) === null) { + await historicalFromIntent(state, intent, options); + const source = `${state}.journal`; const destination = `${state}.journal.v2-retired`; + await boundary(options, "before", "source-rename", destination); + const destinationEntry = await absent(destination, options); + if (destinationEntry !== null) { + if (!same(intent.source.sourceIdentity, destinationEntry) || await absent(source, options) !== null) throw new Error("Migration source retirement destination is a conflicting inode."); + } else { + if (!same(intent.source.sourceIdentity, await directory(source, options))) throw new Error("Migration source was replaced before retirement."); + try { await options.renameFile(source, destination); } + catch (error) { if (options.renameFile !== rename || error?.code !== "ENOENT" || !same(intent.source.sourceIdentity, await absent(destination, options))) throw error; } + } + await boundary(options, "after", "source-rename", destination); + if (!same(intent.source.sourceIdentity, await directory(destination, options))) throw new Error("Migration source retirement has a different destination inode."); + await sync(dirname(source), options); + } + const installed = await prepareInstalled(state, meta, intent, options); + await options.hooks?.afterMigrationInstalled?.({ statePath: state, root: installed.root }); + let repaired = false; + for (let attempt = 0; attempt < 32; attempt++) { + try { await format.withConsumerGenerationLock(installed.root, installed.authority, async () => {}, options); repaired = true; break; } + catch (error) { if (!format.isGenerationBusy(error)) throw error; } + } + if (!repaired) throw new Error("Concurrent migration projection repair exceeded its bounded active-owner joins."); + await finalAuthority(state, meta, intent, options); + const current = await format.prepareConsumerGeneration(installed.root, installed.authority, options); + const complete = { schemaVersion: 3, kind: "pylon-consumer-migration-complete", intentSha256: digest(bytes(intent)), rootIdentity: await directory(installed.root, options), genesisSha256: digest(bytes(installed.checkpoint)) }; + await immutable(meta, "complete.json", complete, options); + await options.hooks?.afterMigrationComplete?.({ statePath: state, checkpoint: current.checkpoint }); + await finalAuthority(state, meta, intent, options); + return { epoch: current.checkpoint.epoch, tipSha256: format.generationEpochAuthority(current, options).tip.tipDigest, sourceAuthoritySha256: installed.checkpoint.sourceAuthoritySha256 }; + } + + async function prepareState(statePath, rawOptions = {}) { + if (typeof statePath !== "string" || !statePath) throw new Error("A consumer-local state path is required."); + const options = optionsFor(rawOptions); const state = resolve(statePath); const meta = `${state}.journal-v3`; + await canonicalAncestors(state, options, true); + let intent = await absent(meta, options) === null ? null : await readIntent(meta, state, options); + if (intent === null) { + for (const path of [`${state}.journal`, `${state}.journal.v2-retired`, `${state}.transactions`, `${state}.lock`, `${state}.lock.v1-retired`]) { + if (await absent(path, options) !== null) throw new Error("Historical consumer authority requires explicit migration with acknowledgement that all legacy processes are stopped."); + } + const projection = await file(state, options, options.stateMaxBytes, false); + const observed = { source: null, legacy: null, projection }; + intent = intentFor(state, observed); + await make(meta, options); await make(join(meta, "receipts"), options); + await immutable(meta, "intent.json", intent, options); + await installGuard(state, meta, intent, options); + } else if (intent.source.kind !== "fresh") { + // Normal entry never advances an unfinished legacy migration. + const completed = await file(join(meta, "complete.json"), options, options.metadataMaxBytes, false); + if (completed === null) throw new Error("Interrupted legacy migration requires the explicitly acknowledged migration command."); + const { checkpoint } = await finalAuthority(state, meta, intent, options); + const selected = canonical(await file(join(meta, "root.json"), options)); + if (typeof selected.goal !== "string" || !new RegExp(`^journal-${digest(bytes(intent))}-${UUID}$`).test(selected.goal)) throw new Error("Migration completion has an invalid selected root."); + const root = join(meta, selected.goal); + const expected = { schemaVersion: 3, kind: "pylon-consumer-migration-complete", intentSha256: digest(bytes(intent)), rootIdentity: await directory(root, options), genesisSha256: digest(bytes(checkpoint)) }; + if (!completed.equals(bytes(expected))) throw new Error("Migration completion differs from exact final source and root authority."); + await receiptFor(meta, "complete.json", join(meta, "complete.json"), completed, options, { repair: false }); + } else await installGuard(state, meta, intent, options); + const installed = await prepareInstalled(state, meta, intent, options); + if (intent.source.kind !== "fresh") { + const complete = { schemaVersion: 3, kind: "pylon-consumer-migration-complete", intentSha256: digest(bytes(intent)), rootIdentity: await directory(installed.root, options), genesisSha256: digest(bytes(installed.checkpoint)) }; + await receiptFor(meta, "complete.json", join(meta, "complete.json"), bytes(complete), options); + } + return { ...installed, state, meta, intent, options }; + } + async function withState(statePath, action, rawOptions = {}) { + if (typeof action !== "function") throw new Error("Consumer high-water lock action must be a function."); + const prepared = await prepareState(statePath, rawOptions); + const { root, authority, state, meta, intent, options } = prepared; + const validate = async () => validateMeta(state, meta, intent, (await finalAuthority(state, meta, intent, options)).observed, options); + const result = await format.withConsumerGenerationLock(root, authority, async (path, transaction) => { + await validate(); + const result = await action(path, transaction); + await validate(); + return result; + }, options); + await validate(); + return result; + } + async function rotate(statePath, rawOptions = {}) { + const { root, authority, state, meta, intent, options } = await prepareState(statePath, rawOptions); + const initial = await format.prepareConsumerGeneration(root, authority, options); + const scan = format.generationEpochAuthority(initial, options); + const latest = scan.claims.at(-1); + const terminal = latest && scan.terminals.get(key(latest)); + if (latest?.type === "normal" && (!terminal || terminal.outcome === "commit" && !scan.applied.has(key(latest)))) { + await format.withConsumerGenerationLock(root, authority, async () => {}, options); + const recovered = await format.prepareConsumerGeneration(root, authority, options); + if (recovered.checkpoint.epoch > initial.checkpoint.epoch) { + await validateMeta(state, meta, intent, (await finalAuthority(state, meta, intent, options)).observed, options); + return { epoch: recovered.checkpoint.epoch, tipSha256: format.generationEpochAuthority(recovered, options).tip.tipDigest }; + } + } + const ready = await format.prepareConsumerGeneration(root, authority, options); + const readyScan = format.generationEpochAuthority(ready, options); + if (ready.checkpoint.epoch > 1 && readyScan.tip.length === 0 && readyScan.claims.at(-1)?.type !== "rotation" && ready.retirementCertificate === null) { + const projection = await file(state, options, options.stateMaxBytes, false, 0); + if (!sameBytes(projection, readyScan.tip.tipBytes)) await format.withConsumerGenerationLock(root, authority, async () => {}, options); + await validateMeta(state, meta, intent, (await finalAuthority(state, meta, intent, options)).observed, options); + return { epoch: ready.checkpoint.epoch, tipSha256: readyScan.tip.tipDigest }; + } + // The reserved rotation slot remains usable when the normal frontier is full. + const result = await format.rotateConsumerGeneration(root, authority, options); + await format.withConsumerGenerationLock(root, authority, async () => {}, options); + await validateMeta(state, meta, intent, (await finalAuthority(state, meta, intent, options)).observed, options); + return result; + } + async function inspect(statePath, rawOptions = {}) { if (typeof statePath !== "string" || !statePath) throw new Error("Migration state path is required."); const options = optionsFor(rawOptions); const state = resolve(statePath); await directory(dirname(state), options); - const projection = await file(state, options, options.stateMaxBytes, false); + const projection = await file(state, options, options.stateMaxBytes, false, 0); const journal = await absent(`${state}.journal`, options); let legacyProjection = projection; if (journal) { @@ -280,10 +878,15 @@ export function createConsumerMigrationApi(format) { if (!bytes(expected).equals(await file(`${state}.lock`, options))) throw new Error("Historical prior-retired guard is not exact."); } } + if (lock && !lock.isDirectory()) { + const expected = { schemaVersion: 1, kind: "pylon-consumer-legacy-lock-guard", statePathSha256: digest(Buffer.from(state)) }; + if (!bytes(expected).equals(await file(`${state}.lock`, options))) throw new Error("Historical regular guard is not exact."); + } + if (await absent(`${state}.journal.v2-retired`, options) !== null) throw new Error("Historical retained source requires its existing migration proof."); const source = journal ? await readV2(state, `${state}.journal`, projection, { ...options, maxJournalBytes: MAX_BYTES }, legacy) : legacy; if (!source) throw new Error("No historical authority exists."); return { source, legacy, projection }; } - return { inspect, readV1, readV2 }; + return { inspect, readV1, readV2, migrate, withState, rotate }; } diff --git a/scripts/migrate-pylon-consumer-journal.mjs b/scripts/migrate-pylon-consumer-journal.mjs index f41413897a..3e5c572925 100755 --- a/scripts/migrate-pylon-consumer-journal.mjs +++ b/scripts/migrate-pylon-consumer-journal.mjs @@ -5,14 +5,17 @@ import { resolve } from "node:path"; import { migrateConsumerStateJournal } from "./lib/pylon-consumer-lock.mjs"; function parseArgs(args) { - if (args.length !== 2 || args[0] !== "--state" || !args[1] || args[1].startsWith("--")) { - throw new Error("Usage: migrate-pylon-consumer-journal --state "); - } - return resolve(args[1]); + const remaining = [...args]; + const acknowledgement = remaining.indexOf("--acknowledge-legacy-processes-stopped"); + if (acknowledgement !== -1) remaining.splice(acknowledgement, 1); + if (acknowledgement === -1 || remaining.length !== 2 || remaining[0] !== "--state" || !remaining[1] || remaining[1].startsWith("--")) { + throw new Error("Usage: migrate-pylon-consumer-journal --state --acknowledge-legacy-processes-stopped"); + } + return resolve(remaining[1]); } try { - const result = await migrateConsumerStateJournal(parseArgs(process.argv.slice(2))); + const result = await migrateConsumerStateJournal(parseArgs(process.argv.slice(2)), { acknowledgeLegacyProcessesStopped: true }); console.log(JSON.stringify({ journalEpoch: result.epoch, tipSha256: result.tipSha256, diff --git a/scripts/pylon-generation-migration.test.mjs b/scripts/pylon-generation-migration.test.mjs index 22d53509a5..b9991604ec 100644 --- a/scripts/pylon-generation-migration.test.mjs +++ b/scripts/pylon-generation-migration.test.mjs @@ -1,10 +1,10 @@ import assert from "node:assert/strict"; import { randomUUID } from "node:crypto"; -import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, writeFile } from "node:fs/promises"; +import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, symlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; -import { inspectConsumerMigrationSource } from "./lib/pylon-consumer-lock.mjs"; +import { inspectConsumerMigrationSource, migrateConsumerGenerationJournal, withConsumerGenerationStateLock, rotateConsumerGenerationStateJournal } from "./lib/pylon-consumer-lock.mjs"; import { generationBytes as bytes, generationDigest as digest, GENERATION_ZERO as ZERO } from "./lib/pylon-generation-format.mjs"; import * as protectedV2 from "./fixtures/protected-publication-v2/pylon-consumer-lock.mjs"; @@ -128,3 +128,368 @@ test("migration historical reader never silently excludes an unauthenticated blo await put(join(clean.source.root, clean.source.headEpoch, "claim-9999999999999999.json"), { schemaVersion: 3, kind: "forged" }); await assert.rejects(inspectConsumerMigrationSource(native), /claim.*malformed/); }); + + +test("v3 migration acknowledgement precedes every filesystem operation", async () => { + let operations = 0; + const forbidden = () => { operations++; throw new Error("must not execute"); }; + await assert.rejects(migrateConsumerGenerationJournal("/absent/test/state", { lstatEntry: forbidden, makeDirectory: forbidden, openFile: forbidden }), /acknowledgement/); + assert.equal(operations, 0); +}); + +test("v3 migration preserves original v1 inodes and freezes both directories", async (t) => { + const { state, value } = await v1(t, true); + const before = await lstat(`${state}.lock`); const txBefore = await lstat(`${state}.transactions`); + const result = await migrateConsumerGenerationJournal(state, { acknowledgeLegacyProcessesStopped: true, ...runtime }); + assert.equal(result.tipSha256, digest(value)); + for (const [path, original] of [[`${state}.lock`, before], [`${state}.transactions`, txBefore]]) { + const actual = await lstat(path); assert.equal(actual.ino, original.ino); assert.equal(actual.dev, original.dev); assert.equal(actual.mode & 0o7777, 0o500); + } + assert.deepEqual(await readFile(state), value); + const resumed = await migrateConsumerGenerationJournal(state, { acknowledgeLegacyProcessesStopped: true, ...runtime }); + assert.equal(resumed.tipSha256, digest(value)); + await chmod(`${state}.lock`, 0o700); await chmod(`${state}.transactions`, 0o700); +}); + +test("v3 migration retains v2 source inode and resumes its canonical root", async (t) => { + const state = await fixture(t); + await protectedV2.withConsumerStateLock(state, async (_path, tx) => tx.commitState("native"), runtime); + const original = await lstat(`${state}.journal`); + const options = { acknowledgeLegacyProcessesStopped: true, ...runtime }; + const result = await migrateConsumerGenerationJournal(state, options); + assert.equal(result.tipSha256, digest(Buffer.from("native"))); + assert.equal((await lstat(`${state}.journal.v2-retired`)).ino, original.ino); + await assert.rejects(lstat(`${state}.journal`), { code: "ENOENT" }); + const resumed = await migrateConsumerGenerationJournal(state, options); + assert.equal(resumed.tipSha256, result.tipSha256); +}); + +test("v3 migration blocker stops exact protected clients before help and projection repair", async (t) => { + for (const family of ["v1", "prior-retired-v1", "v2", "v1-v2", "prior-retired-v1-v2"]) { + let state; + if (family === "v2") state = await fixture(t); + else { + ({ state } = await v1(t)); + if (family.startsWith("prior-retired")) { + await rename(`${state}.lock`, `${state}.lock.v1-retired`); + await put(`${state}.lock`, { schemaVersion: 1, kind: "pylon-consumer-legacy-lock-guard", statePathSha256: digest(Buffer.from(state)) }); + } + if (family.endsWith("-v2")) await protectedV2.migrateConsumerStateJournal(state, runtime); + } + if (family.endsWith("v2")) { + if (family === "v2") await protectedV2.withConsumerStateLock(state, async (_path, tx) => tx.commitState("before"), runtime); + const stop = new Error("retained incomplete committed operation"); + await assert.rejects(protectedV2.withConsumerStateLock(state, async (_path, tx) => tx.commitState("after"), { + ...runtime, hooks: { afterCommitDecision: () => { throw stop; } }, + }), (error) => error === stop); + } + let checkpoints = 0; + const proveStopped = async () => { + checkpoints++; + const before = await readFile(state); const original = await lstat(state); + let callbacks = 0; let helperWrites = 0; + const oldOptions = { ...runtime, hooks: { beforeProjectionWrite: () => { helperWrites++; }, afterMetadataLink: ({ kind }) => { + if (["transition", "applied", "terminal-commit"].includes(kind)) helperWrites++; + } } }; + await assert.rejects(protectedV2.withConsumerStateLock(state, async () => { callbacks++; }, oldOptions)); + await assert.rejects(protectedV2.rotateConsumerStateJournal(state, oldOptions)); + await assert.rejects(protectedV2.migrateConsumerStateJournal(state, oldOptions)); + assert.equal(callbacks, 0, family); assert.equal(helperWrites, 0, family); + assert.deepEqual(await readFile(state), before, family); assert.equal((await lstat(state)).ino, original.ino, family); + }; + await migrateConsumerGenerationJournal(state, { acknowledgeLegacyProcessesStopped: true, ...runtime, hooks: { afterMigrationBlocker: proveStopped, afterMigrationGuard: proveStopped } }); + assert.equal(checkpoints, 2, family); + for (const path of [`${state}.lock`, `${state}.lock.v1-retired`, `${state}.transactions`]) { + try { const stat = await lstat(path); if (stat.isDirectory()) await chmod(path, 0o700); } catch (error) { if (error.code !== "ENOENT") throw error; } + } + } +}); + +test("v3 migration rejects a foreign old bootstrap journal after source retirement", async (t) => { + const state = await fixture(t); + await protectedV2.withConsumerStateLock(state, async (_path, tx) => tx.commitState("original"), runtime); + const options = { acknowledgeLegacyProcessesStopped: true, ...runtime }; + const injected = new Error("cut after protected bootstrap refusal"); + let foreign; + await assert.rejects(migrateConsumerGenerationJournal(state, { ...options, hooks: { migrationBoundary: async ({ phase, operation }) => { + if (phase !== "after" || operation !== "source-rename") return; + await assert.rejects(protectedV2.withConsumerStateLock(state, async () => assert.fail("old callback must never run"), runtime)); + foreign = await lstat(`${state}.journal`); + throw injected; + } } }), (error) => error === injected); + await assert.rejects(migrateConsumerGenerationJournal(state, options), /foreign historical journal inode/); + assert.equal((await lstat(`${state}.journal`)).ino, foreign.ino); + assert.equal((await readFile(state)).toString(), "original"); +}); + +test("v3 migration detects same-byte source replacement on canonical resume", async (t) => { + const state = await fixture(t); + await protectedV2.withConsumerStateLock(state, async (_path, tx) => tx.commitState("original"), runtime); + const options = { acknowledgeLegacyProcessesStopped: true, ...runtime }; + await migrateConsumerGenerationJournal(state, options); + const source = `${state}.journal.v2-retired`; + const name = (await readdir(source)).find((entry) => entry.startsWith("checkpoint-")); + const data = await readFile(join(source, name)); + await rename(join(source, name), `${state}.old-checkpoint`); + await writeFile(join(source, name), data, { mode: 0o600 }); + await assert.rejects(migrateConsumerGenerationJournal(state, options), /pre-block authority.*inode/); +}); + + +test("v3 fresh state keeps exact genesis independent of later commits and rotations", async (t) => { + const state = await fixture(t); + await writeFile(state, "initial", { mode: 0o600 }); + await withConsumerGenerationStateLock(state, async (_path, transaction) => { + assert.equal(transaction.readStateBytes().toString(), "initial"); await transaction.commitState("later"); + }, runtime); + const intentPath = `${state}.journal-v3/intent.json`; const intent = await readFile(intentPath); + await rotateConsumerGenerationStateJournal(state, runtime); + await withConsumerGenerationStateLock(state, async (_path, transaction) => assert.equal(transaction.readStateBytes().toString(), "later"), runtime); + assert.deepEqual(await readFile(intentPath), intent); +}); + +test("v3 normal entry requires explicit migration and preserves callback error staging", async (t) => { + const state = await fixture(t); + await protectedV2.withConsumerStateLock(state, async (_path, tx) => tx.commitState("historical"), runtime); + let called = 0; + await assert.rejects(withConsumerGenerationStateLock(state, async () => { called++; }, runtime), /explicit migration/); + assert.equal(called, 0); + await migrateConsumerGenerationJournal(state, { ...runtime, acknowledgeLegacyProcessesStopped: true }); + const failure = new Error("callback failed after staging"); + await assert.rejects(withConsumerGenerationStateLock(state, async (_path, tx) => { await tx.commitState("must not commit"); throw failure; }, runtime), (error) => error === failure); + await withConsumerGenerationStateLock(state, async (_path, tx) => assert.equal(tx.readStateBytes().toString(), "historical"), runtime); +}); + +test("v3 two migration helpers join one installed source and canonical root", async (t) => { + const state = await fixture(t); + await protectedV2.withConsumerStateLock(state, async (_path, tx) => tx.commitState("historical"), runtime); + const options = { ...runtime, acknowledgeLegacyProcessesStopped: true }; + const attempts = await Promise.allSettled([migrateConsumerGenerationJournal(state, options), migrateConsumerGenerationJournal(state, options)]); + const results = attempts.filter((attempt) => attempt.status === "fulfilled").map((attempt) => attempt.value); + assert.ok(results.length >= 1, attempts.map((attempt) => attempt.reason?.stack).join("\n")); + for (const attempt of attempts) if (attempt.status === "rejected") assert.match(attempt.reason.message, /changed|disappeared|ENOENT|inode|fenced|authority|actively locked|receipt|publication|conflicting exact bytes/); + const resumed = await migrateConsumerGenerationJournal(state, options); + for (const result of results) assert.deepEqual(result, resumed); + const roots = (await readdir(`${state}.journal-v3`)).filter((name) => name.startsWith("journal-")); + const selected = JSON.parse(await readFile(`${state}.journal-v3/root.json`)).goal; + assert.ok(roots.includes(selected)); + for (const root of roots.filter((name) => name !== selected)) assert.deepEqual(await readdir(`${state}.journal-v3/${root}`), []); +}); + +async function historicalFamily(t, family) { + let state; + if (family === "v2") state = await fixture(t); + else { + ({ state } = await v1(t)); + if (family.startsWith("prior-retired")) { + await rename(`${state}.lock`, `${state}.lock.v1-retired`); + await put(`${state}.lock`, { schemaVersion: 1, kind: "pylon-consumer-legacy-lock-guard", statePathSha256: digest(Buffer.from(state)) }); + } + if (family.endsWith("-v2")) await protectedV2.migrateConsumerStateJournal(state, runtime); + } + if (family.endsWith("v2")) { + if (family === "v2") await protectedV2.withConsumerStateLock(state, async (_path, tx) => tx.commitState("before"), runtime); + const failure = new Error("historical committed incomplete decision"); + await assert.rejects(protectedV2.withConsumerStateLock(state, async (_path, tx) => tx.commitState("after"), { + ...runtime, hooks: { afterCommitDecision: () => { throw failure; } }, + }), (error) => error === failure); + } + return state; +} +async function restoreFixtureModes(state) { + for (const path of [`${state}.lock`, `${state}.lock.v1-retired`, `${state}.transactions`]) { + try { const stat = await lstat(path); if (stat.isDirectory()) await chmod(path, 0o700); } + catch (error) { if (error.code !== "ENOENT") throw error; } + } +} +async function protectedDenial(state) { + const before = await readFile(state); const original = await lstat(state); + let callbacks = 0; let writes = 0; + const oldOptions = { ...runtime, hooks: { beforeProjectionWrite: () => { writes++; }, afterMetadataLink: ({ kind }) => { + if (["transition", "applied", "terminal-commit"].includes(kind)) writes++; + } } }; + await assert.rejects(protectedV2.withConsumerStateLock(state, async () => { callbacks++; }, oldOptions)); + await assert.rejects(protectedV2.migrateConsumerStateJournal(state, oldOptions)); + await assert.rejects(protectedV2.rotateConsumerStateJournal(state, oldOptions)); + assert.equal(callbacks, 0); assert.equal(writes, 0); + assert.deepEqual(await readFile(state), before); assert.equal((await lstat(state)).ino, original.ino); +} + +test("protected old clients cannot help at either freeze, source retirement, installation, or canonical completion", async (t) => { + for (const family of ["v1", "prior-retired-v1", "v2", "v1-v2", "prior-retired-v1-v2"]) { + const cuts = ["installed", "complete"]; + if (family !== "v2") cuts.push("lock-frozen", "transactions-frozen"); + if (family.endsWith("v2")) cuts.push("source-retired"); + for (const cut of cuts) { + const state = await historicalFamily(t, family); const failure = new Error(`${family}/${cut}`); let reached = false; + const stop = async () => { reached = true; await protectedDenial(state); throw failure; }; + await assert.rejects(migrateConsumerGenerationJournal(state, { ...runtime, acknowledgeLegacyProcessesStopped: true, hooks: { + migrationBoundary: async ({ phase, operation, path }) => { + if (phase !== "after") return; + if (cut === "source-retired" && operation === "source-rename") await stop(); + if (operation === "freeze" && (cut === "transactions-frozen" ? path.endsWith(".transactions") : cut === "lock-frozen" && !path.endsWith(".transactions"))) await stop(); + }, + afterMigrationInstalled: cut === "installed" ? stop : undefined, + afterMigrationComplete: cut === "complete" ? stop : undefined, + } }), (error) => error === failure); + assert.equal(reached, true, `${family}/${cut}`); + await restoreFixtureModes(state); + } + } +}); + +test("migration source retirement joins the independently authenticated same-inode helper", async (t) => { + const state = await historicalFamily(t, "v2"); let joined; + const options = { ...runtime, acknowledgeLegacyProcessesStopped: true }; + const result = await migrateConsumerGenerationJournal(state, { ...options, hooks: { migrationBoundary: async ({ phase, operation }) => { + if (phase === "before" && operation === "source-rename") joined = await migrateConsumerGenerationJournal(state, options); + } } }); + assert.deepEqual(result, joined); +}); + +test("migration resumes every freeze and source-retirement hook error without classifying it as success", async (t) => { + for (const [family, operation, phase] of [["v1", "freeze", "before"], ["v1", "freeze", "after"], ["v2", "source-rename", "before"], ["v2", "source-rename", "after"]]) { + const state = await historicalFamily(t, family); + const options = { ...runtime, acknowledgeLegacyProcessesStopped: true }; + const failure = Object.assign(new Error(`injected ${operation}/${phase}`), { code: "ENOENT" }); let fired = false; + await assert.rejects(migrateConsumerGenerationJournal(state, { ...options, hooks: { migrationBoundary: (event) => { + if (!fired && event.operation === operation && event.phase === phase) { fired = true; throw failure; } + } } }), (error) => error === failure); + assert.equal(fired, true); + const completed = await migrateConsumerGenerationJournal(state, options); + assert.equal(completed.tipSha256, digest(Buffer.from(family === "v1" ? "committed" : "after"))); + await restoreFixtureModes(state); + } +}); + +test("migration rejects altered blocker commitments and a replaced canonical root", async (t) => { + for (const field of ["authoritySha256", "tipSha256", "legacyMarkerSha256", "sourceIdentity"]) { + const state = await historicalFamily(t, "v2"); const failure = new Error("blocker cut"); + const options = { ...runtime, acknowledgeLegacyProcessesStopped: true }; + await assert.rejects(migrateConsumerGenerationJournal(state, { ...options, hooks: { afterMigrationBlocker: () => { throw failure; } } }), (error) => error === failure); + const epoch = (await readdir(`${state}.journal`)).find((name) => name.startsWith("epoch-")); + const path = join(`${state}.journal`, epoch, "claim-9999999999999999.json"); + const blocker = JSON.parse(await readFile(path)); blocker.source[field] = field === "sourceIdentity" ? { dev: 0, ino: 0 } : "f".repeat(64); + await put(path, blocker); + await assert.rejects(migrateConsumerGenerationJournal(state, options), /blocker differs/); + } + const state = await fixture(t); + await withConsumerGenerationStateLock(state, async (_path, tx) => tx.commitState("current"), runtime); + const meta = `${state}.journal-v3`; const root = join(meta, (await readdir(meta)).find((name) => name.startsWith("journal-"))); + await rename(root, `${state}.saved-root`); await mkdir(root, { mode: 0o700 }); + await assert.rejects(withConsumerGenerationStateLock(state, async () => assert.fail("replacement must not run"), runtime), /different inode/); +}); + +test("fresh v3 entry creates private nested parents and rejects symlink ancestors", async (t) => { + const state = await fixture(t); const nested = join(`${state}.private`, "nested", "state.json"); + await withConsumerGenerationStateLock(nested, async (_path, tx) => tx.commitState("nested"), runtime); + assert.equal((await readFile(nested)).toString(), "nested"); + assert.equal((await lstat(`${state}.private`)).mode & 0o7777, 0o700); + await symlink(`${state}.private`, `${state}.linked`); + await assert.rejects(withConsumerGenerationStateLock(join(`${state}.linked`, "other.json"), async () => {}, runtime), /canonical real directory/); +}); + +test("migration receipt and root construction cuts resume only through exact durable authority", async (t) => { + const cases = [ + ["receipt-create", "after", null], ["file-sync", "before", null], ["file-sync", "after", null], + ["immutable-link", "before", "intent.json"], ["immutable-link", "after", "intent.json"], + ["receipt-rename", "before", null], ["receipt-rename", "after", null], + ["guard-rename", "before", null], ["guard-rename", "after", null], + ["mkdir", "after", "journal-"], ["immutable-link", "after", "root.json"], + ["immutable-link", "after", "complete.json"], + ]; + for (const [operation, phase, suffix] of cases) { + const state = await historicalFamily(t, "v2"); + const options = { ...runtime, acknowledgeLegacyProcessesStopped: true, processKill: () => { throw Object.assign(new Error("dead receipt fixture"), { code: "ESRCH" }); } }; + const failure = Object.assign(new Error(`cut ${operation}/${phase}/${suffix}`), { code: "EIO" }); let fired = false; + await assert.rejects(migrateConsumerGenerationJournal(state, { ...options, hooks: { migrationBoundary: (event) => { + if (!fired && event.operation === operation && event.phase === phase && (suffix === null || (suffix === "journal-" ? event.path.split("/").at(-1).startsWith(suffix) : event.path.endsWith(suffix)))) { fired = true; throw failure; } + } } }), (error) => error === failure); + assert.equal(fired, true); + let callbacks = 0; + await assert.rejects(withConsumerGenerationStateLock(state, async () => { callbacks++; }, runtime)); + assert.equal(callbacks, 0); + const result = await migrateConsumerGenerationJournal(state, options); + assert.equal(result.tipSha256, digest(Buffer.from("after"))); + } +}); + +test("migration preserves empty projection snapshots and full retained rotation authority", async (t) => { + for (const family of ["v1", "v2", "v1-v2"]) { + const state = await historicalFamily(t, family); + await writeFile(state, Buffer.alloc(0), { mode: 0o600 }); + const result = await migrateConsumerGenerationJournal(state, { ...runtime, acknowledgeLegacyProcessesStopped: true }); + assert.equal(result.tipSha256, digest(Buffer.from(family === "v1" ? "committed" : "after"))); + assert.equal(JSON.parse(await readFile(`${state}.journal-v3/intent.json`)).projectionBase64, ""); + await restoreFixtureModes(state); + } + const state = await fixture(t); + await protectedV2.withConsumerStateLock(state, async (_path, tx) => tx.commitState("rotated"), runtime); + await protectedV2.rotateConsumerStateJournal(state, runtime); + const result = await migrateConsumerGenerationJournal(state, { ...runtime, acknowledgeLegacyProcessesStopped: true }); + assert.equal(result.tipSha256, digest(Buffer.from("rotated"))); + await withConsumerGenerationStateLock(state, async (_path, tx) => tx.commitState("v3 later"), runtime); + await rotateConsumerGenerationStateJournal(state, runtime); + await withConsumerGenerationStateLock(state, async (_path, tx) => assert.equal(tx.readStateBytes().toString(), "v3 later"), runtime); + assert.equal((await readdir(`${state}.journal.v2-retired`)).filter((name) => name.startsWith("checkpoint-")).length, 2); +}); + +test("historical malformed guard and live or uncertain unresolved owners fail before source mutation", async (t) => { + const native = await historicalFamily(t, "v2"); + await put(`${native}.lock`, { schemaVersion: 1, kind: "forged" }); + await assert.rejects(migrateConsumerGenerationJournal(native, { ...runtime, acknowledgeLegacyProcessesStopped: true }), /guard is not exact/); + await assert.rejects(lstat(`${native}.journal-v3`), { code: "ENOENT" }); + for (const alive of [true, false]) { + const { state } = await v1(t, true); const original = await readdir(`${state}.lock`); + const options = { ...runtime, acknowledgeLegacyProcessesStopped: true, processKill: () => { + if (!alive) throw Object.assign(new Error("uncertain owner"), { code: "EPERM" }); + } }; + await assert.rejects(migrateConsumerGenerationJournal(state, options), /live or uncertain/); + assert.deepEqual(await readdir(`${state}.lock`), original); assert.deepEqual(await readdir(`${state}.transactions`), []); + } +}); + +test("direct v1 recovery rejects additional valid authority inserted during permitted completion", async (t) => { + const { state } = await v1(t, true); let inserted = false; + await assert.rejects(migrateConsumerGenerationJournal(state, { ...runtime, acknowledgeLegacyProcessesStopped: true, hooks: { migrationBoundary: async ({ phase, operation }) => { + if (inserted || phase !== "after" || operation !== "immutable-link") return; + inserted = true; + const token = randomUUID(); + await put(`${state}.lock/claim-0000000000000002.json`, { schemaVersion: 1, generation: 2, token, ownerPid: 2_000_000_000, createdAtMs: 0 }); + await put(`${state}.lock/heartbeat-0000000000000002-${token}.json`, { schemaVersion: 1, generation: 2, token, refreshedAtMs: 0 }); + await put(`${state}.lock/terminal-0000000000000002-${token}.json`, { schemaVersion: 1, generation: 2, token, outcome: "released" }); + } } }), /unapproved authority changes/); + assert.equal(inserted, true); + await assert.rejects(lstat(`${state}.journal-v3/intent.json`), { code: "ENOENT" }); +}); + +test("exclusive construction never adopts a foreign empty inode or an unrecorded crash directory", async (t) => { + for (const phase of ["before", "after"]) { + const state = await historicalFamily(t, "v2"); let original; let foreign; let target; + await assert.rejects(migrateConsumerGenerationJournal(state, { ...runtime, acknowledgeLegacyProcessesStopped: true, hooks: { migrationBoundary: async (event) => { + if (target || event.phase !== phase || event.operation !== "mkdir" || !event.path.split("/").at(-1).startsWith("journal-")) return; + target = event.path; + if (phase === "after") { original = `${state}.unrecorded-original`; await rename(target, original); } + await mkdir(target, { mode: 0o700 }); foreign = await lstat(target); + } } }), phase === "before" ? { code: "EEXIST" } : /changed inode/); + assert.equal((await lstat(target)).ino, foreign.ino); + const result = await migrateConsumerGenerationJournal(state, { ...runtime, acknowledgeLegacyProcessesStopped: true }); + assert.equal(result.tipSha256, digest(Buffer.from("after"))); + const selected = JSON.parse(await readFile(`${state}.journal-v3/root.json`)); + assert.notEqual(selected.identity.ino, foreign.ino); + assert.deepEqual(await readdir(target), []); + if (original) assert.deepEqual(await readdir(original), []); + } +}); + +test("native v2 initial checkpoint cut gains an authenticated epoch blocker before old-client admission", async (t) => { + const state = await fixture(t); const failure = new Error("initial checkpoint cut"); + await assert.rejects(protectedV2.withConsumerStateLock(state, async () => {}, { ...runtime, hooks: { afterMetadataDirectorySync: ({ kind }) => { if (kind === "checkpoint") throw failure; } } }), (error) => error === failure); + assert.equal((await inspectConsumerMigrationSource(state)).source.epochs.size, 0); + let callbacks = 0; + await migrateConsumerGenerationJournal(state, { ...runtime, acknowledgeLegacyProcessesStopped: true, processKill: () => { throw Object.assign(new Error("dead bootstrap fixture"), { code: "ESRCH" }); }, hooks: { afterMigrationBlocker: async () => { + await assert.rejects(protectedV2.withConsumerStateLock(state, async () => { callbacks++; }, runtime)); + await assert.rejects(protectedV2.rotateConsumerStateJournal(state, runtime)); + } } }); + assert.equal(callbacks, 0); +}); diff --git a/scripts/pylon-public-state.test.mjs b/scripts/pylon-public-state.test.mjs new file mode 100644 index 0000000000..878761eb58 --- /dev/null +++ b/scripts/pylon-public-state.test.mjs @@ -0,0 +1,253 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { chmodSync, lstatSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { test } from "node:test"; +import { recordPreviewHighWater } from "./verify-pylon-preview-history.mjs"; +import { verifyStableHistoryWithState } from "./verify-pylon-stable-history.mjs"; +import { withConsumerStateLock, migrateConsumerStateJournal, rotateConsumerStateJournal } from "./lib/pylon-consumer-lock.mjs"; +import { createReleaseManifest, PYLON_RELEASE_NODE_VERSION, PYLON_RELEASE_NPM_VERSION } from "./lib/pylon-release.mjs"; +import { canonicalJson, createPreviewManifest, createStableManifest, sha256Bytes } from "./lib/pylon-publication.mjs"; + +const source = { + repository: "https://github.com/pylon-code/prime-agent", + commit: "0123456789abcdef0123456789abcdef01234567", + tree: "89abcdef0123456789abcdef0123456789abcdef", +}; +const version = "0.8.1"; +const invocation = { + sequenceEpoch: 1, + sequence: 17, + workflowRunId: "33428882721", + publicationPolicyRevision: 1, +}; + +function fakeReleaseManifest() { + return createReleaseManifest({ + source, + version, + toolchain: { node: PYLON_RELEASE_NODE_VERSION, npm: PYLON_RELEASE_NPM_VERSION }, + lockfileSha256: "a".repeat(64), + artifacts: [ + ["prime-agent", "pylon-prime-agent-0.8.1.tgz", "d"], + ["@earendil-works/pi-ai", "pylon-prime-agent-ai-0.8.1.tgz", "a"], + ["@earendil-works/pi-agent-core", "pylon-prime-agent-core-0.8.1.tgz", "b"], + ["@earendil-works/pi-tui", "pylon-prime-agent-tui-0.8.1.tgz", "c"], + ].map(([packageName, file, byte]) => { + const bytes = Buffer.from(byte); + return { + package: packageName, + file, + size: bytes.byteLength, + sha256: byte.repeat(64), + sha512: byte.repeat(128), + }; + }), + }); +} + +function manifests() { + const release = fakeReleaseManifest(); + const releaseBytes = Buffer.from(`${JSON.stringify(release, null, 2)}\n`); + const preview = createPreviewManifest(release, releaseBytes, invocation); + const previewBytes = Buffer.from(canonicalJson(preview)); + return { release, releaseBytes, preview, previewBytes }; +} + +function firstStable() { + const { preview, previewBytes } = manifests(); + return createStableManifest({ + previewManifest: preview, + previewManifestBytes: previewBytes, + sequence: 1, + previous: null, + promotion: { kind: "promote", policyCommit: source.commit, policyTree: source.tree, publicationPolicyRevision: 1 }, + }); +} + +function secondStable(previous = firstStable(), options = {}) { + const { preview, previewBytes } = manifests(); + const sequence = 2; + const revocation = { + stableTag: previous.tag, + buildTag: previous.build.previewTag, + reason: "security-withdrawal", + revokedBySequence: sequence, + }; + return createStableManifest({ + previewManifest: preview, + previewManifestBytes: previewBytes, + sequence, + previous: { tag: previous.tag, sha256: sha256Bytes(Buffer.from(canonicalJson(previous))) }, + revocations: options.withdraw ? [revocation] : [], + promotion: options.withdraw ? { kind: "withdraw", policyCommit: source.commit, policyTree: source.tree, publicationPolicyRevision: 1, revocation } : { kind: "promote", policyCommit: source.commit, policyTree: source.tree, publicationPolicyRevision: 1 }, + }); +} + + +test("current public v3 consumer preview high-water allows gaps but rejects rollback and same-sequence equivocation", async () => { + const fixture = realpathSync(mkdtempSync(join(tmpdir(), "pylon-preview-state-"))); + try { + const { preview, previewBytes } = manifests(); + const statePath = join(fixture, "consumer", "nested", "preview.json"); + await assert.rejects(() => recordPreviewHighWater(preview, previewBytes, { statePath }), /--initialize/); + assert.equal((await recordPreviewHighWater(preview, previewBytes, { statePath, initialize: true })).advanced, true); + assert.equal((await recordPreviewHighWater(preview, previewBytes, { statePath })).advanced, false); + const later = structuredClone(preview); + later.sequence += 3; + later.workflowRunId = String(Number(later.workflowRunId) + 3); + assert.equal((await recordPreviewHighWater(later, Buffer.from(canonicalJson(later)), { statePath })).state.highWater.sequence, later.sequence); + await assert.rejects(() => recordPreviewHighWater(preview, previewBytes, { statePath }), /older/); + const equivocation = structuredClone(later); + equivocation.build.releaseManifest.sha256 = "f".repeat(64); + await assert.rejects( + () => recordPreviewHighWater(equivocation, Buffer.from(canonicalJson(equivocation)), { statePath }), + /equivocates/, + ); + } finally { + rmSync(fixture, { recursive: true, force: true }); + } +}); + +test("current public v3 consumer stable high-water requires explicit initialization, is idempotent, and advances atomically", async () => { + const fixture = realpathSync(mkdtempSync(join(tmpdir(), "pylon-stable-state-"))); + try { + const first = firstStable(); + const second = secondStable(first); + const firstPath = join(fixture, "first.json"); + const secondPath = join(fixture, "second.json"); + const statePath = join(fixture, "consumer", "nested", "stable.json"); + writeFileSync(firstPath, canonicalJson(first)); + writeFileSync(secondPath, canonicalJson(second)); + await assert.rejects(() => verifyStableHistoryWithState([firstPath], { statePath }), /--initialize/); + const initialized = await verifyStableHistoryWithState([firstPath], { statePath, initialize: true }); + assert.equal(initialized.advanced, true); + assert.equal(initialized.state.highWater.sequence, 1); + const witnessedBytes = readFileSync(statePath, "utf8"); + const repeated = await verifyStableHistoryWithState([firstPath], { statePath }); + assert.equal(repeated.advanced, false); + assert.equal(readFileSync(statePath, "utf8"), witnessedBytes); + const advanced = await verifyStableHistoryWithState([firstPath, secondPath], { statePath }); + assert.equal(advanced.advanced, true); + assert.equal(advanced.state.highWater.sequence, 2); + writeFileSync(statePath, witnessedBytes); + const repairedRollback = await verifyStableHistoryWithState([firstPath, secondPath], { statePath }); + assert.equal(repairedRollback.state.highWater.sequence, 2, "the immutable transaction tip outranks a rolled-back projection"); + assert.equal(JSON.parse(readFileSync(statePath, "utf8")).highWater.sequence, 2); + rmSync(statePath); + await verifyStableHistoryWithState([firstPath, secondPath], { statePath }); + assert.equal(JSON.parse(readFileSync(statePath, "utf8")).highWater.sequence, 2, "a deleted projection is repaired from the journal"); + writeFileSync(statePath, ""); + await verifyStableHistoryWithState([firstPath, secondPath], { statePath }); + assert.equal(JSON.parse(readFileSync(statePath, "utf8")).highWater.sequence, 2, "an empty projection is repaired from the journal"); + const legacyPath = join(fixture, "legacy.json"); + writeFileSync(legacyPath, canonicalJson(initialized.state), { mode: 0o600 }); + const migrated = await verifyStableHistoryWithState([firstPath], { statePath: legacyPath }); + assert.equal(migrated.advanced, false); + assert.equal(readdirSync(`${legacyPath}.journal-v3`).some((name) => name.startsWith("journal-")), true); + await assert.rejects(() => verifyStableHistoryWithState([firstPath, secondPath], { statePath, initialize: true }), /cannot reset/); + } finally { + rmSync(fixture, { recursive: true, force: true }); + } +}); + +test("current public v3 consumer stable high-water rejects rollback and a rewritten witnessed sequence", async () => { + const fixture = realpathSync(mkdtempSync(join(tmpdir(), "pylon-stable-state-"))); + try { + const first = firstStable(); + const second = secondStable(first); + const firstPath = join(fixture, "first.json"); + const secondPath = join(fixture, "second.json"); + const statePath = join(fixture, "stable.json"); + writeFileSync(firstPath, canonicalJson(first)); + writeFileSync(secondPath, canonicalJson(second)); + await verifyStableHistoryWithState([firstPath, secondPath], { statePath, initialize: true }); + await assert.rejects(() => verifyStableHistoryWithState([firstPath], { statePath }), /older than/); + const rewrittenFirst = structuredClone(first); + rewrittenFirst.promotion.policyTree = "f".repeat(40); + writeFileSync(firstPath, canonicalJson(rewrittenFirst)); + await assert.rejects(() => verifyStableHistoryWithState([firstPath], { statePath }), /older than|rewrites/); + const rewrittenSecond = createStableManifest({ + previewManifest: manifests().preview, + previewManifestBytes: manifests().previewBytes, + sequence: 2, + previous: { tag: rewrittenFirst.tag, sha256: sha256Bytes(Buffer.from(canonicalJson(rewrittenFirst))) }, + promotion: { kind: "promote", policyCommit: source.commit, policyTree: "e".repeat(40), publicationPolicyRevision: 1 }, + }); + writeFileSync(secondPath, canonicalJson(rewrittenSecond)); + await assert.rejects(() => verifyStableHistoryWithState([firstPath, secondPath], { statePath }), /rewrites/); + } finally { + rmSync(fixture, { recursive: true, force: true }); + } +}); + +test("current public API and CLI require acknowledgement without filesystem mutation", async () => { + const fixture = realpathSync(mkdtempSync(join(tmpdir(), "pylon-public-ack-"))); + try { + const state = join(fixture, "missing", "state.json"); + await assert.rejects(migrateConsumerStateJournal(state), /acknowledgement/); + for (const args of [["--state", state], ["--state", state, "--acknowledge-legacy-processes-stopped", "--unknown"]]) { + const result = spawnSync(process.execPath, [resolve("scripts/migrate-pylon-consumer-journal.mjs"), ...args], { encoding: "utf8" }); + assert.equal(result.status, 1); assert.match(result.stderr, /Usage:/); + assert.deepEqual(readdirSync(fixture), []); + } + } finally { rmSync(fixture, { recursive: true, force: true }); } +}); + +test("current public operations stage atomically, reject active owners, preserve failures, and rotate exact genesis", async () => { + const fixture = realpathSync(mkdtempSync(join(tmpdir(), "pylon-public-operations-"))); + const state = join(fixture, "state.json"); + const options = { stateMaxBytes: 1024, startHeartbeat: () => async () => {} }; + try { + let release; let ready; + const held = new Promise((resolve) => { release = resolve; }); + const staged = new Promise((resolve) => { ready = resolve; }); + const owner = withConsumerStateLock(state, async (_path, tx) => { + await tx.commitState("committed"); ready(); await held; + }, options); + await staged; + const rejection = await Promise.allSettled([ + withConsumerStateLock(state, async () => assert.fail("competing callback"), options), + rotateConsumerStateJournal(state, options), + ]); + release(); await owner; + for (const result of rejection) { assert.equal(result.status, "rejected"); assert.match(result.reason.message, /actively locked/); } + const failure = new Error("exact callback failure"); + await assert.rejects(withConsumerStateLock(state, async (_path, tx) => { await tx.commitState("discarded"); throw failure; }, options), (error) => error === failure); + const initial = readFileSync(`${state}.journal-v3/intent.json`); + const rotated = await rotateConsumerStateJournal(state, options); assert.equal(rotated.epoch, 2); + await withConsumerStateLock(state, async (_path, tx) => assert.equal(tx.readStateBytes().toString(), "committed"), options); + assert.deepEqual(readFileSync(`${state}.journal-v3/intent.json`), initial); + chmodSync(`${state}.journal-v3`, 0o777); + let callbacks = 0; + await assert.rejects(withConsumerStateLock(state, async () => { callbacks++; }, options), /permissions/); + assert.equal(callbacks, 0); chmodSync(`${state}.journal-v3`, 0o700); + assert.equal(lstatSync(`${state}.lock`).mode & 0o7777, 0o600); + } finally { rmSync(fixture, { recursive: true, force: true }); } +}); + +test("protected and retained v2 regression fixtures match their pinned provenance bytes", () => { + for (const family of ["protected-publication-v2", "retained-publication-v2"]) { + const directory = resolve(import.meta.dirname, "fixtures", family); + const provenance = JSON.parse(readFileSync(join(directory, "provenance.json"))); + assert.match(provenance.commit, /^[0-9a-f]{40}$/); + for (const entry of provenance.files) assert.equal(sha256Bytes(readFileSync(join(directory, entry.path))), entry.sha256, `${family}/${entry.path}`); + } +}); + +test("current public rotation uses reserved capacity without an extra predecessor normal claim", async () => { + const fixture = realpathSync(mkdtempSync(join(tmpdir(), "pylon-public-capacity-"))); + try { + const state = join(fixture, "state.json"); + const options = { stateMaxBytes: 1024, maxLockGenerations: 2, startHeartbeat: () => async () => {} }; + for (const value of ["one", "two"]) await withConsumerStateLock(state, async (_path, tx) => tx.commitState(value), options); + const rotated = await rotateConsumerStateJournal(state, options); + assert.equal(rotated.epoch, 2); assert.equal(rotated.tipSha256, sha256Bytes(Buffer.from("two"))); + assert.equal(readFileSync(state).toString(), "two"); + assert.deepEqual(await rotateConsumerStateJournal(state, options), rotated); + rmSync(state); + assert.deepEqual(await rotateConsumerStateJournal(state, options), rotated); + assert.equal(readFileSync(state).toString(), "two"); + } finally { rmSync(fixture, { recursive: true, force: true }); } +}); diff --git a/scripts/pylon-publication.test.mjs b/scripts/pylon-publication.test.mjs index 459cc5ead3..d5c35e8002 100644 --- a/scripts/pylon-publication.test.mjs +++ b/scripts/pylon-publication.test.mjs @@ -1,3 +1,5 @@ +import "./pylon-generation-migration.test.mjs"; +import "./pylon-public-state.test.mjs"; import "./pylon-generation-operations.test.mjs"; import "./pylon-generation.test.mjs"; import "./pylon-bounded-file.test.mjs"; @@ -86,15 +88,15 @@ import { PYLON_PUBLICATION_RULESET_GRAPHQL_VARIABLES, } from "./lib/pylon-ruleset-auditor.mjs"; import { validatePreviewWorkflowRunEvidence, verifyGhAttestationResult } from "./verify-pylon-publication-attestations.mjs"; -import { recordPreviewHighWater } from "./verify-pylon-preview-history.mjs"; -import { verifyStableHistoryWithState } from "./verify-pylon-stable-history.mjs"; +import { recordPreviewHighWater } from "./fixtures/retained-publication-v2/verify-pylon-preview-history.mjs"; +import { verifyStableHistoryWithState } from "./fixtures/retained-publication-v2/verify-pylon-stable-history.mjs"; import { verifyPreviewPublication } from "./verify-pylon-preview-publication.mjs"; import { ensureDurableConsumerStateDirectory, migrateConsumerStateJournal, rotateConsumerStateJournal, withConsumerStateLock, -} from "./lib/pylon-consumer-lock.mjs"; +} from "./fixtures/retained-publication-v2/pylon-consumer-lock.mjs"; import { BoundedFileLinkRetiredBeforeReadError, BoundedFileLinkRetiredDuringReadError, @@ -3793,7 +3795,7 @@ test("consumer state locking, recovery, transaction fencing, durability, and pat }; const runConsumerChild = (statePath, candidate) => { const source = ` - import { withConsumerStateLock } from ${JSON.stringify(pathToFileURL(resolve("scripts/lib/pylon-consumer-lock.mjs")).href)}; + import { withConsumerStateLock } from ${JSON.stringify(pathToFileURL(resolve("scripts/fixtures/retained-publication-v2/pylon-consumer-lock.mjs")).href)}; const statePath = process.argv[1]; const candidate = process.argv[2]; try { @@ -3820,7 +3822,7 @@ test("consumer state locking, recovery, transaction fencing, durability, and pat }; const runRotationChild = (statePath) => { const source = ` - import { rotateConsumerStateJournal } from ${JSON.stringify(pathToFileURL(resolve("scripts/lib/pylon-consumer-lock.mjs")).href)}; + import { rotateConsumerStateJournal } from ${JSON.stringify(pathToFileURL(resolve("scripts/fixtures/retained-publication-v2/pylon-consumer-lock.mjs")).href)}; try { const result = await rotateConsumerStateJournal(process.argv[1]); process.stdout.write(JSON.stringify(result)); @@ -4395,7 +4397,7 @@ test("consumer state locking, recovery, transaction fencing, durability, and pat const stagedCrashPath = join(fixture, "staged-then-crashed.json"); const crashingSource = ` - import { withConsumerStateLock } from ${JSON.stringify(pathToFileURL(resolve("scripts/lib/pylon-consumer-lock.mjs")).href)}; + import { withConsumerStateLock } from ${JSON.stringify(pathToFileURL(resolve("scripts/fixtures/retained-publication-v2/pylon-consumer-lock.mjs")).href)}; const hold = setInterval(() => {}, 1000); await withConsumerStateLock(process.argv[1], async (_path, transaction) => { await transaction.commitState(Buffer.from(JSON.stringify({ value: "crashed-stage" }) + "\\n")); From 2c2f0d3cbb3fe4f2f6602f0ad61b467b4971ce47 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Fri, 11 Sep 2026 01:38:34 -0600 Subject: [PATCH 07/14] fix(publication): verify generation recovery across processes Fixes #53. --- .github/workflows/ci.yml | 56 ++++++++ .pylon/features.yaml | 5 +- .pylon/upstream-review.md | 9 ++ docs/pylon-publication.md | 37 +++-- package.json | 2 + .../publication-crash/boundaries.json | 70 +++++++++ .../fixtures/publication-crash/support.mjs | 119 +++++++++++++++ scripts/fixtures/publication-crash/worker.mjs | 81 +++++++++++ scripts/lib/pylon-consumer-lock.mjs | 23 ++- scripts/pylon-publication-crash.test.mjs | 48 +++++++ scripts/pylon-publication-stress.test.mjs | 135 ++++++++++++++++++ scripts/pylon-publication.test.mjs | 2 + 12 files changed, 574 insertions(+), 13 deletions(-) create mode 100644 scripts/fixtures/publication-crash/boundaries.json create mode 100644 scripts/fixtures/publication-crash/support.mjs create mode 100644 scripts/fixtures/publication-crash/worker.mjs create mode 100644 scripts/pylon-publication-crash.test.mjs create mode 100644 scripts/pylon-publication-stress.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0cdda7bf38..718bef60a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -339,6 +339,59 @@ jobs: - name: Verify temporary-prefix runtime run: npm run release:pylon:smoke + pylon-publication: + name: Pylon publication (${{ matrix.os }}) + needs: trust + if: needs.trust.outputs.allowed == 'true' + runs-on: ${{ matrix.os }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15] + steps: + - name: Checkout exact source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ inputs.checkout_ref || github.ref }} + persist-credentials: false + + - name: Setup pinned Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22.23.2 + + - name: Install test dependencies + run: npm ci --ignore-scripts + + - name: Record source identity + run: | + mkdir -p publication-evidence + git rev-parse HEAD > publication-evidence/commit.txt + git rev-parse 'HEAD^{tree}' > publication-evidence/tree.txt + node --version > publication-evidence/node.txt + + - name: Complete publication, crash and repeated process suites + shell: bash + run: | + set -o pipefail + npm run test:pylon-publication 2>&1 | tee publication-evidence/publication.log + + - name: Actual 16 MiB maximum, serialized after complete suite + shell: bash + run: | + set -o pipefail + npm run test:pylon-publication-maximum 2>&1 | tee publication-evidence/maximum.log + + - name: Retain exact-source verification evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pylon-publication-${{ matrix.os }} + path: publication-evidence + if-no-files-found: error + retention-days: 7 + build-check-test: name: build-check-test if: always() && (needs.trust.outputs.allowed == 'true' || inputs.require_trusted) @@ -350,6 +403,7 @@ jobs: - pylon-artifact-pack - pylon-artifact-reproducibility - pylon-artifact-install + - pylon-publication runs-on: ubuntu-latest steps: - name: Verify CI results @@ -360,6 +414,7 @@ jobs: WINDOWS_OWNED_SESSION_RESULT: ${{ needs.owned-session-contract-windows.result }} ARTIFACT_PACK_RESULT: ${{ needs.pylon-artifact-pack.result }} ARTIFACT_REPRODUCIBILITY_RESULT: ${{ needs.pylon-artifact-reproducibility.result }} + PUBLICATION_RESULT: ${{ needs.pylon-publication.result }} ARTIFACT_INSTALL_RESULT: ${{ needs.pylon-artifact-install.result }} run: | test "$TRUST_ALLOWED" = true @@ -369,3 +424,4 @@ jobs: test "$ARTIFACT_PACK_RESULT" = success test "$ARTIFACT_REPRODUCIBILITY_RESULT" = success test "$ARTIFACT_INSTALL_RESULT" = success + test "$PUBLICATION_RESULT" = success diff --git a/.pylon/features.yaml b/.pylon/features.yaml index b5bc95a44c..9327e33deb 100644 --- a/.pylon/features.yaml +++ b/.pylon/features.yaml @@ -404,12 +404,13 @@ decisions: area: distribution state: candidate owner: pylon-fork - decision: retain + decision: redesign pylon_refs: - https://github.com/pylon-code/prime-agent/issues/29 + - https://github.com/pylon-code/prime-agent/issues/53 upstream_refs: - https://github.com/PrimeIntellect-ai/prime-agent/pull/32 - fork_change: protected-preview-and-append-only-stable-publication-v1 + fork_change: protected-publication-with-inode-bound-consumer-generations-v3 upstream_support: Prime's inherited publication path targets upstream R2 and npm channels and does not provide canonical Pylon-only exact-SHA admission, six immutable preview subjects, byte-preserving manual promotion, signed monotonic stable history, or append-only withdrawal. revisit_when: - Prime ships a repository-neutral immutable release and promotion primitive that preserves Pylon's exact source, workflow, attestation, stable-history, and withdrawal policy without upstream credentials or channel names. diff --git a/.pylon/upstream-review.md b/.pylon/upstream-review.md index 3dcbf44025..d4f7fc075d 100644 --- a/.pylon/upstream-review.md +++ b/.pylon/upstream-review.md @@ -292,3 +292,12 @@ Follow-up: Task10 builds/packs the exact merged tree into a private prefix and r - Wire classification: **backward-compatible optional metadata under the existing `event_sequence` capability**. Protocol remains 7; revision is 33 and the independently checked identity is `protocol-7-schema-33-5924c5b19b8b`. Existing command/event compatibility remains unchanged because the underlying events still work without replay fields. No new capability, SDK feature token, command, or startup requirement is introduced. Session/transfer identity, attachment epochs, owner and exact-environment checks remain intact. - Validation: nine focused files passed 673 tests with no skips or failures, including daemon/protocol/supervisor/SDK, caller-owned contract, snapshot-cache and transfer containment. The new 14-case regression suite exercises the real worker replay calculation, supervisor framing/cache and SDK commits with faux-provider messages and private agent homes. It covers absent/old-peer metadata, complete/partial/unavailable status, malformed coordinates, session identity and stale assemblies. Removing propagation gives six expected failures; restoring the old synthetic-complete cache behavior also fails the cache regression. `npm run check` passes formatting/lint, TypeScript, installer and browser-bundle checks. - Final private packaging, live Pylon continuity/follow-up/Stop/restart proof and checkpoint review remain Task10 gates. This fix does not claim to resolve the separately observed Pylon teardown timeouts. Revisit when upstream provides the same worker-issued replay propagation without weakening continuity or stock fallback. + +## 2026-09-11 — bounded publication consumer generations + +- Reviewed upstream remains `1eee2938b4eeb7a4d72e17035adda669a89b63de` (v0.9.4). This distribution change resolves the filesystem-authority defects documented in [#53](https://github.com/pylon-code/prime-agent/issues/53); it does not advance the frozen upstream range or restore upstream R2/npm publication. +- `protected-pylon-publication`: **redesign** the local consumer journal as v3 generations while retaining the protected preview/stable workflow policies. The public JSON projection and signed manifest formats are unchanged. A receipted creation identity selects one root; complete hidden generations publish by one directory rename. Exact predecessor claim/index, tip, inode and retirement-certificate commitments govern successor admission and bounded cleanup through deletion of the last proof link. +- Migration requires explicit external legacy-process quiescence and API/CLI acknowledgement before mutation. It authenticates all five historical source families, installs the old-client-visible impossible-generation blocker before the guard/source retirement sequence, preserves original v1 inodes at exact 0500, retains the exact v2 source for provenance, and revalidates canonical completion. A foreign old-client bootstrap inode in the retirement gap is a conflict. Concurrent helpers may join independently authenticated winners; injected errors and unsafe pinned observations remain terminal. +- Callback staging never publishes before successful return. Exact canonical receipt inodes permit interrupted publication recovery without replaying callbacks. Rotation recovery reconstructs the pre-certificate source when its deterministic rotation claim was linked before the winning index CAS; it excludes only that exact claim and still requires the complete certificate and winning-CAS successor checks. +- Verification gates retain the protected and historical v2 matrices alongside current-public v3 verifier tests, complete mutation-boundary SIGKILL traces with fresh-process recovery, repeated four-process migration/rotation competition, pinned-read handoffs and last-proof replacement negatives. CI requires the complete suite followed sequentially by the actual 16 MiB maximum on Ubuntu 24.04 and macOS 15 under Node 22.23.2; both results are mandatory in `build-check-test`. Process crashes do not simulate physical power loss, and resource limits are separately bounded historical inventories, metadata and generation-journal budgets. +- Revisit only when an upstream primitive preserves these exact local authority, bounded-work, migration and no-replay guarantees. Publication still requires current exact-source checks, independent review and every existing protected environment approval; filesystem tests do not authorize a release. diff --git a/docs/pylon-publication.md b/docs/pylon-publication.md index 1a8ca27fb5..3f850a0e8e 100644 --- a/docs/pylon-publication.md +++ b/docs/pylon-publication.md @@ -116,29 +116,50 @@ GH_TOKEN="$(gh auth token)" npm run release:pylon:verify-preview-history -- \ --initialize ``` -Use `--initialize` only after manually inspecting the first full verified receipt. Omit it thereafter. The canonical JSON at `--state` remains the CLI-compatible projection. The adjacent private `.journal` directory is the concurrency authority. Its authenticated checkpoint names one current epoch, anchors the exact prior immutable tip, and carries that tip's bounded canonical state bytes. Within the epoch, base-digest transition links and one contiguous operation-slot namespace are immutable no-replace records. Each slot is bound to its exact checkpoint epoch and generation and carries either a random-token normal operation or a deterministic rotation operation. Normal-operation 10-second heartbeats yield to one permanent `released`, `retired`, or `commit` decision. `transaction.commitState(candidate)` only validates and privately stages one copied candidate for the current callback. It does not publish a commit terminal, a transition, or the projection while callback code is still running. Only after the callback returns successfully does the wrapper publish the immutable commit decision and finish its transitions, projection, and applied marker before the overall call returns. A throw, process exit, or stale-owner retirement after staging but before callback success leaves no state commit. The active heartbeat and unresolved shared operation slot keep every later normal operation and rotation out for the full callback. A stale 30-second claim is retired, and a complete post-callback commit is helpable after every crash point. Owned write temporaries live in the separate bounded `.owned-temporaries-v2` namespace, so authenticated logical-entry caps never make orphan cleanup unreachable. The verifier preserves live-writer fencing, rejects gaps, cycles, unreachable records, orphan markers, symlinks, unexpected entries, and excess record, temporary, depth, or byte work, and repairs a missing or stale JSON projection from the journal tip. +Use `--initialize` only after manually inspecting the first full verified receipt. Omit it thereafter. The canonical JSON at `--state` is a repairable projection. The private `${state}.journal-v3` sidecar selects one inode-bound `journal--` root through a receipted `root.json`. Its immutable `intent.json` retains the actual initial state, and migration additionally retains the full historical source commitment. A directory's existence or a matching checkpoint digest alone never establishes authority. -`${state}.lock` is not the current journal namespace. For a fresh v2 journal, it is a permanent exact regular-file downgrade guard for clients that used `proper-lockfile`. Current tooling publishes that file by fsyncing a named owned temporary, hard-linking it no-replace, and fsyncing the parent. For migrated v1 authority, the original `${state}.lock` directory stays in place and contains an immutable `.pylon-consumer-v1-retired.json` marker. The marker binds the exact complete pre-marker authority digest and tip digest, and only that exact marker is excluded from the v1 authority digest. Its file, lock directory, and parent are fsynced before migration continues. The nonempty directory permanently blocks an old client's `rmdir` and subsequent atomic lock-directory `mkdir`. A directory without that exact marker is treated as a live or ambiguous legacy lease and fails closed. If no `${state}.transactions` authority exists, stop all old clients, confirm no owner remains, and remove that lease directory manually before retrying; current verification never enters or steals it. If the transaction namespace exists, preserve the directory and use the migration command below. +Each v3 generation contains a checkpoint, an epoch and exact receipt links. A complete hidden builder is fsynced before one same-parent directory rename publishes it. Normal claims, their winning index CAS, heartbeats, terminal decisions, transitions and applied records are immutable. Every record is published in this order: write/fsync an owned temporary, fsync its receipt directory, hardlink the canonical target without replacement, fsync the target parent, rename the temporary to its fixed receipt name, then fsync the receipt directory. Recovery authenticates the exact remaining inode and bytes; it never removes the only durable publication proof. -Versions before the checkpoint journal used `${state}.transactions` plus claim, terminal, and applied records in a `${state}.lock` directory. Legacy detection does not depend on that transaction namespace alone. Before guard or journal initialization, current verification independently inspects `${state}.transactions`, `${state}.lock.v1-retired`, an in-place `${state}.lock` directory and retirement marker, and any existing v2 head whose `sourceAuthoritySha256` is non-genesis. Any one signal requires the complete exact legacy source. A missing or deleted companion namespace, malformed entry, wrong mode, or symlink fails closed. The presence of the transaction namespace is always prior authority; current verification refuses to seed or trust a v2 projection around it. After stopping every old client and confirming that every old claim is terminal, migrate once: +`transaction.commitState(candidate)` privately stages one copied candidate. Only successful callback return permits the commit terminal. Callback errors, process death or stale-owner retirement before that return cannot commit staged bytes or replay the callback. A durable commit decision can be completed by another owner. Active claims exclude normal operations and rotations; 10-second heartbeats refresh the ordinary 30-second lease. Expiry permits a permanent retired decision, not callback replay. Projection repair rescans the authenticated tip after every rename and repairs forward if a delayed writer installed an older projection. + +### Explicit historical migration + +Stop every legacy verifier process before migration. This external quiescence is required even if PID probes or heartbeat age suggest inactivity: a previously admitted old process may already be past its projection revalidation. The acknowledgement is mandatory before any migration filesystem mutation: ```sh npm run release:pylon:migrate-consumer-journal -- \ - --state "$HOME/.local/state/pylon-prime/preview-high-water.json" + --state "$HOME/.local/state/pylon-prime/preview-high-water.json" \ + --acknowledge-legacy-processes-stopped ``` -This explicit quiescent command pins and bounds every v1 file, authenticates the complete transition chain and every relevant commit/help record, and accepts a projection only when it is the exact tip or an authenticated stale prefix. A commit is complete only when its exact applied marker and every decided transition are durable. An incomplete commit is recoverable only when `kill(pid, 0)` proves its recorded owner is gone with `ESRCH`; a live PID, PID reuse, `EPERM`, or any uncertain liveness blocks without retiring the authority. After all permitted dead-owner help, the command re-reads the source and atomically publishes the immutable retirement marker inside the existing `${state}.lock` directory. It never renames that directory and never creates or replaces `${state}.lock.v1-retired`. A prior `${state}.lock.v1-retired` layout from an interrupted local migration is accepted only as read-only source evidence and must use the exact regular downgrade guard. The deterministic v2 checkpoint binds the digest and tip of the complete old authority. Concurrent migrators re-read and join an exact marker or checkpoint that appeared after their initial read; conflicting marker, authority, directory, or checkpoint data fails closed. The command re-authenticates the full source immediately before marker publication, immediately before checkpoint publication, before projection repair, and before success. Every step is fsynced, deterministic, concurrently joinable, and retryable after a crash. Corrupt, active, missing, unreachable, extra, symlinked, over-limit, or permission-unsafe old authority fails closed. +The equivalent API is `migrateConsumerStateJournal(statePath, { acknowledgeLegacyProcessesStopped: true })`. Normal operations never implicitly migrate historical authority. Migration supports original in-place v1 lock/transaction directories, prior-retired v1 plus its exact regular guard, native v2, and v2 descended from either v1 layout. The reader authenticates every retained/current epoch, winning claim/index, valid loser, complete or helpable decision, exact source inode, bounded permitted temporary and allowed stale-prefix projection. Missing companions, active unresolved writers, ambiguous recovery, unsafe modes and unknown entries fail closed. + +Migration installs a receipted impossible-generation blocker, `claim-9999999999999999.json`, into each applicable original v1 lock and retained/current v2 epoch before publishing the v3 guard. The blocker binds the original source identity, authority, tip, retirement marker and immutable intent. Both original v1 directories are preserved in place, opened and validated without following symlinks, frozen through their pinned handles to exact `0500`, then fsynced with their parents. Partial freezes are accepted only with their exact prior proof. Native or prior-retired v2 receives its durable regular v3 guard before the exact `.journal` inode moves to `.journal.v2-retired`. Original in-place v1 keeps its `.lock` directory. A new old-client bootstrap `.journal` in this retirement gap is a foreign inode conflict, never overwritten or silently adopted. + +Provenance is reconstructed from that final fenced source, including every retained/current record and the immutable projection snapshot. The selected construction root has its own durable inode receipt before any generation publication. A crash before root selection may leave an inert empty construction directory; resume allocates a new exclusive root instead of claiming the unknown inode. At most 64 such directories are permitted. The selected generation is built and published atomically, the projection is repaired through an owned claim, and canonical completion is separately receipted. Re-running the acknowledged migration revalidates the retained source and exact selected root. Concurrent helpers can join an independently authenticated winner; callers that observe conflicting bytes, inode replacement or an unsafe intermediate read fail closed. Already-running legacy processes are never supported concurrently. -Rotate before an epoch reaches 3,800 transitions or 60,000 claims: +Preserve the original frozen v1 namespaces and `.journal.v2-retired`: they remain required provenance, not disposable generation history. Never remove a guard, reset the authority or copy a replacement journal over a failed migration. Preserve an offline backup and diagnose the exact reported conflict before retrying the same acknowledged command. + +### Rotation, cleanup and resource limits + +Rotate explicitly before the epoch reaches 3,800 transitions or 60,000 claims; normal operations also reserve capacity for rotation: ```sh npm run release:pylon:rotate-consumer-journal -- \ --state "$HOME/.local/state/pylon-prime/preview-high-water.json" ``` -Normal updates and rotation allocate from one immutable next-operation slot namespace. Every allocator scans and resolves the latest slot, rescans the same epoch and intent, and publishes only that exact next generation with no replacement; a lost publication loops from the new authority. No allocator may publish generation `N+1` while `N` is active or unresolved. A normal slot uses a random token and the configured finite normal-claim cap. A rotation slot is cap-exempt and carries the deterministic intent derived only from the exact current checkpoint and immutable tip, so rotators with different caller caps join the same generation, epoch id, directory, and checkpoint. Once a rotation wins its slot it is never released or retired: normal callers and later rotators help it through prior-writer quiescence, and no normal operation can cross it. If a normal operation wins the shared next slot first, rotation re-reads its committed tip and derives a new slot. Immediately before checkpoint linking and before success, rotation scans the complete bounded root set and rejects every competing same-epoch directory or checkpoint. Dead or already-retired normal-operation temporaries are removed; live prior temporaries keep rotation pending until they quiesce, while live helpers for the same deterministic rotation may join the same no-replace checkpoint link. Projection repair and commit help use bounded internal retries only for authenticated replacement or ctime races. Each retry re-walks the immutable journal tip before rereading or repairing the projection; malformed metadata, symlinks, and transaction digest corruption remain terminal errors. The current projection and high-water JSON schema do not change. After the new epoch is durable, a new fenced owner removes only the authenticated retired epoch and predecessor checkpoint, so active fencing data, directory entries, scan depth, and bytes remain bounded. Exact concurrent helpers treat a peer's already-removed retired epoch, predecessor checkpoint, or owned temporary as the same completed cleanup, re-scan an advanced authenticated head, and join the exact deterministic rotation instead of surfacing a transient path error. +Normal operations and rotation share one next-slot CAS. A rotation binds the exact latest winning claim/index, immutable tip, complete predecessor grammar and successor intent. Preparation converges a durable two-final cut before callback entry; successful rotation leaves one current final. Retirement and deletion preserve the observed predecessor inode. Cleanup removes only entries authenticated by the successor's committed retirement certificate; both certificate links survive until all ordinary authority is gone. After the last proof link, the successor permits cleanup only of that exact same-inode empty container. Byte-identical replacement directories and unknown extra entries remain conflicts. Dead temporaries and exact decided losers can be removed despite PID reuse; live unresolved writers block cleanup. + +The supported state size is 16 MiB per field. Checkpoint bounds account for all three base64 fields (`4 * ceil(bytes / 3)` each) and the complete envelope. Separate explicit budgets are 256 MiB for each historical v1/v2 inventory, 512 MiB for migration metadata/receipts, and 512 MiB for the v3 generation journal; these are not one combined memory or disk cap. Root, epoch, receipt and aggregate bounds are checked before nested allocations. Receipt lookup indexes canonical inode identities once. A capacity refusal occurs before commitment; do not reduce the real maximum fixture to make a verification run pass. + +Every relied-on file is owned by the current numeric uid with exact `0600`; directories are exact `0700`, except proven original v1 directories frozen to `0500`. Reads are bounded, no-follow where Node supports it, exact to EOF, and checked against pinned inode/size/mtime/ctime observations. Only direct native unpinned discovery loss may restart bounded discovery. Hook or injected filesystem errors retain their identity, including `ENOENT`, `EIO` and `EPERM`; a later successful rename does not erase an earlier terminal error. These checks assume a trusted user-owned local parent and are not a portable `openat` sandbox. Unsupported numeric-uid platforms fail closed. + +### Required publication verification + +`npm run test:pylon-publication` retains the protected v2 regression oracle and separately exercises current public v3 preview/stable verification, migration, generation grammar, real child-process crashes and repeated four-process competition. The crash inventory fixes each scenario's complete ordered hook/path/occurrence trace. Every listed cut must be reached through an IPC barrier; the parent kills only its captured child with `SIGKILL` and requires a fresh process to recover. Trace changes and missing cuts fail the gate. TAP diagnostics record the scenario, exact boundary, PID/signal, recovered root inode, projection outcome and elapsed time. These are process-crash tests; ordered fsync assertions support durability sequencing but do not simulate physical power loss. -These pathname checks are not a portable `openat` security sandbox. The verifier rejects observed symlinks and non-directories, pins every read to a no-follow file descriptor where Node exposes it, bounds bytes before allocation, and re-stats after an exact read. Every operation requires a numeric current uid. Every relied-on state, guard, journal, temporary namespace, epoch, claim, marker, transition, and migration-authority entry must already be owned by that uid and have exact `0600` file or `0700` directory mode. Group/world-writable entries are rejected before parsing or use and are never chmod-and-trusted, because another process may retain a writable file descriptor. Newly created directories and files use exact `0700` and `0600`; their contents and directory entries are fsynced before success. For old private state with other modes, stop every process that may hold a descriptor, preserve an offline backup, correct the modes while fully quiescent, and retry. Tooling never performs that migration implicitly. The state parent remains a trusted user-owned local directory with no hostile mutation by the same OS user. Platforms without a numeric current uid fail closed. +CI requires this complete suite and the separate `npm run test:pylon-publication-maximum` on Ubuntu 24.04 and macOS 15 with Node 22.23.2. The actual 16 MiB maximum runs sequentially after the complete suite. Both platform results feed `build-check-test`; skipped, cancelled or failed publication jobs cannot make that aggregate succeed. Evidence artifacts bind logs to the tested commit and tree. Focused `test:pylon-publication-crash` and `test:pylon-publication-stress` commands are available for investigation and do not replace the complete gate. ## Stable promotion diff --git a/package.json b/package.json index 8f8f0ebafe..287b40727a 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,8 @@ "release:pylon:smoke": "node scripts/smoke-pylon-prime-agent-release.mjs", "test:pylon-release": "node --test scripts/pylon-prime-agent-release.test.mjs", "test:pylon-publication": "node --test scripts/pylon-publication.test.mjs", + "test:pylon-publication-crash": "node --test scripts/pylon-publication-crash.test.mjs", + "test:pylon-publication-stress": "node --test scripts/pylon-publication-stress.test.mjs", "test:pylon-publication-maximum": "node --test scripts/pylon-generation-maximum.test.mjs", "test:pylon-ruleset-auditor-app": "node --test scripts/pylon-ruleset-auditor-acceptance.test.mjs", "release:patch": "node scripts/release.mjs patch", diff --git a/scripts/fixtures/publication-crash/boundaries.json b/scripts/fixtures/publication-crash/boundaries.json new file mode 100644 index 0000000000..2de40a38cb --- /dev/null +++ b/scripts/fixtures/publication-crash/boundaries.json @@ -0,0 +1,70 @@ +{ + "recover-builder-checkpoint": { + "count": 76, + "sha256": "f03c819ecf4e5cc6b4486326782cc2acdaea2c5dd5e18f2ef5137ad0a9b9c128" + }, + "recover-commit-receipt": { + "count": 62, + "sha256": "3b12d0d6cec5e7bbe099ca4ec585d1365070943b379caba298c1ef5092ba9093" + }, + "recover-migration-intent": { + "count": 171, + "sha256": "580a32a420f41f8203b87c013264aa4dbbd76d42006ae6c4447f00d37a23becb" + }, + "recover-projection": { + "count": 95, + "sha256": "8c53c6c524b603e3e54c1f22e6c611d4bd0a2db7b62f8fea8129221daa048a5c" + }, + "fresh": { + "count": 200, + "sha256": "3fecb7e88d67f631d1eb04d0aeb749c43234a6d877380bebe1aa541260c37825" + }, + "commit": { + "count": 96, + "sha256": "681403b469c5ad85525886658dcb5539b75960977f2e26729ebf9e7a5c93047d" + }, + "rotate": { + "count": 220, + "sha256": "51583d4f3d418c2d876aed13e5f5d2b560df22aee8916134ee0d82c37a1eec30" + }, + "builder": { + "count": 26, + "sha256": "40dc44c30a950465f31b359bacf21443aed3b050326c248a4ad6c54db54ef463" + }, + "loser-cleanup": { + "count": 20, + "sha256": "a8a77cefd6c48155f92f86880505791de3e6325eb31e73b1e0c598e2d155f2e0" + }, + "migrate-v1": { + "count": 192, + "sha256": "11e5dcc90f0a6129a553c79b0fd777c89a92cd4abbf71fee830660542db94b93" + }, + "migrate-prior-retired-v1": { + "count": 213, + "sha256": "583c44f3b52dbe855815189a19be1278abcac4c8062b727ad1609dfa9c2f4139" + }, + "migrate-v2": { + "count": 192, + "sha256": "fc123fb9de7314ad5abafcc4ad50d1961656eb5674a7a89769ab5c6a81caf464" + }, + "migrate-v1-v2": { + "count": 196, + "sha256": "5bd646380aab12e87493d82812cba42fb205b2627e51c3dd285b7905846a266e" + }, + "migrate-prior-retired-v1-v2": { + "count": 217, + "sha256": "ca8f82d69210565d947a33ab0a1cf7701b45aad434fbd7ca6e613fc9a24f61a6" + }, + "migrate-v2-incomplete": { + "count": 203, + "sha256": "3b0da45c87fc68a5c56c0aefb2d434941348cd5ebde220e275fc172fe7c91ff2" + }, + "migrate-v2-retained": { + "count": 205, + "sha256": "ab1b5d33dd6ca0c58aa708ee454f6086d74867c6123b643d6af831c3783d677d" + }, + "recover-dead-receipt": { + "count": 62, + "sha256": "1779428bb04a7a88567cb919b9f03065f004d2aa1f33bbc17b2518a040d7ebef" + } +} diff --git a/scripts/fixtures/publication-crash/support.mjs b/scripts/fixtures/publication-crash/support.mjs new file mode 100644 index 0000000000..3be0aa5af4 --- /dev/null +++ b/scripts/fixtures/publication-crash/support.mjs @@ -0,0 +1,119 @@ +import assert from "node:assert/strict"; +import { fork } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as old from "../protected-publication-v2/pylon-consumer-lock.mjs"; +import { withConsumerStateLock, buildConsumerGeneration, publishConsumerGeneration } from "../../lib/pylon-consumer-lock.mjs"; +import { generationBytes as bytes, generationDigest as digest, GENERATION_ZERO as ZERO } from "../../lib/pylon-generation-format.mjs"; + +export const options = { stateMaxBytes: 1024, now: () => 1000, stale: 100, startHeartbeat: () => async () => {} }; +const oldOptions = { now: () => 1000, stateMaxBytes: 1024, startHeartbeat: () => async () => {} }; +export const families = ["v1", "prior-retired-v1", "v2", "v1-v2", "prior-retired-v1-v2"]; +export const recoveries = { + "recover-builder-checkpoint": { scenario: "builder", match: { hook: "generation", phase: "after", operation: "file-sync" } }, + "recover-dead-receipt": { scenario: "commit", match: { hook: "generation", phase: "after", operation: "file-sync" } }, + "recover-commit-receipt": { scenario: "commit", match: { hook: "generation", phase: "before", operation: "rename" } }, + "recover-migration-intent": { scenario: "migrate-v2", match: { hook: "migration", phase: "after", operation: "immutable-link", path: "state.json.journal-v3/intent.json" } }, + "recover-projection": { scenario: "commit", match: { hook: "generation", phase: "after", operation: "create-projection" } }, +}; +export const scenarios = [...Object.keys(recoveries), "fresh", "commit", "rotate", "builder", "loser-cleanup", ...families.map((family) => `migrate-${family}`), "migrate-v2-incomplete", "migrate-v2-retained"]; +export async function fixture(scenario) { + if (recoveries[scenario]) { + const setup = recoveries[scenario]; const f = await fixture(setup.scenario); const owner = child(f, setup.scenario, { cutMatch: setup.match, marker: "setup-owner" }); + try { const cut = await owner.wait("cut"); assert.equal(cut.pid, owner.process.pid); owner.process.kill("SIGKILL"); assert.equal((await owner.exit).signal, "SIGKILL"); } + finally { await owner.stop(); } + return f; + } + const directory = await realpath(await mkdtemp(join(tmpdir(), "pylon-publication-crash-"))); + await chmod(directory, 0o700); + const state = join(directory, "state.json"); + const root = join(directory, "journal"); + const authority = { genesis: { statePath: state, stateBytes: null } }; + if (["builder", "loser-cleanup"].includes(scenario)) { + await mkdir(root, { mode: 0o700 }); + if (scenario === "loser-cleanup") { + const winner = await buildConsumerGeneration(root, authority, options); + await buildConsumerGeneration(root, authority, options); + await publishConsumerGeneration(winner, options); + } + } else if (["commit", "rotate"].includes(scenario)) { + await withConsumerStateLock(state, async (_path, tx) => tx.commitState("base"), options); + } else if (scenario.startsWith("migrate-")) { + const family = scenario.slice(8); + if (family.includes("v1")) { + await mkdir(`${state}.lock`, { mode: 0o700 }); + await mkdir(`${state}.transactions`, { mode: 0o700 }); + const claim = { schemaVersion: 1, generation: 1, token: randomUUID(), ownerPid: 2_000_000_000, createdAtMs: 0 }; + const value = Buffer.from("base"); + const transaction = { schemaVersion: 1, baseDigest: ZERO, candidateDigest: digest(value), candidateBase64: value.toString("base64") }; + const terminal = { schemaVersion: 1, generation: 1, token: claim.token, outcome: "commit", transactions: [transaction] }; + const put = (path, value) => writeFile(path, bytes(value), { mode: 0o600 }); + await put(`${state}.lock/claim-0000000000000001.json`, claim); + await put(`${state}.lock/heartbeat-0000000000000001-${claim.token}.json`, { schemaVersion: 1, generation: 1, token: claim.token, refreshedAtMs: 0 }); + await put(`${state}.lock/terminal-0000000000000001-${claim.token}.json`, terminal); + await put(`${state}.transactions/${ZERO}.json`, transaction); + await put(`${state}.lock/applied-0000000000000001-${claim.token}.json`, { schemaVersion: 1, generation: 1, token: claim.token, terminalSha256: digest(bytes(terminal)) }); + await writeFile(state, value, { mode: 0o600 }); + if (family.startsWith("prior-retired")) { + await rename(`${state}.lock`, `${state}.lock.v1-retired`); + await put(`${state}.lock`, { schemaVersion: 1, kind: "pylon-consumer-legacy-lock-guard", statePathSha256: digest(Buffer.from(state)) }); + } + if (family.endsWith("-v2")) await old.migrateConsumerStateJournal(state, oldOptions); + } else { + await old.withConsumerStateLock(state, async (_path, tx) => tx.commitState("base"), oldOptions); + if (family === "v2-retained") await old.rotateConsumerStateJournal(state, oldOptions); + if (family === "v2-incomplete") { + const stopped = new Error("fixture incomplete durable commit"); + await assert.rejects(old.withConsumerStateLock(state, async (_path, tx) => tx.commitState("base" + "-advanced"), { ...oldOptions, hooks: { afterCommitDecision: () => { throw stopped; } } }), (error) => error === stopped); + } + } + } + return { directory, state, root, authority }; +} +export async function cleanup(f) { + for (const path of [`${f.state}.lock`, `${f.state}.lock.v1-retired`, `${f.state}.transactions`]) { + try { if ((await lstat(path)).isDirectory()) await chmod(path, 0o700); } catch (error) { if (error.code !== "ENOENT") throw error; } + } + await rm(f.directory, { recursive: true, force: true }); +} +export function child(f, scenario, configuration = {}) { + const process = fork(new URL("./worker.mjs", import.meta.url), [f.directory, scenario, JSON.stringify(configuration)], { stdio: ["ignore", "pipe", "pipe", "ipc"], execArgv: [] }); + const messages = []; const waiters = []; + let output = ""; let ended = false; + process.stdout.on("data", (data) => { output += data; }); process.stderr.on("data", (data) => { output += data; }); + process.on("message", (message) => { messages.push(message); for (const wake of waiters.splice(0)) wake(); }); + const exit = new Promise((resolve, reject) => { + process.once("error", reject); + process.once("close", (code, signal) => { ended = true; resolve({ code, signal }); for (const wake of waiters.splice(0)) wake(); }); + }); + const watchdog = setTimeout(() => { if (!ended) process.kill("SIGKILL"); }, 60000); + exit.finally(() => clearTimeout(watchdog)); + return { process, messages, exit, get output() { return output; }, + async wait(type) { + while (!messages.some((message) => message.type === type)) { + if (ended) throw new Error(`Worker exited without required ${type}: ${JSON.stringify(messages.slice(-2))} ${output}`); + await new Promise((resolve) => waiters.push(resolve)); + } + return messages.find((message) => message.type === type); + }, + async finish() { + const result = await exit; + assert.deepEqual(result, { code: 0, signal: null }, `${JSON.stringify(messages.slice(-2))} ${output}`); + return messages.find((message) => message.type === "done"); + }, + async stop() { if (!ended) process.kill("SIGKILL"); await exit; }, + }; +} +export async function inventory(f) { + const paths = []; + async function visit(path) { + const stat = await lstat(path); + paths.push({ path: path.slice(f.directory.length + 1), dev: stat.dev, ino: stat.ino, size: stat.size }); + if (stat.isDirectory()) for (const name of (await readdir(path)).sort()) await visit(join(path, name)); + } + await visit(f.directory); + return paths; +} +export async function projection(f) { return readFile(f.state, "utf8").catch((error) => { if (error.code === "ENOENT") return null; throw error; }); } diff --git a/scripts/fixtures/publication-crash/worker.mjs b/scripts/fixtures/publication-crash/worker.mjs new file mode 100644 index 0000000000..34bef79a92 --- /dev/null +++ b/scripts/fixtures/publication-crash/worker.mjs @@ -0,0 +1,81 @@ +import { appendFile, lstat, readFile, readdir } from "node:fs/promises"; +import { basename, join } from "node:path"; +import { withConsumerStateLock, migrateConsumerStateJournal, rotateConsumerStateJournal, prepareConsumerGeneration, discoverConsumerGenerations, withConsumerGenerationLock } from "../../lib/pylon-consumer-lock.mjs"; + +const [directory, requestedScenario, encoded] = process.argv.slice(2); +const config = JSON.parse(encoded); +const recoveryScenarios = { "recover-builder-checkpoint": "builder", "recover-dead-receipt": "commit", "recover-commit-receipt": "commit", "recover-migration-intent": "migrate-v2", "recover-projection": "commit" }; +const scenario = recoveryScenarios[requestedScenario] ?? requestedScenario; +if (recoveryScenarios[requestedScenario]) config.recover = true; +const state = join(directory, "state.json"); +const lowRoot = join(directory, "journal"); +const authority = { genesis: { statePath: state, stateBytes: null } }; +const send = (message) => new Promise((resolve, reject) => process.send(message, (error) => error ? reject(error) : resolve())); +const events = []; +const occurrences = new Map(); +let readPaused = false; +async function pauseRead(path, kind) { + if (readPaused || config.pauseRead !== kind || !path.includes("/generation-")) return; + readPaused = true; process.send({ type: "cut", pid: process.pid, path, kind }); + await new Promise((resolve) => process.once("message", resolve)); +} +const normalize = (path) => path.slice(directory.length + 1).replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g, "UUID").replace(/[0-9a-f]{32,}/g, "HASH").replace(/-p[0-9]+/g, "-pPID"); +async function barrier(hook, event) { + const key = `${hook}:${event.phase}:${event.operation}:${normalize(event.path)}`; + const occurrence = (occurrences.get(key) ?? 0) + 1; occurrences.set(key, occurrence); + const cut = { hook, phase: event.phase, operation: event.operation, path: normalize(event.path), occurrence }; + events.push(cut); + if (config.cut === events.length - 1 || config.pause === key || config.cutMatch && Object.entries(config.cutMatch).every(([name, value]) => cut[name] === value)) { + process.send({ type: "cut", pid: process.pid, index: events.length - 1, event: cut, events }); + await new Promise((resolve) => process.once("message", resolve)); + } +} +const options = { stateMaxBytes: config.stateMaxBytes ?? 1024, now: () => config.now ?? (config.recover ? 100000 : 1000), stale: 100, startHeartbeat: () => async () => {}, + hooks: { + afterMigrationBlocker: config.migrationPause ? async () => { + await send({ type: "cut", pid: process.pid, phase: "after-blocker" }); + await new Promise((resolve) => process.once("message", resolve)); + } : undefined, + metadataRead: { afterInitialStat: async ({ path }) => { + const name = basename(path); + const kind = ["checkpoint.json", "claim-index-", "heartbeat-", "terminal-", "transition-", "applied-", "receipt-"].find((prefix) => name.startsWith(prefix)); + if (kind) await pauseRead(path, kind); + } }, + generationBoundary: (event) => barrier("generation", event), migrationBoundary: (event) => barrier("migration", event), + beforeCommitDecision: () => barrier("semantic", { phase: "before", operation: "commit-decision", path: state }), + afterCommitDecision: () => barrier("semantic", { phase: "after", operation: "commit-decision", path: state }), + } }; +if (config.pauseRead === "receipt-") options.lstatEntry = async (path) => { + const stat = await lstat(path); + if (basename(path).startsWith("receipt-")) await pauseRead(path, "receipt-"); + return stat; +}; +async function callback(_path, tx) { + const marker = config.marker ?? "owner"; + await appendFile(join(directory, `${marker}.callbacks`), "entered\n", { mode: 0o600 }); + if (config.recover) return tx.readStateBytes()?.toString() ?? null; + const current = tx.readStateBytes()?.toString(); + await tx.commitState(config.append ? JSON.stringify([...(current === "base" || current == null ? [] : JSON.parse(current)), config.value]) : config.value ?? "candidate"); + await barrier("callback", { phase: "after", operation: "stage", path: state }); + await appendFile(join(directory, `${marker}.callbacks`), "returned\n", { mode: 0o600 }); + return current; +} +try { + if (config.ready) { process.send({ type: "ready", pid: process.pid }); await new Promise((resolve) => process.once("message", resolve)); } + let result; + if (config.pauseRead === "receipt-") { + const record = JSON.parse(await readFile(`${state}.journal-v3/root.json`)); + result = await discoverConsumerGenerations(join(`${state}.journal-v3`, record.goal), authority, options); + } else if (scenario.startsWith("migrate-")) result = await migrateConsumerStateJournal(state, { ...options, acknowledgeLegacyProcessesStopped: true }); + else if (["builder", "loser-cleanup"].includes(scenario)) { + result = await prepareConsumerGeneration(lowRoot, authority, options); + if (config.recover) await withConsumerGenerationLock(lowRoot, authority, async () => {}, options); + } else if (scenario === "rotate" && !config.recover) result = await rotateConsumerStateJournal(state, options); + else result = await withConsumerStateLock(state, callback, options); + let root = lowRoot; + if (!["builder", "loser-cleanup"].includes(scenario)) root = join(`${state}.journal-v3`, JSON.parse(await readFile(`${state}.journal-v3/root.json`)).goal); + const stat = await lstat(root); + const names = await readdir(root); + await send({ type: "done", pid: process.pid, events, root: { dev: stat.dev, ino: stat.ino }, finals: names.filter((name) => name.startsWith("generation-")), entries: names, result: result?.epoch ?? null }); + process.disconnect(); +} catch (error) { await send({ type: "error", pid: process.pid, message: error.message, code: error.code, events }); process.disconnect(); process.exitCode = 1; } diff --git a/scripts/lib/pylon-consumer-lock.mjs b/scripts/lib/pylon-consumer-lock.mjs index f256391812..9dea41aff4 100644 --- a/scripts/lib/pylon-consumer-lock.mjs +++ b/scripts/lib/pylon-consumer-lock.mjs @@ -4231,6 +4231,18 @@ function generationRetirementCertificate(snapshot, slot) { epochIdentity: snapshot.epochIdentity, receiptsIdentity: snapshot.receiptsIdentity, slot, entries: snapshot.canonicalEntries.filter((entry) => entry.name !== "retirement.json").map((entry) => generationCertificateEntry(snapshot, entry)).sort((a, b) => a.name.localeCompare(b.name)) }; } +function generationPendingRotation(snapshot, slot, scan, options) { + const certificateBytes = snapshot.retirementCertificate ?? metadataBytes(generationRetirementCertificate(snapshot, slot)); + const wanted = consumerGenerationRotationClaim(snapshot.checkpoint, slot, { ...scan.tip, previousGenerationIdentity: snapshot.identity, retirementAuthoritySha256: generationRetirementDigest(certificateBytes) }, options.stateMaxBytes); + const name = basename(claimPath({ epochDirectory: "" }, wanted)); + const pending = snapshot.epochRecords.get(name); + if (pending && !pending.equals(metadataBytes(wanted))) throw new Error("Generation pending rotation differs from its exact deterministic claim."); + // The durable certificate precedes this claim's canonical link and index CAS. + // Exclude only that exact claim while reconstructing its pre-publication source. + const prior = pending ? { ...snapshot, canonicalEntries: snapshot.canonicalEntries.filter((entry) => entry.name !== `epoch/${name}`) } : snapshot; + if (!metadataBytes(generationRetirementCertificate(prior, slot)).equals(certificateBytes)) throw new Error("Generation pending rotation certificate differs from its exact predecessor authority."); + return { certificateBytes, wanted }; +} function validateGenerationRetirementCertificate(snapshot, successor, options) { const bytes = snapshot.retirementCertificate; if (!Buffer.isBuffer(bytes) || generationRetirementDigest(bytes) !== successor.retirementAuthoritySha256 || !generationSameInode(snapshot.identity, successor.previousGenerationIdentity)) throw new Error("Generation retirement certificate does not bind the exact predecessor inode."); @@ -4504,7 +4516,9 @@ async function generationQuiesce(snapshot, options, ownTemporary = null, require if (receipt.target === null && !decided && temporaryProcessIsAlive({ pid: receipt.pid }, options)) throw new Error("Generation rotation is pending until its live unresolved receipt writer quiesces."); if (!sameRetiredLinkStat(receipt.stat, await options.lstatEntry(path))) throw new Error("Generation receipt temporary inode changed before recovery."); if (receipt.target === null) { + await generationBoundary(options, "before", "unlink", path); await options.removeFile(path); + await generationBoundary(options, "after", "unlink", path); } else { const fixed = join(snapshot.path, "receipts", `receipt-${digest(Buffer.from(receipt.target))}.json`); await generationBoundary(options, "before", "rename", fixed); @@ -4832,12 +4846,16 @@ async function generationCleanupInstalledBuilders(root, current, options) { for (const directory of ["epoch", "receipts"]) { if (!names.includes(directory)) continue; await generationNames(join(path, directory), 0, options); + await generationBoundary(options, "before", "remove-directory", join(path, directory)); await options.removeFile(join(path, directory), { recursive: true }); + await generationBoundary(options, "after", "remove-directory", join(path, directory)); await generationSync(path, options); } if (!generationSameInode(identity, await generationDirectory(path, options))) throw new Error("Generation losing builder container was replaced."); await generationNames(path, 0, options); + await generationBoundary(options, "before", "remove-directory", path); await options.removeFile(path, { recursive: true }); + await generationBoundary(options, "after", "remove-directory", path); await generationSync(root, options); } } @@ -4870,14 +4888,13 @@ export async function rotateConsumerGeneration(root, authority, rawOptions = {}) if (latest?.type === "normal" && (!scan.terminals.has(`${latest.generation}:${latest.token}`) || (scan.terminals.get(`${latest.generation}:${latest.token}`).outcome === "commit" && !scan.applied.has(`${latest.generation}:${latest.token}`)))) throw new Error("Generation rotation requires a resolved normal operation frontier."); if (latest?.type !== "rotation") { const slot = (latest?.generation ?? 0) + 1; - const certificateBytes = metadataBytes(generationRetirementCertificate(snapshot, slot)); - const wanted = consumerGenerationRotationClaim(snapshot.checkpoint, slot, { ...scan.tip, previousGenerationIdentity: snapshot.identity, retirementAuthoritySha256: generationRetirementDigest(certificateBytes) }, options.stateMaxBytes); + const { certificateBytes, wanted } = generationPendingRotation(snapshot, slot, scan, options); const headroom = 2 * certificateBytes.length + 2 * metadataBytes(wanted).length + 2 * metadataBytes(claimIndexFor(wanted)).length + 2 * metadataBytes(wanted.intent.checkpoint).length; if (await generationRootPreflight(root, options) + headroom > options.maxJournalBytes) throw new Error("Generation lacks reserved rotation headroom."); await options.hooks?.beforeRotationDecision?.({ claim: wanted, intent: wanted.intent }); const result = await generationWriteReceipt(snapshot, "retirement.json", certificateBytes, options, async () => { const current = await generationReadPinned(snapshot, options); - if (!metadataBytes(generationRetirementCertificate(current, slot)).equals(certificateBytes)) throw new Error("Generation retirement authority changed before certificate publication."); + if (!generationPendingRotation(current, slot, generationEpochAuthority(current, options), options).certificateBytes.equals(certificateBytes)) throw new Error("Generation retirement authority changed before certificate publication."); }); if (!result.bytes?.equals(certificateBytes)) throw new Error("Generation retirement certificate lost its immutable publication."); snapshot = await generationReadPinned(snapshot, options); diff --git a/scripts/pylon-publication-crash.test.mjs b/scripts/pylon-publication-crash.test.mjs new file mode 100644 index 0000000000..684db8cb2b --- /dev/null +++ b/scripts/pylon-publication-crash.test.mjs @@ -0,0 +1,48 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { test } from "node:test"; +import { child, cleanup, fixture, inventory, projection, scenarios } from "./fixtures/publication-crash/support.mjs"; + +const manifest = JSON.parse(await readFile(new URL("./fixtures/publication-crash/boundaries.json", import.meta.url))); +const traceDigest = (events) => createHash("sha256").update(JSON.stringify(events)).digest("hex"); + +for (const scenario of scenarios) test(`publication crash boundary inventory ${scenario}`, { timeout: 900000 }, async (t) => { + const original = await fixture(scenario); const recorder = child(original, scenario); + let events; + try { + events = (await recorder.finish()).events; + assert.equal(events.length, manifest[scenario].count, "Changed boundary count requires explicit inventory review"); + assert.equal(traceDigest(events), manifest[scenario].sha256, "Changed boundary ordering/path/occurrence requires explicit inventory review"); + } finally { await recorder.stop(); await cleanup(original); } + for (const [index, expected] of events.entries()) { + const start = performance.now(); const f = await fixture(scenario); const owner = child(f, scenario, { cut: index }); let recovery; + try { + const observed = await owner.wait("cut"); + assert.equal(observed.pid, owner.process.pid); + assert.equal(observed.index, index); + assert.deepEqual(observed.events, events.slice(0, index + 1), `Trace changed before ${scenario} cut ${index}`); + assert.equal(owner.process.kill("SIGKILL"), true); + assert.deepEqual(await owner.exit, { code: null, signal: "SIGKILL" }); + const beforeRecovery = await inventory(f); + recovery = child(f, scenario, { now: 1000000, recover: true, marker: "recovery" }); + const result = await recovery.finish(); + assert.notEqual(result.pid, observed.pid); + assert.equal(result.finals.length, 1); + assert.equal(result.entries.some((name) => name.startsWith(".retired-") || name.startsWith(".deleting-")), false); + const value = await projection(f); + if (scenario.startsWith("recover-")) assert.equal(value, scenario === "recover-projection" ? "candidate" : scenario === "recover-builder-checkpoint" ? null : "base"); + else if (scenario.startsWith("migrate-")) assert.equal(value, scenario.endsWith("incomplete") ? "base-advanced" : "base"); + else if (scenario === "rotate") assert.equal(value, "base"); + else if (["fresh", "commit"].includes(scenario)) { + const markers = await readFile(join(f.directory, "owner.callbacks"), "utf8").catch((error) => { if (error.code === "ENOENT") return ""; throw error; }); + assert.ok(["", "entered\n", "entered\nreturned\n"].includes(markers), "Original callback never replayed"); + const returned = markers.endsWith("returned\n"); + const committed = observed.events.some((event) => event.hook === "generation" && event.operation === "link" && event.phase === "after" && event.path.includes("/terminal-")); + assert.equal(value, returned && committed ? "candidate" : scenario === "fresh" ? null : "base"); + } + t.diagnostic(JSON.stringify({ scenario, index, ...expected, pid: observed.pid, signal: "SIGKILL", recoveryPid: result.pid, root: result.root, preservedEntries: beforeRecovery.length, projection: value, elapsedMs: Math.round(performance.now() - start) })); + } finally { await owner.stop(); if (recovery) await recovery.stop(); await cleanup(f); } + } +}); diff --git a/scripts/pylon-publication-stress.test.mjs b/scripts/pylon-publication-stress.test.mjs new file mode 100644 index 0000000000..695dde2aeb --- /dev/null +++ b/scripts/pylon-publication-stress.test.mjs @@ -0,0 +1,135 @@ +import assert from "node:assert/strict"; +import { chmod, cp, readFile, readdir, rename, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { test } from "node:test"; +import { child, cleanup, fixture, families, projection } from "./fixtures/publication-crash/support.mjs"; + +const rounds = 10; +const refusals = /actively locked|changed|disappeared|ENOENT|EEXIST|inode|receipt|publication|authority|conflicting|writer|ownership|claim|checkpoint|namespace|unfinished|incomplete|unsafe type, owner or exact permissions/; +async function competition(t, f, scenarios, round) { + const controlled = !scenarios[0].startsWith("migrate-") && round % 2 === 0; + const workers = scenarios.map((scenario, index) => child(f, scenario, { pause: controlled && index === 0 ? "callback:after:stage:state.json" : undefined, append: true, stateMaxBytes: 8192, ready: true, marker: `round-${round}-${index}`, value: `value-${round}-${index}` })); + try { + await Promise.all(workers.map((worker) => worker.wait("ready"))); + if (controlled) { + workers[0].process.send({ type: "release" }); const cut = await workers[0].wait("cut"); assert.equal(cut.pid, workers[0].process.pid); + for (const worker of workers.slice(1)) worker.process.send({ type: "release" }); + await Promise.all(workers.slice(1).map((worker) => worker.exit)); + workers[0].process.send({ type: "release" }); + } else for (const worker of workers) worker.process.send({ type: "release" }); + const results = await Promise.all(workers.map(async (worker) => { + const exit = await worker.exit; assert.equal(exit.signal, null); + const result = worker.messages.find((message) => message.type === "done" || message.type === "error"); assert.ok(result, `Missing terminal IPC: ${JSON.stringify(exit)} ${worker.output}`); + if (exit.code !== 0) { assert.equal(exit.code, 1); assert.equal(result.type, "error"); assert.match(result.message, refusals); } + return result; + })); + if (controlled) assert.equal(results[0].type, "done", "The staged owner must complete exactly once"); + for (let index = 0; index < workers.length; index++) { + const markers = await readFile(join(f.directory, `round-${round}-${index}.callbacks`), "utf8").catch((error) => { if (error.code === "ENOENT") return ""; throw error; }); + assert.ok(["", "entered\n", "entered\nreturned\n"].includes(markers), "No callback invocation is replayed"); + } + const recovery = child(f, scenarios[0].startsWith("migrate-") ? scenarios[0] : "commit", { stateMaxBytes: 8192, recover: true, marker: `recovery-${round}` }); + let resumed; + try { resumed = await recovery.finish(); } finally { await recovery.stop(); } + assert.equal(resumed.finals.length, 1); + if (!scenarios[0].startsWith("migrate-")) { + const value = await projection(f); const history = value === "base" ? [] : JSON.parse(value); + assert.equal(new Set(history).size, history.length, "Each admitted value appears exactly once"); + for (const [index, result] of results.entries()) if (result.type === "done" && scenarios[index] === "commit") assert.ok(history.includes(`value-${round}-${index}`)); + for (const entry of history.filter((entry) => entry.startsWith(`value-${round}-`))) { + const index = Number(entry.split("-").at(-1)); + assert.equal(await readFile(join(f.directory, `round-${round}-${index}.callbacks`), "utf8"), "entered\nreturned\n"); + } + } + for (const result of results.filter((result) => result.type === "done")) assert.deepEqual(result.root, resumed.root); + t.diagnostic(JSON.stringify({ round, scenarios, controlled, pids: workers.map((worker) => worker.process.pid), admitted: results.filter((result) => result.type === "done").length, refusals: results.filter((result) => result.type === "error").map((result) => result.message), root: resumed.root })); + } finally { await Promise.all(workers.map((worker) => worker.stop())); } +} + +test("publication repeated four-process normal and rotation competition", { timeout: 600000 }, async (t) => { + const f = await fixture("commit"); + try { + for (let round = 0; round < rounds; round++) await competition(t, f, round % 2 ? ["commit", "commit", "rotate", "rotate"] : ["commit", "commit", "commit", "commit"], round); + assert.ok((await projection(f)) === "base" || Array.isArray(JSON.parse(await projection(f)))); + } finally { await cleanup(f); } +}); +for (const family of families) test(`publication repeated four-process migration ${family}`, { timeout: 600000 }, async (t) => { + for (let round = 0; round < rounds; round++) { + const f = await fixture(`migrate-${family}`); + try { await competition(t, f, Array(4).fill(`migrate-${family}`), round); assert.equal(await projection(f), "base"); } + finally { await cleanup(f); } + } +}); + +for (const kind of ["checkpoint.json", "claim-index-", "heartbeat-", "terminal-", "transition-", "applied-", "receipt-"]) test(`publication pinned ${kind} reader versus process rotation`, { timeout: 60000 }, async () => { + const f = await fixture("commit"); const reader = child(f, "commit", { recover: true, pauseRead: kind, marker: "reader", stateMaxBytes: 8192 }); let rotator; let recovery; + try { + const pinned = await reader.wait("cut"); assert.equal(pinned.pid, reader.process.pid); + rotator = child(f, "rotate", { stateMaxBytes: 8192 }); await rotator.finish(); + reader.process.send({ type: "release" }); + const exit = await reader.exit; assert.equal(exit.signal, null); + const result = reader.messages.find((event) => event.type === "done" || event.type === "error"); assert.ok(result); + if (result.type === "error") assert.match(result.message, /changed|disappeared|ENOENT|inode|retired/); + recovery = child(f, "commit", { recover: true, marker: "recovery", stateMaxBytes: 8192 }); + const settled = await recovery.finish(); assert.equal(settled.finals.length, 1); assert.equal(await projection(f), "base"); + } finally { await reader.stop(); if (rotator) await rotator.stop(); if (recovery) await recovery.stop(); await cleanup(f); } +}); + +test("publication killed staged owner excludes peers and never replays its callback", { timeout: 60000 }, async () => { + const f = await fixture("commit"); const owner = child(f, "commit", { pause: "callback:after:stage:state.json", marker: "held" }); const peers = []; + try { + const cut = await owner.wait("cut"); assert.equal(cut.pid, owner.process.pid); + for (const mode of ["commit", "rotate", "commit"]) peers.push(child(f, mode, { marker: `peer-${peers.length}` })); + for (const peer of peers) { + const exit = await peer.exit; assert.equal(exit.code, 1); assert.equal(exit.signal, null); + assert.match(peer.messages.find((event) => event.type === "error").message, /actively locked/); + } + assert.equal(await projection(f), "base"); owner.process.kill("SIGKILL"); assert.equal((await owner.exit).signal, "SIGKILL"); + const recovery = child(f, "commit", { recover: true, marker: "recovery" }); peers.push(recovery); await recovery.finish(); + assert.equal(await projection(f), "base"); assert.equal(await readFile(join(f.directory, "held.callbacks"), "utf8"), "entered\n"); + } finally { await owner.stop(); await Promise.all(peers.map((peer) => peer.stop())); await cleanup(f); } +}); + +test("publication recovers the linked rotation claim before its index CAS", { timeout: 60000 }, async () => { + const f = await fixture("rotate"); const owner = child(f, "rotate", { cut: 17 }); let recovery; + try { + const cut = await owner.wait("cut"); assert.equal(cut.pid, owner.process.pid); + assert.equal(cut.event.operation, "link"); assert.equal(cut.event.phase, "after"); assert.match(cut.event.path, /epoch\/claim-0000000000000002-HASH.json$/); + owner.process.kill("SIGKILL"); assert.equal((await owner.exit).signal, "SIGKILL"); + recovery = child(f, "rotate", { recover: true }); const result = await recovery.finish(); + assert.equal(result.finals.length, 1); assert.match(result.finals[0], /^generation-0000000000000002-/); assert.equal(await projection(f), "base"); + } finally { await owner.stop(); if (recovery) await recovery.stop(); await cleanup(f); } +}); + +for (const family of families) test(`publication helpers recover killed blocker owner ${family}`, { timeout: 60000 }, async (t) => { + const f = await fixture(`migrate-${family}`); const owner = child(f, `migrate-${family}`, { migrationPause: true }); + try { + const cut = await owner.wait("cut"); assert.equal(cut.pid, owner.process.pid); assert.equal(cut.phase, "after-blocker"); + owner.process.kill("SIGKILL"); assert.equal((await owner.exit).signal, "SIGKILL"); + await competition(t, f, Array(4).fill(`migrate-${family}`), 0); assert.equal(await projection(f), "base"); + } finally { await owner.stop(); await cleanup(f); } +}); + +for (const replacement of ["same-byte-inode", "unknown-entry"]) test(`publication last-proof SIGKILL rejects ${replacement}`, { timeout: 60000 }, async () => { + const reference = await fixture("rotate"); const recorder = child(reference, "rotate"); let index; + try { + const events = (await recorder.finish()).events; + index = events.findLastIndex((event) => event.phase === "after" && event.operation === "unlink"); + assert.ok(index > 0); + } finally { await recorder.stop(); await cleanup(reference); } + const f = await fixture("rotate"); const owner = child(f, "rotate", { cut: index }); let recovery; + try { + const cut = await owner.wait("cut"); assert.equal(cut.pid, owner.process.pid); assert.equal(cut.event.operation, "unlink"); + owner.process.kill("SIGKILL"); assert.equal((await owner.exit).signal, "SIGKILL"); + const record = JSON.parse(await readFile(`${f.state}.journal-v3/root.json`)); const root = join(`${f.state}.journal-v3`, record.goal); + const deleting = (await readdir(root)).filter((name) => name.startsWith(".deleting-")); assert.equal(deleting.length, 1); const path = join(root, deleting[0]); + if (replacement === "same-byte-inode") { + const preserved = join(f.directory, "preserved-deleting"); await rename(path, preserved); await cp(preserved, path, { recursive: true }); + await chmod(path, 0o700); for (const name of await readdir(path)) await chmod(join(path, name), 0o700); + } else await writeFile(join(path, "unknown"), "foreign", { mode: 0o600 }); + recovery = child(f, "rotate", { recover: true }); const exit = await recovery.exit; + assert.equal(exit.code, 1); assert.equal(exit.signal, null); + assert.match(recovery.messages.find((event) => event.type === "error").message, /inode|certificate|unexpected|closed|authority/); + assert.equal(await projection(f), "base"); + } finally { await owner.stop(); if (recovery) await recovery.stop(); await cleanup(f); } +}); diff --git a/scripts/pylon-publication.test.mjs b/scripts/pylon-publication.test.mjs index d5c35e8002..d955ec5913 100644 --- a/scripts/pylon-publication.test.mjs +++ b/scripts/pylon-publication.test.mjs @@ -1,3 +1,5 @@ +import "./pylon-publication-crash.test.mjs"; +import "./pylon-publication-stress.test.mjs"; import "./pylon-generation-migration.test.mjs"; import "./pylon-public-state.test.mjs"; import "./pylon-generation-operations.test.mjs"; From 7eaa37d796139d2eb6112783df1f38364efd74cc Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Fri, 11 Sep 2026 01:48:24 -0600 Subject: [PATCH 08/14] fix(publication): rotate before callback admission exhausts capacity Preserve mandatory core, crash, stress and maximum CI gates as explicit sequential suites. Fixes #53. --- .github/workflows/ci.yml | 14 +++++++++++++- .pylon/upstream-review.md | 4 ++-- docs/pylon-publication.md | 6 +++--- scripts/fixtures/publication-crash/support.mjs | 5 ++++- scripts/lib/pylon-consumer-lock.mjs | 12 +++++++++++- scripts/pylon-publication-stress.test.mjs | 13 +++++++++++++ scripts/pylon-publication.test.mjs | 2 -- 7 files changed, 46 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 718bef60a3..14be185d84 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -371,12 +371,24 @@ jobs: git rev-parse 'HEAD^{tree}' > publication-evidence/tree.txt node --version > publication-evidence/node.txt - - name: Complete publication, crash and repeated process suites + - name: Complete retained and current-public contract suite shell: bash run: | set -o pipefail npm run test:pylon-publication 2>&1 | tee publication-evidence/publication.log + - name: Every captured-process crash boundary + shell: bash + run: | + set -o pipefail + npm run test:pylon-publication-crash 2>&1 | tee publication-evidence/crash.log + + - name: Repeated process stress and read handoffs + shell: bash + run: | + set -o pipefail + npm run test:pylon-publication-stress 2>&1 | tee publication-evidence/stress.log + - name: Actual 16 MiB maximum, serialized after complete suite shell: bash run: | diff --git a/.pylon/upstream-review.md b/.pylon/upstream-review.md index d4f7fc075d..709a9e8555 100644 --- a/.pylon/upstream-review.md +++ b/.pylon/upstream-review.md @@ -298,6 +298,6 @@ Follow-up: Task10 builds/packs the exact merged tree into a private prefix and r - Reviewed upstream remains `1eee2938b4eeb7a4d72e17035adda669a89b63de` (v0.9.4). This distribution change resolves the filesystem-authority defects documented in [#53](https://github.com/pylon-code/prime-agent/issues/53); it does not advance the frozen upstream range or restore upstream R2/npm publication. - `protected-pylon-publication`: **redesign** the local consumer journal as v3 generations while retaining the protected preview/stable workflow policies. The public JSON projection and signed manifest formats are unchanged. A receipted creation identity selects one root; complete hidden generations publish by one directory rename. Exact predecessor claim/index, tip, inode and retirement-certificate commitments govern successor admission and bounded cleanup through deletion of the last proof link. - Migration requires explicit external legacy-process quiescence and API/CLI acknowledgement before mutation. It authenticates all five historical source families, installs the old-client-visible impossible-generation blocker before the guard/source retirement sequence, preserves original v1 inodes at exact 0500, retains the exact v2 source for provenance, and revalidates canonical completion. A foreign old-client bootstrap inode in the retirement gap is a conflict. Concurrent helpers may join independently authenticated winners; injected errors and unsafe pinned observations remain terminal. -- Callback staging never publishes before successful return. Exact canonical receipt inodes permit interrupted publication recovery without replaying callbacks. Rotation recovery reconstructs the pre-certificate source when its deterministic rotation claim was linked before the winning index CAS; it excludes only that exact claim and still requires the complete certificate and winning-CAS successor checks. -- Verification gates retain the protected and historical v2 matrices alongside current-public v3 verifier tests, complete mutation-boundary SIGKILL traces with fresh-process recovery, repeated four-process migration/rotation competition, pinned-read handoffs and last-proof replacement negatives. CI requires the complete suite followed sequentially by the actual 16 MiB maximum on Ubuntu 24.04 and macOS 15 under Node 22.23.2; both results are mandatory in `build-check-test`. Process crashes do not simulate physical power loss, and resource limits are separately bounded historical inventories, metadata and generation-journal budgets. +- Callback staging never publishes before successful return. Exact canonical receipt inodes permit interrupted publication recovery without replaying callbacks. Rotation recovery reconstructs the pre-certificate source when its deterministic rotation claim was linked before the winning index CAS; it excludes only that exact claim and still requires the complete certificate and winning-CAS successor checks. A claim that consumes the remaining heartbeat admission margin is released and rotated before callback entry, preserving recovery at small supported state limits. +- Verification gates retain the protected and historical v2 matrices alongside current-public v3 verifier tests, complete mutation-boundary SIGKILL traces with fresh-process recovery, repeated four-process migration/rotation competition, pinned-read handoffs and last-proof replacement negatives. CI requires separate retained/current-public, exhaustive crash and repeated stress suites followed sequentially by the actual 16 MiB maximum on Ubuntu 24.04 and macOS 15 under Node 22.23.2; both results are mandatory in `build-check-test`. Process crashes do not simulate physical power loss, and resource limits are separately bounded historical inventories, metadata and generation-journal budgets. - Revisit only when an upstream primitive preserves these exact local authority, bounded-work, migration and no-replay guarantees. Publication still requires current exact-source checks, independent review and every existing protected environment approval; filesystem tests do not authorize a release. diff --git a/docs/pylon-publication.md b/docs/pylon-publication.md index 3f850a0e8e..c36660b210 100644 --- a/docs/pylon-publication.md +++ b/docs/pylon-publication.md @@ -151,15 +151,15 @@ npm run release:pylon:rotate-consumer-journal -- \ Normal operations and rotation share one next-slot CAS. A rotation binds the exact latest winning claim/index, immutable tip, complete predecessor grammar and successor intent. Preparation converges a durable two-final cut before callback entry; successful rotation leaves one current final. Retirement and deletion preserve the observed predecessor inode. Cleanup removes only entries authenticated by the successor's committed retirement certificate; both certificate links survive until all ordinary authority is gone. After the last proof link, the successor permits cleanup only of that exact same-inode empty container. Byte-identical replacement directories and unknown extra entries remain conflicts. Dead temporaries and exact decided losers can be removed despite PID reuse; live unresolved writers block cleanup. -The supported state size is 16 MiB per field. Checkpoint bounds account for all three base64 fields (`4 * ceil(bytes / 3)` each) and the complete envelope. Separate explicit budgets are 256 MiB for each historical v1/v2 inventory, 512 MiB for migration metadata/receipts, and 512 MiB for the v3 generation journal; these are not one combined memory or disk cap. Root, epoch, receipt and aggregate bounds are checked before nested allocations. Receipt lookup indexes canonical inode identities once. A capacity refusal occurs before commitment; do not reduce the real maximum fixture to make a verification run pass. +The supported state size is 16 MiB per field. Checkpoint bounds account for all three base64 fields (`4 * ceil(bytes / 3)` each) and the complete envelope. Separate explicit budgets are 256 MiB for each historical v1/v2 inventory, 512 MiB for migration metadata/receipts, and 512 MiB for the v3 generation journal; these are not one combined memory or disk cap. Root, epoch, receipt and aggregate bounds are checked before nested allocations. Receipt lookup indexes canonical inode identities once. If claim/index publication consumes the remaining admission margin, the claim is released and rotation finishes before heartbeat scheduling or callback entry. A capacity refusal occurs before commitment; do not reduce the real maximum fixture to make a verification run pass. Every relied-on file is owned by the current numeric uid with exact `0600`; directories are exact `0700`, except proven original v1 directories frozen to `0500`. Reads are bounded, no-follow where Node supports it, exact to EOF, and checked against pinned inode/size/mtime/ctime observations. Only direct native unpinned discovery loss may restart bounded discovery. Hook or injected filesystem errors retain their identity, including `ENOENT`, `EIO` and `EPERM`; a later successful rename does not erase an earlier terminal error. These checks assume a trusted user-owned local parent and are not a portable `openat` sandbox. Unsupported numeric-uid platforms fail closed. ### Required publication verification -`npm run test:pylon-publication` retains the protected v2 regression oracle and separately exercises current public v3 preview/stable verification, migration, generation grammar, real child-process crashes and repeated four-process competition. The crash inventory fixes each scenario's complete ordered hook/path/occurrence trace. Every listed cut must be reached through an IPC barrier; the parent kills only its captured child with `SIGKILL` and requires a fresh process to recover. Trace changes and missing cuts fail the gate. TAP diagnostics record the scenario, exact boundary, PID/signal, recovered root inode, projection outcome and elapsed time. These are process-crash tests; ordered fsync assertions support durability sequencing but do not simulate physical power loss. +`npm run test:pylon-publication` retains the protected v2 regression oracle and exercises current public v3 preview/stable verification, migration and generation grammar. `npm run test:pylon-publication-crash` adds the exhaustive real child-process crash matrix; `npm run test:pylon-publication-stress` adds repeated four-process competition and read handoffs. These are three mandatory suites. The protected preview pack runs the retained/current-public contract suite within its existing job budget; exact-source admission separately requires the full CI aggregate, including every crash/stress/maximum gate. The crash inventory fixes each scenario's complete ordered hook/path/occurrence trace. Every listed cut must be reached through an IPC barrier; the parent kills only its captured child with `SIGKILL` and requires a fresh process to recover. Trace changes and missing cuts fail the gate. TAP diagnostics record the scenario, exact boundary, PID/signal, recovered root inode, projection outcome and elapsed time. These are process-crash tests; ordered fsync assertions support durability sequencing but do not simulate physical power loss. -CI requires this complete suite and the separate `npm run test:pylon-publication-maximum` on Ubuntu 24.04 and macOS 15 with Node 22.23.2. The actual 16 MiB maximum runs sequentially after the complete suite. Both platform results feed `build-check-test`; skipped, cancelled or failed publication jobs cannot make that aggregate succeed. Evidence artifacts bind logs to the tested commit and tree. Focused `test:pylon-publication-crash` and `test:pylon-publication-stress` commands are available for investigation and do not replace the complete gate. +CI requires the complete retained/current-public suite, exhaustive crash suite, repeated process stress suite and `npm run test:pylon-publication-maximum` on Ubuntu 24.04 and macOS 15 with Node 22.23.2, in that sequential order. The actual 16 MiB maximum starts only after all three preceding suites pass. Both platform results feed `build-check-test`; skipped, cancelled or failed publication jobs cannot make that aggregate succeed. Evidence artifacts bind logs to the tested commit and tree. For a complete local non-maximum proof, run `test:pylon-publication`, `test:pylon-publication-crash` and `test:pylon-publication-stress` sequentially on the same tree. Running only one is a component proof, never the full gate. ## Stable promotion diff --git a/scripts/fixtures/publication-crash/support.mjs b/scripts/fixtures/publication-crash/support.mjs index 3be0aa5af4..b49679775d 100644 --- a/scripts/fixtures/publication-crash/support.mjs +++ b/scripts/fixtures/publication-crash/support.mjs @@ -101,7 +101,10 @@ export function child(f, scenario, configuration = {}) { async finish() { const result = await exit; assert.deepEqual(result, { code: 0, signal: null }, `${JSON.stringify(messages.slice(-2))} ${output}`); - return messages.find((message) => message.type === "done"); + const done = messages.find((message) => message.type === "done"); + assert.ok(done, "Successful exit requires the complete terminal IPC proof"); + assert.equal(done.pid, process.pid); + return done; }, async stop() { if (!ended) process.kill("SIGKILL"); await exit; }, }; diff --git a/scripts/lib/pylon-consumer-lock.mjs b/scripts/lib/pylon-consumer-lock.mjs index 9dea41aff4..11e967c175 100644 --- a/scripts/lib/pylon-consumer-lock.mjs +++ b/scripts/lib/pylon-consumer-lock.mjs @@ -4961,7 +4961,17 @@ export async function withConsumerGenerationLock(root, authority, action, rawOpt } claim = { schemaVersion: 2, generation: slot, token: randomUUID(), type: "normal", ownerPid: process.pid, createdAtMs: options.now() }; if (!(await generationPublishClaim(snapshot, claim, options))) continue; - await generationOwnsClaim(snapshot, claim, options); + const owned = await generationOwnsClaim(snapshot, claim, options); + if (!(await generationHasOperationCapacity(owned.snapshot, options))) { + // Claim/index publication can consume the remaining admission margin. + // Release and rotate before heartbeat scheduling or callback entry. + const released = { schemaVersion: 2, generation: claim.generation, token: claim.token, outcome: "released" }; + const result = await generationWriteReceipt(owned.snapshot, `epoch/terminal-${generationName(claim.generation)}-${claim.token}.json`, metadataBytes(released), options, + async (temporary) => generationOwnsClaim(owned.snapshot, claim, options, temporary)); + if (!result.bytes.equals(metadataBytes(released))) throw new Error("Generation capacity release lost its exact claim ownership."); + await rotateConsumerGeneration(root, authority, options); + continue; + } acquired = true; break; } diff --git a/scripts/pylon-publication-stress.test.mjs b/scripts/pylon-publication-stress.test.mjs index 695dde2aeb..417f6f4d12 100644 --- a/scripts/pylon-publication-stress.test.mjs +++ b/scripts/pylon-publication-stress.test.mjs @@ -133,3 +133,16 @@ for (const replacement of ["same-byte-inode", "unknown-entry"]) test(`publicatio assert.equal(await projection(f), "base"); } finally { await owner.stop(); if (recovery) await recovery.stop(); await cleanup(f); } }); + +test("publication recovery rotates a claim that consumes heartbeat headroom before callback", { timeout: 60000 }, async () => { + const f = await fixture("recover-projection"); const owner = child(f, "recover-projection", { cut: 54 }); let recovery; + try { + const cut = await owner.wait("cut"); assert.equal(cut.pid, owner.process.pid); assert.equal(cut.event.operation, "link"); + assert.equal(cut.event.phase, "after"); assert.match(cut.event.path, /claim-index-0000000000000003.json$/); + owner.process.kill("SIGKILL"); assert.equal((await owner.exit).signal, "SIGKILL"); + recovery = child(f, "recover-projection", { recover: true, now: 1000000, marker: "capacity-recovery" }); + const result = await recovery.finish(); assert.equal(result.finals.length, 1); assert.match(result.finals[0], /^generation-0000000000000002-/); + assert.equal(await projection(f), "candidate"); + assert.equal(await readFile(join(f.directory, "capacity-recovery.callbacks"), "utf8"), "entered\n"); + } finally { await owner.stop(); if (recovery) await recovery.stop(); await cleanup(f); } +}); diff --git a/scripts/pylon-publication.test.mjs b/scripts/pylon-publication.test.mjs index d955ec5913..d5c35e8002 100644 --- a/scripts/pylon-publication.test.mjs +++ b/scripts/pylon-publication.test.mjs @@ -1,5 +1,3 @@ -import "./pylon-publication-crash.test.mjs"; -import "./pylon-publication-stress.test.mjs"; import "./pylon-generation-migration.test.mjs"; import "./pylon-public-state.test.mjs"; import "./pylon-generation-operations.test.mjs"; From c099fabb809286a4d14066287534f8350b234bfe Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Fri, 11 Sep 2026 02:10:11 -0600 Subject: [PATCH 09/14] fix(publication): complete durable recovery joins and checkpoint proofs Fixes #53 --- .pylon/upstream-review.md | 1 + docs/pylon-publication.md | 6 +- .../publication-crash/boundaries.json | 64 ++++++------ .../fixtures/publication-crash/support.mjs | 8 +- scripts/fixtures/publication-crash/worker.mjs | 2 +- scripts/lib/pylon-bounded-file.mjs | 4 +- scripts/lib/pylon-consumer-migration.mjs | 21 +++- scripts/pylon-bounded-file.test.mjs | 16 ++- scripts/pylon-publication-crash.test.mjs | 4 +- scripts/pylon-publication-durability.test.mjs | 99 +++++++++++++++++++ scripts/pylon-publication-stress.test.mjs | 22 +++-- scripts/pylon-publication.test.mjs | 1 + 12 files changed, 194 insertions(+), 54 deletions(-) create mode 100644 scripts/pylon-publication-durability.test.mjs diff --git a/.pylon/upstream-review.md b/.pylon/upstream-review.md index 709a9e8555..891319ebd2 100644 --- a/.pylon/upstream-review.md +++ b/.pylon/upstream-review.md @@ -298,6 +298,7 @@ Follow-up: Task10 builds/packs the exact merged tree into a private prefix and r - Reviewed upstream remains `1eee2938b4eeb7a4d72e17035adda669a89b63de` (v0.9.4). This distribution change resolves the filesystem-authority defects documented in [#53](https://github.com/pylon-code/prime-agent/issues/53); it does not advance the frozen upstream range or restore upstream R2/npm publication. - `protected-pylon-publication`: **redesign** the local consumer journal as v3 generations while retaining the protected preview/stable workflow policies. The public JSON projection and signed manifest formats are unchanged. A receipted creation identity selects one root; complete hidden generations publish by one directory rename. Exact predecessor claim/index, tip, inode and retirement-certificate commitments govern successor admission and bounded cleanup through deletion of the last proof link. - Migration requires explicit external legacy-process quiescence and API/CLI acknowledgement before mutation. It authenticates all five historical source families, installs the old-client-visible impossible-generation blocker before the guard/source retirement sequence, preserves original v1 inodes at exact 0500, retains the exact v2 source for provenance, and revalidates canonical completion. A foreign old-client bootstrap inode in the retirement gap is a conflict. Concurrent helpers may join independently authenticated winners; injected errors and unsafe pinned observations remain terminal. +- Interrupted receipt and guard joins complete and revalidate their canonical/staging directory fsync barriers before dependent publication or source retirement. Async and sync supplied openers preserve their original errors, including ELOOP/EISDIR; only direct native errors receive unsafe-path classification. Focused ordering regressions cover interrupted, concurrent and raced helpers, and the crash inventory includes missing nested parents. Repeated stress preserves exact prior history prefixes and cumulative acknowledged/returned values across all ten rounds. - Callback staging never publishes before successful return. Exact canonical receipt inodes permit interrupted publication recovery without replaying callbacks. Rotation recovery reconstructs the pre-certificate source when its deterministic rotation claim was linked before the winning index CAS; it excludes only that exact claim and still requires the complete certificate and winning-CAS successor checks. A claim that consumes the remaining heartbeat admission margin is released and rotated before callback entry, preserving recovery at small supported state limits. - Verification gates retain the protected and historical v2 matrices alongside current-public v3 verifier tests, complete mutation-boundary SIGKILL traces with fresh-process recovery, repeated four-process migration/rotation competition, pinned-read handoffs and last-proof replacement negatives. CI requires separate retained/current-public, exhaustive crash and repeated stress suites followed sequentially by the actual 16 MiB maximum on Ubuntu 24.04 and macOS 15 under Node 22.23.2; both results are mandatory in `build-check-test`. Process crashes do not simulate physical power loss, and resource limits are separately bounded historical inventories, metadata and generation-journal budgets. - Revisit only when an upstream primitive preserves these exact local authority, bounded-work, migration and no-replay guarantees. Publication still requires current exact-source checks, independent review and every existing protected environment approval; filesystem tests do not authorize a release. diff --git a/docs/pylon-publication.md b/docs/pylon-publication.md index c36660b210..dedfef53bc 100644 --- a/docs/pylon-publication.md +++ b/docs/pylon-publication.md @@ -138,6 +138,8 @@ Migration installs a receipted impossible-generation blocker, `claim-99999999999 Provenance is reconstructed from that final fenced source, including every retained/current record and the immutable projection snapshot. The selected construction root has its own durable inode receipt before any generation publication. A crash before root selection may leave an inert empty construction directory; resume allocates a new exclusive root instead of claiming the unknown inode. At most 64 such directories are permitted. The selected generation is built and published atomically, the projection is repaired through an owned claim, and canonical completion is separately receipted. Re-running the acknowledged migration revalidates the retained source and exact selected root. Concurrent helpers can join an independently authenticated winner; callers that observe conflicting bytes, inode replacement or an unsafe intermediate read fail closed. Already-running legacy processes are never supported concurrently. +Recovery and concurrent helpers complete the same durability sequence as the original publisher: canonical-parent fsync, fixed receipt rename, then receipt-directory fsync. An already fixed receipt still requires the joining caller to synchronize and revalidate its canonical and receipt parents. An already installed guard requires both its staging and canonical parents to be synchronized and its proof-backed inode revalidated before v2 retirement. + Preserve the original frozen v1 namespaces and `.journal.v2-retired`: they remain required provenance, not disposable generation history. Never remove a guard, reset the authority or copy a replacement journal over a failed migration. Preserve an offline backup and diagnose the exact reported conflict before retrying the same acknowledged command. ### Rotation, cleanup and resource limits @@ -153,7 +155,7 @@ Normal operations and rotation share one next-slot CAS. A rotation binds the exa The supported state size is 16 MiB per field. Checkpoint bounds account for all three base64 fields (`4 * ceil(bytes / 3)` each) and the complete envelope. Separate explicit budgets are 256 MiB for each historical v1/v2 inventory, 512 MiB for migration metadata/receipts, and 512 MiB for the v3 generation journal; these are not one combined memory or disk cap. Root, epoch, receipt and aggregate bounds are checked before nested allocations. Receipt lookup indexes canonical inode identities once. If claim/index publication consumes the remaining admission margin, the claim is released and rotation finishes before heartbeat scheduling or callback entry. A capacity refusal occurs before commitment; do not reduce the real maximum fixture to make a verification run pass. -Every relied-on file is owned by the current numeric uid with exact `0600`; directories are exact `0700`, except proven original v1 directories frozen to `0500`. Reads are bounded, no-follow where Node supports it, exact to EOF, and checked against pinned inode/size/mtime/ctime observations. Only direct native unpinned discovery loss may restart bounded discovery. Hook or injected filesystem errors retain their identity, including `ENOENT`, `EIO` and `EPERM`; a later successful rename does not erase an earlier terminal error. These checks assume a trusted user-owned local parent and are not a portable `openat` sandbox. Unsupported numeric-uid platforms fail closed. +Every relied-on file is owned by the current numeric uid with exact `0600`; directories are exact `0700`, except proven original v1 directories frozen to `0500`. Reads are bounded, no-follow where Node supports it, exact to EOF, and checked against pinned inode/size/mtime/ctime observations. Only direct native unpinned discovery loss may restart bounded discovery. Hook or injected filesystem errors retain their identity, including `ENOENT`, `EIO`, `EPERM`, `ELOOP` and `EISDIR`; a later successful rename does not erase an earlier terminal error. Native unsafe-file errors remain terminal path refusals. These checks assume a trusted user-owned local parent and are not a portable `openat` sandbox. Unsupported numeric-uid platforms fail closed. ### Required publication verification @@ -161,6 +163,8 @@ Every relied-on file is owned by the current numeric uid with exact `0600`; dire CI requires the complete retained/current-public suite, exhaustive crash suite, repeated process stress suite and `npm run test:pylon-publication-maximum` on Ubuntu 24.04 and macOS 15 with Node 22.23.2, in that sequential order. The actual 16 MiB maximum starts only after all three preceding suites pass. Both platform results feed `build-check-test`; skipped, cancelled or failed publication jobs cannot make that aggregate succeed. Evidence artifacts bind logs to the tested commit and tree. For a complete local non-maximum proof, run `test:pylon-publication`, `test:pylon-publication-crash` and `test:pylon-publication-stress` sequentially on the same tree. Running only one is a component proof, never the full gate. +The crash matrix includes both an existing state parent and two missing nested parents, with actual file/ancestor-directory fsync instrumentation. Repeated normal/rotation stress requires each earlier recovered history to remain an exact prefix, every cumulatively acknowledged value to survive exactly once, and every recorded value to have a returned callback marker. A callback may have committed durably even when its operation subsequently failed closed; such values also remain in the preserved prefix. + ## Stable promotion Run **Actions → Pylon stable promotion → Run workflow** on `pylon` with `operation=promote`, an immutable `preview_tag`, and no recovery or withdrawal identity. diff --git a/scripts/fixtures/publication-crash/boundaries.json b/scripts/fixtures/publication-crash/boundaries.json index 2de40a38cb..89b41ff4ba 100644 --- a/scripts/fixtures/publication-crash/boundaries.json +++ b/scripts/fixtures/publication-crash/boundaries.json @@ -3,29 +3,37 @@ "count": 76, "sha256": "f03c819ecf4e5cc6b4486326782cc2acdaea2c5dd5e18f2ef5137ad0a9b9c128" }, + "recover-dead-receipt": { + "count": 82, + "sha256": "f325ad16e669f36c143dd3e42a66ef4906e974589c81b84ba594b680750e753e" + }, "recover-commit-receipt": { - "count": 62, - "sha256": "3b12d0d6cec5e7bbe099ca4ec585d1365070943b379caba298c1ef5092ba9093" + "count": 82, + "sha256": "fe8239abceb9692d490fe9d0539bccb6476b09b0403532cac6953c792122826a" }, "recover-migration-intent": { - "count": 171, - "sha256": "580a32a420f41f8203b87c013264aa4dbbd76d42006ae6c4447f00d37a23becb" + "count": 199, + "sha256": "ad2daa927b5c4378189d44c6eee4dc0475371b0cd69a5183ebbfeabcb87c704f" }, "recover-projection": { - "count": 95, - "sha256": "8c53c6c524b603e3e54c1f22e6c611d4bd0a2db7b62f8fea8129221daa048a5c" + "count": 115, + "sha256": "c144f9c471b25fc107dfdcad513d97a358f0a836c2395bed2bb9f23213e45e8e" }, "fresh": { - "count": 200, - "sha256": "3fecb7e88d67f631d1eb04d0aeb749c43234a6d877380bebe1aa541260c37825" + "count": 220, + "sha256": "7c2067aa769e9b601fcc8e3ba0a3284182a1df737f3aef4324b1d5eacf9d52aa" + }, + "fresh-nested": { + "count": 232, + "sha256": "4723b58753989ebc367f9a2a4d5ce03d5a9dfeb804e543af95f60325511e263e" }, "commit": { - "count": 96, - "sha256": "681403b469c5ad85525886658dcb5539b75960977f2e26729ebf9e7a5c93047d" + "count": 116, + "sha256": "da47b8c92b86ecf9122cac5b3d73d328a2a694bd082431c622f8f2f1b6c2a9b5" }, "rotate": { - "count": 220, - "sha256": "51583d4f3d418c2d876aed13e5f5d2b560df22aee8916134ee0d82c37a1eec30" + "count": 240, + "sha256": "ab65cb7e113885e1043cfc993a8254581d6e44f2e7da240c24944317ae442770" }, "builder": { "count": 26, @@ -36,35 +44,31 @@ "sha256": "a8a77cefd6c48155f92f86880505791de3e6325eb31e73b1e0c598e2d155f2e0" }, "migrate-v1": { - "count": 192, - "sha256": "11e5dcc90f0a6129a553c79b0fd777c89a92cd4abbf71fee830660542db94b93" + "count": 220, + "sha256": "ac3502338da7a659713c74df5ebc327700bcd533e6906c430aa9be3957691139" }, "migrate-prior-retired-v1": { - "count": 213, - "sha256": "583c44f3b52dbe855815189a19be1278abcac4c8062b727ad1609dfa9c2f4139" + "count": 245, + "sha256": "b9f4e8739f4ac32fdb5f326af9c5a4f0b2e1266d50e17b645477a47eccb66ccc" }, "migrate-v2": { - "count": 192, - "sha256": "fc123fb9de7314ad5abafcc4ad50d1961656eb5674a7a89769ab5c6a81caf464" + "count": 220, + "sha256": "42f56e23e01fe0fdd0d09672e104454be4fd6b41e6a33ede0331bdc8aa1b4b92" }, "migrate-v1-v2": { - "count": 196, - "sha256": "5bd646380aab12e87493d82812cba42fb205b2627e51c3dd285b7905846a266e" + "count": 224, + "sha256": "d4376743040c554ad95287190d4f4731a236c654c703c1b9b1feac6ac78c2673" }, "migrate-prior-retired-v1-v2": { - "count": 217, - "sha256": "ca8f82d69210565d947a33ab0a1cf7701b45aad434fbd7ca6e613fc9a24f61a6" + "count": 249, + "sha256": "1cc9354d6430f059d24834332a262ba571f3c7858ac4b1271f6c9f86fd76ac3a" }, "migrate-v2-incomplete": { - "count": 203, - "sha256": "3b0da45c87fc68a5c56c0aefb2d434941348cd5ebde220e275fc172fe7c91ff2" + "count": 231, + "sha256": "b80165115441c6fc674bf8b215e33d924527735b69be4247a2da7c1dd28eb813" }, "migrate-v2-retained": { - "count": 205, - "sha256": "ab1b5d33dd6ca0c58aa708ee454f6086d74867c6123b643d6af831c3783d677d" - }, - "recover-dead-receipt": { - "count": 62, - "sha256": "1779428bb04a7a88567cb919b9f03065f004d2aa1f33bbc17b2518a040d7ebef" + "count": 237, + "sha256": "f9164e1a79e770a5e40e700ea9519430bc39befddc92ba2957809414b2fa7d30" } } diff --git a/scripts/fixtures/publication-crash/support.mjs b/scripts/fixtures/publication-crash/support.mjs index b49679775d..a0d40faabe 100644 --- a/scripts/fixtures/publication-crash/support.mjs +++ b/scripts/fixtures/publication-crash/support.mjs @@ -3,7 +3,7 @@ import { fork } from "node:child_process"; import { randomUUID } from "node:crypto"; import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, relative } from "node:path"; import * as old from "../protected-publication-v2/pylon-consumer-lock.mjs"; import { withConsumerStateLock, buildConsumerGeneration, publishConsumerGeneration } from "../../lib/pylon-consumer-lock.mjs"; import { generationBytes as bytes, generationDigest as digest, GENERATION_ZERO as ZERO } from "../../lib/pylon-generation-format.mjs"; @@ -18,7 +18,7 @@ export const recoveries = { "recover-migration-intent": { scenario: "migrate-v2", match: { hook: "migration", phase: "after", operation: "immutable-link", path: "state.json.journal-v3/intent.json" } }, "recover-projection": { scenario: "commit", match: { hook: "generation", phase: "after", operation: "create-projection" } }, }; -export const scenarios = [...Object.keys(recoveries), "fresh", "commit", "rotate", "builder", "loser-cleanup", ...families.map((family) => `migrate-${family}`), "migrate-v2-incomplete", "migrate-v2-retained"]; +export const scenarios = [...Object.keys(recoveries), "fresh", "fresh-nested", "commit", "rotate", "builder", "loser-cleanup", ...families.map((family) => `migrate-${family}`), "migrate-v2-incomplete", "migrate-v2-retained"]; export async function fixture(scenario) { if (recoveries[scenario]) { const setup = recoveries[scenario]; const f = await fixture(setup.scenario); const owner = child(f, setup.scenario, { cutMatch: setup.match, marker: "setup-owner" }); @@ -28,7 +28,7 @@ export async function fixture(scenario) { } const directory = await realpath(await mkdtemp(join(tmpdir(), "pylon-publication-crash-"))); await chmod(directory, 0o700); - const state = join(directory, "state.json"); + const state = join(directory, ...(scenario === "fresh-nested" ? ["one", "two"] : []), "state.json"); const root = join(directory, "journal"); const authority = { genesis: { statePath: state, stateBytes: null } }; if (["builder", "loser-cleanup"].includes(scenario)) { @@ -79,7 +79,7 @@ export async function cleanup(f) { await rm(f.directory, { recursive: true, force: true }); } export function child(f, scenario, configuration = {}) { - const process = fork(new URL("./worker.mjs", import.meta.url), [f.directory, scenario, JSON.stringify(configuration)], { stdio: ["ignore", "pipe", "pipe", "ipc"], execArgv: [] }); + const process = fork(new URL("./worker.mjs", import.meta.url), [f.directory, scenario, JSON.stringify({ ...configuration, stateRelative: relative(f.directory, f.state) })], { stdio: ["ignore", "pipe", "pipe", "ipc"], execArgv: [] }); const messages = []; const waiters = []; let output = ""; let ended = false; process.stdout.on("data", (data) => { output += data; }); process.stderr.on("data", (data) => { output += data; }); diff --git a/scripts/fixtures/publication-crash/worker.mjs b/scripts/fixtures/publication-crash/worker.mjs index 34bef79a92..1777e47f1a 100644 --- a/scripts/fixtures/publication-crash/worker.mjs +++ b/scripts/fixtures/publication-crash/worker.mjs @@ -7,7 +7,7 @@ const config = JSON.parse(encoded); const recoveryScenarios = { "recover-builder-checkpoint": "builder", "recover-dead-receipt": "commit", "recover-commit-receipt": "commit", "recover-migration-intent": "migrate-v2", "recover-projection": "commit" }; const scenario = recoveryScenarios[requestedScenario] ?? requestedScenario; if (recoveryScenarios[requestedScenario]) config.recover = true; -const state = join(directory, "state.json"); +const state = join(directory, config.stateRelative ?? "state.json"); const lowRoot = join(directory, "journal"); const authority = { genesis: { statePath: state, stateBytes: null } }; const send = (message) => new Promise((resolve, reject) => process.send(message, (error) => error ? reject(error) : resolve())); diff --git a/scripts/lib/pylon-bounded-file.mjs b/scripts/lib/pylon-bounded-file.mjs index 66081a3a98..0291df0281 100644 --- a/scripts/lib/pylon-bounded-file.mjs +++ b/scripts/lib/pylon-bounded-file.mjs @@ -159,7 +159,7 @@ export async function readBoundedRegularFile( handle = await openFile(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); } catch (error) { if (openFile === open && error?.code === "ENOENT") return null; - if (["ELOOP", "EISDIR"].includes(error?.code)) { + if (openFile === open && ["ELOOP", "EISDIR"].includes(error?.code)) { throw new Error(`${description} is not one regular non-symlink file.`); } throw error; @@ -288,7 +288,7 @@ export function readBoundedRegularFileSync( descriptor = openFile(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); } catch (error) { if (openFile === openSync && error?.code === "ENOENT") return null; - if (["ELOOP", "EISDIR"].includes(error?.code)) throw new Error(`${description} is not one regular non-symlink file.`); + if (openFile === openSync && ["ELOOP", "EISDIR"].includes(error?.code)) throw new Error(`${description} is not one regular non-symlink file.`); throw error; } try { diff --git a/scripts/lib/pylon-consumer-migration.mjs b/scripts/lib/pylon-consumer-migration.mjs index 76bf123c5c..12745c5bc5 100644 --- a/scripts/lib/pylon-consumer-migration.mjs +++ b/scripts/lib/pylon-consumer-migration.mjs @@ -122,7 +122,11 @@ async function canonicalAncestors(state, options, create = false) { await boundary(options, "after", "mkdir", path); await sync(path, options); const parentHandle = await options.openFile(dirname(path), constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); - try { await parentHandle.sync(); } finally { await parentHandle.close(); } + try { + await boundary(options, "before", "directory-sync", dirname(path)); + await parentHandle.sync(); + await boundary(options, "after", "directory-sync", dirname(path)); + } finally { await parentHandle.close(); } stat = await options.lstatEntry(path); } if (!stat?.isDirectory() || stat.isSymbolicLink()) throw new Error("Migration ancestor must be a canonical real directory."); @@ -140,6 +144,8 @@ async function receiptFor(meta, logical, target, expected, options, { publish = const wanted = await absent(target, options); if (wanted !== null) { if (!sameBytes(await file(target, options), expected)) throw new Error("Migration immutable target has conflicting exact bytes."); + await sync(dirname(target), options, true); + if (!same(wanted, await options.lstatEntry(target))) throw new Error("Migration canonical target changed inode during durability join."); let proof = await absent(fixed, options); if (proof === null) { if (!repair) throw new Error("Interrupted legacy migration receipt requires explicitly acknowledged migration."); @@ -157,6 +163,8 @@ async function receiptFor(meta, logical, target, expected, options, { publish = } if (proof === null || !same(proof, wanted) || ![2, ...(logical === "guard.json" ? [3, 4] : [])].includes(wanted.nlink) || proof.nlink !== wanted.nlink) throw new Error("Migration immutable record lacks its exact durable receipt inode."); if (!sameBytes(await file(fixed, options), expected)) throw new Error("Migration receipt bytes changed."); + await sync(receipts, options); + if (!same(wanted, await options.lstatEntry(target)) || !same(proof, await options.lstatEntry(fixed))) throw new Error("Migration receipt changed inode during durability join."); return identity(wanted); } if (!publish) throw new Error("Migration required immutable record is absent."); @@ -501,9 +509,14 @@ export function createConsumerMigrationApi(format) { const proofPath = join(meta, "guard.json"); await immutable(meta, "guard.json", guardFor(intent), options); const sourceIdentity = identity(await options.lstatEntry(proofPath)); + const synchronizeGuard = async () => { + await sync(meta, options); await sync(dirname(target), options); + if (!same(sourceIdentity, await options.lstatEntry(proofPath)) || !same(sourceIdentity, await options.lstatEntry(target)) || !sameBytes(await file(target, options), guard)) throw new Error("Migration guard durability join changed its exact proof-backed inode."); + }; // guard.json and its receipt are immutable; the third link is the downgrade fence. if (sameBytes(current, guard)) { if (!same(sourceIdentity, await options.lstatEntry(target))) throw new Error("Migration v3 guard is a different inode from its proof."); + await synchronizeGuard(); return; } const oldGuard = { schemaVersion: 1, kind: "pylon-consumer-legacy-lock-guard", statePathSha256: intent.statePathSha256 }; @@ -518,6 +531,7 @@ export function createConsumerMigrationApi(format) { if (same(sourceIdentity, before)) { const pending = await absent(temporary, options); if (pending !== null) { if (!same(sourceIdentity, pending)) throw new Error("Migration guard link was replaced."); await options.removeFile(temporary); await sync(meta, options); } + await synchronizeGuard(); return; } if (previousIdentity === null ? before !== null : !same(previousIdentity, before)) throw new Error("Migration guard changed inode before replacement."); @@ -526,7 +540,7 @@ export function createConsumerMigrationApi(format) { catch (error) { if (options.renameFile !== rename || error?.code !== "ENOENT" || !same(sourceIdentity, await absent(target, options))) throw error; } await boundary(options, "after", "guard-rename", target); if (!same(sourceIdentity, await options.lstatEntry(target)) || !sameBytes(await file(target, options), guard)) throw new Error("Migration guard replacement differs from its exact receipt-backed inode."); - await sync(meta, options); await sync(dirname(target), options); + await synchronizeGuard(); } async function finalAuthority(state, meta, intent, options) { const observed = await historicalFromIntent(state, intent, options, { final: true }); @@ -613,11 +627,14 @@ export function createConsumerMigrationApi(format) { await file(path, options, options.metadataMaxBytes, true, 0); if (stat.nlink === 2) { if (!same(stat, canonicalStat)) throw new Error("Migration linked temporary lost its canonical inode."); + await sync(dirname(target), options, true); + if (!same(canonicalStat, await options.lstatEntry(target))) throw new Error("Migration cleanup canonical target changed inode during durability join."); await boundary(options, "before", "receipt-rename", fixed); if (!same(stat, await options.lstatEntry(path)) || !same(directoryIdentity, await directory(receipts, options))) throw new Error("Migration receipt cleanup was fenced by inode replacement."); try { await options.renameFile(path, fixed); } catch (error) { if (options.renameFile !== rename || error?.code !== "ENOENT" || !same(stat, await absent(fixed, options))) throw error; } await boundary(options, "after", "receipt-rename", fixed); await sync(receipts, options); + if (!same(stat, await options.lstatEntry(fixed)) || !same(canonicalStat, await options.lstatEntry(target))) throw new Error("Migration repaired receipt changed inode during durability join."); } else { const decided = canonicalStat !== null && same(canonicalStat, await absent(fixed, options)); if (!decided && !dead(Number(match[1]), options)) throw new Error("Migration has a live unresolved receipt writer."); diff --git a/scripts/pylon-bounded-file.test.mjs b/scripts/pylon-bounded-file.test.mjs index 1c8f5e3128..5e2bce72a3 100644 --- a/scripts/pylon-bounded-file.test.mjs +++ b/scripts/pylon-bounded-file.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; -import { closeSync, lstatSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { closeSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { lstat, open } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -13,8 +13,8 @@ const readers = [ ]; for (const reader of readers) { - for (const code of ["ENOENT", "EIO", "EPERM"]) { - for (const stage of ["initial lstat", "open", "final lstat", "afterInitialPathStat", "afterInitialStat", "beforeFinalStat", "afterFinalStat", "stat", "read", "close"]) { + for (const code of ["ENOENT", "EIO", "EPERM", "ELOOP", "EISDIR"]) { + for (const stage of ["ELOOP", "EISDIR"].includes(code) ? ["open"] : ["initial lstat", "open", "final lstat", "afterInitialPathStat", "afterInitialStat", "beforeFinalStat", "afterFinalStat", "stat", "read", "close"]) { test(`bounded ${reader.name} retains injected ${code} identity at ${stage}`, async () => { const fixture = mkdtempSync(join(tmpdir(), "pylon-bounded-identity-")); const path = join(fixture, "input"); @@ -59,6 +59,16 @@ for (const reader of readers) { }); } } + for (const replacement of ["symlink", "directory"]) test(`bounded ${reader.name} rejects native ${replacement} replacement before open`, async () => { + const fixture = mkdtempSync(join(tmpdir(), "pylon-bounded-unsafe-")); const path = join(fixture, "input"); + try { + writeFileSync(path, "input"); + await assert.rejects(async () => reader.read(path, { maxBytes: 1024, hooks: { afterInitialPathStat() { + renameSync(path, `${path}.original`); + if (replacement === "symlink") symlinkSync(`${path}.original`, path); else mkdirSync(path); + } } }), /not one regular non-symlink file/); + } finally { rmSync(fixture, { recursive: true, force: true }); } + }); test(`bounded ${reader.name} classifies only native initial and open absence`, async () => { const fixture = mkdtempSync(join(tmpdir(), "pylon-bounded-native-")); const path = join(fixture, "input"); diff --git a/scripts/pylon-publication-crash.test.mjs b/scripts/pylon-publication-crash.test.mjs index 684db8cb2b..f46f1fe362 100644 --- a/scripts/pylon-publication-crash.test.mjs +++ b/scripts/pylon-publication-crash.test.mjs @@ -35,12 +35,12 @@ for (const scenario of scenarios) test(`publication crash boundary inventory ${s if (scenario.startsWith("recover-")) assert.equal(value, scenario === "recover-projection" ? "candidate" : scenario === "recover-builder-checkpoint" ? null : "base"); else if (scenario.startsWith("migrate-")) assert.equal(value, scenario.endsWith("incomplete") ? "base-advanced" : "base"); else if (scenario === "rotate") assert.equal(value, "base"); - else if (["fresh", "commit"].includes(scenario)) { + else if (["fresh", "fresh-nested", "commit"].includes(scenario)) { const markers = await readFile(join(f.directory, "owner.callbacks"), "utf8").catch((error) => { if (error.code === "ENOENT") return ""; throw error; }); assert.ok(["", "entered\n", "entered\nreturned\n"].includes(markers), "Original callback never replayed"); const returned = markers.endsWith("returned\n"); const committed = observed.events.some((event) => event.hook === "generation" && event.operation === "link" && event.phase === "after" && event.path.includes("/terminal-")); - assert.equal(value, returned && committed ? "candidate" : scenario === "fresh" ? null : "base"); + assert.equal(value, returned && committed ? "candidate" : scenario.startsWith("fresh") ? null : "base"); } t.diagnostic(JSON.stringify({ scenario, index, ...expected, pid: observed.pid, signal: "SIGKILL", recoveryPid: result.pid, root: result.root, preservedEntries: beforeRecovery.length, projection: value, elapsedMs: Math.round(performance.now() - start) })); } finally { await owner.stop(); if (recovery) await recovery.stop(); await cleanup(f); } diff --git a/scripts/pylon-publication-durability.test.mjs b/scripts/pylon-publication-durability.test.mjs new file mode 100644 index 0000000000..805d76e0f1 --- /dev/null +++ b/scripts/pylon-publication-durability.test.mjs @@ -0,0 +1,99 @@ +import assert from "node:assert/strict"; +import { open } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { test } from "node:test"; +import { migrateConsumerStateJournal, withConsumerStateLock } from "./lib/pylon-consumer-lock.mjs"; +import { generationDigest as digest } from "./lib/pylon-generation-format.mjs"; +import { cleanup, fixture } from "./fixtures/publication-crash/support.mjs"; + +const runtime = { stateMaxBytes: 1024, startHeartbeat: () => async () => {}, acknowledgeLegacyProcessesStopped: true }; +const isSync = (event, path) => event.phase === "after" && event.operation === "directory-sync" && event.path === path; +test("nested fresh state brackets every actual file and directory fsync", async () => { + const f = await fixture("fresh-nested"); const active = new Set(); const synchronized = []; + const boundary = (event) => { + if (!event.operation.includes("sync")) return; + if (event.phase === "before") active.add(event.path); else active.delete(event.path); + }; + try { + await withConsumerStateLock(f.state, async (_path, tx) => tx.commitState("candidate"), { ...runtime, hooks: { generationBoundary: boundary, migrationBoundary: boundary }, async openFile(path, ...args) { + const handle = await open(path, ...args); + return new Proxy(handle, { get(target, property) { + if (property === "sync") return async () => { assert.ok(active.has(path), `Actual fsync lacks its before boundary: ${path}`); synchronized.push(path); await target.sync(); }; + const value = Reflect.get(target, property); return typeof value === "function" ? value.bind(target) : value; + } }); + } }); + assert.equal(active.size, 0); assert.ok(synchronized.includes(f.directory)); assert.ok(synchronized.includes(join(f.directory, "one"))); + } finally { await cleanup(f); } +}); +for (const mode of ["interrupted", "concurrent-helper", "raced-installed-guard"]) test(`migration ${mode} synchronizes both guard parents before source retirement`, async () => { + const f = await fixture("migrate-v2"); const stop = new Error("captured guard publication cut"); const finished = new Error("guard ordering verified"); + const events = []; + const verify = async () => assert.rejects(migrateConsumerStateJournal(f.state, { ...runtime, hooks: { migrationBoundary(event) { + events.push(event); + if (event.phase === "before" && event.operation === "source-rename") { + assert.ok(events.some((entry) => isSync(entry, `${f.state}.journal-v3`)), "Guard staging parent must be durable before retirement"); + assert.ok(events.some((entry) => isSync(entry, dirname(f.state))), "Guard canonical parent must be durable before retirement"); + throw finished; + } + } } }), (error) => error === finished); + try { + if (mode === "raced-installed-guard") { + await assert.rejects(migrateConsumerStateJournal(f.state, { ...runtime, hooks: { async migrationBoundary(event) { + if (event.phase === "before" && event.operation === "guard-rename") { + await assert.rejects(migrateConsumerStateJournal(f.state, { ...runtime, hooks: { migrationBoundary(inner) { + if (inner.phase === "after" && inner.operation === "guard-rename") throw stop; + } } }), (error) => error === stop); + events.length = 0; + } + events.push(event); + if (event.phase === "before" && event.operation === "source-rename") { + assert.ok(events.some((entry) => isSync(entry, `${f.state}.journal-v3`)), "Raced guard staging parent must be durable"); + assert.ok(events.some((entry) => isSync(entry, dirname(f.state))), "Raced guard canonical parent must be durable"); + throw finished; + } + } } }), (error) => error === finished); + } else { + await assert.rejects(migrateConsumerStateJournal(f.state, { ...runtime, hooks: { async migrationBoundary(event) { + if (event.phase === "after" && event.operation === "guard-rename") { + if (mode === "concurrent-helper") await verify(); + throw stop; + } + } } }), (error) => error === stop); + if (mode === "interrupted") await verify(); + } + } finally { await cleanup(f); } +}); + +for (const mode of ["linked-blocker", "fixed-blocker", "concurrent-blocker", "linked-cleanup"]) test(`migration ${mode} rejoins canonical and receipt durability in order`, async () => { + const f = await fixture(mode === "linked-cleanup" ? "migrate-v1" : "migrate-v2"); const stop = new Error("captured receipt publication cut"); + const events = []; let target; let fixed; let repaired = false; + const resume = () => migrateConsumerStateJournal(f.state, { ...runtime, hooks: { metadataRead: { afterInitialStat({ path }) { + if (mode === "linked-cleanup" && path.includes("/.writing-") && path.endsWith(`${digest(Buffer.from("v1-lock/.pylon-consumer-v1-retired.json"))}.tmp`)) events.length = 0; + } }, migrationBoundary(event) { + events.push(event); + if (event.phase === "before" && event.operation === "receipt-rename" && event.path === fixed) { + assert.ok(events.some((entry) => isSync(entry, dirname(target))), "Canonical parent sync must precede fixed receipt rename"); + repaired = true; + } + }, afterMigrationBlocker() { + if (mode !== "linked-cleanup") assert.ok(events.some((entry) => isSync(entry, `${f.state}.journal-v3/receipts`)), "A joining helper must complete the fixed receipt durability barrier"); + } } }); + try { + await assert.rejects(migrateConsumerStateJournal(f.state, { ...runtime, hooks: { async migrationBoundary(event) { + const matches = mode === "linked-cleanup" ? event.path.endsWith("/.pylon-consumer-v1-retired.json") : event.path.endsWith("/claim-9999999999999999.json"); + if (event.phase === "after" && event.operation === "immutable-link" && matches) { + target = event.path; + const logical = mode === "linked-cleanup" ? "v1-lock/.pylon-consumer-v1-retired.json" : `blocker-v2/${dirname(target).split("/").at(-1)}`; + fixed = join(`${f.state}.journal-v3`, "receipts", `receipt-${digest(Buffer.from(logical))}.json`); + if (mode === "fixed-blocker") return; + if (mode === "concurrent-blocker") await resume(); + throw stop; + } + if (mode === "fixed-blocker" && fixed && event.phase === "after" && event.operation === "receipt-rename" && event.path === fixed) throw stop; + } } }), (error) => error === stop); + if (mode !== "concurrent-blocker") await resume(); + if (mode !== "fixed-blocker") assert.equal(repaired, true, "The actual linked receipt repair path must run"); + assert.ok(events.some((entry) => isSync(entry, dirname(target)))); + assert.ok(events.some((entry) => isSync(entry, `${f.state}.journal-v3/receipts`))); + } finally { await cleanup(f); } +}); diff --git a/scripts/pylon-publication-stress.test.mjs b/scripts/pylon-publication-stress.test.mjs index 417f6f4d12..749c76b58c 100644 --- a/scripts/pylon-publication-stress.test.mjs +++ b/scripts/pylon-publication-stress.test.mjs @@ -6,7 +6,7 @@ import { child, cleanup, fixture, families, projection } from "./fixtures/public const rounds = 10; const refusals = /actively locked|changed|disappeared|ENOENT|EEXIST|inode|receipt|publication|authority|conflicting|writer|ownership|claim|checkpoint|namespace|unfinished|incomplete|unsafe type, owner or exact permissions/; -async function competition(t, f, scenarios, round) { +async function competition(t, f, scenarios, round, historyProof) { const controlled = !scenarios[0].startsWith("migrate-") && round % 2 === 0; const workers = scenarios.map((scenario, index) => child(f, scenario, { pause: controlled && index === 0 ? "callback:after:stage:state.json" : undefined, append: true, stateMaxBytes: 8192, ready: true, marker: `round-${round}-${index}`, value: `value-${round}-${index}` })); try { @@ -35,11 +35,14 @@ async function competition(t, f, scenarios, round) { if (!scenarios[0].startsWith("migrate-")) { const value = await projection(f); const history = value === "base" ? [] : JSON.parse(value); assert.equal(new Set(history).size, history.length, "Each admitted value appears exactly once"); - for (const [index, result] of results.entries()) if (result.type === "done" && scenarios[index] === "commit") assert.ok(history.includes(`value-${round}-${index}`)); - for (const entry of history.filter((entry) => entry.startsWith(`value-${round}-`))) { - const index = Number(entry.split("-").at(-1)); - assert.equal(await readFile(join(f.directory, `round-${round}-${index}.callbacks`), "utf8"), "entered\nreturned\n"); + assert.deepEqual(history.slice(0, historyProof.previous.length), historyProof.previous, "Every prior recovered value remains an exact prefix across later competition"); + for (const [index, result] of results.entries()) if (result.type === "done" && scenarios[index] === "commit") historyProof.acknowledged.add(`value-${round}-${index}`); + for (const entry of historyProof.acknowledged) assert.ok(history.includes(entry), "Every acknowledged commit survives all later rounds"); + for (const entry of history) { + const match = /^value-([0-9]+)-([0-9]+)$/.exec(entry); assert.ok(match); + assert.equal(await readFile(join(f.directory, `round-${match[1]}-${match[2]}.callbacks`), "utf8"), "entered\nreturned\n"); } + historyProof.previous = history; } for (const result of results.filter((result) => result.type === "done")) assert.deepEqual(result.root, resumed.root); t.diagnostic(JSON.stringify({ round, scenarios, controlled, pids: workers.map((worker) => worker.process.pid), admitted: results.filter((result) => result.type === "done").length, refusals: results.filter((result) => result.type === "error").map((result) => result.message), root: resumed.root })); @@ -48,9 +51,10 @@ async function competition(t, f, scenarios, round) { test("publication repeated four-process normal and rotation competition", { timeout: 600000 }, async (t) => { const f = await fixture("commit"); + const historyProof = { previous: [], acknowledged: new Set() }; try { - for (let round = 0; round < rounds; round++) await competition(t, f, round % 2 ? ["commit", "commit", "rotate", "rotate"] : ["commit", "commit", "commit", "commit"], round); - assert.ok((await projection(f)) === "base" || Array.isArray(JSON.parse(await projection(f)))); + for (let round = 0; round < rounds; round++) await competition(t, f, round % 2 ? ["commit", "commit", "rotate", "rotate"] : ["commit", "commit", "commit", "commit"], round, historyProof); + assert.deepEqual(JSON.parse(await projection(f)), historyProof.previous); } finally { await cleanup(f); } }); for (const family of families) test(`publication repeated four-process migration ${family}`, { timeout: 600000 }, async (t) => { @@ -91,7 +95,7 @@ test("publication killed staged owner excludes peers and never replays its callb }); test("publication recovers the linked rotation claim before its index CAS", { timeout: 60000 }, async () => { - const f = await fixture("rotate"); const owner = child(f, "rotate", { cut: 17 }); let recovery; + const f = await fixture("rotate"); const owner = child(f, "rotate", { cutMatch: { hook: "generation", phase: "after", operation: "link", path: "state.json.journal-v3/journal-HASH-UUID/generation-0000000000000001-HASH/epoch/claim-0000000000000002-HASH.json" } }); let recovery; try { const cut = await owner.wait("cut"); assert.equal(cut.pid, owner.process.pid); assert.equal(cut.event.operation, "link"); assert.equal(cut.event.phase, "after"); assert.match(cut.event.path, /epoch\/claim-0000000000000002-HASH.json$/); @@ -135,7 +139,7 @@ for (const replacement of ["same-byte-inode", "unknown-entry"]) test(`publicatio }); test("publication recovery rotates a claim that consumes heartbeat headroom before callback", { timeout: 60000 }, async () => { - const f = await fixture("recover-projection"); const owner = child(f, "recover-projection", { cut: 54 }); let recovery; + const f = await fixture("recover-projection"); const owner = child(f, "recover-projection", { cutMatch: { hook: "generation", phase: "after", operation: "link", path: "state.json.journal-v3/journal-HASH-UUID/generation-0000000000000001-HASH/epoch/claim-index-0000000000000003.json" } }); let recovery; try { const cut = await owner.wait("cut"); assert.equal(cut.pid, owner.process.pid); assert.equal(cut.event.operation, "link"); assert.equal(cut.event.phase, "after"); assert.match(cut.event.path, /claim-index-0000000000000003.json$/); diff --git a/scripts/pylon-publication.test.mjs b/scripts/pylon-publication.test.mjs index d5c35e8002..f8e535a31a 100644 --- a/scripts/pylon-publication.test.mjs +++ b/scripts/pylon-publication.test.mjs @@ -1,4 +1,5 @@ import "./pylon-generation-migration.test.mjs"; +import "./pylon-publication-durability.test.mjs"; import "./pylon-public-state.test.mjs"; import "./pylon-generation-operations.test.mjs"; import "./pylon-generation.test.mjs"; From fc5d956c3893945fcc65b4c4b41db0e5f8aff12b Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Fri, 11 Sep 2026 02:14:57 -0600 Subject: [PATCH 10/14] test(publication): preserve wrapped opener errors in retained coverage Fixes #53 --- scripts/pylon-publication.test.mjs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/pylon-publication.test.mjs b/scripts/pylon-publication.test.mjs index f8e535a31a..c839b89438 100644 --- a/scripts/pylon-publication.test.mjs +++ b/scripts/pylon-publication.test.mjs @@ -1218,15 +1218,24 @@ test("bounded reads authenticate only exact pre-read link retirement transitions const symlinkMoved = join(fixture, "symlink-moved"); writeFileSync(symlinkSource, exactBytes); writeFileSync(symlinkTarget, exactBytes); - await rejectGenericChange(() => readBoundedRegularFile(symlinkSource, { + let symlinkOpenError; + await assert.rejects(() => readBoundedRegularFile(symlinkSource, { maxBytes: 1024, expectedSha256: exactDigest, openFile: async (path, flags) => { renameSync(path, symlinkMoved); symlinkSync(symlinkTarget, path); - return openFileHandle(path, flags); + try { return await openFileHandle(path, flags); } + catch (error) { symlinkOpenError = error; throw error; } }, - })); + }), (error) => { + assert.ok(symlinkOpenError); + assert.equal(error, symlinkOpenError); + assert.equal(error.code, "ELOOP"); + assert.equal(error instanceof BoundedFileLinkRetiredBeforeReadError, false); + assert.equal(error instanceof BoundedFileUnlinkedDuringReadError, false); + return true; + }); const ioCases = [ ["initial lstat", (_path, failure) => ({ lstatEntry: async () => { throw failure; } })], From c958fe117c2e37dd71abaeab1760966ee4cd460b Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Fri, 11 Sep 2026 02:35:46 -0600 Subject: [PATCH 11/14] test(publication): synchronize admitted retained rotation helpers Fixes #53 --- scripts/pylon-publication.test.mjs | 63 +++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/scripts/pylon-publication.test.mjs b/scripts/pylon-publication.test.mjs index c839b89438..da589f8c92 100644 --- a/scripts/pylon-publication.test.mjs +++ b/scripts/pylon-publication.test.mjs @@ -3834,27 +3834,54 @@ test("consumer state locking, recovery, transaction fencing, durability, and pat const source = ` import { rotateConsumerStateJournal } from ${JSON.stringify(pathToFileURL(resolve("scripts/fixtures/retained-publication-v2/pylon-consumer-lock.mjs")).href)}; try { - const result = await rotateConsumerStateJournal(process.argv[1]); + let admitted = false; + const result = await rotateConsumerStateJournal(process.argv[1], { hooks: { afterRotationEpochSync: async () => { + if (admitted) return; + admitted = true; + await new Promise((resolveRelease, rejectRelease) => { + process.once("message", (message) => { + if (message?.type !== "release-rotation") rejectRelease(new Error("Unexpected rotation barrier release")); + else resolveRelease(); + }); + process.send({ type: "rotation-admitted", pid: process.pid }, (error) => { if (error) rejectRelease(error); }); + }); + } } }); process.stdout.write(JSON.stringify(result)); } catch (error) { console.error(error.stack); process.exitCode = 1; } + process.disconnect(); `; const captured = captureChild( ["--input-type=module", "--eval", source, statePath], - { cwd: resolve("."), stdio: ["ignore", "pipe", "pipe"] }, + { cwd: resolve("."), stdio: ["ignore", "pipe", "pipe", "ipc"] }, ); + const ready = deferred(); + let admitted = false; + let protocolError = null; + captured.child.on("message", (message) => { + if (admitted || message?.type !== "rotation-admitted" || message.pid !== captured.child.pid) { + protocolError = new Error("Rotation admission must identify its captured child exactly once."); + ready.reject(protocolError); + return; + } + admitted = true; + ready.resolve(); + }); let stdout = ""; let stderr = ""; captured.child.stdout.setEncoding("utf8"); captured.child.stderr.setEncoding("utf8"); captured.child.stdout.on("data", (chunk) => { stdout += chunk; }); captured.child.stderr.on("data", (chunk) => { stderr += chunk; }); - return captured.closed.then((status) => { - if (status.code === 0 && status.spawnError === null) return JSON.parse(stdout); - throw closedChildError("rotation child", status, stderr); + const result = captured.closed.then((status) => { + if (status.code === 0 && status.spawnError === null && admitted && protocolError === null) return JSON.parse(stdout); + const error = protocolError ?? closedChildError("rotation child", status, stderr); + ready.reject(error); + throw error; }); + return { ready: ready.promise, result, release: () => captured.child.send({ type: "release-rotation" }) }; }; const startLegacyLockChild = async (statePath, afterReleasePath = "") => { const source = ` @@ -5972,15 +5999,23 @@ test("consumer state locking, recovery, transaction fencing, durability, and pat await transaction.commitState(bytes("concurrent-after-rotation")); }, { ...manualRuntime({ value: 3 }), maxLockGenerations: 2 }); - const rotationWavePath = join(fixture, "concurrent-rotation-process-wave.json"); - await withConsumerStateLock(rotationWavePath, async (_path, transaction) => { - await transaction.commitState(bytes("wave-anchor")); - }, manualRuntime({ value: 1 })); - const rotationWave = await Promise.all(Array.from({ length: 12 }, () => runRotationChild(rotationWavePath))); - assert.equal(rotationWave.every((result) => result.epoch === 2), true); - assert.deepEqual(rotationWave, Array.from({ length: 12 }, () => rotationWave[0])); - assert.equal(readdirSync(`${rotationWavePath}.journal`).filter((name) => name.startsWith("checkpoint-")).length, 1); - assert.equal(readdirSync(`${rotationWavePath}.journal`).filter((name) => name.startsWith("epoch-")).length, 1); + // The retained v2 oracle covers anchored convergence. Its unsupported cold + // discovery race is covered by the current v3 root-handoff and stress suites. + for (let round = 0; round < 3; round++) { + const rotationWavePath = join(fixture, `concurrent-rotation-process-wave-${round}.json`); + await withConsumerStateLock(rotationWavePath, async (_path, transaction) => { + await transaction.commitState(bytes("wave-anchor")); + }, manualRuntime({ value: 1 })); + const children = Array.from({ length: 12 }, () => runRotationChild(rotationWavePath)); + const [, rotationWave] = await Promise.all([ + Promise.all(children.map((child) => child.ready)).then(() => { for (const child of children) child.release(); }), + Promise.all(children.map((child) => child.result)), + ]); + assert.equal(rotationWave.every((result) => result.epoch === 2), true); + assert.deepEqual(rotationWave, Array.from({ length: 12 }, () => rotationWave[0])); + assert.equal(readdirSync(`${rotationWavePath}.journal`).filter((name) => name.startsWith("checkpoint-")).length, 1); + assert.equal(readdirSync(`${rotationWavePath}.journal`).filter((name) => name.startsWith("epoch-")).length, 1); + } for (const competitorHook of ["afterRotationEpochSync", "afterMetadataLink"]) { const competingRotationPath = join(fixture, `competing-rotation-${competitorHook}.json`); From b535551c53129c4ee431cd685e97a7b20730fec6 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Fri, 11 Sep 2026 02:38:21 -0600 Subject: [PATCH 12/14] ci(publication): allow complete platform proof within measured budget Fixes #53 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14be185d84..87206eaaaa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -344,7 +344,7 @@ jobs: needs: trust if: needs.trust.outputs.allowed == 'true' runs-on: ${{ matrix.os }} - timeout-minutes: 90 + timeout-minutes: 120 strategy: fail-fast: false matrix: From ac7300274676fbf80b0bcfbd5b20f9339a6ebe2d Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Fri, 11 Sep 2026 03:06:39 -0600 Subject: [PATCH 13/14] test(publication): distinguish retained limits from current handoff fixes #53 --- scripts/pylon-publication-handoff.test.mjs | 122 +++++++++++++ scripts/pylon-publication.test.mjs | 198 ++++++++------------- 2 files changed, 193 insertions(+), 127 deletions(-) create mode 100644 scripts/pylon-publication-handoff.test.mjs diff --git a/scripts/pylon-publication-handoff.test.mjs b/scripts/pylon-publication-handoff.test.mjs new file mode 100644 index 0000000000..cdc3a40791 --- /dev/null +++ b/scripts/pylon-publication-handoff.test.mjs @@ -0,0 +1,122 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { lstat, readFile, readdir } from "node:fs/promises"; +import { join } from "node:path"; +import { test } from "node:test"; +import { child, cleanup, fixture, projection } from "./fixtures/publication-crash/support.mjs"; +import { rotateConsumerStateJournal, withConsumerStateLock } from "./lib/pylon-consumer-lock.mjs"; + +const options = { stateMaxBytes: 8192, startHeartbeat: () => async () => {} }; +const tipSha256 = createHash("sha256").update("base").digest("hex"); +const rotationSource = ` +import { readdir } from "node:fs/promises"; +import { rotateConsumerStateJournal } from ${JSON.stringify(new URL("./lib/pylon-consumer-lock.mjs", import.meta.url).href)}; +const send = message => new Promise((resolve, reject) => process.send(message, error => error ? reject(error) : resolve())); +let paused = false; +try { + const result = await rotateConsumerStateJournal(process.argv[1], { + stateMaxBytes: 8192, startHeartbeat: () => async () => {}, + readDirectory: async path => { + const names = await readdir(path); + if (path === process.argv[2] && !paused) { + paused = true; + await new Promise((resolve, reject) => { + process.once("message", message => message?.type === "release" ? resolve() : reject(new Error("Unexpected root barrier release"))); + process.send({ type: "ready", pid: process.pid, names }, error => { if (error) reject(error); }); + }); + } + return names; + }, + }); + await send({ type: "done", pid: process.pid, result }); +} catch (error) { + await send({ type: "failure", pid: process.pid, message: error.stack }); + process.exitCode = 1; +} +process.disconnect(); +`; + +function rootReader(state, root) { + const process = spawn(globalThis.process.execPath, ["--input-type=module", "--eval", rotationSource, state, root], { stdio: ["ignore", "ignore", "pipe", "ipc"] }); + const messages = []; let ended = false; let output = ""; const waiters = []; + process.stderr.on("data", (data) => { output += data; }); + process.on("message", (message) => { messages.push(message); for (const wake of waiters.splice(0)) wake(); }); + const exit = new Promise((resolve, reject) => { + process.once("error", reject); + process.once("close", (code, signal) => { ended = true; resolve({ code, signal }); for (const wake of waiters.splice(0)) wake(); }); + }); + const watchdog = setTimeout(() => { if (!ended) process.kill("SIGKILL"); }, 60000); + exit.finally(() => clearTimeout(watchdog)); + return { process, messages, exit, + async ready() { + while (!messages.some((message) => message.type === "ready")) { + if (ended) throw new Error(`Root reader exited before its required barrier: ${JSON.stringify(messages)} ${output}`); + await new Promise((resolve) => waiters.push(resolve)); + } + const ready = messages.find((message) => message.type === "ready"); + assert.equal(ready.pid, process.pid); + return ready; + }, + async finish() { + assert.deepEqual(await exit, { code: 0, signal: null }, `${JSON.stringify(messages)} ${output}`); + assert.deepEqual(messages.map((message) => message.type), ["ready", "done"]); + assert.ok(messages.every((message) => message.pid === process.pid)); + return messages[1].result; + }, + async stop() { if (!ended) process.kill("SIGKILL"); await exit; }, + }; +} + +for (let round = 0; round < 3; round++) test(`current-public twelve-process stale root handoff round ${round + 1}`, { timeout: 90000 }, async () => { + const f = await fixture("commit"); const readers = []; let publisher; + try { + const record = JSON.parse(await readFile(`${f.state}.journal-v3/root.json`)); + const root = join(`${f.state}.journal-v3`, record.goal); + const initial = await readdir(root); + assert.equal(initial.length, 1); assert.match(initial[0], /^generation-0000000000000001-/); + for (let index = 0; index < 12; index++) readers.push(rootReader(f.state, root)); + for (const ready of await Promise.all(readers.map((reader) => reader.ready()))) assert.deepEqual(ready.names, initial); + publisher = child(f, "rotate", { stateMaxBytes: 8192 }); + const published = await publisher.finish(); + assert.equal(published.result, 2); assert.equal(published.finals.length, 1); + assert.match(published.finals[0], /^generation-0000000000000002-/); + assert.equal((await readdir(root)).includes(initial[0]), false); + for (const reader of readers) reader.process.send({ type: "release" }); + const results = await Promise.all(readers.map((reader) => reader.finish())); + assert.deepEqual(results, Array.from({ length: 12 }, () => ({ epoch: 2, tipSha256 }))); + assert.deepEqual(await readdir(root), published.finals); + assert.equal(await projection(f), "base"); + } finally { + await Promise.all(readers.map((reader) => reader.stop())); + if (publisher) await publisher.stop(); + await cleanup(f); + } +}); + +test("current-public pinned checkpoint retirement fails closed before a fresh callback converges", async () => { + const f = await fixture("commit"); let retired = false; let callbackCalls = 0; + try { + await assert.rejects(withConsumerStateLock(f.state, async () => { callbackCalls++; }, { ...options, hooks: { + metadataRead: { afterInitialStat: async ({ path, handle, stat }) => { + if (retired || !path.endsWith("/checkpoint.json") || !path.includes("/generation-")) return; + retired = true; + assert.equal(stat.nlink, 2); + assert.deepEqual(await rotateConsumerStateJournal(f.state, options), { epoch: 2, tipSha256 }); + await assert.rejects(lstat(path), { code: "ENOENT" }); + const after = await handle.stat(); + assert.deepEqual([after.dev, after.ino, after.size, after.mtimeMs], [stat.dev, stat.ino, stat.size, stat.mtimeMs]); + assert.equal(after.nlink, 0); + } }, + } }), { message: "Generation discovery checkpoint changed while it was read." }); + assert.equal(retired, true); assert.equal(callbackCalls, 0); assert.equal(await projection(f), "base"); + await withConsumerStateLock(f.state, async (_path, tx) => { + callbackCalls++; + assert.equal(tx.readStateBytes().toString(), "base"); + }, options); + assert.equal(callbackCalls, 1); assert.equal(await projection(f), "base"); + const record = JSON.parse(await readFile(`${f.state}.journal-v3/root.json`)); + const finals = await readdir(join(`${f.state}.journal-v3`, record.goal)); + assert.equal(finals.length, 1); assert.match(finals[0], /^generation-0000000000000002-/); + } finally { await cleanup(f); } +}); diff --git a/scripts/pylon-publication.test.mjs b/scripts/pylon-publication.test.mjs index da589f8c92..4e4e89e2b6 100644 --- a/scripts/pylon-publication.test.mjs +++ b/scripts/pylon-publication.test.mjs @@ -1,3 +1,4 @@ +import "./pylon-publication-handoff.test.mjs"; import "./pylon-generation-migration.test.mjs"; import "./pylon-publication-durability.test.mjs"; import "./pylon-public-state.test.mjs"; @@ -1814,7 +1815,7 @@ test("bounded reads authenticate every exact monotone retirement cut and confirm } }); -test("checkpoint readers converge across exact publication-link and retained-link retirement", async () => { +test("retained checkpoint proofs refuse stale link snapshots and converge across authenticated retirement", async () => { const deferred = () => { let resolvePromise; const promise = new Promise((resolvePromiseValue) => { resolvePromise = resolvePromiseValue; }); @@ -1872,75 +1873,60 @@ test("checkpoint readers converge across exact publication-link and retained-lin await withConsumerStateLock(linkedStatePath, async (_path, transaction) => { await transaction.commitState(linkedAnchor); }, runtime()); - const linkedJournalDirectory = `${linkedStatePath}.journal`; + const retainedLinkedJournal = consumerJournal(linkedStatePath); const publicLinkSynced = deferred(); const releasePublisher = deferred(); - const privateLinkRemoved = deferred(); let publishedCheckpointPath; - let publicPathStat; const publisher = rotateConsumerStateJournal(linkedStatePath, runtime({ afterMetadataDirectorySync: async ({ kind, path, linked }) => { if (kind !== "checkpoint" || !linked) return; publishedCheckpointPath = path; - publicPathStat = lstatSync(path); - assert.equal(publicPathStat.nlink, 2); + assert.equal(lstatSync(path).nlink, 2); publicLinkSynced.resolve(); await releasePublisher.promise; }, - }, { - removeFile: async (path, options) => { - rmSync(path, options); - if ( - publishedCheckpointPath && path !== publishedCheckpointPath && - basename(path).includes("-kcheckpoint-") - ) privateLinkRemoved.resolve(); - }, })); await publicLinkSynced.promise; - - const readerBeforeOpen = deferred(); - const releaseReaderOpen = deferred(); - let checkpointOpens = 0; - let readerPathStat; - let latestPathStat; - const linkedReader = rotateConsumerStateJournal(linkedStatePath, runtime({}, { - lstatEntry: async (path) => { - const entry = await lstatFile(path); - if (path === publishedCheckpointPath) latestPathStat = entry; - return entry; - }, - openFile: async (path, flags, mode) => { - if (path === publishedCheckpointPath) { - checkpointOpens += 1; - if (checkpointOpens === 2) { - readerPathStat = latestPathStat; - readerBeforeOpen.resolve(); - await releaseReaderOpen.promise; - } - } - return openFileHandle(path, flags, mode); + let retiredPublicationLink = false; + let linkedCallbackCalls = 0; + // V2 retains an exact checkpoint stat across proof reads. A legitimate + // publisher retirement can invalidate that snapshot before admission. + await assert.rejects(withConsumerStateLock(linkedStatePath, async () => { + linkedCallbackCalls += 1; + }, runtime({ + beforeStableCheckpointProofRead: async ({ path, target }) => { + if (path !== publishedCheckpointPath || retiredPublicationLink) return; + assert.equal(target, true); + retiredPublicationLink = true; + const before = lstatSync(path); + releasePublisher.resolve(); + await publisher; + const after = lstatSync(path); + assert.deepEqual( + [before.dev, before.ino, before.size, before.mtimeMs], + [after.dev, after.ino, after.size, after.mtimeMs], + ); + assert.deepEqual([before.nlink, after.nlink], [2, 1]); + assert.notEqual(before.ctimeMs, after.ctimeMs); }, - })); - await readerBeforeOpen.promise; - assert.equal(readerPathStat.nlink, 2); - releasePublisher.resolve(); - await privateLinkRemoved.promise; - const retiredLinkStat = lstatSync(publishedCheckpointPath); - assert.deepEqual( - [retiredLinkStat.dev, retiredLinkStat.ino, retiredLinkStat.size, retiredLinkStat.mtimeMs], - [publicPathStat.dev, publicPathStat.ino, publicPathStat.size, publicPathStat.mtimeMs], - ); - assert.deepEqual([publicPathStat.nlink, retiredLinkStat.nlink], [2, 1]); - assert.equal(publicPathStat.ctimeMs === retiredLinkStat.ctimeMs, false); - releaseReaderOpen.resolve(); - const [linkedReaderReceipt, publisherReceipt] = await Promise.all([linkedReader, publisher]); - assert.equal( - Buffer.from(JSON.stringify(linkedReaderReceipt)).equals(Buffer.from(JSON.stringify(publisherReceipt))), - true, - ); - assert.deepEqual(publisherReceipt, { epoch: 2, tipSha256: sha256Bytes(linkedAnchor) }); - assertFinalEpoch(linkedStatePath, sha256Bytes(linkedAnchor)); - assert.equal(dirname(publishedCheckpointPath), linkedJournalDirectory); + })), { + message: "Consumer high-water journal root has neither its byte-exact current checkpoint nor one exact immediate successor.", + }); + assert.equal(retiredPublicationLink, true); + assert.equal(linkedCallbackCalls, 0); + assert.deepEqual(readFileSync(linkedStatePath), linkedAnchor); + assert.deepEqual(await publisher, { epoch: 2, tipSha256: sha256Bytes(linkedAnchor) }); + const publishedCheckpoint = JSON.parse(readFileSync(publishedCheckpointPath)); + assert.deepEqual(readdirSync(`${linkedStatePath}.journal`).sort(), [ + ".owned-temporaries-v2", + basename(retainedLinkedJournal.checkpoint), + basename(retainedLinkedJournal.epoch), + basename(publishedCheckpointPath), + `epoch-0000000000000002-${publishedCheckpoint.epochId}`, + ].sort()); + assert.equal(publishedCheckpoint.epoch, 2); + assert.equal(publishedCheckpoint.anchorDigest, sha256Bytes(linkedAnchor)); + assert.deepEqual(readdirSync(join(`${linkedStatePath}.journal`, ".owned-temporaries-v2")), []); const unlinkedStatePath = join(fixture, "unlinked-retained.json"); const unlinkedAnchor = stateBytes("unlinked-retained-anchor"); @@ -3830,59 +3816,6 @@ test("consumer state locking, recovery, transaction fencing, durability, and pat throw closedChildError("consumer child", status, stderr); }); }; - const runRotationChild = (statePath) => { - const source = ` - import { rotateConsumerStateJournal } from ${JSON.stringify(pathToFileURL(resolve("scripts/fixtures/retained-publication-v2/pylon-consumer-lock.mjs")).href)}; - try { - let admitted = false; - const result = await rotateConsumerStateJournal(process.argv[1], { hooks: { afterRotationEpochSync: async () => { - if (admitted) return; - admitted = true; - await new Promise((resolveRelease, rejectRelease) => { - process.once("message", (message) => { - if (message?.type !== "release-rotation") rejectRelease(new Error("Unexpected rotation barrier release")); - else resolveRelease(); - }); - process.send({ type: "rotation-admitted", pid: process.pid }, (error) => { if (error) rejectRelease(error); }); - }); - } } }); - process.stdout.write(JSON.stringify(result)); - } catch (error) { - console.error(error.stack); - process.exitCode = 1; - } - process.disconnect(); - `; - const captured = captureChild( - ["--input-type=module", "--eval", source, statePath], - { cwd: resolve("."), stdio: ["ignore", "pipe", "pipe", "ipc"] }, - ); - const ready = deferred(); - let admitted = false; - let protocolError = null; - captured.child.on("message", (message) => { - if (admitted || message?.type !== "rotation-admitted" || message.pid !== captured.child.pid) { - protocolError = new Error("Rotation admission must identify its captured child exactly once."); - ready.reject(protocolError); - return; - } - admitted = true; - ready.resolve(); - }); - let stdout = ""; - let stderr = ""; - captured.child.stdout.setEncoding("utf8"); - captured.child.stderr.setEncoding("utf8"); - captured.child.stdout.on("data", (chunk) => { stdout += chunk; }); - captured.child.stderr.on("data", (chunk) => { stderr += chunk; }); - const result = captured.closed.then((status) => { - if (status.code === 0 && status.spawnError === null && admitted && protocolError === null) return JSON.parse(stdout); - const error = protocolError ?? closedChildError("rotation child", status, stderr); - ready.reject(error); - throw error; - }); - return { ready: ready.promise, result, release: () => captured.child.send({ type: "release-rotation" }) }; - }; const startLegacyLockChild = async (statePath, afterReleasePath = "") => { const source = ` import { open } from "node:fs/promises"; @@ -5999,23 +5932,34 @@ test("consumer state locking, recovery, transaction fencing, durability, and pat await transaction.commitState(bytes("concurrent-after-rotation")); }, { ...manualRuntime({ value: 3 }), maxLockGenerations: 2 }); - // The retained v2 oracle covers anchored convergence. Its unsupported cold - // discovery race is covered by the current v3 root-handoff and stress suites. - for (let round = 0; round < 3; round++) { - const rotationWavePath = join(fixture, `concurrent-rotation-process-wave-${round}.json`); - await withConsumerStateLock(rotationWavePath, async (_path, transaction) => { - await transaction.commitState(bytes("wave-anchor")); - }, manualRuntime({ value: 1 })); - const children = Array.from({ length: 12 }, () => runRotationChild(rotationWavePath)); - const [, rotationWave] = await Promise.all([ - Promise.all(children.map((child) => child.ready)).then(() => { for (const child of children) child.release(); }), - Promise.all(children.map((child) => child.result)), - ]); - assert.equal(rotationWave.every((result) => result.epoch === 2), true); - assert.deepEqual(rotationWave, Array.from({ length: 12 }, () => rotationWave[0])); - assert.equal(readdirSync(`${rotationWavePath}.journal`).filter((name) => name.startsWith("checkpoint-")).length, 1); - assert.equal(readdirSync(`${rotationWavePath}.journal`).filter((name) => name.startsWith("epoch-")).length, 1); - } + // V2 cannot recover a cold root listing whose checkpoint is retired before + // its first read. Current-public multiprocess handoff is tested separately. + const oldRootHandoffPath = join(fixture, "retained-cold-root-handoff.json"); + const oldRootAnchor = bytes("wave-anchor"); + await withConsumerStateLock(oldRootHandoffPath, async (_path, transaction) => { + await transaction.commitState(oldRootAnchor); + }, manualRuntime({ value: 1 })); + let oldRootHandedOff = false; + let oldRootCallbackCalls = 0; + await assert.rejects(withConsumerStateLock(oldRootHandoffPath, async () => { + oldRootCallbackCalls += 1; + }, { + ...manualRuntime({ value: 2 }), + readDirectory: async (path) => { + const names = await readDirectoryEntries(path); + if (path === `${oldRootHandoffPath}.journal` && !oldRootHandedOff) { + oldRootHandedOff = true; + await rotateConsumerStateJournal(oldRootHandoffPath, manualRuntime({ value: 3 })); + await rotateConsumerStateJournal(oldRootHandoffPath, manualRuntime({ value: 3 })); + const currentNames = await readDirectoryEntries(path); + assert.ok(names.some((name) => name.startsWith("checkpoint-") && !currentNames.includes(name))); + } + return names; + }, + }), { message: "Consumer high-water journal lost its current checkpoint during an authenticated scan." }); + assert.equal(oldRootHandedOff, true); + assert.equal(oldRootCallbackCalls, 0); + assert.deepEqual(readFileSync(oldRootHandoffPath), oldRootAnchor); for (const competitorHook of ["afterRotationEpochSync", "afterMetadataLink"]) { const competingRotationPath = join(fixture, `competing-rotation-${competitorHook}.json`); From 890e410ccab2c5a5947dd807e364eed23016fee9 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Fri, 11 Sep 2026 04:14:50 -0600 Subject: [PATCH 14/14] fix(publication): finish capacity admission before callbacks fixes #53 --- .github/workflows/ci.yml | 8 +- docs/pylon-publication.md | 4 +- scripts/fixtures/publication-crash/worker.mjs | 11 ++ scripts/lib/pylon-consumer-lock.mjs | 59 +++++++--- scripts/pylon-generation-operations.test.mjs | 46 +++++++- scripts/pylon-publication-stress.test.mjs | 103 +++++++++++++++++- 6 files changed, 204 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87206eaaaa..bb94b45dad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -377,17 +377,17 @@ jobs: set -o pipefail npm run test:pylon-publication 2>&1 | tee publication-evidence/publication.log - - name: Every captured-process crash boundary + - name: Repeated process stress and read handoffs shell: bash run: | set -o pipefail - npm run test:pylon-publication-crash 2>&1 | tee publication-evidence/crash.log + npm run test:pylon-publication-stress 2>&1 | tee publication-evidence/stress.log - - name: Repeated process stress and read handoffs + - name: Every captured-process crash boundary shell: bash run: | set -o pipefail - npm run test:pylon-publication-stress 2>&1 | tee publication-evidence/stress.log + npm run test:pylon-publication-crash 2>&1 | tee publication-evidence/crash.log - name: Actual 16 MiB maximum, serialized after complete suite shell: bash diff --git a/docs/pylon-publication.md b/docs/pylon-publication.md index dedfef53bc..553ba74c47 100644 --- a/docs/pylon-publication.md +++ b/docs/pylon-publication.md @@ -153,7 +153,7 @@ npm run release:pylon:rotate-consumer-journal -- \ Normal operations and rotation share one next-slot CAS. A rotation binds the exact latest winning claim/index, immutable tip, complete predecessor grammar and successor intent. Preparation converges a durable two-final cut before callback entry; successful rotation leaves one current final. Retirement and deletion preserve the observed predecessor inode. Cleanup removes only entries authenticated by the successor's committed retirement certificate; both certificate links survive until all ordinary authority is gone. After the last proof link, the successor permits cleanup only of that exact same-inode empty container. Byte-identical replacement directories and unknown extra entries remain conflicts. Dead temporaries and exact decided losers can be removed despite PID reuse; live unresolved writers block cleanup. -The supported state size is 16 MiB per field. Checkpoint bounds account for all three base64 fields (`4 * ceil(bytes / 3)` each) and the complete envelope. Separate explicit budgets are 256 MiB for each historical v1/v2 inventory, 512 MiB for migration metadata/receipts, and 512 MiB for the v3 generation journal; these are not one combined memory or disk cap. Root, epoch, receipt and aggregate bounds are checked before nested allocations. Receipt lookup indexes canonical inode identities once. If claim/index publication consumes the remaining admission margin, the claim is released and rotation finishes before heartbeat scheduling or callback entry. A capacity refusal occurs before commitment; do not reduce the real maximum fixture to make a verification run pass. +The supported state size is 16 MiB per field. Checkpoint bounds account for all three base64 fields (`4 * ceil(bytes / 3)` each) and the complete envelope. Separate explicit budgets are 256 MiB for each historical v1/v2 inventory, 512 MiB for migration metadata/receipts, and 512 MiB for the v3 generation journal; these are not one combined memory or disk cap. Root, epoch, receipt and aggregate bounds are checked before nested allocations. Receipt lookup indexes canonical inode identities once. If claim/index publication or preparation heartbeats consume the remaining admission margin, the exact owned claim is released and rotation finishes before heartbeat scheduling or callback entry. Preparation errors remain terminal with their original identity; capacity reacquisition never replays a callback. A capacity refusal occurs before commitment; do not reduce the real maximum fixture to make a verification run pass. Every relied-on file is owned by the current numeric uid with exact `0600`; directories are exact `0700`, except proven original v1 directories frozen to `0500`. Reads are bounded, no-follow where Node supports it, exact to EOF, and checked against pinned inode/size/mtime/ctime observations. Only direct native unpinned discovery loss may restart bounded discovery. Hook or injected filesystem errors retain their identity, including `ENOENT`, `EIO`, `EPERM`, `ELOOP` and `EISDIR`; a later successful rename does not erase an earlier terminal error. Native unsafe-file errors remain terminal path refusals. These checks assume a trusted user-owned local parent and are not a portable `openat` sandbox. Unsupported numeric-uid platforms fail closed. @@ -161,7 +161,7 @@ Every relied-on file is owned by the current numeric uid with exact `0600`; dire `npm run test:pylon-publication` retains the protected v2 regression oracle and exercises current public v3 preview/stable verification, migration and generation grammar. `npm run test:pylon-publication-crash` adds the exhaustive real child-process crash matrix; `npm run test:pylon-publication-stress` adds repeated four-process competition and read handoffs. These are three mandatory suites. The protected preview pack runs the retained/current-public contract suite within its existing job budget; exact-source admission separately requires the full CI aggregate, including every crash/stress/maximum gate. The crash inventory fixes each scenario's complete ordered hook/path/occurrence trace. Every listed cut must be reached through an IPC barrier; the parent kills only its captured child with `SIGKILL` and requires a fresh process to recover. Trace changes and missing cuts fail the gate. TAP diagnostics record the scenario, exact boundary, PID/signal, recovered root inode, projection outcome and elapsed time. These are process-crash tests; ordered fsync assertions support durability sequencing but do not simulate physical power loss. -CI requires the complete retained/current-public suite, exhaustive crash suite, repeated process stress suite and `npm run test:pylon-publication-maximum` on Ubuntu 24.04 and macOS 15 with Node 22.23.2, in that sequential order. The actual 16 MiB maximum starts only after all three preceding suites pass. Both platform results feed `build-check-test`; skipped, cancelled or failed publication jobs cannot make that aggregate succeed. Evidence artifacts bind logs to the tested commit and tree. For a complete local non-maximum proof, run `test:pylon-publication`, `test:pylon-publication-crash` and `test:pylon-publication-stress` sequentially on the same tree. Running only one is a component proof, never the full gate. +CI requires the complete retained/current-public suite, repeated process stress suite, exhaustive crash suite and `npm run test:pylon-publication-maximum` on Ubuntu 24.04 and macOS 15 with Node 22.23.2, in that sequential order. The actual 16 MiB maximum starts only after all three preceding suites pass. Both platform results feed `build-check-test`; skipped, cancelled or failed publication jobs cannot make that aggregate succeed. Evidence artifacts bind logs to the tested commit and tree. For a complete local non-maximum proof, run `test:pylon-publication`, `test:pylon-publication-stress` and `test:pylon-publication-crash` sequentially on the same tree. Running only one is a component proof, never the full gate. The crash matrix includes both an existing state parent and two missing nested parents, with actual file/ancestor-directory fsync instrumentation. Repeated normal/rotation stress requires each earlier recovered history to remain an exact prefix, every cumulatively acknowledged value to survive exactly once, and every recorded value to have a returned callback marker. A callback may have committed durably even when its operation subsequently failed closed; such values also remain in the preserved prefix. diff --git a/scripts/fixtures/publication-crash/worker.mjs b/scripts/fixtures/publication-crash/worker.mjs index 1777e47f1a..1ece03bfb5 100644 --- a/scripts/fixtures/publication-crash/worker.mjs +++ b/scripts/fixtures/publication-crash/worker.mjs @@ -50,6 +50,17 @@ if (config.pauseRead === "receipt-") options.lstatEntry = async (path) => { if (basename(path).startsWith("receipt-")) await pauseRead(path, "receipt-"); return stat; }; +if (config.pauseMigrationMetaRead) options.readDirectory = async (path) => { + const names = await readdir(path); + if (path === `${state}.journal-v3` && !readPaused) { + readPaused = true; + await send({ type: "cut", pid: process.pid, phase: "pre-intent-meta-read", names }); + await new Promise((resolve, reject) => process.once("message", (message) => { + if (message?.type === "release") resolve(); else reject(new Error("Unexpected migration metadata barrier release.")); + })); + } + return names; +}; async function callback(_path, tx) { const marker = config.marker ?? "owner"; await appendFile(join(directory, `${marker}.callbacks`), "entered\n", { mode: 0o600 }); diff --git a/scripts/lib/pylon-consumer-lock.mjs b/scripts/lib/pylon-consumer-lock.mjs index 11e967c175..6899c1a72c 100644 --- a/scripts/lib/pylon-consumer-lock.mjs +++ b/scripts/lib/pylon-consumer-lock.mjs @@ -4923,6 +4923,24 @@ async function generationHasOperationCapacity(snapshot, options) { return certificateBytes + 2048 <= options.metadataMaxBytes && snapshot.epochRecords.size + 16 < GENERATION_EPOCH_MAX_ENTRIES && snapshot.receiptEntries.length + 32 < GENERATION_RECEIPT_MAX_ENTRIES && await generationRootPreflight(dirname(snapshot.path), options) + reserve <= options.maxJournalBytes; } +async function generationReleaseOwnedClaim(snapshot, claim, options) { + const owned = await generationOwnsClaim(snapshot, claim, options); + const released = { schemaVersion: 2, generation: claim.generation, token: claim.token, outcome: "released" }; + const result = await generationWriteReceipt(owned.snapshot, `epoch/terminal-${generationName(claim.generation)}-${claim.token}.json`, metadataBytes(released), options, + async (temporary) => generationOwnsClaim(owned.snapshot, claim, options, temporary)); + if (!result.bytes.equals(metadataBytes(released))) throw new Error("Generation capacity release lost its exact claim ownership."); +} + +async function generationBeatOwnedClaim(snapshot, claim, options) { + const owned = await generationOwnsClaim(snapshot, claim, options); + if (!(await generationHasOperationCapacity(owned.snapshot, options))) return false; + const refreshedAtMs = options.now(); + const value = { schemaVersion: 2, generation: claim.generation, token: claim.token, refreshedAtMs }; + await generationWriteReceipt(owned.snapshot, `epoch/heartbeat-${generationName(claim.generation)}-${claim.token}-${generationName(refreshedAtMs)}.json`, metadataBytes(value), options, + async (temporary) => generationOwnsClaim(snapshot, claim, options, temporary)); + return true; +} + export async function withConsumerGenerationLock(root, authority, action, rawOptions = {}) { if (typeof action !== "function" || !authority?.genesis?.statePath) throw new Error("Generation operation requires an action and exact genesis authority."); const options = generationOptions(rawOptions); @@ -4930,6 +4948,7 @@ export async function withConsumerGenerationLock(root, authority, action, rawOpt await generationDirectory(dirname(statePath), options); let snapshot; let claim; + let base; let acquired = false; for (let attempt = 0; attempt < PROJECTION_RETRY_LIMIT; attempt += 1) { snapshot = await prepareConsumerGeneration(root, authority, options); @@ -4965,13 +4984,32 @@ export async function withConsumerGenerationLock(root, authority, action, rawOpt if (!(await generationHasOperationCapacity(owned.snapshot, options))) { // Claim/index publication can consume the remaining admission margin. // Release and rotate before heartbeat scheduling or callback entry. - const released = { schemaVersion: 2, generation: claim.generation, token: claim.token, outcome: "released" }; - const result = await generationWriteReceipt(owned.snapshot, `epoch/terminal-${generationName(claim.generation)}-${claim.token}.json`, metadataBytes(released), options, - async (temporary) => generationOwnsClaim(owned.snapshot, claim, options, temporary)); - if (!result.bytes.equals(metadataBytes(released))) throw new Error("Generation capacity release lost its exact claim ownership."); + await generationReleaseOwnedClaim(owned.snapshot, claim, options); await rotateConsumerGeneration(root, authority, options); continue; } + try { + let prepared = await generationBeatOwnedClaim(snapshot, claim, options); + if (prepared) { + await options.hooks?.afterClaim?.({ claim }); + base = await generationRepairProjection(root, authority, statePath, options); + prepared = await generationBeatOwnedClaim(snapshot, claim, options); + } + if (prepared) { + const current = await generationOwnsClaim(snapshot, claim, options); + prepared = await generationHasOperationCapacity(current.snapshot, options); + } + if (!prepared) { + // Preparation heartbeats also consume certificate space. No callback + // or scheduler has started, so release this exact claim and rotate. + await generationReleaseOwnedClaim(snapshot, claim, options); + await rotateConsumerGeneration(root, authority, options); + continue; + } + } catch (error) { + try { await generationReleaseOwnedClaim(snapshot, claim, options); } catch { /* Preserve the original preparation error for recovery. */ } + throw error; + } acquired = true; break; } @@ -4984,12 +5022,7 @@ export async function withConsumerGenerationLock(root, authority, action, rawOpt const beat = async () => { if (!active) return false; try { - const owned = await generationOwnsClaim(snapshot, claim, options); - if (!(await generationHasOperationCapacity(owned.snapshot, options))) throw new Error("Generation heartbeat must quiesce to preserve rotation headroom."); - const refreshedAtMs = options.now(); - const value = { schemaVersion: 2, generation: claim.generation, token: claim.token, refreshedAtMs }; - await generationWriteReceipt(owned.snapshot, `epoch/heartbeat-${generationName(claim.generation)}-${claim.token}-${generationName(refreshedAtMs)}.json`, metadataBytes(value), options, - async (temporary) => generationOwnsClaim(snapshot, claim, options, temporary)); + if (!(await generationBeatOwnedClaim(snapshot, claim, options))) throw new Error("Generation heartbeat must quiesce to preserve rotation headroom."); return true; } catch (error) { heartbeatFailure = error; throw error; } }; @@ -5002,11 +5035,7 @@ export async function withConsumerGenerationLock(root, authority, action, rawOpt if (!result.bytes.equals(metadataBytes(wanted))) throw new Error("Generation operation lost ownership before its terminal decision."); }; try { - await beat(); - await options.hooks?.afterClaim?.({ claim }); - const base = await generationRepairProjection(root, authority, statePath, options); - // Serialize preparation writes; only a live callback needs concurrent heartbeats. - await beat(); + // Only the fully prepared callback needs concurrent heartbeats. stopHeartbeat = (options.startHeartbeat ?? defaultHeartbeatScheduler)({ interval: options.update ?? PYLON_CONSUMER_LOCK_UPDATE_MS, beat }); const transaction = Object.freeze({ readStateBytes: () => base.tipBytes === null ? null : Buffer.from(base.tipBytes), diff --git a/scripts/pylon-generation-operations.test.mjs b/scripts/pylon-generation-operations.test.mjs index fc69f20d58..c48dbf1da9 100644 --- a/scripts/pylon-generation-operations.test.mjs +++ b/scripts/pylon-generation-operations.test.mjs @@ -1,11 +1,11 @@ import assert from "node:assert/strict"; import { fork } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; -import { chmod, cp, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, writeFile } from "node:fs/promises"; +import { chmod, cp, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { basename, join } from "node:path"; import { test } from "node:test"; -import { buildConsumerGeneration, discoverConsumerGenerations, prepareConsumerGeneration, publishConsumerGeneration, readConsumerGeneration, recoverConsumerGenerationBuilder, rotateConsumerGeneration, withConsumerGenerationLock } from "./lib/pylon-consumer-lock.mjs"; +import { buildConsumerGeneration, discoverConsumerGenerations, prepareConsumerGeneration, publishConsumerGeneration, readConsumerGeneration, recoverConsumerGenerationBuilder, rotateConsumerGeneration, withConsumerGenerationLock, withConsumerStateLock } from "./lib/pylon-consumer-lock.mjs"; async function fixture(t) { const directory = await mkdtemp(join(tmpdir(), "pylon-generation-operations-")); @@ -422,6 +422,48 @@ test("v3 operation every pinned canonical metadata read preserves hook errors ac } }); +test("v3 operation preparation heartbeat capacity rotates before callback admission", async (t) => { + const f = await fixture(t); const state = join(await realpath(f.directory), "state.json"); + const runtime = { ...options, stale: 100, now: () => 1000 }; + await withConsumerStateLock(state, async (_path, tx) => tx.commitState("base"), runtime); + let clock = 2000; + await withConsumerStateLock(state, async () => {}, { ...runtime, now: () => clock++ }); + const interrupted = new Error("Interrupted exact third claim index"); let cut = false; + await assert.rejects(withConsumerStateLock(state, async () => assert.fail("Interrupted claim must not enter its callback"), { + ...runtime, now: () => 100000, hooks: { generationBoundary: ({ phase, operation, path }) => { + if (phase === "after" && operation === "link" && path.endsWith("/claim-index-0000000000000003.json")) { cut = true; throw interrupted; } + } }, + }), (error) => error === interrupted); + assert.ok(cut); + let callbacks = 0; let schedules = 0; let stopped = 0; let rotated = false; + await withConsumerStateLock(state, async (_path, tx) => { + callbacks++; + assert.ok(rotated); assert.equal(schedules, 1); assert.equal(tx.readStateBytes().toString(), "base"); + }, { + ...runtime, now: () => 1000000, + hooks: { afterGenerationRename: () => { assert.equal(callbacks, 0); assert.equal(schedules, 0); rotated = true; } }, + startHeartbeat: () => { schedules++; return async () => { stopped++; }; }, + }); + assert.equal(callbacks, 1); assert.equal(schedules, 1); assert.equal(stopped, 1); + assert.equal((await readFile(state)).toString(), "base"); + const selected = JSON.parse(await readFile(`${state}.journal-v3/root.json`)); + const finals = await readdir(join(`${state}.journal-v3`, selected.goal)); + assert.equal(finals.length, 1); assert.match(finals[0], /^generation-0000000000000002-/); +}); + +for (const stage of ["afterClaim", "afterProjectionRename"]) test(`v3 operation preparation preserves ${stage} errors without callback or scheduler admission`, async (t) => { + const f = await fixture(t); f.authority.genesis.stateBytes = Buffer.from("base"); + const failure = Object.assign(new Error(`Original ${stage} error`), { code: "ENOENT" }); + let fired = false; let callbacks = 0; let schedules = 0; + await assert.rejects(withConsumerGenerationLock(f.root, f.authority, async () => { callbacks++; }, { + ...options, + hooks: { [stage]: () => { fired = true; throw failure; } }, + startHeartbeat: () => { schedules++; return async () => {}; }, + }), (error) => error === failure); + assert.ok(fired); assert.equal(callbacks, 0); assert.equal(schedules, 0); + await withConsumerGenerationLock(f.root, f.authority, async (_path, tx) => assert.equal(tx.readStateBytes().toString(), "base"), options); +}); + test("v3 operation schedules callback heartbeats only after projection preparation", async (t) => { const f = await fixture(t); f.authority.genesis.stateBytes = Buffer.from("base"); diff --git a/scripts/pylon-publication-stress.test.mjs b/scripts/pylon-publication-stress.test.mjs index 749c76b58c..db5497024c 100644 --- a/scripts/pylon-publication-stress.test.mjs +++ b/scripts/pylon-publication-stress.test.mjs @@ -1,11 +1,18 @@ import assert from "node:assert/strict"; -import { chmod, cp, readFile, readdir, rename, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { chmod, cp, lstat, readFile, readdir, rename, writeFile } from "node:fs/promises"; +import { basename, join, relative } from "node:path"; import { test } from "node:test"; import { child, cleanup, fixture, families, projection } from "./fixtures/publication-crash/support.mjs"; +import { generationDigest as digest } from "./lib/pylon-generation-format.mjs"; const rounds = 10; const refusals = /actively locked|changed|disappeared|ENOENT|EEXIST|inode|receipt|publication|authority|conflicting|writer|ownership|claim|checkpoint|namespace|unfinished|incomplete|unsafe type, owner or exact permissions/; +const migrationScenarios = new Set(families.map((family) => `migrate-${family}`)); +const staleMigrationTarget = "Migration temporary has an unknown immutable target."; +function assertRefusal(scenario, message) { + if (migrationScenarios.has(scenario) && message === staleMigrationTarget) return; + assert.match(message, refusals); +} async function competition(t, f, scenarios, round, historyProof) { const controlled = !scenarios[0].startsWith("migrate-") && round % 2 === 0; const workers = scenarios.map((scenario, index) => child(f, scenario, { pause: controlled && index === 0 ? "callback:after:stage:state.json" : undefined, append: true, stateMaxBytes: 8192, ready: true, marker: `round-${round}-${index}`, value: `value-${round}-${index}` })); @@ -17,20 +24,22 @@ async function competition(t, f, scenarios, round, historyProof) { await Promise.all(workers.slice(1).map((worker) => worker.exit)); workers[0].process.send({ type: "release" }); } else for (const worker of workers) worker.process.send({ type: "release" }); - const results = await Promise.all(workers.map(async (worker) => { + const results = await Promise.all(workers.map(async (worker, index) => { const exit = await worker.exit; assert.equal(exit.signal, null); const result = worker.messages.find((message) => message.type === "done" || message.type === "error"); assert.ok(result, `Missing terminal IPC: ${JSON.stringify(exit)} ${worker.output}`); - if (exit.code !== 0) { assert.equal(exit.code, 1); assert.equal(result.type, "error"); assert.match(result.message, refusals); } + if (exit.code !== 0) { assert.equal(exit.code, 1); assert.equal(result.type, "error"); assertRefusal(scenarios[index], result.message); } return result; })); if (controlled) assert.equal(results[0].type, "done", "The staged owner must complete exactly once"); for (let index = 0; index < workers.length; index++) { + if (migrationScenarios.has(scenarios[index])) await assert.rejects(lstat(join(f.directory, `round-${round}-${index}.callbacks`)), { code: "ENOENT" }); const markers = await readFile(join(f.directory, `round-${round}-${index}.callbacks`), "utf8").catch((error) => { if (error.code === "ENOENT") return ""; throw error; }); assert.ok(["", "entered\n", "entered\nreturned\n"].includes(markers), "No callback invocation is replayed"); } const recovery = child(f, scenarios[0].startsWith("migrate-") ? scenarios[0] : "commit", { stateMaxBytes: 8192, recover: true, marker: `recovery-${round}` }); let resumed; try { resumed = await recovery.finish(); } finally { await recovery.stop(); } + if (migrationScenarios.has(scenarios[0])) await assert.rejects(lstat(join(f.directory, `recovery-${round}.callbacks`)), { code: "ENOENT" }); assert.equal(resumed.finals.length, 1); if (!scenarios[0].startsWith("migrate-")) { const value = await projection(f); const history = value === "base" ? [] : JSON.parse(value); @@ -65,6 +74,92 @@ for (const family of families) test(`publication repeated four-process migration } }); +for (const family of families) test(`publication stale pre-intent migration refuses a peer blocker temporary ${family}`, { timeout: 60000 }, async (t) => { + const scenario = `migrate-${family}`; const f = await fixture(scenario); const workers = []; + try { + const legacy = family.includes("v1"); const v2 = family.endsWith("v2"); + const lock = `${f.state}${family.startsWith("prior-retired") ? ".lock.v1-retired" : ".lock"}`; + const sources = [ + ...(legacy ? [{ path: lock, frozen: true }, { path: `${f.state}.transactions`, frozen: true }] : []), + ...(v2 ? [{ path: `${f.state}.journal`, frozen: false }] : []), + ]; + const original = []; + async function record(path, source) { + const stat = await lstat(path); + original.push({ path, source, stat, bytes: stat.isFile() ? await readFile(path) : null }); + if (stat.isDirectory()) for (const name of await readdir(path)) await record(join(path, name), source); + } + for (const source of sources) await record(source.path, source); + const stale = child(f, scenario, { stateMaxBytes: 8192, pauseMigrationMetaRead: true, marker: "stale" }); workers.push(stale); + const staleCut = await stale.wait("cut"); assert.equal(staleCut.pid, stale.process.pid); + assert.equal(staleCut.phase, "pre-intent-meta-read"); assert.deepEqual(staleCut.names, ["receipts"]); + const epochName = v2 ? (await readdir(`${f.state}.journal`)).find((name) => name.startsWith("epoch-")) : null; + const blocker = join(legacy ? lock : join(`${f.state}.journal`, epochName), "claim-9999999999999999.json"); + const logical = legacy ? "blocker-v1" : `blocker-v2/${epochName}`; + const blockerHash = digest(Buffer.from(logical)); + const normalizedBlocker = relative(f.directory, blocker).replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g, "UUID"); + const publisher = child(f, scenario, { stateMaxBytes: 8192, marker: "publisher", cutMatch: { + hook: "migration", phase: "before", operation: "immutable-link", path: normalizedBlocker, + } }); workers.push(publisher); + const publisherCut = await publisher.wait("cut"); assert.equal(publisherCut.pid, publisher.process.pid); + const meta = `${f.state}.journal-v3`; const receipts = join(meta, "receipts"); + const temporaries = (await readdir(receipts)).filter((name) => name.endsWith(`-${blockerHash}.tmp`)); + assert.equal(temporaries.length, 1); assert.ok(temporaries[0].startsWith(`.writing-p${publisher.process.pid}-`)); + const temporary = join(receipts, temporaries[0]); const pending = await lstat(temporary); const pendingBytes = await readFile(temporary); + assert.equal(pending.nlink, 1); assert.equal(pending.mode & 0o7777, 0o600); + await assert.rejects(lstat(blocker), { code: "ENOENT" }); + const intentBytes = await readFile(join(meta, "intent.json")); const intent = JSON.parse(intentBytes); + assert.deepEqual(JSON.parse(pendingBytes), { + schemaVersion: 3, kind: "pylon-consumer-impossible-generation-blocker", generation: "9999999999999999", + statePathSha256: intent.statePathSha256, migrationIntentSha256: digest(intentBytes), source: intent.source, + }); + stale.process.send({ type: "release" }); + assert.deepEqual(await stale.exit, { code: 1, signal: null }); + const refused = stale.messages.find((message) => message.type === "error"); assert.equal(refused.pid, stale.process.pid); + assert.equal(refused.message, staleMigrationTarget); assertRefusal(scenario, refused.message); + for (const denied of ["commit", "rotate", "migrate-unknown"]) assert.throws(() => assertRefusal(denied, refused.message), assert.AssertionError); + await assert.rejects(lstat(join(meta, "complete.json")), { code: "ENOENT" }); + assert.equal(await projection(f), "base"); + publisher.process.send({ type: "release" }); const published = await publisher.finish(); + const recovery = child(f, scenario, { recover: true, stateMaxBytes: 8192, marker: "recovery" }); workers.push(recovery); + const recovered = await recovery.finish(); + assert.deepEqual(recovered.root, published.root); assert.equal(recovered.finals.length, 1); assert.equal(await projection(f), "base"); + assert.deepEqual(await readFile(join(meta, "intent.json")), intentBytes); + const finalBytes = await readFile(join(meta, "final.json")); const final = JSON.parse(finalBytes); + const selected = JSON.parse(await readFile(join(meta, "root.json"))); + const complete = JSON.parse(await readFile(join(meta, "complete.json"))); + const checkpointBytes = await readFile(join(meta, selected.goal, recovered.finals[0], "checkpoint.json")); + const checkpoint = JSON.parse(checkpointBytes); + assert.deepEqual(selected.identity, recovered.root); assert.deepEqual(complete.rootIdentity, recovered.root); + assert.equal(final.intentSha256, digest(intentBytes)); assert.equal(complete.intentSha256, digest(intentBytes)); + assert.equal(final.genesisSha256, digest(checkpointBytes)); assert.equal(complete.genesisSha256, digest(checkpointBytes)); + assert.equal(final.sourceAuthoritySha256, checkpoint.sourceAuthoritySha256); + assert.equal(checkpoint.sourceKind, v2 ? "v2" : "v1"); assert.equal(checkpoint.sourceTipDigest, digest(Buffer.from("base"))); + assert.equal(checkpoint.migrationKind, v2 && legacy ? "v1" : null); + const finalBlocker = legacy ? blocker : blocker.replace(`${f.state}.journal/`, `${f.state}.journal.v2-retired/`); + const committed = await lstat(finalBlocker); const receipt = await lstat(join(receipts, `receipt-${blockerHash}.json`)); + assert.deepEqual([committed.dev, committed.ino, receipt.dev, receipt.ino], [pending.dev, pending.ino, pending.dev, pending.ino]); + assert.deepEqual(await readFile(finalBlocker), pendingBytes); + for (const entry of original) { + const path = entry.source.frozen ? entry.path : entry.path.replace(`${f.state}.journal`, `${f.state}.journal.v2-retired`); + const current = await lstat(path); + assert.deepEqual([current.dev, current.ino], [entry.stat.dev, entry.stat.ino], path); + assert.equal(current.mode & 0o7777, entry.stat.isDirectory() && entry.source.frozen ? 0o500 : entry.stat.mode & 0o7777, path); + if (entry.bytes) assert.deepEqual(await readFile(path), entry.bytes, path); + } + if (legacy) { + const initialLock = original.find((entry) => entry.path === lock).stat; + assert.deepEqual(intent.source.legacyLockIdentity, { dev: initialLock.dev, ino: initialLock.ino }); + } + if (v2) { + const initialJournal = original.find((entry) => entry.path === `${f.state}.journal`).stat; + assert.deepEqual(intent.source.sourceIdentity, { dev: initialJournal.dev, ino: initialJournal.ino }); + } + for (const marker of ["stale", "publisher", "recovery"]) await assert.rejects(lstat(join(f.directory, `${marker}.callbacks`)), { code: "ENOENT" }); + t.diagnostic(JSON.stringify({ family, pids: workers.map((worker) => worker.process.pid), refusal: refused.message, blocker: basename(finalBlocker), root: recovered.root, sourceAuthoritySha256: checkpoint.sourceAuthoritySha256 })); + } finally { await Promise.all(workers.map((worker) => worker.stop())); await cleanup(f); } +}); + for (const kind of ["checkpoint.json", "claim-index-", "heartbeat-", "terminal-", "transition-", "applied-", "receipt-"]) test(`publication pinned ${kind} reader versus process rotation`, { timeout: 60000 }, async () => { const f = await fixture("commit"); const reader = child(f, "commit", { recover: true, pauseRead: kind, marker: "reader", stateMaxBytes: 8192 }); let rotator; let recovery; try {