From 7ab6876c26a6e6ce599f0576d1d4ecd39793767a Mon Sep 17 00:00:00 2001 From: hhh2210 Date: Wed, 22 Jul 2026 23:09:26 +0800 Subject: [PATCH 1/4] Fix compact Codex fork attribution Separate raw cumulative fork-boundary proof from owned usage subtraction and parent invalidation. Selectively migrate legacy parent-dependent cache rows, including known-model artifacts and later last-only snapshots. --- .../Vendored/CostUsage/CostUsageCache.swift | 13 +- .../CostUsageScanner+CacheHelpers.swift | 31 ++- .../Vendored/CostUsage/CostUsageScanner.swift | 105 ++++++++-- .../CodexForkAttributionMigrationTests.swift | 191 ++++++++++++++++++ .../CostUsageScannerBreakdownTests.swift | 141 +++++++++++++ 5 files changed, 462 insertions(+), 19 deletions(-) create mode 100644 Tests/CodexBarTests/CodexForkAttributionMigrationTests.swift diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift index e13e0bcd3b..edd84212d6 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift @@ -1,10 +1,11 @@ import Foundation enum CostUsageCacheIO { - /// Producer keys from older parser hashes whose caches are still valid under the current - /// delta semantics. Cleared for #2037: interleave containment changed how cumulative - /// totals are counted, so every earlier cache must be rebuilt. - private static let compatibleCodexProducerKeys: Set = [] + /// Producer keys from older parser hashes whose caches remain structurally compatible. + /// #2037 invalidated earlier producers because interleave containment changed cumulative + /// accounting. This workspace-era predecessor is safe to admit because the scanner selectively + /// reparses its parent-dependent forked files via `codexForkAttributionVersion`. + private static let compatibleCodexProducerKeys: Set = ["codex:cu:pa15a1040092b4a62"] /// Parsing and attribution changes rotate the Codex parser producer key. /// Increment this artifact version only when the stored schema or cache layout becomes incompatible. @@ -121,6 +122,8 @@ struct CostUsageCache: Codable { var codexPricingKey: String? var codexPriorityMetadataKey: String? var codexProjectMetadataVersion: Int? + /// Optional migration marker; absent caches must inspect parent-dependent fork candidates. + var codexForkAttributionVersion: Int? var codexPriorityTurnKeys: [String: String]? var codexPriorityTurnIDsByDay: [String: [String]]? @@ -151,6 +154,8 @@ struct CostUsageFileUsage: Codable { var sessionId: String? var forkedFromId: String? var forkBaselineDependencyKey: String? + /// Set after this file has passed the fork-attribution parser; nil requires the dependency-key check. + var codexForkAttributionVersion: Int? var projectPath: String? var canonicalProjectPath: String? var codexCostCacheComplete: Bool? diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift index d6c5882db9..9dab0da5bc 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift @@ -10,6 +10,14 @@ import Darwin #endif extension CostUsageScanner { + /// #2285 persisted this key for every compact parent candidate, including known-model rows. + /// Only the sentinel proves the file never depended on parent context. + static func isLegacyForkAttributionCandidate(_ usage: CostUsageFileUsage) -> Bool { + usage.forkedFromId != nil + && usage.codexForkAttributionVersion != codexForkAttributionVersion + && usage.forkBaselineDependencyKey != codexForkDependencyNotRequiredKey + } + private final class CodexModelsDevCatalogResolver { private var catalog: ModelsDevCatalog? private let cacheRoot: URL? @@ -966,6 +974,11 @@ extension CostUsageScanner { else { return false } guard !Self.cachedCodexFileNeedsPriorityRescan(cached, context: context) else { return false } + if context.needsForkAttributionMigration, + Self.isLegacyForkAttributionCandidate(cached) + { + return false + } let sessionAlreadyContributed = cached.sessionId.map { state.contributingSessionIds.contains($0) } ?? false let cachedRows = cached.codexRows ?? [] @@ -1241,7 +1254,13 @@ extension CostUsageScanner { if let cached = input.cached { self.applyFileDays(cache: &cache, fileDays: cached.days, sign: -1) } - let migratedCached = input.cached.map { Self.codexFileUsageWithCostCache($0, context: context) } + // A legacy parent-dependent fork file cannot carry unreparsed days across this migration: the + // current request may not cover its suspect day. Drop its retained projection and stamp + // the file current only from fresh source rows. + let hasLegacyForkCandidate = input.cached.map { Self.isLegacyForkAttributionCandidate($0) } ?? false + let migratedCached = hasLegacyForkCandidate + ? nil + : input.cached.map { Self.codexFileUsageWithCostCache($0, context: context) } var usageDays = context.dropDeferredCodexRows ? [:] : Self.fileDaysOutsideScanWindow(migratedCached?.days ?? [:], range: context.range) @@ -1251,6 +1270,7 @@ extension CostUsageScanner { range: context.range, maxBytesToRead: maxBytesToRead, inheritedTotalsResolver: context.resources.inheritedResolver.inheritedTotals(for:atOrBefore:), + inheritedRawTotalsResolver: context.resources.inheritedResolver.rawTotals(for:atOrBefore:), checkCancellation: context.checkCancellation) let forkBaselineDependencyKey = Self.codexForkBaselineDependencyKey( parentSessionId: parsed.forkedFromId, @@ -1368,6 +1388,7 @@ extension CostUsageScanner { codexJSONLResumeState: parsed.jsonlResumeState, codexBufferedSubagentLines: parsed.bufferedSubagentLines) .refreshingCodexWorkspaceUsageFingerprint() + cache.files[input.metadata.path]?.codexForkAttributionVersion = Self.codexForkAttributionVersion Self.applyFileDays(cache: &cache, fileDays: cache.files[input.metadata.path]?.days ?? [:], sign: 1) Self.rememberScannedCodexFile( input: input, @@ -1498,7 +1519,13 @@ extension CostUsageScanner { catalog: modelsDevCatalog, cacheRoot: modelsDevCacheRoot) var reportCache = cache - for (path, usage) in cache.files where self.needsCodexCostCache(usage, range: range) { + // A compatible predecessor cache may hydrate before migration completes. Do not present a + // parent-dependent candidate; current files and sentinel-owned forks remain visible. + for (path, usage) in cache.files where Self.isLegacyForkAttributionCandidate(usage) { + Self.applyFileDays(cache: &reportCache, fileDays: usage.days, sign: -1) + reportCache.files.removeValue(forKey: path) + } + for (path, usage) in reportCache.files where self.needsCodexCostCache(usage, range: range) { reportCache.files[path] = self.codexFileUsageWithCostCache( usage, range: range, diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift index 3b8a793d99..a683e9b954 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift @@ -9,6 +9,7 @@ import Foundation // swiftlint:disable type_body_length file_length enum CostUsageScanner { static let codexProjectMetadataVersion = 1 + static let codexForkAttributionVersion = 1 typealias CancellationCheck = () throws -> Void static let log = CodexBarLog.logger(LogCategories.tokenCost) @@ -574,6 +575,7 @@ enum CostUsageScanner { let dropDeferredCodexRows: Bool let requiresTurnIDCache: Bool let changedPriorityTurnIDs: Set + let needsForkAttributionMigration: Bool let resources: CodexScanResources let checkCancellation: CancellationCheck? let scanBudget: CodexScanBudget? @@ -686,6 +688,7 @@ enum CostUsageScanner { let priorityMetadataChanged: Bool let priorityTurnsChanged: Bool let needsTurnIDCacheMigration: Bool + let needsForkAttributionMigration: Bool let changedPriorityTurnIDs: Set let shouldRefresh: Bool } @@ -780,10 +783,12 @@ enum CostUsageScanner { } } + /// Reads each parent once and exposes every fork-boundary fact from the same stable snapshot. final class CodexInheritedTotalsResolver { private struct SnapshotResolution { let dependencyKey: String? - let snapshots: [CodexTimestampedTotals]? + let ownedSnapshots: [CodexTimestampedTotals]? + let rawSnapshots: [CodexTimestampedTotals]? } private let fileIndex: CodexSessionFileIndex @@ -814,7 +819,7 @@ enum CostUsageScanner { "Codex cost usage could not parse fork timestamp; falling back to lexical comparison", metadata: ["sessionId": sessionId, "timestamp": cutoffTimestamp]) } - guard let snapshots = try self.snapshotResolution(for: sessionId).snapshots else { return .unresolved } + guard let snapshots = try self.snapshotResolution(for: sessionId).ownedSnapshots else { return .unresolved } var inherited: CostUsageCodexTotals? for snapshot in snapshots { let isAtOrBefore: Bool = if let snapshotDate = snapshot.date, let cutoffDate { @@ -829,6 +834,18 @@ enum CostUsageScanner { return .resolved(inherited) } + /// Exact raw equality is the only proof that a compact child prefix is copied parent + /// history. Owned/deduplicated totals are intentionally not used for this decision. + func rawTotals(for sessionId: String, atOrBefore cutoffTimestamp: String) throws -> CodexForkBaseline { + guard !cutoffTimestamp.isEmpty, + let snapshots = try self.snapshotResolution(for: sessionId).rawSnapshots + else { return .unresolved } + guard let totals = Self.lastTotals(in: snapshots, atOrBefore: cutoffTimestamp) else { + return .unresolved + } + return .resolved(totals) + } + func currentDependencyKey(for sessionId: String) throws -> String { guard let fileURL = try self.fileIndex.fileURL(for: sessionId) else { return "missing:\(sessionId)" @@ -863,7 +880,8 @@ enum CostUsageScanner { metadata: ["sessionId": sessionId]) let resolution = SnapshotResolution( dependencyKey: "missing:\(sessionId)", - snapshots: nil) + ownedSnapshots: nil, + rawSnapshots: nil) self.snapshotResolutions[sessionId] = resolution return resolution } @@ -919,7 +937,8 @@ enum CostUsageScanner { metadata: ["sessionId": sessionId, "path": fileURL.path]) let resolution = SnapshotResolution( dependencyKey: dependencyKeyAfterParse, - snapshots: nil) + ownedSnapshots: nil, + rawSnapshots: nil) self.snapshotResolutions[sessionId] = resolution self.scanBudget?.consume(workBytes: parentMetadata.size) return resolution @@ -934,14 +953,16 @@ enum CostUsageScanner { ]) let resolution = SnapshotResolution( dependencyKey: dependencyKeyAfterParse, - snapshots: nil) + ownedSnapshots: nil, + rawSnapshots: nil) self.snapshotResolutions[sessionId] = resolution self.scanBudget?.consume(workBytes: parentMetadata.size) return resolution } let resolution = SnapshotResolution( dependencyKey: dependencyKeyAfterParse, - snapshots: parsed.snapshots) + ownedSnapshots: parsed.ownedSnapshots, + rawSnapshots: parsed.rawSnapshots) self.snapshotResolutions[sessionId] = resolution self.scanBudget?.consume(workBytes: parentMetadata.size) return resolution @@ -950,10 +971,32 @@ enum CostUsageScanner { CostUsageScanner.log.warning( "Codex cost usage parent session changed while reading; deferring inherited baseline", metadata: ["sessionId": sessionId, "path": fileURL.path]) - let resolution = SnapshotResolution(dependencyKey: nil, snapshots: nil) + let resolution = SnapshotResolution( + dependencyKey: nil, + ownedSnapshots: nil, + rawSnapshots: nil) self.snapshotResolutions[sessionId] = resolution return resolution } + + private static func lastTotals( + in snapshots: [CodexTimestampedTotals], + atOrBefore cutoffTimestamp: String) -> CostUsageCodexTotals? + { + let cutoffDate = CostUsageScanner.dateFromTimestamp(cutoffTimestamp) + var result: CostUsageCodexTotals? + for snapshot in snapshots { + let isAtOrBefore: Bool = if let date = snapshot.date, let cutoffDate { + date <= cutoffDate + } else { + snapshot.timestamp <= cutoffTimestamp + } + if isAtOrBefore { + result = snapshot.totals + } + } + return result + } } struct ClaudeParseResult { @@ -2184,11 +2227,13 @@ enum CostUsageScanner { fileURL: URL, checkCancellation: CancellationCheck? = nil) throws -> ( sessionId: String?, - snapshots: [CodexTimestampedTotals]) + ownedSnapshots: [CodexTimestampedTotals], + rawSnapshots: [CodexTimestampedTotals]) { var sessionId: String? var accumulator = CodexSnapshotAccumulator() - var snapshots: [CodexTimestampedTotals] = [] + var ownedSnapshots: [CodexTimestampedTotals] = [] + var rawSnapshots: [CodexTimestampedTotals] = [] var warnedAboutUnparsedTimestamp = false func parsedSnapshotDate(timestamp: String) -> Date? { @@ -2206,10 +2251,16 @@ enum CostUsageScanner { func appendSnapshot(timestamp: String, last: CostUsageCodexTotals?, total: CostUsageCodexTotals?) { guard last != nil || total != nil else { return } let counted = accumulator.apply(last: last, total: total) - snapshots.append(CodexTimestampedTotals( + ownedSnapshots.append(CodexTimestampedTotals( timestamp: timestamp, date: parsedSnapshotDate(timestamp: timestamp), totals: counted)) + if let raw = total { + rawSnapshots.append(CodexTimestampedTotals( + timestamp: timestamp, + date: parsedSnapshotDate(timestamp: timestamp), + totals: raw)) + } } do { @@ -2293,7 +2344,7 @@ enum CostUsageScanner { metadata: ["path": fileURL.path, "error": error.localizedDescription]) } - return (sessionId, snapshots) + return (sessionId, ownedSnapshots, rawSnapshots) } static func parseCodexFile( @@ -2370,6 +2421,7 @@ enum CostUsageScanner { initialJSONLResumeState: CostUsageJsonl.ResumeState? = nil, maxBytesToRead: Int64? = nil, inheritedTotalsResolver: ((String, String) throws -> CodexForkBaseline)? = nil, + inheritedRawTotalsResolver: ((String, String) throws -> CodexForkBaseline)? = nil, checkCancellation: CancellationCheck? = nil) throws -> CodexParseResult { var currentModel = initialModel @@ -3070,9 +3122,11 @@ enum CostUsageScanner { let parentSessionID = forkedFromId { candidateBoundaryDependsOnParentTotals = true - if let inheritedTotalsResolver { - switch try inheritedTotalsResolver(parentSessionID, forkTimestamp ?? "") { + if let inheritedRawTotalsResolver { + switch try inheritedRawTotalsResolver(parentSessionID, forkTimestamp ?? "") { case let .resolved(parentTotals): + // Candidate baseline and parent totals are both raw cumulative values. + // Do not accept a deduplicated/owned equality as copied-prefix proof. if Self.codexTotalsEqual(parentTotals, candidate.parentTotalsAtBoundary) { subagentCounterSemantics = .copiedPrefix ownedSuffix = candidate.ownedSuffix @@ -3289,6 +3343,7 @@ enum CostUsageScanner { let windowExpanded = Self.requestedWindowExpandsCache(range: range, cache: cache) let needsCostCacheMigration = cache.files.values.contains { Self.needsCodexCostCache($0, range: range) } let needsProjectMetadataMigration = cache.codexProjectMetadataVersion != Self.codexProjectMetadataVersion + let needsForkAttributionMigration = cache.files.values.contains { Self.isLegacyForkAttributionCandidate($0) } let modelsDevLoad = ModelsDevCache.load(now: now, cacheRoot: options.cacheRoot) let modelsDevCatalog = modelsDevLoad.artifact?.catalog let codexPricingKey = Self.codexPricingKey(modelsDevArtifact: modelsDevLoad.artifact) @@ -3308,6 +3363,7 @@ enum CostUsageScanner { || rootsChanged || needsCostCacheMigration || needsProjectMetadataMigration + || needsForkAttributionMigration || needsTurnIDCacheMigration || pricingChanged || priorityMetadataChanged @@ -3339,6 +3395,7 @@ enum CostUsageScanner { || rootsChanged || needsCostCacheMigration || needsProjectMetadataMigration + || needsForkAttributionMigration || needsTurnIDCacheMigration || pricingChanged || priorityMetadataChanged @@ -3366,6 +3423,7 @@ enum CostUsageScanner { priorityMetadataChanged: priorityMetadataChanged, priorityTurnsChanged: priorityTurnsChanged, needsTurnIDCacheMigration: needsTurnIDCacheMigration, + needsForkAttributionMigration: needsForkAttributionMigration, changedPriorityTurnIDs: changedPriorityTurnIDs, shouldRefresh: shouldRefresh) } @@ -3401,6 +3459,21 @@ enum CostUsageScanner { if options.forceRescan { cache = CostUsageCache() } + if plan.needsForkAttributionMigration { + // A retained legacy suspect outside this request cannot be certified without a + // full source read. Remove it now; a later wider request reparses it from disk. + let stalePaths = cache.files.compactMap { path, usage in + Self.isLegacyForkAttributionCandidate(usage) + && !usage.touchesCodexScanWindow(sinceKey: range.scanSinceKey, untilKey: range.scanUntilKey) + ? path + : nil + } + for path in stalePaths { + guard let usage = cache.files[path] else { continue } + Self.applyFileDays(cache: &cache, fileDays: usage.days, sign: -1) + cache.files.removeValue(forKey: path) + } + } let cachedSinceKey = cache.scanSinceKey let cachedUntilKey = cache.scanUntilKey @@ -3533,6 +3606,7 @@ enum CostUsageScanner { let shouldRetainWiderWindow = !options.forceRescan && !plan.pricingChanged && !plan .priorityMetadataChanged && !plan.needsTurnIDCacheMigration && !plan.needsProjectMetadataMigration + && !plan.needsForkAttributionMigration let retainedSinceKey = shouldRetainWiderWindow ? [cachedSinceKey, range.scanSinceKey].compactMap(\.self).min() ?? range.scanSinceKey : range.scanSinceKey @@ -3561,6 +3635,10 @@ enum CostUsageScanner { retainedUntilKey: retainedUntilKey) } cache.lastScanUnixMs = nowMs + cache.codexForkAttributionVersion = cache.files.values + .contains { Self.isLegacyForkAttributionCandidate($0) } + ? nil + : Self.codexForkAttributionVersion try checkCancellation?() Self.saveCodexCache(cache, options: options, range: range) } @@ -3589,6 +3667,7 @@ enum CostUsageScanner { || plan.needsTurnIDCacheMigration, requiresTurnIDCache: plan.needsTurnIDCacheMigration, changedPriorityTurnIDs: plan.changedPriorityTurnIDs, + needsForkAttributionMigration: plan.needsForkAttributionMigration, resources: resources, checkCancellation: checkCancellation, scanBudget: scanBudget) diff --git a/Tests/CodexBarTests/CodexForkAttributionMigrationTests.swift b/Tests/CodexBarTests/CodexForkAttributionMigrationTests.swift new file mode 100644 index 0000000000..57ceca9ac2 --- /dev/null +++ b/Tests/CodexBarTests/CodexForkAttributionMigrationTests.swift @@ -0,0 +1,191 @@ +import Foundation +import Testing +@testable import CodexBarCore + +struct CodexForkAttributionMigrationTests { + private func options(_ env: CostUsageTestEnvironment) -> CostUsageScanner.Options { + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite")) + options.refreshMinIntervalSeconds = 0 + return options + } + + private func writeSession( + _ env: CostUsageTestEnvironment, + day: Date, + events: [(Date, Int)], + model: String? = nil) throws + { + let lines: [[String: Any]] = [[ + "type": "session_meta", + "timestamp": env.isoString(for: day), + "payload": ["id": "migration-session"], + ]] + events.map { timestamp, input in + var info: [String: Any] = [ + "last_token_usage": ["input_tokens": input, "cached_input_tokens": 0, "output_tokens": 0], + ] + if let model { + info["model"] = model + } + return [ + "type": "event_msg", + "timestamp": env.isoString(for: timestamp), + "payload": [ + "type": "token_count", + "info": info, + ], + ] + } + _ = try env.writeCodexSessionFile(day: day, filename: "migration.jsonl", contents: env.jsonl(lines)) + } + + private func markLegacyForkCandidate(_ env: CostUsageTestEnvironment) { + var cache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + for path in cache.files.keys { + cache.files[path]?.forkedFromId = "missing-parent" + cache.files[path]?.forkBaselineDependencyKey = "missing-parent|legacy-raw-boundary" + cache.files[path]?.codexForkAttributionVersion = nil + } + cache.codexForkAttributionVersion = nil + CostUsageCacheIO.save( + provider: .codex, + cache: cache, + cacheRoot: env.cacheRoot, + producerKey: "codex:cu:pa15a1040092b4a62") + } + + @Test + func `public cached snapshot quarantines known model legacy fork candidate`() async throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 18) + try self.writeSession(env, day: day, events: [(day, 408_650_005)], model: "gpt-5.6-sol") + let options = self.options(env) + _ = CostUsageScanner.loadDailyReport(provider: .codex, since: day, until: day, now: day, options: options) + self.markLegacyForkCandidate(env) + + let legacy = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + #expect(legacy.producerKey == "codex:cu:pa15a1040092b4a62") + #expect(legacy.files.values.contains { CostUsageScanner.isLegacyForkAttributionCandidate($0) }) + #expect(legacy.files.values + .contains { $0.forkBaselineDependencyKey != CostUsageScanner.codexForkDependencyNotRequiredKey }) + + let cached = await CostUsageFetcher.loadCachedCodexTokenSnapshot( + now: day, historyDays: 1, scannerOptions: options) + #expect(cached == nil) + + let migrated = CostUsageScanner.loadDailyReport( + provider: .codex, since: day, until: day, now: day.addingTimeInterval(1), options: options) + #expect(migrated.data.first?.modelBreakdowns?.contains { + $0.modelName == "gpt-5.6-sol" && $0.totalTokens == 408_650_005 + } == true) + let refreshed = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + #expect(refreshed.files.values.allSatisfy { + $0.codexForkAttributionVersion == CostUsageScanner.codexForkAttributionVersion + }) + } + + @Test + func `public cached snapshot preserves current legitimate unknown sentinel fork`() async throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 18) + try self.writeSession(env, day: day, events: [(day, 42)]) + let options = self.options(env) + let first = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + var cache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + for path in cache.files.keys { + cache.files[path]?.forkedFromId = "current-fork" + cache.files[path]?.forkBaselineDependencyKey = CostUsageScanner.codexForkDependencyNotRequiredKey + cache.files[path]?.codexForkAttributionVersion = CostUsageScanner.codexForkAttributionVersion + } + cache.codexForkAttributionVersion = CostUsageScanner.codexForkAttributionVersion + CostUsageCacheIO.save(provider: .codex, cache: cache, cacheRoot: env.cacheRoot) + let cached = await CostUsageFetcher.loadCachedCodexTokenSnapshot( + now: day, + historyDays: 1, + scannerOptions: options) + #expect(first.data.first?.totalTokens == 42) + #expect(cached?.last30DaysTokens == 42) + #expect(cached?.daily.first?.modelBreakdowns?.contains { + CostUsagePricing.isCodexUnattributedModel($0.modelName) && $0.totalTokens == 42 + } == true) + } + + @Test + func `legacy migration reparses touched file and drops unreparsed older day`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let older = try env.makeLocalNoon(year: 2026, month: 5, day: 16) + let current = try env.makeLocalNoon(year: 2026, month: 5, day: 18) + try self.writeSession(env, day: current, events: [(older, 10), (current, 20)]) + let options = self.options(env) + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: older, + until: current, + now: current, + options: options) + self.markLegacyForkCandidate(env) + let migrated = CostUsageScanner.loadDailyReport( + provider: .codex, + since: current, + until: current, + now: current.addingTimeInterval(1), + options: options) + let cache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let oldKey = CostUsageScanner.CostUsageDayRange.dayKey(from: older) + #expect(migrated.data.first?.totalTokens == 20) + #expect(cache.files.values.allSatisfy { $0.days[oldKey] == nil }) + #expect(cache.scanSinceKey == CostUsageScanner.CostUsageDayRange(since: current, until: current).scanSinceKey) + #expect(cache.codexForkAttributionVersion == CostUsageScanner.codexForkAttributionVersion) + #expect(cache.files.values.allSatisfy { + $0.codexForkAttributionVersion == CostUsageScanner.codexForkAttributionVersion + }) + } + + @Test + func `out of window legacy suspect is removed and later expansion reparses source`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let older = try env.makeLocalNoon(year: 2026, month: 5, day: 16) + let current = try env.makeLocalNoon(year: 2026, month: 5, day: 18) + try self.writeSession(env, day: older, events: [(older, 33)]) + let options = self.options(env) + _ = CostUsageScanner.loadDailyReport(provider: .codex, since: older, until: older, now: older, options: options) + self.markLegacyForkCandidate(env) + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: current, + until: current, + now: current, + options: options) + let narrowed = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let oldKey = CostUsageScanner.CostUsageDayRange.dayKey(from: older) + #expect(narrowed.files.values.allSatisfy { $0.days[oldKey] == nil }) + #expect(narrowed.scanSinceKey == CostUsageScanner.CostUsageDayRange(since: current, until: current) + .scanSinceKey) + #expect(narrowed.codexForkAttributionVersion == CostUsageScanner.codexForkAttributionVersion) + + let expanded = CostUsageScanner.loadDailyReport( + provider: .codex, + since: older, + until: current, + now: current.addingTimeInterval(1), + options: options) + #expect(expanded.data.first?.totalTokens == 33) + let expandedCache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + #expect(expandedCache.files.count == 1) + #expect(expandedCache.files.values.allSatisfy { + $0.codexForkAttributionVersion == CostUsageScanner.codexForkAttributionVersion + }) + } +} diff --git a/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift b/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift index 547cf7fc0e..27b5a62747 100644 --- a/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift +++ b/Tests/CodexBarTests/CostUsageScannerBreakdownTests.swift @@ -1434,6 +1434,147 @@ struct CostUsageScannerBreakdownTests { #expect(parsed.days[dayKey]?["gpt-5"] == nil) } + @Test + func `codex exact raw parent prefix with known model is suppressed instead of repriced`() async throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let parentDay = try env.makeLocalNoon(year: 2026, month: 5, day: 17) + let childDay = try env.makeLocalNoon(year: 2026, month: 5, day: 18) + let parentID = "parent-exact-raw-prefix" + let childID = "child-exact-raw-prefix" + let copied: Usage = (input: 407_555_823, cached: 399_890_176, output: 1_094_182) + let parentURL = try env.writeCodexSessionFile( + day: parentDay, + filename: "parent-exact-raw-prefix.jsonl", + contents: env.jsonl([ + ["type": "session_meta", "timestamp": env.isoString(for: parentDay), "payload": ["id": parentID]], + self.codexTurnContext(timestamp: env.isoString(for: parentDay), model: "gpt-5.6-sol"), + self.codexTokenCount( + timestamp: env.isoString(for: parentDay.addingTimeInterval(1)), + model: "gpt-5.6-sol", + total: copied, + last: (input: 127_520, cached: 125_696, output: 57)), + // A later last-only status event must not erase the preceding raw cumulative + // snapshot that the compact child copied at its fork boundary. + self.codexTokenCount( + timestamp: env.isoString(for: parentDay.addingTimeInterval(2)), + model: "gpt-5.6-sol", + last: (input: 127_520, cached: 125_696, output: 57)), + ])) + _ = try env.writeCodexSessionFile( + day: childDay, + filename: "child-exact-raw-prefix.jsonl", + contents: env.jsonl([ + [ + "type": "session_meta", + "timestamp": env.isoString(for: childDay), + "payload": ["id": childID, "forked_from_id": parentID, "source": "subagent"], + ], + self.codexTokenCount( + timestamp: env.isoString(for: childDay.addingTimeInterval(1)), + model: "gpt-5.6-sol", + total: copied), + self.codexTurnContext( + timestamp: env.isoString(for: childDay.addingTimeInterval(2)), + model: "gpt-5.6-sol"), + [ + "type": "inter_agent_communication_metadata", + "timestamp": env.isoString(for: childDay.addingTimeInterval(3)), + "payload": ["trigger_turn": true], + ], + self.codexTokenCount( + timestamp: env.isoString(for: childDay.addingTimeInterval(4)), + model: "gpt-5.6-sol", + total: copied), + ])) + + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot, + codexTraceDatabaseURL: env.root.appendingPathComponent("missing.sqlite")) + options.refreshMinIntervalSeconds = 0 + let resolver = CostUsageScanner.CodexInheritedTotalsResolver( + fileIndex: CostUsageScanner.CodexSessionFileIndex(files: [parentURL], roots: []), + checkCancellation: nil) + let cutoff = env.isoString(for: childDay) + let owned = try resolver.inheritedTotals(for: parentID, atOrBefore: cutoff) + let raw = try resolver.rawTotals(for: parentID, atOrBefore: cutoff) + if case let .resolved(ownedTotals) = owned, case let .resolved(rawTotals) = raw { + #expect(ownedTotals != rawTotals) + } else { + Issue.record("Expected parent owned and raw totals") + } + let report = CostUsageScanner.loadDailyReport( + provider: .codex, since: childDay, until: childDay, now: childDay, options: options) + + #expect(report.data.isEmpty) + + // Recreate the compatible predecessor artifact: the child recorded the copied prefix under a known model, + // even though its raw cumulative total exactly equals the parent's fork boundary. + var legacy = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let childPath = try #require(legacy.files.first { $0.value.sessionId == childID }?.key) + let childDayKey = CostUsageScanner.CostUsageDayRange.dayKey(from: childDay) + let copiedRow = [copied.input, copied.cached, copied.output] + legacy.files[childPath]?.days[childDayKey] = ["gpt-5.6-sol": copiedRow] + legacy.files[childPath]?.forkedFromId = parentID + legacy.files[childPath]?.forkBaselineDependencyKey = "file|parent-exact-raw-prefix" + legacy.files[childPath]?.codexForkAttributionVersion = nil + legacy.days[childDayKey] = ["gpt-5.6-sol": copiedRow] + legacy.codexForkAttributionVersion = nil + CostUsageCacheIO.save( + provider: .codex, + cache: legacy, + cacheRoot: env.cacheRoot, + producerKey: "codex:cu:pa15a1040092b4a62") + + let cached = await CostUsageFetcher.loadCachedCodexTokenSnapshot( + now: childDay, historyDays: 1, scannerOptions: options) + #expect(cached == nil) + + let migrated = CostUsageScanner.loadDailyReport( + provider: .codex, + since: childDay, + until: childDay, + now: childDay.addingTimeInterval(1), + options: options) + #expect(migrated.data.isEmpty) + let refreshed = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + #expect(refreshed.files.values.allSatisfy { + $0.codexForkAttributionVersion == CostUsageScanner.codexForkAttributionVersion + }) + } + + @Test + func `codex raw parent proof requires cumulative total usage`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 18) + let parentID = "last-only-parent" + let parentURL = try env.writeCodexSessionFile( + day: day, + filename: "last-only-parent.jsonl", + contents: env.jsonl([ + ["type": "session_meta", "timestamp": env.isoString(for: day), "payload": ["id": parentID]], + self.codexTokenCount( + timestamp: env.isoString(for: day.addingTimeInterval(1)), + model: "gpt-5.6-sol", + last: (input: 127_520, cached: 125_696, output: 57)), + ])) + let resolver = CostUsageScanner.CodexInheritedTotalsResolver( + fileIndex: CostUsageScanner.CodexSessionFileIndex(files: [parentURL], roots: []), + checkCancellation: nil) + + let raw = try resolver.rawTotals( + for: parentID, + atOrBefore: env.isoString(for: day.addingTimeInterval(2))) + if case .resolved = raw { + Issue.record("last_token_usage must not prove a raw cumulative fork boundary") + } + } + @Test func `codex turn context remains authoritative over conflicting token model`() throws { let env = try CostUsageTestEnvironment() From 9fc05af4def50986bc147f77cbeb6bbcfbc0f1c6 Mon Sep 17 00:00:00 2001 From: hhh2210 Date: Thu, 23 Jul 2026 11:22:41 +0800 Subject: [PATCH 2/4] Harden compact fork cache reuse Require complete file parses before fresh-cache reuse, centralize provenance-based presentation projection, and add growing-rollout plus mixed-breakdown regressions. --- .../CostUsageScanner+CacheHelpers.swift | 20 ++-- .../Vendored/CostUsage/CostUsageScanner.swift | 3 +- .../CodexCompactSubagentAccountingTests.swift | 96 +++++++++++++++++++ .../CodexCompactSubagentFixture.swift | 25 ++++- .../CodexForkAttributionMigrationTests.swift | 78 ++++++++++++++- 5 files changed, 206 insertions(+), 16 deletions(-) diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift index 9dab0da5bc..22b6b2291d 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift @@ -18,6 +18,17 @@ extension CostUsageScanner { && usage.forkBaselineDependencyKey != codexForkDependencyNotRequiredKey } + /// Keep every cache-backed presentation surface on the same migration boundary. This is + /// intentionally provenance-based: a stale copied prefix may already carry a known model. + static func codexCacheForPresentation(_ cache: CostUsageCache) -> CostUsageCache { + var projected = cache + for (path, usage) in cache.files where Self.isLegacyForkAttributionCandidate(usage) { + Self.applyFileDays(cache: &projected, fileDays: usage.days, sign: -1) + projected.files.removeValue(forKey: path) + } + return projected + } + private final class CodexModelsDevCatalogResolver { private var catalog: ModelsDevCatalog? private let cacheRoot: URL? @@ -968,7 +979,8 @@ extension CostUsageScanner { let needsSessionId = cached.sessionId == nil guard cached.mtimeUnixMs == input.metadata.mtimeUnixMs, cached.size == input.metadata.size, - cached.codexScanComplete != false, + cached.codexScanComplete == true + || (cached.codexScanComplete == nil && cached.parsedBytes == cached.size), !needsSessionId, !context.forceFullScan else { return false } @@ -1518,13 +1530,9 @@ extension CostUsageScanner { let catalogResolver = CodexModelsDevCatalogResolver( catalog: modelsDevCatalog, cacheRoot: modelsDevCacheRoot) - var reportCache = cache // A compatible predecessor cache may hydrate before migration completes. Do not present a // parent-dependent candidate; current files and sentinel-owned forks remain visible. - for (path, usage) in cache.files where Self.isLegacyForkAttributionCandidate(usage) { - Self.applyFileDays(cache: &reportCache, fileDays: usage.days, sign: -1) - reportCache.files.removeValue(forKey: path) - } + var reportCache = Self.codexCacheForPresentation(cache) for (path, usage) in reportCache.files where self.needsCodexCostCache(usage, range: range) { reportCache.files[path] = self.codexFileUsageWithCostCache( usage, diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift index a683e9b954..bf5cea9fcc 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift @@ -3343,7 +3343,8 @@ enum CostUsageScanner { let windowExpanded = Self.requestedWindowExpandsCache(range: range, cache: cache) let needsCostCacheMigration = cache.files.values.contains { Self.needsCodexCostCache($0, range: range) } let needsProjectMetadataMigration = cache.codexProjectMetadataVersion != Self.codexProjectMetadataVersion - let needsForkAttributionMigration = cache.files.values.contains { Self.isLegacyForkAttributionCandidate($0) } + let needsForkAttributionMigration = cache.codexForkAttributionVersion != Self.codexForkAttributionVersion + && cache.files.values.contains { Self.isLegacyForkAttributionCandidate($0) } let modelsDevLoad = ModelsDevCache.load(now: now, cacheRoot: options.cacheRoot) let modelsDevCatalog = modelsDevLoad.artifact?.catalog let codexPricingKey = Self.codexPricingKey(modelsDevArtifact: modelsDevLoad.artifact) diff --git a/Tests/CodexBarTests/CodexCompactSubagentAccountingTests.swift b/Tests/CodexBarTests/CodexCompactSubagentAccountingTests.swift index 3fa37a5bb9..2b2516d4ad 100644 --- a/Tests/CodexBarTests/CodexCompactSubagentAccountingTests.swift +++ b/Tests/CodexBarTests/CodexCompactSubagentAccountingTests.swift @@ -172,6 +172,102 @@ struct CodexCompactSubagentAccountingTests { #expect(afterChild.days.values.allSatisfy { $0[CostUsagePricing.codexUnattributedModel] == nil }) } + @Test + func `appended first turn marker reclassifies a cached compact child prefix`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = try env.makeLocalNoon(year: 2026, month: 7, day: 21) + let parentModel = "openai/gpt-5.4" + let leafModel = "openai/gpt-5.6-sol" + let prefix: Usage = (input: 407_555, cached: 399_890, output: 1094) + let parentOwned: Usage = (input: 127, cached: 125, output: 1) + let suffix: Usage = (input: 1725, cached: 1568, output: 6) + let fixture = Fixture.Child( + sessionID: "growing-compact-child", + parentID: "growing-compact-parent", + leafModel: leafModel, + prefix: prefix, + suffix: suffix, + preBoundaryLast: parentOwned) + + _ = try env.writeCodexSessionFile( + day: day, + filename: "rollout-0-growing-parent.jsonl", + contents: Fixture.parentContents( + env: env, + day: day, + sessionID: fixture.parentID, + model: parentModel, + totals: prefix, + lastTotals: parentOwned)) + let prefixContents = try Fixture.childPrefixContents(env: env, day: day, fixture: fixture) + let childURL = try env.writeCodexSessionFile( + day: day, + filename: "rollout-1-growing-child.jsonl", + contents: prefixContents) + + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + claudeProjectsRoots: nil, + cacheRoot: env.cacheRoot) + options.refreshMinIntervalSeconds = 0 + let provisional = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + #expect(provisional.data.first?.modelBreakdowns?.contains { + $0.modelName == CostUsagePricing.codexUnattributedModel + } == true) + + let completeContents = try prefixContents + + Fixture.childSuffixContents(env: env, day: day, fixture: fixture) + try completeContents.write(to: childURL, atomically: true, encoding: .utf8) + + // A failed full-file read can retain prefix rows while observing the complete file's + // metadata. `parsedBytes` must keep that cache from becoming permanently fresh. + var partialCache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let partialEntry = try #require(partialCache.files.first { $0.value.sessionId == fixture.sessionID }) + var partialChild = partialEntry.value + let completeMetadata = CostUsageScanner.codexFileMetadata(fileURL: childURL) + let parsedPrefixBytes = try #require(partialChild.parsedBytes) + #expect(parsedPrefixBytes < completeMetadata.size) + #expect(partialChild.forkBaselineDependencyKey == CostUsageScanner.codexForkDependencyNotRequiredKey) + partialChild.mtimeUnixMs = completeMetadata.mtimeUnixMs + partialChild.size = completeMetadata.size + partialChild.parsedBytes = 0 + partialCache.files[partialEntry.key] = partialChild + CostUsageCacheIO.save(provider: .codex, cache: partialCache, cacheRoot: env.cacheRoot) + + let completed = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(1), + options: options) + let daily = try #require(completed.data.first) + #expect(daily.totalTokens == 1859) + #expect(!(daily.modelBreakdowns ?? []).contains { + $0.modelName == CostUsagePricing.codexUnattributedModel + }) + #expect(daily.modelBreakdowns?.first { + $0.modelName == CostUsagePricing.normalizeCodexModel(parentModel) + }?.totalTokens == 128) + #expect(daily.modelBreakdowns?.first { + $0.modelName == CostUsagePricing.normalizeCodexModel(leafModel) + }?.totalTokens == 1731) + let refreshedCache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let refreshedChild = try #require(refreshedCache.files.values.first { + $0.sessionId == fixture.sessionID + }) + #expect(refreshedChild.codexForkAttributionVersion == CostUsageScanner.codexForkAttributionVersion) + #expect(refreshedChild.days.values.allSatisfy { + $0[CostUsagePricing.codexUnattributedModel] == nil + }) + } + @Test func `unconfirmed compact prefix stays independent and parent-dependent`() throws { let env = try CostUsageTestEnvironment() diff --git a/Tests/CodexBarTests/CodexCompactSubagentFixture.swift b/Tests/CodexBarTests/CodexCompactSubagentFixture.swift index 85855038c3..d76be9ac5c 100644 --- a/Tests/CodexBarTests/CodexCompactSubagentFixture.swift +++ b/Tests/CodexBarTests/CodexCompactSubagentFixture.swift @@ -17,7 +17,8 @@ enum CodexCompactSubagentFixture { day: Date, sessionID: String, model: String, - totals: Usage) throws -> String + totals: Usage, + lastTotals: Usage? = nil) throws -> String { try env.jsonl([ [ @@ -32,7 +33,7 @@ enum CodexCompactSubagentFixture { timestamp: env.isoString(for: day.addingTimeInterval(-1)), model: model, total: totals, - last: totals), + last: lastTotals ?? totals), ]) } @@ -40,6 +41,15 @@ enum CodexCompactSubagentFixture { env: CostUsageTestEnvironment, day: Date, fixture: Child) throws -> String + { + try self.childPrefixContents(env: env, day: day, fixture: fixture) + + self.childSuffixContents(env: env, day: day, fixture: fixture) + } + + static func childPrefixContents( + env: CostUsageTestEnvironment, + day: Date, + fixture: Child) throws -> String { let forkTimestamp = env.isoString(for: day) var lines: [[String: Any]] = [ @@ -67,7 +77,15 @@ enum CodexCompactSubagentFixture { timestamp: env.isoString(for: day.addingTimeInterval(0.2)), last: preBoundaryLast)) } - lines.append(contentsOf: [ + return try env.jsonl(lines) + } + + static func childSuffixContents( + env: CostUsageTestEnvironment, + day: Date, + fixture: Child) throws -> String + { + try env.jsonl([ self.turnContext( timestamp: env.isoString(for: day.addingTimeInterval(1)), model: fixture.leafModel), @@ -84,7 +102,6 @@ enum CodexCompactSubagentFixture { output: fixture.prefix.output + fixture.suffix.output), last: fixture.suffix), ]) - return try env.jsonl(lines) } static func tokenCount( diff --git a/Tests/CodexBarTests/CodexForkAttributionMigrationTests.swift b/Tests/CodexBarTests/CodexForkAttributionMigrationTests.swift index 57ceca9ac2..6b86cf7f89 100644 --- a/Tests/CodexBarTests/CodexForkAttributionMigrationTests.swift +++ b/Tests/CodexBarTests/CodexForkAttributionMigrationTests.swift @@ -17,12 +17,19 @@ struct CodexForkAttributionMigrationTests { _ env: CostUsageTestEnvironment, day: Date, events: [(Date, Int)], - model: String? = nil) throws + model: String? = nil, + sessionID: String = "migration-session", + filename: String = "migration.jsonl", + cwd: String? = nil) throws { + var metadata: [String: Any] = ["id": sessionID] + if let cwd { + metadata["cwd"] = cwd + } let lines: [[String: Any]] = [[ "type": "session_meta", "timestamp": env.isoString(for: day), - "payload": ["id": "migration-session"], + "payload": metadata, ]] + events.map { timestamp, input in var info: [String: Any] = [ "last_token_usage": ["input_tokens": input, "cached_input_tokens": 0, "output_tokens": 0], @@ -39,12 +46,18 @@ struct CodexForkAttributionMigrationTests { ], ] } - _ = try env.writeCodexSessionFile(day: day, filename: "migration.jsonl", contents: env.jsonl(lines)) + _ = try env.writeCodexSessionFile(day: day, filename: filename, contents: env.jsonl(lines)) } - private func markLegacyForkCandidate(_ env: CostUsageTestEnvironment) { + private func markLegacyForkCandidate( + _ env: CostUsageTestEnvironment, + sessionID: String? = nil) + { var cache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) - for path in cache.files.keys { + let candidatePaths = cache.files.compactMap { path, usage in + sessionID == nil || usage.sessionId == sessionID ? path : nil + } + for path in candidatePaths { cache.files[path]?.forkedFromId = "missing-parent" cache.files[path]?.forkBaselineDependencyKey = "missing-parent|legacy-raw-boundary" cache.files[path]?.codexForkAttributionVersion = nil @@ -88,6 +101,61 @@ struct CodexForkAttributionMigrationTests { }) } + @Test + func `cached snapshot quarantines legacy fork from daily projects and sessions`() async throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 18) + let currentProject = env.root.appendingPathComponent("current-project", isDirectory: true) + let legacyProject = env.root.appendingPathComponent("legacy-project", isDirectory: true) + try FileManager.default.createDirectory(at: currentProject, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: legacyProject, withIntermediateDirectories: true) + try self.writeSession( + env, + day: day, + events: [(day, 42)], + model: "gpt-5.4", + sessionID: "current-session", + filename: "current.jsonl", + cwd: currentProject.path) + try self.writeSession( + env, + day: day, + events: [(day, 900)], + model: "gpt-5.6-sol", + sessionID: "legacy-session", + filename: "legacy.jsonl", + cwd: legacyProject.path) + let options = self.options(env) + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + self.markLegacyForkCandidate(env, sessionID: "legacy-session") + + let legacy = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let legacyUsage = try #require(legacy.files.values.first { $0.sessionId == "legacy-session" }) + let currentUsage = try #require(legacy.files.values.first { $0.sessionId == "current-session" }) + #expect(CostUsageScanner.isLegacyForkAttributionCandidate(legacyUsage)) + #expect(currentUsage.codexForkAttributionVersion == CostUsageScanner.codexForkAttributionVersion) + + let cached = try #require(await CostUsageFetcher.loadCachedCodexTokenSnapshot( + now: day, + historyDays: 1, + scannerOptions: options)) + #expect(cached.sessionTokens == 42) + #expect(cached.last30DaysTokens == 42) + #expect(cached.daily.count == 1) + #expect(cached.daily.first?.totalTokens == 42) + #expect(cached.sessions.map(\.sessionID) == ["current-session"]) + #expect(cached.sessions.first?.totalTokens == 42) + #expect(cached.projects.map(\.path) == [currentProject.path]) + #expect(cached.projects.first?.totalTokens == 42) + #expect(cached.projects.flatMap(\.sources).allSatisfy { $0.path != legacyProject.path }) + } + @Test func `public cached snapshot preserves current legitimate unknown sentinel fork`() async throws { let env = try CostUsageTestEnvironment() From 9b745d2a91ddf7fc04ddcdc246bf3e3648318d99 Mon Sep 17 00:00:00 2001 From: hhh2210 Date: Wed, 29 Jul 2026 13:59:33 +0800 Subject: [PATCH 3/4] Separate Codex cache compatibility - Persist the accounting contract independently of the raw parser fingerprint and bootstrap only known-safe predecessors. - Quarantine legacy or rejected caches across raw projections and workspace sidecars while preserving pricing-only last-good snapshots. - Cover future markers, roots-scoped candidates, and sidecar migration transitions with regressions. --- .../CodexLocalProjectUsageIndexer.swift | 17 +++ .../CodexWorkspaceUsageSidecar.swift | 4 + .../Vendored/CostUsage/CostUsageCache.swift | 76 +++++++++-- .../CodexForkAttributionMigrationTests.swift | 9 +- .../CodexLocalProjectUsageTests.swift | 118 ++++++++++++++++++ Tests/CodexBarTests/CostUsageCacheTests.swift | 80 ++++++++++++ 6 files changed, 287 insertions(+), 17 deletions(-) diff --git a/Sources/CodexBarCore/CodexLocalProjectUsageIndexer.swift b/Sources/CodexBarCore/CodexLocalProjectUsageIndexer.swift index 3e25b2b47c..54aa7707f2 100644 --- a/Sources/CodexBarCore/CodexLocalProjectUsageIndexer.swift +++ b/Sources/CodexBarCore/CodexLocalProjectUsageIndexer.swift @@ -21,12 +21,29 @@ enum CodexLocalProjectUsageIndexer { _ = now let clampedHistoryDays = max(1, min(365, historyDays)) let stableScopeSignature = self.stableScopeSignature(options: options.scannerOptions) + let cacheProducerKey: String? + switch CostUsageCacheIO.codexCacheAdmission(cacheRoot: options.scannerOptions.cacheRoot) { + case .missing: + cacheProducerKey = nil + case .rejected: + return nil + case let .accepted(cache): + let roots = CostUsageScanner.codexSessionsRoots(options: options.scannerOptions) + let scopedCache = CostUsageScanner.codexCache(cache, scopedTo: roots) + guard !scopedCache.files.values.contains(where: CostUsageScanner.isLegacyForkAttributionCandidate) else { + // The sidecar snapshot is aggregate-only and cannot quarantine individual legacy rows. + // Withhold it until the normal scanner selectively migrates parent-dependent sources. + return nil + } + cacheProducerKey = cache.producerKey + } let sidecar = CodexWorkspaceUsageSidecar(cacheRoot: options.scannerOptions.cacheRoot) let catalogResult = CodexThreadCatalogReader.loadResult(options: options.scannerOptions) let sourceStatus = CodexLocalProjectUsageSourceStatus(catalog: catalogResult.completeness) if let snapshot = sidecar.loadLatestSnapshot( scopeSignature: stableScopeSignature, historyDays: clampedHistoryDays, + cacheProducerKey: cacheProducerKey, catalog: catalogResult.isComplete ? catalogResult.catalog : nil) { return self.projecting(snapshot, sourceStatus: sourceStatus) diff --git a/Sources/CodexBarCore/CodexWorkspaceUsageSidecar.swift b/Sources/CodexBarCore/CodexWorkspaceUsageSidecar.swift index 28f58a518c..5f3cfdfd13 100644 --- a/Sources/CodexBarCore/CodexWorkspaceUsageSidecar.swift +++ b/Sources/CodexBarCore/CodexWorkspaceUsageSidecar.swift @@ -74,6 +74,7 @@ struct CodexWorkspaceUsageSidecar: Sendable { scopeSignature: String, historyDays: Int, rootsFingerprint: [String: Int64]? = nil, + cacheProducerKey: String? = nil, cache: CostUsageCache? = nil, catalog: CodexThreadCatalog? = nil) -> CodexLocalProjectUsageSnapshot? { @@ -125,6 +126,9 @@ struct CodexWorkspaceUsageSidecar: Sendable { persistedRoots == rootsFingerprint else { return nil } } + if let cacheProducerKey, Self.columnString(statement, at: 3) != cacheProducerKey { + return nil + } if let cache { guard Self.columnString(statement, at: 3) == cache.producerKey, Self.columnString(statement, at: 4) == cache.codexPricingKey, diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift index edd84212d6..6efa143469 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift @@ -1,11 +1,25 @@ import Foundation enum CostUsageCacheIO { - /// Producer keys from older parser hashes whose caches remain structurally compatible. + enum CodexCacheAdmission { + case missing + case rejected + case accepted(CostUsageCache) + } + + /// The persisted Codex accounting contract, independent of the raw source fingerprint. + /// Bump when cached fields, cumulative-delta semantics, or ownership semantics become incompatible. + /// Do not bump for provider additions, logging, UI, or other changes that leave cached accounting intact. + static let codexCacheCompatibilityVersion = 1 + + /// Pre-marker producer keys whose persisted accounting contract matches version 1. /// #2037 invalidated earlier producers because interleave containment changed cumulative - /// accounting. This workspace-era predecessor is safe to admit because the scanner selectively - /// reparses its parent-dependent forked files via `codexForkAttributionVersion`. - private static let compatibleCodexProducerKeys: Set = ["codex:cu:pa15a1040092b4a62"] + /// accounting. These two workspace-era predecessors differ only in non-Codex provider dispatch; + /// parent-dependent rows are still selectively reparsed via `codexForkAttributionVersion`. + private static let bootstrapCodexProducerKeys: Set = [ + "codex:cu:pa15a1040092b4a62", + "codex:cu:p7378e1f7e954ea1f", + ] /// Parsing and attribution changes rotate the Codex parser producer key. /// Increment this artifact version only when the stored schema or cache layout becomes incompatible. @@ -39,15 +53,23 @@ enum CostUsageCacheIO { producerKey: String? = nil, calendar: Calendar? = nil) -> CostUsageCache { + if provider == .codex, producerKey == nil { + if case let .accepted(cache) = self.codexCacheAdmission( + cacheRoot: cacheRoot, + calendar: calendar) + { + return cache + } + return CostUsageCache() + } + let url = self.cacheFileURL(provider: provider, cacheRoot: cacheRoot) let expectedProducerKey = producerKey ?? self.currentProducerKey(provider: provider) - let compatibleProducerKeys = producerKey == nil && provider == .codex - ? self.compatibleCodexProducerKeys - : [] if let decoded = self.loadCache( at: url, expectedProducerKey: expectedProducerKey, - compatibleProducerKeys: compatibleProducerKeys) + expectedCodexCompatibilityVersion: nil, + bootstrapCodexProducerKeys: []) { if let calendar, decoded.timeZoneIdentifier != calendar.timeZone.identifier { return CostUsageCache() @@ -57,19 +79,42 @@ enum CostUsageCacheIO { return CostUsageCache() } + static func codexCacheAdmission( + cacheRoot: URL? = nil, + calendar: Calendar? = nil) -> CodexCacheAdmission + { + let url = self.cacheFileURL(provider: .codex, cacheRoot: cacheRoot) + guard FileManager.default.fileExists(atPath: url.path) else { return .missing } + guard let cache = self.loadCache( + at: url, + expectedProducerKey: self.currentProducerKey(provider: .codex), + expectedCodexCompatibilityVersion: self.codexCacheCompatibilityVersion, + bootstrapCodexProducerKeys: self.bootstrapCodexProducerKeys) + else { return .rejected } + if let calendar, cache.timeZoneIdentifier != calendar.timeZone.identifier { + return .rejected + } + return .accepted(cache) + } + private static func loadCache( at url: URL, expectedProducerKey: String?, - compatibleProducerKeys: Set) -> CostUsageCache? + expectedCodexCompatibilityVersion: Int?, + bootstrapCodexProducerKeys: Set) -> CostUsageCache? { guard let data = try? Data(contentsOf: url) else { return nil } guard let decoded = try? JSONDecoder().decode(CostUsageCache.self, from: data) else { return nil } guard decoded.version == 1 else { return nil } - if let expectedProducerKey { - guard decoded.producerKey == expectedProducerKey - || decoded.producerKey.map(compatibleProducerKeys.contains) == true - else { return nil } + if let expectedCodexCompatibilityVersion { + let compatibleContract = decoded.producerKey != nil + && decoded.codexCacheCompatibilityVersion == expectedCodexCompatibilityVersion + let compatibleBootstrap = decoded.codexCacheCompatibilityVersion == nil + && decoded.producerKey.map(bootstrapCodexProducerKeys.contains) == true + guard compatibleContract || compatibleBootstrap else { return nil } + } else if let expectedProducerKey { + guard decoded.producerKey == expectedProducerKey else { return nil } } return decoded } @@ -88,6 +133,9 @@ enum CostUsageCacheIO { var cache = cache cache.producerKey = producerKey ?? self.currentProducerKey(provider: provider) cache.timeZoneIdentifier = calendar.timeZone.identifier + if provider == .codex, producerKey == nil { + cache.codexCacheCompatibilityVersion = self.codexCacheCompatibilityVersion + } let tmp = dir.appendingPathComponent(".tmp-\(UUID().uuidString).json", isDirectory: false) let data = (try? JSONEncoder().encode(cache)) ?? Data() @@ -115,6 +163,8 @@ enum CostUsageCacheIO { struct CostUsageCache: Codable { var version: Int = 1 var producerKey: String? + /// Persisted accounting contract; nil identifies a pre-marker cache eligible only by bootstrap key. + var codexCacheCompatibilityVersion: Int? var lastScanUnixMs: Int64 = 0 var scanSinceKey: String? var scanUntilKey: String? diff --git a/Tests/CodexBarTests/CodexForkAttributionMigrationTests.swift b/Tests/CodexBarTests/CodexForkAttributionMigrationTests.swift index 6b86cf7f89..8dde2dbfe8 100644 --- a/Tests/CodexBarTests/CodexForkAttributionMigrationTests.swift +++ b/Tests/CodexBarTests/CodexForkAttributionMigrationTests.swift @@ -51,7 +51,8 @@ struct CodexForkAttributionMigrationTests { private func markLegacyForkCandidate( _ env: CostUsageTestEnvironment, - sessionID: String? = nil) + sessionID: String? = nil, + producerKey: String = "codex:cu:pa15a1040092b4a62") { var cache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) let candidatePaths = cache.files.compactMap { path, usage in @@ -67,7 +68,7 @@ struct CodexForkAttributionMigrationTests { provider: .codex, cache: cache, cacheRoot: env.cacheRoot, - producerKey: "codex:cu:pa15a1040092b4a62") + producerKey: producerKey) } @Test @@ -78,10 +79,10 @@ struct CodexForkAttributionMigrationTests { try self.writeSession(env, day: day, events: [(day, 408_650_005)], model: "gpt-5.6-sol") let options = self.options(env) _ = CostUsageScanner.loadDailyReport(provider: .codex, since: day, until: day, now: day, options: options) - self.markLegacyForkCandidate(env) + self.markLegacyForkCandidate(env, producerKey: "codex:cu:p7378e1f7e954ea1f") let legacy = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) - #expect(legacy.producerKey == "codex:cu:pa15a1040092b4a62") + #expect(legacy.producerKey == "codex:cu:p7378e1f7e954ea1f") #expect(legacy.files.values.contains { CostUsageScanner.isLegacyForkAttributionCandidate($0) }) #expect(legacy.files.values .contains { $0.forkBaselineDependencyKey != CostUsageScanner.codexForkDependencyNotRequiredKey }) diff --git a/Tests/CodexBarTests/CodexLocalProjectUsageTests.swift b/Tests/CodexBarTests/CodexLocalProjectUsageTests.swift index 2127cbe690..d749ca6295 100644 --- a/Tests/CodexBarTests/CodexLocalProjectUsageTests.swift +++ b/Tests/CodexBarTests/CodexLocalProjectUsageTests.swift @@ -666,6 +666,7 @@ struct CodexLocalProjectUsageTests { fixture: fixture, costNanos: 1) CostUsageCacheIO.save(provider: .codex, cache: cache, cacheRoot: env.cacheRoot) + cache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) let snapshot = try CodexLocalProjectUsageIndexer.buildSnapshotFromCostCache( now: day, historyDays: 1, @@ -689,6 +690,123 @@ struct CodexLocalProjectUsageTests { scannerOptions: options))?.total.totalTokens == 130) } + @Test + func `cached project snapshot rejects unsafe raw cache and scopes legacy candidates`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + + let day = Date() + let project = env.root.appendingPathComponent("CodexBar", isDirectory: true) + try FileManager.default.createDirectory( + at: project.appendingPathComponent(".git", isDirectory: true), + withIntermediateDirectories: true) + let fixture = CodexUsageFixture( + filename: "legacy-sidecar.jsonl", + sessionID: "legacy-sidecar-session", + cwd: project.path, + input: 100, + cached: 20, + output: 30) + let options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + cacheRoot: env.cacheRoot) + let dayKey = CostUsageScanner.CostUsageDayRange.dayKey(from: day) + let safeFileURL = env.codexSessionsRoot.appendingPathComponent(fixture.filename) + var safeUsage = self.makeCachedFileUsage(dayKey: dayKey, fixture: fixture, costNanos: 1) + safeUsage.codexForkAttributionVersion = CostUsageScanner.codexForkAttributionVersion + safeUsage.forkBaselineDependencyKey = CostUsageScanner.codexForkDependencyNotRequiredKey + + var cache = CostUsageCache() + cache.producerKey = "codex:cu:p7378e1f7e954ea1f" + cache.scanSinceKey = dayKey + cache.scanUntilKey = dayKey + cache.codexPricingKey = "pricing-a" + cache.roots = CostUsageScanner.codexRootsFingerprint(options: options) + cache.files[safeFileURL.path] = safeUsage + let snapshot = try CodexLocalProjectUsageIndexer.buildSnapshotFromCostCache( + now: day, + historyDays: 1, + since: day, + until: day, + options: options, + cacheOverride: cache) + let catalog = CodexThreadCatalogReader.load(options: options) + let sidecar = CodexWorkspaceUsageSidecar(cacheRoot: env.cacheRoot) + try sidecar.synchronize( + snapshot: snapshot, + cache: cache, + catalog: catalog, + rootsFingerprint: CostUsageScanner.codexRootsFingerprint(options: options)) + #expect(sidecar.loadLatestSnapshot( + scopeSignature: snapshot.scopeSignature, + historyDays: 1) != nil) + + CostUsageCacheIO.save( + provider: .codex, + cache: cache, + cacheRoot: env.cacheRoot, + producerKey: "codex:cu:p7378e1f7e954ea1f") + #expect(CodexLocalProjectUsageIndexer.cachedSnapshot( + now: day, + historyDays: 1, + options: .init(scannerOptions: options))?.total.totalTokens == 130) + + var legacyUsage = safeUsage + legacyUsage.forkedFromId = "missing-parent" + legacyUsage.forkBaselineDependencyKey = "missing-parent|legacy-raw-boundary" + legacyUsage.codexForkAttributionVersion = nil + let insideLegacyURL = env.codexSessionsRoot.appendingPathComponent("legacy-inside.jsonl") + cache.files[insideLegacyURL.path] = legacyUsage + CostUsageCacheIO.save( + provider: .codex, + cache: cache, + cacheRoot: env.cacheRoot, + producerKey: "codex:cu:p7378e1f7e954ea1f") + #expect(CodexLocalProjectUsageIndexer.cachedSnapshot( + now: day, + historyDays: 1, + options: .init(scannerOptions: options)) == nil) + + cache.files.removeValue(forKey: insideLegacyURL.path) + let outsideLegacyURL = env.root + .appendingPathComponent("other-codex-home/sessions/legacy-outside.jsonl") + cache.files[outsideLegacyURL.path] = legacyUsage + CostUsageCacheIO.save( + provider: .codex, + cache: cache, + cacheRoot: env.cacheRoot, + producerKey: "codex:cu:p7378e1f7e954ea1f") + #expect(CodexLocalProjectUsageIndexer.cachedSnapshot( + now: day, + historyDays: 1, + options: .init(scannerOptions: options))?.total.totalTokens == 130) + + let currentProducer = try #require(CostUsageCacheIO.currentProducerKey(provider: .codex)) + cache.producerKey = currentProducer + cache.codexCacheCompatibilityVersion = CostUsageCacheIO.codexCacheCompatibilityVersion + 1 + CostUsageCacheIO.save( + provider: .codex, + cache: cache, + cacheRoot: env.cacheRoot, + producerKey: currentProducer) + #expect(CodexLocalProjectUsageIndexer.cachedSnapshot( + now: day, + historyDays: 1, + options: .init(scannerOptions: options)) == nil) + + cache.files.removeValue(forKey: outsideLegacyURL.path) + cache.codexCacheCompatibilityVersion = CostUsageCacheIO.codexCacheCompatibilityVersion + CostUsageCacheIO.save( + provider: .codex, + cache: cache, + cacheRoot: env.cacheRoot, + producerKey: currentProducer) + #expect(CodexLocalProjectUsageIndexer.cachedSnapshot( + now: day, + historyDays: 1, + options: .init(scannerOptions: options)) == nil) + } + @Test func `project usage severity separates high usage from unknown cost coverage`() throws { let env = try CostUsageTestEnvironment() diff --git a/Tests/CodexBarTests/CostUsageCacheTests.swift b/Tests/CodexBarTests/CostUsageCacheTests.swift index d06d66c09c..ff37f14b3a 100644 --- a/Tests/CodexBarTests/CostUsageCacheTests.swift +++ b/Tests/CodexBarTests/CostUsageCacheTests.swift @@ -115,6 +115,7 @@ struct CostUsageCacheTests { defer { try? FileManager.default.removeItem(at: root) } var cache = CostUsageCache() + cache.codexCacheCompatibilityVersion = CostUsageCacheIO.codexCacheCompatibilityVersion cache.lastScanUnixMs = 123 cache.days = ["2026-05-18": ["gpt-5.5": [1, 2, 3]]] @@ -141,6 +142,85 @@ struct CostUsageCacheTests { #expect(stale.days.isEmpty) } + @Test + func `runtime codex save stamps compatibility contract`() throws { + let root = try self.makeTemporaryCacheRoot() + defer { try? FileManager.default.removeItem(at: root) } + + var cache = CostUsageCache() + cache.lastScanUnixMs = 123 + CostUsageCacheIO.save(provider: .codex, cache: cache, cacheRoot: root) + + let loaded = CostUsageCacheIO.load(provider: .codex, cacheRoot: root) + #expect(loaded.lastScanUnixMs == 123) + #expect(loaded.codexCacheCompatibilityVersion == CostUsageCacheIO.codexCacheCompatibilityVersion) + } + + @Test + func `runtime accepts current contract across producer hash changes`() throws { + let root = try self.makeTemporaryCacheRoot() + defer { try? FileManager.default.removeItem(at: root) } + + var cache = CostUsageCache() + cache.lastScanUnixMs = 123 + cache.codexCacheCompatibilityVersion = CostUsageCacheIO.codexCacheCompatibilityVersion + CostUsageCacheIO.save( + provider: .codex, + cache: cache, + cacheRoot: root, + producerKey: "codex:cu:p1111111111111111") + + let loaded = CostUsageCacheIO.load(provider: .codex, cacheRoot: root) + #expect(loaded.lastScanUnixMs == 123) + #expect(loaded.producerKey == "codex:cu:p1111111111111111") + } + + @Test + func `runtime accepts only known pre marker bootstrap producers`() throws { + let root = try self.makeTemporaryCacheRoot() + defer { try? FileManager.default.removeItem(at: root) } + + for producerKey in ["codex:cu:pa15a1040092b4a62", "codex:cu:p7378e1f7e954ea1f"] { + var cache = CostUsageCache() + cache.lastScanUnixMs = 123 + CostUsageCacheIO.save( + provider: .codex, + cache: cache, + cacheRoot: root, + producerKey: producerKey) + + let loaded = CostUsageCacheIO.load(provider: .codex, cacheRoot: root) + #expect(loaded.lastScanUnixMs == 123) + #expect(loaded.producerKey == producerKey) + #expect(loaded.codexCacheCompatibilityVersion == nil) + } + + var unknown = CostUsageCache() + unknown.lastScanUnixMs = 321 + CostUsageCacheIO.save( + provider: .codex, + cache: unknown, + cacheRoot: root, + producerKey: "codex:cu:pdeadbeefdeadbeef") + let unknownRejected = CostUsageCacheIO.load(provider: .codex, cacheRoot: root) + #expect(unknownRejected.lastScanUnixMs == 0) + + var incompatible = CostUsageCache() + incompatible.lastScanUnixMs = 456 + incompatible.codexCacheCompatibilityVersion = CostUsageCacheIO.codexCacheCompatibilityVersion + 1 + let currentProducer = try #require(CostUsageCacheIO.currentProducerKey(provider: .codex)) + CostUsageCacheIO.save( + provider: .codex, + cache: incompatible, + cacheRoot: root, + producerKey: currentProducer) + + let rejected = CostUsageCacheIO.load(provider: .codex, cacheRoot: root) + #expect(rejected.lastScanUnixMs == 0) + #expect(rejected.files.isEmpty) + #expect(rejected.days.isEmpty) + } + @Test func `legacy cache without producer key is ignored`() throws { let root = try self.makeTemporaryCacheRoot() From 0ca86c9dc9d4e4069d762a07f0a99e656ff76d8b Mon Sep 17 00:00:00 2001 From: hhh2210 Date: Thu, 30 Jul 2026 00:46:27 +0800 Subject: [PATCH 4/4] Integrate bounded Codex cache migration - Preserve legacy fork provenance and cursors until bounded scans complete, then stamp the attribution contract. - Apply the same quarantine projection to workspace sidecars so deferred rows cannot leak into project usage. - Compose calendar isolation with the one-time predecessor bridge; future parser hashes rely on compatibility v1. --- .../CodexLocalProjectUsageIndexer.swift | 10 ++- .../Generated/CodexParserHash.generated.swift | 2 +- .../Vendored/CostUsage/CostUsageCache.swift | 12 +-- .../CostUsageScanner+CacheHelpers.swift | 52 +++++++++++-- .../Vendored/CostUsage/CostUsageScanner.swift | 19 ++--- .../CodexForkAttributionMigrationTests.swift | 72 ++++++++++++++++++ .../CodexLocalProjectUsageTests.swift | 74 +++++++++++++++++++ Tests/CodexBarTests/CostUsageCacheTests.swift | 57 +++++++++++++- 8 files changed, 269 insertions(+), 29 deletions(-) diff --git a/Sources/CodexBarCore/CodexLocalProjectUsageIndexer.swift b/Sources/CodexBarCore/CodexLocalProjectUsageIndexer.swift index 54aa7707f2..d995445097 100644 --- a/Sources/CodexBarCore/CodexLocalProjectUsageIndexer.swift +++ b/Sources/CodexBarCore/CodexLocalProjectUsageIndexer.swift @@ -22,7 +22,10 @@ enum CodexLocalProjectUsageIndexer { let clampedHistoryDays = max(1, min(365, historyDays)) let stableScopeSignature = self.stableScopeSignature(options: options.scannerOptions) let cacheProducerKey: String? - switch CostUsageCacheIO.codexCacheAdmission(cacheRoot: options.scannerOptions.cacheRoot) { + switch CostUsageCacheIO.codexCacheAdmission( + cacheRoot: options.scannerOptions.cacheRoot, + calendar: options.scannerOptions.calendar) + { case .missing: cacheProducerKey = nil case .rejected: @@ -86,10 +89,13 @@ enum CodexLocalProjectUsageIndexer { checkCancellation: checkCancellation) try checkCancellation?() - let cache = CostUsageCacheIO.load( + let rawCache = CostUsageCacheIO.load( provider: .codex, cacheRoot: scannerOptions.cacheRoot, calendar: scannerOptions.calendar) + // A budget-limited refresh can leave legacy fork candidates pending. Keep the workspace + // sidecar on the same provenance boundary as daily/project/session cache presentation. + let cache = CostUsageScanner.codexCacheForPresentation(rawCache) let catalogResult = CodexThreadCatalogReader.loadResult(options: scannerOptions) let catalog = catalogResult.catalog let sourceStatus = CodexLocalProjectUsageSourceStatus(catalog: catalogResult.completeness) diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index 9b8af37833..a1256a2239 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "21dae5bee0a0ece1" + static let value = "0a042d13ecd8201e" } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift index 6efa143469..d520ece49e 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift @@ -12,13 +12,14 @@ enum CostUsageCacheIO { /// Do not bump for provider additions, logging, UI, or other changes that leave cached accounting intact. static let codexCacheCompatibilityVersion = 1 - /// Pre-marker producer keys whose persisted accounting contract matches version 1. - /// #2037 invalidated earlier producers because interleave containment changed cumulative - /// accounting. These two workspace-era predecessors differ only in non-Codex provider dispatch; - /// parent-dependent rows are still selectively reparsed via `codexForkAttributionVersion`. + /// Audited pre-marker producers whose persisted accounting contract can migrate to version 1. + /// #2037 invalidated earlier producers. This one-time bridge covers released caches plus the + /// immediate upstream predecessor; once version 1 ships, future hashes use the marker instead. private static let bootstrapCodexProducerKeys: Set = [ "codex:cu:pa15a1040092b4a62", "codex:cu:p7378e1f7e954ea1f", + "codex:cu:p6f689d90f8eedcbd", + "codex:cu:p21dae5bee0a0ece1", ] /// Parsing and attribution changes rotate the Codex parser producer key. @@ -111,7 +112,8 @@ enum CostUsageCacheIO { let compatibleContract = decoded.producerKey != nil && decoded.codexCacheCompatibilityVersion == expectedCodexCompatibilityVersion let compatibleBootstrap = decoded.codexCacheCompatibilityVersion == nil - && decoded.producerKey.map(bootstrapCodexProducerKeys.contains) == true + && (decoded.producerKey == expectedProducerKey + || decoded.producerKey.map(bootstrapCodexProducerKeys.contains) == true) guard compatibleContract || compatibleBootstrap else { return nil } } else if let expectedProducerKey { guard decoded.producerKey == expectedProducerKey else { return nil } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift index 22b6b2291d..ae518fea82 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner+CacheHelpers.swift @@ -18,6 +18,29 @@ extension CostUsageScanner { && usage.forkBaselineDependencyKey != codexForkDependencyNotRequiredKey } + static func shouldDropLegacyForkCandidateOutsideWindow( + _ usage: CostUsageFileUsage, + range: CostUsageDayRange) -> Bool + { + self.isLegacyForkAttributionCandidate(usage) + && usage.codexScanComplete != false + && !usage.touchesCodexScanWindow(sinceKey: range.scanSinceKey, untilKey: range.scanUntilKey) + } + + static func dropLegacyForkCandidatesOutsideWindow( + cache: inout CostUsageCache, + range: CostUsageDayRange) + { + let stalePaths = cache.files.compactMap { path, usage in + self.shouldDropLegacyForkCandidateOutsideWindow(usage, range: range) ? path : nil + } + for path in stalePaths { + guard let usage = cache.files[path] else { continue } + self.applyFileDays(cache: &cache, fileDays: usage.days, sign: -1) + cache.files.removeValue(forKey: path) + } + } + /// Keep every cache-backed presentation surface on the same migration boundary. This is /// intentionally provenance-based: a stale copied prefix may already carry a known model. static func codexCacheForPresentation(_ cache: CostUsageCache) -> CostUsageCache { @@ -979,8 +1002,8 @@ extension CostUsageScanner { let needsSessionId = cached.sessionId == nil guard cached.mtimeUnixMs == input.metadata.mtimeUnixMs, cached.size == input.metadata.size, - cached.codexScanComplete == true - || (cached.codexScanComplete == nil && cached.parsedBytes == cached.size), + cached.parsedBytes == cached.size, + cached.codexScanComplete != false, !needsSessionId, !context.forceFullScan else { return false } @@ -1246,6 +1269,11 @@ extension CostUsageScanner { codexJSONLResumeState: delta.jsonlResumeState, codexBufferedSubagentLines: delta.bufferedSubagentLines) .refreshingCodexWorkspaceUsageFingerprint() + if cache.files[input.metadata.path]?.codexScanComplete == true { + cache.files[input.metadata.path]?.codexForkAttributionVersion = Self.codexForkAttributionVersion + } else { + cache.files[input.metadata.path]?.codexForkAttributionVersion = cached.codexForkAttributionVersion + } Self.rememberScannedCodexFile( input: input, session: CodexScannedSession(id: sessionId, days: mergedDays), @@ -1284,6 +1312,7 @@ extension CostUsageScanner { inheritedTotalsResolver: context.resources.inheritedResolver.inheritedTotals(for:atOrBefore:), inheritedRawTotalsResolver: context.resources.inheritedResolver.rawTotals(for:atOrBefore:), checkCancellation: context.checkCancellation) + let scanComplete = parsed.parsedBytes >= input.metadata.size && parsed.jsonlResumeState == nil let forkBaselineDependencyKey = Self.codexForkBaselineDependencyKey( parentSessionId: parsed.forkedFromId, dependsOnParentTotals: parsed.dependsOnParentTotals, @@ -1295,7 +1324,12 @@ extension CostUsageScanner { title: nil, startedAtUnixMs: nil, latestActivityUnixMs: nil) - let parsedCodexSession = cachedSessionMetadata.merging(parsed.codexSession) + var parsedCodexSession = cachedSessionMetadata.merging(parsed.codexSession) + if !scanComplete { + // A bounded prefix may stop before lineage metadata. Preserve the legacy provenance + // until a complete parse can safely reclassify and stamp this row. + parsedCodexSession.forkedFromId = parsedCodexSession.forkedFromId ?? input.cached?.forkedFromId + } let sessionId = parsedCodexSession.sessionId ?? parsed.sessionId ?? input.cached?.sessionId let projectPath = parsed.projectPath ?? input.cached?.projectPath let canonicalProjectPath = parsed.projectPath.map { @@ -1340,8 +1374,10 @@ extension CostUsageScanner { hasInterleavedTotals: parsed.hasInterleavedTotals, lastCodexTurnID: parsed.lastCodexTurnID, sessionId: sessionId, - forkedFromId: parsedCodexSession.forkedFromId ?? parsed.forkedFromId, - forkBaselineDependencyKey: forkBaselineDependencyKey, + forkedFromId: parsedCodexSession.forkedFromId ?? parsed.forkedFromId ?? input.cached?.forkedFromId, + forkBaselineDependencyKey: scanComplete + ? forkBaselineDependencyKey + : input.cached?.forkBaselineDependencyKey ?? forkBaselineDependencyKey, projectPath: projectPath, canonicalProjectPath: canonicalProjectPath, codexSession: parsedCodexSession.isEmpty ? nil : parsedCodexSession, @@ -1396,11 +1432,13 @@ extension CostUsageScanner { modelsDevCacheRoot: context.resources.modelsDevCacheRoot), codexScanFileId: input.metadata.fileId, codexScanTargetSize: input.metadata.size, - codexScanComplete: parsed.parsedBytes >= input.metadata.size && parsed.jsonlResumeState == nil, + codexScanComplete: scanComplete, codexJSONLResumeState: parsed.jsonlResumeState, codexBufferedSubagentLines: parsed.bufferedSubagentLines) .refreshingCodexWorkspaceUsageFingerprint() - cache.files[input.metadata.path]?.codexForkAttributionVersion = Self.codexForkAttributionVersion + if cache.files[input.metadata.path]?.codexScanComplete == true { + cache.files[input.metadata.path]?.codexForkAttributionVersion = Self.codexForkAttributionVersion + } Self.applyFileDays(cache: &cache, fileDays: cache.files[input.metadata.path]?.days ?? [:], sign: 1) Self.rememberScannedCodexFile( input: input, diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift index bf5cea9fcc..fa0c4f8898 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift @@ -902,7 +902,8 @@ enum CostUsageScanner { ]) let resolution = SnapshotResolution( dependencyKey: self.dependencyKey(for: sessionId, fileURL: fileURL), - snapshots: nil) + ownedSnapshots: nil, + rawSnapshots: nil) self.snapshotResolutions[sessionId] = resolution return resolution case .deferBudget: @@ -917,7 +918,8 @@ enum CostUsageScanner { ]) let resolution = SnapshotResolution( dependencyKey: self.dependencyKey(for: sessionId, fileURL: fileURL), - snapshots: nil) + ownedSnapshots: nil, + rawSnapshots: nil) self.snapshotResolutions[sessionId] = resolution return resolution } @@ -3463,17 +3465,8 @@ enum CostUsageScanner { if plan.needsForkAttributionMigration { // A retained legacy suspect outside this request cannot be certified without a // full source read. Remove it now; a later wider request reparses it from disk. - let stalePaths = cache.files.compactMap { path, usage in - Self.isLegacyForkAttributionCandidate(usage) - && !usage.touchesCodexScanWindow(sinceKey: range.scanSinceKey, untilKey: range.scanUntilKey) - ? path - : nil - } - for path in stalePaths { - guard let usage = cache.files[path] else { continue } - Self.applyFileDays(cache: &cache, fileDays: usage.days, sign: -1) - cache.files.removeValue(forKey: path) - } + // Preserve an in-progress bounded scan even before it has emitted an in-window day. + Self.dropLegacyForkCandidatesOutsideWindow(cache: &cache, range: range) } let cachedSinceKey = cache.scanSinceKey diff --git a/Tests/CodexBarTests/CodexForkAttributionMigrationTests.swift b/Tests/CodexBarTests/CodexForkAttributionMigrationTests.swift index 8dde2dbfe8..65917880a0 100644 --- a/Tests/CodexBarTests/CodexForkAttributionMigrationTests.swift +++ b/Tests/CodexBarTests/CodexForkAttributionMigrationTests.swift @@ -19,10 +19,15 @@ struct CodexForkAttributionMigrationTests { events: [(Date, Int)], model: String? = nil, sessionID: String = "migration-session", + forkedFromID: String? = nil, filename: String = "migration.jsonl", cwd: String? = nil) throws { var metadata: [String: Any] = ["id": sessionID] + if let forkedFromID { + metadata["forked_from_id"] = forkedFromID + metadata["source"] = "subagent" + } if let cwd { metadata["cwd"] = cwd } @@ -221,6 +226,73 @@ struct CodexForkAttributionMigrationTests { }) } + @Test + func `bounded migration keeps a partial legacy fork quarantined until completion`() throws { + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 5, day: 18) + let events = (0..<80).map { index in + (day.addingTimeInterval(TimeInterval(index + 1)), (index + 1) * 10) + } + try self.writeSession( + env, + day: day, + events: events, + model: "gpt-5.6-sol", + sessionID: "bounded-legacy-fork", + forkedFromID: "missing-parent", + filename: "bounded-legacy-fork.jsonl") + var options = self.options(env) + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + self.markLegacyForkCandidate( + env, + sessionID: "bounded-legacy-fork", + producerKey: "codex:cu:p6f689d90f8eedcbd") + + let seeded = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let seededEntry = try #require(seeded.files.first { $0.value.sessionId == "bounded-legacy-fork" }) + let slice = max(1, seededEntry.value.size / 4) + options.maxCodexSessionFileBytes = slice + options.maxCodexScanBytesPerRefresh = slice + + let partialReport = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(1), + options: options) + var migrated = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + var migratedUsage = try #require(migrated.files[seededEntry.key]) + #expect(migratedUsage.codexScanComplete == false) + #expect(migratedUsage.codexForkAttributionVersion == nil) + #expect(CostUsageScanner.isLegacyForkAttributionCandidate(migratedUsage)) + #expect(migrated.codexForkAttributionVersion == nil) + #expect(partialReport.data.isEmpty) + + var completedReport = partialReport + for offset in 2...8 where migratedUsage.codexScanComplete != true { + completedReport = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day.addingTimeInterval(TimeInterval(offset)), + options: options) + migrated = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + migratedUsage = try #require(migrated.files[seededEntry.key]) + } + + #expect(migratedUsage.codexScanComplete == true) + #expect(migratedUsage.codexForkAttributionVersion == CostUsageScanner.codexForkAttributionVersion) + #expect(!CostUsageScanner.isLegacyForkAttributionCandidate(migratedUsage)) + #expect(migrated.codexForkAttributionVersion == CostUsageScanner.codexForkAttributionVersion) + #expect(completedReport.data.first?.totalTokens == 32400) + } + @Test func `out of window legacy suspect is removed and later expansion reparses source`() throws { let env = try CostUsageTestEnvironment() diff --git a/Tests/CodexBarTests/CodexLocalProjectUsageTests.swift b/Tests/CodexBarTests/CodexLocalProjectUsageTests.swift index d749ca6295..1ff663ad66 100644 --- a/Tests/CodexBarTests/CodexLocalProjectUsageTests.swift +++ b/Tests/CodexBarTests/CodexLocalProjectUsageTests.swift @@ -807,6 +807,80 @@ struct CodexLocalProjectUsageTests { options: .init(scannerOptions: options)) == nil) } + @Test + func `workspace refresh excludes a budget deferred legacy fork candidate`() throws { + #if canImport(SQLite3) + let env = try CostUsageTestEnvironment() + defer { env.cleanup() } + let day = try env.makeLocalNoon(year: 2026, month: 7, day: 21) + let project = env.root.appendingPathComponent("CodexBar", isDirectory: true) + try FileManager.default.createDirectory( + at: project.appendingPathComponent(".git", isDirectory: true), + withIntermediateDirectories: true) + let safeFixture = CodexUsageFixture( + filename: "safe-sidecar.jsonl", + sessionID: "safe-sidecar", + cwd: project.path, + input: 100, + cached: 20, + output: 30) + let legacyFixture = CodexUsageFixture( + filename: "legacy-sidecar.jsonl", + sessionID: "legacy-sidecar", + cwd: project.path, + input: 900, + cached: 80, + output: 20) + _ = try self.writeCodexUsageFile(env: env, day: day, fixture: safeFixture) + _ = try self.writeCodexUsageFile(env: env, day: day, fixture: legacyFixture) + var options = CostUsageScanner.Options( + codexSessionsRoot: env.codexSessionsRoot, + cacheRoot: env.cacheRoot) + options.refreshMinIntervalSeconds = 0 + _ = CostUsageScanner.loadDailyReport( + provider: .codex, + since: day, + until: day, + now: day, + options: options) + var cache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let safePath = try #require(cache.files.first { $0.value.sessionId == safeFixture.sessionID }?.key) + let legacyPath = try #require(cache.files.first { $0.value.sessionId == legacyFixture.sessionID }?.key) + #expect(cache.files[safePath]?.codexForkAttributionVersion == CostUsageScanner.codexForkAttributionVersion) + cache.files[legacyPath]?.forkedFromId = "missing-parent" + cache.files[legacyPath]?.codexSession?.forkedFromId = "missing-parent" + cache.files[legacyPath]?.forkBaselineDependencyKey = "missing-parent|legacy-raw-boundary" + cache.files[legacyPath]?.codexForkAttributionVersion = nil + cache.codexForkAttributionVersion = nil + CostUsageCacheIO.save(provider: .codex, cache: cache, cacheRoot: env.cacheRoot) + options.maxCodexSessionFileBytes = 1 + options.maxCodexScanBytesPerRefresh = 1 + options.preferNewestCodexSessionsFirst = false + + let snapshot = try CodexLocalProjectUsageIndexer.loadSnapshot( + now: day.addingTimeInterval(1), + historyDays: 1, + options: .init(scannerOptions: options)) + let rawCache = CostUsageCacheIO.load(provider: .codex, cacheRoot: env.cacheRoot) + let pending = try #require(rawCache.files.values.first { $0.sessionId == legacyFixture.sessionID }) + let safeRaw = try #require(rawCache.files.values.first { $0.sessionId == safeFixture.sessionID }) + let presented = CostUsageScanner.codexCacheForPresentation(rawCache) + let sidecarCache = try CodexWorkspaceUsageSidecar(cacheRoot: env.cacheRoot) + .usageCache(roots: cache.roots ?? [:]) + + #expect(CostUsageScanner.isLegacyForkAttributionCandidate(pending)) + #expect(safeRaw.days.values.contains { !$0.isEmpty }) + #expect(presented.files.values.contains { $0.sessionId == safeFixture.sessionID }) + #expect(!presented.files.values.contains { $0.sessionId == legacyFixture.sessionID }) + #expect(snapshot.total.totalTokens == 130) + #expect(snapshot.sessions.map(\.id) == ["safe-sidecar"]) + #expect(sidecarCache.files.values.contains { $0.sessionId == safeFixture.sessionID }) + #expect(!sidecarCache.files.values.contains { $0.sessionId == legacyFixture.sessionID }) + #else + #expect(Bool(true)) + #endif + } + @Test func `project usage severity separates high usage from unknown cost coverage`() throws { let env = try CostUsageTestEnvironment() diff --git a/Tests/CodexBarTests/CostUsageCacheTests.swift b/Tests/CodexBarTests/CostUsageCacheTests.swift index ff37f14b3a..8cdddfa362 100644 --- a/Tests/CodexBarTests/CostUsageCacheTests.swift +++ b/Tests/CodexBarTests/CostUsageCacheTests.swift @@ -175,12 +175,37 @@ struct CostUsageCacheTests { #expect(loaded.producerKey == "codex:cu:p1111111111111111") } + @Test + func `runtime bootstraps the exact current pre marker producer`() throws { + let root = try self.makeTemporaryCacheRoot() + defer { try? FileManager.default.removeItem(at: root) } + let currentProducer = try #require(CostUsageCacheIO.currentProducerKey(provider: .codex)) + + var cache = CostUsageCache() + cache.lastScanUnixMs = 123 + CostUsageCacheIO.save( + provider: .codex, + cache: cache, + cacheRoot: root, + producerKey: currentProducer) + + let loaded = CostUsageCacheIO.load(provider: .codex, cacheRoot: root) + #expect(loaded.lastScanUnixMs == 123) + #expect(loaded.producerKey == currentProducer) + #expect(loaded.codexCacheCompatibilityVersion == nil) + } + @Test func `runtime accepts only known pre marker bootstrap producers`() throws { let root = try self.makeTemporaryCacheRoot() defer { try? FileManager.default.removeItem(at: root) } - for producerKey in ["codex:cu:pa15a1040092b4a62", "codex:cu:p7378e1f7e954ea1f"] { + for producerKey in [ + "codex:cu:pa15a1040092b4a62", + "codex:cu:p7378e1f7e954ea1f", + "codex:cu:p6f689d90f8eedcbd", + "codex:cu:p21dae5bee0a0ece1", + ] { var cache = CostUsageCache() cache.lastScanUnixMs = 123 CostUsageCacheIO.save( @@ -221,6 +246,36 @@ struct CostUsageCacheTests { #expect(rejected.days.isEmpty) } + @Test + func `calendar isolation dominates Codex bootstrap admission`() throws { + let root = try self.makeTemporaryCacheRoot() + defer { try? FileManager.default.removeItem(at: root) } + let producerKey = "codex:cu:p6f689d90f8eedcbd" + var utc = Calendar(identifier: .gregorian) + utc.timeZone = try #require(TimeZone(identifier: "UTC")) + var bangkok = Calendar(identifier: .gregorian) + bangkok.timeZone = try #require(TimeZone(identifier: "Asia/Bangkok")) + + var legacy = CostUsageCache() + legacy.producerKey = producerKey + legacy.lastScanUnixMs = 123 + let url = CostUsageCacheIO.cacheFileURL(provider: .codex, cacheRoot: root) + try FileManager.default.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true) + try JSONEncoder().encode(legacy).write(to: url) + #expect(CostUsageCacheIO.load(provider: .codex, cacheRoot: root, calendar: utc).lastScanUnixMs == 0) + + CostUsageCacheIO.save( + provider: .codex, + cache: legacy, + cacheRoot: root, + producerKey: producerKey, + calendar: utc) + #expect(CostUsageCacheIO.load(provider: .codex, cacheRoot: root, calendar: utc).lastScanUnixMs == 123) + #expect(CostUsageCacheIO.load(provider: .codex, cacheRoot: root, calendar: bangkok).lastScanUnixMs == 0) + } + @Test func `legacy cache without producer key is ignored`() throws { let root = try self.makeTemporaryCacheRoot()