Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 24 additions & 1 deletion Sources/CodexBarCore/CodexLocalProjectUsageIndexer.swift
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,32 @@ enum CodexLocalProjectUsageIndexer {
_ = now
let clampedHistoryDays = max(1, min(365, historyDays))
let stableScopeSignature = self.stableScopeSignature(options: options.scannerOptions)
let cacheProducerKey: String?
switch CostUsageCacheIO.codexCacheAdmission(
cacheRoot: options.scannerOptions.cacheRoot,
calendar: options.scannerOptions.calendar)
{
case .missing:
cacheProducerKey = nil
case .rejected:
return nil
case let .accepted(cache):
let roots = CostUsageScanner.codexSessionsRoots(options: options.scannerOptions)
let scopedCache = CostUsageScanner.codexCache(cache, scopedTo: roots)
guard !scopedCache.files.values.contains(where: CostUsageScanner.isLegacyForkAttributionCandidate) else {
// The sidecar snapshot is aggregate-only and cannot quarantine individual legacy rows.
// Withhold it until the normal scanner selectively migrates parent-dependent sources.
return nil
}
cacheProducerKey = cache.producerKey
}
let sidecar = CodexWorkspaceUsageSidecar(cacheRoot: options.scannerOptions.cacheRoot)
let catalogResult = CodexThreadCatalogReader.loadResult(options: options.scannerOptions)
let sourceStatus = CodexLocalProjectUsageSourceStatus(catalog: catalogResult.completeness)
if let snapshot = sidecar.loadLatestSnapshot(
scopeSignature: stableScopeSignature,
historyDays: clampedHistoryDays,
cacheProducerKey: cacheProducerKey,
catalog: catalogResult.isComplete ? catalogResult.catalog : nil)
{
return self.projecting(snapshot, sourceStatus: sourceStatus)
Expand Down Expand Up @@ -69,10 +89,13 @@ enum CodexLocalProjectUsageIndexer {
checkCancellation: checkCancellation)
try checkCancellation?()

let cache = CostUsageCacheIO.load(
let rawCache = CostUsageCacheIO.load(
provider: .codex,
cacheRoot: scannerOptions.cacheRoot,
calendar: scannerOptions.calendar)
// A budget-limited refresh can leave legacy fork candidates pending. Keep the workspace
// sidecar on the same provenance boundary as daily/project/session cache presentation.
let cache = CostUsageScanner.codexCacheForPresentation(rawCache)
let catalogResult = CodexThreadCatalogReader.loadResult(options: scannerOptions)
let catalog = catalogResult.catalog
let sourceStatus = CodexLocalProjectUsageSourceStatus(catalog: catalogResult.completeness)
Expand Down
4 changes: 4 additions & 0 deletions Sources/CodexBarCore/CodexWorkspaceUsageSidecar.swift
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ struct CodexWorkspaceUsageSidecar: Sendable {
scopeSignature: String,
historyDays: Int,
rootsFingerprint: [String: Int64]? = nil,
cacheProducerKey: String? = nil,
cache: CostUsageCache? = nil,
catalog: CodexThreadCatalog? = nil) -> CodexLocalProjectUsageSnapshot?
{
Expand Down Expand Up @@ -125,6 +126,9 @@ struct CodexWorkspaceUsageSidecar: Sendable {
persistedRoots == rootsFingerprint
else { return nil }
}
if let cacheProducerKey, Self.columnString(statement, at: 3) != cacheProducerKey {
return nil
}
if let cache {
guard Self.columnString(statement, at: 3) == cache.producerKey,
Self.columnString(statement, at: 4) == cache.codexPricingKey,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand.

enum CodexParserHash {
static let value = "21dae5bee0a0ece1"
static let value = "0a042d13ecd8201e"
}
83 changes: 70 additions & 13 deletions Sources/CodexBarCore/Vendored/CostUsage/CostUsageCache.swift
Original file line number Diff line number Diff line change
@@ -1,10 +1,26 @@
import Foundation

enum CostUsageCacheIO {
/// Producer keys from older parser hashes whose caches are still valid under the current
/// delta semantics. Cleared for #2037: interleave containment changed how cumulative
/// totals are counted, so every earlier cache must be rebuilt.
private static let compatibleCodexProducerKeys: Set<String> = []
enum CodexCacheAdmission {
case missing
case rejected
case accepted(CostUsageCache)
}

/// The persisted Codex accounting contract, independent of the raw source fingerprint.
/// Bump when cached fields, cumulative-delta semantics, or ownership semantics become incompatible.
/// Do not bump for provider additions, logging, UI, or other changes that leave cached accounting intact.
static let codexCacheCompatibilityVersion = 1

/// Audited pre-marker producers whose persisted accounting contract can migrate to version 1.
/// #2037 invalidated earlier producers. This one-time bridge covers released caches plus the
/// immediate upstream predecessor; once version 1 ships, future hashes use the marker instead.
private static let bootstrapCodexProducerKeys: Set<String> = [
"codex:cu:pa15a1040092b4a62",
"codex:cu:p7378e1f7e954ea1f",
"codex:cu:p6f689d90f8eedcbd",
"codex:cu:p21dae5bee0a0ece1",
]

/// Parsing and attribution changes rotate the Codex parser producer key.
/// Increment this artifact version only when the stored schema or cache layout becomes incompatible.
Expand Down Expand Up @@ -38,15 +54,23 @@ enum CostUsageCacheIO {
producerKey: String? = nil,
calendar: Calendar? = nil) -> CostUsageCache
{
if provider == .codex, producerKey == nil {
if case let .accepted(cache) = self.codexCacheAdmission(
cacheRoot: cacheRoot,
calendar: calendar)
{
return cache
}
return CostUsageCache()
}

let url = self.cacheFileURL(provider: provider, cacheRoot: cacheRoot)
let expectedProducerKey = producerKey ?? self.currentProducerKey(provider: provider)
let compatibleProducerKeys = producerKey == nil && provider == .codex
? self.compatibleCodexProducerKeys
: []
if let decoded = self.loadCache(
at: url,
expectedProducerKey: expectedProducerKey,
compatibleProducerKeys: compatibleProducerKeys)
expectedCodexCompatibilityVersion: nil,
bootstrapCodexProducerKeys: [])
{
if let calendar, decoded.timeZoneIdentifier != calendar.timeZone.identifier {
return CostUsageCache()
Expand All @@ -56,19 +80,43 @@ enum CostUsageCacheIO {
return CostUsageCache()
}

static func codexCacheAdmission(
cacheRoot: URL? = nil,
calendar: Calendar? = nil) -> CodexCacheAdmission
{
let url = self.cacheFileURL(provider: .codex, cacheRoot: cacheRoot)
guard FileManager.default.fileExists(atPath: url.path) else { return .missing }
guard let cache = self.loadCache(
at: url,
expectedProducerKey: self.currentProducerKey(provider: .codex),
expectedCodexCompatibilityVersion: self.codexCacheCompatibilityVersion,
bootstrapCodexProducerKeys: self.bootstrapCodexProducerKeys)
else { return .rejected }
if let calendar, cache.timeZoneIdentifier != calendar.timeZone.identifier {
return .rejected
}
return .accepted(cache)
}

private static func loadCache(
at url: URL,
expectedProducerKey: String?,
compatibleProducerKeys: Set<String>) -> CostUsageCache?
expectedCodexCompatibilityVersion: Int?,
bootstrapCodexProducerKeys: Set<String>) -> CostUsageCache?
{
guard let data = try? Data(contentsOf: url) else { return nil }
guard let decoded = try? JSONDecoder().decode(CostUsageCache.self, from: data)
else { return nil }
guard decoded.version == 1 else { return nil }
if let expectedProducerKey {
guard decoded.producerKey == expectedProducerKey
|| decoded.producerKey.map(compatibleProducerKeys.contains) == true
else { return nil }
if let expectedCodexCompatibilityVersion {
let compatibleContract = decoded.producerKey != nil
&& decoded.codexCacheCompatibilityVersion == expectedCodexCompatibilityVersion
let compatibleBootstrap = decoded.codexCacheCompatibilityVersion == nil
&& (decoded.producerKey == expectedProducerKey
|| decoded.producerKey.map(bootstrapCodexProducerKeys.contains) == true)
guard compatibleContract || compatibleBootstrap else { return nil }
} else if let expectedProducerKey {
guard decoded.producerKey == expectedProducerKey else { return nil }
}
return decoded
}
Expand All @@ -87,6 +135,9 @@ enum CostUsageCacheIO {
var cache = cache
cache.producerKey = producerKey ?? self.currentProducerKey(provider: provider)
cache.timeZoneIdentifier = calendar.timeZone.identifier
if provider == .codex, producerKey == nil {
cache.codexCacheCompatibilityVersion = self.codexCacheCompatibilityVersion
}

let tmp = dir.appendingPathComponent(".tmp-\(UUID().uuidString).json", isDirectory: false)
let data = (try? JSONEncoder().encode(cache)) ?? Data()
Expand Down Expand Up @@ -114,13 +165,17 @@ enum CostUsageCacheIO {
struct CostUsageCache: Codable {
var version: Int = 1
var producerKey: String?
/// Persisted accounting contract; nil identifies a pre-marker cache eligible only by bootstrap key.
var codexCacheCompatibilityVersion: Int?
var lastScanUnixMs: Int64 = 0
var scanSinceKey: String?
var scanUntilKey: String?
var timeZoneIdentifier: String?
var codexPricingKey: String?
var codexPriorityMetadataKey: String?
var codexProjectMetadataVersion: Int?
/// Optional migration marker; absent caches must inspect parent-dependent fork candidates.
var codexForkAttributionVersion: Int?
var codexPriorityTurnKeys: [String: String]?
var codexPriorityTurnIDsByDay: [String: [String]]?

Expand Down Expand Up @@ -151,6 +206,8 @@ struct CostUsageFileUsage: Codable {
var sessionId: String?
var forkedFromId: String?
var forkBaselineDependencyKey: String?
/// Set after this file has passed the fork-attribution parser; nil requires the dependency-key check.
var codexForkAttributionVersion: Int?
var projectPath: String?
var canonicalProjectPath: String?
var codexCostCacheComplete: Bool?
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,48 @@ import Darwin
#endif

extension CostUsageScanner {
/// #2285 persisted this key for every compact parent candidate, including known-model rows.
/// Only the sentinel proves the file never depended on parent context.
static func isLegacyForkAttributionCandidate(_ usage: CostUsageFileUsage) -> Bool {
usage.forkedFromId != nil
&& usage.codexForkAttributionVersion != codexForkAttributionVersion
&& usage.forkBaselineDependencyKey != codexForkDependencyNotRequiredKey
}

static func shouldDropLegacyForkCandidateOutsideWindow(
_ usage: CostUsageFileUsage,
range: CostUsageDayRange) -> Bool
{
self.isLegacyForkAttributionCandidate(usage)
&& usage.codexScanComplete != false
&& !usage.touchesCodexScanWindow(sinceKey: range.scanSinceKey, untilKey: range.scanUntilKey)
}

static func dropLegacyForkCandidatesOutsideWindow(
cache: inout CostUsageCache,
range: CostUsageDayRange)
{
let stalePaths = cache.files.compactMap { path, usage in
self.shouldDropLegacyForkCandidateOutsideWindow(usage, range: range) ? path : nil
}
for path in stalePaths {
guard let usage = cache.files[path] else { continue }
self.applyFileDays(cache: &cache, fileDays: usage.days, sign: -1)
cache.files.removeValue(forKey: path)
}
}

/// Keep every cache-backed presentation surface on the same migration boundary. This is
/// intentionally provenance-based: a stale copied prefix may already carry a known model.
static func codexCacheForPresentation(_ cache: CostUsageCache) -> CostUsageCache {
var projected = cache
for (path, usage) in cache.files where Self.isLegacyForkAttributionCandidate(usage) {
Self.applyFileDays(cache: &projected, fileDays: usage.days, sign: -1)
projected.files.removeValue(forKey: path)
}
return projected
}

private final class CodexModelsDevCatalogResolver {
private var catalog: ModelsDevCatalog?
private let cacheRoot: URL?
Expand Down Expand Up @@ -960,12 +1002,18 @@ extension CostUsageScanner {
let needsSessionId = cached.sessionId == nil
guard cached.mtimeUnixMs == input.metadata.mtimeUnixMs,
cached.size == input.metadata.size,
cached.parsedBytes == cached.size,
cached.codexScanComplete != false,
!needsSessionId,
!context.forceFullScan
else { return false }

guard !Self.cachedCodexFileNeedsPriorityRescan(cached, context: context) else { return false }
if context.needsForkAttributionMigration,
Self.isLegacyForkAttributionCandidate(cached)
{
return false
}

let sessionAlreadyContributed = cached.sessionId.map { state.contributingSessionIds.contains($0) } ?? false
let cachedRows = cached.codexRows ?? []
Expand Down Expand Up @@ -1221,6 +1269,11 @@ extension CostUsageScanner {
codexJSONLResumeState: delta.jsonlResumeState,
codexBufferedSubagentLines: delta.bufferedSubagentLines)
.refreshingCodexWorkspaceUsageFingerprint()
if cache.files[input.metadata.path]?.codexScanComplete == true {
cache.files[input.metadata.path]?.codexForkAttributionVersion = Self.codexForkAttributionVersion
} else {
cache.files[input.metadata.path]?.codexForkAttributionVersion = cached.codexForkAttributionVersion
}
Self.rememberScannedCodexFile(
input: input,
session: CodexScannedSession(id: sessionId, days: mergedDays),
Expand All @@ -1241,7 +1294,13 @@ extension CostUsageScanner {
if let cached = input.cached {
self.applyFileDays(cache: &cache, fileDays: cached.days, sign: -1)
}
let migratedCached = input.cached.map { Self.codexFileUsageWithCostCache($0, context: context) }
// A legacy parent-dependent fork file cannot carry unreparsed days across this migration: the
// current request may not cover its suspect day. Drop its retained projection and stamp
// the file current only from fresh source rows.
let hasLegacyForkCandidate = input.cached.map { Self.isLegacyForkAttributionCandidate($0) } ?? false
let migratedCached = hasLegacyForkCandidate
? nil
: input.cached.map { Self.codexFileUsageWithCostCache($0, context: context) }
var usageDays = context.dropDeferredCodexRows
? [:]
: Self.fileDaysOutsideScanWindow(migratedCached?.days ?? [:], range: context.range)
Expand All @@ -1251,7 +1310,9 @@ extension CostUsageScanner {
range: context.range,
maxBytesToRead: maxBytesToRead,
inheritedTotalsResolver: context.resources.inheritedResolver.inheritedTotals(for:atOrBefore:),
inheritedRawTotalsResolver: context.resources.inheritedResolver.rawTotals(for:atOrBefore:),
checkCancellation: context.checkCancellation)
let scanComplete = parsed.parsedBytes >= input.metadata.size && parsed.jsonlResumeState == nil
let forkBaselineDependencyKey = Self.codexForkBaselineDependencyKey(
parentSessionId: parsed.forkedFromId,
dependsOnParentTotals: parsed.dependsOnParentTotals,
Expand All @@ -1263,7 +1324,12 @@ extension CostUsageScanner {
title: nil,
startedAtUnixMs: nil,
latestActivityUnixMs: nil)
let parsedCodexSession = cachedSessionMetadata.merging(parsed.codexSession)
var parsedCodexSession = cachedSessionMetadata.merging(parsed.codexSession)
if !scanComplete {
// A bounded prefix may stop before lineage metadata. Preserve the legacy provenance
// until a complete parse can safely reclassify and stamp this row.
parsedCodexSession.forkedFromId = parsedCodexSession.forkedFromId ?? input.cached?.forkedFromId
}
let sessionId = parsedCodexSession.sessionId ?? parsed.sessionId ?? input.cached?.sessionId
let projectPath = parsed.projectPath ?? input.cached?.projectPath
let canonicalProjectPath = parsed.projectPath.map {
Expand Down Expand Up @@ -1308,8 +1374,10 @@ extension CostUsageScanner {
hasInterleavedTotals: parsed.hasInterleavedTotals,
lastCodexTurnID: parsed.lastCodexTurnID,
sessionId: sessionId,
forkedFromId: parsedCodexSession.forkedFromId ?? parsed.forkedFromId,
forkBaselineDependencyKey: forkBaselineDependencyKey,
forkedFromId: parsedCodexSession.forkedFromId ?? parsed.forkedFromId ?? input.cached?.forkedFromId,
forkBaselineDependencyKey: scanComplete
? forkBaselineDependencyKey
: input.cached?.forkBaselineDependencyKey ?? forkBaselineDependencyKey,
projectPath: projectPath,
canonicalProjectPath: canonicalProjectPath,
codexSession: parsedCodexSession.isEmpty ? nil : parsedCodexSession,
Expand Down Expand Up @@ -1364,10 +1432,13 @@ extension CostUsageScanner {
modelsDevCacheRoot: context.resources.modelsDevCacheRoot),
codexScanFileId: input.metadata.fileId,
codexScanTargetSize: input.metadata.size,
codexScanComplete: parsed.parsedBytes >= input.metadata.size && parsed.jsonlResumeState == nil,
codexScanComplete: scanComplete,
codexJSONLResumeState: parsed.jsonlResumeState,
codexBufferedSubagentLines: parsed.bufferedSubagentLines)
.refreshingCodexWorkspaceUsageFingerprint()
if cache.files[input.metadata.path]?.codexScanComplete == true {
cache.files[input.metadata.path]?.codexForkAttributionVersion = Self.codexForkAttributionVersion
}
Self.applyFileDays(cache: &cache, fileDays: cache.files[input.metadata.path]?.days ?? [:], sign: 1)
Self.rememberScannedCodexFile(
input: input,
Expand Down Expand Up @@ -1497,8 +1568,10 @@ extension CostUsageScanner {
let catalogResolver = CodexModelsDevCatalogResolver(
catalog: modelsDevCatalog,
cacheRoot: modelsDevCacheRoot)
var reportCache = cache
for (path, usage) in cache.files where self.needsCodexCostCache(usage, range: range) {
// A compatible predecessor cache may hydrate before migration completes. Do not present a
// parent-dependent candidate; current files and sentinel-owned forks remain visible.
var reportCache = Self.codexCacheForPresentation(cache)
for (path, usage) in reportCache.files where self.needsCodexCostCache(usage, range: range) {
reportCache.files[path] = self.codexFileUsageWithCostCache(
usage,
range: range,
Expand Down
Loading