diff --git a/src/parser.ts b/src/parser.ts index 1f2ac824..a20f43f1 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -129,14 +129,14 @@ const LARGE_JSONL_LINE_BYTES = 32 * 1024 export function parseJsonlLine(line: string | Buffer): JournalEntry | null { if (Buffer.isBuffer(line)) { - if (line.length > LARGE_JSONL_LINE_BYTES) return parseLargeJsonlBuffer(line) + if (line.length > LARGE_JSONL_LINE_BYTES) return parseLargeJsonl(line) try { return JSON.parse(line.toString('utf-8')) as JournalEntry } catch { return null } } - if (line.length > LARGE_JSONL_LINE_BYTES) return parseLargeJsonlLine(line) + if (line.length > LARGE_JSONL_LINE_BYTES) return parseLargeJsonl(line) try { return JSON.parse(line) as JournalEntry } catch { @@ -152,125 +152,121 @@ type JsonValueBounds = { kind: 'string' | 'object' | 'array' | 'scalar' } -function findJsonStringEnd(source: string, start: number, limit = source.length): number { - for (let i = start + 1; i < limit; i++) { - const ch = source.charCodeAt(i) - if (ch === 0x5c) { - i++ - continue - } - if (ch === 0x22) return i - } - return -1 +type JsonIndexedSource = string | Buffer + +type JsonSource = { + readonly raw: JsonIndexedSource + readonly length: number + readonly slice: (start: number, end: number, maxChars?: number) => string } -function findJsonContainerEnd(source: string, start: number, open: number, close: number, limit = source.length): number { - let depth = 0 - let inString = false - for (let i = start; i < limit; i++) { - const ch = source.charCodeAt(i) - if (inString) { - if (ch === 0x5c) { - i++ - } else if (ch === 0x22) { - inString = false - } - continue - } - if (ch === 0x22) { - inString = true - } else if (ch === open) { - depth++ - } else if (ch === close) { - depth-- - if (depth === 0) return i - } - } - return -1 +function isAsciiWhitespace(ch: number | undefined): boolean { + return ch === 0x20 || ch === 0x0a || ch === 0x0d || ch === 0x09 || ch === 0x0b || ch === 0x0c } -function findJsonValueBounds(source: string, start: number, limit = source.length): JsonValueBounds | null { - let i = start - while (i < limit && /\s/.test(source[i]!)) i++ - if (i >= limit) return null - const ch = source.charCodeAt(i) - if (ch === 0x22) { - const end = findJsonStringEnd(source, i, limit) - return end === -1 ? null : { start: i, end: end + 1, kind: 'string' } - } - if (ch === 0x7b) { - const end = findJsonContainerEnd(source, i, 0x7b, 0x7d, limit) - return end === -1 ? null : { start: i, end: end + 1, kind: 'object' } - } - if (ch === 0x5b) { - const end = findJsonContainerEnd(source, i, 0x5b, 0x5d, limit) - return end === -1 ? null : { start: i, end: end + 1, kind: 'array' } +function isBufferWhitespaceAt(source: Buffer, index: number): boolean { + const byte = source[index] + if (isAsciiWhitespace(byte)) return true + if (byte === undefined || byte < 0x80) return false + + let start = index + while (start > 0) { + const preceding = source[start] + if (preceding === undefined || (preceding & 0xc0) !== 0x80) break + start-- } - let end = i - while (end < limit) { - const c = source.charCodeAt(end) - if (c === 0x2c || c === 0x7d || c === 0x5d || /\s/.test(source[end]!)) break - end++ + const first = source[start] + if (first === undefined) return false + let codePoint: number | undefined + let byteLength = 0 + if (first >= 0xc2 && first <= 0xdf) { + const second = source[start + 1] + if (second === undefined || (second & 0xc0) !== 0x80) return false + codePoint = ((first & 0x1f) << 6) | (second & 0x3f) + byteLength = 2 + } else if (first >= 0xe0 && first <= 0xef) { + const second = source[start + 1] + const third = source[start + 2] + if (second === undefined || third === undefined || (second & 0xc0) !== 0x80 || (third & 0xc0) !== 0x80) return false + codePoint = ((first & 0x0f) << 12) | ((second & 0x3f) << 6) | (third & 0x3f) + byteLength = 3 + } else if (first >= 0xf0 && first <= 0xf4) { + const second = source[start + 1] + const third = source[start + 2] + const fourth = source[start + 3] + if (second === undefined || third === undefined || fourth === undefined || (second & 0xc0) !== 0x80 || (third & 0xc0) !== 0x80 || (fourth & 0xc0) !== 0x80) { + return false + } + codePoint = ((first & 0x07) << 18) | ((second & 0x3f) << 12) | ((third & 0x3f) << 6) | (fourth & 0x3f) + byteLength = 4 } - return { start: i, end, kind: 'scalar' } + if (codePoint === undefined || index >= start + byteLength) return false + return codePoint === 0x00a0 || codePoint === 0x1680 || (codePoint >= 0x2000 && codePoint <= 0x200a) || codePoint === 0x2028 || codePoint === 0x2029 || codePoint === 0x202f || codePoint === 0x205f || codePoint === 0x3000 || codePoint === 0xfeff } -function findObjectFieldValue(source: string, objectStart: number, objectEnd: number, field: string): JsonValueBounds | null { - if (source.charCodeAt(objectStart) !== 0x7b) return null - let i = objectStart + 1 - while (i < objectEnd - 1) { - while (i < objectEnd && /\s/.test(source[i]!)) i++ - if (source.charCodeAt(i) === 0x2c) { - i++ - continue - } - if (source.charCodeAt(i) !== 0x22) { - i++ - continue +function safeBufferSegmentEnd(source: Buffer, index: number): number { + while (index > 0 && ((source[index] ?? 0) & 0xc0) === 0x80) index-- + return index +} + +function createJsonSource(source: string | Buffer): JsonSource { + if (typeof source === 'string') { + return { + raw: source, + length: source.length, + slice: (start, end, maxChars = Number.POSITIVE_INFINITY) => source.slice(start, Math.min(end, start + maxChars)), } - const keyEnd = findJsonStringEnd(source, i, objectEnd) - if (keyEnd === -1) return null - const key = source.slice(i + 1, keyEnd) - i = keyEnd + 1 - while (i < objectEnd && /\s/.test(source[i]!)) i++ - if (source.charCodeAt(i) !== 0x3a) continue - const value = findJsonValueBounds(source, i + 1, objectEnd) - if (!value) return null - if (key === field) return value - i = value.end } - return null + + return { + raw: source, + length: source.length, + slice: (start, end, maxChars = Number.POSITIVE_INFINITY) => { + const cappedEnd = Number.isFinite(maxChars) ? safeBufferSegmentEnd(source, Math.min(end, start + maxChars * 4)) : end + return source.subarray(start, cappedEnd).toString('utf-8').slice(0, maxChars) + }, + } } -function readJsonString(source: string, bounds: JsonValueBounds | null, cap = Number.POSITIVE_INFINITY): string | undefined { - if (!bounds || bounds.kind !== 'string') return undefined - let out = '' - for (let i = bounds.start + 1; i < bounds.end - 1 && out.length < cap; i++) { - const ch = source[i]! - if (ch !== '\\') { - out += ch - continue - } - const next = source[++i] - if (!next) break - if (next === 'n') out += '\n' - else if (next === 'r') out += '\r' - else if (next === 't') out += '\t' - else if (next === 'b') out += '\b' - else if (next === 'f') out += '\f' - else if (next === 'u' && i + 4 < bounds.end) { - const hex = source.slice(i + 1, i + 5) - const code = Number.parseInt(hex, 16) - if (Number.isFinite(code)) out += String.fromCharCode(code) - i += 4 - } else { - out += next - } +function jsonCharCodeAt(source: JsonSource, index: number): number { + return typeof source.raw === 'string' ? source.raw.charCodeAt(index) : source.raw[index] ?? Number.NaN +} + +function skipJsonWhitespace(source: JsonSource, start: number, limit = source.length): number { + if (typeof source.raw === 'string') { + let i = start + while (i < limit && /\s/.test(source.raw[i]!)) i++ + return i } - return out + let i = start + while (i < limit && isBufferWhitespaceAt(source.raw, i)) i++ + return i +} + +function findJsonStringEnd(source: JsonSource, start: number, limit = source.length): number { + return typeof source.raw === 'string' + ? findJsonStringEndString(source.raw, start, limit) + : findJsonStringEndBuffer(source.raw, start, limit) } -function readJsonNumberField(source: string, objectBounds: JsonValueBounds | null, field: string): number | undefined { +function findJsonContainerEnd(source: JsonSource, start: number, open: number, close: number, limit = source.length): number { + return typeof source.raw === 'string' + ? findJsonContainerEndString(source.raw, start, open, close, limit) + : findJsonContainerEndBuffer(source.raw, start, open, close, limit) +} + +function findObjectFieldValue(source: JsonSource, objectStart: number, objectEnd: number, field: string): JsonValueBounds | null { + return typeof source.raw === 'string' + ? findObjectFieldValueString(source.raw, objectStart, objectEnd, field) + : findObjectFieldValueBuffer(source.raw, objectStart, objectEnd, field) +} + +function readJsonString(source: JsonSource, bounds: JsonValueBounds | null, cap = Number.POSITIVE_INFINITY): string | undefined { + if (typeof source.raw === 'string') return readJsonStringString(source.raw, bounds, cap) + return readJsonStringBuffer(source.raw, bounds, cap) +} + +function readJsonNumberField(source: JsonSource, objectBounds: JsonValueBounds | null, field: string): number | undefined { if (!objectBounds || objectBounds.kind !== 'object') return undefined const bounds = findObjectFieldValue(source, objectBounds.start, objectBounds.end, field) if (!bounds) return undefined @@ -298,7 +294,7 @@ function extractAdvisorIterations(usageObjectJson: string): ApiUsageIteration[] return advisor.length > 0 ? advisor : undefined } -function parseLargeUsage(source: string, usageBounds: JsonValueBounds | null) { +function parseLargeUsage(source: JsonSource, usageBounds: JsonValueBounds | null) { const usage: AssistantMessageContent['usage'] = { input_tokens: readJsonNumberField(source, usageBounds, 'input_tokens') ?? 0, output_tokens: readJsonNumberField(source, usageBounds, 'output_tokens') ?? 0, @@ -337,17 +333,17 @@ function parseLargeUsage(source: string, usageBounds: JsonValueBounds | null) { return usage } -function extractLargeToolBlocks(source: string, contentBounds: JsonValueBounds | null): ToolUseBlock[] { +function extractLargeToolBlocks(source: JsonSource, contentBounds: JsonValueBounds | null): ToolUseBlock[] { if (!contentBounds || contentBounds.kind !== 'array') return [] const tools: ToolUseBlock[] = [] let i = contentBounds.start + 1 while (i < contentBounds.end - 1 && tools.length < MAX_TOOL_BLOCKS) { - while (i < contentBounds.end && /\s/.test(source[i]!)) i++ - if (source.charCodeAt(i) === 0x2c) { + i = skipJsonWhitespace(source, i, contentBounds.end) + if (jsonCharCodeAt(source, i) === 0x2c) { i++ continue } - if (source.charCodeAt(i) !== 0x7b) { + if (jsonCharCodeAt(source, i) !== 0x7b) { i++ continue } @@ -384,7 +380,7 @@ function extractLargeToolBlocks(source: string, contentBounds: JsonValueBounds | return tools } -function extractLargeUserText(source: string, contentBounds: JsonValueBounds | null): string | undefined { +function extractLargeUserText(source: JsonSource, contentBounds: JsonValueBounds | null): string | undefined { if (!contentBounds) return undefined if (contentBounds.kind === 'string') return readJsonString(source, contentBounds, USER_TEXT_CAP) if (contentBounds.kind !== 'array') return undefined @@ -392,12 +388,12 @@ function extractLargeUserText(source: string, contentBounds: JsonValueBounds | n let text = '' let i = contentBounds.start + 1 while (i < contentBounds.end - 1 && text.length < USER_TEXT_CAP) { - while (i < contentBounds.end && /\s/.test(source[i]!)) i++ - if (source.charCodeAt(i) === 0x2c) { + i = skipJsonWhitespace(source, i, contentBounds.end) + if (jsonCharCodeAt(source, i) === 0x2c) { i++ continue } - if (source.charCodeAt(i) !== 0x7b) { + if (jsonCharCodeAt(source, i) !== 0x7b) { i++ continue } @@ -418,7 +414,7 @@ function extractLargeUserText(source: string, contentBounds: JsonValueBounds | n return text || undefined } -function extractLargeAddedNames(source: string, attachmentBounds: JsonValueBounds | null): string[] { +function extractLargeAddedNames(source: JsonSource, attachmentBounds: JsonValueBounds | null): string[] { if (!attachmentBounds || attachmentBounds.kind !== 'object') return [] const attachmentType = readJsonString(source, findObjectFieldValue(source, attachmentBounds.start, attachmentBounds.end, 'type')) if (attachmentType !== 'deferred_tools_delta') return [] @@ -427,12 +423,12 @@ function extractLargeAddedNames(source: string, attachmentBounds: JsonValueBound const names: string[] = [] let i = addedNames.start + 1 while (i < addedNames.end - 1 && names.length < MAX_ADDED_NAMES) { - while (i < addedNames.end && /\s/.test(source[i]!)) i++ - if (source.charCodeAt(i) === 0x2c) { + i = skipJsonWhitespace(source, i, addedNames.end) + if (jsonCharCodeAt(source, i) === 0x2c) { i++ continue } - if (source.charCodeAt(i) !== 0x22) { + if (jsonCharCodeAt(source, i) !== 0x22) { i++ continue } @@ -445,65 +441,119 @@ function extractLargeAddedNames(source: string, attachmentBounds: JsonValueBound return names } -function parseLargeJsonlLine(line: string): JournalEntry | null { - const rootEnd = findJsonContainerEnd(line, 0, 0x7b, 0x7d) +function parseLargeJsonl(line: string | Buffer): JournalEntry | null { + const source = createJsonSource(line) + const rootStart = skipJsonWhitespace(source, 0) + const rootEnd = findJsonContainerEnd(source, rootStart, 0x7b, 0x7d) if (rootEnd === -1) return null - const rootStart = 0 const rootLimit = rootEnd + 1 - const type = readJsonString(line, findObjectFieldValue(line, rootStart, rootLimit, 'type')) + const type = readJsonString(source, findObjectFieldValue(source, rootStart, rootLimit, 'type')) if (!type) return null const entry: JournalEntry = { type } - const timestamp = readJsonString(line, findObjectFieldValue(line, rootStart, rootLimit, 'timestamp')) - const sessionId = readJsonString(line, findObjectFieldValue(line, rootStart, rootLimit, 'sessionId')) - const cwd = readJsonString(line, findObjectFieldValue(line, rootStart, rootLimit, 'cwd')) + const timestamp = readJsonString(source, findObjectFieldValue(source, rootStart, rootLimit, 'timestamp')) + const sessionId = readJsonString(source, findObjectFieldValue(source, rootStart, rootLimit, 'sessionId')) + const cwd = readJsonString(source, findObjectFieldValue(source, rootStart, rootLimit, 'cwd')) if (timestamp !== undefined) entry.timestamp = timestamp if (sessionId !== undefined) entry.sessionId = sessionId if (cwd !== undefined) entry.cwd = cwd - const addedNames = extractLargeAddedNames(line, findObjectFieldValue(line, rootStart, rootLimit, 'attachment')) + const addedNames = extractLargeAddedNames(source, findObjectFieldValue(source, rootStart, rootLimit, 'attachment')) if (addedNames.length > 0) { ;(entry as Record)['attachment'] = { type: 'deferred_tools_delta', addedNames } } if (type === 'user') { - const message = findObjectFieldValue(line, rootStart, rootLimit, 'message') + const message = findObjectFieldValue(source, rootStart, rootLimit, 'message') if (message?.kind === 'object') { - const content = findObjectFieldValue(line, message.start, message.end, 'content') - const text = extractLargeUserText(line, content) + const content = findObjectFieldValue(source, message.start, message.end, 'content') + const text = extractLargeUserText(source, content) if (text !== undefined) entry.message = { role: 'user', content: text } } return entry } if (type !== 'assistant') return entry - const message = findObjectFieldValue(line, rootStart, rootLimit, 'message') + const message = findObjectFieldValue(source, rootStart, rootLimit, 'message') if (message?.kind !== 'object') return entry - const model = readJsonString(line, findObjectFieldValue(line, message.start, message.end, 'model')) - const usageBounds = findObjectFieldValue(line, message.start, message.end, 'usage') + const model = readJsonString(source, findObjectFieldValue(source, message.start, message.end, 'model')) + const usageBounds = findObjectFieldValue(source, message.start, message.end, 'usage') if (!model || usageBounds?.kind !== 'object') return entry - const id = readJsonString(line, findObjectFieldValue(line, message.start, message.end, 'id')) - const contentBounds = findObjectFieldValue(line, message.start, message.end, 'content') + const id = readJsonString(source, findObjectFieldValue(source, message.start, message.end, 'id')) + const contentBounds = findObjectFieldValue(source, message.start, message.end, 'content') entry.message = { type: 'message', role: 'assistant', model, ...(id !== undefined ? { id } : {}), - content: extractLargeToolBlocks(line, contentBounds), - usage: parseLargeUsage(line, usageBounds), + content: extractLargeToolBlocks(source, contentBounds), + usage: parseLargeUsage(source, usageBounds), } return entry } -type BufferJsonValueBounds = { - start: number - end: number - kind: 'string' | 'object' | 'array' | 'scalar' +function findJsonStringEndString(source: string, start: number, limit = source.length): number { + for (let i = start + 1; i < limit; i++) { + const ch = source.charCodeAt(i) + if (ch === 0x5c) { + i++ + continue + } + if (ch === 0x22) return i + } + return -1 +} + +function findJsonContainerEndString(source: string, start: number, open: number, close: number, limit = source.length): number { + let depth = 0 + let inString = false + for (let i = start; i < limit; i++) { + const ch = source.charCodeAt(i) + if (inString) { + if (ch === 0x5c) { + i++ + } else if (ch === 0x22) { + inString = false + } + continue + } + if (ch === 0x22) { + inString = true + } else if (ch === open) { + depth++ + } else if (ch === close) { + depth-- + if (depth === 0) return i + } + } + return -1 } -function isJsonWhitespaceByte(ch: number | undefined): boolean { - return ch === 0x20 || ch === 0x0a || ch === 0x0d || ch === 0x09 +function findJsonValueBoundsString(source: string, start: number, limit = source.length): JsonValueBounds | null { + let i = start + while (i < limit && /\s/.test(source[i]!)) i++ + if (i >= limit) return null + const ch = source.charCodeAt(i) + if (ch === 0x22) { + const end = findJsonStringEndString(source, i, limit) + return end === -1 ? null : { start: i, end: end + 1, kind: 'string' } + } + if (ch === 0x7b) { + const end = findJsonContainerEndString(source, i, 0x7b, 0x7d, limit) + return end === -1 ? null : { start: i, end: end + 1, kind: 'object' } + } + if (ch === 0x5b) { + const end = findJsonContainerEndString(source, i, 0x5b, 0x5d, limit) + return end === -1 ? null : { start: i, end: end + 1, kind: 'array' } + } + let end = i + while (end < limit) { + const c = source.charCodeAt(end) + if (c === 0x2c || c === 0x7d || c === 0x5d || /\s/.test(source[end]!)) break + end++ + } + return { start: i, end, kind: 'scalar' } } function findJsonStringEndBuffer(source: Buffer, start: number, limit = source.length): number { @@ -543,9 +593,9 @@ function findJsonContainerEndBuffer(source: Buffer, start: number, open: number, return -1 } -function findJsonValueBoundsBuffer(source: Buffer, start: number, limit = source.length): BufferJsonValueBounds | null { +function findJsonValueBoundsBuffer(source: Buffer, start: number, limit = source.length): JsonValueBounds | null { let i = start - while (i < limit && isJsonWhitespaceByte(source[i])) i++ + while (i < limit && isBufferWhitespaceAt(source, i)) i++ if (i >= limit) return null const ch = source[i] if (ch === 0x22) { @@ -563,22 +613,44 @@ function findJsonValueBoundsBuffer(source: Buffer, start: number, limit = source let end = i while (end < limit) { const c = source[end] - if (c === 0x2c || c === 0x7d || c === 0x5d || isJsonWhitespaceByte(c)) break + if (c === 0x2c || c === 0x7d || c === 0x5d || isBufferWhitespaceAt(source, end)) break end++ } return { start: i, end, kind: 'scalar' } } -function bufferKeyEquals(source: Buffer, keyStart: number, keyEnd: number, field: string): boolean { - if (keyEnd - keyStart !== field.length) return false - return source.subarray(keyStart, keyEnd).equals(Buffer.from(field)) +function findObjectFieldValueString(source: string, objectStart: number, objectEnd: number, field: string): JsonValueBounds | null { + if (source.charCodeAt(objectStart) !== 0x7b) return null + let i = objectStart + 1 + while (i < objectEnd - 1) { + while (i < objectEnd && /\s/.test(source[i]!)) i++ + if (source.charCodeAt(i) === 0x2c) { + i++ + continue + } + if (source.charCodeAt(i) !== 0x22) { + i++ + continue + } + const keyEnd = findJsonStringEndString(source, i, objectEnd) + if (keyEnd === -1) return null + const keyStart = i + 1 + i = keyEnd + 1 + while (i < objectEnd && /\s/.test(source[i]!)) i++ + if (source.charCodeAt(i) !== 0x3a) continue + const value = findJsonValueBoundsString(source, i + 1, objectEnd) + if (!value) return null + if (source.slice(keyStart, keyEnd) === field) return value + i = value.end + } + return null } -function findObjectFieldValueBuffer(source: Buffer, objectStart: number, objectEnd: number, field: string): BufferJsonValueBounds | null { +function findObjectFieldValueBuffer(source: Buffer, objectStart: number, objectEnd: number, field: string): JsonValueBounds | null { if (source[objectStart] !== 0x7b) return null let i = objectStart + 1 while (i < objectEnd - 1) { - while (i < objectEnd && isJsonWhitespaceByte(source[i])) i++ + while (i < objectEnd && isBufferWhitespaceAt(source, i)) i++ if (source[i] === 0x2c) { i++ continue @@ -591,262 +663,114 @@ function findObjectFieldValueBuffer(source: Buffer, objectStart: number, objectE if (keyEnd === -1) return null const keyStart = i + 1 i = keyEnd + 1 - while (i < objectEnd && isJsonWhitespaceByte(source[i])) i++ + while (i < objectEnd && isBufferWhitespaceAt(source, i)) i++ if (source[i] !== 0x3a) continue const value = findJsonValueBoundsBuffer(source, i + 1, objectEnd) if (!value) return null - if (bufferKeyEquals(source, keyStart, keyEnd, field)) return value + if (keyEnd - keyStart === field.length && source.subarray(keyStart, keyEnd).equals(Buffer.from(field))) return value i = value.end } return null } +function appendStringJsonSegment(source: string, start: number, end: number, current: string, cap: number): string { + if (start >= end || current.length >= cap) return current + return current + source.slice(start, Math.min(end, start + cap - current.length)) +} + function appendBufferJsonSegment(source: Buffer, start: number, end: number, current: string, cap: number): string { if (start >= end || current.length >= cap) return current const remaining = cap - current.length - const cappedEnd = Number.isFinite(cap) ? Math.min(end, start + remaining * 4) : end + const cappedEnd = Number.isFinite(cap) ? safeBufferSegmentEnd(source, Math.min(end, start + remaining * 4)) : end return current + source.subarray(start, cappedEnd).toString('utf-8').slice(0, remaining) } -function readJsonStringBuffer(source: Buffer, bounds: BufferJsonValueBounds | null, cap = Number.POSITIVE_INFINITY): string | undefined { +function readJsonStringString(source: string, bounds: JsonValueBounds | null, cap = Number.POSITIVE_INFINITY): string | undefined { if (!bounds || bounds.kind !== 'string') return undefined let out = '' + const contentEnd = bounds.end - 1 let segmentStart = bounds.start + 1 - for (let i = bounds.start + 1; i < bounds.end - 1 && out.length < cap; i++) { - const ch = source[i] - if (ch !== 0x5c) continue - - out = appendBufferJsonSegment(source, segmentStart, i, out, cap) + let i = segmentStart + let scanLimit = Number.isFinite(cap) ? Math.min(contentEnd, segmentStart + cap) : contentEnd + while (i < contentEnd && out.length < cap) { + if (i >= scanLimit) { + out = appendStringJsonSegment(source, segmentStart, i, out, cap) + if (out.length >= cap) break + segmentStart = i + scanLimit = Number.isFinite(cap) ? Math.min(contentEnd, i + cap - out.length) : contentEnd + continue + } + const ch = source.charCodeAt(i) + if (ch !== 0x5c) { + i++ + continue + } + out = appendStringJsonSegment(source, segmentStart, i, out, cap) if (out.length >= cap) break - const next = source[++i] - if (next === undefined) break + i++ + const next = source.charCodeAt(i) + if (Number.isNaN(next)) break if (next === 0x6e) out += '\n' else if (next === 0x72) out += '\r' else if (next === 0x74) out += '\t' else if (next === 0x62) out += '\b' else if (next === 0x66) out += '\f' else if (next === 0x75 && i + 4 < bounds.end) { - const hex = source.subarray(i + 1, i + 5).toString('ascii') - const code = Number.parseInt(hex, 16) + const code = Number.parseInt(source.slice(i + 1, i + 5), 16) if (Number.isFinite(code)) out += String.fromCharCode(code) i += 4 } else { out += String.fromCharCode(next) } segmentStart = i + 1 + i++ } - return appendBufferJsonSegment(source, segmentStart, bounds.end - 1, out, cap) -} - -function readJsonNumberFieldBuffer(source: Buffer, objectBounds: BufferJsonValueBounds | null, field: string): number | undefined { - if (!objectBounds || objectBounds.kind !== 'object') return undefined - const bounds = findObjectFieldValueBuffer(source, objectBounds.start, objectBounds.end, field) - if (!bounds) return undefined - const value = Number(source.subarray(bounds.start, bounds.end).toString('ascii')) - return Number.isFinite(value) ? value : undefined -} - -function parseLargeUsageBuffer(source: Buffer, usageBounds: BufferJsonValueBounds | null) { - const usage: AssistantMessageContent['usage'] = { - input_tokens: readJsonNumberFieldBuffer(source, usageBounds, 'input_tokens') ?? 0, - output_tokens: readJsonNumberFieldBuffer(source, usageBounds, 'output_tokens') ?? 0, - cache_creation_input_tokens: readJsonNumberFieldBuffer(source, usageBounds, 'cache_creation_input_tokens'), - cache_read_input_tokens: readJsonNumberFieldBuffer(source, usageBounds, 'cache_read_input_tokens'), - } - - if (usageBounds?.kind === 'object') { - const cacheCreation = findObjectFieldValueBuffer(source, usageBounds.start, usageBounds.end, 'cache_creation') - const ephemeral5m = readJsonNumberFieldBuffer(source, cacheCreation, 'ephemeral_5m_input_tokens') - const ephemeral1h = readJsonNumberFieldBuffer(source, cacheCreation, 'ephemeral_1h_input_tokens') - if (ephemeral5m !== undefined || ephemeral1h !== undefined) { - ;(usage as AssistantMessageContent['usage']).cache_creation = { - ...(ephemeral5m !== undefined ? { ephemeral_5m_input_tokens: ephemeral5m } : {}), - ...(ephemeral1h !== undefined ? { ephemeral_1h_input_tokens: ephemeral1h } : {}), - } - } - - const serverToolUse = findObjectFieldValueBuffer(source, usageBounds.start, usageBounds.end, 'server_tool_use') - const webSearch = readJsonNumberFieldBuffer(source, serverToolUse, 'web_search_requests') - const webFetch = readJsonNumberFieldBuffer(source, serverToolUse, 'web_fetch_requests') - if (webSearch !== undefined || webFetch !== undefined) { - ;(usage as AssistantMessageContent['usage']).server_tool_use = { - ...(webSearch !== undefined ? { web_search_requests: webSearch } : {}), - ...(webFetch !== undefined ? { web_fetch_requests: webFetch } : {}), - } - } - - const speed = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, usageBounds.start, usageBounds.end, 'speed')) - if (speed === 'standard' || speed === 'fast') usage.speed = speed - - const advisor = extractAdvisorIterations(source.subarray(usageBounds.start, usageBounds.end).toString('utf8')) - if (advisor) usage.iterations = advisor - } - - return usage -} - -function extractLargeToolBlocksBuffer(source: Buffer, contentBounds: BufferJsonValueBounds | null): ToolUseBlock[] { - if (!contentBounds || contentBounds.kind !== 'array') return [] - const tools: ToolUseBlock[] = [] - let i = contentBounds.start + 1 - while (i < contentBounds.end - 1 && tools.length < MAX_TOOL_BLOCKS) { - while (i < contentBounds.end && isJsonWhitespaceByte(source[i])) i++ - if (source[i] === 0x2c) { - i++ - continue - } - if (source[i] !== 0x7b) { - i++ - continue - } - const objectEnd = findJsonContainerEndBuffer(source, i, 0x7b, 0x7d, contentBounds.end) - if (objectEnd === -1) break - const objectBounds = { start: i, end: objectEnd + 1, kind: 'object' as const } - const blockType = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, objectBounds.start, objectBounds.end, 'type')) - if (blockType === 'tool_use') { - const name = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, objectBounds.start, objectBounds.end, 'name')) ?? '' - const id = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, objectBounds.start, objectBounds.end, 'id')) ?? '' - const inputBounds = findObjectFieldValueBuffer(source, objectBounds.start, objectBounds.end, 'input') - const input: Record = {} - if (inputBounds?.kind === 'object') { - if (name === 'Skill') { - const skill = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, inputBounds.start, inputBounds.end, 'skill'), 200) - const skillName = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, inputBounds.start, inputBounds.end, 'name'), 200) - if (skill !== undefined) input['skill'] = skill - if (skillName !== undefined) input['name'] = skillName - } else if (name === 'Read' || name === 'FileReadTool' || EDIT_TOOLS.has(name)) { - const filePath = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, inputBounds.start, inputBounds.end, 'file_path'), BASH_COMMAND_CAP) - if (filePath !== undefined) input['file_path'] = filePath - } else if (name === 'Agent' || name === 'Task') { - const subagentType = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, inputBounds.start, inputBounds.end, 'subagent_type'), 200) - if (subagentType !== undefined) input['subagent_type'] = subagentType - } else if (BASH_TOOLS.has(name)) { - const command = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, inputBounds.start, inputBounds.end, 'command'), BASH_COMMAND_CAP) - if (command !== undefined) input['command'] = command - } - } - tools.push({ type: 'tool_use', id, name, input }) - } - i = objectEnd + 1 - } - return tools -} - -function extractLargeUserTextBuffer(source: Buffer, contentBounds: BufferJsonValueBounds | null): string | undefined { - if (!contentBounds) return undefined - if (contentBounds.kind === 'string') return readJsonStringBuffer(source, contentBounds, USER_TEXT_CAP) - if (contentBounds.kind !== 'array') return undefined - - let text = '' - let i = contentBounds.start + 1 - while (i < contentBounds.end - 1 && text.length < USER_TEXT_CAP) { - while (i < contentBounds.end && isJsonWhitespaceByte(source[i])) i++ - if (source[i] === 0x2c) { - i++ - continue - } - if (source[i] !== 0x7b) { - i++ - continue - } - const objectEnd = findJsonContainerEndBuffer(source, i, 0x7b, 0x7d, contentBounds.end) - if (objectEnd === -1) break - const objectBounds = { start: i, end: objectEnd + 1, kind: 'object' as const } - const type = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, objectBounds.start, objectBounds.end, 'type')) - if (type === 'text' || type === 'input_text') { - const part = readJsonStringBuffer( - source, - findObjectFieldValueBuffer(source, objectBounds.start, objectBounds.end, 'text'), - USER_TEXT_CAP - text.length, - ) - if (part) text += (text ? ' ' : '') + part - } - i = objectEnd + 1 - } - return text || undefined + return appendStringJsonSegment(source, segmentStart, contentEnd, out, cap) } -function extractLargeAddedNamesBuffer(source: Buffer, attachmentBounds: BufferJsonValueBounds | null): string[] { - if (!attachmentBounds || attachmentBounds.kind !== 'object') return [] - const attachmentType = readJsonStringBuffer( - source, - findObjectFieldValueBuffer(source, attachmentBounds.start, attachmentBounds.end, 'type'), - ) - if (attachmentType !== 'deferred_tools_delta') return [] - const addedNames = findObjectFieldValueBuffer(source, attachmentBounds.start, attachmentBounds.end, 'addedNames') - if (!addedNames || addedNames.kind !== 'array') return [] - const names: string[] = [] - let i = addedNames.start + 1 - while (i < addedNames.end - 1 && names.length < MAX_ADDED_NAMES) { - while (i < addedNames.end && isJsonWhitespaceByte(source[i])) i++ - if (source[i] === 0x2c) { - i++ +function readJsonStringBuffer(source: Buffer, bounds: JsonValueBounds | null, cap = Number.POSITIVE_INFINITY): string | undefined { + if (!bounds || bounds.kind !== 'string') return undefined + let out = '' + const contentEnd = bounds.end - 1 + let segmentStart = bounds.start + 1 + let i = segmentStart + let scanLimit = Number.isFinite(cap) ? Math.min(contentEnd, segmentStart + cap * 4) : contentEnd + while (i < contentEnd && out.length < cap) { + if (i >= scanLimit) { + const segmentEnd = safeBufferSegmentEnd(source, i) + out = appendBufferJsonSegment(source, segmentStart, segmentEnd, out, cap) + if (out.length >= cap) break + segmentStart = segmentEnd + i = segmentEnd + scanLimit = Number.isFinite(cap) ? Math.min(contentEnd, i + (cap - out.length) * 4) : contentEnd continue } - if (source[i] !== 0x22) { + const ch = source[i] + if (ch !== 0x5c) { i++ continue } - const end = findJsonStringEndBuffer(source, i, addedNames.end) - if (end === -1) break - const name = readJsonStringBuffer(source, { start: i, end: end + 1, kind: 'string' }, 500) - if (name) names.push(name) - i = end + 1 - } - return names -} - -function parseLargeJsonlBuffer(line: Buffer): JournalEntry | null { - let rootStart = 0 - while (rootStart < line.length && isJsonWhitespaceByte(line[rootStart])) rootStart++ - if (line[rootStart] !== 0x7b) return null - const rootEnd = findJsonContainerEndBuffer(line, rootStart, 0x7b, 0x7d) - if (rootEnd === -1) return null - const rootLimit = rootEnd + 1 - const type = readJsonStringBuffer(line, findObjectFieldValueBuffer(line, rootStart, rootLimit, 'type')) - if (!type) return null - - const entry: JournalEntry = { type } - const timestamp = readJsonStringBuffer(line, findObjectFieldValueBuffer(line, rootStart, rootLimit, 'timestamp')) - const sessionId = readJsonStringBuffer(line, findObjectFieldValueBuffer(line, rootStart, rootLimit, 'sessionId')) - const cwd = readJsonStringBuffer(line, findObjectFieldValueBuffer(line, rootStart, rootLimit, 'cwd')) - if (timestamp !== undefined) entry.timestamp = timestamp - if (sessionId !== undefined) entry.sessionId = sessionId - if (cwd !== undefined) entry.cwd = cwd - const addedNames = extractLargeAddedNamesBuffer(line, findObjectFieldValueBuffer(line, rootStart, rootLimit, 'attachment')) - if (addedNames.length > 0) { - ;(entry as Record)['attachment'] = { type: 'deferred_tools_delta', addedNames } - } - - if (type === 'user') { - const message = findObjectFieldValueBuffer(line, rootStart, rootLimit, 'message') - if (message?.kind === 'object') { - const content = findObjectFieldValueBuffer(line, message.start, message.end, 'content') - const text = extractLargeUserTextBuffer(line, content) - if (text !== undefined) entry.message = { role: 'user', content: text } + out = appendBufferJsonSegment(source, segmentStart, i, out, cap) + if (out.length >= cap) break + i++ + const next = source[i] + if (next === undefined) break + if (next === 0x6e) out += '\n' + else if (next === 0x72) out += '\r' + else if (next === 0x74) out += '\t' + else if (next === 0x62) out += '\b' + else if (next === 0x66) out += '\f' + else if (next === 0x75 && i + 4 < bounds.end) { + const code = Number.parseInt(source.subarray(i + 1, i + 5).toString('ascii'), 16) + if (Number.isFinite(code)) out += String.fromCharCode(code) + i += 4 + } else { + out += String.fromCharCode(next) } - return entry - } - - if (type !== 'assistant') return entry - const message = findObjectFieldValueBuffer(line, rootStart, rootLimit, 'message') - if (message?.kind !== 'object') return entry - const model = readJsonStringBuffer(line, findObjectFieldValueBuffer(line, message.start, message.end, 'model')) - const usageBounds = findObjectFieldValueBuffer(line, message.start, message.end, 'usage') - if (!model || usageBounds?.kind !== 'object') return entry - const id = readJsonStringBuffer(line, findObjectFieldValueBuffer(line, message.start, message.end, 'id')) - const contentBounds = findObjectFieldValueBuffer(line, message.start, message.end, 'content') - - entry.message = { - type: 'message', - role: 'assistant', - model, - ...(id !== undefined ? { id } : {}), - content: extractLargeToolBlocksBuffer(line, contentBounds), - usage: parseLargeUsageBuffer(line, usageBounds), + segmentStart = i + 1 + i++ } - - return entry + return appendBufferJsonSegment(source, segmentStart, contentEnd, out, cap) } function getTopLevelRawJsonStringField(head: string, field: string): string | null { @@ -862,15 +786,15 @@ function getTopLevelRawJsonStringField(head: string, field: string): string | nu } if (head.charCodeAt(i) === 0x7d) return null if (head.charCodeAt(i) !== 0x22) return null - const keyEnd = findJsonStringEnd(head, i) + const keyEnd = findJsonStringEndString(head, i) if (keyEnd === -1) return null const key = head.slice(i + 1, keyEnd) i = keyEnd + 1 while (i < head.length && /\s/.test(head[i]!)) i++ if (head.charCodeAt(i) !== 0x3a) return null - const value = findJsonValueBounds(head, i + 1) + const value = findJsonValueBoundsString(head, i + 1) if (!value) return null - if (key === field) return readJsonString(head, value) ?? null + if (key === field) return readJsonStringString(head, value) ?? null i = value.end } return null diff --git a/tests/parser-large-json-scanner-parity.test.ts b/tests/parser-large-json-scanner-parity.test.ts new file mode 100644 index 00000000..b7c4d272 --- /dev/null +++ b/tests/parser-large-json-scanner-parity.test.ts @@ -0,0 +1,196 @@ +import { describe, expect, it } from 'vitest' + +import { parseJsonlLine, shouldSkipLine } from '../src/parser.js' + +const LARGE_PADDING = 'x'.repeat(40_000) + +function withUnicodeEscapes(line: string): string { + return line.replaceAll('☃', '\\u2603') +} + +function jsonStringContent(value: string): string { + return JSON.stringify(value).slice(1, -1) +} + +function largeUserLine(rawJsonContent: string, prefix = ''): string { + return `${prefix}{"type":"user","sessionId":"parity-edge","timestamp":"2026-05-01T00:00:00Z","message":{"role":"user","content":"${rawJsonContent}"},"padding":"${LARGE_PADDING}"}` +} + +function largeUserArrayLine(rawJsonText: string): string { + return `{"type":"user","sessionId":"parity-edge","message":{"role":"user","content":[{"type":"text","text":"${rawJsonText}"}]},"padding":"${LARGE_PADDING}"}` +} + +function expectStringBufferParity(line: string): void { + expect(parseJsonlLine(Buffer.from(line))).toEqual(parseJsonlLine(line)) +} + +function userLine(): { line: string; expected: ReturnType } { + const text = `escaped "quotes" and \\slashes\\; unicode ☃; nested {"brace":[1,2]} ${LARGE_PADDING}` + const line = withUnicodeEscapes(JSON.stringify({ + type: 'user', + sessionId: 'parity-user', + timestamp: '2026-05-01T00:00:00Z', + cwd: '/repo', + message: { role: 'user', content: [{ type: 'text', text }] }, + })) + return { + line, + expected: { + type: 'user', + sessionId: 'parity-user', + timestamp: '2026-05-01T00:00:00Z', + cwd: '/repo', + message: { role: 'user', content: text.slice(0, 2000) }, + }, + } +} + +function assistantLine(): { line: string; expected: ReturnType } { + const command = `printf '{"brace":[1,2]}' \\tmp\\file ☃ ${LARGE_PADDING}` + const line = withUnicodeEscapes(JSON.stringify({ + type: 'assistant', + sessionId: 'parity-assistant', + timestamp: '2026-05-01T00:00:01Z', + cwd: '/repo', + message: { + id: 'message-☃', + type: 'message', + role: 'assistant', + model: 'claude-sonnet-4-5', + content: [ + { type: 'tool_use', id: 'tool-☃', name: 'Bash', input: { command, ignored: { nested: ['value'] } } }, + ], + usage: { input_tokens: 100, output_tokens: 20, cache_read_input_tokens: 300 }, + }, + })) + return { + line, + expected: { + type: 'assistant', + sessionId: 'parity-assistant', + timestamp: '2026-05-01T00:00:01Z', + cwd: '/repo', + message: { + type: 'message', + role: 'assistant', + model: 'claude-sonnet-4-5', + id: 'message-☃', + content: [{ type: 'tool_use', id: 'tool-☃', name: 'Bash', input: { command: command.slice(0, 2000) } }], + usage: { input_tokens: 100, output_tokens: 20, cache_read_input_tokens: 300 }, + }, + }, + } +} + +function attachmentLine(): { line: string; expected: ReturnType } { + const line = withUnicodeEscapes(JSON.stringify({ + type: 'attachment', + sessionId: 'parity-attachment', + timestamp: '2026-05-01T00:00:02Z', + cwd: '/repo', + padding: LARGE_PADDING, + attachment: { type: 'deferred_tools_delta', addedNames: ['mcp__svc__tool', 'tool-☃\\path'] }, + })) + return { + line, + expected: { + type: 'attachment', + sessionId: 'parity-attachment', + timestamp: '2026-05-01T00:00:02Z', + cwd: '/repo', + attachment: { type: 'deferred_tools_delta', addedNames: ['mcp__svc__tool', 'tool-☃\\path'] }, + }, + } +} + +describe('large JSONL scanner string/Buffer parity', () => { + it('keeps nasty strings, nested delimiters, unicode escapes, and truncation identical', () => { + const completeCases = [userLine(), assistantLine(), attachmentLine()] + + for (const { line, expected } of completeCases) { + const stringResult = parseJsonlLine(line) + const bufferResult = parseJsonlLine(Buffer.from(line)) + expect(stringResult).toEqual(expected) + expect(bufferResult).toEqual(expected) + expect(bufferResult).toEqual(stringResult) + } + + const truncatedLine = userLine().line.slice(0, -1) + expect(parseJsonlLine(truncatedLine)).toBeNull() + expect(parseJsonlLine(Buffer.from(truncatedLine))).toBeNull() + }) + + it('accepts leading JSON whitespace through both entry paths', () => { + const line = ` ${userLine().line}` + expect(parseJsonlLine(line)).toEqual(parseJsonlLine(userLine().line)) + expect(parseJsonlLine(Buffer.from(line))).toEqual(parseJsonlLine(line)) + }) + + it('preserves escaped Unicode followed by multibyte text at the output cap', () => { + const expectedContent = '☃'.repeat(700) + '亜'.repeat(1300) + const rawJsonContent = '\\u2603'.repeat(700) + '亜'.repeat(5000) + const line = largeUserLine(rawJsonContent) + const arrayLine = largeUserArrayLine(rawJsonContent) + const expected = { + type: 'user', + sessionId: 'parity-edge', + timestamp: '2026-05-01T00:00:00Z', + message: { role: 'user', content: expectedContent }, + } + + expect(parseJsonlLine(line)).toEqual(expected) + expect(parseJsonlLine(Buffer.from(line))).toEqual(expected) + const expectedArray = { + type: 'user', + sessionId: 'parity-edge', + message: { role: 'user', content: expectedContent }, + } + expect(parseJsonlLine(arrayLine)).toEqual(expectedArray) + expect(parseJsonlLine(Buffer.from(arrayLine))).toEqual(parseJsonlLine(arrayLine)) + }) + + it('keeps raw emoji cap truncation identical across UTF-16 and UTF-8', () => { + const rawText = '😀'.repeat(3000) + const line = largeUserLine(jsonStringContent(rawText)) + const expectedContent = rawText.slice(0, 2000) + const expected = { + type: 'user', + sessionId: 'parity-edge', + timestamp: '2026-05-01T00:00:00Z', + message: { role: 'user', content: expectedContent }, + } + + expect(parseJsonlLine(line)).toEqual(expected) + expect(parseJsonlLine(Buffer.from(line))).toEqual(expected) + }) + + it('keeps parity when a large line truncates mid-string or mid-escape', () => { + const midString = largeUserLine('y'.repeat(40_000)).slice(0, -LARGE_PADDING.length - 4) + const midEscape = largeUserLine(`${'y'.repeat(2_000)}\\u26`) + + expect(parseJsonlLine(midString)).toBeNull() + expect(parseJsonlLine(Buffer.from(midString))).toBeNull() + expectStringBufferParity(midEscape) + }) + + it('keeps parity when a Buffer contains a truncated UTF-8 sequence', () => { + const line = largeUserLine(jsonStringContent('亜'.repeat(12_000))) + const encoded = Buffer.from(line) + const characterStart = encoded.indexOf(Buffer.from('亜')) + const truncated = Buffer.concat([encoded.subarray(0, characterStart + 1), encoded.subarray(characterStart + 3)]) + + expect(parseJsonlLine(truncated)).toEqual(parseJsonlLine(truncated.toString('utf-8'))) + }) + + it('keeps the string skip filter equivalent to JSON parsing', () => { + const before = ` {"type":"user","timestamp":"2026-04-30T23:59:59Z","padding":"${LARGE_PADDING}"}` + const after = ` {"type":"user","timestamp":"2026-05-01T00:00:01Z","padding":"${LARGE_PADDING}"}` + const unrelated = ` {"type":"attachment","timestamp":"2026-04-30T00:00:00Z","padding":"${LARGE_PADDING}"}` + const threshold = '2026-05-01T00:00:00Z' + + for (const [line, expected] of [[before, true], [after, false], [unrelated, false]] as const) { + expect(parseJsonlLine(Buffer.from(line))).toEqual(parseJsonlLine(line)) + expect(shouldSkipLine(line, threshold)).toBe(expected) + } + }) +})