Skip to content

[Checkpoints V2] Tag "thinking" lines for compact transcript #973

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
computermode wants to merge 4 commits into
base: main
Choose a base branch
from
+126 −40
Draft

[Checkpoints V2] Tag "thinking" lines for compact transcript #973

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8253516
Preserve claude's 'thinking' output
computermode Apr 16, 2026
46c8809
Add gemini thinking lines
computermode Apr 16, 2026
1e86992
Merge branch 'main' of https://github.com/entireio/cli into add-think…
computermode Apr 22, 2026
7c15490
Fix linter issues
computermode Apr 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 20 additions & 7 deletions cmd/entire/cli/transcript/compact/compact.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,10 @@ func compactJSONL(content []byte, opts MetadataFields) ([]byte, error) {

func compactJSONLWith(content []byte, opts MetadataFields, preprocess linePreprocessor) ([]byte, error) {
base := newTranscriptLine(opts)
preserveThinking := opts.Agent == "claude-code"

// Pass 1: parse all lines into intermediate entries.
entries, err := parseJSONLEntries(content, preprocess)
entries, err := parseJSONLEntries(content, preprocess, preserveThinking)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -283,7 +284,7 @@ func appendLine(result *[]byte, line transcriptLine) {

// parseJSONLEntries parses all JSONL lines into intermediate entries,
// filtering dropped types and malformed lines.
func parseJSONLEntries(content []byte, preprocess linePreprocessor) ([]parsedEntry, error) {
func parseJSONLEntries(content []byte, preprocess linePreprocessor, preserveThinking bool) ([]parsedEntry, error) {
reader := bufio.NewReader(bytes.NewReader(content))
var entries []parsedEntry

Expand All @@ -294,7 +295,7 @@ func parseJSONLEntries(content []byte, preprocess linePreprocessor) ([]parsedEnt
}

if len(bytes.TrimSpace(lineBytes)) > 0 {
if e, ok := parseLine(lineBytes, preprocess); ok {
if e, ok := parseLine(lineBytes, preprocess, preserveThinking); ok {
entries = append(entries, e)
}
}
Expand All @@ -309,7 +310,7 @@ func parseJSONLEntries(content []byte, preprocess linePreprocessor) ([]parsedEnt

// parseLine converts a single JSONL line into a parsedEntry.
// Returns ok=false for dropped/malformed lines.
func parseLine(lineBytes []byte, preprocess linePreprocessor) (parsedEntry, bool) {
func parseLine(lineBytes []byte, preprocess linePreprocessor, preserveThinking bool) (parsedEntry, bool) {
var raw map[string]json.RawMessage
if err := json.Unmarshal(lineBytes, &raw); err != nil {
return parsedEntry{}, false
Expand All @@ -336,7 +337,7 @@ func parseLine(lineBytes []byte, preprocess linePreprocessor) (parsedEntry, bool
if msg != nil {
e.id = unquote(msg["id"])
if contentRaw, ok := msg["content"]; ok {
e.content = stripAssistantContent(contentRaw)
e.content = stripAssistantContent(contentRaw, preserveThinking)
}
e.inputTokens, e.outputTokens = extractUsageTokens(msg)
}
Expand Down Expand Up @@ -620,7 +621,7 @@ func extractUserContent(contentRaw json.RawMessage) userContent {
return uc
}

func stripAssistantContent(contentRaw json.RawMessage) json.RawMessage {
func stripAssistantContent(contentRaw json.RawMessage, preserveThinking bool) json.RawMessage {
var str string
if json.Unmarshal(contentRaw, &str) == nil {
return contentRaw
Expand All @@ -635,7 +636,19 @@ func stripAssistantContent(contentRaw json.RawMessage) json.RawMessage {
for _, block := range blocks {
blockType := unquote(block["type"])

if blockType == "thinking" || blockType == "redacted_thinking" {
if blockType == "redacted_thinking" {
continue
}

if preserveThinking && blockType == "thinking" {
stripped := make(map[string]json.RawMessage)
copyField(stripped, block, "type")
copyField(stripped, block, "thinking")
Comment thread
computermode marked this conversation as resolved.
result = append(result, stripped)
continue
}

if blockType == "thinking" {
continue
}

Expand Down
22 changes: 12 additions & 10 deletions cmd/entire/cli/transcript/compact/compact_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ func TestCompact_AssistantStripping(t *testing.T) {
`)

expected := []string{
`{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"assistant","ts":"2026-01-01T00:00:01Z","id":"msg-1","content":[{"type":"text","text":"Here's my answer."},{"type":"tool_use","id":"tu-1","name":"Bash","input":{"command":"ls"}},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"abc"}}]}`,
`{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"assistant","ts":"2026-01-01T00:00:01Z","id":"msg-1","content":[{"thinking":"hmm...","type":"thinking"},{"type":"text","text":"Here's my answer."},{"type":"tool_use","id":"tu-1","name":"Bash","input":{"command":"ls"}},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"abc"}}]}`,
}

result, err := Compact(redact.AlreadyRedacted(input), defaultOpts)
Expand All @@ -65,16 +65,18 @@ func TestCompact_AssistantStripping(t *testing.T) {
func TestCompact_AssistantThinkingOnly(t *testing.T) {
t.Parallel()

// Assistant lines with only thinking content should be dropped entirely
// (streaming intermediates that carry no user-visible content).
input := []byte(`{"type":"assistant","timestamp":"2026-01-01T00:00:01Z","requestId":"req-1","message":{"id":"msg-1","content":[{"type":"thinking","thinking":"hmm..."}]}}
`)

expected := []string{
`{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"assistant","ts":"2026-01-01T00:00:01Z","id":"msg-1","content":[{"thinking":"hmm...","type":"thinking"}]}`,
}

result, err := Compact(redact.AlreadyRedacted(input), defaultOpts)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
assertJSONLines(t, result, nil)
assertJSONLines(t, result, expected)
}

func TestCompact_UserWithToolResult(t *testing.T) {
Expand Down Expand Up @@ -217,7 +219,7 @@ func TestCompact_StreamingFragmentTokenMerge(t *testing.T) {
`)

expected := []string{
`{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"assistant","ts":"t2","id":"m1","input_tokens":100,"output_tokens":42,"content":[{"type":"text","text":"done"}]}`,
`{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"assistant","ts":"t2","id":"m1","input_tokens":100,"output_tokens":42,"content":[{"thinking":"hmm","type":"thinking"},{"type":"text","text":"done"}]}`,
}

result, err := Compact(redact.AlreadyRedacted(input), defaultOpts)
Expand Down Expand Up @@ -366,7 +368,7 @@ func TestCompact_FullFixture_WithTruncation(t *testing.T) {
// to inline into, so user text is emitted and tool result is lost.
expected := []string{
`{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"user","ts":"2026-01-01T00:01:00Z","content":[{"text":"now fix the bug"}]}`,
`{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"assistant","ts":"2026-01-01T00:01:01Z","id":"msg-2","content":[{"type":"text","text":"I found the issue."},{"type":"tool_use","id":"tu-2","name":"Edit","input":{"file_path":"/repo/bug.go","old_string":"bad","new_string":"good"}}]}`,
`{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"assistant","ts":"2026-01-01T00:01:01Z","id":"msg-2","content":[{"thinking":"analyzing the bug...","type":"thinking"},{"type":"text","text":"I found the issue."},{"type":"tool_use","id":"tu-2","name":"Edit","input":{"file_path":"/repo/bug.go","old_string":"bad","new_string":"good"}}]}`,
}

result, err := Compact(redact.AlreadyRedacted([]byte(fixtureFullJSONL)), opts)
Expand All @@ -382,13 +384,13 @@ func TestCompact_FullFixture_NoTruncation(t *testing.T) {
expected := []string{
// Line 0: user "hello"
`{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"user","ts":"2026-01-01T00:00:00Z","content":[{"text":"hello"}]}`,
// Line 1: assistant (thinking stripped, caller stripped, tool result inlined from line 3 with file metadata)
`{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"assistant","ts":"2026-01-01T00:00:01Z","id":"msg-1","content":[{"type":"text","text":"Hi there!"},{"type":"tool_use","id":"tu-1","name":"Bash","input":{"command":"ls"},"result":{"output":"file1.txt\nfile2.txt","status":"success","file":{"filePath":"/repo/file1.txt","numLines":10}}}]}`,
// Line 1: assistant (thinking preserved, caller stripped, tool result inlined from line 3 with file metadata)
`{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"assistant","ts":"2026-01-01T00:00:01Z","id":"msg-1","content":[{"thinking":"let me think...","type":"thinking"},{"type":"text","text":"Hi there!"},{"type":"tool_use","id":"tu-1","name":"Bash","input":{"command":"ls"},"result":{"output":"file1.txt\nfile2.txt","status":"success","file":{"filePath":"/repo/file1.txt","numLines":10}}}]}`,
// Line 2: progress — dropped
// Line 3: user with tool_result — inlined above, user text emitted
`{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"user","ts":"2026-01-01T00:01:00Z","content":[{"text":"now fix the bug"}]}`,
// Line 4: assistant (thinking + redacted_thinking stripped)
`{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"assistant","ts":"2026-01-01T00:01:01Z","id":"msg-2","content":[{"type":"text","text":"I found the issue."},{"type":"tool_use","id":"tu-2","name":"Edit","input":{"file_path":"/repo/bug.go","old_string":"bad","new_string":"good"}}]}`,
// Line 4: assistant (thinking preserved, redacted_thinking stripped)
`{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"assistant","ts":"2026-01-01T00:01:01Z","id":"msg-2","content":[{"thinking":"analyzing the bug...","type":"thinking"},{"type":"text","text":"I found the issue."},{"type":"tool_use","id":"tu-2","name":"Edit","input":{"file_path":"/repo/bug.go","old_string":"bad","new_string":"good"}}]}`,
// Lines 5-6: file-history-snapshot, system — dropped
}

Expand Down
45 changes: 44 additions & 1 deletion cmd/entire/cli/transcript/compact/gemini.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,16 @@ type geminiMessage struct {
Timestamp string `json:"timestamp"`
Type string `json:"type"`
Content json.RawMessage `json:"content"`
Thoughts []geminiThought `json:"thoughts"`
ToolCalls []geminiToolCall `json:"toolCalls"`
Tokens *geminiTokens `json:"tokens"`
}

type geminiThought struct {
Subject string `json:"subject"`
Description string `json:"description"`
}

// geminiTokens holds token usage from a Gemini message.
type geminiTokens struct {
Input int `json:"input"`
Expand Down Expand Up @@ -152,7 +158,28 @@ func emitGeminiUser(result *[]byte, base transcriptLine, msg geminiMessage, ts j
// emitGeminiAssistant produces a single assistant line. The content array
// contains text blocks (from content field) and tool_use blocks (from toolCalls).
func emitGeminiAssistant(result *[]byte, base transcriptLine, msg geminiMessage, ts json.RawMessage) {
content := make([]map[string]json.RawMessage, 0, 1+len(msg.ToolCalls))
content := make([]map[string]json.RawMessage, 0, len(msg.Thoughts)+1+len(msg.ToolCalls))

for _, thought := range msg.Thoughts {
thinkingText := geminiThoughtText(thought)
if thinkingText == "" {
continue
}

blockType, err := json.Marshal("thinking")
if err != nil {
continue
}
thinking, err := json.Marshal(thinkingText)
if err != nil {
continue
}

content = append(content, map[string]json.RawMessage{
"type": blockType,
"thinking": thinking,
})
}
Comment thread
computermode marked this conversation as resolved.

if contentText := geminiContentText(msg.Content); contentText != "" {
b, err := json.Marshal(transcript.ContentTypeText)
Expand Down Expand Up @@ -214,6 +241,22 @@ func emitGeminiAssistant(result *[]byte, base transcriptLine, msg geminiMessage,
appendLine(result, line)
}

func geminiThoughtText(thought geminiThought) string {
subject := strings.TrimSpace(thought.Subject)
description := strings.TrimSpace(thought.Description)

switch {
case subject != "" && description != "":
return subject + "\n\n" + description
case description != "":
return description
case subject != "":
return subject
default:
return ""
}
}

// geminiContentText extracts the text from a Gemini content field which may
// be either a plain JSON string or an array of content parts (each with a
// "text" field). Returns the concatenated text or "" if content is absent.
Expand Down
28 changes: 28 additions & 0 deletions cmd/entire/cli/transcript/compact/gemini_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,34 @@ func TestCompact_GeminiStartLine(t *testing.T) {
})
}

func TestCompact_GeminiThoughtOnlyAssistant(t *testing.T) {
t.Parallel()

input := []byte(`{
"sessionId": "s1",
"messages": [
{
"id":"m1",
"timestamp":"2026-01-01T00:00:01Z",
"type":"gemini",
"content":"",
"thoughts":[{"subject":"Planning","description":"I should inspect the project first."}],
"tokens":{"input":10,"output":5}
}
]
}`)

expected := []string{
`{"v":1,"agent":"gemini-cli","cli_version":"0.5.1","type":"assistant","ts":"2026-01-01T00:00:01Z","id":"m1","input_tokens":10,"output_tokens":5,"content":[{"thinking":"Planning\n\nI should inspect the project first.","type":"thinking"}]}`,
}

result, err := Compact(redact.AlreadyRedacted(input), agentOpts("gemini-cli"))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
assertJSONLines(t, result, expected)
}

func TestIsGeminiFormat(t *testing.T) {
t.Parallel()

Expand Down
2 changes: 1 addition & 1 deletion cmd/entire/cli/transcript/compact/testdata/claude_expected.jsonl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"user","ts":"2026-03-18T00:03:14.992Z","content":[{"id":"1a789a48-ac83-46b6-a746-90ff5bb455ad","text":"I need to revert a change to my last commit message"}]}
{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"assistant","ts":"2026-03-18T00:03:19.789Z","id":"msg_01A8w8vJNhp54SKt6N4Pdg6W","input_tokens":3,"output_tokens":102,"content":[{"text":"What would you like the new commit message to be? The last commit on this branch is:\n\n```\n42e6c78 test\n```\n\nTo change it, I can run `git commit --amend -m \"new message\"`. What message would you like?","type":"text"}]}
{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"assistant","ts":"2026-03-18T00:03:19.789Z","id":"msg_01A8w8vJNhp54SKt6N4Pdg6W","input_tokens":3,"output_tokens":102,"content":[{"thinking":"","type":"thinking"},{"text":"What would you like the new commit message to be? The last commit on this branch is:\n\n```\n42e6c78 test\n```\n\nTo change it, I can run `git commit --amend -m \"new message\"`. What message would you like?","type":"text"}]}
{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"user","ts":"2026-03-18T00:05:10.762Z","content":[{"id":"156e4210-ce44-45e0-a2bf-76416bd3f82b","text":"No I want to revert the extra amendment to the commit message where the new entire-checkpoint text was added"}]}
{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"assistant","ts":"2026-03-18T00:05:14.398Z","id":"msg_01XBRjv2YCFQs34eaLK7w9XF","input_tokens":3,"output_tokens":113,"content":[{"text":"\n\nLet me check the current full commit message first.","type":"text"},{"id":"toolu_01NFoPKsjrNsBZ4PJ4r8uSF2","input":{"command":"git log -1 --format=\"%B\"","description":"Show full last commit message"},"name":"Bash","result":{"output":"test\n\nEntire-Checkpoint: ecf782729563\nEntire-Checkpoint: c81467b72ca0","status":"success"},"type":"tool_use"}]}
{"v":1,"agent":"claude-code","cli_version":"0.5.1","type":"assistant","ts":"2026-03-18T00:05:18.719Z","id":"msg_01YDHWjQkYdHk5EjK4amSHy7","input_tokens":1,"output_tokens":39,"content":[{"text":"I see — the commit message has `Entire-Checkpoint` lines appended. What should the message be after reverting? Just `test` with no checkpoint lines?","type":"text"}]}
Expand Down
34 changes: 17 additions & 17 deletions cmd/entire/cli/transcript/compact/testdata/claude_expected2.jsonl
View file
Open in desktop

Large diffs are not rendered by default.

Loading
Loading