diff --git a/.changeset/mastra-thread-view-llm-io.md b/.changeset/mastra-thread-view-llm-io.md new file mode 100644 index 000000000..95d3468e3 --- /dev/null +++ b/.changeset/mastra-thread-view-llm-io.md @@ -0,0 +1,17 @@ +--- +"braintrust": patch +--- + +fix(mastra): Transform LLM input/output for the Braintrust Thread view + +The model-generation spans from the Mastra observability exporter +(`model_generation`, `model_step`, `model_chunk`) now log input as a bare +chat-message array (unwrapping Mastra's `{ messages: [...] }` container, or +wrapping a single `{ role, content }` object) and output as an +`{ role: 'assistant', content }` message (unwrapping Mastra's `{ text, ... }`), +so Braintrust's Thread view renders the conversation correctly. All three map to +the `llm` span type and are surfaced together in Thread view, so they share the +transform; the app's dedup coalesces them into one turn. Embedding (`rag_embedding`) +and non-model spans are unchanged. + +Ports mastra-ai/mastra#10794 (fixes #9848). diff --git a/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.log-payloads.json b/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.log-payloads.json index 75d3077e7..27c5f9dc0 100644 --- a/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.log-payloads.json +++ b/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.log-payloads.json @@ -31,28 +31,26 @@ "type": "task" }, { - "input": { - "messages": [ - { - "content": "Answer weather questions with the provided mock forecast.", - "role": "system" - }, - { - "content": [ - { - "providerOptions": { - "mastra": { - "createdAt": 0 - } - }, - "text": "What is the weather in Paris?", - "type": "text" - } - ], - "role": "user" - } - ] - }, + "input": [ + { + "content": "Answer weather questions with the provided mock forecast.", + "role": "system" + }, + { + "content": [ + { + "providerOptions": { + "mastra": { + "createdAt": 0 + } + }, + "text": "What is the weather in Paris?", + "type": "text" + } + ], + "role": "user" + } + ], "metadata": { "entity_id": "weather-agent", "entity_name": "Weather Agent", @@ -67,10 +65,11 @@ ], "name": "llm: 'mock-model-id'", "output": { + "content": "The forecast is sunny.", "files": [], "reasoning": [], + "role": "assistant", "sources": [], - "text": "The forecast is sunny.", "warnings": [] }, "type": "llm" @@ -89,7 +88,8 @@ ], "name": "step: 0", "output": { - "text": "The forecast is sunny.", + "content": "The forecast is sunny.", + "role": "assistant", "toolCalls": [] }, "type": "llm" @@ -103,7 +103,8 @@ "metric_keys": [], "name": "chunk: 'text'", "output": { - "text": "The forecast is sunny." + "content": "The forecast is sunny.", + "role": "assistant" }, "type": "llm" }, @@ -131,28 +132,26 @@ "type": "task" }, { - "input": { - "messages": [ - { - "content": "Answer weather questions with the provided mock forecast.", - "role": "system" - }, - { - "content": [ - { - "providerOptions": { - "mastra": { - "createdAt": 0 - } - }, - "text": "Stream the Paris forecast.", - "type": "text" - } - ], - "role": "user" - } - ] - }, + "input": [ + { + "content": "Answer weather questions with the provided mock forecast.", + "role": "system" + }, + { + "content": [ + { + "providerOptions": { + "mastra": { + "createdAt": 0 + } + }, + "text": "Stream the Paris forecast.", + "type": "text" + } + ], + "role": "user" + } + ], "metadata": { "entity_id": "weather-agent", "entity_name": "Weather Agent", @@ -167,10 +166,11 @@ ], "name": "llm: 'mock-model-id'", "output": { + "content": "The forecast is sunny.", "files": [], "reasoning": [], + "role": "assistant", "sources": [], - "text": "The forecast is sunny.", "warnings": [] }, "type": "llm" @@ -189,7 +189,8 @@ ], "name": "step: 0", "output": { - "text": "The forecast is sunny.", + "content": "The forecast is sunny.", + "role": "assistant", "toolCalls": [] }, "type": "llm" @@ -203,7 +204,8 @@ "metric_keys": [], "name": "chunk: 'text'", "output": { - "text": "The forecast is sunny." + "content": "The forecast is sunny.", + "role": "assistant" }, "type": "llm" }, diff --git a/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.span-tree.json b/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.span-tree.json index e7ee35483..79bd1efde 100644 --- a/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.span-tree.json +++ b/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.span-tree.json @@ -24,7 +24,8 @@ "type": "llm", "children": [], "output": { - "text": "The forecast is sunny." + "content": "The forecast is sunny.", + "role": "assistant" }, "metadata": { "entity_id": "weather-agent", @@ -34,7 +35,8 @@ } ], "output": { - "text": "The forecast is sunny.", + "content": "The forecast is sunny.", + "role": "assistant", "toolCalls": [] }, "metadata": { @@ -49,33 +51,32 @@ ] } ], - "input": { - "messages": [ - { - "content": "Answer weather questions with the provided mock forecast.", - "role": "system" - }, - { - "content": [ - { - "providerOptions": { - "mastra": { - "createdAt": 0 - } - }, - "text": "What is the weather in Paris?", - "type": "text" - } - ], - "role": "user" - } - ] - }, + "input": [ + { + "content": "Answer weather questions with the provided mock forecast.", + "role": "system" + }, + { + "content": [ + { + "providerOptions": { + "mastra": { + "createdAt": 0 + } + }, + "text": "What is the weather in Paris?", + "type": "text" + } + ], + "role": "user" + } + ], "output": { + "content": "The forecast is sunny.", "files": [], "reasoning": [], + "role": "assistant", "sources": [], - "text": "The forecast is sunny.", "warnings": [] }, "metadata": { @@ -120,7 +121,8 @@ "type": "llm", "children": [], "output": { - "text": "The forecast is sunny." + "content": "The forecast is sunny.", + "role": "assistant" }, "metadata": { "entity_id": "weather-agent", @@ -130,7 +132,8 @@ } ], "output": { - "text": "The forecast is sunny.", + "content": "The forecast is sunny.", + "role": "assistant", "toolCalls": [] }, "metadata": { @@ -145,33 +148,32 @@ ] } ], - "input": { - "messages": [ - { - "content": "Answer weather questions with the provided mock forecast.", - "role": "system" - }, - { - "content": [ - { - "providerOptions": { - "mastra": { - "createdAt": 0 - } - }, - "text": "Stream the Paris forecast.", - "type": "text" - } - ], - "role": "user" - } - ] - }, + "input": [ + { + "content": "Answer weather questions with the provided mock forecast.", + "role": "system" + }, + { + "content": [ + { + "providerOptions": { + "mastra": { + "createdAt": 0 + } + }, + "text": "Stream the Paris forecast.", + "type": "text" + } + ], + "role": "user" + } + ], "output": { + "content": "The forecast is sunny.", "files": [], "reasoning": [], + "role": "assistant", "sources": [], - "text": "The forecast is sunny.", "warnings": [] }, "metadata": { diff --git a/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.span-tree.txt b/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.span-tree.txt index e94ee3248..950086b95 100644 --- a/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.span-tree.txt +++ b/e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.span-tree.txt @@ -19,33 +19,32 @@ span_tree: │ │ "entity_type": "agent" │ │ } │ │ └── llm: 'mock-model-id' [llm] - │ │ input: { - │ │ "messages": [ - │ │ { - │ │ "content": "Answer weather questions with the provided mock forecast.", - │ │ "role": "system" - │ │ }, - │ │ { - │ │ "content": [ - │ │ { - │ │ "providerOptions": { - │ │ "mastra": { - │ │ "createdAt": 0 - │ │ } - │ │ }, - │ │ "text": "What is the weather in Paris?", - │ │ "type": "text" - │ │ } - │ │ ], - │ │ "role": "user" - │ │ } - │ │ ] - │ │ } + │ │ input: [ + │ │ { + │ │ "content": "Answer weather questions with the provided mock forecast.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "mastra": { + │ │ "createdAt": 0 + │ │ } + │ │ }, + │ │ "text": "What is the weather in Paris?", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ] │ │ output: { + │ │ "content": "The forecast is sunny.", │ │ "files": [], │ │ "reasoning": [], + │ │ "role": "assistant", │ │ "sources": [], - │ │ "text": "The forecast is sunny.", │ │ "warnings": [] │ │ } │ │ metadata: { @@ -62,7 +61,8 @@ span_tree: │ │ ] │ │ └── step: 0 [llm] │ │ output: { - │ │ "text": "The forecast is sunny.", + │ │ "content": "The forecast is sunny.", + │ │ "role": "assistant", │ │ "toolCalls": [] │ │ } │ │ metadata: { @@ -77,7 +77,8 @@ span_tree: │ │ ] │ │ └── chunk: 'text' [llm] │ │ output: { - │ │ "text": "The forecast is sunny." + │ │ "content": "The forecast is sunny.", + │ │ "role": "assistant" │ │ } │ │ metadata: { │ │ "entity_id": "weather-agent", @@ -96,33 +97,32 @@ span_tree: │ │ "entity_type": "agent" │ │ } │ │ └── llm: 'mock-model-id' [llm] - │ │ input: { - │ │ "messages": [ - │ │ { - │ │ "content": "Answer weather questions with the provided mock forecast.", - │ │ "role": "system" - │ │ }, - │ │ { - │ │ "content": [ - │ │ { - │ │ "providerOptions": { - │ │ "mastra": { - │ │ "createdAt": 0 - │ │ } - │ │ }, - │ │ "text": "Stream the Paris forecast.", - │ │ "type": "text" - │ │ } - │ │ ], - │ │ "role": "user" - │ │ } - │ │ ] - │ │ } + │ │ input: [ + │ │ { + │ │ "content": "Answer weather questions with the provided mock forecast.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "mastra": { + │ │ "createdAt": 0 + │ │ } + │ │ }, + │ │ "text": "Stream the Paris forecast.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ] │ │ output: { + │ │ "content": "The forecast is sunny.", │ │ "files": [], │ │ "reasoning": [], + │ │ "role": "assistant", │ │ "sources": [], - │ │ "text": "The forecast is sunny.", │ │ "warnings": [] │ │ } │ │ metadata: { @@ -139,7 +139,8 @@ span_tree: │ │ ] │ │ └── step: 0 [llm] │ │ output: { - │ │ "text": "The forecast is sunny.", + │ │ "content": "The forecast is sunny.", + │ │ "role": "assistant", │ │ "toolCalls": [] │ │ } │ │ metadata: { @@ -154,7 +155,8 @@ span_tree: │ │ ] │ │ └── chunk: 'text' [llm] │ │ output: { - │ │ "text": "The forecast is sunny." + │ │ "content": "The forecast is sunny.", + │ │ "role": "assistant" │ │ } │ │ metadata: { │ │ "entity_id": "weather-agent", diff --git a/js/src/wrappers/mastra.test.ts b/js/src/wrappers/mastra.test.ts index dcced568b..1877567c0 100644 --- a/js/src/wrappers/mastra.test.ts +++ b/js/src/wrappers/mastra.test.ts @@ -16,11 +16,32 @@ try { // Best-effort initialization for test environments. } -type ExportedSpan = Parameters< +type MastraExportedSpan = Parameters< BraintrustObservabilityExporter["exportTracingEvent"] >[0]["exportedSpan"]; -const span = (overrides: Partial): ExportedSpan => ({ +let backgroundLogger: ReturnType< + typeof _exportsForTestingOnly.useTestBackgroundLogger +>; +let logger: ReturnType; + +beforeAll(async () => { + await _exportsForTestingOnly.simulateLoginForTests(); +}); + +beforeEach(() => { + backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); + logger = initLogger({ + projectId: "test-project-id", + projectName: "mastra.test.ts", + }); +}); + +afterEach(() => { + _exportsForTestingOnly.clearTestBackgroundLogger(); +}); + +const span = (overrides: Partial): MastraExportedSpan => ({ id: "span-1", traceId: "trace-1", name: "agent run", @@ -29,28 +50,25 @@ const span = (overrides: Partial): ExportedSpan => ({ ...overrides, }); -describe("BraintrustObservabilityExporter", () => { - let backgroundLogger: ReturnType< - typeof _exportsForTestingOnly.useTestBackgroundLogger - >; - let logger: ReturnType; - - beforeAll(async () => { - await _exportsForTestingOnly.simulateLoginForTests(); +// Run a span through the exporter start->end and return the logged row. +async function logSpan(overrides: Partial): Promise { + const exporter = new BraintrustObservabilityExporter(); + const exportedSpan = span({ + name: "llm: 'mock-model'", + type: "model_generation", + ...overrides, }); - - beforeEach(() => { - backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); - logger = initLogger({ - projectId: "test-project-id", - projectName: "mastra.test.ts", - }); - }); - - afterEach(() => { - _exportsForTestingOnly.clearTestBackgroundLogger(); + await exporter.exportTracingEvent({ type: "span_started", exportedSpan }); + await exporter.exportTracingEvent({ + type: "span_ended", + exportedSpan: { ...exportedSpan, endTime: 1_000_001 }, }); + await backgroundLogger.flush(); + const events = (await backgroundLogger.drain()) as any[]; + return events.find((e) => e.span_attributes?.name === exportedSpan.name); +} +describe("BraintrustObservabilityExporter", () => { test("logFeedback keyed on the Mastra span id merges into the span row", async () => { const exporter = new BraintrustObservabilityExporter(); const id = "mastra-span-123"; @@ -79,3 +97,107 @@ describe("BraintrustObservabilityExporter", () => { expect(merged[0].scores?.quality).toBe(0.9); }); }); + +describe("mastra thread-view IO transform", () => { + test("unwraps model_generation input { messages } into a bare array", async () => { + const messages = [ + { role: "system", content: "You are helpful." }, + { role: "user", content: "Hi" }, + ]; + const row = await logSpan({ input: { messages }, output: { text: "hi" } }); + expect(row?.input).toEqual(messages); + }); + + test("wraps a single model_generation input object into an array", async () => { + const row = await logSpan({ + input: { role: "user", content: "Just this" }, + output: { text: "ok" }, + }); + expect(row?.input).toEqual([{ role: "user", content: "Just this" }]); + }); + + test("reshapes model_generation output text into an assistant message", async () => { + const row = await logSpan({ + input: { messages: [{ role: "user", content: "Hi" }] }, + output: { text: "Hello there!", files: [], warnings: [] }, + }); + expect(row?.output).toEqual({ + role: "assistant", + content: "Hello there!", + files: [], + warnings: [], + }); + }); + + test("passes through model_generation output that has no text (tool-call turn)", async () => { + // A tool-call turn carries no `text`. Forcing `content: undefined` here + // makes the downstream message importer drop the whole turn, so it must + // pass through unchanged rather than be reshaped into an assistant message. + const output = { + files: [], + reasoning: [], + warnings: [], + toolCalls: [{ toolCallId: "c1", toolName: "getWeather" }], + }; + const row = await logSpan({ + input: { messages: [{ role: "user", content: "Weather?" }] }, + output, + }); + expect(row?.output).toEqual(output); + expect(row?.output).not.toHaveProperty("role"); + expect(row?.output).not.toHaveProperty("content"); + }); + + test("assistant reshape wins over colliding fields in the raw output", async () => { + const row = await logSpan({ + input: { messages: [{ role: "user", content: "Hi" }] }, + output: { text: "final", role: "tool", content: "stale", files: [] }, + }); + expect(row?.output).toEqual({ + role: "assistant", + content: "final", + files: [], + }); + }); + + test("leaves non-model spans untouched", async () => { + const input = { messages: [{ role: "user", content: "Hi" }] }; + const output = { text: "raw" }; + const row = await logSpan({ + name: "tool call", + type: "tool_call", + input, + output, + }); + expect(row?.input).toEqual(input); + expect(row?.output).toEqual(output); + }); + + // model_step and model_chunk are also `llm`-typed and reach Thread view, so + // they get the same transform as model_generation. + for (const type of ["model_step", "model_chunk"]) { + test(`transforms ${type} input and output like model_generation`, async () => { + const row = await logSpan({ + type, + input: { messages: [{ role: "user", content: "Hi" }] }, + output: { text: "Hello there!", toolCalls: [] }, + }); + expect(row?.input).toEqual([{ role: "user", content: "Hi" }]); + expect(row?.output).toEqual({ + role: "assistant", + content: "Hello there!", + toolCalls: [], + }); + }); + } + + test("leaves rag_embedding spans untouched (llm-typed but not chat)", async () => { + // rag_embedding maps to the `llm` span type but its payload is embedding + // text/vectors, not chat messages, so it must not be reshaped. + const input = { content: "text to embed" }; + const output = { text: "text to embed", embedding: [0.1, 0.2] }; + const row = await logSpan({ type: "rag_embedding", input, output }); + expect(row?.input).toEqual(input); + expect(row?.output).toEqual(output); + }); +}); diff --git a/js/src/wrappers/mastra.ts b/js/src/wrappers/mastra.ts index 28bba3410..5e18ba691 100644 --- a/js/src/wrappers/mastra.ts +++ b/js/src/wrappers/mastra.ts @@ -170,6 +170,59 @@ function modelMetrics( return Object.keys(out).length > 0 ? out : undefined; } +/** Mastra span types whose input/output carry the AI-SDK chat shapes that + * Braintrust's Thread view renders as a conversation. All three map to the + * Braintrust `llm` span type, and Thread view collects every `llm` span in + * the tree (`getThreadSpans` in the app), so a child `model_step` / + * `model_chunk` whose output stays as raw `{ text }` renders as JSON next to + * the properly-formatted `model_generation` turn. Transforming all three + * lets the app's hash dedup coalesce them into one assistant turn. + * + * `rag_embedding` is deliberately excluded: it is also `llm`-typed but its + * payload is embedding text/vectors, not chat messages. */ +const MODEL_SPAN_TYPES = new Set([ + "model_generation", + "model_step", + "model_chunk", +]); + +/** + * Unwrap a model span's input into the message-array shape Braintrust's Thread + * view expects. Mastra logs LLM input as `{ messages: [...] }` (or occasionally + * a single `{ role, content }` object); Braintrust renders a bare array of chat + * messages. Non-model spans and unrecognized shapes pass through unchanged. + */ +function transformModelInput(input: unknown, mastraType: string): unknown { + if (!MODEL_SPAN_TYPES.has(mastraType)) return input; + if (isObject(input) && Array.isArray(input.messages)) { + return input.messages; + } + if (isObject(input) && "content" in input) { + return [{ role: input.role, content: input.content }]; + } + return input; +} + +/** + * Reshape a model span's output into the assistant-message shape Braintrust's + * Thread view expects. Mastra logs a text turn as `{ text, ...rest }`; + * Braintrust renders `{ role: 'assistant', content }`, so we lift `text` into + * `content` (the explicit fields win over any collisions in `rest`). + * + * Only a string `text` is reshaped. A tool-call turn has no `text`; forcing + * `content: undefined` there makes the downstream message importer drop the + * whole turn, so those outputs pass through unchanged and are rendered by the + * importer's own tool-call handling. Non-model spans and non-object outputs + * also pass through unchanged. + */ +function transformModelOutput(output: unknown, mastraType: string): unknown { + if (!MODEL_SPAN_TYPES.has(mastraType)) return output; + if (!isObject(output)) return output; + if (typeof output.text !== "string") return output; + const { text, ...rest } = output; + return { ...rest, role: "assistant", content: text }; +} + /** Build the metadata payload Braintrust shows on the span, merging * Mastra's own `metadata`, `attributes` (sans usage), and entity fields. */ function buildMetadata(exported: MastraExportedSpan): Record { @@ -340,11 +393,11 @@ export class BraintrustObservabilityExporter implements MastraObservabilityExpor const event: Record = {}; if (exported.input !== undefined) { - event.input = exported.input; + event.input = transformModelInput(exported.input, exported.type); record.hasLoggedInput = true; } if (exported.output !== undefined) { - event.output = exported.output; + event.output = transformModelOutput(exported.output, exported.type); } const metadata = buildMetadata(exported);