diff --git a/AGENTS.md b/AGENTS.md index 0782e6ba..dca26750 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,6 +17,8 @@ Lingua is a universal message format that compiles to provider-specific formats - **Ask when non-lossy mapping is unclear**: If the universal type cannot represent a provider feature non-lossily, stop and ask for clarification on the intended canonical representation before implementing a workaround. - **No unapproved fallback logic**: Do not add ad-hoc fallback parsing/translation paths (for example `fallback_*` helpers) without checking with the programmer first. - **Typed boundaries only**: At provider boundaries, parse into well-defined typed structs/enums. Do not add lenient raw-JSON parsing that guesses defaults for required fields (for example defaulting missing `role` to `user`, lowercasing unknown roles, or inventing empty `content`). +- **Do not handwrite provider-format structs**: Do not manually define Rust structs/enums that represent provider wire formats when generated or canonical provider types already exist. Fix generation or add typed adapters around canonical types instead. +- **Do not inspect `serde_json::Value` directly for provider semantics**: Do not branch on provider-format fields via ad-hoc `Value` map access. Deserialize into typed provider or typed compatibility structs first, then convert. - **Fix via types or explicit errors**: If fuzzing finds unsupported/ambiguous shapes, either model them explicitly in types/converters or return a clear error. Do not silently coerce invalid input into a "best effort" shape. - **Typed-boundary CI gate**: CI enforces `make typed-boundary-check-branch BASE=origin/` on pull requests. Running `make typed-boundary-check` locally is recommended for faster feedback, but not required as a pre-commit hook. - **Typed extras views over raw map access**: If provider extras must be read, deserialize extras into a typed view struct first; do not pluck fields ad-hoc with `map.get(...)`. diff --git a/TODO.md b/TODO.md new file mode 100644 index 00000000..508e5ce2 --- /dev/null +++ b/TODO.md @@ -0,0 +1,54 @@ +# Import fixtures follow-up TODO + +## Failing new cases + +- [x] openai-responses-function-call-output-input +- [x] openai-responses-image-attachments +- [x] openai-responses-image-generation-call +- [x] openai-responses-mixed-input-order +- [x] openai-responses-mixed-output-order +- [x] openai-responses-real-world-tool-loop +- [x] openai-responses-reasoning-blocks +- [x] openai-responses-reasoning-only-output +- [x] openai-responses-web-search + +## Work items + +- [x] Loosen importer pre-check for Responses item arrays + - Current gap: `has_message_structure` rejects arrays that do not have `role` or nested `message.role`. + - Goal: allow Responses-only arrays (`reasoning`, `function_call_output`, `web_search_call`, etc.) to reach typed OpenAI conversion logic. + - Acceptance: mixed/output-only Responses fixtures are not dropped at import pre-check. + +- [x] Handle raw string span input as user messages + - Current gap: string-valued `span.input` is ignored. + - Goal: map string input to `Message::User` with string content. + - Acceptance: string-input fixtures (for example image generation and web search) include the expected leading user message. + +- [x] Expand lenient text-type parsing for content blocks + - Current gap: lenient parser only accepts `type: "text"`. + - Goal: also accept OpenAI Responses block types `input_text` and `output_text`. + - Acceptance: fixtures containing these block types parse into expected user/assistant text messages. + +- [x] Add typed compatibility for `callId` aliasing + - Current gap: some fixtures use `callId` while generated OpenAI types expect `call_id`. + - Goal: normalize or alias `callId` at import boundary before typed conversion. + - Acceptance: tool call and tool result linkage is preserved for both `call_id` and `callId`. + +- [x] Add typed compatibility for `function_call_result` + - Current gap: fixtures include `type: "function_call_result"` which is not represented in generated enums. + - Goal: normalize this to the canonical supported shape before conversion, without raw fallback parsing. + - Acceptance: output/input-order and tool-loop fixtures parse tool result messages correctly. + +- [x] Add typed compatibility for non-string tool output payloads + - Current gap: fixtures include object-valued `output`, while generated OpenAI types model output as string. + - Goal: normalize object payloads to canonical representation for strict typed conversion. + - Acceptance: function/tool-result fixtures preserve structured output content in imported tool messages. + +- [x] Decide and implement reasoning message aggregation behavior + - Current gap: reasoning output items become standalone assistant messages; some fixtures expect reasoning merged with adjacent assistant text. + - Goal: define canonical import behavior for reasoning-plus-message sequences and implement consistently. + - Acceptance: `openai-responses-reasoning-blocks` and `openai-responses-reasoning-only-output` match expected message counts and roles. + +- [x] Re-run and verify fixture suite after each fix + - Command: `cargo test -p lingua --test import_fixtures -- --nocapture` + - Process: update one behavior at a time and confirm no regressions in previously passing fixtures. diff --git a/crates/lingua/src/processing/import.rs b/crates/lingua/src/processing/import.rs index 12bf16cd..f4bb0509 100644 --- a/crates/lingua/src/processing/import.rs +++ b/crates/lingua/src/processing/import.rs @@ -1,6 +1,7 @@ use crate::providers::anthropic::generated as anthropic; -use crate::providers::openai::convert::ChatCompletionRequestMessageExt; -use crate::providers::openai::generated as openai; +use crate::providers::openai::convert::{ + try_parse_responses_items_for_import, ChatCompletionRequestMessageExt, +}; use crate::serde_json; use crate::serde_json::Value; use crate::universal::convert::TryFromLLM; @@ -22,43 +23,37 @@ pub struct Span { pub other: serde_json::Map, } -/// Cheap check to see if a value looks like it might contain messages -/// Returns early to avoid expensive deserialization attempts on non-message data -fn has_message_structure(data: &Value) -> bool { - match data { - // Check if it's an array where ANY element has "role" field or is a choice object - Value::Array(arr) => { - if arr.is_empty() { - return false; - } - // Check if ANY element in the array looks like a message (not just the first) - // This handles mixed-type arrays from Responses API - for item in arr { - if let Value::Object(obj) = item { - // Direct message format: has "role" field - if obj.contains_key("role") { +/// Try to convert a value to lingua messages by attempting multiple format conversions +fn try_converting_to_messages(data: &Value) -> Vec { + if let Some(messages) = try_parse_responses_items_for_import(data) { + return messages; + } + + // Cheap check to see if a value looks like it might contain messages. + // Returns early to avoid expensive deserialization attempts on non-message data. + let has_message_structure = match data { + // Check if it's an array where any element has "role" or nested "message.role". + Value::Array(arr) => arr.iter().any(|item| match item { + Value::Object(obj) => { + if obj.contains_key("role") { + return true; + } + if let Some(Value::Object(msg)) = obj.get("message") { + if msg.contains_key("role") { return true; } - // Chat completions response choices format: has "message" field with role inside - if let Some(Value::Object(msg)) = obj.get("message") { - if msg.contains_key("role") { - return true; - } - } } + false } - false - } + _ => false, + }), // Check if it's an object with "role" field (single message) Value::Object(obj) => obj.contains_key("role"), _ => false, - } -} + }; -/// Try to convert a value to lingua messages by attempting multiple format conversions -fn try_converting_to_messages(data: &Value) -> Vec { // Early bailout: if data doesn't have message structure, skip expensive deserializations - if !has_message_structure(data) { + if !has_message_structure { // Still try nested object search (for wrapped messages like {messages: [...]}) if let Value::Object(obj) = data { for key in [ @@ -104,32 +99,6 @@ fn try_converting_to_messages(data: &Value) -> Vec { } } - // Try Responses API format - if let Ok(provider_messages) = - serde_json::from_value::>(data_to_parse.clone()) - { - if let Ok(messages) = - as TryFromLLM>>::try_from(provider_messages) - { - if !messages.is_empty() { - return messages; - } - } - } - - // Try Responses API output format - if let Ok(provider_messages) = - serde_json::from_value::>(data_to_parse.clone()) - { - if let Ok(messages) = - as TryFromLLM>>::try_from(provider_messages) - { - if !messages.is_empty() { - return messages; - } - } - } - // Try Anthropic format (including role-based system/developer messages). if let Some(anthropic_messages) = try_anthropic_or_system_messages(data_to_parse) { if !anthropic_messages.is_empty() { @@ -265,7 +234,7 @@ fn parse_user_content(value: &Value) -> Option { for item in arr { if let Some(obj) = item.as_object() { if let Some(Value::String(text_type)) = obj.get("type") { - if text_type == "text" { + if matches!(text_type.as_str(), "text" | "input_text" | "output_text") { if let Some(Value::String(text)) = obj.get("text") { parts.push(UserContentPart::Text(TextContentPart { text: text.clone(), @@ -296,7 +265,7 @@ fn parse_assistant_content(value: &Value) -> Option { for item in arr { if let Some(obj) = item.as_object() { if let Some(Value::String(text_type)) = obj.get("type") { - if text_type == "text" { + if matches!(text_type.as_str(), "text" | "input_text" | "output_text") { if let Some(Value::String(text)) = obj.get("text") { parts.push(crate::universal::AssistantContentPart::Text( TextContentPart { @@ -389,9 +358,8 @@ fn try_choices_array_parsing(data: &Value) -> Option> { for item in arr { let obj = item.as_object()?; - // Check if this looks like a choice object (has "message" or "finish_reason") - // Note: has_message_structure only checks the first element, so we need to validate - // each element here to ensure the entire array is a valid choices array + // Check if this looks like a choice object (has "message" or "finish_reason"). + // We still validate each element here to ensure the entire array is a valid choices array. if !obj.contains_key("message") && !obj.contains_key("finish_reason") { return None; // Not a choices array } @@ -426,7 +394,11 @@ pub fn import_messages_from_spans(spans: Vec) -> Vec { for span in spans { // Try to extract messages from input - if let Some(input) = &span.input { + if let Some(Value::String(input_text)) = &span.input { + messages.push(Message::User { + content: UserContent::String(input_text.clone()), + }); + } else if let Some(input) = &span.input { let input_messages = try_converting_to_messages(input); messages.extend(input_messages); } diff --git a/crates/lingua/src/providers/openai/convert.rs b/crates/lingua/src/providers/openai/convert.rs index 59dc3f29..0de435ca 100644 --- a/crates/lingua/src/providers/openai/convert.rs +++ b/crates/lingua/src/providers/openai/convert.rs @@ -8,6 +8,7 @@ use crate::universal::{ ToolCallArguments, ToolContentPart, ToolResultContentPart, UserContent, UserContentPart, }; use crate::util::media::parse_base64_data_url; +use serde::de::Deserializer; use serde::{Deserialize, Serialize}; /// Extended ChatCompletionRequest/ResponseMessage with reasoning support. @@ -104,6 +105,207 @@ fn parse_builtin_field( } } +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +enum ResponsesImportKnownType { + // Some SDK/frontend traces use compatibility shapes (`function_call_result`, + // camelCase `callId`, JSON-valued `output`/`result`) that are not in the + // canonical OpenAI schema used to generate `openai::*` types. + #[serde(rename = "function_call_output", alias = "function_call_result")] + FunctionCallOutput, + #[serde(rename = "custom_tool_call_output")] + CustomToolCallOutput, + #[serde(rename = "function_call")] + FunctionCall, + #[serde(rename = "custom_tool_call")] + CustomToolCall, + #[serde(rename = "image_generation_call")] + ImageGenerationCall, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +enum ResponsesImportItemType { + Known(ResponsesImportKnownType), + Other(String), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct ResponsesImportCompatItem { + #[serde(rename = "type", skip_serializing_if = "Option::is_none")] + item_type: Option, + #[serde(default, alias = "callId", skip_serializing_if = "Option::is_none")] + call_id: Option, + #[serde( + default, + deserialize_with = "deserialize_optional_json_as_string", + skip_serializing_if = "Option::is_none" + )] + output: Option, + #[serde( + default, + deserialize_with = "deserialize_optional_json_as_string", + skip_serializing_if = "Option::is_none" + )] + result: Option, + #[serde(flatten)] + extra: serde_json::Map, +} + +fn deserialize_optional_json_as_string<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + match value { + None | Some(serde_json::Value::Null) => Ok(None), + Some(serde_json::Value::String(text)) => Ok(Some(text)), + Some(other) => Ok(Some(other.to_string())), + } +} + +fn normalize_responses_items_for_import(data: &serde_json::Value) -> Option { + let wrapped; + let candidate = if data.is_object() { + wrapped = serde_json::Value::Array(vec![data.clone()]); + &wrapped + } else { + data + }; + + let compat_items = + serde_json::from_value::>(candidate.clone()).ok()?; + let normalized = serde_json::to_value(compat_items).ok()?; + + if normalized == *candidate { + None + } else { + Some(normalized) + } +} + +fn is_reasoning_only_assistant_message(message: &Message) -> bool { + match message { + Message::Assistant { + content: AssistantContent::Array(parts), + .. + } => { + !parts.is_empty() + && parts + .iter() + .all(|part| matches!(part, AssistantContentPart::Reasoning { .. })) + } + _ => false, + } +} + +fn merge_adjacent_reasoning_assistant_messages(messages: Vec) -> Vec { + let mut merged = Vec::with_capacity(messages.len()); + + for message in messages { + let should_merge = matches!(merged.last(), Some(prev) if is_reasoning_only_assistant_message(prev)) + && matches!(message, Message::Assistant { .. }); + + if !should_merge { + merged.push(message); + continue; + } + + let Some(previous) = merged.pop() else { + merged.push(message); + continue; + }; + + let Message::Assistant { + content: AssistantContent::Array(reasoning_parts), + id: reasoning_id, + } = previous + else { + merged.push(previous); + merged.push(message); + continue; + }; + + let Message::Assistant { + content: next_content, + id: next_id, + } = message + else { + merged.push(Message::Assistant { + content: AssistantContent::Array(reasoning_parts), + id: reasoning_id, + }); + merged.push(message); + continue; + }; + + let mut combined_parts = reasoning_parts; + match next_content { + AssistantContent::Array(parts) => combined_parts.extend(parts), + AssistantContent::String(text) => { + combined_parts.push(AssistantContentPart::Text(TextContentPart { + text, + encrypted_content: None, + provider_options: None, + })); + } + } + + merged.push(Message::Assistant { + content: AssistantContent::Array(combined_parts), + id: next_id.or(reasoning_id), + }); + } + + merged +} + +fn try_from_responses_items_candidate(candidate: &serde_json::Value) -> Option> { + let wrapped; + let candidate = if candidate.is_object() { + wrapped = serde_json::Value::Array(vec![candidate.clone()]); + &wrapped + } else { + candidate + }; + + if let Ok(provider_messages) = + serde_json::from_value::>(candidate.clone()) + { + if let Ok(messages) = + as TryFromLLM>>::try_from(provider_messages) + { + if !messages.is_empty() { + return Some(merge_adjacent_reasoning_assistant_messages(messages)); + } + } + } + + if let Ok(provider_messages) = + serde_json::from_value::>(candidate.clone()) + { + if let Ok(messages) = + as TryFromLLM>>::try_from(provider_messages) + { + if !messages.is_empty() { + return Some(merge_adjacent_reasoning_assistant_messages(messages)); + } + } + } + + None +} + +pub(crate) fn try_parse_responses_items_for_import( + data: &serde_json::Value, +) -> Option> { + if let Some(messages) = try_from_responses_items_candidate(data) { + return Some(messages); + } + + let normalized = normalize_responses_items_for_import(data)?; + try_from_responses_items_candidate(&normalized) +} + /// Convert OpenAI InputItem collection to universal Message collection /// This handles OpenAI-specific logic for combining or transforming multiple items impl TryFromLLM> for Vec { diff --git a/payloads/import-cases/README.md b/payloads/import-cases/README.md index 2b7ce585..0bfaf9ce 100644 --- a/payloads/import-cases/README.md +++ b/payloads/import-cases/README.md @@ -69,6 +69,10 @@ Supported keys: - `expectedRolesInOrder` (string array) - `mustContainText` (string array) +Notes: + +- Unknown keys are ignored by the current test runner. We use `_migrationNote` to document intentional expectation changes when porting from older frontend converter tests. + ## Test modes Default mode (strict): diff --git a/payloads/import-cases/adk-basic-input-output.assertions.json b/payloads/import-cases/adk-basic-input-output.assertions.json new file mode 100644 index 00000000..9ad3ab64 --- /dev/null +++ b/payloads/import-cases/adk-basic-input-output.assertions.json @@ -0,0 +1,6 @@ +{ + "_migrationNote": "Diverges from app/ui/trace/converters/adk-converter.test.ts (basic ADK input/output): the old converter normalized ADK/Gemini-shaped `contents/parts` + `content.parts` into user+assistant messages, but lingua import_messages_from_spans currently imports 0 messages for this ADK span shape.", + "expectedMessageCount": 0, + "expectedRolesInOrder": [], + "mustContainText": [] +} diff --git a/payloads/import-cases/adk-basic-input-output.spans.json b/payloads/import-cases/adk-basic-input-output.spans.json new file mode 100644 index 00000000..1a3b903f --- /dev/null +++ b/payloads/import-cases/adk-basic-input-output.spans.json @@ -0,0 +1,25 @@ +[ + { + "input": { + "model": "gemini-2.0-flash-exp", + "contents": [ + { + "role": "user", + "parts": [{ "text": "What is the capital of France?" }] + } + ] + }, + "output": { + "content": { + "role": "model", + "parts": [{ "text": "The capital of France is Paris." }] + }, + "usageMetadata": { + "promptTokenCount": 10, + "candidatesTokenCount": 8, + "totalTokenCount": 18 + }, + "finishReason": "STOP" + } + } +] diff --git a/payloads/import-cases/ai-sdk-legacy-messages-output.assertions.json b/payloads/import-cases/ai-sdk-legacy-messages-output.assertions.json new file mode 100644 index 00000000..7f8d05ca --- /dev/null +++ b/payloads/import-cases/ai-sdk-legacy-messages-output.assertions.json @@ -0,0 +1,10 @@ +{ + "_migrationNote": "Diverges from app/ui/trace/converters/ai-sdk-converter.test.ts (legacy AI SDK format): lingua imports the nested input `messages` array but currently drops the legacy output object (`content`/`text` fields), while the old converter normalized both user and assistant messages.", + "expectedMessageCount": 1, + "expectedRolesInOrder": [ + "user" + ], + "mustContainText": [ + "Hello, how are you?" + ] +} diff --git a/payloads/import-cases/ai-sdk-legacy-messages-output.spans.json b/payloads/import-cases/ai-sdk-legacy-messages-output.spans.json new file mode 100644 index 00000000..cecfa114 --- /dev/null +++ b/payloads/import-cases/ai-sdk-legacy-messages-output.spans.json @@ -0,0 +1,28 @@ +[ + { + "input": { + "messages": [ + { + "role": "user", + "content": "Hello, how are you?" + } + ] + }, + "output": { + "content": [ + { + "type": "text", + "text": "I'm doing well, thank you!" + } + ], + "text": "I'm doing well, thank you!", + "finishReason": "stop", + "usage": { + "inputTokens": 10, + "outputTokens": 8, + "totalTokens": 18 + }, + "response": { "body": "" } + } + } +] diff --git a/payloads/import-cases/ai-sdk-openai-responses-steps.assertions.json b/payloads/import-cases/ai-sdk-openai-responses-steps.assertions.json new file mode 100644 index 00000000..5903f4f6 --- /dev/null +++ b/payloads/import-cases/ai-sdk-openai-responses-steps.assertions.json @@ -0,0 +1,6 @@ +{ + "_migrationNote": "Diverges from app/ui/trace/converters/ai-sdk-converter.test.ts (OpenAI Responses steps format): the old AI SDK converter normalized `prompt` + `steps.content` into user+assistant messages, but lingua import_messages_from_spans currently imports 0 messages for this AI SDK wrapper shape.", + "expectedMessageCount": 0, + "expectedRolesInOrder": [], + "mustContainText": [] +} diff --git a/payloads/import-cases/ai-sdk-openai-responses-steps.spans.json b/payloads/import-cases/ai-sdk-openai-responses-steps.spans.json new file mode 100644 index 00000000..c3cfe480 --- /dev/null +++ b/payloads/import-cases/ai-sdk-openai-responses-steps.spans.json @@ -0,0 +1,46 @@ +[ + { + "input": { + "model": { + "config": { + "fileIdPrefixes": ["file-"], + "provider": "openai.responses" + }, + "modelId": "gpt-5-mini", + "specificationVersion": "v2" + }, + "prompt": "What is the capital of France?" + }, + "output": { + "body": "", + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "rs_123", + "reasoningEncryptedContent": null + } + }, + "text": "", + "type": "reasoning" + }, + { + "providerMetadata": { + "openai": { + "itemId": "msg_456" + } + }, + "text": "The capital of France is Paris.", + "type": "text" + } + ], + "finishReason": "stop", + "request": { "body": "" }, + "response": { "body": "" } + } + ] + } + } +] diff --git a/payloads/import-cases/anthropic-multiple-tool-use-blocks.assertions.json b/payloads/import-cases/anthropic-multiple-tool-use-blocks.assertions.json new file mode 100644 index 00000000..911d9241 --- /dev/null +++ b/payloads/import-cases/anthropic-multiple-tool-use-blocks.assertions.json @@ -0,0 +1,5 @@ +{ + "expectedMessageCount": 1, + "expectedRolesInOrder": ["assistant"], + "mustContainText": ["tool1", "tool2", "toolu_1", "toolu_2"] +} diff --git a/payloads/import-cases/anthropic-multiple-tool-use-blocks.spans.json b/payloads/import-cases/anthropic-multiple-tool-use-blocks.spans.json new file mode 100644 index 00000000..7a7d7b9c --- /dev/null +++ b/payloads/import-cases/anthropic-multiple-tool-use-blocks.spans.json @@ -0,0 +1,23 @@ +[ + { + "input": [ + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_1", + "name": "tool1", + "input": { "param": "value1" } + }, + { + "type": "tool_use", + "id": "toolu_2", + "name": "tool2", + "input": { "param": "value2" } + } + ] + } + ] + } +] diff --git a/payloads/import-cases/anthropic-tool-result-blocks.assertions.json b/payloads/import-cases/anthropic-tool-result-blocks.assertions.json new file mode 100644 index 00000000..0f957d5b --- /dev/null +++ b/payloads/import-cases/anthropic-tool-result-blocks.assertions.json @@ -0,0 +1,5 @@ +{ + "expectedMessageCount": 1, + "expectedRolesInOrder": ["tool"], + "mustContainText": ["toolu_123", "4"] +} diff --git a/payloads/import-cases/anthropic-tool-result-blocks.spans.json b/payloads/import-cases/anthropic-tool-result-blocks.spans.json new file mode 100644 index 00000000..9ac33603 --- /dev/null +++ b/payloads/import-cases/anthropic-tool-result-blocks.spans.json @@ -0,0 +1,16 @@ +[ + { + "input": [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_123", + "content": "4" + } + ] + } + ] + } +] diff --git a/payloads/import-cases/anthropic-tool-use-blocks.assertions.json b/payloads/import-cases/anthropic-tool-use-blocks.assertions.json new file mode 100644 index 00000000..f8fd1b8c --- /dev/null +++ b/payloads/import-cases/anthropic-tool-use-blocks.assertions.json @@ -0,0 +1,5 @@ +{ + "expectedMessageCount": 1, + "expectedRolesInOrder": ["assistant"], + "mustContainText": ["Let me calculate", "calculate"] +} diff --git a/payloads/import-cases/anthropic-tool-use-blocks.spans.json b/payloads/import-cases/anthropic-tool-use-blocks.spans.json new file mode 100644 index 00000000..6de963c0 --- /dev/null +++ b/payloads/import-cases/anthropic-tool-use-blocks.spans.json @@ -0,0 +1,18 @@ +[ + { + "input": [ + { + "role": "assistant", + "content": [ + { "type": "text", "text": "Let me calculate that for you." }, + { + "type": "tool_use", + "id": "toolu_123", + "name": "calculate", + "input": { "operation": "add", "a": 2, "b": 2 } + } + ] + } + ] + } +] diff --git a/payloads/import-cases/converter-test-porting-todo.md b/payloads/import-cases/converter-test-porting-todo.md new file mode 100644 index 00000000..82e84b72 --- /dev/null +++ b/payloads/import-cases/converter-test-porting-todo.md @@ -0,0 +1,88 @@ +# Converter test porting todo + +This tracks porting from `app/ui/trace/converters/*.test.ts` into `payloads/import-cases`. + +Notes: + +- `import-cases` only exercises `import_messages_from_spans`, so detector tests and metadata-transform-only tests are not directly portable. +- When a fixture expectation differs from the old converter tests, the corresponding `*.assertions.json` includes `_migrationNote`. + +## Ported (high confidence / direct importer coverage) + +- `lingua-converter.test.ts` + - simple chat messages + - tool calls / tool results + - developer role variants + - multi-message conversations + - anthropic-style content block output + - try prompt (input-only) +- `anthropic-tools-converter.test.ts` + - tool_use blocks + - tool_result blocks + - multiple tool_use blocks + - plus existing anthropic fixtures already in this folder +- `openai-response-converter.test.ts` + - mixed responses ordering (input/output) + - function_call_output input case + - real-world tool loop + - image attachments / image generation / web search / reasoning blocks / reasoning-only output + - input_text/output_text message arrays +- `mastra-response-converter.test.ts` + - llm_generation conversation/tool loop + - legacy tool message + - tool_call span + - agent_run variants + +## Ported (representative unsupported/raw wrapper coverage) + +- `gemini-converter.test.ts` + - basic raw `contents/parts` request shape +- `adk-converter.test.ts` + - basic raw ADK input/output shape +- `langchain-converter.test.ts` + - basic human/ai wrapper + `generations` +- `pydantic-ai-converter.test.ts` + - basic wrapper `user_prompt` + `response.parts` +- `ai-sdk-converter.test.ts` + - OpenAI Responses `steps` wrapper + - legacy AI SDK `messages` + output object + +## Still to port (maximalist backlog) + +- `ai-sdk-converter.test.ts` (many scenarios; highest volume) + - v3/v4 output shapes + - streaming/steps variants + - tool call/result extraction across steps + - attachments (image/document) + - reasoning/thinking variants + - doGenerate/doStream/provider-level formats + - streamObject/object output variants +- `langchain-converter.test.ts` (many scenarios) + - tool_call transformations + - tool messages/tool_call_id + - metadata extraction variations + - multimodal image content + - Anthropic image conversions + - batch/multiple generation shapes +- `pydantic-ai-converter.test.ts` (many scenarios) + - message_history and internal message formats + - tool calls/returns and grouping + - multipart image/document attachments + - toolset/tool definition extraction + - reasoning/thinking parts +- `gemini-converter.test.ts` (more scenarios) + - thinking tokens + - image inputs + - function calls and snake_case variants +- `adk-converter.test.ts` (more scenarios) + - function calls and snake_case variants + - error responses and finishReason/usageMetadata edge cases + - Go library PascalCase format +- `openai-response-converter.test.ts` (remaining non-portable unit tests) + - `isOpenAIResponse` detection/rejection cases + - `transformMetadataForChatCompletions` + - metadata-driven system message / response-format transformation assertions +- `anthropic-tools-converter.test.ts` (remaining non-portable unit tests) + - tool metadata detection/transformation assertions +- `mastra-response-converter.test.ts` (remaining non-portable) + - `isMastraSpan` detector assertions diff --git a/payloads/import-cases/gemini-basic-generate-content.assertions.json b/payloads/import-cases/gemini-basic-generate-content.assertions.json new file mode 100644 index 00000000..e6521c90 --- /dev/null +++ b/payloads/import-cases/gemini-basic-generate-content.assertions.json @@ -0,0 +1,6 @@ +{ + "_migrationNote": "Diverges from app/ui/trace/converters/gemini-converter.test.ts (Gemini input without output): the old converter normalized Gemini `contents/parts` into a user chat message, but lingua import_messages_from_spans currently imports 0 messages for raw Gemini generateContent request shape.", + "expectedMessageCount": 0, + "expectedRolesInOrder": [], + "mustContainText": [] +} diff --git a/payloads/import-cases/gemini-basic-generate-content.spans.json b/payloads/import-cases/gemini-basic-generate-content.spans.json new file mode 100644 index 00000000..b694bd2b --- /dev/null +++ b/payloads/import-cases/gemini-basic-generate-content.spans.json @@ -0,0 +1,13 @@ +[ + { + "input": { + "model": "gemini-2.0-flash-exp", + "contents": [ + { + "role": "user", + "parts": [{ "text": "What is the capital of France?" }] + } + ] + } + } +] diff --git a/payloads/import-cases/langchain-human-ai-basic.assertions.json b/payloads/import-cases/langchain-human-ai-basic.assertions.json new file mode 100644 index 00000000..9535ddf3 --- /dev/null +++ b/payloads/import-cases/langchain-human-ai-basic.assertions.json @@ -0,0 +1,6 @@ +{ + "_migrationNote": "Diverges from app/ui/trace/converters/langchain-converter.test.ts (human/ai basic): the old converter normalized LangChain message wrappers (`type: human/ai`, nested `generations`) into user+assistant messages, but lingua import_messages_from_spans currently imports 0 messages for this raw LangChain shape.", + "expectedMessageCount": 0, + "expectedRolesInOrder": [], + "mustContainText": [] +} diff --git a/payloads/import-cases/langchain-human-ai-basic.spans.json b/payloads/import-cases/langchain-human-ai-basic.spans.json new file mode 100644 index 00000000..f2b27eef --- /dev/null +++ b/payloads/import-cases/langchain-human-ai-basic.spans.json @@ -0,0 +1,34 @@ +[ + { + "input": [ + [ + { + "content": "What is the capital of France?", + "type": "human", + "additional_kwargs": {}, + "response_metadata": {}, + "name": null, + "id": null, + "example": false + } + ] + ], + "output": { + "generations": [ + [ + { + "message": { + "content": "The capital of France is Paris.", + "type": "ai", + "additional_kwargs": { "refusal": null } + }, + "generation_info": { + "finish_reason": "stop", + "logprobs": null + } + } + ] + ] + } + } +] diff --git a/payloads/import-cases/lingua-anthropic-content-block-output.assertions.json b/payloads/import-cases/lingua-anthropic-content-block-output.assertions.json new file mode 100644 index 00000000..2a5499f9 --- /dev/null +++ b/payloads/import-cases/lingua-anthropic-content-block-output.assertions.json @@ -0,0 +1,5 @@ +{ + "expectedMessageCount": 2, + "expectedRolesInOrder": ["user", "assistant"], + "mustContainText": ["Tell me about AI", "Artificial Intelligence"] +} diff --git a/payloads/import-cases/lingua-anthropic-content-block-output.spans.json b/payloads/import-cases/lingua-anthropic-content-block-output.spans.json new file mode 100644 index 00000000..f45aa038 --- /dev/null +++ b/payloads/import-cases/lingua-anthropic-content-block-output.spans.json @@ -0,0 +1,18 @@ +[ + { + "input": [ + { "role": "user", "content": "Tell me about AI" } + ], + "output": [ + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "AI stands for Artificial Intelligence." + } + ] + } + ] + } +] diff --git a/payloads/import-cases/lingua-developer-array-content.assertions.json b/payloads/import-cases/lingua-developer-array-content.assertions.json new file mode 100644 index 00000000..5f571a4d --- /dev/null +++ b/payloads/import-cases/lingua-developer-array-content.assertions.json @@ -0,0 +1,5 @@ +{ + "expectedMessageCount": 2, + "expectedRolesInOrder": ["developer", "assistant"], + "mustContainText": ["specialized assistant", "Understood"] +} diff --git a/payloads/import-cases/lingua-developer-array-content.spans.json b/payloads/import-cases/lingua-developer-array-content.spans.json new file mode 100644 index 00000000..058609ab --- /dev/null +++ b/payloads/import-cases/lingua-developer-array-content.spans.json @@ -0,0 +1,18 @@ +[ + { + "input": [ + { + "role": "developer", + "content": [ + { + "type": "text", + "text": "You are a specialized assistant." + } + ] + } + ], + "output": [ + { "role": "assistant", "content": "Understood." } + ] + } +] diff --git a/payloads/import-cases/lingua-developer-role.assertions.json b/payloads/import-cases/lingua-developer-role.assertions.json new file mode 100644 index 00000000..7967a87c --- /dev/null +++ b/payloads/import-cases/lingua-developer-role.assertions.json @@ -0,0 +1,5 @@ +{ + "expectedMessageCount": 3, + "expectedRolesInOrder": ["developer", "user", "assistant"], + "mustContainText": ["helpful assistant", "What can you help me with", "available tools"] +} diff --git a/payloads/import-cases/lingua-developer-role.spans.json b/payloads/import-cases/lingua-developer-role.spans.json new file mode 100644 index 00000000..8539dd54 --- /dev/null +++ b/payloads/import-cases/lingua-developer-role.spans.json @@ -0,0 +1,20 @@ +[ + { + "input": [ + { + "role": "developer", + "content": "You are a helpful assistant with access to tools." + }, + { + "role": "user", + "content": "What can you help me with?" + } + ], + "output": [ + { + "role": "assistant", + "content": "I can help you with various tasks using the available tools." + } + ] + } +] diff --git a/payloads/import-cases/lingua-multi-message-conversation.assertions.json b/payloads/import-cases/lingua-multi-message-conversation.assertions.json new file mode 100644 index 00000000..0e88af24 --- /dev/null +++ b/payloads/import-cases/lingua-multi-message-conversation.assertions.json @@ -0,0 +1,5 @@ +{ + "expectedMessageCount": 5, + "expectedRolesInOrder": ["system", "user", "assistant", "user", "assistant"], + "mustContainText": ["You are helpful", "First question", "Follow-up answer"] +} diff --git a/payloads/import-cases/lingua-multi-message-conversation.spans.json b/payloads/import-cases/lingua-multi-message-conversation.spans.json new file mode 100644 index 00000000..ca58cbd2 --- /dev/null +++ b/payloads/import-cases/lingua-multi-message-conversation.spans.json @@ -0,0 +1,13 @@ +[ + { + "input": [ + { "role": "system", "content": "You are helpful." }, + { "role": "user", "content": "First question" } + ], + "output": [ + { "role": "assistant", "content": "First answer" }, + { "role": "user", "content": "Follow-up question" }, + { "role": "assistant", "content": "Follow-up answer" } + ] + } +] diff --git a/payloads/import-cases/lingua-tool-calls-only.assertions.json b/payloads/import-cases/lingua-tool-calls-only.assertions.json new file mode 100644 index 00000000..27dd8499 --- /dev/null +++ b/payloads/import-cases/lingua-tool-calls-only.assertions.json @@ -0,0 +1,5 @@ +{ + "expectedMessageCount": 2, + "expectedRolesInOrder": ["user", "assistant"], + "mustContainText": ["Calculate 5 + 3", "calculate"] +} diff --git a/payloads/import-cases/lingua-tool-calls-only.spans.json b/payloads/import-cases/lingua-tool-calls-only.spans.json new file mode 100644 index 00000000..d7f0732c --- /dev/null +++ b/payloads/import-cases/lingua-tool-calls-only.spans.json @@ -0,0 +1,23 @@ +[ + { + "input": [ + { "role": "user", "content": "Calculate 5 + 3" } + ], + "output": [ + { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_abc", + "type": "function", + "function": { + "name": "calculate", + "arguments": "{\"operation\":\"add\",\"a\":5,\"b\":3}" + } + } + ] + } + ] + } +] diff --git a/payloads/import-cases/lingua-tool-results-conversation.assertions.json b/payloads/import-cases/lingua-tool-results-conversation.assertions.json new file mode 100644 index 00000000..0828cb60 --- /dev/null +++ b/payloads/import-cases/lingua-tool-results-conversation.assertions.json @@ -0,0 +1,5 @@ +{ + "expectedMessageCount": 4, + "expectedRolesInOrder": ["user", "assistant", "tool", "assistant"], + "mustContainText": ["What's the weather?", "get_weather", "Sunny, 75F", "It's sunny"] +} diff --git a/payloads/import-cases/lingua-tool-results-conversation.spans.json b/payloads/import-cases/lingua-tool-results-conversation.spans.json new file mode 100644 index 00000000..8eab7a88 --- /dev/null +++ b/payloads/import-cases/lingua-tool-results-conversation.spans.json @@ -0,0 +1,29 @@ +[ + { + "input": [ + { "role": "user", "content": "What's the weather?" } + ], + "output": [ + { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_weather", + "type": "function", + "function": { "name": "get_weather", "arguments": "{}" } + } + ] + }, + { + "role": "tool", + "content": "Sunny, 75F", + "tool_call_id": "call_weather" + }, + { + "role": "assistant", + "content": "It's sunny and 75F!" + } + ] + } +] diff --git a/payloads/import-cases/lingua-try-prompt-input-only.assertions.json b/payloads/import-cases/lingua-try-prompt-input-only.assertions.json new file mode 100644 index 00000000..57cecfdc --- /dev/null +++ b/payloads/import-cases/lingua-try-prompt-input-only.assertions.json @@ -0,0 +1,5 @@ +{ + "expectedMessageCount": 2, + "expectedRolesInOrder": ["system", "user"], + "mustContainText": ["helpful assistant", "Hello"] +} diff --git a/payloads/import-cases/lingua-try-prompt-input-only.spans.json b/payloads/import-cases/lingua-try-prompt-input-only.spans.json new file mode 100644 index 00000000..164e59e5 --- /dev/null +++ b/payloads/import-cases/lingua-try-prompt-input-only.spans.json @@ -0,0 +1,8 @@ +[ + { + "input": [ + { "role": "system", "content": "You are a helpful assistant." }, + { "role": "user", "content": "Hello" } + ] + } +] diff --git a/payloads/import-cases/mastra-agent-run-object.assertions.json b/payloads/import-cases/mastra-agent-run-object.assertions.json new file mode 100644 index 00000000..0f12d592 --- /dev/null +++ b/payloads/import-cases/mastra-agent-run-object.assertions.json @@ -0,0 +1,6 @@ +{ + "_migrationNote": "Diverges from app/ui/trace/converters/mastra-response-converter.test.ts (agent_run span): the old converter synthesized an assistant message from `{text: ...}` output, but lingua import_messages_from_spans currently imports 0 messages for this object-shaped Mastra span.", + "expectedMessageCount": 0, + "expectedRolesInOrder": [], + "mustContainText": [] +} diff --git a/payloads/import-cases/mastra-agent-run-object.spans.json b/payloads/import-cases/mastra-agent-run-object.spans.json new file mode 100644 index 00000000..86e202d4 --- /dev/null +++ b/payloads/import-cases/mastra-agent-run-object.spans.json @@ -0,0 +1,7 @@ +[ + { + "input": { "query": "test" }, + "output": { "text": "Agent response" }, + "metadata": { "spanType": "agent_run" } + } +] diff --git a/payloads/import-cases/mastra-agent-run-parts-input.assertions.json b/payloads/import-cases/mastra-agent-run-parts-input.assertions.json new file mode 100644 index 00000000..92f83c9d --- /dev/null +++ b/payloads/import-cases/mastra-agent-run-parts-input.assertions.json @@ -0,0 +1,6 @@ +{ + "_migrationNote": "Diverges from app/ui/trace/converters/mastra-response-converter.test.ts (agent_run parts-based input): the old converter normalized `parts` input plus `{text}` output into user+assistant messages, but lingua import_messages_from_spans currently imports 0 messages for this Mastra-specific shape.", + "expectedMessageCount": 0, + "expectedRolesInOrder": [], + "mustContainText": [] +} diff --git a/payloads/import-cases/mastra-agent-run-parts-input.spans.json b/payloads/import-cases/mastra-agent-run-parts-input.spans.json new file mode 100644 index 00000000..e2f61ce0 --- /dev/null +++ b/payloads/import-cases/mastra-agent-run-parts-input.spans.json @@ -0,0 +1,13 @@ +[ + { + "input": [ + { + "id": "3By20HRG0qPzog54", + "role": "user", + "parts": [{ "type": "text", "text": "why did this task fail?" }] + } + ], + "output": { "text": "Agent response text" }, + "metadata": { "spanType": "agent_run" } + } +] diff --git a/payloads/import-cases/mastra-legacy-tool-message.assertions.json b/payloads/import-cases/mastra-legacy-tool-message.assertions.json new file mode 100644 index 00000000..8bf2cd32 --- /dev/null +++ b/payloads/import-cases/mastra-legacy-tool-message.assertions.json @@ -0,0 +1,10 @@ +{ + "_migrationNote": "Diverges from app/ui/trace/converters/mastra-response-converter.test.ts (legacy tool messages with toolId): the old converter emitted user + tool + assistant, but lingua import_messages_from_spans currently imports only the nested user message from `input.messages` and drops the tool/toolId variant plus `{text}` output.", + "expectedMessageCount": 1, + "expectedRolesInOrder": [ + "user" + ], + "mustContainText": [ + "Question" + ] +} diff --git a/payloads/import-cases/mastra-legacy-tool-message.spans.json b/payloads/import-cases/mastra-legacy-tool-message.spans.json new file mode 100644 index 00000000..84cbf3f0 --- /dev/null +++ b/payloads/import-cases/mastra-legacy-tool-message.spans.json @@ -0,0 +1,11 @@ +[ + { + "input": { + "messages": [ + { "role": "user", "content": "Question" }, + { "role": "tool", "content": "Result", "toolId": "myTool" } + ] + }, + "output": { "text": "Answer" } + } +] diff --git a/payloads/import-cases/mastra-llm-generation-tool-loop.assertions.json b/payloads/import-cases/mastra-llm-generation-tool-loop.assertions.json new file mode 100644 index 00000000..a6254570 --- /dev/null +++ b/payloads/import-cases/mastra-llm-generation-tool-loop.assertions.json @@ -0,0 +1,17 @@ +{ + "_migrationNote": "Diverges from app/ui/trace/converters/mastra-response-converter.test.ts (llm_generation tool loop): lingua imports the nested `input.messages` conversation (including tool-call/tool-result content blocks) but drops the final assistant message synthesized by the old converter from the object-shaped `{text}` output.", + "expectedMessageCount": 6, + "expectedRolesInOrder": [ + "system", + "user", + "assistant", + "tool", + "assistant", + "tool" + ], + "mustContainText": [ + "helpful assistant", + "weatherTool", + "Go hiking" + ] +} diff --git a/payloads/import-cases/mastra-llm-generation-tool-loop.spans.json b/payloads/import-cases/mastra-llm-generation-tool-loop.spans.json new file mode 100644 index 00000000..d0421961 --- /dev/null +++ b/payloads/import-cases/mastra-llm-generation-tool-loop.spans.json @@ -0,0 +1,56 @@ +[ + { + "input": { + "messages": [ + { "role": "system", "content": "You are a helpful assistant." }, + { "role": "user", "content": [{ "type": "text", "text": "What's the weather?" }] }, + { + "role": "assistant", + "content": [ + { + "type": "tool-call", + "toolCallId": "call_abc", + "toolName": "weatherTool", + "input": { "location": "SF" } + } + ] + }, + { + "role": "tool", + "content": [ + { + "type": "tool-result", + "toolCallId": "call_abc", + "toolName": "weatherTool", + "output": { "type": "json", "value": { "temp": 72, "sky": "Clear" } } + } + ] + }, + { + "role": "assistant", + "content": [ + { "type": "text", "text": "Based on that, " }, + { + "type": "tool-call", + "toolCallId": "call_xyz", + "toolName": "activitiesTool", + "input": { "weather": "Clear" } + } + ] + }, + { + "role": "tool", + "content": [ + { + "type": "tool-result", + "toolCallId": "call_xyz", + "toolName": "activitiesTool", + "output": "Go hiking" + } + ] + } + ] + }, + "output": { "text": "It's 72 and clear. Try hiking!", "files": [] } + } +] diff --git a/payloads/import-cases/mastra-tool-call-span.assertions.json b/payloads/import-cases/mastra-tool-call-span.assertions.json new file mode 100644 index 00000000..82fdffaf --- /dev/null +++ b/payloads/import-cases/mastra-tool-call-span.assertions.json @@ -0,0 +1,6 @@ +{ + "_migrationNote": "Diverges from app/ui/trace/converters/mastra-response-converter.test.ts (tool_call span): the old converter synthesized assistant tool-call + tool result messages from raw input/output objects and metadata, but lingua import_messages_from_spans currently imports 0 messages for this Mastra tool_call span shape.", + "expectedMessageCount": 0, + "expectedRolesInOrder": [], + "mustContainText": [] +} diff --git a/payloads/import-cases/mastra-tool-call-span.spans.json b/payloads/import-cases/mastra-tool-call-span.spans.json new file mode 100644 index 00000000..2162e2ab --- /dev/null +++ b/payloads/import-cases/mastra-tool-call-span.spans.json @@ -0,0 +1,7 @@ +[ + { + "input": { "location": "NYC" }, + "output": { "temp": 65, "conditions": "Cloudy" }, + "metadata": { "spanType": "tool_call", "toolId": "weatherTool", "toolDescription": "Get weather data" } + } +] diff --git a/payloads/import-cases/openai-responses-function-call-output-input.assertions.json b/payloads/import-cases/openai-responses-function-call-output-input.assertions.json new file mode 100644 index 00000000..53d6f581 --- /dev/null +++ b/payloads/import-cases/openai-responses-function-call-output-input.assertions.json @@ -0,0 +1,10 @@ +{ + "expectedMessageCount": 2, + "expectedRolesInOrder": [ + "tool", + "assistant" + ], + "mustContainText": [ + "weather in Chicago" + ] +} diff --git a/payloads/import-cases/openai-responses-function-call-output-input.spans.json b/payloads/import-cases/openai-responses-function-call-output-input.spans.json new file mode 100644 index 00000000..4950469b --- /dev/null +++ b/payloads/import-cases/openai-responses-function-call-output-input.spans.json @@ -0,0 +1,23 @@ +[ + { + "input": [ + { + "type": "function_call_output", + "call_id": "call_weather_1", + "output": "{\"temperature\":18.2,\"windspeed\":23.2}" + } + ], + "output": [ + { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "The current weather in Chicago is approximately 18.2 C." + } + ] + } + ] + } +] diff --git a/payloads/import-cases/openai-responses-image-attachments.assertions.json b/payloads/import-cases/openai-responses-image-attachments.assertions.json new file mode 100644 index 00000000..ab7d9791 --- /dev/null +++ b/payloads/import-cases/openai-responses-image-attachments.assertions.json @@ -0,0 +1,8 @@ +{ + "expectedMessageCount": 2, + "expectedRolesInOrder": [ + "user", + "assistant" + ], + "mustContainText": [] +} diff --git a/payloads/import-cases/openai-responses-image-attachments.spans.json b/payloads/import-cases/openai-responses-image-attachments.spans.json new file mode 100644 index 00000000..24da3ed2 --- /dev/null +++ b/payloads/import-cases/openai-responses-image-attachments.spans.json @@ -0,0 +1,42 @@ +[ + { + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Generate an image using this photo where the person looks like a werewolf." + }, + { + "type": "input_image", + "image": { + "type": "braintrust_attachment", + "content_type": "image/jpeg", + "filename": "file.jpeg", + "key": "input-image-key" + } + } + ] + } + ], + "output": [ + { + "type": "braintrust_attachment", + "content_type": "image/jpeg", + "filename": "file.jpeg", + "key": "output-image-key" + }, + { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Here is the transformed image with a spooky werewolf vibe." + } + ] + } + ] + } +] diff --git a/payloads/import-cases/openai-responses-image-generation-call.assertions.json b/payloads/import-cases/openai-responses-image-generation-call.assertions.json new file mode 100644 index 00000000..1d73a619 --- /dev/null +++ b/payloads/import-cases/openai-responses-image-generation-call.assertions.json @@ -0,0 +1,9 @@ +{ + "expectedMessageCount": 3, + "expectedRolesInOrder": [ + "user", + "assistant", + "assistant" + ], + "mustContainText": [] +} diff --git a/payloads/import-cases/openai-responses-image-generation-call.spans.json b/payloads/import-cases/openai-responses-image-generation-call.spans.json new file mode 100644 index 00000000..30599dfa --- /dev/null +++ b/payloads/import-cases/openai-responses-image-generation-call.spans.json @@ -0,0 +1,29 @@ +[ + { + "input": "Generate an image of a serene mountain landscape at sunset.", + "output": [ + { + "type": "image_generation_call", + "id": "ig_1", + "status": "completed", + "output_format": "png", + "result": { + "type": "braintrust_attachment", + "content_type": "image/png", + "filename": "mountain.png", + "key": "img_key_1" + } + }, + { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Here is the serene mountain landscape at sunset." + } + ] + } + ] + } +] diff --git a/payloads/import-cases/openai-responses-input-text-output-text.assertions.json b/payloads/import-cases/openai-responses-input-text-output-text.assertions.json new file mode 100644 index 00000000..e95e76e7 --- /dev/null +++ b/payloads/import-cases/openai-responses-input-text-output-text.assertions.json @@ -0,0 +1,11 @@ +{ + "expectedMessageCount": 2, + "expectedRolesInOrder": [ + "user", + "assistant" + ], + "mustContainText": [ + "Is this a question?", + "Question." + ] +} diff --git a/payloads/import-cases/openai-responses-input-text-output-text.spans.json b/payloads/import-cases/openai-responses-input-text-output-text.spans.json new file mode 100644 index 00000000..a8aec068 --- /dev/null +++ b/payloads/import-cases/openai-responses-input-text-output-text.spans.json @@ -0,0 +1,27 @@ +[ + { + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Is this a question?" + } + ] + } + ], + "output": [ + { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Question." + } + ] + } + ] + } +] diff --git a/payloads/import-cases/openai-responses-mixed-input-order.assertions.json b/payloads/import-cases/openai-responses-mixed-input-order.assertions.json new file mode 100644 index 00000000..478cec97 --- /dev/null +++ b/payloads/import-cases/openai-responses-mixed-input-order.assertions.json @@ -0,0 +1,10 @@ +{ + "expectedMessageCount": 4, + "expectedRolesInOrder": [ + "user", + "assistant", + "tool", + "user" + ], + "mustContainText": [] +} diff --git a/payloads/import-cases/openai-responses-mixed-input-order.spans.json b/payloads/import-cases/openai-responses-mixed-input-order.spans.json new file mode 100644 index 00000000..7dcc3d74 --- /dev/null +++ b/payloads/import-cases/openai-responses-mixed-input-order.spans.json @@ -0,0 +1,27 @@ +[ + { + "input": [ + { + "type": "message", + "role": "user", + "content": [{ "type": "input_text", "text": "Initial snapshot" }] + }, + { + "type": "function_call", + "name": "fill_field", + "arguments": "{\"value\":\"Example\"}", + "callId": "call_input_steps" + }, + { + "type": "function_call_output", + "callId": "call_input_steps", + "output": { "text": "Filled value" } + }, + { + "type": "message", + "role": "user", + "content": [{ "type": "input_text", "text": "Current state after tool" }] + } + ] + } +] diff --git a/payloads/import-cases/openai-responses-mixed-output-order.assertions.json b/payloads/import-cases/openai-responses-mixed-output-order.assertions.json new file mode 100644 index 00000000..478cec97 --- /dev/null +++ b/payloads/import-cases/openai-responses-mixed-output-order.assertions.json @@ -0,0 +1,10 @@ +{ + "expectedMessageCount": 4, + "expectedRolesInOrder": [ + "user", + "assistant", + "tool", + "user" + ], + "mustContainText": [] +} diff --git a/payloads/import-cases/openai-responses-mixed-output-order.spans.json b/payloads/import-cases/openai-responses-mixed-output-order.spans.json new file mode 100644 index 00000000..8e176458 --- /dev/null +++ b/payloads/import-cases/openai-responses-mixed-output-order.spans.json @@ -0,0 +1,27 @@ +[ + { + "output": [ + { + "type": "message", + "role": "user", + "content": [{ "type": "input_text", "text": "Before running tool" }] + }, + { + "type": "function_call", + "name": "click_search_result", + "arguments": "{\"elementDescription\":\"Example\"}", + "call_id": "call_output_steps" + }, + { + "type": "function_call_result", + "call_id": "call_output_steps", + "output": { "text": "Clicked result" } + }, + { + "type": "message", + "role": "user", + "content": [{ "type": "input_text", "text": "After running tool" }] + } + ] + } +] diff --git a/payloads/import-cases/openai-responses-real-world-tool-loop.assertions.json b/payloads/import-cases/openai-responses-real-world-tool-loop.assertions.json new file mode 100644 index 00000000..389c7d1a --- /dev/null +++ b/payloads/import-cases/openai-responses-real-world-tool-loop.assertions.json @@ -0,0 +1,13 @@ +{ + "expectedMessageCount": 4, + "expectedRolesInOrder": [ + "user", + "assistant", + "tool", + "assistant" + ], + "mustContainText": [ + "indoor shrimp farming", + "mystical 8-ball" + ] +} diff --git a/payloads/import-cases/openai-responses-real-world-tool-loop.spans.json b/payloads/import-cases/openai-responses-real-world-tool-loop.spans.json new file mode 100644 index 00000000..181f4fae --- /dev/null +++ b/payloads/import-cases/openai-responses-real-world-tool-loop.spans.json @@ -0,0 +1,38 @@ +[ + { + "input": [ + { + "type": "message", + "role": "user", + "content": "I have invested considerable money into indoor shrimp farming. Will my venture pay off?" + }, + { + "type": "function_call", + "name": "magic_8_ball", + "arguments": "{\"question\":\"Will my indoor shrimp farming venture pay off?\"}", + "callId": "call_magic_8" + }, + { + "type": "function_call_result", + "callId": "call_magic_8", + "name": "magic_8_ball", + "output": { + "text": "The magic 8-ball says: Signs point to yes", + "type": "text" + } + } + ], + "output": [ + { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "I consulted the mystical 8-ball and the outlook is favorable." + } + ] + } + ] + } +] diff --git a/payloads/import-cases/openai-responses-reasoning-blocks.assertions.json b/payloads/import-cases/openai-responses-reasoning-blocks.assertions.json new file mode 100644 index 00000000..ccbe4459 --- /dev/null +++ b/payloads/import-cases/openai-responses-reasoning-blocks.assertions.json @@ -0,0 +1,12 @@ +{ + "expectedMessageCount": 2, + "expectedRolesInOrder": [ + "user", + "assistant" + ], + "mustContainText": [ + "South American countries", + "Step 1: List all countries", + "Estimating South American capitals" + ] +} diff --git a/payloads/import-cases/openai-responses-reasoning-blocks.spans.json b/payloads/import-cases/openai-responses-reasoning-blocks.spans.json new file mode 100644 index 00000000..2771989a --- /dev/null +++ b/payloads/import-cases/openai-responses-reasoning-blocks.spans.json @@ -0,0 +1,37 @@ +[ + { + "input": [ + { + "type": "message", + "role": "user", + "content": "List South American countries by Pacific capital population." + } + ], + "output": [ + { + "type": "reasoning", + "id": "rs_1", + "summary": [ + { + "type": "summary_text", + "text": "**Estimating South American capitals**" + }, + { + "type": "summary_text", + "text": "**Estimating capital populations**" + } + ] + }, + { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Step 1: List all countries in South America..." + } + ] + } + ] + } +] diff --git a/payloads/import-cases/openai-responses-reasoning-only-output.assertions.json b/payloads/import-cases/openai-responses-reasoning-only-output.assertions.json new file mode 100644 index 00000000..82cdbc8d --- /dev/null +++ b/payloads/import-cases/openai-responses-reasoning-only-output.assertions.json @@ -0,0 +1,11 @@ +{ + "expectedMessageCount": 3, + "expectedRolesInOrder": [ + "system", + "user", + "assistant" + ], + "mustContainText": [ + "Evaluate this content" + ] +} diff --git a/payloads/import-cases/openai-responses-reasoning-only-output.spans.json b/payloads/import-cases/openai-responses-reasoning-only-output.spans.json new file mode 100644 index 00000000..1562440b --- /dev/null +++ b/payloads/import-cases/openai-responses-reasoning-only-output.spans.json @@ -0,0 +1,19 @@ +[ + { + "input": [ + { "role": "system", "content": "Content evaluator" }, + { "role": "user", "content": "Evaluate this content" } + ], + "output": [ + { + "type": "reasoning", + "id": "rs_reasoning_only", + "summary": [ + { "type": "summary_text", "text": "aaaaa" }, + { "type": "summary_text", "text": "aaaa" }, + { "type": "summary_text", "text": "aaaaa" } + ] + } + ] + } +] diff --git a/payloads/import-cases/openai-responses-web-search.assertions.json b/payloads/import-cases/openai-responses-web-search.assertions.json new file mode 100644 index 00000000..5fac5608 --- /dev/null +++ b/payloads/import-cases/openai-responses-web-search.assertions.json @@ -0,0 +1,12 @@ +{ + "expectedMessageCount": 3, + "expectedRolesInOrder": [ + "user", + "assistant", + "assistant" + ], + "mustContainText": [ + "latest AI news", + "comprehensive overview" + ] +} diff --git a/payloads/import-cases/openai-responses-web-search.spans.json b/payloads/import-cases/openai-responses-web-search.spans.json new file mode 100644 index 00000000..9d5b4bee --- /dev/null +++ b/payloads/import-cases/openai-responses-web-search.spans.json @@ -0,0 +1,26 @@ +[ + { + "input": "What are the latest developments in artificial intelligence from this week?", + "output": [ + { + "type": "web_search_call", + "id": "ws_1", + "status": "completed", + "action": { + "type": "search", + "query": "latest AI news this week" + } + }, + { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Here is a comprehensive overview of recent AI developments." + } + ] + } + ] + } +] diff --git a/payloads/import-cases/pydantic-ai-wrapper-user-prompt.assertions.json b/payloads/import-cases/pydantic-ai-wrapper-user-prompt.assertions.json new file mode 100644 index 00000000..d99cee68 --- /dev/null +++ b/payloads/import-cases/pydantic-ai-wrapper-user-prompt.assertions.json @@ -0,0 +1,6 @@ +{ + "_migrationNote": "Diverges from app/ui/trace/converters/pydantic-ai-converter.test.ts (wrapper format with user_prompt): the old converter synthesized user+assistant messages from pydantic-ai wrapper fields (`user_prompt`, `response.parts`), but lingua import_messages_from_spans currently imports 0 messages for this raw wrapper shape.", + "expectedMessageCount": 0, + "expectedRolesInOrder": [], + "mustContainText": [] +} diff --git a/payloads/import-cases/pydantic-ai-wrapper-user-prompt.spans.json b/payloads/import-cases/pydantic-ai-wrapper-user-prompt.spans.json new file mode 100644 index 00000000..cd1cd41b --- /dev/null +++ b/payloads/import-cases/pydantic-ai-wrapper-user-prompt.spans.json @@ -0,0 +1,22 @@ +[ + { + "input": { + "user_prompt": "What is the capital of France?" + }, + "output": { + "output": "The capital of France is Paris.", + "response": { + "finish_reason": "stop", + "kind": "response", + "model_name": "gpt-4o-2024-08-06", + "parts": [ + { + "content": "The capital of France is Paris.", + "id": null, + "part_kind": "text" + } + ] + } + } + } +]