From 48be39a0f8ef5d0f2674fde3050102095bb1abf0 Mon Sep 17 00:00:00 2001 From: jewoodev Date: Fri, 3 Jul 2026 08:59:19 +0900 Subject: [PATCH] Merge streaming tool call deltas by index OpenAI omits tool call id/name on continuation deltas, while some OpenAI-compatible endpoints send empty strings. Key streaming tool call merging on the required delta index so split fields and parallel tool calls stay associated. Keep final streamed tool call conversion strict: the merged tool call must have a function, non-blank id, and non-blank function name before it is exposed as an assistant tool call. Arguments remain optional and default to an empty string. Fixes #6374 Signed-off-by: jewoodev --- .../ai/openai/OpenAiChatModel.java | 71 +++--- .../OpenAiChatModelChunkMergerTests.java | 212 ++++++++++++++++++ .../ai/openai/OpenAiChatModelTests.java | 82 +++++++ 3 files changed, 336 insertions(+), 29 deletions(-) create mode 100644 models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelChunkMergerTests.java diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java index 1002452d3c..68267e1f76 100644 --- a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java @@ -1039,7 +1039,7 @@ private Prompt buildRequestPrompt(Prompt prompt) { } } - private static final class ChunkMerger { + static final class ChunkMerger { static boolean hasToolCall(ChatCompletionChunk chunk) { return !chunk.choices().isEmpty() @@ -1077,35 +1077,18 @@ private static Choice mergeChoices(Choice c1, Choice c2) { } private static Delta mergeDeltas(Delta left, Delta right) { + // Deltas of the same logical tool call share the required 'index' field. + // Some OpenAI-compatible providers (e.g. DeepSeek) send an empty-string id + // on continuation deltas instead of omitting it, so id presence cannot be + // used to detect the start of a new tool call. var tcs = Stream.of(left.toolCalls(), right.toolCalls()).flatMap(Optional::stream).reduce((tcs1, tcs2) -> { if (tcs2.isEmpty()) { return tcs1; } - Assert.isTrue(tcs2.size() == 1, "no more than one tool call per message currently supported"); - ToolCall toolCall = tcs2.get(0); - if (toolCall.id().isPresent()) { - List result = new ArrayList<>(tcs1); - result.add(toolCall); - return result; - } - else { - ToolCall lastFromTc1 = tcs1.get(tcs1.size() - 1); - Function lastFromTc1F = lastFromTc1.function().get(); - - var concatenatedArgs = Stream - .of(lastFromTc1F.arguments(), toolCall.function().flatMap(Function::arguments)) - .flatMap(Optional::stream) - .reduce((args1, args2) -> args1 + args2) - .orElse(""); - - List result = new ArrayList<>(tcs1); - result.set(tcs1.size() - 1, - lastFromTc1.toBuilder() - .putAllAdditionalProperties(toolCall._additionalProperties()) - .function(lastFromTc1F.toBuilder().arguments(concatenatedArgs).build()) - .build()); - return result; - } + Map mergedByIndex = new LinkedHashMap<>(); + tcs1.forEach(tc -> mergedByIndex.merge(tc.index(), tc, ChunkMerger::mergeToolCalls)); + tcs2.forEach(tc -> mergedByIndex.merge(tc.index(), tc, ChunkMerger::mergeToolCalls)); + return List.copyOf(mergedByIndex.values()); }).orElse(List.of()); Delta.Builder deltaBuilder = left.toBuilder().toolCalls(tcs); @@ -1124,6 +1107,28 @@ private static Optional stringProperty(Delta delta, String key) { return Optional.ofNullable(delta._additionalProperties().get(key)).flatMap(JsonValue::asString); } + private static ToolCall mergeToolCalls(ToolCall previous, ToolCall current) { + String arguments = Stream + .of(previous.function().flatMap(Function::arguments), current.function().flatMap(Function::arguments)) + .flatMap(Optional::stream) + .collect(Collectors.joining()); + return previous.toBuilder() + .id(firstWithText(previous.id(), current.id())) + .putAllAdditionalProperties(current._additionalProperties()) + .function(previous.function() + .map(Function::toBuilder) + .orElseGet(Function::builder) + .name(firstWithText(previous.function().flatMap(Function::name), + current.function().flatMap(Function::name))) + .arguments(arguments) + .build()) + .build(); + } + + private static String firstWithText(Optional first, Optional second) { + return first.filter(StringUtils::hasText).or(() -> second.filter(StringUtils::hasText)).orElse(""); + } + /** * Convert a ChatCompletionChunk into a ChatCompletion. */ @@ -1160,11 +1165,19 @@ static ChatCompletion chunkToChatCompletion(ChatCompletionChunk chunk) { msgBuilder.toolCalls(ccctcs.stream().map(tc -> { ChatCompletionMessageFunctionToolCall.Builder toolCallBuilder = ChatCompletionMessageFunctionToolCall .builder(); + Function function = tc.function() + .orElseThrow(() -> new IllegalStateException("Tool call function is missing")); + String id = tc.id() + .filter(StringUtils::hasText) + .orElseThrow(() -> new IllegalStateException("Tool call id is missing")); + String name = function.name() + .filter(StringUtils::hasText) + .orElseThrow(() -> new IllegalStateException("Tool call function name is missing")); toolCallBuilder.putAllAdditionalProperties(tc._additionalProperties()); - toolCallBuilder.id(tc.id().get()); + toolCallBuilder.id(id); toolCallBuilder.function(ChatCompletionMessageFunctionToolCall.Function.builder() - .name(tc.function().get().name().get()) - .arguments(tc.function().get().arguments().get()) + .name(name) + .arguments(function.arguments().orElse("")) .build()); return ChatCompletionMessageToolCall.ofFunction(toolCallBuilder.build()); }).toList()); diff --git a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelChunkMergerTests.java b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelChunkMergerTests.java new file mode 100644 index 0000000000..8413290c37 --- /dev/null +++ b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelChunkMergerTests.java @@ -0,0 +1,212 @@ +/* + * Copyright 2023-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.openai; + +import java.util.List; + +import com.openai.models.chat.completions.ChatCompletion; +import com.openai.models.chat.completions.ChatCompletionChunk; +import com.openai.models.chat.completions.ChatCompletionChunk.Choice; +import com.openai.models.chat.completions.ChatCompletionChunk.Choice.Delta; +import com.openai.models.chat.completions.ChatCompletionChunk.Choice.Delta.ToolCall; +import com.openai.models.chat.completions.ChatCompletionChunk.Choice.Delta.ToolCall.Function; +import com.openai.models.chat.completions.ChatCompletionChunk.Choice.FinishReason; +import com.openai.models.chat.completions.ChatCompletionMessageFunctionToolCall; +import org.jspecify.annotations.Nullable; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalStateException; + +/** + * Unit tests for {@link OpenAiChatModel.ChunkMerger}, covering the merging of streamed + * tool call deltas by their {@code index} field. OpenAI omits {@code id} and + * {@code function.name} on continuation deltas, while some OpenAI-compatible providers + * (e.g. DeepSeek) send them as empty strings instead. + * + * @author Jewoo Shin + */ +class OpenAiChatModelChunkMergerTests { + + @Test + void mergeToolCallDeltasWithAbsentContinuationIds() { + ChatCompletionChunk merged = OpenAiChatModel.ChunkMerger + .mergeChunks(List.of(chunk(null, toolCallDelta(0, "call_1", "get_weather", "")), + chunk(null, toolCallDelta(0, null, null, "{\"city\": ")), + chunk(null, toolCallDelta(0, null, null, "\"Seoul\"}")), finishChunk())); + + List toolCalls = merged.choices().get(0).delta().toolCalls().orElseThrow(); + assertThat(toolCalls).hasSize(1); + assertThat(toolCalls.get(0).id()).contains("call_1"); + assertThat(toolCalls.get(0).function().orElseThrow().name()).contains("get_weather"); + assertThat(toolCalls.get(0).function().orElseThrow().arguments()).contains("{\"city\": \"Seoul\"}"); + } + + @Test // gh-6374 + void mergeToolCallDeltasWithEmptyContinuationIds() { + ChatCompletionChunk merged = OpenAiChatModel.ChunkMerger + .mergeChunks(List.of(chunk(null, toolCallDelta(0, "call_1d70a7ee", "show_options", "")), + chunk(null, toolCallDelta(0, "", "", "{\"options\": [\"a\"")), + chunk(null, toolCallDelta(0, "", "", ", \"b\"]}")), finishChunk())); + + assertThat(merged.choices().get(0).finishReason()).contains(FinishReason.TOOL_CALLS); + List toolCalls = merged.choices().get(0).delta().toolCalls().orElseThrow(); + assertThat(toolCalls).hasSize(1); + assertThat(toolCalls.get(0).id()).contains("call_1d70a7ee"); + assertThat(toolCalls.get(0).function().orElseThrow().name()).contains("show_options"); + assertThat(toolCalls.get(0).function().orElseThrow().arguments()).contains("{\"options\": [\"a\", \"b\"]}"); + + ChatCompletion completion = OpenAiChatModel.ChunkMerger.chunkToChatCompletion(merged); + ChatCompletionMessageFunctionToolCall functionToolCall = completion.choices() + .get(0) + .message() + .toolCalls() + .orElseThrow() + .get(0) + .function() + .orElseThrow(); + assertThat(functionToolCall.id()).isEqualTo("call_1d70a7ee"); + assertThat(functionToolCall.function().name()).isEqualTo("show_options"); + assertThat(functionToolCall.function().arguments()).isEqualTo("{\"options\": [\"a\", \"b\"]}"); + } + + @Test + void mergeParallelToolCallDeltasByIndex() { + ChatCompletionChunk merged = OpenAiChatModel.ChunkMerger + .mergeChunks(List.of(chunk(null, toolCallDelta(0, "call_a", "tool_a", "")), + chunk(null, toolCallDelta(1, "call_b", "tool_b", "")), + chunk(null, toolCallDelta(0, null, null, "{\"a\": 1}")), + chunk(null, toolCallDelta(1, null, null, "{\"b\": 2}")), finishChunk())); + + List toolCalls = merged.choices().get(0).delta().toolCalls().orElseThrow(); + assertThat(toolCalls).hasSize(2); + assertThat(toolCalls.get(0).id()).contains("call_a"); + assertThat(toolCalls.get(0).function().orElseThrow().arguments()).contains("{\"a\": 1}"); + assertThat(toolCalls.get(1).id()).contains("call_b"); + assertThat(toolCalls.get(1).function().orElseThrow().arguments()).contains("{\"b\": 2}"); + } + + @Test + void mergeMultipleToolCallDeltasInSingleChunk() { + ChatCompletionChunk merged = OpenAiChatModel.ChunkMerger.mergeChunks(List.of( + chunk(null, toolCallDelta(0, "call_a", "tool_a", ""), toolCallDelta(1, "call_b", "tool_b", "")), + chunk(null, toolCallDelta(0, null, null, "{\"a\": 1}"), toolCallDelta(1, null, null, "{\"b\": 2}")), + finishChunk())); + + List toolCalls = merged.choices().get(0).delta().toolCalls().orElseThrow(); + assertThat(toolCalls).hasSize(2); + assertThat(toolCalls.get(0).function().orElseThrow().name()).contains("tool_a"); + assertThat(toolCalls.get(0).function().orElseThrow().arguments()).contains("{\"a\": 1}"); + assertThat(toolCalls.get(1).function().orElseThrow().name()).contains("tool_b"); + assertThat(toolCalls.get(1).function().orElseThrow().arguments()).contains("{\"b\": 2}"); + } + + @Test + void chunkToChatCompletionRequiresToolCallId() { + ChatCompletionChunk chunk = chunk(FinishReason.TOOL_CALLS, + toolCallDelta(0, null, "get_weather", "{\"city\":\"Seoul\"}")); + ChatCompletionChunk blankIdChunk = chunk(FinishReason.TOOL_CALLS, + toolCallDelta(0, " ", "get_weather", "{\"city\":\"Seoul\"}")); + + assertThatIllegalStateException() + .isThrownBy(() -> OpenAiChatModel.ChunkMerger.chunkToChatCompletion(chunk)) + .withMessage("Tool call id is missing"); + assertThatIllegalStateException() + .isThrownBy(() -> OpenAiChatModel.ChunkMerger.chunkToChatCompletion(blankIdChunk)) + .withMessage("Tool call id is missing"); + } + + @Test + void chunkToChatCompletionRequiresToolCallFunction() { + ChatCompletionChunk chunk = chunk(FinishReason.TOOL_CALLS, toolCallDeltaWithoutFunction(0, "call_1")); + + assertThatIllegalStateException().isThrownBy(() -> OpenAiChatModel.ChunkMerger.chunkToChatCompletion(chunk)) + .withMessage("Tool call function is missing"); + } + + @Test + void chunkToChatCompletionRequiresToolCallFunctionName() { + ChatCompletionChunk chunk = chunk(FinishReason.TOOL_CALLS, + toolCallDelta(0, "call_1", null, "{\"city\":\"Seoul\"}")); + ChatCompletionChunk blankNameChunk = chunk(FinishReason.TOOL_CALLS, + toolCallDelta(0, "call_1", " ", "{\"city\":\"Seoul\"}")); + + assertThatIllegalStateException().isThrownBy(() -> OpenAiChatModel.ChunkMerger.chunkToChatCompletion(chunk)) + .withMessage("Tool call function name is missing"); + assertThatIllegalStateException() + .isThrownBy(() -> OpenAiChatModel.ChunkMerger.chunkToChatCompletion(blankNameChunk)) + .withMessage("Tool call function name is missing"); + } + + @Test + void chunkToChatCompletionUsesEmptyDefaultForMissingArguments() { + ChatCompletionChunk chunk = chunk(FinishReason.TOOL_CALLS, + toolCallDeltaWithoutArguments(0, "call_1", "get_weather")); + + ChatCompletion completion = OpenAiChatModel.ChunkMerger.chunkToChatCompletion(chunk); + + ChatCompletionMessageFunctionToolCall functionToolCall = completion.choices() + .get(0) + .message() + .toolCalls() + .orElseThrow() + .get(0) + .function() + .orElseThrow(); + assertThat(functionToolCall.id()).isEqualTo("call_1"); + assertThat(functionToolCall.function().name()).isEqualTo("get_weather"); + assertThat(functionToolCall.function().arguments()).isEmpty(); + } + + private static ChatCompletionChunk chunk(@Nullable FinishReason finishReason, ToolCall... toolCalls) { + Delta.Builder delta = Delta.builder(); + if (toolCalls.length > 0) { + delta.toolCalls(List.of(toolCalls)); + } + return ChatCompletionChunk.builder() + .id("chatcmpl-test") + .choices(List.of(Choice.builder().index(0L).delta(delta.build()).finishReason(finishReason).build())) + .created(1L) + .model("test-model") + .build(); + } + + private static ChatCompletionChunk finishChunk() { + return chunk(FinishReason.TOOL_CALLS); + } + + private static ToolCall toolCallDelta(long index, @Nullable String id, @Nullable String name, String arguments) { + Function.Builder function = Function.builder().arguments(arguments); + if (name != null) { + function.name(name); + } + ToolCall.Builder toolCall = ToolCall.builder().index(index).function(function.build()); + if (id != null) { + toolCall.id(id); + } + return toolCall.build(); + } + + private static ToolCall toolCallDeltaWithoutArguments(long index, String id, String name) { + return ToolCall.builder().index(index).id(id).function(Function.builder().name(name).build()).build(); + } + + private static ToolCall toolCallDeltaWithoutFunction(long index, String id) { + return ToolCall.builder().index(index).id(id).build(); + } + +} diff --git a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelTests.java b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelTests.java index df79987d30..7ebf618956 100644 --- a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelTests.java +++ b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelTests.java @@ -270,6 +270,88 @@ void preserveUnmappedToolCallAdditionalPropertiesFromStream() { Map.of("google", Map.of("thought_signature", "signature-123"))); } + @Test + void mergeStreamToolCallWhenIdNameAndArgumentsArriveInSeparateChunks() { + ChatServiceAsync chatServiceAsync = mock(ChatServiceAsync.class); + ChatCompletionServiceAsync chatCompletionServiceAsync = mock(ChatCompletionServiceAsync.class); + when(this.openAiClientAsync.chat()).thenReturn(chatServiceAsync); + when(chatServiceAsync.completions()).thenReturn(chatCompletionServiceAsync); + when(chatCompletionServiceAsync.createStreaming(any(ChatCompletionCreateParams.class))) + .thenReturn(asyncStreamResponse(ChatCompletionChunk.builder() + .id("chatcmpl-stream-test") + .created(1777799928) + .model("test-model") + .addChoice(ChatCompletionChunk.Choice.builder() + .index(0) + .finishReason(Optional.empty()) + .delta(ChatCompletionChunk.Choice.Delta.builder() + .addToolCall(ChatCompletionChunk.Choice.Delta.ToolCall.builder().index(0).id("call_1").build()) + .build()) + .build()) + .build(), + ChatCompletionChunk.builder() + .id("chatcmpl-stream-test") + .created(1777799928) + .model("test-model") + .addChoice(ChatCompletionChunk.Choice.builder() + .index(0) + .finishReason(Optional.empty()) + .delta(ChatCompletionChunk.Choice.Delta.builder() + .addToolCall(ChatCompletionChunk.Choice.Delta.ToolCall.builder() + .index(0) + .function(ChatCompletionChunk.Choice.Delta.ToolCall.Function.builder() + .name("get_current_weather") + .build()) + .build()) + .build()) + .build()) + .build(), + ChatCompletionChunk.builder() + .id("chatcmpl-stream-test") + .created(1777799928) + .model("test-model") + .addChoice(ChatCompletionChunk.Choice.builder() + .index(0) + .finishReason(Optional.empty()) + .delta(ChatCompletionChunk.Choice.Delta.builder() + .addToolCall(ChatCompletionChunk.Choice.Delta.ToolCall.builder() + .index(0) + .function(ChatCompletionChunk.Choice.Delta.ToolCall.Function.builder() + .arguments("{\"location\":\"Seoul\"}") + .build()) + .build()) + .build()) + .build()) + .build(), + ChatCompletionChunk.builder() + .id("chatcmpl-stream-test") + .created(1777799928) + .model("test-model") + .addChoice(ChatCompletionChunk.Choice.builder() + .index(0) + .finishReason(ChatCompletionChunk.Choice.FinishReason.TOOL_CALLS) + .delta(ChatCompletionChunk.Choice.Delta.builder().build()) + .build()) + .build())); + + OpenAiChatOptions options = OpenAiChatOptions.builder().model("test-model").build(); + OpenAiChatModel chatModel = OpenAiChatModel.builder() + .openAiClient(this.openAiClient) + .openAiClientAsync(this.openAiClientAsync) + .options(options) + .build(); + + ChatResponse response = chatModel.stream(new Prompt("hi", options)).blockLast(); + + assertThat(response).isNotNull(); + AssistantMessage assistantMessage = response.getResult().getOutput(); + assertThat(assistantMessage.getToolCalls()).singleElement().satisfies(toolCall -> { + assertThat(toolCall.id()).isEqualTo("call_1"); + assertThat(toolCall.name()).isEqualTo("get_current_weather"); + assertThat(toolCall.arguments()).isEqualTo("{\"location\":\"Seoul\"}"); + }); + } + @Test void restoreUnmappedToolCallAdditionalProperties() { OpenAiChatOptions options = OpenAiChatOptions.builder().model("gemini-3.5-flash").build();