diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java index 1002452d3c..68267e1f76 100644 --- a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java @@ -1039,7 +1039,7 @@ private Prompt buildRequestPrompt(Prompt prompt) { } } - private static final class ChunkMerger { + static final class ChunkMerger { static boolean hasToolCall(ChatCompletionChunk chunk) { return !chunk.choices().isEmpty() @@ -1077,35 +1077,18 @@ private static Choice mergeChoices(Choice c1, Choice c2) { } private static Delta mergeDeltas(Delta left, Delta right) { + // Deltas of the same logical tool call share the required 'index' field. + // Some OpenAI-compatible providers (e.g. DeepSeek) send an empty-string id + // on continuation deltas instead of omitting it, so id presence cannot be + // used to detect the start of a new tool call. var tcs = Stream.of(left.toolCalls(), right.toolCalls()).flatMap(Optional::stream).reduce((tcs1, tcs2) -> { if (tcs2.isEmpty()) { return tcs1; } - Assert.isTrue(tcs2.size() == 1, "no more than one tool call per message currently supported"); - ToolCall toolCall = tcs2.get(0); - if (toolCall.id().isPresent()) { - List result = new ArrayList<>(tcs1); - result.add(toolCall); - return result; - } - else { - ToolCall lastFromTc1 = tcs1.get(tcs1.size() - 1); - Function lastFromTc1F = lastFromTc1.function().get(); - - var concatenatedArgs = Stream - .of(lastFromTc1F.arguments(), toolCall.function().flatMap(Function::arguments)) - .flatMap(Optional::stream) - .reduce((args1, args2) -> args1 + args2) - .orElse(""); - - List result = new ArrayList<>(tcs1); - result.set(tcs1.size() - 1, - lastFromTc1.toBuilder() - .putAllAdditionalProperties(toolCall._additionalProperties()) - .function(lastFromTc1F.toBuilder().arguments(concatenatedArgs).build()) - .build()); - return result; - } + Map mergedByIndex = new LinkedHashMap<>(); + tcs1.forEach(tc -> mergedByIndex.merge(tc.index(), tc, ChunkMerger::mergeToolCalls)); + tcs2.forEach(tc -> mergedByIndex.merge(tc.index(), tc, ChunkMerger::mergeToolCalls)); + return List.copyOf(mergedByIndex.values()); }).orElse(List.of()); Delta.Builder deltaBuilder = left.toBuilder().toolCalls(tcs); @@ -1124,6 +1107,28 @@ private static Optional stringProperty(Delta delta, String key) { return Optional.ofNullable(delta._additionalProperties().get(key)).flatMap(JsonValue::asString); } + private static ToolCall mergeToolCalls(ToolCall previous, ToolCall current) { + String arguments = Stream + .of(previous.function().flatMap(Function::arguments), current.function().flatMap(Function::arguments)) + .flatMap(Optional::stream) + .collect(Collectors.joining()); + return previous.toBuilder() + .id(firstWithText(previous.id(), current.id())) + .putAllAdditionalProperties(current._additionalProperties()) + .function(previous.function() + .map(Function::toBuilder) + .orElseGet(Function::builder) + .name(firstWithText(previous.function().flatMap(Function::name), + current.function().flatMap(Function::name))) + .arguments(arguments) + .build()) + .build(); + } + + private static String firstWithText(Optional first, Optional second) { + return first.filter(StringUtils::hasText).or(() -> second.filter(StringUtils::hasText)).orElse(""); + } + /** * Convert a ChatCompletionChunk into a ChatCompletion. */ @@ -1160,11 +1165,19 @@ static ChatCompletion chunkToChatCompletion(ChatCompletionChunk chunk) { msgBuilder.toolCalls(ccctcs.stream().map(tc -> { ChatCompletionMessageFunctionToolCall.Builder toolCallBuilder = ChatCompletionMessageFunctionToolCall .builder(); + Function function = tc.function() + .orElseThrow(() -> new IllegalStateException("Tool call function is missing")); + String id = tc.id() + .filter(StringUtils::hasText) + .orElseThrow(() -> new IllegalStateException("Tool call id is missing")); + String name = function.name() + .filter(StringUtils::hasText) + .orElseThrow(() -> new IllegalStateException("Tool call function name is missing")); toolCallBuilder.putAllAdditionalProperties(tc._additionalProperties()); - toolCallBuilder.id(tc.id().get()); + toolCallBuilder.id(id); toolCallBuilder.function(ChatCompletionMessageFunctionToolCall.Function.builder() - .name(tc.function().get().name().get()) - .arguments(tc.function().get().arguments().get()) + .name(name) + .arguments(function.arguments().orElse("")) .build()); return ChatCompletionMessageToolCall.ofFunction(toolCallBuilder.build()); }).toList()); diff --git a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelChunkMergerTests.java b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelChunkMergerTests.java new file mode 100644 index 0000000000..8413290c37 --- /dev/null +++ b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelChunkMergerTests.java @@ -0,0 +1,212 @@ +/* + * Copyright 2023-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.openai; + +import java.util.List; + +import com.openai.models.chat.completions.ChatCompletion; +import com.openai.models.chat.completions.ChatCompletionChunk; +import com.openai.models.chat.completions.ChatCompletionChunk.Choice; +import com.openai.models.chat.completions.ChatCompletionChunk.Choice.Delta; +import com.openai.models.chat.completions.ChatCompletionChunk.Choice.Delta.ToolCall; +import com.openai.models.chat.completions.ChatCompletionChunk.Choice.Delta.ToolCall.Function; +import com.openai.models.chat.completions.ChatCompletionChunk.Choice.FinishReason; +import com.openai.models.chat.completions.ChatCompletionMessageFunctionToolCall; +import org.jspecify.annotations.Nullable; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalStateException; + +/** + * Unit tests for {@link OpenAiChatModel.ChunkMerger}, covering the merging of streamed + * tool call deltas by their {@code index} field. OpenAI omits {@code id} and + * {@code function.name} on continuation deltas, while some OpenAI-compatible providers + * (e.g. DeepSeek) send them as empty strings instead. + * + * @author Jewoo Shin + */ +class OpenAiChatModelChunkMergerTests { + + @Test + void mergeToolCallDeltasWithAbsentContinuationIds() { + ChatCompletionChunk merged = OpenAiChatModel.ChunkMerger + .mergeChunks(List.of(chunk(null, toolCallDelta(0, "call_1", "get_weather", "")), + chunk(null, toolCallDelta(0, null, null, "{\"city\": ")), + chunk(null, toolCallDelta(0, null, null, "\"Seoul\"}")), finishChunk())); + + List toolCalls = merged.choices().get(0).delta().toolCalls().orElseThrow(); + assertThat(toolCalls).hasSize(1); + assertThat(toolCalls.get(0).id()).contains("call_1"); + assertThat(toolCalls.get(0).function().orElseThrow().name()).contains("get_weather"); + assertThat(toolCalls.get(0).function().orElseThrow().arguments()).contains("{\"city\": \"Seoul\"}"); + } + + @Test // gh-6374 + void mergeToolCallDeltasWithEmptyContinuationIds() { + ChatCompletionChunk merged = OpenAiChatModel.ChunkMerger + .mergeChunks(List.of(chunk(null, toolCallDelta(0, "call_1d70a7ee", "show_options", "")), + chunk(null, toolCallDelta(0, "", "", "{\"options\": [\"a\"")), + chunk(null, toolCallDelta(0, "", "", ", \"b\"]}")), finishChunk())); + + assertThat(merged.choices().get(0).finishReason()).contains(FinishReason.TOOL_CALLS); + List toolCalls = merged.choices().get(0).delta().toolCalls().orElseThrow(); + assertThat(toolCalls).hasSize(1); + assertThat(toolCalls.get(0).id()).contains("call_1d70a7ee"); + assertThat(toolCalls.get(0).function().orElseThrow().name()).contains("show_options"); + assertThat(toolCalls.get(0).function().orElseThrow().arguments()).contains("{\"options\": [\"a\", \"b\"]}"); + + ChatCompletion completion = OpenAiChatModel.ChunkMerger.chunkToChatCompletion(merged); + ChatCompletionMessageFunctionToolCall functionToolCall = completion.choices() + .get(0) + .message() + .toolCalls() + .orElseThrow() + .get(0) + .function() + .orElseThrow(); + assertThat(functionToolCall.id()).isEqualTo("call_1d70a7ee"); + assertThat(functionToolCall.function().name()).isEqualTo("show_options"); + assertThat(functionToolCall.function().arguments()).isEqualTo("{\"options\": [\"a\", \"b\"]}"); + } + + @Test + void mergeParallelToolCallDeltasByIndex() { + ChatCompletionChunk merged = OpenAiChatModel.ChunkMerger + .mergeChunks(List.of(chunk(null, toolCallDelta(0, "call_a", "tool_a", "")), + chunk(null, toolCallDelta(1, "call_b", "tool_b", "")), + chunk(null, toolCallDelta(0, null, null, "{\"a\": 1}")), + chunk(null, toolCallDelta(1, null, null, "{\"b\": 2}")), finishChunk())); + + List toolCalls = merged.choices().get(0).delta().toolCalls().orElseThrow(); + assertThat(toolCalls).hasSize(2); + assertThat(toolCalls.get(0).id()).contains("call_a"); + assertThat(toolCalls.get(0).function().orElseThrow().arguments()).contains("{\"a\": 1}"); + assertThat(toolCalls.get(1).id()).contains("call_b"); + assertThat(toolCalls.get(1).function().orElseThrow().arguments()).contains("{\"b\": 2}"); + } + + @Test + void mergeMultipleToolCallDeltasInSingleChunk() { + ChatCompletionChunk merged = OpenAiChatModel.ChunkMerger.mergeChunks(List.of( + chunk(null, toolCallDelta(0, "call_a", "tool_a", ""), toolCallDelta(1, "call_b", "tool_b", "")), + chunk(null, toolCallDelta(0, null, null, "{\"a\": 1}"), toolCallDelta(1, null, null, "{\"b\": 2}")), + finishChunk())); + + List toolCalls = merged.choices().get(0).delta().toolCalls().orElseThrow(); + assertThat(toolCalls).hasSize(2); + assertThat(toolCalls.get(0).function().orElseThrow().name()).contains("tool_a"); + assertThat(toolCalls.get(0).function().orElseThrow().arguments()).contains("{\"a\": 1}"); + assertThat(toolCalls.get(1).function().orElseThrow().name()).contains("tool_b"); + assertThat(toolCalls.get(1).function().orElseThrow().arguments()).contains("{\"b\": 2}"); + } + + @Test + void chunkToChatCompletionRequiresToolCallId() { + ChatCompletionChunk chunk = chunk(FinishReason.TOOL_CALLS, + toolCallDelta(0, null, "get_weather", "{\"city\":\"Seoul\"}")); + ChatCompletionChunk blankIdChunk = chunk(FinishReason.TOOL_CALLS, + toolCallDelta(0, " ", "get_weather", "{\"city\":\"Seoul\"}")); + + assertThatIllegalStateException() + .isThrownBy(() -> OpenAiChatModel.ChunkMerger.chunkToChatCompletion(chunk)) + .withMessage("Tool call id is missing"); + assertThatIllegalStateException() + .isThrownBy(() -> OpenAiChatModel.ChunkMerger.chunkToChatCompletion(blankIdChunk)) + .withMessage("Tool call id is missing"); + } + + @Test + void chunkToChatCompletionRequiresToolCallFunction() { + ChatCompletionChunk chunk = chunk(FinishReason.TOOL_CALLS, toolCallDeltaWithoutFunction(0, "call_1")); + + assertThatIllegalStateException().isThrownBy(() -> OpenAiChatModel.ChunkMerger.chunkToChatCompletion(chunk)) + .withMessage("Tool call function is missing"); + } + + @Test + void chunkToChatCompletionRequiresToolCallFunctionName() { + ChatCompletionChunk chunk = chunk(FinishReason.TOOL_CALLS, + toolCallDelta(0, "call_1", null, "{\"city\":\"Seoul\"}")); + ChatCompletionChunk blankNameChunk = chunk(FinishReason.TOOL_CALLS, + toolCallDelta(0, "call_1", " ", "{\"city\":\"Seoul\"}")); + + assertThatIllegalStateException().isThrownBy(() -> OpenAiChatModel.ChunkMerger.chunkToChatCompletion(chunk)) + .withMessage("Tool call function name is missing"); + assertThatIllegalStateException() + .isThrownBy(() -> OpenAiChatModel.ChunkMerger.chunkToChatCompletion(blankNameChunk)) + .withMessage("Tool call function name is missing"); + } + + @Test + void chunkToChatCompletionUsesEmptyDefaultForMissingArguments() { + ChatCompletionChunk chunk = chunk(FinishReason.TOOL_CALLS, + toolCallDeltaWithoutArguments(0, "call_1", "get_weather")); + + ChatCompletion completion = OpenAiChatModel.ChunkMerger.chunkToChatCompletion(chunk); + + ChatCompletionMessageFunctionToolCall functionToolCall = completion.choices() + .get(0) + .message() + .toolCalls() + .orElseThrow() + .get(0) + .function() + .orElseThrow(); + assertThat(functionToolCall.id()).isEqualTo("call_1"); + assertThat(functionToolCall.function().name()).isEqualTo("get_weather"); + assertThat(functionToolCall.function().arguments()).isEmpty(); + } + + private static ChatCompletionChunk chunk(@Nullable FinishReason finishReason, ToolCall... toolCalls) { + Delta.Builder delta = Delta.builder(); + if (toolCalls.length > 0) { + delta.toolCalls(List.of(toolCalls)); + } + return ChatCompletionChunk.builder() + .id("chatcmpl-test") + .choices(List.of(Choice.builder().index(0L).delta(delta.build()).finishReason(finishReason).build())) + .created(1L) + .model("test-model") + .build(); + } + + private static ChatCompletionChunk finishChunk() { + return chunk(FinishReason.TOOL_CALLS); + } + + private static ToolCall toolCallDelta(long index, @Nullable String id, @Nullable String name, String arguments) { + Function.Builder function = Function.builder().arguments(arguments); + if (name != null) { + function.name(name); + } + ToolCall.Builder toolCall = ToolCall.builder().index(index).function(function.build()); + if (id != null) { + toolCall.id(id); + } + return toolCall.build(); + } + + private static ToolCall toolCallDeltaWithoutArguments(long index, String id, String name) { + return ToolCall.builder().index(index).id(id).function(Function.builder().name(name).build()).build(); + } + + private static ToolCall toolCallDeltaWithoutFunction(long index, String id) { + return ToolCall.builder().index(index).id(id).build(); + } + +} diff --git a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelTests.java b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelTests.java index df79987d30..7ebf618956 100644 --- a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelTests.java +++ b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelTests.java @@ -270,6 +270,88 @@ void preserveUnmappedToolCallAdditionalPropertiesFromStream() { Map.of("google", Map.of("thought_signature", "signature-123"))); } + @Test + void mergeStreamToolCallWhenIdNameAndArgumentsArriveInSeparateChunks() { + ChatServiceAsync chatServiceAsync = mock(ChatServiceAsync.class); + ChatCompletionServiceAsync chatCompletionServiceAsync = mock(ChatCompletionServiceAsync.class); + when(this.openAiClientAsync.chat()).thenReturn(chatServiceAsync); + when(chatServiceAsync.completions()).thenReturn(chatCompletionServiceAsync); + when(chatCompletionServiceAsync.createStreaming(any(ChatCompletionCreateParams.class))) + .thenReturn(asyncStreamResponse(ChatCompletionChunk.builder() + .id("chatcmpl-stream-test") + .created(1777799928) + .model("test-model") + .addChoice(ChatCompletionChunk.Choice.builder() + .index(0) + .finishReason(Optional.empty()) + .delta(ChatCompletionChunk.Choice.Delta.builder() + .addToolCall(ChatCompletionChunk.Choice.Delta.ToolCall.builder().index(0).id("call_1").build()) + .build()) + .build()) + .build(), + ChatCompletionChunk.builder() + .id("chatcmpl-stream-test") + .created(1777799928) + .model("test-model") + .addChoice(ChatCompletionChunk.Choice.builder() + .index(0) + .finishReason(Optional.empty()) + .delta(ChatCompletionChunk.Choice.Delta.builder() + .addToolCall(ChatCompletionChunk.Choice.Delta.ToolCall.builder() + .index(0) + .function(ChatCompletionChunk.Choice.Delta.ToolCall.Function.builder() + .name("get_current_weather") + .build()) + .build()) + .build()) + .build()) + .build(), + ChatCompletionChunk.builder() + .id("chatcmpl-stream-test") + .created(1777799928) + .model("test-model") + .addChoice(ChatCompletionChunk.Choice.builder() + .index(0) + .finishReason(Optional.empty()) + .delta(ChatCompletionChunk.Choice.Delta.builder() + .addToolCall(ChatCompletionChunk.Choice.Delta.ToolCall.builder() + .index(0) + .function(ChatCompletionChunk.Choice.Delta.ToolCall.Function.builder() + .arguments("{\"location\":\"Seoul\"}") + .build()) + .build()) + .build()) + .build()) + .build(), + ChatCompletionChunk.builder() + .id("chatcmpl-stream-test") + .created(1777799928) + .model("test-model") + .addChoice(ChatCompletionChunk.Choice.builder() + .index(0) + .finishReason(ChatCompletionChunk.Choice.FinishReason.TOOL_CALLS) + .delta(ChatCompletionChunk.Choice.Delta.builder().build()) + .build()) + .build())); + + OpenAiChatOptions options = OpenAiChatOptions.builder().model("test-model").build(); + OpenAiChatModel chatModel = OpenAiChatModel.builder() + .openAiClient(this.openAiClient) + .openAiClientAsync(this.openAiClientAsync) + .options(options) + .build(); + + ChatResponse response = chatModel.stream(new Prompt("hi", options)).blockLast(); + + assertThat(response).isNotNull(); + AssistantMessage assistantMessage = response.getResult().getOutput(); + assertThat(assistantMessage.getToolCalls()).singleElement().satisfies(toolCall -> { + assertThat(toolCall.id()).isEqualTo("call_1"); + assertThat(toolCall.name()).isEqualTo("get_current_weather"); + assertThat(toolCall.arguments()).isEqualTo("{\"location\":\"Seoul\"}"); + }); + } + @Test void restoreUnmappedToolCallAdditionalProperties() { OpenAiChatOptions options = OpenAiChatOptions.builder().model("gemini-3.5-flash").build();