diff --git a/inception/inception-assistant/src/main/java/de/tudarmstadt/ukp/inception/assistant/config/AssistantAutoConfiguration.java b/inception/inception-assistant/src/main/java/de/tudarmstadt/ukp/inception/assistant/config/AssistantAutoConfiguration.java index 5c6e3eca08..3d64f5b8ed 100644 --- a/inception/inception-assistant/src/main/java/de/tudarmstadt/ukp/inception/assistant/config/AssistantAutoConfiguration.java +++ b/inception/inception-assistant/src/main/java/de/tudarmstadt/ukp/inception/assistant/config/AssistantAutoConfiguration.java @@ -51,6 +51,7 @@ import de.tudarmstadt.ukp.inception.documents.api.DocumentService; import de.tudarmstadt.ukp.inception.documents.api.RepositoryProperties; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ToolLibraryExtensionPoint; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClientExtensionPoint; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaClient; import de.tudarmstadt.ukp.inception.scheduling.SchedulingService; @@ -87,9 +88,9 @@ public EncodingRegistry encodingRegistry() @Bean public EmbeddingService EmbeddingService(AssistantProperties aProperties, - OllamaClient aOllamaClient) + LlmChatClientExtensionPoint aChatClientExtensionPoint) { - return new EmbeddingServiceImpl(aProperties, aOllamaClient); + return new EmbeddingServiceImpl(aProperties, aChatClientExtensionPoint); } @Bean diff --git a/inception/inception-assistant/src/main/java/de/tudarmstadt/ukp/inception/assistant/embedding/EmbeddingServiceImpl.java b/inception/inception-assistant/src/main/java/de/tudarmstadt/ukp/inception/assistant/embedding/EmbeddingServiceImpl.java index e67ad2b97c..685b154994 100644 --- a/inception/inception-assistant/src/main/java/de/tudarmstadt/ukp/inception/assistant/embedding/EmbeddingServiceImpl.java +++ b/inception/inception-assistant/src/main/java/de/tudarmstadt/ukp/inception/assistant/embedding/EmbeddingServiceImpl.java @@ -22,7 +22,9 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.function.Function; @@ -33,22 +35,27 @@ import org.springframework.context.event.EventListener; import de.tudarmstadt.ukp.inception.assistant.config.AssistantProperties; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaClient; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaEmbedRequest; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaOptions; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClient; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClientExtensionPoint; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmEndpoint; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaLlmChatClient; public class EmbeddingServiceImpl implements EmbeddingService { private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final String OPT_NUM_CTX = "num_ctx"; + private static final String OPT_SEED = "seed"; + private final AssistantProperties properties; - private final OllamaClient ollamaClient; + private final LlmChatClientExtensionPoint chatClientExtensionPoint; - public EmbeddingServiceImpl(AssistantProperties aProperties, OllamaClient aOllamaClient) + public EmbeddingServiceImpl(AssistantProperties aProperties, + LlmChatClientExtensionPoint aChatClientExtensionPoint) { properties = aProperties; - ollamaClient = aOllamaClient; + chatClientExtensionPoint = aChatClientExtensionPoint; } @EventListener @@ -96,23 +103,11 @@ public List> embed(Function aExtractor, Iterable objects.add(o); } - var request = OllamaEmbedRequest.builder() // - .withModel(properties.getEmbedding().getModel()) // - .withInput(strings.toArray(String[]::new)) // - .withOption(OllamaOptions.NUM_CTX, properties.getEmbedding().getContextLength()) // - .withOption(OllamaOptions.SEED, properties.getEmbedding().getSeed()) // - // The following options should not be relevant for embeddings - // .withOption(OllamaOptions.TEMPERATURE, 0.0) // - // .withOption(OllamaOptions.TOP_P, 0.0) // - // .withOption(OllamaOptions.TOP_K, 0) // - // .withOption(OllamaOptions.REPEAT_PENALTY, 1.0) // - .build(); - - var response = ollamaClient.embed(properties.getUrl(), request); + var vectors = client().embed(endpoint(), strings, embeddingOptions()); var result = new ArrayList>(); - for (var i = 0; i < response.size(); i++) { - result.add(Pair.of(objects.get(i), response.get(i).getValue())); + for (var i = 0; i < vectors.size(); i++) { + result.add(Pair.of(objects.get(i), vectors.get(i))); } return result; } @@ -124,7 +119,6 @@ public List> embed(String... aStrings) throws IOException var strings = new ArrayList(); for (var s : aStrings) { - s = removeEmptyLinesAndTrim(s); if (s.isEmpty() || hasHighProportionOfShortSequences(s) @@ -135,18 +129,37 @@ public List> embed(String... aStrings) throws IOException strings.add(s); } - var request = OllamaEmbedRequest.builder() // - .withModel(properties.getEmbedding().getModel()) // - .withInput(strings.toArray(String[]::new)) // - .withOption(OllamaOptions.NUM_CTX, properties.getEmbedding().getContextLength()) // - .withOption(OllamaOptions.SEED, properties.getEmbedding().getSeed()) // - // The following options should not be relevant for embeddings - // .withOption(OllamaOptions.TEMPERATURE, 0.0) // - // .withOption(OllamaOptions.TOP_P, 0.0) // - // .withOption(OllamaOptions.TOP_K, 0) // - // .withOption(OllamaOptions.REPEAT_PENALTY, 1.0) // - .build(); - return ollamaClient.embed(properties.getUrl(), request); + var vectors = client().embed(endpoint(), strings, embeddingOptions()); + + var result = new ArrayList>(); + for (var i = 0; i < vectors.size(); i++) { + result.add(Pair.of(strings.get(i), vectors.get(i))); + } + return result; + } + + private LlmChatClient client() + { + // Provider is hardcoded to Ollama for now; once assistant config moves to UI-driven + // traits, this becomes traits.getProviderId(). + return chatClientExtensionPoint.getExtension(OllamaLlmChatClient.ID) // + .orElseThrow(() -> new IllegalStateException( + "Ollama LLM client not registered — is the inception-imls-ollama module on " + + "the classpath?")); + } + + private LlmEndpoint endpoint() + { + return new LlmEndpoint(OllamaLlmChatClient.ID, properties.getUrl(), + properties.getEmbedding().getModel(), null); + } + + private Map embeddingOptions() + { + var options = new LinkedHashMap(); + options.put(OPT_NUM_CTX, properties.getEmbedding().getContextLength()); + options.put(OPT_SEED, properties.getEmbedding().getSeed()); + return options; } private void autoDetectEmbeddingDimension() @@ -157,15 +170,14 @@ private void autoDetectEmbeddingDimension() try { LOG.info("Contacting [{}] to auto-detect dimension of model [{}]...", properties.getUrl(), embeddingProperties.getModel()); - var embedding = ollamaClient.embed(properties.getUrl(), OllamaEmbedRequest - .builder() // - .withModel(embeddingProperties.getModel()) // - .withInput( - "We just need to know the dimension of the generated embedding. Thanks!") // - .build()).get(0).getValue(); - embeddingProperties.setDimension(embedding.length); + var vectors = client().embed(endpoint(), + List.of("We just need to know the dimension of the generated " + + "embedding. Thanks!"), + null); + var dim = vectors.get(0).length; + embeddingProperties.setDimension(dim); LOG.info("Auto-detected embedding dimension of model [{}]: {}", - embeddingProperties.getModel(), embeddingProperties.getDimension()); + embeddingProperties.getModel(), dim); } catch (Exception e) { if (LOG.isDebugEnabled()) { @@ -229,5 +241,4 @@ static boolean hasHighProportionOfWhitespaceOrLineBreaks(String aString) double proportion = (double) whitespaceOrLineBreakCount / totalChars; return proportion > 0.5; } - } diff --git a/inception/inception-assistant/src/main/java/de/tudarmstadt/ukp/inception/assistant/recommender/AssistantRecommenderFactory.java b/inception/inception-assistant/src/main/java/de/tudarmstadt/ukp/inception/assistant/recommender/AssistantRecommenderFactory.java index ebda13adf9..6b72fb84ba 100644 --- a/inception/inception-assistant/src/main/java/de/tudarmstadt/ukp/inception/assistant/recommender/AssistantRecommenderFactory.java +++ b/inception/inception-assistant/src/main/java/de/tudarmstadt/ukp/inception/assistant/recommender/AssistantRecommenderFactory.java @@ -62,6 +62,15 @@ public boolean isEvaluable() return false; } + @Override + public boolean isDeprecated() + { + // Hide from the recommender-tool dropdown: the assistant recommender is not + // user-configurable + // - its instance is created/managed by the assistant subsystem. + return true; + } + @Override public RecommendationEngine build(Recommender aRecommender) { diff --git a/inception/inception-assistant/src/main/java/de/tudarmstadt/ukp/inception/assistant/tools/AssistantRuntimeContext.java b/inception/inception-assistant/src/main/java/de/tudarmstadt/ukp/inception/assistant/tools/AssistantRuntimeContext.java new file mode 100644 index 0000000000..e653c0ef52 --- /dev/null +++ b/inception/inception-assistant/src/main/java/de/tudarmstadt/ukp/inception/assistant/tools/AssistantRuntimeContext.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.assistant.tools; + +import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; +import de.tudarmstadt.ukp.clarin.webanno.security.model.User; +import de.tudarmstadt.ukp.inception.assistant.CommandDispatcher; + +/** + * Snapshot of the assistant's per-chat-turn runtime state, captured into each + * {@link AssistantToolInvoker} when the tool registry is built. Any field may be {@code null} when + * the surrounding chat session does not have that context (e.g. no document open). + */ +public record AssistantRuntimeContext( // + User sessionOwner, // + Project project, // + SourceDocument document, // + String dataOwner, // + CommandDispatcher commandDispatcher) +{} diff --git a/inception/inception-assistant/src/main/java/de/tudarmstadt/ukp/inception/assistant/tools/AssistantToolInvoker.java b/inception/inception-assistant/src/main/java/de/tudarmstadt/ukp/inception/assistant/tools/AssistantToolInvoker.java new file mode 100644 index 0000000000..4209fe68ac --- /dev/null +++ b/inception/inception-assistant/src/main/java/de/tudarmstadt/ukp/inception/assistant/tools/AssistantToolInvoker.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.assistant.tools; + +import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.ToolUtils.getParameterName; +import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.ToolUtils.isParameter; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; + +import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; +import de.tudarmstadt.ukp.inception.assistant.CommandDispatcher; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.AnnotationEditorContext; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ToolInvoker; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ToolDescriptor; +import de.tudarmstadt.ukp.inception.support.json.JSONUtil; +import tools.jackson.databind.JsonNode; + +/** + * {@link ToolInvoker} backed by a {@code @Tool}-annotated Java {@link Method}, with binding for the + * assistant's runtime context. Captures the per-turn {@link AssistantRuntimeContext} at + * construction so {@link #invoke} is a pure (arguments) → result call. + *

+ * Parameter binding mirrors what {@code MToolCall.invoke} did pre-abstraction: + *

    + *
  • {@code @ToolParam} parameters → Jackson-converted from the LLM-supplied JSON arguments. + *
  • {@link AnnotationEditorContext} → built per-call from the captured runtime context. + *
  • {@link Project} / {@link SourceDocument} / {@link CommandDispatcher} → captured runtime + * context. + *
  • Anything else → {@link IllegalStateException} at invocation time. + *
+ * Exceptions thrown by the target method are unwrapped from {@link InvocationTargetException} so + * callers see the original cause. + */ +public class AssistantToolInvoker + implements ToolInvoker +{ + private final Object instance; + private final Method method; + private final ToolDescriptor descriptor; + private final AssistantRuntimeContext context; + + public AssistantToolInvoker(Object aInstance, Method aMethod, AssistantRuntimeContext aContext) + { + instance = aInstance; + method = aMethod; + descriptor = ToolDescriptor.fromMethod(aMethod); + context = aContext; + } + + @Override + public ToolDescriptor descriptor() + { + return descriptor; + } + + @Override + public Object invoke(JsonNode aArguments) throws Exception + { + var mapper = JSONUtil.getObjectMapper(); + var typeFactory = mapper.getTypeFactory(); + var args = new ArrayList<>(); + + for (var param : method.getParameters()) { + if (isParameter(param)) { + var paramName = getParameterName(param); + var raw = aArguments != null ? aArguments.get(paramName) : null; + var type = typeFactory.constructType(param.getParameterizedType()); + args.add(mapper.convertValue(raw, type)); + continue; + } + + args.add(resolveContextParameter(param.getType(), param.getName())); + } + + try { + return method.invoke(instance, args.toArray()); + } + catch (InvocationTargetException e) { + if (e.getCause() instanceof Exception cause) { + throw cause; + } + throw e; + } + } + + private Object resolveContextParameter(Class aType, String aParamName) + { + // Strict direction: the parameter type IS-A injectable type. Using the loose direction + // (param.getType().isAssignableFrom(KnownType.class)) — as the pre-abstraction + // MToolCall.invoke did — silently matched parameters declared as Object or other + // supertypes against whichever known type appeared first in the chain. + if (AnnotationEditorContext.class.isAssignableFrom(aType)) { + return AnnotationEditorContext.builder() // + .withSessionOwner(context.sessionOwner()) // + .withProject(context.project()) // + .withDocument(context.document()) // + .withDataOwner(context.dataOwner()) // + .build(); + } + if (CommandDispatcher.class.isAssignableFrom(aType)) { + return context.commandDispatcher(); + } + if (Project.class.isAssignableFrom(aType)) { + return context.project(); + } + if (SourceDocument.class.isAssignableFrom(aType)) { + return context.document(); + } + throw new IllegalStateException("Tool [" + descriptor.name() + "] declares parameter [" + + aParamName + "] of unsupported type [" + aType.getName() + + "]. Supported context types are: AnnotationEditorContext, " + + "CommandDispatcher, Project, SourceDocument."); + } + + @Override + public String toString() + { + return "AssistantToolInvoker[" + descriptor.name() + " -> " + method.toGenericString() + + "]"; + } +} diff --git a/inception/inception-assistant/src/test/java/de/tudarmstadt/ukp/inception/assistant/tools/AssistantToolInvokerTest.java b/inception/inception-assistant/src/test/java/de/tudarmstadt/ukp/inception/assistant/tools/AssistantToolInvokerTest.java new file mode 100644 index 0000000000..51523d2db7 --- /dev/null +++ b/inception/inception-assistant/src/test/java/de/tudarmstadt/ukp/inception/assistant/tools/AssistantToolInvokerTest.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.assistant.tools; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalStateException; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; + +import java.lang.reflect.Method; + +import org.junit.jupiter.api.Test; + +import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; +import de.tudarmstadt.ukp.clarin.webanno.security.model.User; +import de.tudarmstadt.ukp.inception.assistant.CommandDispatcher; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.AnnotationEditorContext; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.Tool; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ToolParam; +import de.tudarmstadt.ukp.inception.support.json.JSONUtil; + +class AssistantToolInvokerTest +{ + static class Tools + { + Object[] received; + + @Tool(value = "echo", description = "Returns its input.") + public String echo(@ToolParam(value = "text", description = "what to echo") String aText) + { + received = new Object[] { aText }; + return aText; + } + + @Tool(value = "current_project", description = "Returns the current project name.") + public String currentProject(Project aProject) + { + received = new Object[] { aProject }; + return aProject.getName(); + } + + @Tool(value = "current_document", description = "Returns the current document name.") + public String currentDocument(SourceDocument aDocument) + { + received = new Object[] { aDocument }; + return aDocument.getName(); + } + + @Tool(value = "dispatch", description = "Has access to the command dispatcher.") + public String dispatch(CommandDispatcher aDispatcher) + { + received = new Object[] { aDispatcher }; + return aDispatcher.getClass().getSimpleName(); + } + + @Tool(value = "editor", description = "Receives an AnnotationEditorContext.") + public String editor(AnnotationEditorContext aEditorContext) + { + received = new Object[] { aEditorContext }; + return aEditorContext.getProject() != null ? aEditorContext.getProject().getName() + : "no-project"; + } + + @Tool(value = "mixed", description = "Mix of model args and runtime injection.") + public String mixed(@ToolParam(value = "prefix", description = "prefix") String aPrefix, + Project aProject) + { + received = new Object[] { aPrefix, aProject }; + return aPrefix + ":" + aProject.getName(); + } + + @Tool(value = "unsupported", description = "Declares an unsupported param.") + public String unsupported(Object aOpaque) + { + return aOpaque.toString(); + } + + @Tool(value = "boom", description = "Throws.") + public String boom(@ToolParam(value = "msg", description = "msg") String aMsg) + { + throw new IllegalArgumentException(aMsg); + } + } + + private static Method methodNamed(String aName) + { + for (var m : Tools.class.getDeclaredMethods()) { + if (m.getName().equals(aName)) { + return m; + } + } + throw new AssertionError("No method " + aName); + } + + private static AssistantRuntimeContext ctx(Project aProject, SourceDocument aDocument, + CommandDispatcher aDispatcher) + { + return new AssistantRuntimeContext(new User("alice"), aProject, aDocument, "alice", + aDispatcher); + } + + @Test + void descriptorIsBuiltFromMethod() + { + var sut = new AssistantToolInvoker(new Tools(), methodNamed("echo"), ctx(null, null, null)); + assertThat(sut.descriptor().name()).isEqualTo("echo"); + assertThat(sut.descriptor().description()).contains("input"); + } + + @Test + void invokeBindsToolParamsFromArguments() throws Exception + { + var tools = new Tools(); + var sut = new AssistantToolInvoker(tools, methodNamed("echo"), ctx(null, null, null)); + var args = JSONUtil.getObjectMapper().createObjectNode().put("text", "hello"); + + assertThat(sut.invoke(args)).isEqualTo("hello"); + assertThat(tools.received).containsExactly("hello"); + } + + @Test + void invokeInjectsProjectFromContext() throws Exception + { + var project = new Project(); + project.setName("demo"); + var sut = new AssistantToolInvoker(new Tools(), methodNamed("currentProject"), + ctx(project, null, null)); + + assertThat(sut.invoke(null)).isEqualTo("demo"); + } + + @Test + void invokeInjectsSourceDocumentFromContext() throws Exception + { + var doc = new SourceDocument(); + doc.setName("doc-1"); + var sut = new AssistantToolInvoker(new Tools(), methodNamed("currentDocument"), + ctx(null, doc, null)); + + assertThat(sut.invoke(null)).isEqualTo("doc-1"); + } + + @Test + void invokeInjectsCommandDispatcherFromContext() throws Exception + { + var dispatcher = mock(CommandDispatcher.class); + var sut = new AssistantToolInvoker(new Tools(), methodNamed("dispatch"), + ctx(null, null, dispatcher)); + + // Mockito gives back a generated subclass — its simple name reflects that. + assertThat((String) sut.invoke(null)).startsWith("CommandDispatcher"); + } + + @Test + void invokeBuildsAnnotationEditorContextFromRuntimeContext() throws Exception + { + var project = new Project(); + project.setName("ctx-project"); + var sut = new AssistantToolInvoker(new Tools(), methodNamed("editor"), + ctx(project, null, null)); + + assertThat(sut.invoke(null)).isEqualTo("ctx-project"); + } + + @Test + void invokeMixesToolParamsAndRuntimeInjection() throws Exception + { + var project = new Project(); + project.setName("mix"); + var sut = new AssistantToolInvoker(new Tools(), methodNamed("mixed"), + ctx(project, null, null)); + var args = JSONUtil.getObjectMapper().createObjectNode().put("prefix", "hi"); + + assertThat(sut.invoke(args)).isEqualTo("hi:mix"); + } + + @Test + void unsupportedParameterTypeFailsWithClearMessage() + { + var sut = new AssistantToolInvoker(new Tools(), methodNamed("unsupported"), + ctx(null, null, null)); + + assertThatIllegalStateException() // + .isThrownBy(() -> sut.invoke(null)) // + .withMessageContaining("unsupported") // + .withMessageContaining(Object.class.getName()); + } + + @Test + void targetExceptionIsUnwrapped() + { + var sut = new AssistantToolInvoker(new Tools(), methodNamed("boom"), ctx(null, null, null)); + var args = JSONUtil.getObjectMapper().createObjectNode().put("msg", "kapow"); + + assertThatThrownBy(() -> sut.invoke(args)) // + .isInstanceOf(IllegalArgumentException.class) // + .hasMessage("kapow"); + } +} diff --git a/inception/inception-assistant/src/test/java/de/tudarmstadt/ukp/inception/assistant/userguide/UserGuideQueryServiceImplTest.java b/inception/inception-assistant/src/test/java/de/tudarmstadt/ukp/inception/assistant/userguide/UserGuideQueryServiceImplTest.java index 25b7dd1636..5a6f9f6cab 100644 --- a/inception/inception-assistant/src/test/java/de/tudarmstadt/ukp/inception/assistant/userguide/UserGuideQueryServiceImplTest.java +++ b/inception/inception-assistant/src/test/java/de/tudarmstadt/ukp/inception/assistant/userguide/UserGuideQueryServiceImplTest.java @@ -23,6 +23,7 @@ import java.lang.invoke.MethodHandles; import java.nio.file.Path; +import java.util.List; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -39,8 +40,9 @@ import de.tudarmstadt.ukp.inception.assistant.config.AssistantDocumentIndexPropertiesImpl; import de.tudarmstadt.ukp.inception.assistant.config.AssistantPropertiesImpl; import de.tudarmstadt.ukp.inception.assistant.embedding.EmbeddingServiceImpl; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaClient; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClientExtensionPointImpl; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaClientImpl; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaLlmChatClient; import de.tudarmstadt.ukp.inception.scheduling.SchedulingService; import de.tudarmstadt.ukp.inception.support.test.http.HttpTestUtils; @@ -52,7 +54,7 @@ class UserGuideQueryServiceImplTest private @Mock SchedulingService schedulingService; private AssistantPropertiesImpl assistantProperties; private AssistantDocumentIndexProperties assistantDocumentIndexProperties; - private OllamaClient ollamaClient; + private LlmChatClientExtensionPointImpl chatClientExtensionPoint; private UserGuideQueryServiceImpl sut; private EmbeddingServiceImpl embeddingService; @@ -71,8 +73,12 @@ void setup() assistantDocumentIndexProperties = new AssistantDocumentIndexPropertiesImpl(); assistantProperties = new AssistantPropertiesImpl(); assistantProperties.setDocumentIndex(assistantDocumentIndexProperties); - ollamaClient = new OllamaClientImpl(); - embeddingService = new EmbeddingServiceImpl(assistantProperties, ollamaClient); + + var ollamaAdapter = new OllamaLlmChatClient(new OllamaClientImpl()); + chatClientExtensionPoint = new LlmChatClientExtensionPointImpl(List.of(ollamaAdapter)); + chatClientExtensionPoint.init(); + + embeddingService = new EmbeddingServiceImpl(assistantProperties, chatClientExtensionPoint); sut = new UserGuideQueryServiceImpl(assistantProperties, schedulingService, embeddingService); } diff --git a/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/EditorAjaxRequestHandlerExtensionPointImpl.java b/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/EditorAjaxRequestHandlerExtensionPointImpl.java index 1cccdb5a68..8625469544 100644 --- a/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/EditorAjaxRequestHandlerExtensionPointImpl.java +++ b/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/EditorAjaxRequestHandlerExtensionPointImpl.java @@ -51,6 +51,15 @@ public Optional getHandler(Request aRequest) .findFirst(); } + @Override + protected boolean enforceUniqueIds() + { + // Multiple handlers intentionally share a command id (e.g. SelectAnnotationHandler and + // FillSlotWithExistingAnnotationHandler both register as "selectAnnotation"); accepts() + // + @Order discriminate between them at dispatch time. + return false; + } + @Override public List getExtensions(Request aContext) { diff --git a/inception/inception-imls-azureai-openai/pom.xml b/inception/inception-imls-azureai-openai/pom.xml index ea034b5cc9..5bd5d026c8 100644 --- a/inception/inception-imls-azureai-openai/pom.xml +++ b/inception/inception-imls-azureai-openai/pom.xml @@ -113,7 +113,8 @@ com.fasterxml.jackson.core jackson-annotations - + + tools.jackson.core jackson-databind @@ -121,6 +122,7 @@ org.slf4j slf4j-api + test - \ No newline at end of file + diff --git a/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/azureaiopenai/AzureAiOpenAiRecommender.java b/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/azureaiopenai/AzureAiOpenAiRecommender.java index 3f50497c06..0667bcbed7 100644 --- a/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/azureaiopenai/AzureAiOpenAiRecommender.java +++ b/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/azureaiopenai/AzureAiOpenAiRecommender.java @@ -17,95 +17,28 @@ */ package de.tudarmstadt.ukp.inception.recommendation.imls.llm.azureaiopenai; -import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.azureaiopenai.client.AzureAiResponseFormatType.JSON_OBJECT; -import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.azureaiopenai.client.AzureAiResponseFormatType.JSON_SCHEMA; -import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.response.ResponseFormat.JSON; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import de.tudarmstadt.ukp.inception.recommendation.api.model.Recommender; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.AnnotationTaskCodecExtensionPoint; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ChatBasedLlmRecommenderImplBase; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ChatMessage; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.azureaiopenai.client.AzureAiChatCompletionMessage; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.azureaiopenai.client.AzureAiChatCompletionRequest; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.azureaiopenai.client.AzureAiGenerateResponseFormat; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.azureaiopenai.client.AzureAiOpenAiClient; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.response.ResponseFormat; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.azureaiopenai.client.AzureAiOpenAiLlmChatClient; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClientExtensionPoint; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; -import de.tudarmstadt.ukp.inception.security.client.auth.apikey.ApiKeyAuthenticationTraits; -import tools.jackson.databind.JsonNode; public class AzureAiOpenAiRecommender extends ChatBasedLlmRecommenderImplBase { - private final static Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private final AzureAiOpenAiClient client; - public AzureAiOpenAiRecommender(Recommender aRecommender, - AzureAiOpenAiRecommenderTraits aTraits, AzureAiOpenAiClient aClient, - AnnotationSchemaService aSchemaService, - AnnotationTaskCodecExtensionPoint aResponseExtractorExtensionPoint) + AzureAiOpenAiRecommenderTraits aTraits, AnnotationSchemaService aSchemaService, + AnnotationTaskCodecExtensionPoint aResponseExtractorExtensionPoint, + LlmChatClientExtensionPoint aChatClientExtensionPoint) { - super(aRecommender, aTraits, aSchemaService, aResponseExtractorExtensionPoint); - - client = aClient; + super(aRecommender, aTraits, aSchemaService, aResponseExtractorExtensionPoint, + aChatClientExtensionPoint); } @Override - protected String exchange(List aMessages, ResponseFormat aResponseformat, - JsonNode aJsonSchema) - throws IOException + protected String getProviderId() { - var format = getResponseFormat(aResponseformat, aJsonSchema); - - var messages = aMessages.stream() // - .map(m -> new AzureAiChatCompletionMessage(m.role().getName(), m.content())) // - .toList(); - - LOG.trace("Querying Azure AI OpenAI: [{}]", aMessages); - var request = AzureAiChatCompletionRequest.builder() // - .withApiKey(((ApiKeyAuthenticationTraits) traits.getAuthentication()).getApiKey()) // - .withMessages(messages) // - .withFormat(format); - - var options = traits.getOptions(); - // https://platform.openai.com/docs/api-reference/chat/create recommends to set temperature - // or top_p but not both. - if (!options.containsKey(AzureAiChatCompletionRequest.TEMPERATURE.getName()) - && !options.containsKey(AzureAiChatCompletionRequest.TOP_P.getName())) { - request.withOption(AzureAiChatCompletionRequest.TEMPERATURE, 0.0d); - } - request.withOption(AzureAiChatCompletionRequest.SEED, 0xdeadbeef); - request.withExtraOptions(options); - - var response = client.generate(traits.getUrl(), request.build()).trim(); - LOG.trace("Azure AI OpenAI responds: [{}]", response); - return response; - } - - private AzureAiGenerateResponseFormat getResponseFormat(ResponseFormat aResponseformat, - JsonNode aSchema) - { - if (aSchema != null) { - return AzureAiGenerateResponseFormat.builder() // - .withType(JSON_SCHEMA) // - .withSchema("response", aSchema) // - .build(); - } - - if (aResponseformat == JSON) { - return AzureAiGenerateResponseFormat.builder() // - .withType(JSON_OBJECT) // - .build(); - } - - return null; + return AzureAiOpenAiLlmChatClient.ID; } } diff --git a/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/azureaiopenai/AzureAiOpenAiRecommenderFactory.java b/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/azureaiopenai/AzureAiOpenAiRecommenderFactory.java index 0d6fbb9d1e..e65f0477bc 100644 --- a/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/azureaiopenai/AzureAiOpenAiRecommenderFactory.java +++ b/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/azureaiopenai/AzureAiOpenAiRecommenderFactory.java @@ -31,7 +31,7 @@ import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationEngine; import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationEngineFactoryImplBase; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.AnnotationTaskCodecExtensionPoint; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.azureaiopenai.client.AzureAiOpenAiClient; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClientExtensionPoint; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.preset.Presets; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; @@ -42,17 +42,17 @@ public class AzureAiOpenAiRecommenderFactory // and without the database starting to refer to non-existing recommendation tools. public static final String ID = "de.tudarmstadt.ukp.inception.recommendation.imls.azureaiopenai.AzureAiOpenAiRecommender"; - private final AzureAiOpenAiClient client; private final AnnotationSchemaService schemaService; private final AnnotationTaskCodecExtensionPoint responseExtractorExtensionPoint; + private final LlmChatClientExtensionPoint chatClientExtensionPoint; - public AzureAiOpenAiRecommenderFactory(AzureAiOpenAiClient aClient, - AnnotationSchemaService aSchemaService, - AnnotationTaskCodecExtensionPoint aResponseExtractorExtensionPoint) + public AzureAiOpenAiRecommenderFactory(AnnotationSchemaService aSchemaService, + AnnotationTaskCodecExtensionPoint aResponseExtractorExtensionPoint, + LlmChatClientExtensionPoint aChatClientExtensionPoint) { - client = aClient; schemaService = aSchemaService; responseExtractorExtensionPoint = aResponseExtractorExtensionPoint; + chatClientExtensionPoint = aChatClientExtensionPoint; } @Override @@ -71,8 +71,8 @@ public String getName() public RecommendationEngine build(Recommender aRecommender) { var traits = readTraits(aRecommender); - return new AzureAiOpenAiRecommender(aRecommender, traits, client, schemaService, - responseExtractorExtensionPoint); + return new AzureAiOpenAiRecommender(aRecommender, traits, schemaService, + responseExtractorExtensionPoint, chatClientExtensionPoint); } @Override diff --git a/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/azureaiopenai/client/AzureAiOpenAiLlmChatClient.java b/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/azureaiopenai/client/AzureAiOpenAiLlmChatClient.java new file mode 100644 index 0000000000..c9ebc20c9c --- /dev/null +++ b/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/azureaiopenai/client/AzureAiOpenAiLlmChatClient.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.azureaiopenai.client; + +import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.azureaiopenai.client.AzureAiResponseFormatType.JSON_OBJECT; +import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.azureaiopenai.client.AzureAiResponseFormatType.JSON_SCHEMA; + +import java.io.IOException; +import java.util.List; + +import java.util.EnumSet; +import java.util.Set; + +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ChatMessage; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ChatOptions; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ChatResult; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClient; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmEndpoint; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ModelCapability; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.response.ResponseFormat; +import de.tudarmstadt.ukp.inception.security.client.auth.apikey.ApiKeyAuthenticationTraits; + +/** + * {@link LlmChatClient} adapter for Azure OpenAI Service. The Azure URL embeds the deployment name + * and API version, so {@link LlmEndpoint#model()} is not sent on the wire (it is captured by the + * URL). Delegates to the existing {@link AzureAiOpenAiClient}; pure pass-through with no + * recommender-specific defaults applied here. + */ +public class AzureAiOpenAiLlmChatClient + implements LlmChatClient +{ + public static final String ID = "azure-openai"; + + private final AzureAiOpenAiClient client; + + public AzureAiOpenAiLlmChatClient(AzureAiOpenAiClient aClient) + { + client = aClient; + } + + @Override + public String getId() + { + return ID; + } + + @Override + public Set supportedCapabilities() + { + return EnumSet.of(ModelCapability.CHAT, ModelCapability.JSON_SCHEMA); + } + + @Override + public ChatResult chat(LlmEndpoint aEndpoint, List aMessages, ChatOptions aOptions) + throws IOException + { + var messages = aMessages.stream() // + .map(m -> new AzureAiChatCompletionMessage(m.role().getName(), m.content())) // + .toList(); + + var builder = AzureAiChatCompletionRequest.builder() // + .withApiKey(apiKey(aEndpoint)) // + .withModel(aEndpoint.model()) // + .withMessages(messages) // + .withFormat(toResponseFormat(aOptions)); + + if (aOptions.options() != null) { + builder.withExtraOptions(aOptions.options()); + } + + var responseText = client.generate(aEndpoint.url(), builder.build()).trim(); + + return ChatResult.of(new ChatMessage(ChatMessage.Role.ASSISTANT, responseText)); + } + + private static String apiKey(LlmEndpoint aEndpoint) + { + var auth = aEndpoint.auth(); + if (auth == null) { + throw new IllegalArgumentException( + "Azure OpenAI requires an API key but none is configured"); + } + if (auth instanceof ApiKeyAuthenticationTraits apiKeyAuth) { + var key = apiKeyAuth.getApiKey(); + if (key == null || key.isBlank()) { + throw new IllegalArgumentException( + "Azure OpenAI requires an API key but the configured key is blank"); + } + return key; + } + throw new IllegalArgumentException( + "Azure OpenAI client requires " + ApiKeyAuthenticationTraits.class.getSimpleName() + + " but got [" + auth.getClass().getName() + "]"); + } + + private static AzureAiGenerateResponseFormat toResponseFormat(ChatOptions aOptions) + { + if (aOptions.jsonSchema() != null) { + return AzureAiGenerateResponseFormat.builder() // + .withType(JSON_SCHEMA) // + .withSchema("response", aOptions.jsonSchema()) // + .build(); + } + + if (aOptions.responseFormat() == ResponseFormat.JSON) { + return AzureAiGenerateResponseFormat.builder() // + .withType(JSON_OBJECT) // + .build(); + } + + return null; + } +} diff --git a/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/azureaiopenai/config/AzureAiOpenAiRecommenderAutoConfiguration.java b/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/azureaiopenai/config/AzureAiOpenAiRecommenderAutoConfiguration.java index b0403538f2..5d37a7a187 100644 --- a/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/azureaiopenai/config/AzureAiOpenAiRecommenderAutoConfiguration.java +++ b/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/azureaiopenai/config/AzureAiOpenAiRecommenderAutoConfiguration.java @@ -25,6 +25,8 @@ import de.tudarmstadt.ukp.inception.recommendation.imls.llm.azureaiopenai.AzureAiOpenAiRecommenderFactory; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.azureaiopenai.client.AzureAiOpenAiClient; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.azureaiopenai.client.AzureAiOpenAiClientImpl; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.azureaiopenai.client.AzureAiOpenAiLlmChatClient; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClientExtensionPoint; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; @Configuration @@ -38,12 +40,19 @@ public AzureAiOpenAiClient azureAiOpenAiClient() return new AzureAiOpenAiClientImpl(); } + @Bean + public AzureAiOpenAiLlmChatClient azureAiOpenAiLlmChatClient(AzureAiOpenAiClient aClient) + { + return new AzureAiOpenAiLlmChatClient(aClient); + } + @Bean public AzureAiOpenAiRecommenderFactory azureAiOpenAiRecommenderFactory( - AzureAiOpenAiClient aClient, AnnotationSchemaService aSchemaService, - AnnotationTaskCodecExtensionPoint aResponseExtractorExtensionPoint) + AnnotationSchemaService aSchemaService, + AnnotationTaskCodecExtensionPoint aResponseExtractorExtensionPoint, + LlmChatClientExtensionPoint aChatClientExtensionPoint) { - return new AzureAiOpenAiRecommenderFactory(aClient, aSchemaService, - aResponseExtractorExtensionPoint); + return new AzureAiOpenAiRecommenderFactory(aSchemaService, aResponseExtractorExtensionPoint, + aChatClientExtensionPoint); } } diff --git a/inception/inception-imls-chatgpt/pom.xml b/inception/inception-imls-chatgpt/pom.xml index cd573f2646..da197fd4e3 100644 --- a/inception/inception-imls-chatgpt/pom.xml +++ b/inception/inception-imls-chatgpt/pom.xml @@ -123,5 +123,12 @@ org.slf4j slf4j-api + + + de.tudarmstadt.ukp.inception.app + inception-testing + ${project.version} + test + \ No newline at end of file diff --git a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/ChatGptRecommender.java b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/ChatGptRecommender.java index c47120d24a..b64ca8a770 100644 --- a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/ChatGptRecommender.java +++ b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/ChatGptRecommender.java @@ -17,93 +17,28 @@ */ package de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt; -import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatCompletionRequest.SEED; -import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatCompletionRequest.TEMPERATURE; -import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatCompletionRequest.TOP_P; -import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatGptResponseFormatType.JSON_SCHEMA; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import de.tudarmstadt.ukp.inception.recommendation.api.model.Recommender; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.AnnotationTaskCodecExtensionPoint; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ChatBasedLlmRecommenderImplBase; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ChatMessage; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatCompletionMessage; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatCompletionRequest; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatGptClient; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatGptResponseFormat; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatGptResponseFormatType; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.response.ResponseFormat; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatGptLlmChatClient; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClientExtensionPoint; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; -import de.tudarmstadt.ukp.inception.security.client.auth.apikey.ApiKeyAuthenticationTraits; -import tools.jackson.databind.JsonNode; public class ChatGptRecommender extends ChatBasedLlmRecommenderImplBase { - private final static Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private final ChatGptClient client; - public ChatGptRecommender(Recommender aRecommender, ChatGptRecommenderTraits aTraits, - ChatGptClient aClient, AnnotationSchemaService aSchemaService, - AnnotationTaskCodecExtensionPoint aResponseExtractorExtensionPoint) + AnnotationSchemaService aSchemaService, + AnnotationTaskCodecExtensionPoint aResponseExtractorExtensionPoint, + LlmChatClientExtensionPoint aChatClientExtensionPoint) { - super(aRecommender, aTraits, aSchemaService, aResponseExtractorExtensionPoint); - - client = aClient; + super(aRecommender, aTraits, aSchemaService, aResponseExtractorExtensionPoint, + aChatClientExtensionPoint); } @Override - protected String exchange(List aMessages, - de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.response.ResponseFormat aFormat, - JsonNode aJsonSchema) - throws IOException - { - var messages = aMessages.stream() // - .map(m -> new ChatCompletionMessage(m.role().getName(), m.content())) // - .toList(); - - var request = ChatCompletionRequest.builder() // - .withApiKey(((ApiKeyAuthenticationTraits) traits.getAuthentication()).getApiKey()) // - .withMessages(messages) // - .withResponseFormat(getResponseFormat(aFormat, aJsonSchema)) // - .withModel(traits.getModel()); - - var options = traits.getOptions(); - // https://platform.openai.com/docs/api-reference/chat/create recommends to set temperature - // or top_p but not both. - if (!options.containsKey(TEMPERATURE.getName()) && !options.containsKey(TOP_P.getName())) { - request.withOption(TEMPERATURE, 0.0d); - } - request.withOption(SEED, 0xdeadbeef); - request.withExtraOptions(options); - - var response = client.chat(traits.getUrl(), request.build()).trim(); - LOG.trace("Response: [{}]", response); - return response; - } - - private ChatGptResponseFormat getResponseFormat(ResponseFormat aFormat, JsonNode aSchema) + protected String getProviderId() { - if (aSchema != null) { - return ChatGptResponseFormat.builder() // - .withType(JSON_SCHEMA) // - .withSchema("response", aSchema) // - .build(); - } - - if (aFormat == ResponseFormat.JSON) { - return ChatGptResponseFormat.builder() // - .withType(ChatGptResponseFormatType.JSON_OBJECT) // - .build(); - } - - return null; + return ChatGptLlmChatClient.ID; } } diff --git a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/ChatGptRecommenderFactory.java b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/ChatGptRecommenderFactory.java index a5a28fcf56..9a5e6a1925 100644 --- a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/ChatGptRecommenderFactory.java +++ b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/ChatGptRecommenderFactory.java @@ -32,7 +32,7 @@ import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationEngineFactoryImplBase; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.AnnotationTaskCodecExtensionPoint; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatCompletionRequest; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatGptClient; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClientExtensionPoint; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.preset.Presets; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; @@ -43,16 +43,17 @@ public class ChatGptRecommenderFactory // and without the database starting to refer to non-existing recommendation tools. public static final String ID = "de.tudarmstadt.ukp.inception.recommendation.imls.chatgpt.ChatGptRecommender"; - private final ChatGptClient client; private final AnnotationSchemaService schemaService; private final AnnotationTaskCodecExtensionPoint responseExtractorExtensionPoint; + private final LlmChatClientExtensionPoint chatClientExtensionPoint; - public ChatGptRecommenderFactory(ChatGptClient aClient, AnnotationSchemaService aSchemaService, - AnnotationTaskCodecExtensionPoint aResponseExtractorExtensionPoint) + public ChatGptRecommenderFactory(AnnotationSchemaService aSchemaService, + AnnotationTaskCodecExtensionPoint aResponseExtractorExtensionPoint, + LlmChatClientExtensionPoint aChatClientExtensionPoint) { - client = aClient; schemaService = aSchemaService; responseExtractorExtensionPoint = aResponseExtractorExtensionPoint; + chatClientExtensionPoint = aChatClientExtensionPoint; } @Override @@ -71,8 +72,8 @@ public String getName() public RecommendationEngine build(Recommender aRecommender) { var traits = readTraits(aRecommender); - return new ChatGptRecommender(aRecommender, traits, client, schemaService, - responseExtractorExtensionPoint); + return new ChatGptRecommender(aRecommender, traits, schemaService, + responseExtractorExtensionPoint, chatClientExtensionPoint); } @Override diff --git a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/ChatGptRecommenderTraitsEditor.java b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/ChatGptRecommenderTraitsEditor.java index 38092b3525..61994dcfb8 100644 --- a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/ChatGptRecommenderTraitsEditor.java +++ b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/ChatGptRecommenderTraitsEditor.java @@ -35,9 +35,9 @@ import de.tudarmstadt.ukp.inception.recommendation.api.model.Recommender; import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationEngineFactory; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatGptClientImpl; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatGptModel; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ListModelsRequest; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatGptLlmChatClient; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmEndpoint; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ModelInfo; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.preset.Preset; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.traits.LlmRecommenderTraits; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.traits.LlmRecommenderTraitsEditor_ImplBase; @@ -52,6 +52,7 @@ public class ChatGptRecommenderTraitsEditor private static final long serialVersionUID = 1677442652521110324L; private @SpringBean RecommendationEngineFactory toolFactory; + private @SpringBean ChatGptLlmChatClient client; public ChatGptRecommenderTraitsEditor(String aId, IModel aRecommender, IModel> aPresets, IModel>> aOptions) @@ -86,20 +87,19 @@ protected List listUrls() @Override protected List listModels() { - var url = getTraits().map(LlmRecommenderTraits::getUrl).orElse(null).getObject(); - var apiKey = ((ApiKeyAuthenticationTraits) getTraits() - .map(LlmRecommenderTraits::getAuthentication).orElse(null).getObject()).getApiKey(); + var traits = getTraits().getObject(); + var url = traits.getUrl(); + var auth = (ApiKeyAuthenticationTraits) traits.getAuthentication(); - if (!new UrlValidator(new String[] { "http", "https" }).isValid(url) || isBlank(apiKey)) { + if (!new UrlValidator(new String[] { "http", "https" }).isValid(url) || auth == null + || isBlank(auth.getApiKey())) { return emptyList(); } - var client = new ChatGptClientImpl(); try { - return client.listModels(url, ListModelsRequest.builder() // - .withApiKey(apiKey) // - .build()).stream() // - .map(ChatGptModel::getId) // + var endpoint = new LlmEndpoint(ChatGptLlmChatClient.ID, url, null, auth); + return client.listModels(endpoint).stream() // + .map(ModelInfo::id) // .toList(); } catch (IOException e) { diff --git a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/client/ChatGptClient.java b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/client/ChatGptClient.java index fa88b2c406..9ae2535600 100644 --- a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/client/ChatGptClient.java +++ b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/client/ChatGptClient.java @@ -18,8 +18,11 @@ package de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client; import java.io.IOException; +import java.util.List; public interface ChatGptClient { String chat(String aUrl, ChatCompletionRequest aRequest) throws IOException; + + List listModels(String aUrl, ListModelsRequest aRequest) throws IOException; } diff --git a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/client/ChatGptClientImpl.java b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/client/ChatGptClientImpl.java index ef80749c95..a91e7a5748 100644 --- a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/client/ChatGptClientImpl.java +++ b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/client/ChatGptClientImpl.java @@ -106,12 +106,15 @@ public String chat(String aUrl, ChatCompletionRequest aRequest) throws IOExcepti var request = HttpRequest.newBuilder() // .uri(URI.create(appendIfMissing(aUrl, "/") + "v1/chat/completions")) // - .header(CONTENT_TYPE, "application/json") // - .header(AUTHORIZATION, "Bearer " + aRequest.getApiKey()) // - .POST(BodyPublishers.ofString(JSONUtil.toJsonString(aRequest), UTF_8)) // - .build(); + .header(CONTENT_TYPE, "application/json"); - var response = sendRequest(request); + if (aRequest.getApiKey() != null) { + request.header(AUTHORIZATION, "Bearer " + aRequest.getApiKey()); + } + + request.POST(BodyPublishers.ofString(JSONUtil.toJsonString(aRequest), UTF_8)); + + var response = sendRequest(request.build()); handleError(response); @@ -174,16 +177,19 @@ public String chat(String aUrl, ChatCompletionRequest aRequest) throws IOExcepti return result.toString(); } + @Override public List listModels(String aUrl, ListModelsRequest aRequest) throws IOException { var request = HttpRequest.newBuilder() // .uri(URI.create(appendIfMissing(aUrl, "/") + "v1/models")) // .header(CONTENT_TYPE, "application/json").GET() // - .header(AUTHORIZATION, "Bearer " + aRequest.getApiKey()) // - .timeout(Duration.ofSeconds(10)) // - .build(); + .timeout(Duration.ofSeconds(10)); + + if (aRequest.getApiKey() != null) { + request.header(AUTHORIZATION, "Bearer " + aRequest.getApiKey()); + } - var response = sendRequest(request); + var response = sendRequest(request.build()); handleError(response); diff --git a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/client/ChatGptLlmChatClient.java b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/client/ChatGptLlmChatClient.java new file mode 100644 index 0000000000..70da80b6de --- /dev/null +++ b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/client/ChatGptLlmChatClient.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client; + +import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatGptResponseFormatType.JSON_OBJECT; +import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatGptResponseFormatType.JSON_SCHEMA; + +import java.io.IOException; +import java.util.List; + +import java.util.EnumSet; +import java.util.Set; + +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ChatMessage; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ChatOptions; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ChatResult; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClient; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmEndpoint; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ModelCapability; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ModelInfo; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.response.ResponseFormat; +import de.tudarmstadt.ukp.inception.security.client.auth.apikey.ApiKeyAuthenticationTraits; + +/** + * {@link LlmChatClient} adapter for the OpenAI chat-completions API (and OpenAI-compatible + * endpoints such as Groq, Cerebras, or local Ollama in OpenAI mode). Delegates to the existing + * {@link ChatGptClient}; behaviour is a faithful translation, with no recommender-specific defaults + * applied here. + */ +public class ChatGptLlmChatClient + implements LlmChatClient +{ + public static final String ID = "openai"; + + private final ChatGptClient client; + + public ChatGptLlmChatClient(ChatGptClient aClient) + { + client = aClient; + } + + @Override + public String getId() + { + return ID; + } + + @Override + public Set supportedCapabilities() + { + return EnumSet.of(ModelCapability.CHAT, ModelCapability.JSON_SCHEMA); + } + + @Override + public ChatResult chat(LlmEndpoint aEndpoint, List aMessages, ChatOptions aOptions) + throws IOException + { + var messages = aMessages.stream() // + .map(m -> new ChatCompletionMessage(m.role().getName(), m.content())) // + .toList(); + + var builder = ChatCompletionRequest.builder() // + .withApiKey(apiKey(aEndpoint)) // + .withModel(aEndpoint.model()) // + .withMessages(messages) // + .withResponseFormat(toResponseFormat(aOptions)); + + if (aOptions.options() != null) { + builder.withExtraOptions(aOptions.options()); + } + + var responseText = client.chat(aEndpoint.url(), builder.build()).trim(); + + return ChatResult.of(new ChatMessage(ChatMessage.Role.ASSISTANT, responseText)); + } + + @Override + public List listModels(LlmEndpoint aEndpoint) throws IOException + { + var request = ListModelsRequest.builder() // + .withApiKey(apiKey(aEndpoint)) // + .build(); + + return client.listModels(aEndpoint.url(), request).stream() // + .map(m -> new ModelInfo(m.getId())) // + .toList(); + } + + private static String apiKey(LlmEndpoint aEndpoint) + { + var auth = aEndpoint.auth(); + if (auth instanceof ApiKeyAuthenticationTraits apiKeyAuth) { + return apiKeyAuth.getApiKey(); + } + if (auth == null) { + return null; + } + throw new IllegalArgumentException( + "ChatGPT client requires " + ApiKeyAuthenticationTraits.class.getSimpleName() + + " but got [" + auth.getClass().getName() + "]"); + } + + private static ChatGptResponseFormat toResponseFormat(ChatOptions aOptions) + { + if (aOptions.jsonSchema() != null) { + return ChatGptResponseFormat.builder() // + .withType(JSON_SCHEMA) // + .withSchema("response", aOptions.jsonSchema()) // + .build(); + } + + if (aOptions.responseFormat() == ResponseFormat.JSON) { + return ChatGptResponseFormat.builder() // + .withType(JSON_OBJECT) // + .build(); + } + + return null; + } +} diff --git a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/config/ChatGptRecommenderAutoConfiguration.java b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/config/ChatGptRecommenderAutoConfiguration.java index 70d272bb1c..6ab295b9de 100644 --- a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/config/ChatGptRecommenderAutoConfiguration.java +++ b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/config/ChatGptRecommenderAutoConfiguration.java @@ -25,6 +25,8 @@ import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.ChatGptRecommenderFactory; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatGptClient; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatGptClientImpl; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatGptLlmChatClient; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClientExtensionPoint; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; @Configuration @@ -39,11 +41,18 @@ public ChatGptClient chatGptClient() } @Bean - public ChatGptRecommenderFactory chatGptRecommenderFactory(ChatGptClient aClient, + public ChatGptLlmChatClient chatGptLlmChatClient(ChatGptClient aClient) + { + return new ChatGptLlmChatClient(aClient); + } + + @Bean + public ChatGptRecommenderFactory chatGptRecommenderFactory( AnnotationSchemaService aSchemaService, - AnnotationTaskCodecExtensionPoint aResponseExtractorExtensionPoint) + AnnotationTaskCodecExtensionPoint aResponseExtractorExtensionPoint, + LlmChatClientExtensionPoint aChatClientExtensionPoint) { - return new ChatGptRecommenderFactory(aClient, aSchemaService, - aResponseExtractorExtensionPoint); + return new ChatGptRecommenderFactory(aSchemaService, aResponseExtractorExtensionPoint, + aChatClientExtensionPoint); } } diff --git a/inception/inception-imls-chatgpt/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/client/ChatGptLlmChatClientTest.java b/inception/inception-imls-chatgpt/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/client/ChatGptLlmChatClientTest.java new file mode 100644 index 0000000000..585f71eeca --- /dev/null +++ b/inception/inception-imls-chatgpt/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/client/ChatGptLlmChatClientTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client; + +import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.ChatMessage.Role.USER; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; + +import java.lang.invoke.MethodHandles; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ChatMessage; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ChatOptions; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmEndpoint; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.response.ResponseFormat; +import de.tudarmstadt.ukp.inception.security.client.auth.apikey.ApiKeyAuthenticationTraits; +import de.tudarmstadt.ukp.inception.support.json.JSONUtil; +import de.tudarmstadt.ukp.inception.support.test.http.HttpTestUtils; + +/** + * Exercises {@link ChatGptLlmChatClient} against Ollama's OpenAI-compatible endpoint. The + * {@code /v1} path segment is appended by the client, so the endpoint URL points at the Ollama + * root. Requires a local Ollama with {@code ministral-3:8b} pulled; skipped otherwise. + */ +class ChatGptLlmChatClientTest +{ + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final String OLLAMA_URL = "http://localhost:11434"; + private static final String MODEL = "ministral-3:8b"; + + private final ChatGptLlmChatClient sut = new ChatGptLlmChatClient(new ChatGptClientImpl()); + + @BeforeAll + static void checkIfOllamaIsRunning() + { + assumeThat(HttpTestUtils.checkURL(OLLAMA_URL)).isTrue(); + } + + private static LlmEndpoint endpoint(String aModel) + { + var auth = new ApiKeyAuthenticationTraits(); + auth.setApiKey("not-used-by-ollama"); + return new LlmEndpoint(ChatGptLlmChatClient.ID, OLLAMA_URL, aModel, auth); + } + + @Test + void testChat() throws Exception + { + var messages = List.of(new ChatMessage(USER, "Tell me a joke in one sentence.")); + + var result = sut.chat(endpoint(MODEL), messages, ChatOptions.defaults()); + + LOG.info("Response: [{}]", result.message().content()); + assertThat(result.message().content()).isNotBlank(); + assertThat(result.message().role()).isEqualTo(ChatMessage.Role.ASSISTANT); + } + + @Test + void testChatWithJsonResponseFormat() throws Exception + { + var messages = List.of(new ChatMessage(USER, + "Return a JSON object with the keys `a` set to 1 and `b` set to 2.")); + var options = new ChatOptions(ResponseFormat.JSON, null, List.of(), null); + + var result = sut.chat(endpoint(MODEL), messages, options); + + LOG.info("Response: [{}]", result.message().content()); + // Provider promised JSON object response; verify it parses. + assertThat(JSONUtil.getObjectMapper().readTree(result.message().content())).isNotNull(); + } + + @Test + void testListModels() throws Exception + { + var models = sut.listModels(endpoint(null)); + + LOG.info("Models: {}", models); + assertThat(models).extracting("id").contains(MODEL); + } +} diff --git a/inception/inception-imls-chatgpt/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/client/OpenAiClientTest.java b/inception/inception-imls-chatgpt/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/client/OpenAiClientTest.java similarity index 86% rename from inception/inception-imls-chatgpt/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/client/OpenAiClientTest.java rename to inception/inception-imls-chatgpt/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/client/OpenAiClientTest.java index 2ca835f841..d14dab39cc 100644 --- a/inception/inception-imls-chatgpt/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/client/OpenAiClientTest.java +++ b/inception/inception-imls-chatgpt/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/client/OpenAiClientTest.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package de.tudarmstadt.ukp.inception.recommendation.imls.azureaiopenai.client; +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client; import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.ChatGptRecommenderTraits.OPENAI_API_URL; import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatGptResponseFormatType.JSON_OBJECT; @@ -28,11 +28,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatCompletionRequest; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatGptClientImpl; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ChatGptResponseFormat; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.chatgpt.client.ListModelsRequest; - class OpenAiClientTest { private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ChatBasedLlmRecommenderImplBase.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ChatBasedLlmRecommenderImplBase.java index 1e9d4188e1..bc853c4728 100644 --- a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ChatBasedLlmRecommenderImplBase.java +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ChatBasedLlmRecommenderImplBase.java @@ -20,6 +20,7 @@ import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.prompt.PromptContextGenerator.VAR_EXAMPLES; import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.prompt.PromptContextGenerator.getPromptContextGenerator; import static de.tudarmstadt.ukp.inception.scheduling.ProgressScope.SCOPE_UNITS; +import static java.util.Collections.emptyList; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.exception.ExceptionUtils.getRootCauseMessage; @@ -38,6 +39,9 @@ import de.tudarmstadt.ukp.inception.recommendation.api.recommender.NonTrainableRecommenderEngineImplBase; import de.tudarmstadt.ukp.inception.recommendation.api.recommender.PredictionContext; import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationException; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ChatOptions; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClientExtensionPoint; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmEndpoint; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.prompt.JinjaPromptRenderer; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.response.ResponseFormat; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.traits.LlmRecommenderTraits; @@ -51,25 +55,38 @@ public abstract class ChatBasedLlmRecommenderImplBase prepareGlobalBindings(CAS aCas) { var globalBindings = new LinkedHashMap(); @@ -134,7 +151,41 @@ public Range predict(PredictionContext aContext, CAS aCas, int aBegin, int aEnd) return new Range(aBegin, aEnd); } - protected abstract String exchange(List aPrompt, ResponseFormat aResponseformat, + protected String exchange(List aPrompt, ResponseFormat aResponseformat, JsonNode aJsonSchema) - throws IOException; + throws IOException + { + var providerId = getProviderId(); + var client = chatClientExtensionPoint.getExtension(providerId) + .orElseThrow(() -> new IOException("No LLM client registered for provider [" + + providerId + "] - is the corresponding module enabled?")); + + var endpoint = new LlmEndpoint(providerId, traits.getUrl(), traits.getModel(), + traits.getAuthentication(), traits.getCapabilities()); + + var options = recommenderDefaults(traits.getOptions()); + var chatOptions = new ChatOptions(aResponseformat, aJsonSchema, emptyList(), options); + + var result = client.chat(endpoint, aPrompt, chatOptions); + LOG.trace("[{}] response: [{}]", providerId, result.message().content()); + return result.message().content(); + } + + /** + * Apply recommender-use-case defaults to the option bag without mutating the traits-owned map: + * deterministic seed for reproducibility, and {@code temperature=0} unless the user has already + * set either temperature or top_p. + */ + private static Map recommenderDefaults(Map aOptions) + { + var out = new LinkedHashMap(); + if (aOptions != null) { + out.putAll(aOptions); + } + if (!out.containsKey(OPT_TEMPERATURE) && !out.containsKey(OPT_TOP_P)) { + out.put(OPT_TEMPERATURE, 0.0d); + } + out.putIfAbsent(OPT_SEED, 0xdeadbeef); + return out; + } } diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ChatMessage.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ChatMessage.java index 13ea9c1acb..b02776e440 100644 --- a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ChatMessage.java +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ChatMessage.java @@ -19,13 +19,33 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +/** + * A single chat message exchanged with the LLM. + * + * @param role + * who is speaking + * @param content + * the message text; may be empty for assistant messages that only carry tool calls + * @param thinking + * reasoning / chain-of-thought emitted by the model alongside {@code content}, when the + * model exposes it; {@code null} otherwise + * @param toolCallId + * for {@link Role#TOOL} messages, the id of the call this is the result of (set by the + * provider in the preceding assistant turn); {@code null} for all other roles + */ @JsonIgnoreProperties(ignoreUnknown = true) -public record ChatMessage(Role role, String content) { +public record ChatMessage(Role role, String content, String thinking, String toolCallId) { + public ChatMessage(Role aRole, String aContent) + { + this(aRole, aContent, null, null); + } + public static enum Role { SYSTEM("system"), // ASSISTANT("assistant"), // - USER("user"); + USER("user"), // + TOOL("tool"); private final String name; @@ -39,5 +59,4 @@ public String getName() return name; } } - } diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ChatChunk.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ChatChunk.java new file mode 100644 index 0000000000..e38ac7c2db --- /dev/null +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ChatChunk.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.client; + +/** + * One incremental chunk delivered during a streaming chat exchange. Fields are deltas (only the new + * text since the previous chunk), not cumulative. Any field may be {@code null} when not present in + * the chunk. + * + * @param contentDelta + * new visible content text + * @param thinkingDelta + * new reasoning / chain-of-thought text, when the model exposes it + */ +public record ChatChunk( // + String contentDelta, // + String thinkingDelta) +{} diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ChatOptions.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ChatOptions.java new file mode 100644 index 0000000000..265529aa0c --- /dev/null +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ChatOptions.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.client; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; + +import java.util.List; +import java.util.Map; + +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.response.ResponseFormat; +import tools.jackson.databind.JsonNode; + +/** + * Per-call generation parameters for {@link LlmChatClient#chat}. Provider-specific knobs + * (temperature, top_p, seed, ...) ride in {@link #options}. + * + * @param responseFormat + * requested response format, or {@code null} for unconstrained + * @param jsonSchema + * JSON schema for structured output; honored only when supported by the provider + * @param tools + * tools the model may call; empty list disables tool calling + * @param options + * free-form provider-specific options (temperature, top_p, seed, ...) + */ +public record ChatOptions( // + ResponseFormat responseFormat, // + JsonNode jsonSchema, // + List tools, // + Map options) +{ + public static ChatOptions defaults() + { + return new ChatOptions(null, null, emptyList(), emptyMap()); + } +} diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ChatResult.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ChatResult.java new file mode 100644 index 0000000000..e5d8144d38 --- /dev/null +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ChatResult.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.client; + +import static java.util.Collections.emptyList; + +import java.util.List; + +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ChatMessage; + +/** + * Result of a single chat exchange. + * + * @param message + * the assistant message returned by the model (may have empty content if the model only + * requested tool calls) + * @param toolCalls + * tool invocations requested by the model; empty when none + * @param finishReason + * why the model stopped, or {@code null} if the provider did not report one + * @param usage + * token usage reported by the provider, or {@code null} if unavailable + */ +public record ChatResult( // + ChatMessage message, // + List toolCalls, // + FinishReason finishReason, // + UsageInfo usage) +{ + public static ChatResult of(ChatMessage aMessage) + { + return new ChatResult(aMessage, emptyList(), null, null); + } +} diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/FinishReason.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/FinishReason.java new file mode 100644 index 0000000000..1a2808b4bf --- /dev/null +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/FinishReason.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.client; + +/** + * Why the model stopped generating. Maps to the union of finish/stop reasons exposed by common + * providers (OpenAI {@code finish_reason}, Ollama {@code done_reason}, Azure OpenAI). + */ +public enum FinishReason +{ + /** Natural stop, end-of-turn or stop sequence hit. */ + STOP, + + /** Reached the maximum token limit before completing. */ + LENGTH, + + /** Model wants to invoke one or more tools; see {@code ChatResult.toolCalls}. */ + TOOL_CALLS, + + /** Output was blocked by a content filter / safety system. */ + CONTENT_FILTER, + + /** Generation aborted due to an error reported by the provider. */ + ERROR, + + /** Provider returned a reason that does not map to any of the above. */ + OTHER +} diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/LlmChatClient.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/LlmChatClient.java new file mode 100644 index 0000000000..4ea6693188 --- /dev/null +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/LlmChatClient.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.client; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; + +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ChatMessage; +import de.tudarmstadt.ukp.inception.support.extensionpoint.Extension; + +/** + * Provider-neutral chat client. One implementation per provider (OpenAI / OpenAI-compatible, Azure + * OpenAI, Ollama, ...) registered as a Spring bean and selected via + * {@link LlmChatClientExtensionPoint}. + *

+ * {@link #getId()} returns the provider id (e.g. {@code openai}, {@code azure-openai}, + * {@code ollama}) and must match the {@code providerId} carried by {@link LlmEndpoint}. + */ +public interface LlmChatClient + extends Extension +{ + @Override + String getId(); + + @Override + default boolean accepts(LlmEndpoint aEndpoint) + { + return aEndpoint != null && getId().equals(aEndpoint.providerId()); + } + + /** + * Capabilities this adapter is currently able to translate on its wire protocol. This + * is an adapter-implementation property, not a model property: + *

    + *
  • {@link ModelCapability#STREAMING} / {@link ModelCapability#EMBEDDINGS} indicate that + * {@link #chatStream} / {@link #embed} are implemented (instead of throwing + * {@code UnsupportedOperationException}).
    • + *
  • {@link ModelCapability#TOOLS} / {@link ModelCapability#JSON_SCHEMA} / + * {@link ModelCapability#VISION} / {@link ModelCapability#THINKING} indicate that the adapter + * translates the corresponding {@link ChatOptions} field or {@link ChatMessage} content onto + * the wire — silently ignored otherwise.
    • + *
+ * This set is typically a constant per adapter and reflects adapter maturity. + *

+ * Distinct from {@link LlmEndpoint#capabilities()}, which describes what the configured + * model is declared to support. The endpoint set is what the caller wants to use; this set + * is the upper bound of what the adapter can deliver. A valid configuration has + * {@code endpoint.capabilities() ⊆ adapter.supportedCapabilities()}; declaring a capability the + * adapter cannot deliver has no effect (best case) or is silently ignored. + *

+ * UIs should use this set to gate which capability checkboxes are even sensible to show for a + * given provider. + */ + default Set supportedCapabilities() + { + return EnumSet.of(ModelCapability.CHAT); + } + + /** + * Perform a non-streaming chat exchange. + */ + ChatResult chat(LlmEndpoint aEndpoint, List aMessages, ChatOptions aOptions) + throws IOException; + + /** + * Perform a streaming chat exchange. {@code aOnChunk} is invoked for each delivered fragment + * (deltas, not cumulative); the returned {@link ChatResult} carries the assembled final message + * together with usage and finish reason. + */ + default ChatResult chatStream(LlmEndpoint aEndpoint, List aMessages, + ChatOptions aOptions, Consumer aOnChunk) + throws IOException + { + throw new UnsupportedOperationException( + "Provider [" + getId() + "] does not support streaming"); + } + + /** + * Compute embeddings for the given inputs. Result order matches {@code aInputs}. + * + * @param aOptions + * provider-specific options (e.g. {@code num_ctx}, {@code seed} for Ollama; + * {@code dimensions}, {@code encoding_format} for OpenAI). May be {@code null} or + * empty. + */ + default List embed(LlmEndpoint aEndpoint, List aInputs, + Map aOptions) + throws IOException + { + throw new UnsupportedOperationException( + "Provider [" + getId() + "] does not support embeddings"); + } + + /** + * List models available at the endpoint. Returns an empty list if the provider does not expose + * a discovery API. + */ + default List listModels(LlmEndpoint aEndpoint) throws IOException + { + return List.of(); + } +} diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/LlmChatClientExtensionPoint.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/LlmChatClientExtensionPoint.java new file mode 100644 index 0000000000..5adfce5ca3 --- /dev/null +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/LlmChatClientExtensionPoint.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.client; + +import de.tudarmstadt.ukp.inception.support.extensionpoint.ExtensionPoint; + +public interface LlmChatClientExtensionPoint + extends ExtensionPoint +{ +} diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/LlmChatClientExtensionPointImpl.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/LlmChatClientExtensionPointImpl.java new file mode 100644 index 0000000000..0c09fa90c3 --- /dev/null +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/LlmChatClientExtensionPointImpl.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.client; + +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; + +import de.tudarmstadt.ukp.inception.support.extensionpoint.ExtensionPoint_ImplBase; + +public class LlmChatClientExtensionPointImpl + extends ExtensionPoint_ImplBase + implements LlmChatClientExtensionPoint +{ + public LlmChatClientExtensionPointImpl( + @Lazy @Autowired(required = false) List aExtensions) + { + super(aExtensions); + } +} diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/LlmEndpoint.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/LlmEndpoint.java new file mode 100644 index 0000000000..f2f6d59940 --- /dev/null +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/LlmEndpoint.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.client; + +import java.util.EnumSet; +import java.util.Set; + +import de.tudarmstadt.ukp.inception.security.client.auth.AuthenticationTraits; + +/** + * Identifies a concrete LLM endpoint: which provider, where it lives, how to authenticate, which + * model to use, and which {@link ModelCapability capabilities} the configured model is declared to + * support. Typically derived from {@code LlmRecommenderTraits} (recommender side) or from assistant + * configuration. + */ +public record LlmEndpoint( // + String providerId, // + String url, // + String model, // + AuthenticationTraits auth, // + Set capabilities) +{ + public LlmEndpoint + { + capabilities = capabilities != null // + ? EnumSet.copyOf(capabilities.isEmpty() // + ? EnumSet.noneOf(ModelCapability.class) // + : capabilities) + : EnumSet.noneOf(ModelCapability.class); + } + + public LlmEndpoint(String aProviderId, String aUrl, String aModel, AuthenticationTraits aAuth) + { + this(aProviderId, aUrl, aModel, aAuth, EnumSet.noneOf(ModelCapability.class)); + } +} diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ModelCapability.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ModelCapability.java new file mode 100644 index 0000000000..b7819d1ef5 --- /dev/null +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ModelCapability.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.client; + +/** + * Capabilities a configured model+endpoint supports. The source varies by provider: Ollama can + * probe via {@code ollama show}, OpenAI-compatible endpoints (incl. LM Studio, vLLM, Groq, ...) + * have no probe so the user declares them in the traits/config UI. + *

+ * Distinct from {@code LlmChatClient.supportsX()}, which describes what the adapter can translate + * at all on its wire protocol (a static, adapter-level capability). The adapter flag gates + * which UI controls are sensible to show; this enum gates what the caller actually sends per + * request. + */ +public enum ModelCapability +{ + CHAT, TOOLS, JSON_SCHEMA, STREAMING, EMBEDDINGS, VISION, THINKING +} diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ModelInfo.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ModelInfo.java new file mode 100644 index 0000000000..51caf82bd1 --- /dev/null +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ModelInfo.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.client; + +/** + * A model offered by a provider endpoint. Used by traits editors to populate model dropdowns. + * + * @param id + * identifier accepted by the provider (e.g. {@code gpt-4o-mini}, {@code llama3.1:8b}) + * @param displayName + * human-readable label; falls back to {@code id} when {@code null}, so callers can + * always render {@link #displayName()} without checking for {@code null} + */ +public record ModelInfo( // + String id, // + String displayName) +{ + public ModelInfo + { + if (displayName == null) { + displayName = id; + } + } + + public ModelInfo(String aId) + { + this(aId, null); + } +} diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ToolCall.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ToolCall.java new file mode 100644 index 0000000000..4acabc6769 --- /dev/null +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ToolCall.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.client; + +import tools.jackson.databind.JsonNode; + +/** + * An invocation requested by the model. Returned as part of {@link ChatResult#toolCalls()} when the + * model decides to call a tool. The provider-specific {@code id} (when present) must be echoed back + * with the tool result so the model can correlate. + * + * @param id + * provider-assigned call id, may be {@code null} for providers that do not use one + * @param name + * function name as declared via {@link ToolDescriptor#name()} + * @param arguments + * JSON object with the arguments the model wants to pass + */ +public record ToolCall( // + String id, // + String name, // + JsonNode arguments) +{} diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ToolDescriptor.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ToolDescriptor.java new file mode 100644 index 0000000000..4f8dd3c1a0 --- /dev/null +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ToolDescriptor.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.client; + +import static com.github.victools.jsonschema.generator.OptionPreset.PLAIN_JSON; +import static com.github.victools.jsonschema.generator.SchemaVersion.DRAFT_2020_12; +import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.ToolUtils.getFunctionDescription; +import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.ToolUtils.getFunctionName; +import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.ToolUtils.getParameterDescription; +import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.ToolUtils.getParameterName; +import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.ToolUtils.isParameter; + +import java.lang.reflect.Method; + +import com.github.victools.jsonschema.generator.Option; +import com.github.victools.jsonschema.generator.SchemaGenerator; +import com.github.victools.jsonschema.generator.SchemaGeneratorConfigBuilder; +import com.github.victools.jsonschema.module.jackson.JacksonModule; +import com.github.victools.jsonschema.module.jackson.JacksonOption; + +import de.tudarmstadt.ukp.inception.support.json.JSONUtil; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.node.ObjectNode; + +/** + * Provider-neutral description of a tool the model may call. Carried in {@link ChatOptions#tools()} + * on the request side. The execution-side counterpart (with the actual Java method to invoke) lives + * in the assistant/tool-library layer. + * + * @param name + * function name as exposed to the model + * @param description + * free-text description used by the model to decide when to call + * @param parametersSchema + * JSON schema describing the function parameters + */ +public record ToolDescriptor( // + String name, // + String description, // + JsonNode parametersSchema) +{ + /** + * Builds a {@link ToolDescriptor} from a {@code @Tool}-annotated method by deriving the JSON + * schema of its parameters via the victools schema generator. The method itself is not carried + * in the result; the invocation-side {@code (instance, method)} pair stays with the caller + * (typically in a parallel name-keyed map). + */ + public static ToolDescriptor fromMethod(Method aMethod) + { + var generator = new SchemaGenerator( + new SchemaGeneratorConfigBuilder(DRAFT_2020_12, PLAIN_JSON) // + .with(Option.FORBIDDEN_ADDITIONAL_PROPERTIES_BY_DEFAULT) // + .with(new JacksonModule(JacksonOption.RESPECT_JSONPROPERTY_REQUIRED)) // + .build()); + + var schema = JSONUtil.getObjectMapper().createObjectNode(); + schema.put("type", "object"); + var properties = (ObjectNode) schema.putObject("properties"); + var required = schema.putArray("required"); + + for (var param : aMethod.getParameters()) { + if (!isParameter(param)) { + continue; + } + + var paramName = getParameterName(param); + var propertySchema = generator.generateSchema(param.getParameterizedType()); + getParameterDescription(param).ifPresent( + description -> propertySchema.put("description", description.strip())); + + properties.set(paramName, propertySchema); + required.add(paramName); + } + + return new ToolDescriptor(getFunctionName(aMethod), + getFunctionDescription(aMethod).orElse(null), schema); + } +} diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ToolInvoker.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ToolInvoker.java new file mode 100644 index 0000000000..e1c10a935a --- /dev/null +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ToolInvoker.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.client; + +import tools.jackson.databind.JsonNode; + +/** + * Dispatch handle for one tool the LLM may call: pairs the wire-side {@link ToolDescriptor} with + * the logic that runs when the model picks the tool. Implementations are caller-defined since the + * dispatch typically depends on caller-specific runtime context (e.g. the assistant binds + * {@code Project}/{@code SourceDocument}/... captured at construction time). Registered into a + * {@link ToolInvokerSet} and looked up by name when a {@link ToolCall} comes back from the model. + */ +public interface ToolInvoker +{ + /** + * The provider-neutral, wire-side description of the tool this invoker handles. Sent to the + * model via {@link ChatOptions#tools()}. + */ + ToolDescriptor descriptor(); + + /** + * Run the tool. + * + * @param aArguments + * arguments the model supplied; typically an {@code ObjectNode} matching the tool's + * parameter schema. May be {@code null} if the tool has no parameters. + * @return the value the tool produced; the caller decides how to map it into the next + * conversation turn. + */ + Object invoke(JsonNode aArguments) throws Exception; +} diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ToolInvokerSet.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ToolInvokerSet.java new file mode 100644 index 0000000000..7f9b7798da --- /dev/null +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ToolInvokerSet.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.client; + +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +/** + * A short-lived collection of {@link ToolInvoker}s — typically composed for a single chat exchange + * from whatever tool sources apply (Java {@code @Tool} methods, MCP-discovered tools, ad-hoc + * callbacks, ...) and consulted by name when a {@link ToolCall} is returned by the model. + *

+ * Tool names must be unique within a set; adding a duplicate name fails fast. + */ +public interface ToolInvokerSet +{ + /** + * Add an invoker to the set. + * + * @throws IllegalStateException + * if an invoker with the same {@link ToolDescriptor#name() name} is already + * present. + */ + void add(ToolInvoker aInvoker); + + /** + * @return the invoker for the given tool name, or empty if none. + */ + Optional findByName(String aName); + + /** + * @return all invokers, in insertion order. + */ + Collection all(); + + /** + * @return the wire-side {@link ToolDescriptor}s for all invokers, in insertion order. Suitable + * for passing into {@link ChatOptions#tools()}. + */ + List toDescriptors(); +} diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ToolInvokerSetImpl.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ToolInvokerSetImpl.java new file mode 100644 index 0000000000..5485821cbf --- /dev/null +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ToolInvokerSetImpl.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.client; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Default {@link ToolInvokerSet} implementation backed by a {@link LinkedHashMap} keyed by tool + * name. Not thread-safe; the typical use is per-request setup from a single thread. + */ +public class ToolInvokerSetImpl + implements ToolInvokerSet +{ + private final Map byName = new LinkedHashMap<>(); + + public ToolInvokerSetImpl() + { + } + + public ToolInvokerSetImpl(Collection aInvokers) + { + if (aInvokers != null) { + aInvokers.forEach(this::add); + } + } + + @Override + public void add(ToolInvoker aInvoker) + { + var name = aInvoker.descriptor().name(); + var existing = byName.put(name, aInvoker); + if (existing != null) { + byName.put(name, existing); // restore — fail-fast wins + throw new IllegalStateException( + "Duplicate tool name [" + name + "]: [" + existing + "] vs [" + aInvoker + "]"); + } + } + + @Override + public Optional findByName(String aName) + { + if (aName == null) { + return Optional.empty(); + } + return Optional.ofNullable(byName.get(aName)); + } + + @Override + public Collection all() + { + return Collections.unmodifiableCollection(byName.values()); + } + + @Override + public List toDescriptors() + { + return byName.values().stream() // + .map(ToolInvoker::descriptor) // + .toList(); + } +} diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/UsageInfo.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/UsageInfo.java new file mode 100644 index 0000000000..efba9df2bb --- /dev/null +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/UsageInfo.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.client; + +/** + * Token usage reported by the provider. Any field may be {@code null} if the provider did not + * report it. + */ +public record UsageInfo( // + Integer promptTokens, // + Integer completionTokens, // + Integer totalTokens) +{} diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/config/LlmChatClientAutoConfiguration.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/config/LlmChatClientAutoConfiguration.java new file mode 100644 index 0000000000..3d973416d2 --- /dev/null +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/config/LlmChatClientAutoConfiguration.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.config; + +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Lazy; + +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClient; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClientExtensionPoint; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClientExtensionPointImpl; + +@Configuration +public class LlmChatClientAutoConfiguration +{ + @Bean + public LlmChatClientExtensionPoint llmChatClientExtensionPoint( + @Lazy @Autowired(required = false) List aClients) + { + return new LlmChatClientExtensionPointImpl(aClients); + } +} diff --git a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support/traits/LlmRecommenderTraits.java b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support/traits/LlmRecommenderTraits.java index bffa4cc2ea..34df29fb15 100644 --- a/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support/traits/LlmRecommenderTraits.java +++ b/inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support/traits/LlmRecommenderTraits.java @@ -21,12 +21,16 @@ import java.io.Serializable; import java.util.Collections; +import java.util.EnumSet; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ModelCapability; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.prompt.PromptingMode; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.response.ExtractionMode; import de.tudarmstadt.ukp.inception.security.client.auth.AuthenticationTraits; @@ -49,7 +53,8 @@ public class LlmRecommenderTraits private @JsonInclude(NON_EMPTY) Map options = new LinkedHashMap(); - private boolean structuredOutputSupported = true; + private @JsonInclude(NON_EMPTY) Set capabilities = EnumSet + .of(ModelCapability.JSON_SCHEMA); private boolean interactive; @@ -138,14 +143,37 @@ public void setAuthentication(AuthenticationTraits aAuthentication) authentication = aAuthentication; } + public Set getCapabilities() + { + return Collections.unmodifiableSet(capabilities); + } + + public void setCapabilities(Set aCapabilities) + { + capabilities = aCapabilities != null && !aCapabilities.isEmpty() // + ? EnumSet.copyOf(aCapabilities) // + : EnumSet.noneOf(ModelCapability.class); + } + + /** + * Derived view of {@link ModelCapability#JSON_SCHEMA} membership in {@link #getCapabilities()}. + * Suppressed from JSON output so persisted traits use the {@code capabilities} field as the + * source of truth; legacy JSON rows carrying this boolean still deserialize via the setter. + */ + @JsonIgnore public boolean isStructuredOutputSupported() { - return structuredOutputSupported; + return capabilities.contains(ModelCapability.JSON_SCHEMA); } public void setStructuredOutputSupported(boolean aStructuredOutputSupported) { - structuredOutputSupported = aStructuredOutputSupported; + if (aStructuredOutputSupported) { + capabilities.add(ModelCapability.JSON_SCHEMA); + } + else { + capabilities.remove(ModelCapability.JSON_SCHEMA); + } } public boolean isJustificationEnabled() diff --git a/inception/inception-imls-llm-support/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/inception/inception-imls-llm-support/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 0000000000..83c66cf174 --- /dev/null +++ b/inception/inception-imls-llm-support/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1 @@ +de.tudarmstadt.ukp.inception.recommendation.imls.llm.config.LlmChatClientAutoConfiguration diff --git a/inception/inception-imls-llm-support/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ToolInvokerSetImplTest.java b/inception/inception-imls-llm-support/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ToolInvokerSetImplTest.java new file mode 100644 index 0000000000..8b1114b6ee --- /dev/null +++ b/inception/inception-imls-llm-support/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/client/ToolInvokerSetImplTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.client; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalStateException; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import tools.jackson.databind.JsonNode; + +class ToolInvokerSetImplTest +{ + private static ToolInvoker stub(String aName) + { + return new ToolInvoker() + { + private final ToolDescriptor descriptor = new ToolDescriptor(aName, null, null); + + @Override + public ToolDescriptor descriptor() + { + return descriptor; + } + + @Override + public Object invoke(JsonNode aArguments) + { + return aName; + } + + @Override + public String toString() + { + return "stub:" + aName; + } + }; + } + + @Test + void emptySetHasNothing() + { + var sut = new ToolInvokerSetImpl(); + assertThat(sut.all()).isEmpty(); + assertThat(sut.toDescriptors()).isEmpty(); + assertThat(sut.findByName("anything")).isEmpty(); + } + + @Test + void addAndFindByName() + { + var foo = stub("foo"); + var sut = new ToolInvokerSetImpl(); + sut.add(foo); + + assertThat(sut.findByName("foo")).containsSame(foo); + assertThat(sut.findByName("bar")).isEmpty(); + assertThat(sut.findByName(null)).isEmpty(); + } + + @Test + void duplicateNameFailsAndOriginalSurvives() + { + var first = stub("foo"); + var second = stub("foo"); + var sut = new ToolInvokerSetImpl(); + sut.add(first); + + assertThatIllegalStateException() // + .isThrownBy(() -> sut.add(second)) // + .withMessageContaining("foo"); + + assertThat(sut.findByName("foo")).containsSame(first); + } + + @Test + void allAndToDescriptorsPreserveInsertionOrder() + { + var sut = new ToolInvokerSetImpl(); + sut.add(stub("alpha")); + sut.add(stub("zulu")); + sut.add(stub("mike")); + + assertThat(sut.all()).extracting(t -> t.descriptor().name()) // + .containsExactly("alpha", "zulu", "mike"); + assertThat(sut.toDescriptors()).extracting(ToolDescriptor::name) // + .containsExactly("alpha", "zulu", "mike"); + } + + @Test + void constructorSeedsFromCollection() + { + var sut = new ToolInvokerSetImpl(List.of(stub("a"), stub("b"))); + assertThat(sut.toDescriptors()).extracting(ToolDescriptor::name) // + .containsExactly("a", "b"); + } +} diff --git a/inception/inception-imls-ollama/pom.xml b/inception/inception-imls-ollama/pom.xml index bd628a9254..a97da90793 100644 --- a/inception/inception-imls-ollama/pom.xml +++ b/inception/inception-imls-ollama/pom.xml @@ -67,6 +67,11 @@ inception-api-render ${project.version} + + de.tudarmstadt.ukp.inception.app + inception-security + ${project.version} + org.apache.commons diff --git a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/OllamaRecommender.java b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/OllamaRecommender.java index ca990930b7..af0f631c64 100644 --- a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/OllamaRecommender.java +++ b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/OllamaRecommender.java @@ -17,85 +17,28 @@ */ package de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama; -import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaOptions.SEED; -import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaOptions.TEMPERATURE; -import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaOptions.TOP_P; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import de.tudarmstadt.ukp.inception.recommendation.api.model.Recommender; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.AnnotationTaskCodecExtensionPoint; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ChatBasedLlmRecommenderImplBase; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ChatMessage; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaChatMessage; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaChatRequest; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaClient; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.response.ResponseFormat; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClientExtensionPoint; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaLlmChatClient; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; -import tools.jackson.databind.JsonNode; -import tools.jackson.databind.node.JsonNodeFactory; public class OllamaRecommender extends ChatBasedLlmRecommenderImplBase { - private final static Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private final OllamaClient client; - public OllamaRecommender(Recommender aRecommender, OllamaRecommenderTraits aTraits, - OllamaClient aClient, AnnotationSchemaService aSchemaService, - AnnotationTaskCodecExtensionPoint aResponseExtractorExtensionPoint) + AnnotationSchemaService aSchemaService, + AnnotationTaskCodecExtensionPoint aResponseExtractorExtensionPoint, + LlmChatClientExtensionPoint aChatClientExtensionPoint) { - super(aRecommender, aTraits, aSchemaService, aResponseExtractorExtensionPoint); - - client = aClient; + super(aRecommender, aTraits, aSchemaService, aResponseExtractorExtensionPoint, + aChatClientExtensionPoint); } @Override - protected String exchange(List aMessages, ResponseFormat aFormat, JsonNode aSchema) - throws IOException + protected String getProviderId() { - var format = getResponseFormat(aFormat, aSchema); - var messages = aMessages.stream() // - .map(m -> new OllamaChatMessage(m.role().getName(), m.content())) // - .toList(); - - var request = OllamaChatRequest.builder() // - .withModel(traits.getModel()) // - .withMessages(messages) // - .withFormat(format) // - .withThink(false) // - .withStream(false); - - var options = traits.getOptions(); - // https://platform.openai.com/docs/api-reference/chat/create recommends to set temperature - // or top_p but not both. - if (!options.containsKey(TEMPERATURE.getName()) && !options.containsKey(TOP_P.getName())) { - request.withOption(TEMPERATURE, 0.0d); - } - request.withOption(SEED, 0xdeadbeef); - request.withExtraOptions(options); - - var response = client.chat(traits.getUrl(), request.build(), null).getMessage().content(); - - return response; - } - - private JsonNode getResponseFormat(ResponseFormat aFormat, JsonNode aSchema) - { - if (aSchema != null) { - return aSchema; - } - - if (aFormat == ResponseFormat.JSON) { - return JsonNodeFactory.instance.textNode("json"); - } - - return null; + return OllamaLlmChatClient.ID; } } diff --git a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/OllamaRecommenderFactory.java b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/OllamaRecommenderFactory.java index c37594accd..89f4a47e45 100644 --- a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/OllamaRecommenderFactory.java +++ b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/OllamaRecommenderFactory.java @@ -31,7 +31,7 @@ import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationEngine; import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationEngineFactoryImplBase; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.AnnotationTaskCodecExtensionPoint; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaClient; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClientExtensionPoint; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.preset.Presets; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; @@ -43,15 +43,16 @@ public class OllamaRecommenderFactory public static final String ID = "de.tudarmstadt.ukp.inception.recommendation.imls.ollama.OllamaRecommenderFactory"; private final AnnotationSchemaService schemaService; - private final OllamaClient client; private final AnnotationTaskCodecExtensionPoint responseExtractorExtensionPoint; + private final LlmChatClientExtensionPoint chatClientExtensionPoint; - public OllamaRecommenderFactory(OllamaClient aClient, AnnotationSchemaService aSchemaService, - AnnotationTaskCodecExtensionPoint aResponseExtractorExtensionPoint) + public OllamaRecommenderFactory(AnnotationSchemaService aSchemaService, + AnnotationTaskCodecExtensionPoint aResponseExtractorExtensionPoint, + LlmChatClientExtensionPoint aChatClientExtensionPoint) { - client = aClient; schemaService = aSchemaService; responseExtractorExtensionPoint = aResponseExtractorExtensionPoint; + chatClientExtensionPoint = aChatClientExtensionPoint; } @Override @@ -70,8 +71,8 @@ public String getName() public RecommendationEngine build(Recommender aRecommender) { OllamaRecommenderTraits traits = readTraits(aRecommender); - return new OllamaRecommender(aRecommender, traits, client, schemaService, - responseExtractorExtensionPoint); + return new OllamaRecommender(aRecommender, traits, schemaService, + responseExtractorExtensionPoint, chatClientExtensionPoint); } @Override diff --git a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/OllamaRecommenderTraitsEditor.java b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/OllamaRecommenderTraitsEditor.java index 6eabcb8340..d6e74dcbaa 100644 --- a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/OllamaRecommenderTraitsEditor.java +++ b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/OllamaRecommenderTraitsEditor.java @@ -31,9 +31,10 @@ import de.tudarmstadt.ukp.inception.recommendation.api.model.Recommender; import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationEngineFactory; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaClient; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmEndpoint; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ModelInfo; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaLlmChatClient; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaOptions; -import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaTag; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.preset.Preset; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.traits.LlmRecommenderTraits; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.traits.LlmRecommenderTraitsEditor_ImplBase; @@ -44,7 +45,7 @@ public class OllamaRecommenderTraitsEditor private static final long serialVersionUID = 1677442652521110324L; private @SpringBean RecommendationEngineFactory toolFactory; - private @SpringBean OllamaClient ollamaClient; + private @SpringBean OllamaLlmChatClient client; public OllamaRecommenderTraitsEditor(String aId, IModel aRecommender, IModel> aPresets) @@ -68,7 +69,8 @@ protected List listModels() } try { - return ollamaClient.listModels(url).stream().map(OllamaTag::name).toList(); + var endpoint = new LlmEndpoint(OllamaLlmChatClient.ID, url, null, null); + return client.listModels(endpoint).stream().map(ModelInfo::id).toList(); } catch (IOException e) { return emptyList(); diff --git a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/client/OllamaClient.java b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/client/OllamaClient.java index 8c97eeab5d..a2b358a0f6 100644 --- a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/client/OllamaClient.java +++ b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/client/OllamaClient.java @@ -37,7 +37,7 @@ OllamaChatResponse chat(String aUrl, OllamaChatRequest aRequest, Consumer aCallback) throws IOException; - List listModels(String aUrl) throws IOException; + List listModels(String aUrl, String aApiKey) throws IOException; OllamaShowResponse getModelInfo(String aUrl, OllamaShowRequest aRequest) throws IOException; diff --git a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/client/OllamaClientImpl.java b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/client/OllamaClientImpl.java index 7ed04f4260..9ba7ec9fe2 100644 --- a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/client/OllamaClientImpl.java +++ b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/client/OllamaClientImpl.java @@ -261,11 +261,15 @@ public List> embed(String aUrl, OllamaEmbedRequest aReques { var request = HttpRequest.newBuilder() // .uri(URI.create(CS.appendIfMissing(aUrl, "/") + "api/embed")) // - .header(CONTENT_TYPE, "application/json") - .POST(BodyPublishers.ofString(JSONUtil.toJsonString(aRequest), UTF_8)) // - .build(); + .header(CONTENT_TYPE, "application/json"); + + if (aRequest.apiKey() != null) { + request.header(AUTHORIZATION, "Bearer " + aRequest.apiKey()); + } - var rawResponse = sendRequest(request); + request.POST(BodyPublishers.ofString(JSONUtil.toJsonString(aRequest), UTF_8)); + + var rawResponse = sendRequest(request.build()); if (aRequest.input().size() == 1 && rawResponse.statusCode() >= HTTP_BAD_REQUEST) { LOG.error("Error embedding string [{}]", aRequest.input().get(0)); @@ -328,15 +332,18 @@ private void collectMetrics(OllamaEmbedResponse response) } @Override - public List listModels(String aUrl) throws IOException + public List listModels(String aUrl, String aApiKey) throws IOException { var request = HttpRequest.newBuilder() // .uri(URI.create(CS.appendIfMissing(aUrl, "/") + "api/tags")) // .header(CONTENT_TYPE, APPLICATION_JSON).GET() // - .timeout(TIMEOUT) // - .build(); + .timeout(TIMEOUT); - var response = sendRequest(request); + if (aApiKey != null) { + request.header(AUTHORIZATION, "Bearer " + aApiKey); + } + + var response = sendRequest(request.build()); handleError(response); diff --git a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/client/OllamaEmbedRequest.java b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/client/OllamaEmbedRequest.java index 1f9b2c8877..d97c62b73f 100644 --- a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/client/OllamaEmbedRequest.java +++ b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/client/OllamaEmbedRequest.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; @@ -31,13 +32,13 @@ import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.traits.Option; @JsonIgnoreProperties(ignoreUnknown = true) -public record OllamaEmbedRequest(String model, List input, boolean truncate, - @JsonInclude(Include.NON_EMPTY) Map options) +public record OllamaEmbedRequest(@JsonIgnore String apiKey, String model, List input, + boolean truncate, @JsonInclude(Include.NON_EMPTY) Map options) { private OllamaEmbedRequest(Builder builder) { - this(builder.model, builder.input, builder.truncate, builder.options); + this(builder.apiKey, builder.model, builder.input, builder.truncate, builder.options); } public static Builder builder() @@ -47,6 +48,7 @@ public static Builder builder() public static final class Builder { + private String apiKey; private String model; private boolean truncate = true; private List input = new ArrayList<>(); @@ -56,6 +58,12 @@ private Builder() { } + public Builder withApiKey(String aApiKey) + { + apiKey = aApiKey; + return this; + } + public Builder withModel(String aModel) { model = aModel; diff --git a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/client/OllamaLlmChatClient.java b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/client/OllamaLlmChatClient.java new file mode 100644 index 0000000000..e157784fa2 --- /dev/null +++ b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/client/OllamaLlmChatClient.java @@ -0,0 +1,299 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client; + +import java.io.IOException; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; + +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ChatMessage; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ChatChunk; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ChatOptions; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ChatResult; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.FinishReason; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClient; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmEndpoint; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ModelCapability; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ModelInfo; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ToolCall; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ToolDescriptor; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.UsageInfo; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.response.ResponseFormat; +import de.tudarmstadt.ukp.inception.security.client.auth.apikey.ApiKeyAuthenticationTraits; +import de.tudarmstadt.ukp.inception.support.json.JSONUtil; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.node.JsonNodeFactory; +import tools.jackson.databind.node.ObjectNode; + +/** + * {@link LlmChatClient} adapter for Ollama, exposing chat, streaming, embeddings, and model + * discovery through the provider-neutral abstraction. Delegates to the existing + * {@link OllamaClient}; pure pass-through with no recommender-specific defaults applied here. + *

+ * Tool calling is bridged here: {@link ChatOptions#tools()} are translated from + * {@code ToolDescriptor} into {@link OllamaTool} when building the chat request. + */ +public class OllamaLlmChatClient + implements LlmChatClient +{ + public static final String ID = "ollama"; + + private final OllamaClient client; + + public OllamaLlmChatClient(OllamaClient aClient) + { + client = aClient; + } + + @Override + public String getId() + { + return ID; + } + + @Override + public Set supportedCapabilities() + { + return EnumSet.of( // + ModelCapability.CHAT, // + ModelCapability.JSON_SCHEMA, // + ModelCapability.STREAMING, // + ModelCapability.EMBEDDINGS, // + ModelCapability.TOOLS); + } + + @Override + public ChatResult chat(LlmEndpoint aEndpoint, List aMessages, ChatOptions aOptions) + throws IOException + { + var request = buildChatRequest(aEndpoint, aMessages, aOptions, false); + var response = client.chat(aEndpoint.url(), request); + return toChatResult(response); + } + + @Override + public ChatResult chatStream(LlmEndpoint aEndpoint, List aMessages, + ChatOptions aOptions, Consumer aOnChunk) + throws IOException + { + var request = buildChatRequest(aEndpoint, aMessages, aOptions, true); + Consumer callback = chunk -> { + var msg = chunk.getMessage(); + if (msg == null) { + return; + } + if (msg.content() == null && msg.thinking() == null) { + return; + } + aOnChunk.accept(new ChatChunk(msg.content(), msg.thinking())); + }; + var response = client.chat(aEndpoint.url(), request, callback); + return toChatResult(response); + } + + @Override + public List embed(LlmEndpoint aEndpoint, List aInputs, + Map aOptions) + throws IOException + { + var builder = OllamaEmbedRequest.builder() // + .withApiKey(apiKey(aEndpoint)) // + .withModel(aEndpoint.model()) // + .withInput(aInputs); + + if (aOptions != null && !aOptions.isEmpty()) { + builder.withOptions(aOptions); + } + + return client.embed(aEndpoint.url(), builder.build()).stream() // + .map(p -> p.getRight()) // + .toList(); + } + + @Override + public List listModels(LlmEndpoint aEndpoint) throws IOException + { + return client.listModels(aEndpoint.url(), apiKey(aEndpoint)).stream() // + .map(t -> new ModelInfo(t.name())) // + .toList(); + } + + private OllamaChatRequest buildChatRequest(LlmEndpoint aEndpoint, List aMessages, + ChatOptions aOptions, boolean aStream) + { + var messages = aMessages.stream() // + .map(OllamaLlmChatClient::toOllamaMessage) // + .toList(); + + var builder = OllamaChatRequest.builder() // + .withApiKey(apiKey(aEndpoint)) // + .withModel(aEndpoint.model()) // + .withMessages(messages) // + .withFormat(toFormat(aOptions)) // + .withThink(false) // + .withStream(aStream); + + if (aOptions.options() != null) { + builder.withExtraOptions(aOptions.options()); + } + + if (aOptions.tools() != null && !aOptions.tools().isEmpty()) { + builder.withTools(aOptions.tools().stream() // + .map(OllamaLlmChatClient::toOllamaTool) // + .toList()); + } + + return builder.build(); + } + + private static OllamaChatMessage toOllamaMessage(ChatMessage aMessage) + { + // tool_call_id is not currently part of the OllamaChatMessage DTO; Ollama matches tool + // results to calls positionally. Drop the id on the way out. + return new OllamaChatMessage(aMessage.role().getName(), aMessage.content()); + } + + private static OllamaTool toOllamaTool(ToolDescriptor aDescriptor) + { + var function = OllamaFunction.builder() // + .withName(aDescriptor.name()) // + .withDescription(aDescriptor.description()) // + .withParameters(toOllamaParameters(aDescriptor.parametersSchema())) // + .build(); + return OllamaTool.builder() // + .withType("function") // + .withFunction(function) // + .build(); + } + + private static OllamaFunctionParameters toOllamaParameters(JsonNode aSchema) + { + var builder = OllamaFunctionParameters.builder(); + if (aSchema == null || !aSchema.isObject()) { + return builder.build(); + } + + var typeNode = aSchema.get("type"); + if (typeNode != null && typeNode.isTextual()) { + builder.withType(typeNode.asText()); + } + + var requiredNode = aSchema.get("required"); + if (requiredNode != null && requiredNode.isArray()) { + for (var item : requiredNode) { + if (item.isTextual()) { + builder.addRequired(item.asText()); + } + } + } + + var propsNode = aSchema.get("properties"); + if (propsNode != null && propsNode.isObject()) { + for (var entry : propsNode.properties()) { + if (entry.getValue() instanceof ObjectNode propDef) { + builder.addProperty(entry.getKey(), propDef); + } + } + } + + return builder.build(); + } + + private static ChatResult toChatResult(OllamaChatResponse aResponse) + { + var ollamaMessage = aResponse.getMessage(); + var content = ollamaMessage != null && ollamaMessage.content() != null // + ? ollamaMessage.content() // + : ""; + var role = ollamaMessage != null && ollamaMessage.role() != null // + ? roleFromOllama(ollamaMessage.role()) // + : ChatMessage.Role.ASSISTANT; + + var toolCalls = ollamaMessage != null && ollamaMessage.toolCalls() != null // + ? ollamaMessage.toolCalls().stream() // + .map(OllamaLlmChatClient::toToolCall) // + .toList() // + : Collections. emptyList(); + + var finishReason = aResponse.isDone() // + ? (toolCalls.isEmpty() ? FinishReason.STOP : FinishReason.TOOL_CALLS) // + : null; + + var usage = new UsageInfo( // + aResponse.getPromptEvalCount() != 0 ? aResponse.getPromptEvalCount() : null, // + aResponse.getEvalCount() != 0 ? aResponse.getEvalCount() : null, // + aResponse.getPromptEvalCount() + aResponse.getEvalCount() != 0 // + ? aResponse.getPromptEvalCount() + aResponse.getEvalCount() // + : null); + + var thinking = ollamaMessage != null ? ollamaMessage.thinking() : null; + return new ChatResult(new ChatMessage(role, content, thinking, null), toolCalls, + finishReason, usage); + } + + private static ToolCall toToolCall(OllamaToolCall aCall) + { + var fn = aCall.getFunction(); + if (fn == null) { + return new ToolCall(null, null, null); + } + // Round-trip through the object mapper so values become proper TextNode / IntNode / ... + // — putPOJO would wrap them in POJONode, which makes JsonNode#isTextual / isNumber lie. + var args = (ObjectNode) JSONUtil.getObjectMapper().valueToTree(fn.getArguments()); + return new ToolCall(null, fn.getName(), args); + } + + private static ChatMessage.Role roleFromOllama(String aRole) + { + return switch (aRole) { + case "system" -> ChatMessage.Role.SYSTEM; + case "user" -> ChatMessage.Role.USER; + case "tool" -> ChatMessage.Role.TOOL; + default -> ChatMessage.Role.ASSISTANT; + }; + } + + private static JsonNode toFormat(ChatOptions aOptions) + { + if (aOptions.jsonSchema() != null) { + return aOptions.jsonSchema(); + } + if (aOptions.responseFormat() == ResponseFormat.JSON) { + return JsonNodeFactory.instance.textNode("json"); + } + return null; + } + + private static String apiKey(LlmEndpoint aEndpoint) + { + var auth = aEndpoint.auth(); + if (auth instanceof ApiKeyAuthenticationTraits apiKeyAuth) { + return apiKeyAuth.getApiKey(); + } + if (auth == null) { + return null; + } + throw new IllegalArgumentException( + "Ollama client only supports " + ApiKeyAuthenticationTraits.class.getSimpleName() + + " or no auth; got [" + auth.getClass().getName() + "]"); + } +} diff --git a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/config/OllamaRecommenderAutoConfiguration.java b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/config/OllamaRecommenderAutoConfiguration.java index 4f8ede7958..5eee305207 100644 --- a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/config/OllamaRecommenderAutoConfiguration.java +++ b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/config/OllamaRecommenderAutoConfiguration.java @@ -24,9 +24,11 @@ import org.springframework.context.annotation.Configuration; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.AnnotationTaskCodecExtensionPoint; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClientExtensionPoint; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.OllamaRecommenderFactory; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaClient; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaClientImpl; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaLlmChatClient; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaMetrics; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaMetricsImpl; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; @@ -47,14 +49,21 @@ public OllamaMetrics ollamaMetrics() return new OllamaMetricsImpl(); } + // Like ollamaClient itself, the adapter is unconditional because the assistant needs it. + @Bean + public OllamaLlmChatClient ollamaLlmChatClient(OllamaClient aClient) + { + return new OllamaLlmChatClient(aClient); + } + @ConditionalOnProperty(prefix = "recommender.ollama", name = "enabled", havingValue = "true", // matchIfMissing = false) @Bean - public OllamaRecommenderFactory ollamaRecommenderFactory(OllamaClient aClient, - AnnotationSchemaService aSchemaService, - AnnotationTaskCodecExtensionPoint aResponseExtractorExtensionPoint) + public OllamaRecommenderFactory ollamaRecommenderFactory(AnnotationSchemaService aSchemaService, + AnnotationTaskCodecExtensionPoint aResponseExtractorExtensionPoint, + LlmChatClientExtensionPoint aChatClientExtensionPoint) { - return new OllamaRecommenderFactory(aClient, aSchemaService, - aResponseExtractorExtensionPoint); + return new OllamaRecommenderFactory(aSchemaService, aResponseExtractorExtensionPoint, + aChatClientExtensionPoint); } } diff --git a/inception/inception-imls-ollama/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommenderTest.java b/inception/inception-imls-ollama/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommenderTest.java index 38ebe38857..bf29e88f65 100644 --- a/inception/inception-imls-ollama/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommenderTest.java +++ b/inception/inception-imls-ollama/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommenderTest.java @@ -53,9 +53,12 @@ import de.tudarmstadt.ukp.inception.recommendation.api.recommender.PredictionContext; import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommenderContext; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.AnnotationTaskCodecExtensionPointImpl; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmChatClientExtensionPointImpl; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.OllamaRecommender; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.OllamaRecommenderTraits; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaClientImpl; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaLlmChatClient; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaMetricsImpl; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.response.ExtractionMode; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.response.LabellingAnnotationTaskCodec; import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.response.SpanJsonAnnotationTaskCodec; @@ -77,6 +80,7 @@ class OllamaRecommenderTest private Recommender recommender; private CAS cas; private AnnotationTaskCodecExtensionPointImpl responseExtractorExtensionPoint; + private LlmChatClientExtensionPointImpl chatClientExtensionPoint; @BeforeAll static void checkIfOllamaIsRunning() @@ -100,6 +104,11 @@ void setup() throws Exception new LabellingAnnotationTaskCodec())); responseExtractorExtensionPoint.init(); + var ollamaAdapter = new OllamaLlmChatClient(new OllamaClientImpl( + java.net.http.HttpClient.newBuilder().build(), new OllamaMetricsImpl())); + chatClientExtensionPoint = new LlmChatClientExtensionPointImpl(asList(ollamaAdapter)); + chatClientExtensionPoint.init(); + var tsd = TypeSystemDescriptionFactory.createTypeSystemDescription(); RecommenderTypeSystemUtils.addPredictionFeaturesToTypeSystem(tsd, asList(feature)); cas = CasFactory.createCas(tsd); @@ -116,35 +125,8 @@ void testPerDocumentUsingReponseAsLabel() throws Exception traits.setPromptingMode(PER_DOCUMENT); traits.setExtractionMode(ExtractionMode.RESPONSE_AS_LABEL); - var sut = new OllamaRecommender(recommender, traits, new OllamaClientImpl(), schemaSerivce, - responseExtractorExtensionPoint); - sut.predict(new PredictionContext(new RecommenderContext()), cas); - - var predictions = cas.select(NamedEntity.class) // - .filter(ne -> getFeature(ne, FEATURE_NAME_IS_PREDICTION, Boolean.class)) // - .toList(); - - predictions.forEach($ -> LOG.info("Prediction: {} {}", $.getCoveredText(), $.getValue())); - assertThat(predictions).as("predictions").isNotEmpty(); - } - - @Test - void testPerDocumentUsingMentionsFromJsonList_Numbers() throws Exception - { - cas.setDocumentText("1 2 3 4 5 6 7 8 9 10"); - - var traits = new OllamaRecommenderTraits(); - traits.setModel(DEFAULT_MODEL); - traits.setPrompt(""" - Identify all even numbers in the following list and return them as JSON. - - {{ text }}"""); - traits.setPromptingMode(PER_DOCUMENT); - traits.setExtractionMode(MENTIONS_FROM_JSON); - traits.setStructuredOutputSupported(false); - - var sut = new OllamaRecommender(recommender, traits, new OllamaClientImpl(), schemaSerivce, - responseExtractorExtensionPoint); + var sut = new OllamaRecommender(recommender, traits, schemaSerivce, + responseExtractorExtensionPoint, chatClientExtensionPoint); sut.predict(new PredictionContext(new RecommenderContext()), cas); var predictions = cas.select(NamedEntity.class) // @@ -155,59 +137,6 @@ void testPerDocumentUsingMentionsFromJsonList_Numbers() throws Exception assertThat(predictions).as("predictions").isNotEmpty(); } - @Test - void testPerDocumentUsingMentionsFromJsonList_Entities() throws Exception - { - cas.setDocumentText( - "John is going to work at the diner tomorrow. There, he meets a guy working at Starbucks."); - - var traits = new OllamaRecommenderTraits(); - traits.setModel(DEFAULT_MODEL); - traits.setPrompt("Identify all named entities in the following text.\n\n{{ text }}"); - traits.setPromptingMode(PER_DOCUMENT); - traits.setExtractionMode(MENTIONS_FROM_JSON); - traits.setStructuredOutputSupported(false); - - var sut = new OllamaRecommender(recommender, traits, new OllamaClientImpl(), schemaSerivce, - responseExtractorExtensionPoint); - sut.predict(new PredictionContext(new RecommenderContext()), cas); - - var predictions = cas.select(NamedEntity.class) - .filter(ne -> getFeature(ne, FEATURE_NAME_IS_PREDICTION, Boolean.class)).toList(); - - predictions.forEach($ -> LOG.info("Prediction: {} {}", $.getCoveredText(), $.getValue())); - assertThat(predictions).as("predictions").isNotEmpty(); - } - - @Test - void testPerDocumentUsingMentionsFromJsonList_Politicians() throws Exception - { - cas.setDocumentText(""" - John is will meet President Livingston tomorrow. - They will lunch together with the minister of foreign affairs. - Later they meet the the Lord of Darkness, Don Horny."""); - - var traits = new OllamaRecommenderTraits(); - traits.setModel(DEFAULT_MODEL); - traits.setPrompt(""" - Identify all politicians in the following text and return them as JSON. - - {{ text }}"""); - traits.setPromptingMode(PER_DOCUMENT); - traits.setExtractionMode(MENTIONS_FROM_JSON); - traits.setStructuredOutputSupported(false); - - var sut = new OllamaRecommender(recommender, traits, new OllamaClientImpl(), schemaSerivce, - responseExtractorExtensionPoint); - sut.predict(new PredictionContext(new RecommenderContext()), cas); - - var predictions = cas.select(NamedEntity.class) - .filter(ne -> getFeature(ne, FEATURE_NAME_IS_PREDICTION, Boolean.class)).toList(); - - predictions.forEach($ -> LOG.info("Prediction: {} {}", $.getCoveredText(), $.getValue())); - assertThat(predictions).as("predictions").isNotEmpty(); - } - @Test void testPerSentenceUsingMentionsFromJsonList_Politicians_fewShjot() throws Exception { @@ -248,8 +177,8 @@ void testPerSentenceUsingMentionsFromJsonList_Politicians_fewShjot() throws Exce traits.setPromptingMode(PER_SENTENCE); traits.setExtractionMode(MENTIONS_FROM_JSON); - var sut = new OllamaRecommender(recommender, traits, new OllamaClientImpl(), schemaSerivce, - responseExtractorExtensionPoint); + var sut = new OllamaRecommender(recommender, traits, schemaSerivce, + responseExtractorExtensionPoint, chatClientExtensionPoint); sut.predict(new PredictionContext(new RecommenderContext()), cas); var predictions = cas.select(NamedEntity.class) diff --git a/inception/inception-imls-ollama/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/client/OllamaClientImplTest.java b/inception/inception-imls-ollama/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/client/OllamaClientImplTest.java index 9ded54a793..87db272de0 100644 --- a/inception/inception-imls-ollama/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/client/OllamaClientImplTest.java +++ b/inception/inception-imls-ollama/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/client/OllamaClientImplTest.java @@ -139,7 +139,7 @@ void testJson() throws Exception @Test void testListModels() throws Exception { - var response = sut.listModels(DEFAULT_OLLAMA_URL); + var response = sut.listModels(DEFAULT_OLLAMA_URL, null); LOG.info("Response: [{}]", response); } @@ -195,9 +195,12 @@ public String getWeather( static List functionCallModels() throws IOException { var sut = new OllamaClientImpl(); - var allModels = sut.listModels(DEFAULT_OLLAMA_URL); + var allModels = sut.listModels(DEFAULT_OLLAMA_URL, null); return allModels.stream() // + // Cloud-only models (":cloud" tag) are proxied through Ollama's hosted service + // and require a paid subscription - skip them so the test doesn't 403. + .filter(model -> !model.name().endsWith(":cloud")) // .filter(model -> { try { var modelInfo = sut.getModelInfo(DEFAULT_OLLAMA_URL, // diff --git a/inception/inception-imls-ollama/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/client/OllamaLlmChatClientTest.java b/inception/inception-imls-ollama/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/client/OllamaLlmChatClientTest.java new file mode 100644 index 0000000000..f8abeaeffd --- /dev/null +++ b/inception/inception-imls-ollama/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/client/OllamaLlmChatClientTest.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.ollama.client; + +import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.ChatMessage.Role.USER; +import static de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.OllamaRecommenderTraits.DEFAULT_OLLAMA_URL; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; + +import java.lang.invoke.MethodHandles; +import java.net.http.HttpClient; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ChatMessage; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.Tool; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ToolParam; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ChatChunk; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ChatOptions; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.FinishReason; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.LlmEndpoint; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.client.ToolDescriptor; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaClientImpl; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaLlmChatClient; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.ollama.client.OllamaMetricsImpl; +import de.tudarmstadt.ukp.inception.recommendation.imls.llm.support.response.ResponseFormat; +import de.tudarmstadt.ukp.inception.support.json.JSONUtil; +import de.tudarmstadt.ukp.inception.support.test.http.HttpTestUtils; + +/** + * Exercises {@link OllamaLlmChatClient} against a locally running Ollama. Requires {@code + * ministral-3:8b} and {@code granite-embedding:278m-fp16} to be pulled; skipped if no Ollama is + * reachable. + */ +class OllamaLlmChatClientTest +{ + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final String CHAT_MODEL = "ministral-3:8b"; + private static final String EMBED_MODEL = "granite-embedding:278m-fp16"; + + private final OllamaLlmChatClient sut = new OllamaLlmChatClient( + new OllamaClientImpl(HttpClient.newBuilder().build(), new OllamaMetricsImpl())); + + @BeforeAll + static void checkIfOllamaIsRunning() + { + assumeThat(HttpTestUtils.checkURL(DEFAULT_OLLAMA_URL)).isTrue(); + } + + private static LlmEndpoint endpoint(String aModel) + { + return new LlmEndpoint(OllamaLlmChatClient.ID, DEFAULT_OLLAMA_URL, aModel, null); + } + + @Test + void testChat() throws Exception + { + var messages = List.of(new ChatMessage(USER, "Tell me a joke in one sentence.")); + + var result = sut.chat(endpoint(CHAT_MODEL), messages, ChatOptions.defaults()); + + LOG.info("Response: [{}]", result.message().content()); + LOG.info("Usage: {}", result.usage()); + assertThat(result.message().content()).isNotBlank(); + assertThat(result.message().role()).isEqualTo(ChatMessage.Role.ASSISTANT); + assertThat(result.finishReason()).isNotNull(); + assertThat(result.usage()).isNotNull(); + assertThat(result.usage().totalTokens()).isPositive(); + } + + @Test + void testChatWithJsonResponseFormat() throws Exception + { + var messages = List.of(new ChatMessage(USER, + "Return a JSON object with the keys `a` set to 1 and `b` set to 2.")); + var options = new ChatOptions(ResponseFormat.JSON, null, List.of(), null); + + var result = sut.chat(endpoint(CHAT_MODEL), messages, options); + + LOG.info("Response: [{}]", result.message().content()); + assertThat(JSONUtil.getObjectMapper().readTree(result.message().content())).isNotNull(); + } + + @Test + void testChatStream() throws Exception + { + var messages = List.of(new ChatMessage(USER, "Count from 1 to 5.")); + var chunks = new ArrayList(); + + var result = sut.chatStream(endpoint(CHAT_MODEL), messages, ChatOptions.defaults(), + chunks::add); + + LOG.info("Got {} chunks, final: [{}]", chunks.size(), result.message().content()); + assertThat(chunks).isNotEmpty(); + assertThat(result.message().content()).isNotBlank(); + } + + @Test + void testEmbed() throws Exception + { + var vectors = sut.embed(endpoint(EMBED_MODEL), List.of("hello world", "good morning"), + null); + + assertThat(vectors).hasSize(2); + assertThat(vectors.get(0)).isNotEmpty(); + assertThat(vectors.get(0).length).isEqualTo(vectors.get(1).length); + LOG.info("Embedding dim: {}", vectors.get(0).length); + } + + @Test + void testListModels() throws Exception + { + var models = sut.listModels(endpoint(null)); + + LOG.info("Models: {}", models); + assertThat(models).extracting("id").contains(CHAT_MODEL); + } + + static class WeatherService + { + @Tool(value = "get_current_weather", description = "Get the current weather for a given location.") + @SuppressWarnings("unused") + String getCurrentWeather(@ToolParam(value = "location", // + description = "The city to get the weather for.") String aLocation) + { + return "sunny"; + } + } + + @Test + void testChatWithTool() throws Exception + { + var weatherTool = ToolDescriptor.fromMethod( + WeatherService.class.getDeclaredMethod("getCurrentWeather", String.class)); + + var messages = List + .of(new ChatMessage(USER, "What is the current weather in Berlin? Use the tool.")); + var options = new ChatOptions(null, null, List.of(weatherTool), null); + + var result = sut.chat(endpoint(CHAT_MODEL), messages, options); + + LOG.info("Finish reason: {}, tool calls: {}", result.finishReason(), result.toolCalls()); + + assertThat(result.toolCalls()).as("tool calls").isNotEmpty(); + assertThat(result.finishReason()).isEqualTo(FinishReason.TOOL_CALLS); + + var call = result.toolCalls().get(0); + assertThat(call.name()).isEqualTo("get_current_weather"); + assertThat(call.arguments()).as("arguments JsonNode").isNotNull(); + assertThat(call.arguments().isObject()).as("arguments is an object node").isTrue(); + + // valueToTree should yield proper value nodes — not POJONode wrappers — so the + // JSON-tree-style type predicates report correctly. + var location = call.arguments().get("location"); + assertThat(location).as("location argument node").isNotNull(); + assertThat(location.isTextual()).as("location node is TextNode").isTrue(); + assertThat(location.asText()).containsIgnoringCase("berlin"); + } +} diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/llm/InteractiveRecommenderSidebar.java b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/llm/InteractiveRecommenderSidebar.java index bcd4edc191..b8a6d37c4a 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/llm/InteractiveRecommenderSidebar.java +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/llm/InteractiveRecommenderSidebar.java @@ -350,10 +350,17 @@ private List listInteractiveRecommenders() { return recommendationService.listEnabledRecommenders(getModelObject().getProject()).stream() // .filter(rec -> recommendationService.getRecommenderFactory(rec) - .map(factory -> factory.isInteractive(rec)).orElse(false)) // + .map(factory -> factory.isInteractive(rec) && hasLlmTraits(factory)) + .orElse(false)) // .toList(); } + private static boolean hasLlmTraits(RecommendationEngineFactory aFactory) + { + var traits = aFactory.createTraits(); + return traits instanceof LlmRecommenderTraits; + } + private List listFeatures() { if (!recommender.isPresent().getObject()) { diff --git a/inception/inception-support/src/main/java/de/tudarmstadt/ukp/inception/support/extensionpoint/ExtensionPoint_ImplBase.java b/inception/inception-support/src/main/java/de/tudarmstadt/ukp/inception/support/extensionpoint/ExtensionPoint_ImplBase.java index 93cc6b3b0a..34f052b1b8 100644 --- a/inception/inception-support/src/main/java/de/tudarmstadt/ukp/inception/support/extensionpoint/ExtensionPoint_ImplBase.java +++ b/inception/inception-support/src/main/java/de/tudarmstadt/ukp/inception/support/extensionpoint/ExtensionPoint_ImplBase.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Optional; @@ -71,9 +72,46 @@ public void init() BOOT_LOG.info("Found [{}] {} extensions", extensions.size(), getClass().getSimpleName()); + if (enforceUniqueIds()) { + checkForDuplicateIds(extensions); + } + extensionsList = unmodifiableList(extensions); } + /** + * Whether two extensions sharing the same id should fail this extension point at startup. + * Default is {@code true} since most extension points look up extensions by id via + * {@link #getExtension(String)} and silently shadowing a duplicate would be a real bug. + * Override to {@code false} for extension points that intentionally dispatch by + * {@link Extension#accepts(Object)} on extensions that share an id (e.g. multiple handlers + * registered for the same command). + */ + protected boolean enforceUniqueIds() + { + return true; + } + + private void checkForDuplicateIds(List aExtensions) + { + // Null ids typically only occur in test wiring that bypasses Spring's BeanNameAware + // initialization; lookup via getExtension(id) cannot find them anyway, so don't conflate + // them with duplicate-id bugs. + var seen = new HashMap(); + for (var ext : aExtensions) { + var id = ext.getId(); + if (id == null) { + continue; + } + var existing = seen.put(id, ext); + if (existing != null) { + throw new IllegalStateException("Duplicate extension id [" + id + "] in " + + getClass().getSimpleName() + ": [" + existing.getClass().getName() + + "] vs [" + ext.getClass().getName() + "]"); + } + } + } + @SuppressWarnings({ "unchecked", "rawtypes" }) protected Comparator makeComparator() { @@ -106,8 +144,13 @@ public List getExtensions(C aContext) @Override public Optional getExtension(String aId) { + if (aId == null) { + return Optional.empty(); + } + // Null-safe: an extension that returns null from getId() (e.g. a Spring-managed + // FeatureSupport in a test that bypassed BeanNameAware) is simply not findable here. return (Optional) getExtensions().stream() // - .filter(fs -> fs.getId().equals(aId)) // + .filter(fs -> aId.equals(fs.getId())) // .findFirst(); } }