diff --git a/deploys/standalone/docker-compose-dev.yaml b/deploys/standalone/docker-compose-dev.yaml index 57bb94afff..4c4097290a 100644 --- a/deploys/standalone/docker-compose-dev.yaml +++ b/deploys/standalone/docker-compose-dev.yaml @@ -21,6 +21,7 @@ services: ZK_HOST: zoo:2181 SOLR_HOST: localhost SOLR_OPTS: "-Dsolr.environment=dev,label=DEV+ENV" + SOLR_MODULES: language-models depends_on: - zoo volumes: diff --git a/deploys/standalone/docker-compose-vector-test.yaml b/deploys/standalone/docker-compose-vector-test.yaml new file mode 100644 index 0000000000..9f86585be6 --- /dev/null +++ b/deploys/standalone/docker-compose-vector-test.yaml @@ -0,0 +1,67 @@ +# Minimal stack for live-testing Solr text-to-vector semantic search. +# Brings up just ZooKeeper + Solr (with the +# language-models module enabled) + PostgreSQL - enough for RODA to bootstrap +# its schema (incl. registering the text-to-vector model against an external +# embedding API) without the full docker-compose-dev.yaml stack. +# +# Usage: +# docker compose -f deploys/standalone/docker-compose-vector-test.yaml up -d +# +# Then run RODA (or a bootstrap-only script) with: +# RODA_CORE_SOLR_TYPE=CLOUD +# RODA_CORE_SOLR_CLOUD_URLS=localhost:2182 +# SPRING_DATASOURCE_URL=jdbc:postgresql://localhost:5433/roda_core_db +# SPRING_DATASOURCE_USERNAME=admin +# SPRING_DATASOURCE_PASSWORD=roda +# RODA_CORE_INDEX_EMBEDDING_ENABLED=true +# RODA_CORE_INDEX_EMBEDDING_BASE_URL= +# RODA_CORE_INDEX_EMBEDDING_MODEL_NAME= +# RODA_CORE_INDEX_EMBEDDING_SOLR_MODEL=roda-embedding-model +# RODA_CORE_INDEX_EMBEDDING_DIMENSIONS= +services: + zoo-vector-test: + image: docker.io/zookeeper:3.9.5-jre-17 + container_name: zoo-vector-test + restart: unless-stopped + ports: + - "2182:2181" + environment: + - ZOO_4LW_COMMANDS_WHITELIST=mntr,conf,ruok + volumes: + - zoo_vector_test_data:/data + - zoo_vector_test_datalog:/datalog + + solr-vector-test: + image: docker.io/solr:10.0.0 + container_name: solr-vector-test + restart: unless-stopped + ports: + - "8984:8983" + environment: + ZK_HOST: zoo-vector-test:2181 + SOLR_HOST: localhost + SOLR_OPTS: "-Dsolr.environment=vector-test,label=VECTOR+TEST" + SOLR_MODULES: language-models + depends_on: + - zoo-vector-test + volumes: + - solr_vector_test_data:/var/solr + + postgres-vector-test: + image: postgres:17 + container_name: postgres-vector-test + restart: unless-stopped + environment: + POSTGRES_USER: admin + POSTGRES_PASSWORD: roda + POSTGRES_DB: roda_core_db + ports: + - "5433:5432" + volumes: + - pg_vector_test_data:/var/lib/postgresql/data + +volumes: + zoo_vector_test_data: + zoo_vector_test_datalog: + solr_vector_test_data: + pg_vector_test_data: diff --git a/deploys/standalone/docker-compose.yaml b/deploys/standalone/docker-compose.yaml index 82f4dd2c6a..f97f213636 100644 --- a/deploys/standalone/docker-compose.yaml +++ b/deploys/standalone/docker-compose.yaml @@ -16,6 +16,7 @@ services: environment: SOLR_HEAP: 2g ZK_HOST: zoo:2181 + SOLR_MODULES: language-models volumes: - solr_data:/var/solr diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java index b39f3640d4..86e85eba1b 100644 --- a/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java @@ -48,6 +48,14 @@ public final class RodaConstants { public static final String CORE_SOLR_CLOUD_URLS = "core.solr.cloud.urls"; public static final String CORE_SOLR_STEMMING_LANGUAGE = "core.solr.stemming.language"; + public static final String CORE_INDEX_EMBEDDING_ENABLED = "core.index.embedding.enabled"; + public static final String CORE_INDEX_EMBEDDING_BASE_URL = "core.index.embedding.base_url"; + public static final String CORE_INDEX_EMBEDDING_API_KEY = "core.index.embedding.api_key"; + public static final String CORE_INDEX_EMBEDDING_MODEL_NAME = "core.index.embedding.model_name"; + public static final String CORE_INDEX_EMBEDDING_SOLR_MODEL = "core.index.embedding.solr_model"; + public static final String CORE_INDEX_EMBEDDING_DIMENSIONS = "core.index.embedding.dimensions"; + public static final String CORE_INDEX_EMBEDDING_DEFAULT_TOP_K = "core.index.embedding.default_top_k"; + public static final String CORE_EVENTS_ENABLED = "core.events.enabled"; public static final String CORE_EVENTS_NOTIFIER_AND_HANDLER_ARE_THE_SAME = "core.events.notifier_and_handler_are_the_same"; public static final String CORE_EVENTS_NOTIFIER_CLASS = "core.events.notifier_class"; @@ -1061,6 +1069,9 @@ public enum OrchestratorType { public static final String INDEX_SEARCH = "search"; public static final String INDEX_WILDCARD = "*"; + public static final String INDEX_EMBEDDING_VECTOR = "embedding_vector"; + public static final String INDEX_VECTORIZED = "vectorized_b"; + public static final String INDEX_INSTANCE_ID = "instanceId"; public static final String INDEX_INSTANCE_NAME = "instanceName"; diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/filter/FilterParameter.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/filter/FilterParameter.java index 327d4b1564..62e5ede0d9 100644 --- a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/filter/FilterParameter.java +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/filter/FilterParameter.java @@ -39,12 +39,14 @@ @JsonSubTypes.Type(value = AndFiltersParameters.class, name = "AndFiltersParameters"), @JsonSubTypes.Type(value = AllFilterParameter.class, name = "AllFilterParameter"), @JsonSubTypes.Type(value = ParentWhichFilterParameter.class, name = "ParentWhichFilterParameter"), - @JsonSubTypes.Type(value = ChildOfFilterParameter.class, name = "ChildOfFilterParameter"),}) + @JsonSubTypes.Type(value = ChildOfFilterParameter.class, name = "ChildOfFilterParameter"), + @JsonSubTypes.Type(value = TextToVectorFilterParameter.class, name = "TextToVectorFilterParameter"),}) @Schema(type = "object", subTypes = {BasicSearchFilterParameter.class, EmptyKeyFilterParameter.class, LikeFilterParameter.class, NotSimpleFilterParameter.class, OneOfManyFilterParameter.class, DateIntervalFilterParameter.class, DateRangeFilterParameter.class, LongRangeFilterParameter.class, StringRangeFilterParameter.class, SimpleFilterParameter.class, OrFiltersParameters.class, AndFiltersParameters.class, - AllFilterParameter.class, ParentWhichFilterParameter.class, ChildOfFilterParameter.class}, discriminatorMapping = { + AllFilterParameter.class, ParentWhichFilterParameter.class, ChildOfFilterParameter.class, + TextToVectorFilterParameter.class}, discriminatorMapping = { @DiscriminatorMapping(value = "BasicSearchFilterParameter", schema = BasicSearchFilterParameter.class), @DiscriminatorMapping(value = "LikeFilterParameter", schema = LikeFilterParameter.class), @DiscriminatorMapping(value = "NotSimpleFilterParameter", schema = NotSimpleFilterParameter.class), @@ -58,7 +60,9 @@ @DiscriminatorMapping(value = "AndFiltersParameters", schema = AndFiltersParameters.class), @DiscriminatorMapping(value = "AllFilterParameter", schema = AllFilterParameter.class), @DiscriminatorMapping(value = "ParentWhichFilterParameter", schema = ParentWhichFilterParameter.class), - @DiscriminatorMapping(value = "ChildOfFilterParameter", schema = ChildOfFilterParameter.class)}, discriminatorProperty = "type") + @DiscriminatorMapping(value = "ChildOfFilterParameter", schema = ChildOfFilterParameter.class), + @DiscriminatorMapping(value = "TextToVectorFilterParameter", + schema = TextToVectorFilterParameter.class)}, discriminatorProperty = "type") public abstract class FilterParameter implements Serializable { @Serial private static final long serialVersionUID = 3744111668897879761L; diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/filter/TextToVectorFilterParameter.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/filter/TextToVectorFilterParameter.java new file mode 100644 index 0000000000..7ad0ae3e0c --- /dev/null +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/filter/TextToVectorFilterParameter.java @@ -0,0 +1,133 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE file at the root of the source + * tree and available online at + * + * https://github.com/keeps/roda + */ +package org.roda.core.data.v2.index.filter; + +import java.io.Serial; + +import com.fasterxml.jackson.annotation.JsonTypeName; + +/** + * Semantic search parameter that vectorizes {@link #getQuery()} at query time + * (via Solr's text-to-vector module) and restricts results to the + * {@link #getTopK()} nearest neighbours of {@link #getField()}. Vectors + * themselves are populated out-of-band by an external enrichment service, not + * by RODA. + */ +@JsonTypeName("TextToVectorFilterParameter") +public class TextToVectorFilterParameter extends FilterParameter { + @Serial + private static final long serialVersionUID = 1L; + + private String field; + private String query; + private String model; + private int topK; + + /** + * Constructs an empty {@link TextToVectorFilterParameter}. + */ + public TextToVectorFilterParameter() { + // do nothing + } + + public TextToVectorFilterParameter(String field, String query, String model, int topK) { + setField(field); + setQuery(query); + setModel(model); + setTopK(topK); + } + + public TextToVectorFilterParameter(TextToVectorFilterParameter other) { + this(other.getField(), other.getQuery(), other.getModel(), other.getTopK()); + } + + /** + * @return the name of the {@code knn_vector} field to search against. + */ + public String getField() { + return field; + } + + public void setField(String field) { + this.field = field; + } + + /** + * @return the free-text query to vectorize. + */ + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + /** + * @return the name the embedding model is registered under in Solr's + * text-to-vector-model-store. + */ + public String getModel() { + return model; + } + + public void setModel(String model) { + this.model = model; + } + + /** + * @return the number of nearest neighbours to retrieve. + */ + public int getTopK() { + return topK; + } + + public void setTopK(int topK) { + this.topK = topK; + } + + @Override + public String toString() { + return "TextToVectorFilterParameter(field=" + field + ", query=" + query + ", model=" + model + ", topK=" + topK + + ")"; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((field == null) ? 0 : field.hashCode()); + result = prime * result + ((query == null) ? 0 : query.hashCode()); + result = prime * result + ((model == null) ? 0 : model.hashCode()); + result = prime * result + topK; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!super.equals(obj)) { + return false; + } + if (!(obj instanceof TextToVectorFilterParameter other)) { + return false; + } + if (topK != other.topK) { + return false; + } + if (field == null ? other.field != null : !field.equals(other.field)) { + return false; + } + if (query == null ? other.query != null : !query.equals(other.query)) { + return false; + } + return model == null ? other.model == null : model.equals(other.model); + } +} diff --git a/roda-core/roda-core-tests/src/main/java/org/roda/core/index/SolrUtilsTest.java b/roda-core/roda-core-tests/src/main/java/org/roda/core/index/SolrUtilsTest.java index 6be98d7077..ce9d176bf6 100644 --- a/roda-core/roda-core-tests/src/main/java/org/roda/core/index/SolrUtilsTest.java +++ b/roda-core/roda-core-tests/src/main/java/org/roda/core/index/SolrUtilsTest.java @@ -46,6 +46,7 @@ import org.roda.core.data.v2.index.filter.LongRangeFilterParameter; import org.roda.core.data.v2.index.filter.OneOfManyFilterParameter; import org.roda.core.data.v2.index.filter.SimpleFilterParameter; +import org.roda.core.data.v2.index.filter.TextToVectorFilterParameter; import org.roda.core.data.v2.index.sort.SortParameter; import org.roda.core.data.v2.index.sort.Sorter; import org.roda.core.index.utils.SolrUtils; @@ -263,6 +264,40 @@ public void testParseWithOneDateIntervalFilterParameter() { } } + @Test + public void testParserWithOneTextToVectorFilterParameter() { + try { + Filter filter = new Filter(); + filter.add( + new TextToVectorFilterParameter(RodaConstants.INDEX_EMBEDDING_VECTOR, "iraqi ministry of defence meeting", + "roda-embedding-model", 10)); + String stringFilter = SolrUtils.parseFilter(filter); + assertNotNull(stringFilter); + assertEquals(String.format("({!knn_text_to_vector model=%s f=%s topK=%d}%s)", "roda-embedding-model", + RodaConstants.INDEX_EMBEDDING_VECTOR, 10, "iraqi ministry of defence meeting"), stringFilter); + } catch (RODAException e) { + Assert.fail("An exception was not expected!"); + } + } + + @Test + public void testParserWithTextToVectorAndSimpleFilterParameter() { + try { + Filter filter = new Filter(); + filter.add(new TextToVectorFilterParameter(RodaConstants.INDEX_EMBEDDING_VECTOR, "meeting minutes", + "roda-embedding-model", 5)); + filter.add(new SimpleFilterParameter(RodaConstants.INDEX_SEARCH, FONDS)); + String stringFilter = SolrUtils.parseFilter(filter); + assertNotNull(stringFilter); + assertEquals( + String.format("({!knn_text_to_vector model=%s f=%s topK=%d}%s) AND (%s:\"%s\")", "roda-embedding-model", + RodaConstants.INDEX_EMBEDDING_VECTOR, 5, "meeting minutes", RodaConstants.INDEX_SEARCH, FONDS), + stringFilter); + } catch (RODAException e) { + Assert.fail("An exception was not expected!"); + } + } + @Test public void testParseSorter() { Sorter sorter = null; diff --git a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/Field.java b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/Field.java index 0478b30f8a..968f87871c 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/Field.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/Field.java @@ -33,6 +33,7 @@ public class Field { public static final String TYPE_INT = "pint"; public static final String TYPE_DATE = "pdate"; public static final String TYPE_STRING = "string"; + public static final String TYPE_KNN_VECTOR = "knn_vector"; public static final String FIELD_SEARCH = RodaConstants.INDEX_SEARCH; diff --git a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/SolrBootstrapUtils.java b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/SolrBootstrapUtils.java index 1e5b9ee10a..c14818518c 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/SolrBootstrapUtils.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/SolrBootstrapUtils.java @@ -8,26 +8,37 @@ package org.roda.core.index.schema; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.client.solrj.request.schema.SchemaRequest; import org.apache.solr.client.solrj.response.schema.SchemaResponse.CopyFieldsResponse; import org.apache.solr.client.solrj.response.schema.SchemaResponse.DynamicFieldsResponse; import org.apache.solr.client.solrj.response.schema.SchemaResponse.FieldsResponse; +import org.roda.core.RodaCoreFactory; +import org.roda.core.data.common.RodaConstants; import org.roda.core.data.exceptions.GenericException; import org.roda.core.data.v2.IsModelObject; import org.roda.core.data.v2.index.IsIndexed; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.ObjectMapper; + public class SolrBootstrapUtils { private static final Logger LOGGER = LoggerFactory.getLogger(SolrBootstrapUtils.class); + private static final String TEXT_TO_VECTOR_MODEL_STORE_PATH = "/schema/text-to-vector-model-store"; + private static final String LANGCHAIN4J_OPENAI_MODEL_CLASS = "dev.langchain4j.model.openai.OpenAiEmbeddingModel"; private static Map getFields(SolrClient client, String collectionName) throws GenericException { @@ -111,6 +122,68 @@ private static void bootstrapColl } else { LOGGER.info("Collection {} is up to date", collection.getIndexName()); } + + boolean hasVectorField = collection.getFields().stream() + .anyMatch(f -> Field.TYPE_KNN_VECTOR.equals(f.getType())); + if (hasVectorField) { + registerTextToVectorModel(client, collection.getIndexName()); + } + } + + /** + * Registers (or re-registers) the embedding model used for query-time + * text-to-vector search, so that {@code {!knn_text_to_vector model=...}} + * queries can resolve it. This only wires up query-time vectorization - + * index-time vectors are written by an external enrichment service, not by + * RODA. Best-effort: semantic search is an optional feature, so failures + * here are logged but must not prevent RODA from starting. + */ + private static void registerTextToVectorModel(SolrClient client, String collectionName) { + boolean enabled = Boolean + .parseBoolean(RodaCoreFactory.getRodaConfigurationAsString(RodaConstants.CORE_INDEX_EMBEDDING_ENABLED)); + if (!enabled) { + return; + } + + String solrModel = RodaCoreFactory.getRodaConfigurationAsString(RodaConstants.CORE_INDEX_EMBEDDING_SOLR_MODEL); + String baseUrl = RodaCoreFactory.getRodaConfigurationAsString(RodaConstants.CORE_INDEX_EMBEDDING_BASE_URL); + String modelName = RodaCoreFactory.getRodaConfigurationAsString(RodaConstants.CORE_INDEX_EMBEDDING_MODEL_NAME); + String apiKey = RodaCoreFactory.getRodaConfigurationAsString(RodaConstants.CORE_INDEX_EMBEDDING_API_KEY); + + if (StringUtils.isBlank(solrModel) || StringUtils.isBlank(baseUrl) || StringUtils.isBlank(modelName)) { + LOGGER.warn( + "Semantic search is enabled (core.index.embedding.enabled=true) but base_url/model_name/solr_model " + + "are not fully configured; skipping text-to-vector model registration for collection {}", + collectionName); + return; + } + + Map params = new LinkedHashMap<>(); + params.put("baseUrl", baseUrl); + params.put("modelName", modelName); + // LangChain4j's OpenAI client rejects a null/absent apiKey even against + // auth-free servers, so always send a value. + params.put("apiKey", StringUtils.isNotBlank(apiKey) ? apiKey : "not-needed"); + + Map payload = new LinkedHashMap<>(); + payload.put("class", LANGCHAIN4J_OPENAI_MODEL_CLASS); + payload.put("name", solrModel); + payload.put("params", params); + + try { + byte[] body = new ObjectMapper().writeValueAsBytes(payload); + GenericSolrRequest request = new GenericSolrRequest(SolrRequest.METHOD.PUT, TEXT_TO_VECTOR_MODEL_STORE_PATH) + .withContent(body, "application/json"); + request.setRequiresCollection(true); + request.process(client, collectionName); + LOGGER.info("Registered text-to-vector model '{}' for collection {}", solrModel, collectionName); + } catch (SolrServerException | IOException | RuntimeException e) { + LOGGER.warn( + "Could not register text-to-vector model '{}' for collection {} - semantic search queries against " + + "this collection will fail with an unknown model error until this is resolved (requires the " + + "language-models Solr module to be enabled, see docker-compose SOLR_MODULES)", + solrModel, collectionName, e); + } } public static void bootstrapSchemas(SolrClient client) throws GenericException { diff --git a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/AIPCollection.java b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/AIPCollection.java index f5dbf7f309..d22959e766 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/AIPCollection.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/AIPCollection.java @@ -129,6 +129,12 @@ public List getFields() { fields.add(SolrCollection.getSortFieldOf(RodaConstants.AIP_TITLE)); + // Semantic search: vector is populated by an external enrichment service on an + // asynchronous second pass, not by RODA. vectorized_b lets that service find + // documents still awaiting vectorization. + fields.add(new Field(RodaConstants.INDEX_EMBEDDING_VECTOR, Field.TYPE_KNN_VECTOR).setStored(false)); + fields.add(new Field(RodaConstants.INDEX_VECTORIZED, Field.TYPE_BOOLEAN).setDefaultValue("false")); + return fields; } diff --git a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/FileCollection.java b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/FileCollection.java index b9b6e6b60c..b835611ec4 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/FileCollection.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/FileCollection.java @@ -116,6 +116,12 @@ public List getFields() { fields.add(new Field(RodaConstants.INGEST_UPDATE_JOB_IDS, Field.TYPE_STRING).setMultiValued(true)); fields.add(new Field(RodaConstants.FILE_CREATED_ON, Field.TYPE_DATE)); + // Semantic search: vector is populated by an external enrichment service on an + // asynchronous second pass, not by RODA. vectorized_b lets that service find + // documents still awaiting vectorization. + fields.add(new Field(RodaConstants.INDEX_EMBEDDING_VECTOR, Field.TYPE_KNN_VECTOR).setStored(false)); + fields.add(new Field(RodaConstants.INDEX_VECTORIZED, Field.TYPE_BOOLEAN).setDefaultValue("false")); + return fields; } diff --git a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/RepresentationCollection.java b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/RepresentationCollection.java index 0cb27e880b..d0dfa8a78c 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/RepresentationCollection.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/RepresentationCollection.java @@ -101,6 +101,12 @@ public List getFields() { fields.add(SolrCollection.getSortFieldOf(RodaConstants.REPRESENTATION_TYPE)); // pataki@ END + // Semantic search: vector is populated by an external enrichment service on an + // asynchronous second pass, not by RODA. vectorized_b lets that service find + // documents still awaiting vectorization. + fields.add(new Field(RodaConstants.INDEX_EMBEDDING_VECTOR, Field.TYPE_KNN_VECTOR).setStored(false)); + fields.add(new Field(RodaConstants.INDEX_VECTORIZED, Field.TYPE_BOOLEAN).setDefaultValue("false")); + return fields; } diff --git a/roda-core/roda-core/src/main/java/org/roda/core/index/utils/SolrUtils.java b/roda-core/roda-core/src/main/java/org/roda/core/index/utils/SolrUtils.java index 8ea50319be..2985575f8d 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/index/utils/SolrUtils.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/index/utils/SolrUtils.java @@ -108,6 +108,7 @@ import org.roda.core.data.v2.index.filter.OrFiltersParameters; import org.roda.core.data.v2.index.filter.ParentWhichFilterParameter; import org.roda.core.data.v2.index.filter.SimpleFilterParameter; +import org.roda.core.data.v2.index.filter.TextToVectorFilterParameter; import org.roda.core.data.v2.index.sort.SortParameter; import org.roda.core.data.v2.index.sort.Sorter; import org.roda.core.data.v2.index.sublist.Sublist; @@ -1022,6 +1023,8 @@ private static void parseFilterParameter(StringBuilder ret, FilterParameter para appendBlockJoinFilterParameter(ret, nestParentFilterParameter, prefixWithANDOperatorIfBuilderNotEmpty); } else if (parameter instanceof ChildOfFilterParameter nestChildOfFilterParameter) { appendBlockJoinChildrenFilterParameter(ret, nestChildOfFilterParameter, prefixWithANDOperatorIfBuilderNotEmpty); + } else if (parameter instanceof TextToVectorFilterParameter param) { + appendTextToVector(ret, param, prefixWithANDOperatorIfBuilderNotEmpty); } else { LOGGER.error("Unsupported filter parameter class: {}", parameter.getClass().getName()); throw new RequestNotValidException("Unsupported filter parameter class: " + parameter.getClass().getName()); @@ -1078,6 +1081,31 @@ private static void appendOROperator(StringBuilder ret, boolean prefixWithOROper } } + /** + * Renders a semantic-search clause vectorizing {@code parameter.getQuery()} at + * query time via Solr's {@code knn_text_to_vector} query parser (see + * https://solr.apache.org/guide/solr/latest/query-guide/text-to-vector.html), + * restricting results to the top-K nearest neighbours of the target + * {@code knn_vector} field. + */ + private static void appendTextToVector(StringBuilder ret, TextToVectorFilterParameter parameter, + boolean prefixWithANDOperatorIfBuilderNotEmpty) { + appendANDOperator(ret, prefixWithANDOperatorIfBuilderNotEmpty); + // The query text is passed via the "v" local param (quoted) rather than as trailing + // text after the local params block. Trailing text is only unambiguous for a single + // word - as soon as the free-text query contains a space, the outer query parser + // (this clause is combined with others via AND/OR into one larger query string) can + // split on it and try to resolve the extra words against the default search field, + // which RODA doesn't define, causing "undefined field _text_". + ret.append("({!knn_text_to_vector model=").append(parameter.getModel()).append(" f=") + .append(parameter.getField()).append(" topK=").append(parameter.getTopK()).append(" v='") + .append(escapeSolrLocalParamValue(parameter.getQuery())).append("'})"); + } + + private static String escapeSolrLocalParamValue(String value) { + return value.replace("\\", "\\\\").replace("'", "\\'"); + } + private static void appendExactMatch(StringBuilder ret, String key, String value, boolean appendDoubleQuotes, boolean prefixWithANDOperatorIfBuilderNotEmpty) { appendANDOperator(ret, prefixWithANDOperatorIfBuilderNotEmpty); diff --git a/roda-core/roda-core/src/main/resources/config/index/common/conf/managed-schema.xml b/roda-core/roda-core/src/main/resources/config/index/common/conf/managed-schema.xml index a6410e8d07..a8101b6f4b 100644 --- a/roda-core/roda-core/src/main/resources/config/index/common/conf/managed-schema.xml +++ b/roda-core/roda-core/src/main/resources/config/index/common/conf/managed-schema.xml @@ -1047,4 +1047,14 @@ + + + + \ No newline at end of file diff --git a/roda-core/roda-core/src/main/resources/config/index/common/conf/solrconfig.xml b/roda-core/roda-core/src/main/resources/config/index/common/conf/solrconfig.xml index 0be4ef0ddd..fb19c6bc2f 100644 --- a/roda-core/roda-core/src/main/resources/config/index/common/conf/solrconfig.xml +++ b/roda-core/roda-core/src/main/resources/config/index/common/conf/solrconfig.xml @@ -74,6 +74,12 @@ --> + + + + + +