Add semantic (vector) search support via Solr's text-to-vector module

deploys/standalone/docker-compose-dev.yaml

@@ -21,6 +21,7 @@ services:
       ZK_HOST: zoo:2181
       SOLR_HOST: localhost
       SOLR_OPTS: "-Dsolr.environment=dev,label=DEV+ENV"
+      SOLR_MODULES: language-models
     depends_on:
       - zoo
     volumes:

deploys/standalone/docker-compose-vector-test.yaml

@@ -0,0 +1,67 @@
+# Minimal stack for live-testing Solr text-to-vector semantic search.
+# Brings up just ZooKeeper + Solr (with the
+# language-models module enabled) + PostgreSQL - enough for RODA to bootstrap
+# its schema (incl. registering the text-to-vector model against an external
+# embedding API) without the full docker-compose-dev.yaml stack.
+#
+# Usage:
+#   docker compose -f deploys/standalone/docker-compose-vector-test.yaml up -d
+#
+# Then run RODA (or a bootstrap-only script) with:
+#   RODA_CORE_SOLR_TYPE=CLOUD
+#   RODA_CORE_SOLR_CLOUD_URLS=localhost:2182
+#   SPRING_DATASOURCE_URL=jdbc:postgresql://localhost:5433/roda_core_db
+#   SPRING_DATASOURCE_USERNAME=admin
+#   SPRING_DATASOURCE_PASSWORD=roda
+#   RODA_CORE_INDEX_EMBEDDING_ENABLED=true
+#   RODA_CORE_INDEX_EMBEDDING_BASE_URL=<embedding API base URL, e.g. http://host:8000/v1>
+#   RODA_CORE_INDEX_EMBEDDING_MODEL_NAME=<embedding model name>
+#   RODA_CORE_INDEX_EMBEDDING_SOLR_MODEL=roda-embedding-model
+#   RODA_CORE_INDEX_EMBEDDING_DIMENSIONS=<must match managed-schema.xml's knn_vector fieldType>
+services:
+  zoo-vector-test:
+    image: docker.io/zookeeper:3.9.5-jre-17
+    container_name: zoo-vector-test
+    restart: unless-stopped
+    ports:
+      - "2182:2181"
+    environment:
+      - ZOO_4LW_COMMANDS_WHITELIST=mntr,conf,ruok
+    volumes:
+      - zoo_vector_test_data:/data
+      - zoo_vector_test_datalog:/datalog
+
+  solr-vector-test:
+    image: docker.io/solr:10.0.0
+    container_name: solr-vector-test
+    restart: unless-stopped
+    ports:
+      - "8984:8983"
+    environment:
+      ZK_HOST: zoo-vector-test:2181
+      SOLR_HOST: localhost
+      SOLR_OPTS: "-Dsolr.environment=vector-test,label=VECTOR+TEST"
+      SOLR_MODULES: language-models
+    depends_on:
+      - zoo-vector-test
+    volumes:
+      - solr_vector_test_data:/var/solr
+
+  postgres-vector-test:
+    image: postgres:17
+    container_name: postgres-vector-test
+    restart: unless-stopped
+    environment:
+      POSTGRES_USER: admin
+      POSTGRES_PASSWORD: roda
+      POSTGRES_DB: roda_core_db
+    ports:
+      - "5433:5432"
+    volumes:
+      - pg_vector_test_data:/var/lib/postgresql/data
+
+volumes:
+  zoo_vector_test_data:
+  zoo_vector_test_datalog:
+  solr_vector_test_data:
+  pg_vector_test_data:

deploys/standalone/docker-compose.yaml

@@ -16,6 +16,7 @@ services:
     environment:
       SOLR_HEAP: 2g
       ZK_HOST: zoo:2181
+      SOLR_MODULES: language-models
     volumes:
       - solr_data:/var/solr
 

roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java

@@ -48,6 +48,14 @@ public final class RodaConstants {
   public static final String CORE_SOLR_CLOUD_URLS = "core.solr.cloud.urls";
   public static final String CORE_SOLR_STEMMING_LANGUAGE = "core.solr.stemming.language";
 
+  public static final String CORE_INDEX_EMBEDDING_ENABLED = "core.index.embedding.enabled";
+  public static final String CORE_INDEX_EMBEDDING_BASE_URL = "core.index.embedding.base_url";
+  public static final String CORE_INDEX_EMBEDDING_API_KEY = "core.index.embedding.api_key";
+  public static final String CORE_INDEX_EMBEDDING_MODEL_NAME = "core.index.embedding.model_name";
+  public static final String CORE_INDEX_EMBEDDING_SOLR_MODEL = "core.index.embedding.solr_model";
+  public static final String CORE_INDEX_EMBEDDING_DIMENSIONS = "core.index.embedding.dimensions";
+  public static final String CORE_INDEX_EMBEDDING_DEFAULT_TOP_K = "core.index.embedding.default_top_k";
+
   public static final String CORE_EVENTS_ENABLED = "core.events.enabled";
   public static final String CORE_EVENTS_NOTIFIER_AND_HANDLER_ARE_THE_SAME = "core.events.notifier_and_handler_are_the_same";
   public static final String CORE_EVENTS_NOTIFIER_CLASS = "core.events.notifier_class";
@@ -1061,6 +1069,9 @@ public enum OrchestratorType {
   public static final String INDEX_SEARCH = "search";
   public static final String INDEX_WILDCARD = "*";
 
+  public static final String INDEX_EMBEDDING_VECTOR = "embedding_vector";
+  public static final String INDEX_VECTORIZED = "vectorized_b";
+
   public static final String INDEX_INSTANCE_ID = "instanceId";
   public static final String INDEX_INSTANCE_NAME = "instanceName";
 

roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/filter/FilterParameter.java

@@ -39,12 +39,14 @@
   @JsonSubTypes.Type(value = AndFiltersParameters.class, name = "AndFiltersParameters"),
   @JsonSubTypes.Type(value = AllFilterParameter.class, name = "AllFilterParameter"),
   @JsonSubTypes.Type(value = ParentWhichFilterParameter.class, name = "ParentWhichFilterParameter"),
-  @JsonSubTypes.Type(value = ChildOfFilterParameter.class, name = "ChildOfFilterParameter"),})
+  @JsonSubTypes.Type(value = ChildOfFilterParameter.class, name = "ChildOfFilterParameter"),
+  @JsonSubTypes.Type(value = TextToVectorFilterParameter.class, name = "TextToVectorFilterParameter"),})
 @Schema(type = "object", subTypes = {BasicSearchFilterParameter.class, EmptyKeyFilterParameter.class,
   LikeFilterParameter.class, NotSimpleFilterParameter.class, OneOfManyFilterParameter.class,
   DateIntervalFilterParameter.class, DateRangeFilterParameter.class, LongRangeFilterParameter.class,
   StringRangeFilterParameter.class, SimpleFilterParameter.class, OrFiltersParameters.class, AndFiltersParameters.class,
-  AllFilterParameter.class, ParentWhichFilterParameter.class, ChildOfFilterParameter.class}, discriminatorMapping = {
+  AllFilterParameter.class, ParentWhichFilterParameter.class, ChildOfFilterParameter.class,
+  TextToVectorFilterParameter.class}, discriminatorMapping = {
     @DiscriminatorMapping(value = "BasicSearchFilterParameter", schema = BasicSearchFilterParameter.class),
     @DiscriminatorMapping(value = "LikeFilterParameter", schema = LikeFilterParameter.class),
     @DiscriminatorMapping(value = "NotSimpleFilterParameter", schema = NotSimpleFilterParameter.class),
@@ -58,7 +60,9 @@
     @DiscriminatorMapping(value = "AndFiltersParameters", schema = AndFiltersParameters.class),
     @DiscriminatorMapping(value = "AllFilterParameter", schema = AllFilterParameter.class),
     @DiscriminatorMapping(value = "ParentWhichFilterParameter", schema = ParentWhichFilterParameter.class),
-    @DiscriminatorMapping(value = "ChildOfFilterParameter", schema = ChildOfFilterParameter.class)}, discriminatorProperty = "type")
+    @DiscriminatorMapping(value = "ChildOfFilterParameter", schema = ChildOfFilterParameter.class),
+    @DiscriminatorMapping(value = "TextToVectorFilterParameter",
+      schema = TextToVectorFilterParameter.class)}, discriminatorProperty = "type")
 public abstract class FilterParameter implements Serializable {
   @Serial
   private static final long serialVersionUID = 3744111668897879761L;

roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/filter/TextToVectorFilterParameter.java

@@ -0,0 +1,133 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE file at the root of the source
+ * tree and available online at
+ *
+ * https://github.com/keeps/roda
+ */
+package org.roda.core.data.v2.index.filter;
+
+import java.io.Serial;
+
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+/**
+ * Semantic search parameter that vectorizes {@link #getQuery()} at query time
+ * (via Solr's text-to-vector module) and restricts results to the
+ * {@link #getTopK()} nearest neighbours of {@link #getField()}. Vectors
+ * themselves are populated out-of-band by an external enrichment service, not
+ * by RODA.
+ */
+@JsonTypeName("TextToVectorFilterParameter")
+public class TextToVectorFilterParameter extends FilterParameter {
+  @Serial
+  private static final long serialVersionUID = 1L;
+
+  private String field;
+  private String query;
+  private String model;
+  private int topK;
+
+  /**
+   * Constructs an empty {@link TextToVectorFilterParameter}.
+   */
+  public TextToVectorFilterParameter() {
+    // do nothing
+  }
+
+  public TextToVectorFilterParameter(String field, String query, String model, int topK) {
+    setField(field);
+    setQuery(query);
+    setModel(model);
+    setTopK(topK);
+  }
+
+  public TextToVectorFilterParameter(TextToVectorFilterParameter other) {
+    this(other.getField(), other.getQuery(), other.getModel(), other.getTopK());
+  }
+
+  /**
+   * @return the name of the {@code knn_vector} field to search against.
+   */
+  public String getField() {
+    return field;
+  }
+
+  public void setField(String field) {
+    this.field = field;
+  }
+
+  /**
+   * @return the free-text query to vectorize.
+   */
+  public String getQuery() {
+    return query;
+  }
+
+  public void setQuery(String query) {
+    this.query = query;
+  }
+
+  /**
+   * @return the name the embedding model is registered under in Solr's
+   *         text-to-vector-model-store.
+   */
+  public String getModel() {
+    return model;
+  }
+
+  public void setModel(String model) {
+    this.model = model;
+  }
+
+  /**
+   * @return the number of nearest neighbours to retrieve.
+   */
+  public int getTopK() {
+    return topK;
+  }
+
+  public void setTopK(int topK) {
+    this.topK = topK;
+  }
+
+  @Override
+  public String toString() {
+    return "TextToVectorFilterParameter(field=" + field + ", query=" + query + ", model=" + model + ", topK=" + topK
+      + ")";
+  }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = super.hashCode();
+    result = prime * result + ((field == null) ? 0 : field.hashCode());
+    result = prime * result + ((query == null) ? 0 : query.hashCode());
+    result = prime * result + ((model == null) ? 0 : model.hashCode());
+    result = prime * result + topK;
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (!super.equals(obj)) {
+      return false;
+    }
+    if (!(obj instanceof TextToVectorFilterParameter other)) {
+      return false;
+    }
+    if (topK != other.topK) {
+      return false;
+    }
+    if (field == null ? other.field != null : !field.equals(other.field)) {
+      return false;
+    }
+    if (query == null ? other.query != null : !query.equals(other.query)) {
+      return false;
+    }
+    return model == null ? other.model == null : model.equals(other.model);
+  }
+}

roda-core/roda-core-tests/src/main/java/org/roda/core/index/SolrUtilsTest.java

@@ -46,6 +46,7 @@
 import org.roda.core.data.v2.index.filter.LongRangeFilterParameter;
 import org.roda.core.data.v2.index.filter.OneOfManyFilterParameter;
 import org.roda.core.data.v2.index.filter.SimpleFilterParameter;
+import org.roda.core.data.v2.index.filter.TextToVectorFilterParameter;
 import org.roda.core.data.v2.index.sort.SortParameter;
 import org.roda.core.data.v2.index.sort.Sorter;
 import org.roda.core.index.utils.SolrUtils;
@@ -263,6 +264,40 @@ public void testParseWithOneDateIntervalFilterParameter() {
     }
   }
 
+  @Test
+  public void testParserWithOneTextToVectorFilterParameter() {
+    try {
+      Filter filter = new Filter();
+      filter.add(
+        new TextToVectorFilterParameter(RodaConstants.INDEX_EMBEDDING_VECTOR, "iraqi ministry of defence meeting",
+          "roda-embedding-model", 10));
+      String stringFilter = SolrUtils.parseFilter(filter);
+      assertNotNull(stringFilter);
+      assertEquals(String.format("({!knn_text_to_vector model=%s f=%s topK=%d}%s)", "roda-embedding-model",
+        RodaConstants.INDEX_EMBEDDING_VECTOR, 10, "iraqi ministry of defence meeting"), stringFilter);
+    } catch (RODAException e) {
+      Assert.fail("An exception was not expected!");
+    }
+  }
+
+  @Test
+  public void testParserWithTextToVectorAndSimpleFilterParameter() {
+    try {
+      Filter filter = new Filter();
+      filter.add(new TextToVectorFilterParameter(RodaConstants.INDEX_EMBEDDING_VECTOR, "meeting minutes",
+        "roda-embedding-model", 5));
+      filter.add(new SimpleFilterParameter(RodaConstants.INDEX_SEARCH, FONDS));
+      String stringFilter = SolrUtils.parseFilter(filter);
+      assertNotNull(stringFilter);
+      assertEquals(
+        String.format("({!knn_text_to_vector model=%s f=%s topK=%d}%s) AND (%s:\"%s\")", "roda-embedding-model",
+          RodaConstants.INDEX_EMBEDDING_VECTOR, 5, "meeting minutes", RodaConstants.INDEX_SEARCH, FONDS),
+        stringFilter);
+    } catch (RODAException e) {
+      Assert.fail("An exception was not expected!");
+    }
+  }
+
   @Test
   public void testParseSorter() {
     Sorter sorter = null;

roda-core/roda-core/src/main/java/org/roda/core/index/schema/Field.java

@@ -33,6 +33,7 @@ public class Field {
   public static final String TYPE_INT = "pint";
   public static final String TYPE_DATE = "pdate";
   public static final String TYPE_STRING = "string";
+  public static final String TYPE_KNN_VECTOR = "knn_vector";
 
   public static final String FIELD_SEARCH = RodaConstants.INDEX_SEARCH;