Decouple HybridIndex from BaseInvertedIndex internals

src/main/java/com/uber/ussi/searchablestructure/index/inverted/BaseInvertedIndex.java

@@ -3,10 +3,8 @@
 
 import com.carrotsearch.hppc.LongDoubleHashMap;
 import com.carrotsearch.hppc.LongHashSet;
-import com.carrotsearch.hppc.LongIntHashMap;
 import com.carrotsearch.hppc.LongObjectHashMap;
 import com.carrotsearch.hppc.cursors.LongCursor;
-import com.carrotsearch.hppc.cursors.LongIntCursor;
 import com.carrotsearch.hppc.cursors.LongObjectCursor;
 import com.uber.ussi.MemoryFootprint;
 import com.uber.ussi.ProcessorAllowance;
@@ -37,7 +35,6 @@
 import com.uber.ussi.searchablestructure.utils.parallel.ParallelShardSearch;
 import com.uber.ussi.searchablestructure.utils.parallel.SharedMinSimilarity;
 import com.uber.ussi.utils.BoundedSizeMaxHeap;
-import com.uber.ussi.utils.ConfigKeys;
 import com.uber.ussi.utils.MathUtils;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -172,11 +169,12 @@ protected BaseInvertedIndex(
     this.mergeScoresFromAccumulatedConjunction = sparsMerge.mergeScoresFromAccumulatedConjunction();
     this.partialConjunctionPolicy = sparsMerge.partialConjunctionPolicy();
     this.doesStoreMergePostingValues = sparsMerge.doesStoreMergePostingValues();
-    this.maxFractionIdsPerTerm = parseMaxFractionIdsPerTerm(namespaceConfig);
+    this.maxFractionIdsPerTerm = PopularTermDiscardPolicy.maxFractionIdsPerTerm(namespaceConfig);
     validateRows();
     this.discardedTerms =
         structureDiscardedTerms == null
-            ? discardedTermsOf(rowNumToTermsAndValuesMap, maxFractionIdsPerTerm)
+            ? PopularTermDiscardPolicy.discardedTermsOf(
+                rowNumToTermsAndValuesMap, maxFractionIdsPerTerm)
             : structureDiscardedTerms;
     LongObjectHashMap<LongTermsAndValues> discardedTermFreeRows = buildDiscardedTermFreeRows();
     this.verificationRowNumToTermsAndValuesMap =
@@ -301,7 +299,7 @@ final long[] getDiscardedTermsForTests() {
   }
 
   final boolean discardsPopularTerms() {
-    return maxFractionIdsPerTerm < 1.0;
+    return PopularTermDiscardPolicy.doesDiscardPopularTerms(maxFractionIdsPerTerm);
   }
 
   final long[] getRowNumsForKeyForTests(int shard, long key) {
@@ -623,51 +621,6 @@ private KeyAndPrefixFilteringData[] collectFilteredSearchQueryKeys(
     return keyData;
   }
 
-  /** The terms a structure holding {@code rowNumToTermsAndValuesMap} discards as popular. */
-  static LongHashSet discardedTermsOf(
-      NamespaceConfig namespaceConfig,
-      LongObjectHashMap<LongTermsAndValues> rowNumToTermsAndValuesMap) {
-    return discardedTermsOf(
-        rowNumToTermsAndValuesMap, parseMaxFractionIdsPerTerm(namespaceConfig));
-  }
-
-  /**
-   * Identifies the high-popularity terms to discard. The structure sees the complete dataset, so
-   * observed popularity is true popularity: a term is discarded when it occurs in more than
-   * floor(numRows * maxFractionIdsPerTerm) rows.
-   *
-   * <p>Counted over the terms of each row, never over the keys the rows are indexed under, so a
-   * frequent key is never discarded for being frequent. Where the keys are signatures they are
-   * generated afterwards, from the rows these terms have been removed from.
-   */
-  private static LongHashSet discardedTermsOf(
-      LongObjectHashMap<LongTermsAndValues> rowNumToTermsAndValuesMap,
-      double maxFractionIdsPerTerm) {
-    LongIntHashMap numRowsByTerm = new LongIntHashMap();
-    // A row counts once per distinct term, so a term repeated within one row stays one row.
-    LongHashSet termsInRow = new LongHashSet();
-    for (LongObjectCursor<LongTermsAndValues> entry : rowNumToTermsAndValuesMap) {
-      termsInRow.clear();
-      for (int i = 0; i < entry.value.termsLength(); ++i) {
-        long term = entry.value.getTerm(i);
-        if (!termsInRow.add(term)) {
-          continue;
-        }
-        int numRows = numRowsByTerm.containsKey(term) ? numRowsByTerm.get(term) + 1 : 1;
-        numRowsByTerm.put(term, numRows);
-      }
-    }
-    int maxNumRowsPerTerm =
-        (int) Math.floor(rowNumToTermsAndValuesMap.size() * maxFractionIdsPerTerm);
-    LongHashSet popularTerms = new LongHashSet();
-    for (LongIntCursor entry : numRowsByTerm) {
-      if (entry.value > maxNumRowsPerTerm) {
-        popularTerms.add(entry.key);
-      }
-    }
-    return popularTerms;
-  }
-
   /** Returns the rows with the high-popularity terms dropped, which is what gets indexed. */
   private LongObjectHashMap<LongTermsAndValues> buildDiscardedTermFreeRows() {
     if (discardedTerms.isEmpty()) {
@@ -813,11 +766,6 @@ private void validateRows() {
     }
   }
 
-  private static double parseMaxFractionIdsPerTerm(NamespaceConfig namespaceConfig) {
-    return namespaceConfig.readDoubleIndexParam(
-        ConfigKeys.MAX_FRACTION_IDS_PER_TERM, ConfigKeys.DEFAULT_MAX_FRACTION_IDS_PER_TERM);
-  }
-
   /**
    * Returns the one record type the structure stores and the comparator reads. Both failures are
    * backstops: {@code IndexConfigValidator} rejects a pairing with none in common, and no

src/main/java/com/uber/ussi/searchablestructure/index/inverted/HybridIndex.java

@@ -17,17 +17,6 @@
 /** Hybrid inverted index using exact keys for short rows and signatures for long rows. */
 public final class HybridIndex extends Index {
 
-  /**
-   * The largest term count a row can have and still be keyed by its own terms. Above it a row is
-   * keyed by signatures instead.
-   *
-   * <p>It is the signature count because that is where signatures stop being a saving: a row with
-   * fewer terms than that would be replaced by more signatures than it had terms, costing list
-   * entries and buying no pruning. The two quantities are derived separately and happen to
-   * coincide, so the cutoff names itself rather than reading as a signature count here.
-   */
-  private static final int TERM_KEYING_CUTOFF = SignatureIndex.NUM_SIGNATURES_PER_ROW;
-
   private final TermIndex termIndex;
   private final SignatureIndex signatureIndex;
   private final boolean termPopularityFilteringEnabled;
@@ -37,10 +26,11 @@ public HybridIndex(
       LongObjectHashMap<LongTermsAndValues> rowNumToTermsAndValuesMap,
       LongObjectHashMap<LongMeta> rowNumToMetaMap) {
     super(namespaceConfig);
+    double maxFractionIdsPerTerm = PopularTermDiscardPolicy.maxFractionIdsPerTerm(namespaceConfig);
     LongObjectHashMap<LongTermsAndValues> exactRows = new LongObjectHashMap<>();
     LongObjectHashMap<LongTermsAndValues> signatureRows = new LongObjectHashMap<>();
     for (LongObjectCursor<LongTermsAndValues> entry : rowNumToTermsAndValuesMap) {
-      if (entry.value.termsLength() <= TERM_KEYING_CUTOFF) {
+      if (entry.value.termsLength() <= InvertedHybridConfiguration.TERM_KEYING_CUTOFF) {
         exactRows.put(entry.key, entry.value);
       } else {
         signatureRows.put(entry.key, entry.value);
@@ -50,13 +40,15 @@ public HybridIndex(
     // term index can collect it: a signature list holds one entry per row whatever that row's
     // terms are, so discarding leaves a signature index's lists exactly as long and only moves the
     // signatures its rows are keyed by.
-    LongHashSet discardedTerms = BaseInvertedIndex.discardedTermsOf(namespaceConfig, exactRows);
+    LongHashSet discardedTerms =
+        PopularTermDiscardPolicy.discardedTermsOf(exactRows, maxFractionIdsPerTerm);
     // The signature index is built first so a comparator without a generator is rejected before
     // the term index is populated.
     this.signatureIndex =
         new SignatureIndex(namespaceConfig, signatureRows, rowNumToMetaMap, new LongHashSet());
     this.termIndex = new TermIndex(namespaceConfig, exactRows, rowNumToMetaMap, discardedTerms);
-    this.termPopularityFilteringEnabled = termIndex.discardsPopularTerms();
+    this.termPopularityFilteringEnabled =
+        PopularTermDiscardPolicy.doesDiscardPopularTerms(maxFractionIdsPerTerm);
   }
 
   @Override
@@ -66,7 +58,8 @@ public List<RowNumAndSimilarity> getNearestNeighborRowNums(
       throw new IllegalArgumentException("k must be greater than 0.");
     }
     int maxResults = Math.min(k, namespaceConfig.getMaxNumSimilarities());
-    boolean queryUsesExactIndex = record.termsLength() <= TERM_KEYING_CUTOFF;
+    boolean queryUsesExactIndex =
+        record.termsLength() <= InvertedHybridConfiguration.TERM_KEYING_CUTOFF;
     Index firstIndex = queryUsesExactIndex ? termIndex : signatureIndex;
     Index secondIndex = queryUsesExactIndex ? signatureIndex : termIndex;
     boolean secondIndexIsExact = !queryUsesExactIndex;
@@ -189,8 +182,9 @@ private boolean maySearchIndex(
     if (termPopularityFilteringEnabled) {
       return true;
     }
-    int minNumTerms = exactIndex ? 0 : TERM_KEYING_CUTOFF + 1;
-    int maxNumTerms = exactIndex ? TERM_KEYING_CUTOFF : Integer.MAX_VALUE;
+    int minNumTerms = exactIndex ? 0 : InvertedHybridConfiguration.TERM_KEYING_CUTOFF + 1;
+    int maxNumTerms =
+        exactIndex ? InvertedHybridConfiguration.TERM_KEYING_CUTOFF : Integer.MAX_VALUE;
     return comparator.mayPassNumTermsFiltering(query, minNumTerms, maxNumTerms, minSimilarity);
   }
 

src/main/java/com/uber/ussi/searchablestructure/index/inverted/InvertedHybridConfiguration.java

@@ -0,0 +1,25 @@
+/* AUTHOR: Ahmed Metwally (ametwally@uber.com) */
+package com.uber.ussi.searchablestructure.index.inverted;
+
+/**
+ * Shared constants for hybrid inverted indexes that combine term-keyed and signature-keyed halves.
+ */
+public final class InvertedHybridConfiguration {
+
+  /**
+   * How many signatures stand in for one long row in the signature-keyed half. Drawn from
+   * experiments trading recall against list length.
+   */
+  public static final int NUM_SIGNATURES_PER_ROW = 270;
+
+  /**
+   * The largest term count a row can have and still be keyed by its own terms in the hybrid index.
+   * Above this count a row is keyed by signatures instead.
+   *
+   * <p>It equals {@link #NUM_SIGNATURES_PER_ROW} because that is where signatures stop being a
+   * saving: a row with fewer terms would be replaced by more signatures than it had terms.
+   */
+  public static final int TERM_KEYING_CUTOFF = NUM_SIGNATURES_PER_ROW;
+
+  private InvertedHybridConfiguration() {}
+}

src/main/java/com/uber/ussi/searchablestructure/index/inverted/PopularTermDiscardPolicy.java

@@ -0,0 +1,89 @@
+/* AUTHOR: Ahmed Metwally (ametwally@uber.com) */
+package com.uber.ussi.searchablestructure.index.inverted;
+
+import com.carrotsearch.hppc.LongHashSet;
+import com.carrotsearch.hppc.LongIntHashMap;
+import com.carrotsearch.hppc.LongObjectHashMap;
+import com.carrotsearch.hppc.cursors.LongIntCursor;
+import com.carrotsearch.hppc.cursors.LongObjectCursor;
+import com.uber.ussi.config.NamespaceConfig;
+import com.uber.ussi.entity.termsandvalues.LongTermsAndValues;
+import com.uber.ussi.utils.ConfigKeys;
+import java.util.Objects;
+
+/**
+ * Popularity-based term discard policy for inverted term-keyed indexes.
+ *
+ * <p>Hybrid and term indexes share this policy so a composite structure can compute discards over
+ * its term-keyed partition without depending on {@link BaseInvertedIndex}.
+ */
+public final class PopularTermDiscardPolicy {
+
+  private PopularTermDiscardPolicy() {}
+
+  /** Returns the configured maximum fraction of rows a term may appear in before it is discarded. */
+  public static double maxFractionIdsPerTerm(NamespaceConfig namespaceConfig) {
+    Objects.requireNonNull(namespaceConfig, "namespaceConfig is null.");
+    return namespaceConfig.readDoubleIndexParam(
+        ConfigKeys.MAX_FRACTION_IDS_PER_TERM, ConfigKeys.DEFAULT_MAX_FRACTION_IDS_PER_TERM);
+  }
+
+  /** Returns whether any term may be discarded under the configured popularity threshold. */
+  public static boolean doesDiscardPopularTerms(NamespaceConfig namespaceConfig) {
+    return maxFractionIdsPerTerm(namespaceConfig) < 1.0;
+  }
+
+  /** Returns whether any term may be discarded under {@code maxFractionIdsPerTerm}. */
+  public static boolean doesDiscardPopularTerms(double maxFractionIdsPerTerm) {
+    return maxFractionIdsPerTerm < 1.0;
+  }
+
+  /**
+   * Returns the terms a structure holding {@code rowNumToTermsAndValuesMap} discards as popular.
+   */
+  public static LongHashSet discardedTermsOf(
+      NamespaceConfig namespaceConfig,
+      LongObjectHashMap<LongTermsAndValues> rowNumToTermsAndValuesMap) {
+    Objects.requireNonNull(namespaceConfig, "namespaceConfig is null.");
+    Objects.requireNonNull(rowNumToTermsAndValuesMap, "rowNumToTermsAndValuesMap is null.");
+    return discardedTermsOf(rowNumToTermsAndValuesMap, maxFractionIdsPerTerm(namespaceConfig));
+  }
+
+  /**
+   * Identifies the high-popularity terms to discard. The structure sees the complete dataset, so
+   * observed popularity is true popularity: a term is discarded when it occurs in more than
+   * floor(numRows * maxFractionIdsPerTerm) rows.
+   *
+   * <p>Counted over the terms of each row, never over the keys the rows are indexed under, so a
+   * frequent key is never discarded for being frequent. Where the keys are signatures they are
+   * generated afterwards, from the rows these terms have been removed from.
+   */
+  public static LongHashSet discardedTermsOf(
+      LongObjectHashMap<LongTermsAndValues> rowNumToTermsAndValuesMap,
+      double maxFractionIdsPerTerm) {
+    Objects.requireNonNull(rowNumToTermsAndValuesMap, "rowNumToTermsAndValuesMap is null.");
+    LongIntHashMap numRowsByTerm = new LongIntHashMap();
+    // A row counts once per distinct term, so a term repeated within one row stays one row.
+    LongHashSet termsInRow = new LongHashSet();
+    for (LongObjectCursor<LongTermsAndValues> entry : rowNumToTermsAndValuesMap) {
+      termsInRow.clear();
+      for (int i = 0; i < entry.value.termsLength(); ++i) {
+        long term = entry.value.getTerm(i);
+        if (!termsInRow.add(term)) {
+          continue;
+        }
+        int numRows = numRowsByTerm.containsKey(term) ? numRowsByTerm.get(term) + 1 : 1;
+        numRowsByTerm.put(term, numRows);
+      }
+    }
+    int maxNumRowsPerTerm =
+        (int) Math.floor(rowNumToTermsAndValuesMap.size() * maxFractionIdsPerTerm);
+    LongHashSet popularTerms = new LongHashSet();
+    for (LongIntCursor entry : numRowsByTerm) {
+      if (entry.value > maxNumRowsPerTerm) {
+        popularTerms.add(entry.key);
+      }
+    }
+    return popularTerms;
+  }
+}

src/main/java/com/uber/ussi/searchablestructure/index/inverted/SignatureIndex.java

@@ -23,11 +23,9 @@
  */
 public final class SignatureIndex extends BaseInvertedIndex {
 
-  /**
-   * How many signatures stand in for one row, which fixes the length of every signature list. Drawn
-   * from experiments trading recall against list length.
-   */
-  public static final int NUM_SIGNATURES_PER_ROW = 270;
+  /** How many signatures stand in for one row. See {@link InvertedHybridConfiguration}. */
+  public static final int NUM_SIGNATURES_PER_ROW =
+      InvertedHybridConfiguration.NUM_SIGNATURES_PER_ROW;
 
   public SignatureIndex(
       NamespaceConfig namespaceConfig,

src/test/java/com/uber/ussi/searchablestructure/index/inverted/PopularTermDiscardPolicyTest.java

@@ -0,0 +1,59 @@
+/* AUTHOR: Ahmed Metwally (ametwally@uber.com) */
+package com.uber.ussi.searchablestructure.index.inverted;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.carrotsearch.hppc.LongHashSet;
+import com.carrotsearch.hppc.LongObjectHashMap;
+import com.uber.ussi.config.NamespaceConfig;
+import com.uber.ussi.entity.termsandvalues.LongTermsAndValues;
+import com.uber.ussi.entity.termsandvalues.LongTermsAndValuesTestFactory;
+import com.uber.ussi.utils.ConfigKeys;
+import java.util.Map;
+import org.junit.jupiter.api.Test;
+
+class PopularTermDiscardPolicyTest {
+
+  @Test
+  void doesDiscardPopularTermsWhenMaxFractionIsBelowOne() {
+    NamespaceConfig config =
+        NamespaceConfig.builder()
+            .minTermsAndValuesLength(0)
+            .maxTermsAndValuesLength(100)
+            .maxCacheSize(100)
+            .cacheType("scan")
+            .indexType("inverted_term")
+            .indexParams(Map.of(ConfigKeys.MAX_FRACTION_IDS_PER_TERM, "0.5"))
+            .comparatorType("jaccard")
+            .comparatorNormalizerType("complement")
+            .maxNumSearchableStructures(3)
+            .maxNumSimilarities(100)
+            .build();
+    assertTrue(PopularTermDiscardPolicy.doesDiscardPopularTerms(config));
+    assertFalse(PopularTermDiscardPolicy.doesDiscardPopularTerms(1.0));
+  }
+
+  @Test
+  void discardedTermsOfMarksTermsAboveTheRowFractionThreshold() {
+    LongObjectHashMap<LongTermsAndValues> rows = new LongObjectHashMap<>();
+    rows.put(1L, terms(new long[] {10, 20}));
+    rows.put(2L, terms(new long[] {10, 30}));
+    rows.put(3L, terms(new long[] {10, 40}));
+    rows.put(4L, terms(new long[] {50}));
+
+    LongHashSet discarded = PopularTermDiscardPolicy.discardedTermsOf(rows, 0.5);
+
+    assertEquals(1, discarded.size());
+    assertTrue(discarded.contains(10L));
+  }
+
+  private static LongTermsAndValues terms(long[] termValues) {
+    float[] values = new float[termValues.length];
+    for (int i = 0; i < values.length; ++i) {
+      values[i] = 1f;
+    }
+    return LongTermsAndValuesTestFactory.create(termValues, values, termValues.length);
+  }
+}