diff --git a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/BaseInvertedIndex.java b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/BaseInvertedIndex.java index 5f7611f..549cbc4 100644 --- a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/BaseInvertedIndex.java +++ b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/BaseInvertedIndex.java @@ -3,10 +3,8 @@ import com.carrotsearch.hppc.LongDoubleHashMap; import com.carrotsearch.hppc.LongHashSet; -import com.carrotsearch.hppc.LongIntHashMap; import com.carrotsearch.hppc.LongObjectHashMap; import com.carrotsearch.hppc.cursors.LongCursor; -import com.carrotsearch.hppc.cursors.LongIntCursor; import com.carrotsearch.hppc.cursors.LongObjectCursor; import com.uber.ussi.MemoryFootprint; import com.uber.ussi.ProcessorAllowance; @@ -37,7 +35,6 @@ import com.uber.ussi.searchablestructure.utils.parallel.ParallelShardSearch; import com.uber.ussi.searchablestructure.utils.parallel.SharedMinSimilarity; import com.uber.ussi.utils.BoundedSizeMaxHeap; -import com.uber.ussi.utils.ConfigKeys; import com.uber.ussi.utils.MathUtils; import java.util.ArrayList; import java.util.Arrays; @@ -172,11 +169,12 @@ protected BaseInvertedIndex( this.mergeScoresFromAccumulatedConjunction = sparsMerge.mergeScoresFromAccumulatedConjunction(); this.partialConjunctionPolicy = sparsMerge.partialConjunctionPolicy(); this.doesStoreMergePostingValues = sparsMerge.doesStoreMergePostingValues(); - this.maxFractionIdsPerTerm = parseMaxFractionIdsPerTerm(namespaceConfig); + this.maxFractionIdsPerTerm = PopularTermDiscardPolicy.maxFractionIdsPerTerm(namespaceConfig); validateRows(); this.discardedTerms = structureDiscardedTerms == null - ? discardedTermsOf(rowNumToTermsAndValuesMap, maxFractionIdsPerTerm) + ? PopularTermDiscardPolicy.discardedTermsOf( + rowNumToTermsAndValuesMap, maxFractionIdsPerTerm) : structureDiscardedTerms; LongObjectHashMap discardedTermFreeRows = buildDiscardedTermFreeRows(); this.verificationRowNumToTermsAndValuesMap = @@ -301,7 +299,7 @@ final long[] getDiscardedTermsForTests() { } final boolean discardsPopularTerms() { - return maxFractionIdsPerTerm < 1.0; + return PopularTermDiscardPolicy.doesDiscardPopularTerms(maxFractionIdsPerTerm); } final long[] getRowNumsForKeyForTests(int shard, long key) { @@ -623,51 +621,6 @@ private KeyAndPrefixFilteringData[] collectFilteredSearchQueryKeys( return keyData; } - /** The terms a structure holding {@code rowNumToTermsAndValuesMap} discards as popular. */ - static LongHashSet discardedTermsOf( - NamespaceConfig namespaceConfig, - LongObjectHashMap rowNumToTermsAndValuesMap) { - return discardedTermsOf( - rowNumToTermsAndValuesMap, parseMaxFractionIdsPerTerm(namespaceConfig)); - } - - /** - * Identifies the high-popularity terms to discard. The structure sees the complete dataset, so - * observed popularity is true popularity: a term is discarded when it occurs in more than - * floor(numRows * maxFractionIdsPerTerm) rows. - * - *

Counted over the terms of each row, never over the keys the rows are indexed under, so a - * frequent key is never discarded for being frequent. Where the keys are signatures they are - * generated afterwards, from the rows these terms have been removed from. - */ - private static LongHashSet discardedTermsOf( - LongObjectHashMap rowNumToTermsAndValuesMap, - double maxFractionIdsPerTerm) { - LongIntHashMap numRowsByTerm = new LongIntHashMap(); - // A row counts once per distinct term, so a term repeated within one row stays one row. - LongHashSet termsInRow = new LongHashSet(); - for (LongObjectCursor entry : rowNumToTermsAndValuesMap) { - termsInRow.clear(); - for (int i = 0; i < entry.value.termsLength(); ++i) { - long term = entry.value.getTerm(i); - if (!termsInRow.add(term)) { - continue; - } - int numRows = numRowsByTerm.containsKey(term) ? numRowsByTerm.get(term) + 1 : 1; - numRowsByTerm.put(term, numRows); - } - } - int maxNumRowsPerTerm = - (int) Math.floor(rowNumToTermsAndValuesMap.size() * maxFractionIdsPerTerm); - LongHashSet popularTerms = new LongHashSet(); - for (LongIntCursor entry : numRowsByTerm) { - if (entry.value > maxNumRowsPerTerm) { - popularTerms.add(entry.key); - } - } - return popularTerms; - } - /** Returns the rows with the high-popularity terms dropped, which is what gets indexed. */ private LongObjectHashMap buildDiscardedTermFreeRows() { if (discardedTerms.isEmpty()) { @@ -813,11 +766,6 @@ private void validateRows() { } } - private static double parseMaxFractionIdsPerTerm(NamespaceConfig namespaceConfig) { - return namespaceConfig.readDoubleIndexParam( - ConfigKeys.MAX_FRACTION_IDS_PER_TERM, ConfigKeys.DEFAULT_MAX_FRACTION_IDS_PER_TERM); - } - /** * Returns the one record type the structure stores and the comparator reads. Both failures are * backstops: {@code IndexConfigValidator} rejects a pairing with none in common, and no diff --git a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/HybridIndex.java b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/HybridIndex.java index 451aa25..3d4899d 100644 --- a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/HybridIndex.java +++ b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/HybridIndex.java @@ -17,17 +17,6 @@ /** Hybrid inverted index using exact keys for short rows and signatures for long rows. */ public final class HybridIndex extends Index { - /** - * The largest term count a row can have and still be keyed by its own terms. Above it a row is - * keyed by signatures instead. - * - *

It is the signature count because that is where signatures stop being a saving: a row with - * fewer terms than that would be replaced by more signatures than it had terms, costing list - * entries and buying no pruning. The two quantities are derived separately and happen to - * coincide, so the cutoff names itself rather than reading as a signature count here. - */ - private static final int TERM_KEYING_CUTOFF = SignatureIndex.NUM_SIGNATURES_PER_ROW; - private final TermIndex termIndex; private final SignatureIndex signatureIndex; private final boolean termPopularityFilteringEnabled; @@ -37,10 +26,11 @@ public HybridIndex( LongObjectHashMap rowNumToTermsAndValuesMap, LongObjectHashMap rowNumToMetaMap) { super(namespaceConfig); + double maxFractionIdsPerTerm = PopularTermDiscardPolicy.maxFractionIdsPerTerm(namespaceConfig); LongObjectHashMap exactRows = new LongObjectHashMap<>(); LongObjectHashMap signatureRows = new LongObjectHashMap<>(); for (LongObjectCursor entry : rowNumToTermsAndValuesMap) { - if (entry.value.termsLength() <= TERM_KEYING_CUTOFF) { + if (entry.value.termsLength() <= InvertedHybridConfiguration.TERM_KEYING_CUTOFF) { exactRows.put(entry.key, entry.value); } else { signatureRows.put(entry.key, entry.value); @@ -50,13 +40,15 @@ public HybridIndex( // term index can collect it: a signature list holds one entry per row whatever that row's // terms are, so discarding leaves a signature index's lists exactly as long and only moves the // signatures its rows are keyed by. - LongHashSet discardedTerms = BaseInvertedIndex.discardedTermsOf(namespaceConfig, exactRows); + LongHashSet discardedTerms = + PopularTermDiscardPolicy.discardedTermsOf(exactRows, maxFractionIdsPerTerm); // The signature index is built first so a comparator without a generator is rejected before // the term index is populated. this.signatureIndex = new SignatureIndex(namespaceConfig, signatureRows, rowNumToMetaMap, new LongHashSet()); this.termIndex = new TermIndex(namespaceConfig, exactRows, rowNumToMetaMap, discardedTerms); - this.termPopularityFilteringEnabled = termIndex.discardsPopularTerms(); + this.termPopularityFilteringEnabled = + PopularTermDiscardPolicy.doesDiscardPopularTerms(maxFractionIdsPerTerm); } @Override @@ -66,7 +58,8 @@ public List getNearestNeighborRowNums( throw new IllegalArgumentException("k must be greater than 0."); } int maxResults = Math.min(k, namespaceConfig.getMaxNumSimilarities()); - boolean queryUsesExactIndex = record.termsLength() <= TERM_KEYING_CUTOFF; + boolean queryUsesExactIndex = + record.termsLength() <= InvertedHybridConfiguration.TERM_KEYING_CUTOFF; Index firstIndex = queryUsesExactIndex ? termIndex : signatureIndex; Index secondIndex = queryUsesExactIndex ? signatureIndex : termIndex; boolean secondIndexIsExact = !queryUsesExactIndex; @@ -189,8 +182,9 @@ private boolean maySearchIndex( if (termPopularityFilteringEnabled) { return true; } - int minNumTerms = exactIndex ? 0 : TERM_KEYING_CUTOFF + 1; - int maxNumTerms = exactIndex ? TERM_KEYING_CUTOFF : Integer.MAX_VALUE; + int minNumTerms = exactIndex ? 0 : InvertedHybridConfiguration.TERM_KEYING_CUTOFF + 1; + int maxNumTerms = + exactIndex ? InvertedHybridConfiguration.TERM_KEYING_CUTOFF : Integer.MAX_VALUE; return comparator.mayPassNumTermsFiltering(query, minNumTerms, maxNumTerms, minSimilarity); } diff --git a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/InvertedHybridConfiguration.java b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/InvertedHybridConfiguration.java new file mode 100644 index 0000000..2a2413e --- /dev/null +++ b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/InvertedHybridConfiguration.java @@ -0,0 +1,25 @@ +/* AUTHOR: Ahmed Metwally (ametwally@uber.com) */ +package com.uber.ussi.searchablestructure.index.inverted; + +/** + * Shared constants for hybrid inverted indexes that combine term-keyed and signature-keyed halves. + */ +public final class InvertedHybridConfiguration { + + /** + * How many signatures stand in for one long row in the signature-keyed half. Drawn from + * experiments trading recall against list length. + */ + public static final int NUM_SIGNATURES_PER_ROW = 270; + + /** + * The largest term count a row can have and still be keyed by its own terms in the hybrid index. + * Above this count a row is keyed by signatures instead. + * + *

It equals {@link #NUM_SIGNATURES_PER_ROW} because that is where signatures stop being a + * saving: a row with fewer terms would be replaced by more signatures than it had terms. + */ + public static final int TERM_KEYING_CUTOFF = NUM_SIGNATURES_PER_ROW; + + private InvertedHybridConfiguration() {} +} diff --git a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/PopularTermDiscardPolicy.java b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/PopularTermDiscardPolicy.java new file mode 100644 index 0000000..e8e99eb --- /dev/null +++ b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/PopularTermDiscardPolicy.java @@ -0,0 +1,89 @@ +/* AUTHOR: Ahmed Metwally (ametwally@uber.com) */ +package com.uber.ussi.searchablestructure.index.inverted; + +import com.carrotsearch.hppc.LongHashSet; +import com.carrotsearch.hppc.LongIntHashMap; +import com.carrotsearch.hppc.LongObjectHashMap; +import com.carrotsearch.hppc.cursors.LongIntCursor; +import com.carrotsearch.hppc.cursors.LongObjectCursor; +import com.uber.ussi.config.NamespaceConfig; +import com.uber.ussi.entity.termsandvalues.LongTermsAndValues; +import com.uber.ussi.utils.ConfigKeys; +import java.util.Objects; + +/** + * Popularity-based term discard policy for inverted term-keyed indexes. + * + *

Hybrid and term indexes share this policy so a composite structure can compute discards over + * its term-keyed partition without depending on {@link BaseInvertedIndex}. + */ +public final class PopularTermDiscardPolicy { + + private PopularTermDiscardPolicy() {} + + /** Returns the configured maximum fraction of rows a term may appear in before it is discarded. */ + public static double maxFractionIdsPerTerm(NamespaceConfig namespaceConfig) { + Objects.requireNonNull(namespaceConfig, "namespaceConfig is null."); + return namespaceConfig.readDoubleIndexParam( + ConfigKeys.MAX_FRACTION_IDS_PER_TERM, ConfigKeys.DEFAULT_MAX_FRACTION_IDS_PER_TERM); + } + + /** Returns whether any term may be discarded under the configured popularity threshold. */ + public static boolean doesDiscardPopularTerms(NamespaceConfig namespaceConfig) { + return maxFractionIdsPerTerm(namespaceConfig) < 1.0; + } + + /** Returns whether any term may be discarded under {@code maxFractionIdsPerTerm}. */ + public static boolean doesDiscardPopularTerms(double maxFractionIdsPerTerm) { + return maxFractionIdsPerTerm < 1.0; + } + + /** + * Returns the terms a structure holding {@code rowNumToTermsAndValuesMap} discards as popular. + */ + public static LongHashSet discardedTermsOf( + NamespaceConfig namespaceConfig, + LongObjectHashMap rowNumToTermsAndValuesMap) { + Objects.requireNonNull(namespaceConfig, "namespaceConfig is null."); + Objects.requireNonNull(rowNumToTermsAndValuesMap, "rowNumToTermsAndValuesMap is null."); + return discardedTermsOf(rowNumToTermsAndValuesMap, maxFractionIdsPerTerm(namespaceConfig)); + } + + /** + * Identifies the high-popularity terms to discard. The structure sees the complete dataset, so + * observed popularity is true popularity: a term is discarded when it occurs in more than + * floor(numRows * maxFractionIdsPerTerm) rows. + * + *

Counted over the terms of each row, never over the keys the rows are indexed under, so a + * frequent key is never discarded for being frequent. Where the keys are signatures they are + * generated afterwards, from the rows these terms have been removed from. + */ + public static LongHashSet discardedTermsOf( + LongObjectHashMap rowNumToTermsAndValuesMap, + double maxFractionIdsPerTerm) { + Objects.requireNonNull(rowNumToTermsAndValuesMap, "rowNumToTermsAndValuesMap is null."); + LongIntHashMap numRowsByTerm = new LongIntHashMap(); + // A row counts once per distinct term, so a term repeated within one row stays one row. + LongHashSet termsInRow = new LongHashSet(); + for (LongObjectCursor entry : rowNumToTermsAndValuesMap) { + termsInRow.clear(); + for (int i = 0; i < entry.value.termsLength(); ++i) { + long term = entry.value.getTerm(i); + if (!termsInRow.add(term)) { + continue; + } + int numRows = numRowsByTerm.containsKey(term) ? numRowsByTerm.get(term) + 1 : 1; + numRowsByTerm.put(term, numRows); + } + } + int maxNumRowsPerTerm = + (int) Math.floor(rowNumToTermsAndValuesMap.size() * maxFractionIdsPerTerm); + LongHashSet popularTerms = new LongHashSet(); + for (LongIntCursor entry : numRowsByTerm) { + if (entry.value > maxNumRowsPerTerm) { + popularTerms.add(entry.key); + } + } + return popularTerms; + } +} diff --git a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/SignatureIndex.java b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/SignatureIndex.java index fd71b46..7f78af1 100644 --- a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/SignatureIndex.java +++ b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/SignatureIndex.java @@ -23,11 +23,9 @@ */ public final class SignatureIndex extends BaseInvertedIndex { - /** - * How many signatures stand in for one row, which fixes the length of every signature list. Drawn - * from experiments trading recall against list length. - */ - public static final int NUM_SIGNATURES_PER_ROW = 270; + /** How many signatures stand in for one row. See {@link InvertedHybridConfiguration}. */ + public static final int NUM_SIGNATURES_PER_ROW = + InvertedHybridConfiguration.NUM_SIGNATURES_PER_ROW; public SignatureIndex( NamespaceConfig namespaceConfig, diff --git a/src/test/java/com/uber/ussi/searchablestructure/index/inverted/PopularTermDiscardPolicyTest.java b/src/test/java/com/uber/ussi/searchablestructure/index/inverted/PopularTermDiscardPolicyTest.java new file mode 100644 index 0000000..42058db --- /dev/null +++ b/src/test/java/com/uber/ussi/searchablestructure/index/inverted/PopularTermDiscardPolicyTest.java @@ -0,0 +1,59 @@ +/* AUTHOR: Ahmed Metwally (ametwally@uber.com) */ +package com.uber.ussi.searchablestructure.index.inverted; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.carrotsearch.hppc.LongHashSet; +import com.carrotsearch.hppc.LongObjectHashMap; +import com.uber.ussi.config.NamespaceConfig; +import com.uber.ussi.entity.termsandvalues.LongTermsAndValues; +import com.uber.ussi.entity.termsandvalues.LongTermsAndValuesTestFactory; +import com.uber.ussi.utils.ConfigKeys; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class PopularTermDiscardPolicyTest { + + @Test + void doesDiscardPopularTermsWhenMaxFractionIsBelowOne() { + NamespaceConfig config = + NamespaceConfig.builder() + .minTermsAndValuesLength(0) + .maxTermsAndValuesLength(100) + .maxCacheSize(100) + .cacheType("scan") + .indexType("inverted_term") + .indexParams(Map.of(ConfigKeys.MAX_FRACTION_IDS_PER_TERM, "0.5")) + .comparatorType("jaccard") + .comparatorNormalizerType("complement") + .maxNumSearchableStructures(3) + .maxNumSimilarities(100) + .build(); + assertTrue(PopularTermDiscardPolicy.doesDiscardPopularTerms(config)); + assertFalse(PopularTermDiscardPolicy.doesDiscardPopularTerms(1.0)); + } + + @Test + void discardedTermsOfMarksTermsAboveTheRowFractionThreshold() { + LongObjectHashMap rows = new LongObjectHashMap<>(); + rows.put(1L, terms(new long[] {10, 20})); + rows.put(2L, terms(new long[] {10, 30})); + rows.put(3L, terms(new long[] {10, 40})); + rows.put(4L, terms(new long[] {50})); + + LongHashSet discarded = PopularTermDiscardPolicy.discardedTermsOf(rows, 0.5); + + assertEquals(1, discarded.size()); + assertTrue(discarded.contains(10L)); + } + + private static LongTermsAndValues terms(long[] termValues) { + float[] values = new float[termValues.length]; + for (int i = 0; i < values.length; ++i) { + values[i] = 1f; + } + return LongTermsAndValuesTestFactory.create(termValues, values, termValues.length); + } +}