Filter-aware aliases: make filtered_stats cost track the view, and cache it

server/src/internalClusterTest/java/org/opensearch/aliases/FilterAwareAliasIT.java

@@ -9,6 +9,7 @@
 package org.opensearch.aliases;
 
 import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
+import org.opensearch.action.bulk.BulkRequestBuilder;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.search.SearchType;
 import org.opensearch.action.support.WriteRequest.RefreshPolicy;
@@ -45,15 +46,27 @@ public class FilterAwareAliasIT extends OpenSearchIntegTestCase {
     /** Build a corpus where RESTRICTED_TERM appears only in filtered-out docs, plus a single visible
      *  sample doc that contains both the restricted term and the absent term. */
     private void buildCorpus() throws Exception {
+        buildCorpus(200);
+    }
+
+    /**
+     * As {@link #buildCorpus()}, with the number of restricted documents under the caller's control. The visible
+     * subset stays at 21 documents, so {@code restrictedDocs} sets how selective the view is.
+     */
+    private void buildCorpus(int restrictedDocs) throws Exception {
         assertAcked(
             prepareCreate(INDEX).setMapping("dept", "type=keyword", "content", "type=text")
                 .setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0))
         );
 
         // Many restricted docs carrying the restricted term -> raises its corpus-wide df.
-        for (int i = 0; i < 200; i++) {
-            client().prepareIndex(INDEX).setSource("dept", RESTRICTED_DEPT, "content", "filler " + RESTRICTED_TERM + " noise" + i).get();
+        BulkRequestBuilder bulk = client().prepareBulk();
+        for (int i = 0; i < restrictedDocs; i++) {
+            bulk.add(
+                client().prepareIndex(INDEX).setSource("dept", RESTRICTED_DEPT, "content", "filler " + RESTRICTED_TERM + " noise" + i)
+            );
         }
+        assertFalse(bulk.get().hasFailures());
         // A handful of visible docs (without the restricted term of their own).
         for (int i = 0; i < 20; i++) {
             client().prepareIndex(INDEX).setSource("dept", VISIBLE_DEPT, "content", "cardio note " + i).get();
@@ -184,7 +197,20 @@ public void testPreFilterPrefixScoresOverVisibleSubsetOnly() throws Exception {
      * duration of the assertions and clear it afterwards.
      */
     public void testFilteredStatisticsMatchPhysicallyFilteredIndex() throws Exception {
-        buildCorpus();
+        assertFilteredStatisticsMatchPhysicallyFilteredIndex(200);   // 21 visible of 221 documents
+    }
+
+    /**
+     * The same equivalence, on a highly selective view: 21 visible of 2,021 documents, about 1%. Cost scales with
+     * the size of the view rather than the size of the index, so a selective view exercises a very different ratio
+     * of visible to total documents, and the statistics must come out identical either way.
+     */
+    public void testFilteredStatisticsMatchPhysicallyFilteredIndexForSelectiveView() throws Exception {
+        assertFilteredStatisticsMatchPhysicallyFilteredIndex(2000);
+    }
+
+    private void assertFilteredStatisticsMatchPhysicallyFilteredIndex(int restrictedDocs) throws Exception {
+        buildCorpus(restrictedDocs);
         addAlias("fs_pre", "pre_filter");
 
         // Build a physical "visible-only" index: reindex just the cardiology docs. This is the
@@ -226,6 +252,59 @@ public void testFilteredStatisticsMatchPhysicallyFilteredIndex() throws Exceptio
         }
     }
 
+    /**
+     * Small-view guardrail: filtered statistics cost O(visible postings) to build, so above a configured
+     * visible-doc budget the request falls back to the default constant_score behavior instead of paying that
+     * cost. Asserts both sides of the switch on the same corpus and alias: with a generous budget the sample
+     * scores with real BM25 (not 1.0), and with a budget below the visible-subset size every hit scores exactly
+     * 1.0 -- still leak-free, just unranked.
+     */
+    public void testFilteredStatisticsGuardrailFallsBackToConstantScore() throws Exception {
+        buildCorpus();
+        addAlias("fs_guard", "pre_filter");
+
+        final String gate = "opensearch.filter_aware_alias.filtered_stats";
+        final String budget = "opensearch.filter_aware_alias.filtered_stats.max_visible_docs";
+        final String previousGate = System.getProperty(gate);
+        final String previousBudget = System.getProperty(budget);
+        try {
+            System.setProperty(gate, "true");
+
+            // Budget well above the ~21 visible docs: filtered_stats applies, so the restricted-only term is
+            // scored with real (visible-subset) BM25 rather than flattened.
+            System.setProperty(budget, "1000000");
+            float rankedScore = sampleScore("fs_guard", RESTRICTED_TERM);
+            assertTrue("sample found with filtered_stats in budget", rankedScore > 0f);
+            assertNotEquals(
+                "with filtered_stats applied the score should be real BM25, not the constant_score 1.0",
+                1.0f,
+                rankedScore,
+                0.0001f
+            );
+
+            // Budget below the visible-subset size: the guardrail trips and we fall back to constant_score,
+            // which scores every hit exactly 1.0.
+            System.setProperty(budget, "1");
+            float guardedScore = sampleScore("fs_guard", RESTRICTED_TERM);
+            assertEquals("guardrail should fall back to constant_score (flat 1.0)", 1.0f, guardedScore, 0.0001f);
+
+            // The fallback is still leak-free: a term confined to filtered-out docs is indistinguishable from
+            // a term that exists nowhere.
+            assertEquals("fallback stays leak-free", sampleScore("fs_guard", ABSENT_TERM), guardedScore, 0.0001f);
+        } finally {
+            restoreProperty(gate, previousGate);
+            restoreProperty(budget, previousBudget);
+        }
+    }
+
+    private static void restoreProperty(String key, String previous) {
+        if (previous == null) {
+            System.clearProperty(key);
+        } else {
+            System.setProperty(key, previous);
+        }
+    }
+
     /** Build a multi-shard corpus for the dfs test. Same shape as {@link #buildCorpus()} but spread
      *  across several shards so dfs_query_then_fetch must aggregate per-shard statistics. */
     private void buildMultiShardCorpus(String index, int shards) throws Exception {

server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java

@@ -50,6 +50,7 @@
 import org.opensearch.index.MergeSchedulerConfig;
 import org.opensearch.index.SearchSlowLog;
 import org.opensearch.index.TieredMergePolicyProvider;
+import org.opensearch.index.cache.filteredstats.FilteredStatsWarmer;
 import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings;
 import org.opensearch.index.engine.EngineConfig;
 import org.opensearch.index.fielddata.IndexFieldDataService;
@@ -209,6 +210,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING,
                 MapperService.INDEX_MAPPING_DYNAMIC_PROPERTIES_LUCENE_FIELD_LIMIT_SETTING,
                 IndicesBitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING,
+                FilteredStatsWarmer.INDEX_WARM_FILTERED_STATS_SETTING,
                 IndexModule.INDEX_STORE_TYPE_SETTING,
                 IndexModule.INDEX_COMPOSITE_STORE_TYPE_SETTING,
                 IndexModule.INDEX_STORE_FACTORY_SETTING,

server/src/main/java/org/opensearch/index/IndexService.java

@@ -60,6 +60,7 @@
 import org.opensearch.common.util.concurrent.AbstractRunnable;
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.Assertions;
+import org.opensearch.core.common.breaker.CircuitBreaker;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.core.indices.breaker.CircuitBreakerService;
@@ -72,6 +73,8 @@
 import org.opensearch.index.analysis.IndexAnalyzers;
 import org.opensearch.index.cache.IndexCache;
 import org.opensearch.index.cache.bitset.BitsetFilterCache;
+import org.opensearch.index.cache.filteredstats.FilteredStatsCache;
+import org.opensearch.index.cache.filteredstats.FilteredStatsWarmer;
 import org.opensearch.index.cache.query.QueryCache;
 import org.opensearch.index.compositeindex.CompositeIndexSettings;
 import org.opensearch.index.engine.Engine;
@@ -326,12 +329,17 @@ public IndexService(
                 return shard == null ? IndicesFieldDataCache.Key.NO_SHARD_IDENTITY : System.identityHashCode(shard);
             });
             this.bitsetFilterCache = new BitsetFilterCache(indexSettings, indicesBitsetFilterCache, new BitsetCacheListener(this));
+            FilteredStatsCache filteredStatsCache = new FilteredStatsCache(
+                indexSettings,
+                circuitBreakerService.getBreaker(CircuitBreaker.FIELDDATA)
+            );
             this.warmer = new IndexWarmer(
                 threadPool,
                 indexFieldData,
-                indicesBitsetFilterCache != null ? indicesBitsetFilterCache.createListener(threadPool) : null
+                indicesBitsetFilterCache != null ? indicesBitsetFilterCache.createListener(threadPool) : null,
+                new FilteredStatsWarmer(threadPool, filteredStatsCache)
             );
-            this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache);
+            this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache, filteredStatsCache);
         } else {
             assert indexAnalyzers == null;
             this.mapperService = null;

server/src/main/java/org/opensearch/index/cache/IndexCache.java

@@ -37,6 +37,7 @@
 import org.opensearch.index.AbstractIndexComponent;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.cache.bitset.BitsetFilterCache;
+import org.opensearch.index.cache.filteredstats.FilteredStatsCache;
 import org.opensearch.index.cache.query.QueryCache;
 
 import java.io.Closeable;
@@ -52,11 +53,22 @@ public class IndexCache extends AbstractIndexComponent implements Closeable {
 
     private final QueryCache queryCache;
     private final BitsetFilterCache bitsetFilterCache;
+    private final FilteredStatsCache filteredStatsCache;
 
     public IndexCache(IndexSettings indexSettings, QueryCache queryCache, BitsetFilterCache bitsetFilterCache) {
+        this(indexSettings, queryCache, bitsetFilterCache, null);
+    }
+
+    public IndexCache(
+        IndexSettings indexSettings,
+        QueryCache queryCache,
+        BitsetFilterCache bitsetFilterCache,
+        FilteredStatsCache filteredStatsCache
+    ) {
         super(indexSettings);
         this.queryCache = queryCache;
         this.bitsetFilterCache = bitsetFilterCache;
+        this.filteredStatsCache = filteredStatsCache;
     }
 
     public QueryCache query() {
@@ -70,14 +82,25 @@ public BitsetFilterCache bitsetFilterCache() {
         return bitsetFilterCache;
     }
 
+    /**
+     * Return the {@link FilteredStatsCache} for this index, or {@code null} when filtered-stats caching is unavailable
+     * (e.g. a cacheless index context). Callers must tolerate {@code null} and fall back to inline computation.
+     */
+    public FilteredStatsCache filteredStatsCache() {
+        return filteredStatsCache;
+    }
+
     @Override
     public void close() throws IOException {
-        IOUtils.close(queryCache, bitsetFilterCache);
+        IOUtils.close(queryCache, bitsetFilterCache, filteredStatsCache);
     }
 
     public void clear(String reason) {
         queryCache.clear(reason);
         bitsetFilterCache.clear(reason);
+        if (filteredStatsCache != null) {
+            filteredStatsCache.clear(reason);
+        }
     }
 
 }