diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 67976d58188e2..1ba3ee562317a 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -48,4 +48,5 @@ BWC_VERSION: - "2.1.0" - "2.1.1" - "2.2.0" + - "2.2.1" - "2.3.0" diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index ce84d9658a808..8c2a6b4889122 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -4,7 +4,8 @@ - [Install Prerequisites](#install-prerequisites) - [JDK 11](#jdk-11) - [JDK 14](#jdk-14) - - [Runtime JDK](#runtime-jdk) + - [JDK 17](#jdk-17) + - [Custom Runtime JDK](#custom-runtime-jdk) - [Windows](#windows) - [Docker](#docker) - [Build](#build) @@ -12,6 +13,7 @@ - [Run OpenSearch](#run-opensearch) - [Use an Editor](#use-an-editor) - [IntelliJ IDEA](#intellij-idea) + - [Remote development using JetBrains Gateway](#remote-development-using-jetbrains-gateway) - [Visual Studio Code](#visual-studio-code) - [Eclipse](#eclipse) - [Project Layout](#project-layout) @@ -35,6 +37,7 @@ - [testImplementation](#testimplementation) - [Gradle Plugins](#gradle-plugins) - [Distribution Download Plugin](#distribution-download-plugin) + - [Creating fat-JAR of a Module](#creating-fat-jar-of-a-module) - [Misc](#misc) - [git-secrets](#git-secrets) - [Installation](#installation) @@ -49,7 +52,7 @@ - [Submitting Changes](#submitting-changes) - [Backports](#backports) - [LineLint](#linelint) - - [Lucene Snapshots](#lucene-snapshots) +- [Lucene Snapshots](#lucene-snapshots) # Developer Guide @@ -374,6 +377,42 @@ The Distribution Download plugin downloads the latest version of OpenSearch by d ./gradlew integTest -PcustomDistributionUrl="https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/1127/linux/x64/dist/opensearch-1.2.0-linux-x64.tar.gz" ``` +### Creating fat-JAR of a Module + +A fat-JAR (or an uber-JAR) is the JAR, which contains classes from all the libraries, on which your project depends and, of course, the classes of current project. + +There might be cases where a developer would like to add some custom logic to the code of a module (or multiple modules) and generate a fat-JAR that can be directly used by the dependency management tool. For example, in [#3665](https://github.com/opensearch-project/OpenSearch/pull/3665) a developer wanted to provide a tentative patch as a fat-JAR to a consumer for changes made in the high level REST client. + +Use [Gradle Shadow plugin](https://imperceptiblethoughts.com/shadow/). +Add the following to the `build.gradle` file of the module for which you want to create the fat-JAR, e.g. `client/rest-high-level/build.gradle`: + +``` +apply plugin: 'com.github.johnrengelman.shadow' +``` + +Run the `shadowJar` command using: +``` +./gradlew :client:rest-high-level:shadowJar +``` + +This will generate a fat-JAR in the `build/distributions` folder of the module, e.g. .`/client/rest-high-level/build/distributions/opensearch-rest-high-level-client-1.4.0-SNAPSHOT.jar`. + +You can further customize your fat-JAR by customising the plugin, More information about shadow plugin can be found [here](https://imperceptiblethoughts.com/shadow/). + +To use the generated JAR, install the JAR locally, e.g. +``` +mvn install:install-file -Dfile=src/main/resources/opensearch-rest-high-level-client-1.4.0-SNAPSHOT.jar -DgroupId=org.opensearch.client -DartifactId=opensearch-rest-high-level-client -Dversion=1.4.0-SNAPSHOT -Dpackaging=jar -DgeneratePom=true +``` + +Refer the installed JAR as any other maven artifact, e.g. + +``` + + org.opensearch.client + opensearch-rest-high-level-client + 1.4.0-SNAPSHOT + +``` ## Misc diff --git a/build.gradle b/build.gradle index e0bb961ce14c2..ce5ea6cdd7e11 100644 --- a/build.gradle +++ b/build.gradle @@ -55,7 +55,7 @@ plugins { id 'lifecycle-base' id 'opensearch.docker-support' id 'opensearch.global-build-info' - id "com.diffplug.spotless" version "6.9.0" apply false + id "com.diffplug.spotless" version "6.9.1" apply false id "org.gradle.test-retry" version "1.4.0" apply false id "test-report-aggregation" id 'jacoco-report-aggregation' diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 8e9911294977a..4af1acfed0ab2 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ opensearch = 3.0.0 -lucene = 9.3.0 +lucene = 9.4.0-snapshot-ddf0d0a bundled_jdk_vendor = adoptium bundled_jdk = 17.0.4+8 diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java index c4901f7e03dbf..0ec2935e71492 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java @@ -112,10 +112,6 @@ import org.opensearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; import org.opensearch.search.aggregations.bucket.filter.ParsedFilter; import org.opensearch.search.aggregations.bucket.filter.ParsedFilters; -import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid; -import org.opensearch.search.aggregations.bucket.geogrid.ParsedGeoTileGrid; import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.opensearch.search.aggregations.bucket.global.ParsedGlobal; import org.opensearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; @@ -161,7 +157,6 @@ import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder; import org.opensearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.opensearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder; -import org.opensearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; import org.opensearch.search.aggregations.metrics.GeoCentroidAggregationBuilder; import org.opensearch.search.aggregations.metrics.InternalHDRPercentileRanks; import org.opensearch.search.aggregations.metrics.InternalHDRPercentiles; @@ -173,7 +168,6 @@ import org.opensearch.search.aggregations.metrics.ParsedAvg; import org.opensearch.search.aggregations.metrics.ParsedCardinality; import org.opensearch.search.aggregations.metrics.ParsedExtendedStats; -import org.opensearch.search.aggregations.metrics.ParsedGeoBounds; import org.opensearch.search.aggregations.metrics.ParsedGeoCentroid; import org.opensearch.search.aggregations.metrics.ParsedHDRPercentileRanks; import org.opensearch.search.aggregations.metrics.ParsedHDRPercentiles; @@ -2240,7 +2234,6 @@ static List getDefaultNamedXContents() { map.put(StatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedStatsBucket.fromXContent(p, (String) c)); map.put(ExtendedStatsAggregationBuilder.NAME, (p, c) -> ParsedExtendedStats.fromXContent(p, (String) c)); map.put(ExtendedStatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedExtendedStatsBucket.fromXContent(p, (String) c)); - map.put(GeoBoundsAggregationBuilder.NAME, (p, c) -> ParsedGeoBounds.fromXContent(p, (String) c)); map.put(GeoCentroidAggregationBuilder.NAME, (p, c) -> ParsedGeoCentroid.fromXContent(p, (String) c)); map.put(HistogramAggregationBuilder.NAME, (p, c) -> ParsedHistogram.fromXContent(p, (String) c)); map.put(DateHistogramAggregationBuilder.NAME, (p, c) -> ParsedDateHistogram.fromXContent(p, (String) c)); @@ -2257,8 +2250,6 @@ static List getDefaultNamedXContents() { map.put(GlobalAggregationBuilder.NAME, (p, c) -> ParsedGlobal.fromXContent(p, (String) c)); map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c)); map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c)); - map.put(GeoHashGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c)); - map.put(GeoTileGridAggregationBuilder.NAME, (p, c) -> ParsedGeoTileGrid.fromXContent(p, (String) c)); map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c)); map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c)); map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c)); diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 24c164f0f1e12..58e9a16f424db 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -11,7 +11,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.5-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=97a52d145762adc241bad7fd18289bf7f6801e08ece6badf80402fe2b9f250b1 +distributionSha256Sum=db9c8211ed63f61f60292c69e80d89196f9eb36665e369e7f00ac4cc841c2219 diff --git a/modules/geo/build.gradle b/modules/geo/build.gradle index 1146b6924b2f8..7f687a414e566 100644 --- a/modules/geo/build.gradle +++ b/modules/geo/build.gradle @@ -28,18 +28,25 @@ * under the License. */ apply plugin: 'opensearch.yaml-rest-test' +apply plugin: 'opensearch.internal-cluster-test' opensearchplugin { - description 'Placeholder plugin for geospatial features in OpenSearch. only registers geo_shape field mapper for now' + description 'Plugin for geospatial features in OpenSearch. Registering the geo_shape and aggregations GeoBounds on Geo_Shape and Geo_Point' classname 'org.opensearch.geo.GeoModulePlugin' } restResources { restApi { - includeCore '_common', 'indices', 'index', 'search' + includeCore '_common', 'indices', 'index', 'search', 'bulk' } } artifacts { restTests(project.file('src/yamlRestTest/resources/rest-api-spec/test')) } -test.enabled = false +/** + * These compiler arguments needs to be removed, as there are raw types being used in the GeoGrid and GeoTile aggregations. + */ +tasks.withType(JavaCompile).configureEach { + options.compilerArgs -= '-Xlint:rawtypes' + options.compilerArgs -= '-Xlint:unchecked' +} diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java new file mode 100644 index 0000000000000..7dc6f2c1b89b7 --- /dev/null +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo; + +import org.opensearch.index.mapper.GeoShapeFieldMapper; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.TestGeoShapeFieldMapperPlugin; + +import java.util.Collection; +import java.util.Collections; + +/** + * This is the base class for all the Geo related integration tests. Use this class to add the features and settings + * for the test cluster on which integration tests are running. + */ +public abstract class GeoModulePluginIntegTestCase extends OpenSearchIntegTestCase { + /** + * Returns a collection of plugins that should be loaded on each node for doing the integration tests. As this + * geo plugin is not getting packaged in a zip, we need to load it before the tests run. + * + * @return List of {@link Plugin} + */ + @Override + protected Collection> nodePlugins() { + return Collections.singletonList(GeoModulePlugin.class); + } + + /** + * This was added as a backdoor to Mock the implementation of {@link GeoShapeFieldMapper} which was coming from + * {@link GeoModulePlugin}. Mock implementation is {@link TestGeoShapeFieldMapperPlugin}. Now we are using the + * {@link GeoModulePlugin} in our integration tests we need to override this functionality to avoid multiple mapper + * error. + * + * @return boolean + */ + @Override + protected boolean addMockGeoShapeFieldMapper() { + return false; + } +} diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/MissingValueIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/MissingValueIT.java new file mode 100644 index 0000000000000..2ac73728b2dab --- /dev/null +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/MissingValueIT.java @@ -0,0 +1,59 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.geo.GeoModulePluginIntegTestCase; +import org.opensearch.geo.search.aggregations.metrics.GeoBounds; +import org.opensearch.geo.tests.common.AggregationBuilders; +import org.opensearch.test.OpenSearchIntegTestCase; + +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.closeTo; + +@OpenSearchIntegTestCase.SuiteScopeTestCase +public class MissingValueIT extends GeoModulePluginIntegTestCase { + + @Override + protected void setupSuiteScopeCluster() throws Exception { + assertAcked(prepareCreate("idx").setMapping("date", "type=date", "location", "type=geo_point", "str", "type=keyword").get()); + indexRandom( + true, + client().prepareIndex("idx").setId("1").setSource(), + client().prepareIndex("idx") + .setId("2") + .setSource("str", "foo", "long", 3L, "double", 5.5, "date", "2015-05-07", "location", "1,2") + ); + } + + public void testUnmappedGeoBounds() { + SearchResponse response = client().prepareSearch("idx") + .addAggregation(AggregationBuilders.geoBounds("bounds").field("non_existing_field").missing("2,1")) + .get(); + assertSearchResponse(response); + GeoBounds bounds = response.getAggregations().get("bounds"); + assertThat(bounds.bottomRight().lat(), closeTo(2.0, 1E-5)); + assertThat(bounds.bottomRight().lon(), closeTo(1.0, 1E-5)); + assertThat(bounds.topLeft().lat(), closeTo(2.0, 1E-5)); + assertThat(bounds.topLeft().lon(), closeTo(1.0, 1E-5)); + } + + public void testGeoBounds() { + SearchResponse response = client().prepareSearch("idx") + .addAggregation(AggregationBuilders.geoBounds("bounds").field("location").missing("2,1")) + .get(); + assertSearchResponse(response); + GeoBounds bounds = response.getAggregations().get("bounds"); + assertThat(bounds.bottomRight().lat(), closeTo(1.0, 1E-5)); + assertThat(bounds.bottomRight().lon(), closeTo(2.0, 1E-5)); + assertThat(bounds.topLeft().lat(), closeTo(2.0, 1E-5)); + assertThat(bounds.topLeft().lon(), closeTo(1.0, 1E-5)); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/GeoHashGridIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridIT.java similarity index 89% rename from server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/GeoHashGridIT.java rename to modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridIT.java index 56d918feef9d8..6ab7dd5254679 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/GeoHashGridIT.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridIT.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket; +package org.opensearch.geo.search.aggregations.bucket; import com.carrotsearch.hppc.ObjectIntHashMap; import com.carrotsearch.hppc.ObjectIntMap; @@ -41,12 +41,12 @@ import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.geo.GeoModulePluginIntegTestCase; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGrid; +import org.opensearch.geo.tests.common.AggregationBuilders; import org.opensearch.index.query.GeoBoundingBoxQueryBuilder; -import org.opensearch.search.aggregations.AggregationBuilders; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.bucket.filter.Filter; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGrid; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGrid.Bucket; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.VersionUtils; @@ -57,17 +57,16 @@ import java.util.Random; import java.util.Set; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.geometry.utils.Geohash.PRECISION; import static org.opensearch.geometry.utils.Geohash.stringEncode; -import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.search.aggregations.AggregationBuilders.geohashGrid; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class GeoHashGridIT extends OpenSearchIntegTestCase { +public class GeoHashGridIT extends GeoModulePluginIntegTestCase { @Override protected boolean forbidPrivateIndexSettings() { @@ -158,13 +157,13 @@ public void setupSuiteScopeCluster() throws Exception { public void testSimple() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx") - .addAggregation(geohashGrid("geohashgrid").field("location").precision(precision)) + .addAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").precision(precision)) .get(); assertSearchResponse(response); GeoGrid geoGrid = response.getAggregations().get("geohashgrid"); - List buckets = geoGrid.getBuckets(); + List buckets = geoGrid.getBuckets(); Object[] propertiesKeys = (Object[]) ((InternalAggregation) geoGrid).getProperty("_key"); Object[] propertiesDocCounts = (Object[]) ((InternalAggregation) geoGrid).getProperty("_count"); for (int i = 0; i < buckets.size(); i++) { @@ -185,7 +184,7 @@ public void testSimple() throws Exception { public void testMultivalued() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("multi_valued_idx") - .addAggregation(geohashGrid("geohashgrid").field("location").precision(precision)) + .addAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").precision(precision)) .get(); assertSearchResponse(response); @@ -208,8 +207,8 @@ public void testFiltered() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx") .addAggregation( - AggregationBuilders.filter("filtered", bbox) - .subAggregation(geohashGrid("geohashgrid").field("location").precision(precision)) + org.opensearch.search.aggregations.AggregationBuilders.filter("filtered", bbox) + .subAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").precision(precision)) ) .get(); @@ -233,7 +232,7 @@ public void testFiltered() throws Exception { public void testUnmapped() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx_unmapped") - .addAggregation(geohashGrid("geohashgrid").field("location").precision(precision)) + .addAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").precision(precision)) .get(); assertSearchResponse(response); @@ -247,7 +246,7 @@ public void testUnmapped() throws Exception { public void testPartiallyUnmapped() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx", "idx_unmapped") - .addAggregation(geohashGrid("geohashgrid").field("location").precision(precision)) + .addAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").precision(precision)) .get(); assertSearchResponse(response); @@ -267,7 +266,9 @@ public void testPartiallyUnmapped() throws Exception { public void testTopMatch() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx") - .addAggregation(geohashGrid("geohashgrid").field("location").size(1).shardSize(100).precision(precision)) + .addAggregation( + AggregationBuilders.geohashGrid("geohashgrid").field("location").size(1).shardSize(100).precision(precision) + ) .get(); assertSearchResponse(response); @@ -296,7 +297,7 @@ public void testSizeIsZero() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> client().prepareSearch("idx") - .addAggregation(geohashGrid("geohashgrid").field("location").size(size).shardSize(shardSize)) + .addAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").size(size).shardSize(shardSize)) .get() ); assertThat(exception.getMessage(), containsString("[size] must be greater than 0. Found [0] in [geohashgrid]")); @@ -308,7 +309,7 @@ public void testShardSizeIsZero() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> client().prepareSearch("idx") - .addAggregation(geohashGrid("geohashgrid").field("location").size(size).shardSize(shardSize)) + .addAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").size(size).shardSize(shardSize)) .get() ); assertThat(exception.getMessage(), containsString("[shardSize] must be greater than 0. Found [0] in [geohashgrid]")); diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/ShardReduceIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/ShardReduceIT.java new file mode 100644 index 0000000000000..5b4dd052a2f65 --- /dev/null +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/ShardReduceIT.java @@ -0,0 +1,107 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.bucket; + +import org.opensearch.action.index.IndexRequestBuilder; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.geo.GeoModulePluginIntegTestCase; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGrid; +import org.opensearch.geo.tests.common.AggregationBuilders; +import org.opensearch.geometry.utils.Geohash; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.opensearch.search.aggregations.bucket.histogram.Histogram; +import org.opensearch.test.OpenSearchIntegTestCase; + +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; + +/** + * Tests making sure that the reduce is propagated to all aggregations in the hierarchy when executing on a single shard + * These tests are based on the date histogram in combination of min_doc_count=0. In order for the date histogram to + * compute empty buckets, its {@code reduce()} method must be called. So by adding the date histogram under other buckets, + * we can make sure that the reduce is properly propagated by checking that empty buckets were created. + */ +@OpenSearchIntegTestCase.SuiteScopeTestCase +public class ShardReduceIT extends GeoModulePluginIntegTestCase { + + private IndexRequestBuilder indexDoc(String date, int value) throws Exception { + return client().prepareIndex("idx") + .setSource( + jsonBuilder().startObject() + .field("value", value) + .field("ip", "10.0.0." + value) + .field("location", Geohash.stringEncode(5, 52, Geohash.PRECISION)) + .field("date", date) + .field("term-l", 1) + .field("term-d", 1.5) + .field("term-s", "term") + .startObject("nested") + .field("date", date) + .endObject() + .endObject() + ); + } + + @Override + public void setupSuiteScopeCluster() throws Exception { + assertAcked( + prepareCreate("idx").setMapping( + "nested", + "type=nested", + "ip", + "type=ip", + "location", + "type=geo_point", + "term-s", + "type=keyword" + ) + ); + + indexRandom(true, indexDoc("2014-01-01", 1), indexDoc("2014-01-02", 2), indexDoc("2014-01-04", 3)); + ensureSearchable(); + } + + public void testGeoHashGrid() throws Exception { + SearchResponse response = client().prepareSearch("idx") + .setQuery(QueryBuilders.matchAllQuery()) + .addAggregation( + AggregationBuilders.geohashGrid("grid") + .field("location") + .subAggregation(dateHistogram("histo").field("date").fixedInterval(DateHistogramInterval.DAY).minDocCount(0)) + ) + .get(); + + assertSearchResponse(response); + + GeoGrid grid = response.getAggregations().get("grid"); + Histogram histo = grid.getBuckets().iterator().next().getAggregations().get("histo"); + assertThat(histo.getBuckets().size(), equalTo(4)); + } + + public void testGeoTileGrid() throws Exception { + SearchResponse response = client().prepareSearch("idx") + .setQuery(QueryBuilders.matchAllQuery()) + .addAggregation( + AggregationBuilders.geotileGrid("grid") + .field("location") + .subAggregation(dateHistogram("histo").field("date").fixedInterval(DateHistogramInterval.DAY).minDocCount(0)) + ) + .get(); + + assertSearchResponse(response); + + GeoGrid grid = response.getAggregations().get("grid"); + Histogram histo = grid.getBuckets().iterator().next().getAggregations().get("histo"); + assertThat(histo.getBuckets().size(), equalTo(4)); + } +} diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java new file mode 100644 index 0000000000000..92987d407f51d --- /dev/null +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java @@ -0,0 +1,295 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.metrics; + +import com.carrotsearch.hppc.ObjectIntHashMap; +import com.carrotsearch.hppc.ObjectIntMap; +import com.carrotsearch.hppc.ObjectObjectHashMap; +import com.carrotsearch.hppc.ObjectObjectMap; +import org.opensearch.action.index.IndexRequestBuilder; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.Strings; +import org.opensearch.common.document.DocumentField; +import org.opensearch.common.geo.GeoPoint; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.ToXContent; +import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.geo.GeoModulePluginIntegTestCase; +import org.opensearch.geo.tests.common.RandomGeoGenerator; +import org.opensearch.geometry.utils.Geohash; +import org.opensearch.search.SearchHit; +import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; + +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; + +/** + * This is base class for all Geo Aggregations Integration Tests. This class is similar to what we have in the server + * folder of the OpenSearch repo. As part of moving the Geo based aggregation into separate module and plugin we need + * to copy the code as we cannot depend on this class. + * GitHub issue + */ +public abstract class AbstractGeoAggregatorModulePluginTestCase extends GeoModulePluginIntegTestCase { + + protected static final String SINGLE_VALUED_FIELD_NAME = "geo_value"; + protected static final String MULTI_VALUED_FIELD_NAME = "geo_values"; + protected static final String NUMBER_FIELD_NAME = "l_values"; + protected static final String UNMAPPED_IDX_NAME = "idx_unmapped"; + protected static final String IDX_NAME = "idx"; + protected static final String EMPTY_IDX_NAME = "empty_idx"; + protected static final String DATELINE_IDX_NAME = "dateline_idx"; + protected static final String HIGH_CARD_IDX_NAME = "high_card_idx"; + protected static final String IDX_ZERO_NAME = "idx_zero"; + + protected static int numDocs; + protected static int numUniqueGeoPoints; + protected static GeoPoint[] singleValues, multiValues; + protected static GeoPoint singleTopLeft, singleBottomRight, multiTopLeft, multiBottomRight, singleCentroid, multiCentroid, + unmappedCentroid; + protected static ObjectIntMap expectedDocCountsForGeoHash = null; + protected static ObjectObjectMap expectedCentroidsForGeoHash = null; + protected static final double GEOHASH_TOLERANCE = 1E-5D; + + @Override + public void setupSuiteScopeCluster() throws Exception { + createIndex(UNMAPPED_IDX_NAME); + assertAcked( + prepareCreate(IDX_NAME).setMapping( + SINGLE_VALUED_FIELD_NAME, + "type=geo_point", + MULTI_VALUED_FIELD_NAME, + "type=geo_point", + NUMBER_FIELD_NAME, + "type=long", + "tag", + "type=keyword" + ) + ); + + singleTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); + singleBottomRight = new GeoPoint(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY); + multiTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); + multiBottomRight = new GeoPoint(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY); + singleCentroid = new GeoPoint(0, 0); + multiCentroid = new GeoPoint(0, 0); + unmappedCentroid = new GeoPoint(0, 0); + + numDocs = randomIntBetween(6, 20); + numUniqueGeoPoints = randomIntBetween(1, numDocs); + expectedDocCountsForGeoHash = new ObjectIntHashMap<>(numDocs * 2); + expectedCentroidsForGeoHash = new ObjectObjectHashMap<>(numDocs * 2); + + singleValues = new GeoPoint[numUniqueGeoPoints]; + for (int i = 0; i < singleValues.length; i++) { + singleValues[i] = RandomGeoGenerator.randomPoint(random()); + updateBoundsTopLeft(singleValues[i], singleTopLeft); + updateBoundsBottomRight(singleValues[i], singleBottomRight); + } + + multiValues = new GeoPoint[numUniqueGeoPoints]; + for (int i = 0; i < multiValues.length; i++) { + multiValues[i] = RandomGeoGenerator.randomPoint(random()); + updateBoundsTopLeft(multiValues[i], multiTopLeft); + updateBoundsBottomRight(multiValues[i], multiBottomRight); + } + + List builders = new ArrayList<>(); + + GeoPoint singleVal; + final GeoPoint[] multiVal = new GeoPoint[2]; + double newMVLat, newMVLon; + for (int i = 0; i < numDocs; i++) { + singleVal = singleValues[i % numUniqueGeoPoints]; + multiVal[0] = multiValues[i % numUniqueGeoPoints]; + multiVal[1] = multiValues[(i + 1) % numUniqueGeoPoints]; + builders.add( + client().prepareIndex(IDX_NAME) + .setSource( + jsonBuilder().startObject() + .array(SINGLE_VALUED_FIELD_NAME, singleVal.lon(), singleVal.lat()) + .startArray(MULTI_VALUED_FIELD_NAME) + .startArray() + .value(multiVal[0].lon()) + .value(multiVal[0].lat()) + .endArray() + .startArray() + .value(multiVal[1].lon()) + .value(multiVal[1].lat()) + .endArray() + .endArray() + .field(NUMBER_FIELD_NAME, i) + .field("tag", "tag" + i) + .endObject() + ) + ); + singleCentroid = singleCentroid.reset( + singleCentroid.lat() + (singleVal.lat() - singleCentroid.lat()) / (i + 1), + singleCentroid.lon() + (singleVal.lon() - singleCentroid.lon()) / (i + 1) + ); + newMVLat = (multiVal[0].lat() + multiVal[1].lat()) / 2d; + newMVLon = (multiVal[0].lon() + multiVal[1].lon()) / 2d; + multiCentroid = multiCentroid.reset( + multiCentroid.lat() + (newMVLat - multiCentroid.lat()) / (i + 1), + multiCentroid.lon() + (newMVLon - multiCentroid.lon()) / (i + 1) + ); + } + + assertAcked(prepareCreate(EMPTY_IDX_NAME).setMapping(SINGLE_VALUED_FIELD_NAME, "type=geo_point")); + + assertAcked( + prepareCreate(DATELINE_IDX_NAME).setMapping( + SINGLE_VALUED_FIELD_NAME, + "type=geo_point", + MULTI_VALUED_FIELD_NAME, + "type=geo_point", + NUMBER_FIELD_NAME, + "type=long", + "tag", + "type=keyword" + ) + ); + + GeoPoint[] geoValues = new GeoPoint[5]; + geoValues[0] = new GeoPoint(38, 178); + geoValues[1] = new GeoPoint(12, -179); + geoValues[2] = new GeoPoint(-24, 170); + geoValues[3] = new GeoPoint(32, -175); + geoValues[4] = new GeoPoint(-11, 178); + + for (int i = 0; i < 5; i++) { + builders.add( + client().prepareIndex(DATELINE_IDX_NAME) + .setSource( + jsonBuilder().startObject() + .array(SINGLE_VALUED_FIELD_NAME, geoValues[i].lon(), geoValues[i].lat()) + .field(NUMBER_FIELD_NAME, i) + .field("tag", "tag" + i) + .endObject() + ) + ); + } + assertAcked( + prepareCreate(HIGH_CARD_IDX_NAME).setSettings(Settings.builder().put("number_of_shards", 2)) + .setMapping( + SINGLE_VALUED_FIELD_NAME, + "type=geo_point", + MULTI_VALUED_FIELD_NAME, + "type=geo_point", + NUMBER_FIELD_NAME, + "type=long,store=true", + "tag", + "type=keyword" + ) + ); + + for (int i = 0; i < 2000; i++) { + singleVal = singleValues[i % numUniqueGeoPoints]; + builders.add( + client().prepareIndex(HIGH_CARD_IDX_NAME) + .setSource( + jsonBuilder().startObject() + .array(SINGLE_VALUED_FIELD_NAME, singleVal.lon(), singleVal.lat()) + .startArray(MULTI_VALUED_FIELD_NAME) + .startArray() + .value(multiValues[i % numUniqueGeoPoints].lon()) + .value(multiValues[i % numUniqueGeoPoints].lat()) + .endArray() + .startArray() + .value(multiValues[(i + 1) % numUniqueGeoPoints].lon()) + .value(multiValues[(i + 1) % numUniqueGeoPoints].lat()) + .endArray() + .endArray() + .field(NUMBER_FIELD_NAME, i) + .field("tag", "tag" + i) + .endObject() + ) + ); + updateGeohashBucketsCentroid(singleVal); + } + + builders.add( + client().prepareIndex(IDX_ZERO_NAME) + .setSource(jsonBuilder().startObject().array(SINGLE_VALUED_FIELD_NAME, 0.0, 1.0).endObject()) + ); + assertAcked(prepareCreate(IDX_ZERO_NAME).setMapping(SINGLE_VALUED_FIELD_NAME, "type=geo_point")); + + indexRandom(true, builders); + ensureSearchable(); + + // Added to debug a test failure where the terms aggregation seems to be reporting two documents with the same + // value for NUMBER_FIELD_NAME. This will check that after random indexing each document only has 1 value for + // NUMBER_FIELD_NAME and it is the correct value. Following this initial change its seems that this call was getting + // more that 2000 hits (actual value was 2059) so now it will also check to ensure all hits have the correct index and type. + SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME) + .addStoredField(NUMBER_FIELD_NAME) + .addSort(SortBuilders.fieldSort(NUMBER_FIELD_NAME).order(SortOrder.ASC)) + .setSize(5000) + .get(); + assertSearchResponse(response); + long totalHits = response.getHits().getTotalHits().value; + XContentBuilder builder = XContentFactory.jsonBuilder(); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + logger.info("Full high_card_idx Response Content:\n{ {} }", Strings.toString(builder)); + for (int i = 0; i < totalHits; i++) { + SearchHit searchHit = response.getHits().getAt(i); + assertThat("Hit " + i + " with id: " + searchHit.getId(), searchHit.getIndex(), equalTo("high_card_idx")); + DocumentField hitField = searchHit.field(NUMBER_FIELD_NAME); + + assertThat("Hit " + i + " has wrong number of values", hitField.getValues().size(), equalTo(1)); + Long value = hitField.getValue(); + assertThat("Hit " + i + " has wrong value", value.intValue(), equalTo(i)); + } + assertThat(totalHits, equalTo(2000L)); + } + + private void updateGeohashBucketsCentroid(final GeoPoint location) { + String hash = Geohash.stringEncode(location.lon(), location.lat(), Geohash.PRECISION); + for (int precision = Geohash.PRECISION; precision > 0; --precision) { + final String h = hash.substring(0, precision); + expectedDocCountsForGeoHash.put(h, expectedDocCountsForGeoHash.getOrDefault(h, 0) + 1); + expectedCentroidsForGeoHash.put(h, updateHashCentroid(h, location)); + } + } + + private GeoPoint updateHashCentroid(String hash, final GeoPoint location) { + GeoPoint centroid = expectedCentroidsForGeoHash.getOrDefault(hash, null); + if (centroid == null) { + return new GeoPoint(location.lat(), location.lon()); + } + final int docCount = expectedDocCountsForGeoHash.get(hash); + final double newLon = centroid.lon() + (location.lon() - centroid.lon()) / docCount; + final double newLat = centroid.lat() + (location.lat() - centroid.lat()) / docCount; + return centroid.reset(newLat, newLon); + } + + private void updateBoundsBottomRight(GeoPoint geoPoint, GeoPoint currentBound) { + if (geoPoint.lat() < currentBound.lat()) { + currentBound.resetLat(geoPoint.lat()); + } + if (geoPoint.lon() > currentBound.lon()) { + currentBound.resetLon(geoPoint.lon()); + } + } + + private void updateBoundsTopLeft(GeoPoint geoPoint, GeoPoint currentBound) { + if (geoPoint.lat() > currentBound.lat()) { + currentBound.resetLat(geoPoint.lat()); + } + if (geoPoint.lon() < currentBound.lon()) { + currentBound.resetLon(geoPoint.lon()); + } + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoBoundsIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsITTestCase.java similarity index 97% rename from server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoBoundsIT.java rename to modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsITTestCase.java index 3af3b9e5212f8..8cc82da12d69a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoBoundsIT.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsITTestCase.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.geo.GeoPoint; @@ -43,21 +43,21 @@ import java.util.List; -import static org.opensearch.index.query.QueryBuilders.matchAllQuery; -import static org.opensearch.search.aggregations.AggregationBuilders.geoBounds; -import static org.opensearch.search.aggregations.AggregationBuilders.global; -import static org.opensearch.search.aggregations.AggregationBuilders.terms; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.closeTo; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; -import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.sameInstance; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.closeTo; +import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.aggregations.AggregationBuilders.global; +import static org.opensearch.search.aggregations.AggregationBuilders.terms; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; +import static org.opensearch.geo.tests.common.AggregationBuilders.geoBounds; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class GeoBoundsIT extends AbstractGeoTestCase { +public class GeoBoundsITTestCase extends AbstractGeoAggregatorModulePluginTestCase { private static final String aggName = "geoBounds"; public void testSingleValuedField() throws Exception { @@ -226,7 +226,8 @@ public void testSingleValuedFieldNearDateLineWrapLongitude() throws Exception { } /** - * This test forces the {@link GeoBoundsAggregator} to resize the {@link BigArray}s it uses to ensure they are resized correctly + * This test forces the {@link GeoBoundsAggregator} to resize the {@link BigArray}s it uses to ensure they are + * resized correctly */ public void testSingleValuedFieldAsSubAggToHighCardTermsAgg() { SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME) diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoCentroidITTestCase.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoCentroidITTestCase.java new file mode 100644 index 0000000000000..e6d45e27b8f70 --- /dev/null +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoCentroidITTestCase.java @@ -0,0 +1,84 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.geo.search.aggregations.metrics; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.geo.GeoPoint; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGrid; +import org.opensearch.geo.tests.common.AggregationBuilders; +import org.opensearch.search.aggregations.metrics.GeoCentroid; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.List; + +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.opensearch.search.aggregations.AggregationBuilders.geoCentroid; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; + +@OpenSearchIntegTestCase.SuiteScopeTestCase +public class GeoCentroidITTestCase extends AbstractGeoAggregatorModulePluginTestCase { + private static final String aggName = "geoCentroid"; + + public void testSingleValueFieldAsSubAggToGeohashGrid() throws Exception { + SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME) + .addAggregation( + AggregationBuilders.geohashGrid("geoGrid") + .field(SINGLE_VALUED_FIELD_NAME) + .subAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME)) + ) + .get(); + assertSearchResponse(response); + + GeoGrid grid = response.getAggregations().get("geoGrid"); + assertThat(grid, notNullValue()); + assertThat(grid.getName(), equalTo("geoGrid")); + List buckets = grid.getBuckets(); + for (GeoGrid.Bucket cell : buckets) { + String geohash = cell.getKeyAsString(); + GeoPoint expectedCentroid = expectedCentroidsForGeoHash.get(geohash); + GeoCentroid centroidAgg = cell.getAggregations().get(aggName); + assertThat( + "Geohash " + geohash + " has wrong centroid latitude ", + expectedCentroid.lat(), + closeTo(centroidAgg.centroid().lat(), GEOHASH_TOLERANCE) + ); + assertThat( + "Geohash " + geohash + " has wrong centroid longitude", + expectedCentroid.lon(), + closeTo(centroidAgg.centroid().lon(), GEOHASH_TOLERANCE) + ); + } + } +} diff --git a/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java b/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java index ed8c7c06fb3e8..25dcf8db2c407 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java +++ b/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java @@ -32,18 +32,74 @@ package org.opensearch.geo; +import org.opensearch.geo.search.aggregations.bucket.composite.GeoTileGridValuesSourceBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregator; +import org.opensearch.geo.search.aggregations.bucket.geogrid.InternalGeoHashGrid; +import org.opensearch.geo.search.aggregations.bucket.geogrid.InternalGeoTileGrid; +import org.opensearch.geo.search.aggregations.metrics.GeoBounds; +import org.opensearch.geo.search.aggregations.metrics.GeoBoundsAggregationBuilder; +import org.opensearch.geo.search.aggregations.metrics.InternalGeoBounds; import org.opensearch.index.mapper.GeoShapeFieldMapper; import org.opensearch.index.mapper.Mapper; import org.opensearch.plugins.MapperPlugin; import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.SearchPlugin; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregation; import java.util.Collections; +import java.util.List; import java.util.Map; -public class GeoModulePlugin extends Plugin implements MapperPlugin { +public class GeoModulePlugin extends Plugin implements MapperPlugin, SearchPlugin { @Override public Map getMappers() { return Collections.singletonMap(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser()); } + + /** + * Registering {@link GeoBounds} aggregation on GeoPoint field. + */ + @Override + public List getAggregations() { + final AggregationSpec geoBounds = new AggregationSpec( + GeoBoundsAggregationBuilder.NAME, + GeoBoundsAggregationBuilder::new, + GeoBoundsAggregationBuilder.PARSER + ).addResultReader(InternalGeoBounds::new).setAggregatorRegistrar(GeoBoundsAggregationBuilder::registerAggregators); + + final AggregationSpec geoHashGrid = new AggregationSpec( + GeoHashGridAggregationBuilder.NAME, + GeoHashGridAggregationBuilder::new, + GeoHashGridAggregationBuilder.PARSER + ).addResultReader(InternalGeoHashGrid::new).setAggregatorRegistrar(GeoHashGridAggregationBuilder::registerAggregators); + + final AggregationSpec geoTileGrid = new AggregationSpec( + GeoTileGridAggregationBuilder.NAME, + GeoTileGridAggregationBuilder::new, + GeoTileGridAggregationBuilder.PARSER + ).addResultReader(InternalGeoTileGrid::new).setAggregatorRegistrar(GeoTileGridAggregationBuilder::registerAggregators); + return List.of(geoBounds, geoHashGrid, geoTileGrid); + } + + /** + * Registering the {@link GeoTileGridAggregator} in the {@link CompositeAggregation}. + * + * @return a {@link List} of {@link CompositeAggregationSpec} + */ + @Override + public List getCompositeAggregations() { + return Collections.singletonList( + new CompositeAggregationSpec( + GeoTileGridValuesSourceBuilder::register, + GeoTileGridValuesSourceBuilder.class, + GeoTileGridValuesSourceBuilder.COMPOSITE_AGGREGATION_SERIALISATION_BYTE_CODE, + GeoTileGridValuesSourceBuilder::new, + GeoTileGridValuesSourceBuilder::parse, + GeoTileGridValuesSourceBuilder.TYPE + ) + ); + } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java similarity index 87% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java index 4b01a08d29a43..84d5943da287f 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.composite; +package org.opensearch.geo.search.aggregations.bucket.composite; import org.apache.lucene.index.IndexReader; import org.opensearch.LegacyESVersion; @@ -43,12 +43,15 @@ import org.opensearch.common.xcontent.ObjectParser; import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.geo.search.aggregations.bucket.geogrid.CellIdSource; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.DocValueFormat; -import org.opensearch.search.aggregations.bucket.geogrid.CellIdSource; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceConfig; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceParserHelper; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.search.aggregations.bucket.missing.MissingOrder; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.aggregations.support.ValuesSource; @@ -88,13 +91,19 @@ CompositeValuesSourceConfig apply( ); } - static final String TYPE = "geotile_grid"; + public static final String TYPE = "geotile_grid"; + /* + use the TYPE parameter instead of Byte code. The byte code is added for backward compatibility and will be + removed in the next version. + */ + @Deprecated + public static final Byte COMPOSITE_AGGREGATION_SERIALISATION_BYTE_CODE = 3; static final ValuesSourceRegistry.RegistryKey REGISTRY_KEY = new ValuesSourceRegistry.RegistryKey( TYPE, GeoTileCompositeSuppier.class ); - private static final ObjectParser PARSER; + static final ObjectParser PARSER; static { PARSER = new ObjectParser<>(GeoTileGridValuesSourceBuilder.TYPE); PARSER.declareInt(GeoTileGridValuesSourceBuilder::precision, new ParseField("precision")); @@ -106,11 +115,11 @@ CompositeValuesSourceConfig apply( CompositeValuesSourceParserHelper.declareValuesSourceFields(PARSER); } - static GeoTileGridValuesSourceBuilder parse(String name, XContentParser parser) throws IOException { + public static GeoTileGridValuesSourceBuilder parse(String name, XContentParser parser) throws IOException { return PARSER.parse(parser, new GeoTileGridValuesSourceBuilder(name), null); } - static void register(ValuesSourceRegistry.Builder builder) { + public static void register(ValuesSourceRegistry.Builder builder) { builder.register( REGISTRY_KEY, @@ -163,7 +172,7 @@ static void register(ValuesSourceRegistry.Builder builder) { super(name); } - GeoTileGridValuesSourceBuilder(StreamInput in) throws IOException { + public GeoTileGridValuesSourceBuilder(StreamInput in) throws IOException { super(in); this.precision = in.readInt(); if (in.getVersion().onOrAfter(LegacyESVersion.V_7_6_0)) { @@ -203,7 +212,7 @@ protected void doXContentBody(XContentBuilder builder, Params params) throws IOE } @Override - String type() { + protected String type() { return TYPE; } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/GeoTileValuesSource.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileValuesSource.java similarity index 88% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/composite/GeoTileValuesSource.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileValuesSource.java index 819dfc573bbe4..303e577e99e7b 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/GeoTileValuesSource.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileValuesSource.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.composite; +package org.opensearch.geo.search.aggregations.bucket.composite; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; @@ -38,7 +38,9 @@ import org.opensearch.common.util.BigArrays; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.search.DocValueFormat; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.opensearch.search.aggregations.bucket.composite.LongValuesSource; +import org.opensearch.search.aggregations.bucket.composite.SingleDimensionValuesSource; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.search.aggregations.bucket.missing.MissingOrder; import java.io.IOException; @@ -68,7 +70,7 @@ class GeoTileValuesSource extends LongValuesSource { } @Override - void setAfter(Comparable value) { + protected void setAfter(Comparable value) { if (missingBucket && value == null) { afterValue = null; } else if (value instanceof Number) { diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/BoundedCellValues.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BoundedCellValues.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/BoundedCellValues.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BoundedCellValues.java index ba824fc8f21dd..06d2dcaee3932 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/BoundedCellValues.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BoundedCellValues.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.index.fielddata.MultiGeoPointValues; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/BucketPriorityQueue.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BucketPriorityQueue.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/BucketPriorityQueue.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BucketPriorityQueue.java index d6cfde0c46eae..70d0552b3e80b 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/BucketPriorityQueue.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BucketPriorityQueue.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.apache.lucene.util.PriorityQueue; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/CellIdSource.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellIdSource.java similarity index 98% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/CellIdSource.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellIdSource.java index 12d9043a2fd5f..d40029e9a762d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/CellIdSource.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellIdSource.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/CellValues.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellValues.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/CellValues.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellValues.java index 9dc357659aae8..d01896c8136fa 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/CellValues.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellValues.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.index.fielddata.AbstractSortingNumericDocValues; import org.opensearch.index.fielddata.MultiGeoPointValues; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGrid.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGrid.java index cfdb08f9ee3d7..4ae888640efc8 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGrid.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java similarity index 99% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java index b08c40268c5cf..4a904b3aa2b16 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.LegacyESVersion; import org.opensearch.OpenSearchException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregator.java similarity index 99% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregator.java index 1ef8ba6c697f4..909772c61a960 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregator.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java index 4049bf2c73640..bbaf9613fb216 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.common.geo.GeoUtils; @@ -40,7 +40,7 @@ import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorFactory; -import org.opensearch.search.aggregations.metrics.GeoGridAggregatorSupplier; +import org.opensearch.geo.search.aggregations.metrics.GeoGridAggregatorSupplier; import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java index 1106320c7431f..6ca7a4d8a9cb8 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java similarity index 98% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java index cdc801aaedffb..1914c07e831f7 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.geometry.utils.Geohash; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java similarity index 95% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java index f73360e3cb826..76ad515f34fe5 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.common.io.stream.StreamInput; @@ -39,7 +39,8 @@ import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorFactory; -import org.opensearch.search.aggregations.metrics.GeoGridAggregatorSupplier; +import org.opensearch.geo.search.aggregations.metrics.GeoGridAggregatorSupplier; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java index 7a2b908148c4c..a205a9afde41e 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java index ef8cd11a22498..b830988a3d410 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.index.query.QueryShardContext; @@ -40,6 +40,7 @@ import org.opensearch.search.aggregations.CardinalityUpperBound; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.NonCollectingAggregator; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.aggregations.support.ValuesSource; import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoGrid.java similarity index 99% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoGrid.java index 94a5ad5717854..9dbed7b27307a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoGrid.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java similarity index 98% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java index a187bfefb661f..93fcdbd098400 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java index 7811b8774d04f..ff1247300939a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java index f9c45dc41ceb1..659909e868651 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java index efbd9a05d6a4d..fa544b5893f0c 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java similarity index 94% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java index f200f55232e00..65d736cfceb32 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java @@ -30,11 +30,12 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoGrid.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoGrid.java index 3f85cf350c89c..adfffeddba59d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoGrid.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.CheckedFunction; import org.opensearch.common.xcontent.ObjectParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java index 08e5c15188ee6..80124cda50b19 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.search.aggregations.Aggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java index f20f972c1ce0a..109524e755c4d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.xcontent.ObjectParser; import org.opensearch.common.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java index 05c7a1c8d1663..4e6e454b08324 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java index 06915cc4210e1..8734c96a15578 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.xcontent.ObjectParser; import org.opensearch.common.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java similarity index 93% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java index c8dec16f322ef..fd47c35f13de1 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java @@ -30,10 +30,11 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/UnboundedCellValues.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/UnboundedCellValues.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/UnboundedCellValues.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/UnboundedCellValues.java index f5a139cdb8d9d..c628c7bfdc8ec 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/UnboundedCellValues.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/UnboundedCellValues.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.index.fielddata.MultiGeoPointValues; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/package-info.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/package-info.java similarity index 79% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/package-info.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/package-info.java index c59685e06cf79..d9183a0f742ef 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/package-info.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/package-info.java @@ -7,4 +7,4 @@ */ /** geo_grid Aggregation package. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoBoundsAggregator.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoBoundsAggregator.java new file mode 100644 index 0000000000000..4a39fa1da04eb --- /dev/null +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoBoundsAggregator.java @@ -0,0 +1,128 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.metrics; + +import org.opensearch.common.lease.Releasables; +import org.opensearch.common.util.BigArrays; +import org.opensearch.common.util.DoubleArray; +import org.opensearch.search.aggregations.Aggregator; +import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.metrics.MetricsAggregator; +import org.opensearch.search.aggregations.support.ValuesSource; +import org.opensearch.search.aggregations.support.ValuesSourceConfig; +import org.opensearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Map; + +/** + * Abstract class for doing the {@link GeoBounds} Aggregation over fields of type geo_shape and geo_point. + * + * @param Class extending the {@link ValuesSource} which will provide the data on which aggregation will happen. + * @opensearch.internal + */ +public abstract class AbstractGeoBoundsAggregator extends MetricsAggregator { + + protected final T valuesSource; + protected final boolean wrapLongitude; + protected DoubleArray tops; + protected DoubleArray bottoms; + protected DoubleArray posLefts; + protected DoubleArray posRights; + protected DoubleArray negLefts; + protected DoubleArray negRights; + + @SuppressWarnings("unchecked") + protected AbstractGeoBoundsAggregator( + String name, + SearchContext searchContext, + Aggregator aggregator, + ValuesSourceConfig valuesSourceConfig, + boolean wrapLongitude, + Map metaData + ) throws IOException { + super(name, searchContext, aggregator, metaData); + this.wrapLongitude = wrapLongitude; + valuesSource = valuesSourceConfig.hasValues() ? (T) valuesSourceConfig.getValuesSource() : null; + + if (valuesSource != null) { + final BigArrays bigArrays = context.bigArrays(); + tops = bigArrays.newDoubleArray(1, false); + tops.fill(0, tops.size(), Double.NEGATIVE_INFINITY); + bottoms = bigArrays.newDoubleArray(1, false); + bottoms.fill(0, bottoms.size(), Double.POSITIVE_INFINITY); + posLefts = bigArrays.newDoubleArray(1, false); + posLefts.fill(0, posLefts.size(), Double.POSITIVE_INFINITY); + posRights = bigArrays.newDoubleArray(1, false); + posRights.fill(0, posRights.size(), Double.NEGATIVE_INFINITY); + negLefts = bigArrays.newDoubleArray(1, false); + negLefts.fill(0, negLefts.size(), Double.POSITIVE_INFINITY); + negRights = bigArrays.newDoubleArray(1, false); + negRights.fill(0, negRights.size(), Double.NEGATIVE_INFINITY); + } + } + + /** + * Build an empty aggregation. + */ + @Override + public InternalAggregation buildEmptyAggregation() { + return new InternalGeoBounds( + name, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY, + Double.POSITIVE_INFINITY, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY, + Double.NEGATIVE_INFINITY, + wrapLongitude, + metadata() + ); + } + + /** + * Build an aggregation for data that has been collected into owningBucketOrd. + */ + @Override + public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOException { + if (valuesSource == null) { + return buildEmptyAggregation(); + } + double top = tops.get(owningBucketOrdinal); + double bottom = bottoms.get(owningBucketOrdinal); + double posLeft = posLefts.get(owningBucketOrdinal); + double posRight = posRights.get(owningBucketOrdinal); + double negLeft = negLefts.get(owningBucketOrdinal); + double negRight = negRights.get(owningBucketOrdinal); + return new InternalGeoBounds(name, top, bottom, posLeft, posRight, negLeft, negRight, wrapLongitude, metadata()); + } + + @Override + public void doClose() { + Releasables.close(tops, bottoms, posLefts, posRights, negLefts, negRights); + } + + protected void setBucketSize(final long bucket, final BigArrays bigArrays) { + if (bucket >= tops.size()) { + long from = tops.size(); + tops = bigArrays.grow(tops, bucket + 1); + tops.fill(from, tops.size(), Double.NEGATIVE_INFINITY); + bottoms = bigArrays.resize(bottoms, tops.size()); + bottoms.fill(from, bottoms.size(), Double.POSITIVE_INFINITY); + posLefts = bigArrays.resize(posLefts, tops.size()); + posLefts.fill(from, posLefts.size(), Double.POSITIVE_INFINITY); + posRights = bigArrays.resize(posRights, tops.size()); + posRights.fill(from, posRights.size(), Double.NEGATIVE_INFINITY); + negLefts = bigArrays.resize(negLefts, tops.size()); + negLefts.fill(from, negLefts.size(), Double.POSITIVE_INFINITY); + negRights = bigArrays.resize(negRights, tops.size()); + negRights.fill(from, negRights.size(), Double.NEGATIVE_INFINITY); + } + } +} diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBounds.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBounds.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBounds.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBounds.java index 380fbce85ada7..81ef502dda130 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBounds.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBounds.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; import org.opensearch.common.geo.GeoPoint; import org.opensearch.search.aggregations.Aggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregationBuilder.java similarity index 93% rename from server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregationBuilder.java index 64e27fa7e13d1..b2c441f9a951c 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregationBuilder.java @@ -30,8 +30,9 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; +import org.opensearch.common.ParseField; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.ObjectParser; @@ -40,6 +41,7 @@ import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorFactory; +import org.opensearch.search.aggregations.metrics.GeoBoundsAggregatorSupplier; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.opensearch.search.aggregations.support.ValuesSourceConfig; @@ -57,6 +59,7 @@ */ public class GeoBoundsAggregationBuilder extends ValuesSourceAggregationBuilder { public static final String NAME = "geo_bounds"; + private static final ParseField WRAP_LONGITUDE_FIELD = new ParseField("wrap_longitude"); public static final ValuesSourceRegistry.RegistryKey REGISTRY_KEY = new ValuesSourceRegistry.RegistryKey<>( NAME, GeoBoundsAggregatorSupplier.class @@ -68,7 +71,7 @@ public class GeoBoundsAggregationBuilder extends ValuesSourceAggregationBuilder< ); static { ValuesSourceAggregationBuilder.declareFields(PARSER, false, false, false); - PARSER.declareBoolean(GeoBoundsAggregationBuilder::wrapLongitude, GeoBoundsAggregator.WRAP_LONGITUDE_FIELD); + PARSER.declareBoolean(GeoBoundsAggregationBuilder::wrapLongitude, WRAP_LONGITUDE_FIELD); } public static void registerAggregators(ValuesSourceRegistry.Builder builder) { @@ -121,13 +124,6 @@ public GeoBoundsAggregationBuilder wrapLongitude(boolean wrapLongitude) { return this; } - /** - * Get whether to wrap longitudes. - */ - public boolean wrapLongitude() { - return wrapLongitude; - } - @Override public BucketCardinality bucketCardinality() { return BucketCardinality.NONE; @@ -145,7 +141,7 @@ protected GeoBoundsAggregatorFactory innerBuild( @Override public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { - builder.field(GeoBoundsAggregator.WRAP_LONGITUDE_FIELD.getPreferredName(), wrapLongitude); + builder.field(WRAP_LONGITUDE_FIELD.getPreferredName(), wrapLongitude); return builder; } diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregator.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregator.java similarity index 51% rename from server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregator.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregator.java index 054e8d4cb1c6c..a6518ea702be6 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregator.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregator.java @@ -30,17 +30,13 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; import org.apache.lucene.index.LeafReaderContext; -import org.opensearch.common.ParseField; import org.opensearch.common.geo.GeoPoint; -import org.opensearch.common.lease.Releasables; import org.opensearch.common.util.BigArrays; -import org.opensearch.common.util.DoubleArray; import org.opensearch.index.fielddata.MultiGeoPointValues; import org.opensearch.search.aggregations.Aggregator; -import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.LeafBucketCollector; import org.opensearch.search.aggregations.LeafBucketCollectorBase; import org.opensearch.search.aggregations.support.ValuesSource; @@ -51,22 +47,11 @@ import java.util.Map; /** - * Aggregate all docs into a geographic bounds + * Aggregate all docs into a geographic bounds for field GeoPoint. * * @opensearch.internal */ -final class GeoBoundsAggregator extends MetricsAggregator { - - static final ParseField WRAP_LONGITUDE_FIELD = new ParseField("wrap_longitude"); - - private final ValuesSource.GeoPoint valuesSource; - private final boolean wrapLongitude; - DoubleArray tops; - DoubleArray bottoms; - DoubleArray posLefts; - DoubleArray posRights; - DoubleArray negLefts; - DoubleArray negRights; +final class GeoBoundsAggregator extends AbstractGeoBoundsAggregator { GeoBoundsAggregator( String name, @@ -76,25 +61,7 @@ final class GeoBoundsAggregator extends MetricsAggregator { boolean wrapLongitude, Map metadata ) throws IOException { - super(name, aggregationContext, parent, metadata); - // TODO: stop expecting nulls here - this.valuesSource = valuesSourceConfig.hasValues() ? (ValuesSource.GeoPoint) valuesSourceConfig.getValuesSource() : null; - this.wrapLongitude = wrapLongitude; - if (valuesSource != null) { - final BigArrays bigArrays = context.bigArrays(); - tops = bigArrays.newDoubleArray(1, false); - tops.fill(0, tops.size(), Double.NEGATIVE_INFINITY); - bottoms = bigArrays.newDoubleArray(1, false); - bottoms.fill(0, bottoms.size(), Double.POSITIVE_INFINITY); - posLefts = bigArrays.newDoubleArray(1, false); - posLefts.fill(0, posLefts.size(), Double.POSITIVE_INFINITY); - posRights = bigArrays.newDoubleArray(1, false); - posRights.fill(0, posRights.size(), Double.NEGATIVE_INFINITY); - negLefts = bigArrays.newDoubleArray(1, false); - negLefts.fill(0, negLefts.size(), Double.POSITIVE_INFINITY); - negRights = bigArrays.newDoubleArray(1, false); - negRights.fill(0, negRights.size(), Double.NEGATIVE_INFINITY); - } + super(name, aggregationContext, parent, valuesSourceConfig, wrapLongitude, metadata); } @Override @@ -107,25 +74,10 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCol return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { - if (bucket >= tops.size()) { - long from = tops.size(); - tops = bigArrays.grow(tops, bucket + 1); - tops.fill(from, tops.size(), Double.NEGATIVE_INFINITY); - bottoms = bigArrays.resize(bottoms, tops.size()); - bottoms.fill(from, bottoms.size(), Double.POSITIVE_INFINITY); - posLefts = bigArrays.resize(posLefts, tops.size()); - posLefts.fill(from, posLefts.size(), Double.POSITIVE_INFINITY); - posRights = bigArrays.resize(posRights, tops.size()); - posRights.fill(from, posRights.size(), Double.NEGATIVE_INFINITY); - negLefts = bigArrays.resize(negLefts, tops.size()); - negLefts.fill(from, negLefts.size(), Double.POSITIVE_INFINITY); - negRights = bigArrays.resize(negRights, tops.size()); - negRights.fill(from, negRights.size(), Double.NEGATIVE_INFINITY); - } + setBucketSize(bucket, bigArrays); if (values.advanceExact(doc)) { final int valuesCount = values.docValueCount(); - for (int i = 0; i < valuesCount; ++i) { GeoPoint value = values.nextValue(); double top = tops.get(bucket); @@ -163,38 +115,4 @@ public void collect(int doc, long bucket) throws IOException { } }; } - - @Override - public InternalAggregation buildAggregation(long owningBucketOrdinal) { - if (valuesSource == null) { - return buildEmptyAggregation(); - } - double top = tops.get(owningBucketOrdinal); - double bottom = bottoms.get(owningBucketOrdinal); - double posLeft = posLefts.get(owningBucketOrdinal); - double posRight = posRights.get(owningBucketOrdinal); - double negLeft = negLefts.get(owningBucketOrdinal); - double negRight = negRights.get(owningBucketOrdinal); - return new InternalGeoBounds(name, top, bottom, posLeft, posRight, negLeft, negRight, wrapLongitude, metadata()); - } - - @Override - public InternalAggregation buildEmptyAggregation() { - return new InternalGeoBounds( - name, - Double.NEGATIVE_INFINITY, - Double.POSITIVE_INFINITY, - Double.POSITIVE_INFINITY, - Double.NEGATIVE_INFINITY, - Double.POSITIVE_INFINITY, - Double.NEGATIVE_INFINITY, - wrapLongitude, - metadata() - ); - } - - @Override - public void doClose() { - Releasables.close(tops, bottoms, posLefts, posRights, negLefts, negRights); - } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregatorFactory.java similarity index 98% rename from server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregatorFactory.java index 2c6b75842b6f5..149e052b4db7d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregatorFactory.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.aggregations.Aggregator; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoGridAggregatorSupplier.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoGridAggregatorSupplier.java similarity index 93% rename from server/src/main/java/org/opensearch/search/aggregations/metrics/GeoGridAggregatorSupplier.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoGridAggregatorSupplier.java index 183c64f4e4af2..43ccb8b89545a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoGridAggregatorSupplier.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoGridAggregatorSupplier.java @@ -30,13 +30,13 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; import org.opensearch.common.geo.GeoBoundingBox; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGridAggregator; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.CardinalityUpperBound; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGridAggregator; import org.opensearch.search.aggregations.support.ValuesSource; import org.opensearch.search.internal.SearchContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalGeoBounds.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/InternalGeoBounds.java similarity index 99% rename from server/src/main/java/org/opensearch/search/aggregations/metrics/InternalGeoBounds.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/InternalGeoBounds.java index 87018242ee8df..7c708de88a49c 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalGeoBounds.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/InternalGeoBounds.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.common.geo.GeoPoint; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/ParsedGeoBounds.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/ParsedGeoBounds.java similarity index 98% rename from server/src/main/java/org/opensearch/search/aggregations/metrics/ParsedGeoBounds.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/ParsedGeoBounds.java index a482fcfdf08dd..7643ac9d9a010 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/ParsedGeoBounds.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/ParsedGeoBounds.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; import org.opensearch.common.Nullable; import org.opensearch.common.collect.Tuple; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/GeoHashGridTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridAggregationBuilderTests.java similarity index 72% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/GeoHashGridTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridAggregationBuilderTests.java index 5e230a445ec98..00cb162e64c19 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/GeoHashGridTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridAggregationBuilderTests.java @@ -30,14 +30,23 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket; +package org.opensearch.geo.search.aggregations.bucket; -import org.opensearch.common.geo.GeoBoundingBoxTests; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; +import org.opensearch.geo.tests.common.RandomGeoGenerator; +import org.opensearch.plugins.Plugin; import org.opensearch.search.aggregations.BaseAggregationTestCase; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; -public class GeoHashGridTests extends BaseAggregationTestCase { +import java.util.Collection; +import java.util.Collections; + +public class GeoHashGridAggregationBuilderTests extends BaseAggregationTestCase { + + protected Collection> getPlugins() { + return Collections.singletonList(GeoModulePlugin.class); + } @Override protected GeoHashGridAggregationBuilder createTestAggregatorBuilder() { @@ -55,7 +64,7 @@ protected GeoHashGridAggregationBuilder createTestAggregatorBuilder() { factory.shardSize(randomIntBetween(1, Integer.MAX_VALUE)); } if (randomBoolean()) { - factory.setGeoBoundingBox(GeoBoundingBoxTests.randomBBox()); + factory.setGeoBoundingBox(RandomGeoGenerator.randomBBox()); } return factory; } diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/GeoTileGridTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoTileGridAggregationBuilderTests.java similarity index 70% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/GeoTileGridTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoTileGridAggregationBuilderTests.java index d54667fb4f1a6..c7c0be21273bd 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/GeoTileGridTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoTileGridAggregationBuilderTests.java @@ -30,15 +30,24 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket; +package org.opensearch.geo.search.aggregations.bucket; -import org.opensearch.common.geo.GeoBoundingBoxTests; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.opensearch.geo.tests.common.RandomGeoGenerator; +import org.opensearch.plugins.Plugin; import org.opensearch.search.aggregations.BaseAggregationTestCase; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; -public class GeoTileGridTests extends BaseAggregationTestCase { +import java.util.Collection; +import java.util.Collections; + +public class GeoTileGridAggregationBuilderTests extends BaseAggregationTestCase { + + protected Collection> getPlugins() { + return Collections.singletonList(GeoModulePlugin.class); + } @Override protected GeoTileGridAggregationBuilder createTestAggregatorBuilder() { @@ -55,7 +64,7 @@ protected GeoTileGridAggregationBuilder createTestAggregatorBuilder() { factory.shardSize(randomIntBetween(1, Integer.MAX_VALUE)); } if (randomBoolean()) { - factory.setGeoBoundingBox(GeoBoundingBoxTests.randomBBox()); + factory.setGeoBoundingBox(RandomGeoGenerator.randomBBox()); } return factory; } diff --git a/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridAggregationCompositeAggregatorTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridAggregationCompositeAggregatorTests.java new file mode 100644 index 0000000000000..3c7c292f9d193 --- /dev/null +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridAggregationCompositeAggregatorTests.java @@ -0,0 +1,174 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.bucket.composite; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.LatLonPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.junit.Before; +import org.opensearch.common.geo.GeoPoint; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregator; +import org.opensearch.index.mapper.GeoPointFieldMapper; +import org.opensearch.plugins.SearchPlugin; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; +import org.opensearch.search.aggregations.composite.BaseCompositeAggregatorTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Testing the {@link GeoTileGridAggregator} as part of CompositeAggregation. + */ +public class GeoTileGridAggregationCompositeAggregatorTests extends BaseCompositeAggregatorTestCase { + + protected List getSearchPlugins() { + return Collections.singletonList(new GeoModulePlugin()); + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + FIELD_TYPES.add(new GeoPointFieldMapper.GeoPointFieldType("geo_point")); + } + + public void testUnmappedFieldWithGeopoint() throws Exception { + final List>> dataset = new ArrayList<>(); + final String mappedFieldName = "geo_point"; + dataset.addAll( + Arrays.asList( + createDocument(mappedFieldName, new GeoPoint(48.934059, 41.610741)), + createDocument(mappedFieldName, new GeoPoint(-23.065941, 113.610741)), + createDocument(mappedFieldName, new GeoPoint(90.0, 0.0)), + createDocument(mappedFieldName, new GeoPoint(37.2343, -115.8067)), + createDocument(mappedFieldName, new GeoPoint(90.0, 0.0)) + ) + ); + + // just unmapped = no results + testSearchCase( + Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), + dataset, + () -> new CompositeAggregationBuilder("name", Arrays.asList(new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped"))), + (result) -> assertEquals(0, result.getBuckets().size()) + ); + + // unmapped missing bucket = one result + testSearchCase( + Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), + dataset, + () -> new CompositeAggregationBuilder( + "name", + Arrays.asList(new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped").missingBucket(true)) + ), + (result) -> { + assertEquals(1, result.getBuckets().size()); + assertEquals("{unmapped=null}", result.afterKey().toString()); + assertEquals("{unmapped=null}", result.getBuckets().get(0).getKeyAsString()); + assertEquals(5L, result.getBuckets().get(0).getDocCount()); + } + ); + + // field + unmapped, no missing bucket = no results + testSearchCase( + Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), + dataset, + () -> new CompositeAggregationBuilder( + "name", + Arrays.asList( + new GeoTileGridValuesSourceBuilder(mappedFieldName).field(mappedFieldName), + new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped") + ) + ), + (result) -> assertEquals(0, result.getBuckets().size()) + ); + + // field + unmapped with missing bucket = multiple results + testSearchCase( + Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), + dataset, + () -> new CompositeAggregationBuilder( + "name", + Arrays.asList( + new GeoTileGridValuesSourceBuilder(mappedFieldName).field(mappedFieldName), + new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped").missingBucket(true) + ) + ), + (result) -> { + assertEquals(2, result.getBuckets().size()); + assertEquals("{geo_point=7/64/56, unmapped=null}", result.afterKey().toString()); + assertEquals("{geo_point=7/32/56, unmapped=null}", result.getBuckets().get(0).getKeyAsString()); + assertEquals(2L, result.getBuckets().get(0).getDocCount()); + assertEquals("{geo_point=7/64/56, unmapped=null}", result.getBuckets().get(1).getKeyAsString()); + assertEquals(3L, result.getBuckets().get(1).getDocCount()); + } + ); + + } + + public void testWithGeoPoint() throws Exception { + final List>> dataset = new ArrayList<>(); + dataset.addAll( + Arrays.asList( + createDocument("geo_point", new GeoPoint(48.934059, 41.610741)), + createDocument("geo_point", new GeoPoint(-23.065941, 113.610741)), + createDocument("geo_point", new GeoPoint(90.0, 0.0)), + createDocument("geo_point", new GeoPoint(37.2343, -115.8067)), + createDocument("geo_point", new GeoPoint(90.0, 0.0)) + ) + ); + testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("geo_point")), dataset, () -> { + GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder("geo_point").field("geo_point"); + return new CompositeAggregationBuilder("name", Collections.singletonList(geoTile)); + }, (result) -> { + assertEquals(2, result.getBuckets().size()); + assertEquals("{geo_point=7/64/56}", result.afterKey().toString()); + assertEquals("{geo_point=7/32/56}", result.getBuckets().get(0).getKeyAsString()); + assertEquals(2L, result.getBuckets().get(0).getDocCount()); + assertEquals("{geo_point=7/64/56}", result.getBuckets().get(1).getKeyAsString()); + assertEquals(3L, result.getBuckets().get(1).getDocCount()); + }); + + testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("geo_point")), dataset, () -> { + GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder("geo_point").field("geo_point"); + return new CompositeAggregationBuilder("name", Collections.singletonList(geoTile)).aggregateAfter( + Collections.singletonMap("geo_point", "7/32/56") + ); + }, (result) -> { + assertEquals(1, result.getBuckets().size()); + assertEquals("{geo_point=7/64/56}", result.afterKey().toString()); + assertEquals("{geo_point=7/64/56}", result.getBuckets().get(0).getKeyAsString()); + assertEquals(3L, result.getBuckets().get(0).getDocCount()); + }); + } + + @Override + protected boolean addValueToDocument(final Document doc, final String name, final Object value) { + if (value instanceof GeoPoint) { + GeoPoint point = (GeoPoint) value; + doc.add( + new SortedNumericDocValuesField( + name, + GeoTileUtils.longEncode(point.lon(), point.lat(), GeoTileGridAggregationBuilder.DEFAULT_PRECISION) + ) + ); + doc.add(new LatLonPoint(name, point.lat(), point.lon())); + return true; + } + return false; + } +} diff --git a/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridCompositeAggregationBuilderTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridCompositeAggregationBuilderTests.java new file mode 100644 index 0000000000000..ea7a2a83945c2 --- /dev/null +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridCompositeAggregationBuilderTests.java @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.bucket.composite; + +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.tests.common.RandomGeoGenerator; +import org.opensearch.plugins.Plugin; +import org.opensearch.search.aggregations.BaseAggregationTestCase; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +public class GeoTileGridCompositeAggregationBuilderTests extends BaseAggregationTestCase { + + protected Collection> getPlugins() { + return Collections.singletonList(GeoModulePlugin.class); + } + + private GeoTileGridValuesSourceBuilder randomGeoTileGridValuesSourceBuilder() { + GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder(randomAlphaOfLengthBetween(5, 10)); + if (randomBoolean()) { + geoTile.precision(randomIntBetween(0, GeoTileUtils.MAX_ZOOM)); + } + if (randomBoolean()) { + geoTile.geoBoundingBox(RandomGeoGenerator.randomBBox()); + } + return geoTile; + } + + @Override + protected CompositeAggregationBuilder createTestAggregatorBuilder() { + int numSources = randomIntBetween(1, 10); + List> sources = new ArrayList<>(); + for (int i = 0; i < numSources; i++) { + sources.add(randomGeoTileGridValuesSourceBuilder()); + } + return new CompositeAggregationBuilder(randomAlphaOfLength(10), sources); + } +} diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java similarity index 90% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java index 2b1700676f549..c6276c06c4511 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java @@ -30,8 +30,9 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.composite; +package org.opensearch.geo.search.aggregations.bucket.composite; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; import org.opensearch.test.OpenSearchTestCase; public class GeoTileGridValuesSourceBuilderTests extends OpenSearchTestCase { diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java similarity index 95% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java index 17fddb8978499..d6153637f656d 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; @@ -45,17 +45,19 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.common.CheckedConsumer; import org.opensearch.common.geo.GeoBoundingBox; -import org.opensearch.common.geo.GeoBoundingBoxTests; import org.opensearch.common.geo.GeoUtils; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.tests.common.AggregationInspectionHelper; +import org.opensearch.geo.tests.common.RandomGeoGenerator; import org.opensearch.index.mapper.GeoPointFieldMapper; import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.plugins.SearchPlugin; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorTestCase; import org.opensearch.search.aggregations.MultiBucketConsumerService; import org.opensearch.search.aggregations.bucket.terms.StringTerms; import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.opensearch.search.aggregations.support.AggregationInspectionHelper; import java.io.IOException; import java.util.ArrayList; @@ -91,6 +93,16 @@ public abstract class GeoGridAggregatorTestCase */ protected abstract GeoGridAggregationBuilder createBuilder(String name); + /** + * Overriding the Search Plugins list with {@link GeoModulePlugin} so that the testcase will know that this plugin is + * to be loaded during the tests. + * @return List of {@link SearchPlugin} + */ + @Override + protected List getSearchPlugins() { + return Collections.singletonList(new GeoModulePlugin()); + } + public void testNoDocs() throws IOException { testCase( new MatchAllDocsQuery(), @@ -225,7 +237,7 @@ public void testBounds() throws IOException { // only consider bounding boxes that are at least GEOHASH_TOLERANCE wide and have quantized coordinates GeoBoundingBox bbox = randomValueOtherThanMany( (b) -> Math.abs(GeoUtils.normalizeLon(b.right()) - GeoUtils.normalizeLon(b.left())) < GEOHASH_TOLERANCE, - GeoBoundingBoxTests::randomBBox + RandomGeoGenerator::randomBBox ); Function encodeDecodeLat = (lat) -> GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); Function encodeDecodeLon = (lon) -> GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lon)); diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridTestCase.java similarity index 79% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridTestCase.java index ce286a4443660..432736a2b43fe 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridTestCase.java @@ -29,9 +29,16 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.apache.lucene.index.IndexWriter; +import org.opensearch.common.ParseField; +import org.opensearch.common.xcontent.ContextParser; +import org.opensearch.common.xcontent.NamedXContentRegistry; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.search.aggregations.metrics.ParsedGeoBounds; +import org.opensearch.plugins.SearchPlugin; +import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.aggregations.ParsedMultiBucketAggregation; import org.opensearch.test.InternalMultiBucketAggregationTestCase; @@ -76,6 +83,36 @@ protected int maxNumberOfBuckets() { return 3; } + /** + * Overriding the method so that tests can get the aggregation specs for namedWriteable. + * + * @return GeoPlugin + */ + @Override + protected SearchPlugin registerPlugin() { + return new GeoModulePlugin(); + } + + /** + * Overriding with the {@link ParsedGeoBounds} so that it can be parsed. We need to do this as {@link GeoModulePlugin} + * is registering this Aggregation. + * + * @return a List of {@link NamedXContentRegistry.Entry} + */ + @Override + protected List getNamedXContents() { + final List namedXContents = new ArrayList<>(getDefaultNamedXContents()); + final ContextParser hashGridParser = (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c); + final ContextParser geoTileParser = (p, c) -> ParsedGeoTileGrid.fromXContent(p, (String) c); + namedXContents.add( + new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(GeoHashGridAggregationBuilder.NAME), hashGridParser) + ); + namedXContents.add( + new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(GeoTileGridAggregationBuilder.NAME), geoTileParser) + ); + return namedXContents; + } + @Override protected T createTestInstance(String name, Map metadata, InternalAggregations aggregations) { final int precision = randomPrecision(); diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java similarity index 96% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java index 5c63b15c7f614..04fa815366f6b 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import static org.opensearch.geometry.utils.Geohash.stringEncode; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java similarity index 99% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java index e81e22b3b562f..44f292e898a61 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.greaterThanOrEqualTo; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridTests.java similarity index 97% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridTests.java index 5a26ec759281c..c84c6ef5ec076 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridTests.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.geometry.utils.Geohash; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java similarity index 94% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java index 4e88111ac2dfc..f2f641ea794c0 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java @@ -30,7 +30,9 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; + +import org.opensearch.search.aggregations.bucket.GeoTileUtils; public class GeoTileGridAggregatorTests extends GeoGridAggregatorTestCase { diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java similarity index 97% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java index 567bcd57d23e5..a5b000d5e6ab3 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.ExceptionsHelper; import org.opensearch.common.xcontent.XContentParseException; @@ -37,6 +37,7 @@ import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.geo.GeometryTestUtils; import org.opensearch.geometry.Rectangle; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.test.OpenSearchTestCase; import static org.hamcrest.Matchers.containsString; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridTests.java similarity index 94% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridTests.java index 50b9a8cd762d1..ead67e0455d94 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridTests.java @@ -29,9 +29,10 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.search.aggregations.InternalAggregations; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import java.util.List; import java.util.Map; diff --git a/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregationBuilderTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregationBuilderTests.java new file mode 100644 index 0000000000000..49b455bbf389e --- /dev/null +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregationBuilderTests.java @@ -0,0 +1,43 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.metrics; + +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.search.aggregations.BaseAggregationTestCase; + +import java.util.Collection; +import java.util.Collections; + +public class GeoBoundsAggregationBuilderTests extends BaseAggregationTestCase { + + /** + * This registers the GeoShape mapper with the Tests so that it can be used for testing the aggregation builders + * + * @return A Collection containing {@link GeoModulePlugin} + */ + protected Collection> getPlugins() { + return Collections.singletonList(GeoModulePlugin.class); + } + + @Override + protected GeoBoundsAggregationBuilder createTestAggregatorBuilder() { + GeoBoundsAggregationBuilder factory = new GeoBoundsAggregationBuilder(randomAlphaOfLengthBetween(1, 20)); + String field = randomAlphaOfLengthBetween(3, 20); + factory.field(field); + if (randomBoolean()) { + factory.wrapLongitude(randomBoolean()); + } + if (randomBoolean()) { + factory.missing("0,0"); + } + return factory; + } + +} diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregatorTests.java similarity index 88% rename from server/src/test/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregatorTests.java index 6440c62e58e18..ee7a3c7e3faa2 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregatorTests.java @@ -30,26 +30,42 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; import org.apache.lucene.document.Document; import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; import org.opensearch.common.geo.GeoPoint; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.tests.common.AggregationInspectionHelper; +import org.opensearch.geo.tests.common.RandomGeoGenerator; import org.opensearch.index.mapper.GeoPointFieldMapper; import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.plugins.SearchPlugin; import org.opensearch.search.aggregations.AggregatorTestCase; -import org.opensearch.search.aggregations.support.AggregationInspectionHelper; -import org.opensearch.test.geo.RandomGeoGenerator; -import static org.opensearch.search.aggregations.metrics.InternalGeoBoundsTests.GEOHASH_TOLERANCE; +import java.util.Collections; +import java.util.List; + import static org.hamcrest.Matchers.closeTo; public class GeoBoundsAggregatorTests extends AggregatorTestCase { + public static final double GEOHASH_TOLERANCE = 1E-5D; + + /** + * Overriding the Search Plugins list with {@link GeoModulePlugin} so that the testcase will know that this plugin is + * to be loaded during the tests. + * @return List of {@link SearchPlugin} + */ + @Override + protected List getSearchPlugins() { + return Collections.singletonList(new GeoModulePlugin()); + } + public void testEmpty() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { GeoBoundsAggregationBuilder aggBuilder = new GeoBoundsAggregationBuilder("my_agg").field("field").wrapLongitude(false); diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalGeoBoundsTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/InternalGeoBoundsTests.java similarity index 81% rename from server/src/test/java/org/opensearch/search/aggregations/metrics/InternalGeoBoundsTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/InternalGeoBoundsTests.java index e3857efff5d4d..22915212ff415 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalGeoBoundsTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/InternalGeoBoundsTests.java @@ -30,11 +30,18 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; +import org.opensearch.common.ParseField; +import org.opensearch.common.xcontent.ContextParser; +import org.opensearch.common.xcontent.NamedXContentRegistry; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.plugins.SearchPlugin; +import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.ParsedAggregation; import org.opensearch.test.InternalAggregationTestCase; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -44,6 +51,30 @@ public class InternalGeoBoundsTests extends InternalAggregationTestCase { static final double GEOHASH_TOLERANCE = 1E-5D; + /** + * Overriding the method so that tests can get the aggregation specs for namedWriteable. + * + * @return GeoPlugin + */ + @Override + protected SearchPlugin registerPlugin() { + return new GeoModulePlugin(); + } + + /** + * Overriding with the {@link ParsedGeoBounds} so that it can be parsed. We need to do this as {@link GeoModulePlugin} + * is registering this Aggregation. + * + * @return a List of {@link NamedXContentRegistry.Entry} + */ + @Override + protected List getNamedXContents() { + final List namedXContents = new ArrayList<>(getDefaultNamedXContents()); + final ContextParser parser = (p, c) -> ParsedGeoBounds.fromXContent(p, (String) c); + namedXContents.add(new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(GeoBoundsAggregationBuilder.NAME), parser)); + return namedXContents; + } + @Override protected InternalGeoBounds createTestInstance(String name, Map metadata) { // we occasionally want to test top = Double.NEGATIVE_INFINITY since this triggers empty xContent object diff --git a/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationBuilders.java b/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationBuilders.java new file mode 100644 index 0000000000000..c0d7e51047c6b --- /dev/null +++ b/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationBuilders.java @@ -0,0 +1,39 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.tests.common; + +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.InternalGeoHashGrid; +import org.opensearch.geo.search.aggregations.bucket.geogrid.InternalGeoTileGrid; +import org.opensearch.geo.search.aggregations.metrics.GeoBounds; +import org.opensearch.geo.search.aggregations.metrics.GeoBoundsAggregationBuilder; + +public class AggregationBuilders { + /** + * Create a new {@link GeoBounds} aggregation with the given name. + */ + public static GeoBoundsAggregationBuilder geoBounds(String name) { + return new GeoBoundsAggregationBuilder(name); + } + + /** + * Create a new {@link InternalGeoHashGrid} aggregation with the given name. + */ + public static GeoHashGridAggregationBuilder geohashGrid(String name) { + return new GeoHashGridAggregationBuilder(name); + } + + /** + * Create a new {@link InternalGeoTileGrid} aggregation with the given name. + */ + public static GeoTileGridAggregationBuilder geotileGrid(String name) { + return new GeoTileGridAggregationBuilder(name); + } +} diff --git a/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationInspectionHelper.java b/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationInspectionHelper.java new file mode 100644 index 0000000000000..3473cf2d94b76 --- /dev/null +++ b/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationInspectionHelper.java @@ -0,0 +1,23 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.tests.common; + +import org.opensearch.geo.search.aggregations.bucket.geogrid.InternalGeoGrid; +import org.opensearch.geo.search.aggregations.metrics.InternalGeoBounds; + +public class AggregationInspectionHelper { + + public static boolean hasValue(InternalGeoBounds agg) { + return (agg.topLeft() == null && agg.bottomRight() == null) == false; + } + + public static boolean hasValue(InternalGeoGrid agg) { + return agg.getBuckets().stream().anyMatch(bucket -> bucket.getDocCount() > 0); + } +} diff --git a/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java b/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java new file mode 100644 index 0000000000000..2fb403155e2bc --- /dev/null +++ b/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java @@ -0,0 +1,97 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.tests.common; + +import org.opensearch.common.geo.GeoBoundingBox; +import org.opensearch.common.geo.GeoPoint; +import org.opensearch.geo.GeometryTestUtils; +import org.opensearch.geometry.Rectangle; + +import java.util.Random; + +/** + * Random geo generation utilities for randomized {@code geo_point} type testing + * does not depend on jts or spatial4j. Use RandomShapeGenerator to create random OGC compliant shapes. + * This is a copy of the file present in the server folder. We need to keep both as there are tests which are + * dependent on that file. + */ +public class RandomGeoGenerator { + + public static void randomPoint(Random r, double[] pt) { + final double[] min = { -180, -90 }; + final double[] max = { 180, 90 }; + randomPointIn(r, min[0], min[1], max[0], max[1], pt); + } + + public static void randomPointIn( + Random r, + final double minLon, + final double minLat, + final double maxLon, + final double maxLat, + double[] pt + ) { + assert pt != null && pt.length == 2; + + // normalize min and max + double[] min = { normalizeLongitude(minLon), normalizeLatitude(minLat) }; + double[] max = { normalizeLongitude(maxLon), normalizeLatitude(maxLat) }; + final double[] tMin = new double[2]; + final double[] tMax = new double[2]; + tMin[0] = Math.min(min[0], max[0]); + tMax[0] = Math.max(min[0], max[0]); + tMin[1] = Math.min(min[1], max[1]); + tMax[1] = Math.max(min[1], max[1]); + + pt[0] = tMin[0] + r.nextDouble() * (tMax[0] - tMin[0]); + pt[1] = tMin[1] + r.nextDouble() * (tMax[1] - tMin[1]); + } + + public static GeoPoint randomPoint(Random r) { + return randomPointIn(r, -180, -90, 180, 90); + } + + public static GeoPoint randomPointIn(Random r, final double minLon, final double minLat, final double maxLon, final double maxLat) { + double[] pt = new double[2]; + randomPointIn(r, minLon, minLat, maxLon, maxLat, pt); + return new GeoPoint(pt[1], pt[0]); + } + + /** Puts latitude in range of -90 to 90. */ + private static double normalizeLatitude(double latitude) { + if (latitude >= -90 && latitude <= 90) { + return latitude; // common case, and avoids slight double precision shifting + } + double off = Math.abs((latitude + 90) % 360); + return (off <= 180 ? off : 360 - off) - 90; + } + + /** Puts longitude in range of -180 to +180. */ + private static double normalizeLongitude(double longitude) { + if (longitude >= -180 && longitude <= 180) { + return longitude; // common case, and avoids slight double precision shifting + } + double off = (longitude + 180) % 360; + if (off < 0) { + return 180 + off; + } else if (off == 0 && longitude > 0) { + return 180; + } else { + return -180 + off; + } + } + + public static GeoBoundingBox randomBBox() { + Rectangle rectangle = GeometryTestUtils.randomRectangle(); + return new GeoBoundingBox( + new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), + new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon()) + ); + } +} diff --git a/modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/230_composite.yml b/modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/230_composite.yml new file mode 100644 index 0000000000000..211f3c3f46b88 --- /dev/null +++ b/modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/230_composite.yml @@ -0,0 +1,168 @@ +--- +setup: + - do: + indices.create: + index: test + body: + mappings: + properties: + date: + type: date + keyword: + type: keyword + long: + type: long + geo_point: + type: geo_point + nested: + type: nested + properties: + nested_long: + type: long + + - do: + indices.create: + index: other + body: + mappings: + properties: + date: + type: date + long: + type: long + nested: + type: nested + properties: + nested_long: + type: long + + - do: + index: + index: test + id: 1 + body: { "keyword": "foo", "long": [10, 20], "geo_point": "37.2343,-115.8067", "nested": [{"nested_long": 10}, {"nested_long": 20}] } + + - do: + index: + index: test + id: 2 + body: { "keyword": ["foo", "bar"], "geo_point": "41.12,-71.34" } + + - do: + index: + index: test + id: 3 + body: { "keyword": "bar", "long": [100, 0], "geo_point": "90.0,0.0", "nested": [{"nested_long": 10}, {"nested_long": 0}] } + + - do: + index: + index: test + id: 4 + body: { "keyword": "bar", "long": [1000, 0], "geo_point": "41.12,-71.34", "nested": [{"nested_long": 1000}, {"nested_long": 20}] } + + - do: + index: + index: test + id: 5 + body: { "date": "2017-10-20T03:08:45" } + + - do: + index: + index: test + id: 6 + body: { "date": "2017-10-21T07:00:00" } + + - do: + index: + index: other + id: 0 + body: { "date": "2017-10-20T03:08:45" } + + - do: + indices.refresh: + index: [test, other] +--- +"Simple Composite aggregation with GeoTile grid": + - skip: + version: " - 7.4.99" + reason: geotile_grid is not supported until 7.5.0 + - do: + search: + rest_total_hits_as_int: true + index: test + body: + aggregations: + test: + composite: + sources: [ + "geo": { + "geotile_grid": { + "field": "geo_point", + "precision": 12 + } + }, + { + "kw": { + "terms": { + "field": "keyword" + } + } + } + ] + + - match: {hits.total: 6} + - length: { aggregations.test.buckets: 4 } + - match: { aggregations.test.buckets.0.key.geo: "12/730/1590" } + - match: { aggregations.test.buckets.0.key.kw: "foo" } + - match: { aggregations.test.buckets.0.doc_count: 1 } + - match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" } + - match: { aggregations.test.buckets.1.key.kw: "bar" } + - match: { aggregations.test.buckets.1.doc_count: 2 } + - match: { aggregations.test.buckets.2.key.geo: "12/1236/1533" } + - match: { aggregations.test.buckets.2.key.kw: "foo" } + - match: { aggregations.test.buckets.2.doc_count: 1 } + - match: { aggregations.test.buckets.3.key.geo: "12/2048/0" } + - match: { aggregations.test.buckets.3.key.kw: "bar" } + - match: { aggregations.test.buckets.3.doc_count: 1 } + +--- +"Simple Composite aggregation with geotile grid add aggregate after": + - skip: + version: " - 7.4.99" + reason: geotile_grid is not supported until 7.5.0 + - do: + search: + index: test + body: + aggregations: + test: + composite: + sources: [ + "geo": { + "geotile_grid": { + "field": "geo_point", + "precision": 12 + } + }, + { + "kw": { + "terms": { + "field": "keyword" + } + } + } + ] + after: { "geo": "12/730/1590", "kw": "foo" } + + - match: { hits.total.value: 6 } + - match: { hits.total.relation: "eq" } + - length: { aggregations.test.buckets: 3 } + - match: { aggregations.test.buckets.0.key.geo: "12/1236/1533" } + - match: { aggregations.test.buckets.0.key.kw: "bar" } + - match: { aggregations.test.buckets.0.doc_count: 2 } + - match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" } + - match: { aggregations.test.buckets.1.key.kw: "foo" } + - match: { aggregations.test.buckets.1.doc_count: 1 } + - match: { aggregations.test.buckets.2.key.geo: "12/2048/0" } + - match: { aggregations.test.buckets.2.key.kw: "bar" } + - match: { aggregations.test.buckets.2.doc_count: 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/280_geohash_grid.yml b/modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/280_geohash_grid.yml similarity index 100% rename from rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/280_geohash_grid.yml rename to modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/280_geohash_grid.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/290_geotile_grid.yml b/modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/290_geotile_grid.yml similarity index 100% rename from rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/290_geotile_grid.yml rename to modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/290_geotile_grid.yml diff --git a/modules/lang-expression/licenses/lucene-expressions-9.3.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.3.0.jar.sha1 deleted file mode 100644 index 2d216277b3a8e..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5583bcd3a24d3aae40b0a3152458021844ac09aa \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.4.0-snapshot-ddf0d0a.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..ec6906d730ac1 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +9f23e695b0c864fa9722e4f67d950266ca64d37b \ No newline at end of file diff --git a/modules/repository-s3/licenses/commons-logging-1.2.jar.sha1 b/modules/repository-s3/licenses/commons-logging-1.2.jar.sha1 deleted file mode 100644 index f40f0242448e8..0000000000000 --- a/modules/repository-s3/licenses/commons-logging-1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4bfc12adfe4842bf07b657f0369c4cb522955686 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0.jar.sha1 deleted file mode 100644 index df4ae8d72dd2b..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -11dd9be0448fe594cf918f5260e193b3ab4e07a0 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.4.0-snapshot-ddf0d0a.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..83c10845cd35a --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +2f6cb0fd7387c6e0db3b86eef7d8677cea3e88a0 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0.jar.sha1 deleted file mode 100644 index 675bf726d2a65..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -87c1357612f2f483174d1a63ea8c6680a1696bac \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.4.0-snapshot-ddf0d0a.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..29387f38bc10c --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +6aff23715a2fba88d844ac83c61decce8ed480bd \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0.jar.sha1 deleted file mode 100644 index 8987f89c913df..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5d032dbeb3f4015741336a877dd4b0e62099246c \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.4.0-snapshot-ddf0d0a.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..54b451abf5049 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +f82d3eba195134f663865e9de3f511e16fbc7351 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0.jar.sha1 deleted file mode 100644 index 00d66c733c548..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fe6ac8772b545e0abd0c755cd4bd07caad58edb9 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.4.0-snapshot-ddf0d0a.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..87474064fbe0f --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +2af6e1996e696b1721a2ec7382bac9aa5096eecb \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0.jar.sha1 deleted file mode 100644 index 0c521b5f5ef6a..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -288726e13b598c341e81aef8b5c9ce53f51889d0 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.4.0-snapshot-ddf0d0a.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..6d35832a1a643 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +ec01d7f91f711abd75b539bb66a437db7cf1ca67 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0.jar.sha1 deleted file mode 100644 index ba98dd7e06f71..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -166d02f7f98f18c6607335030a404fcad8f57cd6 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.4.0-snapshot-ddf0d0a.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..f93d1a153cd26 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +7041b3fa92b8687a84c4ce666b5718bbbc315db1 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0.jar.sha1 deleted file mode 100644 index 88ac9a13e8ce3..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3c0e4177aa87a4be2826a360f656f3559ea3f997 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.4.0-snapshot-ddf0d0a.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..77589a361badf --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +0a5ec9a237c2539e3cbabfadff707252e33b3599 \ No newline at end of file diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml index 2e298441918bc..09278690f5d05 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml @@ -651,92 +651,6 @@ setup: } ] ---- -"Simple Composite aggregation with GeoTile grid": - - skip: - version: " - 7.4.99" - reason: geotile_grid is not supported until 7.5.0 - - do: - search: - rest_total_hits_as_int: true - index: test - body: - aggregations: - test: - composite: - sources: [ - "geo": { - "geotile_grid": { - "field": "geo_point", - "precision": 12 - } - }, - { - "kw": { - "terms": { - "field": "keyword" - } - } - } - ] - - - match: {hits.total: 6} - - length: { aggregations.test.buckets: 4 } - - match: { aggregations.test.buckets.0.key.geo: "12/730/1590" } - - match: { aggregations.test.buckets.0.key.kw: "foo" } - - match: { aggregations.test.buckets.0.doc_count: 1 } - - match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" } - - match: { aggregations.test.buckets.1.key.kw: "bar" } - - match: { aggregations.test.buckets.1.doc_count: 2 } - - match: { aggregations.test.buckets.2.key.geo: "12/1236/1533" } - - match: { aggregations.test.buckets.2.key.kw: "foo" } - - match: { aggregations.test.buckets.2.doc_count: 1 } - - match: { aggregations.test.buckets.3.key.geo: "12/2048/0" } - - match: { aggregations.test.buckets.3.key.kw: "bar" } - - match: { aggregations.test.buckets.3.doc_count: 1 } - ---- -"Simple Composite aggregation with geotile grid add aggregate after": - - skip: - version: " - 7.4.99" - reason: geotile_grid is not supported until 7.5.0 - - do: - search: - index: test - body: - aggregations: - test: - composite: - sources: [ - "geo": { - "geotile_grid": { - "field": "geo_point", - "precision": 12 - } - }, - { - "kw": { - "terms": { - "field": "keyword" - } - } - } - ] - after: { "geo": "12/730/1590", "kw": "foo" } - - - match: { hits.total.value: 6 } - - match: { hits.total.relation: "eq" } - - length: { aggregations.test.buckets: 3 } - - match: { aggregations.test.buckets.0.key.geo: "12/1236/1533" } - - match: { aggregations.test.buckets.0.key.kw: "bar" } - - match: { aggregations.test.buckets.0.doc_count: 2 } - - match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" } - - match: { aggregations.test.buckets.1.key.kw: "foo" } - - match: { aggregations.test.buckets.1.doc_count: 1 } - - match: { aggregations.test.buckets.2.key.geo: "12/2048/0" } - - match: { aggregations.test.buckets.2.key.kw: "bar" } - - match: { aggregations.test.buckets.2.doc_count: 1 } - --- "Mixed ip and unmapped fields": - skip: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml index 6f276f669f815..25d3dd160e031 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml @@ -17,6 +17,12 @@ setup: id: 1 body: { text: some short words with a stupendously long one } + - do: + index: + index: test + id: 2 + body: { text: sentence with UPPERCASE WORDS } + - do: indices.refresh: index: [test] @@ -76,6 +82,25 @@ setup: - match: {hits.max_score: 1} - match: {hits.hits.0._score: 1} +--- +"search with uppercase regex": + - skip: + version: " - 2.99.99" + reason: uppercase regex not supported before 3.0.0 + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + query_string: + default_field: text + query: /UPPERCASE/ + + - match: {hits.total: 1} + - match: {hits.max_score: 1} + - match: {hits.hits.0._score: 1} + --- "search index prefixes with span_multi": - do: diff --git a/server/licenses/lucene-analysis-common-9.3.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.3.0.jar.sha1 deleted file mode 100644 index 2e260eb028f4c..0000000000000 --- a/server/licenses/lucene-analysis-common-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -03496708a19a8a55a0dc4f61f8aa2febc6e8977c \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-analysis-common-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..0805590fd6efd --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +3920c527fd5eee69e09f614391cef4e05c581c7f \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.3.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.3.0.jar.sha1 deleted file mode 100644 index 1dda17ee92fdb..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -95ea01ee0d1e543e18e3cf58d8a6a27a587a7239 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-backward-codecs-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..a8c648e4c192a --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +d1dfcd42ea257355d5cbc64ac2f47142a550ae52 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.3.0.jar.sha1 b/server/licenses/lucene-core-9.3.0.jar.sha1 deleted file mode 100644 index fd870008c5bd4..0000000000000 --- a/server/licenses/lucene-core-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a030180999bc3f1a65f23f53b38098ca9daeee79 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-core-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..779c9796ceae7 --- /dev/null +++ b/server/licenses/lucene-core-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +ae4757f88e97036b30eb1eac1d21da6dabc85a5e \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.3.0.jar.sha1 b/server/licenses/lucene-grouping-9.3.0.jar.sha1 deleted file mode 100644 index 6f63ca177d3c3..0000000000000 --- a/server/licenses/lucene-grouping-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -883071196e53ec93d2a53dcc8211ee30be6c00dc \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-grouping-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..9167482284f9d --- /dev/null +++ b/server/licenses/lucene-grouping-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +75485e3222b096027463378fe3bb2c8d1f529d25 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.3.0.jar.sha1 b/server/licenses/lucene-highlighter-9.3.0.jar.sha1 deleted file mode 100644 index 78264d8ee3713..0000000000000 --- a/server/licenses/lucene-highlighter-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7e895c49b9991ea2ec08855c425b9eae44a08764 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-highlighter-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..2090b009a57fe --- /dev/null +++ b/server/licenses/lucene-highlighter-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +d2a3d1f326f6d3bd6033b5620dc84f3c20a0412d \ No newline at end of file diff --git a/server/licenses/lucene-join-9.3.0.jar.sha1 b/server/licenses/lucene-join-9.3.0.jar.sha1 deleted file mode 100644 index 5e641f5f01075..0000000000000 --- a/server/licenses/lucene-join-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -04baaae4ce4a35ae919150dd17cd1e63b0da9d24 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-join-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..df74fa911f7d2 --- /dev/null +++ b/server/licenses/lucene-join-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +2c6f6058c765a955e0544c6050aeee3a5e376e47 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.3.0.jar.sha1 b/server/licenses/lucene-memory-9.3.0.jar.sha1 deleted file mode 100644 index c8e86c7674ede..0000000000000 --- a/server/licenses/lucene-memory-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1a2203b332edc1366b9789f5286296e109dbc8c4 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-memory-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..3e9d65d73d36d --- /dev/null +++ b/server/licenses/lucene-memory-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +2a155679022106c7db356da32563580d8de043d7 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.3.0.jar.sha1 b/server/licenses/lucene-misc-9.3.0.jar.sha1 deleted file mode 100644 index 11a459a9f52ba..0000000000000 --- a/server/licenses/lucene-misc-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -61b502c9557247b6803a346c0bab20c9dc89d125 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-misc-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..f056cfe5b86ef --- /dev/null +++ b/server/licenses/lucene-misc-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +7c7ac2027a12bf02657ec3a421c8039736b98344 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.3.0.jar.sha1 b/server/licenses/lucene-queries-9.3.0.jar.sha1 deleted file mode 100644 index 2b577bd33b46a..0000000000000 --- a/server/licenses/lucene-queries-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d8fe3bce3c05015c5fdb78279f36b9f1a75b98d8 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-queries-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..41ea4a342f949 --- /dev/null +++ b/server/licenses/lucene-queries-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +25978bb82b9f78537f0511f0be64253be19de6fd \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.3.0.jar.sha1 b/server/licenses/lucene-queryparser-9.3.0.jar.sha1 deleted file mode 100644 index b106860bf9f3e..0000000000000 --- a/server/licenses/lucene-queryparser-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -78f259a66d48f77a2d2b96a0a858efa08eba72dc \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-queryparser-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..e0687571df957 --- /dev/null +++ b/server/licenses/lucene-queryparser-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +e3b6ce41d5bd73fdcc80b5b2a40283c03525aa96 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.3.0.jar.sha1 b/server/licenses/lucene-sandbox-9.3.0.jar.sha1 deleted file mode 100644 index 82c2c6d85ca4c..0000000000000 --- a/server/licenses/lucene-sandbox-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5ee318cf8e9a70c2c99e03e157465316a3d4a17a \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-sandbox-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..e03c731914757 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +ae8649d2d01a416acdbe7c29f14b47a5594acb85 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.3.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.3.0.jar.sha1 deleted file mode 100644 index 8bbc5359487ff..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c9b226b49ae987a4226791f023562187583eb9ad \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-spatial-extras-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..ea8b5cd1ddb1d --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +58049352bb5fc8683c389eb2eb879fb4f16ff9b3 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.3.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.3.0.jar.sha1 deleted file mode 100644 index 31132ef0ad6df..0000000000000 --- a/server/licenses/lucene-spatial3d-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -201aa61856ae44fa494504591aed54fd9b75af16 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-spatial3d-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..a0c0dbe07af8f --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +2d3a8f802e1bb439d945de81ba6b16d01b24d58a \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.3.0.jar.sha1 b/server/licenses/lucene-suggest-9.3.0.jar.sha1 deleted file mode 100644 index 71a263aa163f8..0000000000000 --- a/server/licenses/lucene-suggest-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fb5d7243ba67616edbda1ecf421c615dd595752d \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-suggest-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..1f332eac16c72 --- /dev/null +++ b/server/licenses/lucene-suggest-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +11cdb21cf08feb19e074b4a101e1550dfd544379 \ No newline at end of file diff --git a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java index 2bf73b34247b3..3d8da7eac7690 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java @@ -78,6 +78,7 @@ import org.opensearch.index.mapper.SourceToParse; import org.opensearch.index.seqno.RetentionLeaseSyncer; import org.opensearch.index.seqno.SequenceNumbers; +import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.TestTranslog; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.TranslogStats; @@ -675,6 +676,7 @@ public static final IndexShard newIndexShard( () -> {}, RetentionLeaseSyncer.EMPTY, cbs, + new InternalTranslogFactory(), SegmentReplicationCheckpointPublisher.EMPTY, null ); diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index a1cc0148dcdac..8566cc5556861 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -15,10 +15,14 @@ import org.opensearch.action.admin.indices.segments.IndicesSegmentResponse; import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequest; import org.opensearch.action.admin.indices.segments.ShardSegments; +import org.opensearch.action.support.WriteRequest; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.allocation.command.CancelAllocationCommand; +import org.opensearch.common.Nullable; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.Index; @@ -29,6 +33,7 @@ import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.BackgroundIndexer; +import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; @@ -36,6 +41,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; @@ -70,6 +77,109 @@ protected boolean addMockInternalEngine() { return false; } + public void testPrimaryStopped_ReplicaPromoted() throws Exception { + final String primary = internalCluster().startNode(); + createIndex(INDEX_NAME); + ensureYellowAndNoInitializingShards(INDEX_NAME); + final String replica = internalCluster().startNode(); + ensureGreen(INDEX_NAME); + + client().prepareIndex(INDEX_NAME).setId("1").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + refresh(INDEX_NAME); + + waitForReplicaUpdate(); + assertHitCount(client(primary).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 1); + assertHitCount(client(replica).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 1); + + // index another doc but don't refresh, we will ensure this is searchable once replica is promoted. + client().prepareIndex(INDEX_NAME).setId("2").setSource("bar", "baz").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + + // stop the primary node - we only have one shard on here. + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary)); + ensureYellowAndNoInitializingShards(INDEX_NAME); + + final ShardRouting replicaShardRouting = getShardRoutingForNodeName(replica); + assertNotNull(replicaShardRouting); + assertTrue(replicaShardRouting + " should be promoted as a primary", replicaShardRouting.primary()); + assertHitCount(client(replica).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 2); + + // assert we can index into the new primary. + client().prepareIndex(INDEX_NAME).setId("3").setSource("bar", "baz").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + assertHitCount(client(replica).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 3); + + // start another node, index another doc and replicate. + String nodeC = internalCluster().startNode(); + ensureGreen(INDEX_NAME); + client().prepareIndex(INDEX_NAME).setId("4").setSource("baz", "baz").get(); + refresh(INDEX_NAME); + waitForReplicaUpdate(); + assertHitCount(client(nodeC).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 4); + assertHitCount(client(replica).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 4); + assertSegmentStats(REPLICA_COUNT); + } + + public void testRestartPrimary() throws Exception { + final String primary = internalCluster().startNode(); + createIndex(INDEX_NAME); + ensureYellowAndNoInitializingShards(INDEX_NAME); + final String replica = internalCluster().startNode(); + ensureGreen(INDEX_NAME); + + assertEquals(getNodeContainingPrimaryShard().getName(), primary); + + final int initialDocCount = 1; + client().prepareIndex(INDEX_NAME).setId("1").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + refresh(INDEX_NAME); + + waitForReplicaUpdate(); + assertDocCounts(initialDocCount, replica, primary); + + internalCluster().restartNode(primary); + ensureGreen(INDEX_NAME); + + assertEquals(getNodeContainingPrimaryShard().getName(), replica); + + flushAndRefresh(INDEX_NAME); + waitForReplicaUpdate(); + + assertDocCounts(initialDocCount, replica, primary); + assertSegmentStats(REPLICA_COUNT); + } + + public void testCancelPrimaryAllocation() throws Exception { + // this test cancels allocation on the primary - promoting the new replica and recreating the former primary as a replica. + final String primary = internalCluster().startNode(); + createIndex(INDEX_NAME); + ensureYellowAndNoInitializingShards(INDEX_NAME); + final String replica = internalCluster().startNode(); + ensureGreen(INDEX_NAME); + + final int initialDocCount = 1; + + client().prepareIndex(INDEX_NAME).setId("1").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + refresh(INDEX_NAME); + + waitForReplicaUpdate(); + assertDocCounts(initialDocCount, replica, primary); + + final IndexShard indexShard = getIndexShard(primary); + client().admin() + .cluster() + .prepareReroute() + .add(new CancelAllocationCommand(INDEX_NAME, indexShard.shardId().id(), primary, true)) + .execute() + .actionGet(); + ensureGreen(INDEX_NAME); + + assertEquals(getNodeContainingPrimaryShard().getName(), replica); + + flushAndRefresh(INDEX_NAME); + waitForReplicaUpdate(); + + assertDocCounts(initialDocCount, replica, primary); + assertSegmentStats(REPLICA_COUNT); + } + public void testReplicationAfterPrimaryRefreshAndFlush() throws Exception { final String nodeA = internalCluster().startNode(); final String nodeB = internalCluster().startNode(); @@ -111,6 +221,54 @@ public void testReplicationAfterPrimaryRefreshAndFlush() throws Exception { } } + public void testMultipleShards() throws Exception { + Settings indexSettings = Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .build(); + final String nodeA = internalCluster().startNode(); + final String nodeB = internalCluster().startNode(); + createIndex(INDEX_NAME, indexSettings); + ensureGreen(INDEX_NAME); + + final int initialDocCount = scaledRandomIntBetween(1, 200); + try ( + BackgroundIndexer indexer = new BackgroundIndexer( + INDEX_NAME, + "_doc", + client(), + -1, + RandomizedTest.scaledRandomIntBetween(2, 5), + false, + random() + ) + ) { + indexer.start(initialDocCount); + waitForDocs(initialDocCount, indexer); + refresh(INDEX_NAME); + waitForReplicaUpdate(); + + assertHitCount(client(nodeA).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), initialDocCount); + assertHitCount(client(nodeB).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), initialDocCount); + + final int additionalDocCount = scaledRandomIntBetween(0, 200); + final int expectedHitCount = initialDocCount + additionalDocCount; + indexer.start(additionalDocCount); + waitForDocs(expectedHitCount, indexer); + + flushAndRefresh(INDEX_NAME); + waitForReplicaUpdate(); + assertHitCount(client(nodeA).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedHitCount); + assertHitCount(client(nodeB).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedHitCount); + + ensureGreen(INDEX_NAME); + assertSegmentStats(REPLICA_COUNT); + } + } + public void testReplicationAfterForceMerge() throws Exception { final String nodeA = internalCluster().startNode(); final String nodeB = internalCluster().startNode(); @@ -189,9 +347,8 @@ public void testStartReplicaAfterPrimaryIndexesDocs() throws Exception { assertHitCount(client(primaryNode).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 3); assertHitCount(client(replicaNode).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 3); - final Index index = resolveIndex(INDEX_NAME); - IndexShard primaryShard = getIndexShard(index, primaryNode); - IndexShard replicaShard = getIndexShard(index, replicaNode); + IndexShard primaryShard = getIndexShard(primaryNode); + IndexShard replicaShard = getIndexShard(replicaNode); assertEquals( primaryShard.translogStats().estimatedNumberOfOperations(), replicaShard.translogStats().estimatedNumberOfOperations() @@ -199,6 +356,69 @@ public void testStartReplicaAfterPrimaryIndexesDocs() throws Exception { assertSegmentStats(REPLICA_COUNT); } + public void testDeleteOperations() throws Exception { + final String nodeA = internalCluster().startNode(); + final String nodeB = internalCluster().startNode(); + + createIndex(INDEX_NAME); + ensureGreen(INDEX_NAME); + final int initialDocCount = scaledRandomIntBetween(0, 200); + try ( + BackgroundIndexer indexer = new BackgroundIndexer( + INDEX_NAME, + "_doc", + client(), + -1, + RandomizedTest.scaledRandomIntBetween(2, 5), + false, + random() + ) + ) { + indexer.start(initialDocCount); + waitForDocs(initialDocCount, indexer); + refresh(INDEX_NAME); + waitForReplicaUpdate(); + + // wait a short amount of time to give replication a chance to complete. + assertHitCount(client(nodeA).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), initialDocCount); + assertHitCount(client(nodeB).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), initialDocCount); + + final int additionalDocCount = scaledRandomIntBetween(0, 200); + final int expectedHitCount = initialDocCount + additionalDocCount; + indexer.start(additionalDocCount); + waitForDocs(expectedHitCount, indexer); + waitForReplicaUpdate(); + + assertHitCount(client(nodeA).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedHitCount); + assertHitCount(client(nodeB).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedHitCount); + + ensureGreen(INDEX_NAME); + + Set ids = indexer.getIds(); + String id = ids.toArray()[0].toString(); + client(nodeA).prepareDelete(INDEX_NAME, id).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + + refresh(INDEX_NAME); + waitForReplicaUpdate(); + assertBusy(() -> { + final long nodeA_Count = client(nodeA).prepareSearch(INDEX_NAME) + .setSize(0) + .setPreference("_only_local") + .get() + .getHits() + .getTotalHits().value; + assertEquals(expectedHitCount - 1, nodeA_Count); + final long nodeB_Count = client(nodeB).prepareSearch(INDEX_NAME) + .setSize(0) + .setPreference("_only_local") + .get() + .getHits() + .getTotalHits().value; + assertEquals(expectedHitCount - 1, nodeB_Count); + }, 5, TimeUnit.SECONDS); + } + } + private void assertSegmentStats(int numberOfReplicas) throws IOException { final IndicesSegmentResponse indicesSegmentResponse = client().admin().indices().segments(new IndicesSegmentsRequest()).actionGet(); @@ -237,8 +457,7 @@ private void assertSegmentStats(int numberOfReplicas) throws IOException { final ShardRouting replicaShardRouting = shardSegment.getShardRouting(); ClusterState state = client(internalCluster().getMasterName()).admin().cluster().prepareState().get().getState(); final DiscoveryNode replicaNode = state.nodes().resolveNode(replicaShardRouting.currentNodeId()); - final Index index = resolveIndex(INDEX_NAME); - IndexShard indexShard = getIndexShard(index, replicaNode.getName()); + IndexShard indexShard = getIndexShard(replicaNode.getName()); final String lastCommitSegmentsFileName = SegmentInfos.getLastCommitSegmentsFileName(indexShard.store().directory()); // calls to readCommit will fail if a valid commit point and all its segments are not in the store. SegmentInfos.readCommit(indexShard.store().directory(), lastCommitSegmentsFileName); @@ -262,21 +481,24 @@ private void waitForReplicaUpdate() throws Exception { final Map> segmentListMap = segmentsByShardType(replicationGroupSegments); final List primaryShardSegmentsList = segmentListMap.get(true); final List replicaShardSegments = segmentListMap.get(false); - + // if we don't have any segments yet, proceed. final ShardSegments primaryShardSegments = primaryShardSegmentsList.stream().findFirst().get(); - final Map latestPrimarySegments = getLatestSegments(primaryShardSegments); - final Long latestPrimaryGen = latestPrimarySegments.values().stream().findFirst().map(Segment::getGeneration).get(); - for (ShardSegments shardSegments : replicaShardSegments) { - final boolean isReplicaCaughtUpToPrimary = shardSegments.getSegments() - .stream() - .anyMatch(segment -> segment.getGeneration() == latestPrimaryGen); - assertTrue(isReplicaCaughtUpToPrimary); + if (primaryShardSegments.getSegments().isEmpty() == false) { + final Map latestPrimarySegments = getLatestSegments(primaryShardSegments); + final Long latestPrimaryGen = latestPrimarySegments.values().stream().findFirst().map(Segment::getGeneration).get(); + for (ShardSegments shardSegments : replicaShardSegments) { + final boolean isReplicaCaughtUpToPrimary = shardSegments.getSegments() + .stream() + .anyMatch(segment -> segment.getGeneration() == latestPrimaryGen); + assertTrue(isReplicaCaughtUpToPrimary); + } } } }); } - private IndexShard getIndexShard(Index index, String node) { + private IndexShard getIndexShard(String node) { + final Index index = resolveIndex(INDEX_NAME); IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); IndexService indexService = indicesService.indexServiceSafe(index); final Optional shardId = indexService.shardIds().stream().findFirst(); @@ -293,7 +515,8 @@ private List getShardSegments(IndicesSegmentResponse indicesSeg } private Map getLatestSegments(ShardSegments segments) { - final Long latestPrimaryGen = segments.getSegments().stream().map(Segment::getGeneration).max(Long::compare).get(); + final Optional generation = segments.getSegments().stream().map(Segment::getGeneration).max(Long::compare); + final Long latestPrimaryGen = generation.get(); return segments.getSegments() .stream() .filter(s -> s.getGeneration() == latestPrimaryGen) @@ -303,4 +526,31 @@ private Map getLatestSegments(ShardSegments segments) { private Map> segmentsByShardType(ShardSegments[] replicationGroupSegments) { return Arrays.stream(replicationGroupSegments).collect(Collectors.groupingBy(s -> s.getShardRouting().primary())); } + + @Nullable + private ShardRouting getShardRoutingForNodeName(String nodeName) { + final ClusterState state = client(internalCluster().getClusterManagerName()).admin().cluster().prepareState().get().getState(); + for (IndexShardRoutingTable shardRoutingTable : state.routingTable().index(INDEX_NAME)) { + for (ShardRouting shardRouting : shardRoutingTable.activeShards()) { + final String nodeId = shardRouting.currentNodeId(); + final DiscoveryNode discoveryNode = state.nodes().resolveNode(nodeId); + if (discoveryNode.getName().equals(nodeName)) { + return shardRouting; + } + } + } + return null; + } + + private void assertDocCounts(int expectedDocCount, String... nodeNames) { + for (String node : nodeNames) { + assertHitCount(client(node).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedDocCount); + } + } + + private DiscoveryNode getNodeContainingPrimaryShard() { + final ClusterState state = client(internalCluster().getClusterManagerName()).admin().cluster().prepareState().get().getState(); + final ShardRouting primaryShard = state.routingTable().index(INDEX_NAME).shard(0).primaryShard(); + return state.nodes().resolveNode(primaryShard.currentNodeId()); + } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MissingValueIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MissingValueIT.java index 7d3f06760882d..26bfe59618275 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MissingValueIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MissingValueIT.java @@ -39,7 +39,6 @@ import org.opensearch.search.aggregations.bucket.terms.Terms; import org.opensearch.search.aggregations.bucket.terms.TermsAggregatorFactory.ExecutionMode; import org.opensearch.search.aggregations.metrics.Cardinality; -import org.opensearch.search.aggregations.metrics.GeoBounds; import org.opensearch.search.aggregations.metrics.GeoCentroid; import org.opensearch.search.aggregations.metrics.Percentiles; import org.opensearch.search.aggregations.metrics.Stats; @@ -47,7 +46,6 @@ import static org.opensearch.search.aggregations.AggregationBuilders.cardinality; import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; -import static org.opensearch.search.aggregations.AggregationBuilders.geoBounds; import static org.opensearch.search.aggregations.AggregationBuilders.geoCentroid; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.percentiles; @@ -213,28 +211,6 @@ public void testStats() { assertEquals(4, stats.getAvg(), 0); } - public void testUnmappedGeoBounds() { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(geoBounds("bounds").field("non_existing_field").missing("2,1")) - .get(); - assertSearchResponse(response); - GeoBounds bounds = response.getAggregations().get("bounds"); - assertThat(bounds.bottomRight().lat(), closeTo(2.0, 1E-5)); - assertThat(bounds.bottomRight().lon(), closeTo(1.0, 1E-5)); - assertThat(bounds.topLeft().lat(), closeTo(2.0, 1E-5)); - assertThat(bounds.topLeft().lon(), closeTo(1.0, 1E-5)); - } - - public void testGeoBounds() { - SearchResponse response = client().prepareSearch("idx").addAggregation(geoBounds("bounds").field("location").missing("2,1")).get(); - assertSearchResponse(response); - GeoBounds bounds = response.getAggregations().get("bounds"); - assertThat(bounds.bottomRight().lat(), closeTo(1.0, 1E-5)); - assertThat(bounds.bottomRight().lon(), closeTo(2.0, 1E-5)); - assertThat(bounds.topLeft().lat(), closeTo(2.0, 1E-5)); - assertThat(bounds.topLeft().lon(), closeTo(1.0, 1E-5)); - } - public void testGeoCentroid() { SearchResponse response = client().prepareSearch("idx") .addAggregation(geoCentroid("centroid").field("location").missing("2,1")) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardReduceIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardReduceIT.java index 7352dc7170a21..faa6a54394b00 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardReduceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardReduceIT.java @@ -37,7 +37,6 @@ import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.opensearch.search.aggregations.bucket.filter.Filter; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGrid; import org.opensearch.search.aggregations.bucket.global.Global; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.opensearch.search.aggregations.bucket.histogram.Histogram; @@ -51,8 +50,6 @@ import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.opensearch.search.aggregations.AggregationBuilders.dateRange; import static org.opensearch.search.aggregations.AggregationBuilders.filter; -import static org.opensearch.search.aggregations.AggregationBuilders.geohashGrid; -import static org.opensearch.search.aggregations.AggregationBuilders.geotileGrid; import static org.opensearch.search.aggregations.AggregationBuilders.global; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.ipRange; @@ -338,36 +335,4 @@ public void testDateHistogram() throws Exception { } - public void testGeoHashGrid() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .setQuery(QueryBuilders.matchAllQuery()) - .addAggregation( - geohashGrid("grid").field("location") - .subAggregation(dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.DAY).minDocCount(0)) - ) - .get(); - - assertSearchResponse(response); - - GeoGrid grid = response.getAggregations().get("grid"); - Histogram histo = grid.getBuckets().iterator().next().getAggregations().get("histo"); - assertThat(histo.getBuckets().size(), equalTo(4)); - } - - public void testGeoTileGrid() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .setQuery(QueryBuilders.matchAllQuery()) - .addAggregation( - geotileGrid("grid").field("location") - .subAggregation(dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.DAY).minDocCount(0)) - ) - .get(); - - assertSearchResponse(response); - - GeoGrid grid = response.getAggregations().get("grid"); - Histogram histo = grid.getBuckets().iterator().next().getAggregations().get("histo"); - assertThat(histo.getBuckets().size(), equalTo(4)); - } - } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoCentroidIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoCentroidIT.java index 7cd8b3ed39051..ffc31b7cdb7c4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoCentroidIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoCentroidIT.java @@ -35,15 +35,11 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.common.geo.GeoPoint; import org.opensearch.search.aggregations.InternalAggregation; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGrid; import org.opensearch.search.aggregations.bucket.global.Global; import org.opensearch.test.OpenSearchIntegTestCase; -import java.util.List; - import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.search.aggregations.AggregationBuilders.geoCentroid; -import static org.opensearch.search.aggregations.AggregationBuilders.geohashGrid; import static org.opensearch.search.aggregations.AggregationBuilders.global; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.closeTo; @@ -168,33 +164,4 @@ public void testMultiValuedField() throws Exception { assertThat(centroid.lon(), closeTo(multiCentroid.lon(), GEOHASH_TOLERANCE)); assertEquals(2 * numDocs, geoCentroid.count()); } - - public void testSingleValueFieldAsSubAggToGeohashGrid() throws Exception { - SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME) - .addAggregation( - geohashGrid("geoGrid").field(SINGLE_VALUED_FIELD_NAME).subAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME)) - ) - .get(); - assertSearchResponse(response); - - GeoGrid grid = response.getAggregations().get("geoGrid"); - assertThat(grid, notNullValue()); - assertThat(grid.getName(), equalTo("geoGrid")); - List buckets = grid.getBuckets(); - for (GeoGrid.Bucket cell : buckets) { - String geohash = cell.getKeyAsString(); - GeoPoint expectedCentroid = expectedCentroidsForGeoHash.get(geohash); - GeoCentroid centroidAgg = cell.getAggregations().get(aggName); - assertThat( - "Geohash " + geohash + " has wrong centroid latitude ", - expectedCentroid.lat(), - closeTo(centroidAgg.centroid().lat(), GEOHASH_TOLERANCE) - ); - assertThat( - "Geohash " + geohash + " has wrong centroid longitude", - expectedCentroid.lon(), - closeTo(centroidAgg.centroid().lon(), GEOHASH_TOLERANCE) - ); - } - } } diff --git a/server/src/main/java/org/opensearch/Version.java b/server/src/main/java/org/opensearch/Version.java index 57f49f9c54591..ba512d3fbcdd9 100644 --- a/server/src/main/java/org/opensearch/Version.java +++ b/server/src/main/java/org/opensearch/Version.java @@ -95,8 +95,9 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_1_0 = new Version(2010099, org.apache.lucene.util.Version.LUCENE_9_2_0); public static final Version V_2_1_1 = new Version(2010199, org.apache.lucene.util.Version.LUCENE_9_2_0); public static final Version V_2_2_0 = new Version(2020099, org.apache.lucene.util.Version.LUCENE_9_3_0); + public static final Version V_2_2_1 = new Version(2020199, org.apache.lucene.util.Version.LUCENE_9_3_0); public static final Version V_2_3_0 = new Version(2030099, org.apache.lucene.util.Version.LUCENE_9_3_0); - public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_3_0); + public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_4_0); public static final Version CURRENT = V_3_0_0; public static Version readVersion(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index a02f9580c0857..4acf777dadc95 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -240,6 +240,7 @@ import org.opensearch.action.search.DeletePitAction; import org.opensearch.action.search.GetAllPitsAction; import org.opensearch.action.search.MultiSearchAction; +import org.opensearch.action.search.NodesGetAllPitsAction; import org.opensearch.action.search.SearchAction; import org.opensearch.action.search.SearchScrollAction; import org.opensearch.action.search.TransportClearScrollAction; @@ -247,6 +248,7 @@ import org.opensearch.action.search.TransportDeletePitAction; import org.opensearch.action.search.TransportGetAllPitsAction; import org.opensearch.action.search.TransportMultiSearchAction; +import org.opensearch.action.search.TransportNodesGetAllPitsAction; import org.opensearch.action.search.TransportSearchAction; import org.opensearch.action.search.TransportSearchScrollAction; import org.opensearch.action.support.ActionFilters; @@ -678,6 +680,7 @@ public void reg actions.register(GetAllPitsAction.INSTANCE, TransportGetAllPitsAction.class); actions.register(DeletePitAction.INSTANCE, TransportDeletePitAction.class); actions.register(PitSegmentsAction.INSTANCE, TransportPitSegmentsAction.class); + actions.register(NodesGetAllPitsAction.INSTANCE, TransportNodesGetAllPitsAction.class); // Remote Store actions.register(RestoreRemoteStoreAction.INSTANCE, TransportRestoreRemoteStoreAction.class); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/pit/package-info.java b/server/src/main/java/org/opensearch/action/admin/cluster/pit/package-info.java new file mode 100644 index 0000000000000..e18db6b68fc50 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/cluster/pit/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Node level transport handlers. */ +package org.opensearch.action.admin.cluster.pit; diff --git a/server/src/main/java/org/opensearch/action/search/DeletePitRequest.java b/server/src/main/java/org/opensearch/action/search/DeletePitRequest.java index 945fcfd17eb6c..926e9c19a33f5 100644 --- a/server/src/main/java/org/opensearch/action/search/DeletePitRequest.java +++ b/server/src/main/java/org/opensearch/action/search/DeletePitRequest.java @@ -48,6 +48,11 @@ public DeletePitRequest(List pitIds) { this.pitIds.addAll(pitIds); } + public void clearAndSetPitIds(List pitIds) { + this.pitIds.clear(); + this.pitIds.addAll(pitIds); + } + public DeletePitRequest() {} public List getPitIds() { diff --git a/server/src/main/java/org/opensearch/action/search/GetAllPitNodesRequest.java b/server/src/main/java/org/opensearch/action/search/GetAllPitNodesRequest.java index b4ad2f6641087..cf858baa2b696 100644 --- a/server/src/main/java/org/opensearch/action/search/GetAllPitNodesRequest.java +++ b/server/src/main/java/org/opensearch/action/search/GetAllPitNodesRequest.java @@ -21,11 +21,19 @@ */ public class GetAllPitNodesRequest extends BaseNodesRequest { + private GetAllPitNodesResponse getAllPitNodesResponse; @Inject public GetAllPitNodesRequest(DiscoveryNode... concreteNodes) { super(concreteNodes); } + public void setGetAllPitNodesResponse(GetAllPitNodesResponse getAllPitNodesResponse) { + this.getAllPitNodesResponse = getAllPitNodesResponse; + } + + public GetAllPitNodesResponse getGetAllPitNodesResponse() { + return getAllPitNodesResponse; + } public GetAllPitNodesRequest(StreamInput in) throws IOException { super(in); } diff --git a/server/src/main/java/org/opensearch/action/search/GetAllPitNodesResponse.java b/server/src/main/java/org/opensearch/action/search/GetAllPitNodesResponse.java index 4a454e7145eff..98feb4a30f764 100644 --- a/server/src/main/java/org/opensearch/action/search/GetAllPitNodesResponse.java +++ b/server/src/main/java/org/opensearch/action/search/GetAllPitNodesResponse.java @@ -77,4 +77,9 @@ public void writeNodesTo(StreamOutput out, List nodes) th public List getPitInfos() { return Collections.unmodifiableList(new ArrayList<>(pitInfos)); } + + public void clearAndSetPitInfos(List listPitInfos) { + pitInfos.clear(); + pitInfos.addAll(listPitInfos); + } } diff --git a/server/src/main/java/org/opensearch/action/search/GetAllPitsAction.java b/server/src/main/java/org/opensearch/action/search/GetAllPitsAction.java index 16e65cb785a7d..427deb2f92022 100644 --- a/server/src/main/java/org/opensearch/action/search/GetAllPitsAction.java +++ b/server/src/main/java/org/opensearch/action/search/GetAllPitsAction.java @@ -15,7 +15,7 @@ */ public class GetAllPitsAction extends ActionType { public static final GetAllPitsAction INSTANCE = new GetAllPitsAction(); - public static final String NAME = "indices:data/read/point_in_time/readall"; + public static final String NAME = "indices:data/read/point_in_time/read"; private GetAllPitsAction() { super(NAME, GetAllPitNodesResponse::new); diff --git a/server/src/main/java/org/opensearch/action/search/NodesGetAllPitsAction.java b/server/src/main/java/org/opensearch/action/search/NodesGetAllPitsAction.java new file mode 100644 index 0000000000000..7c722c2adb587 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/search/NodesGetAllPitsAction.java @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.action.search; + +import org.opensearch.action.ActionType; + +/** + * docs + */ +public class NodesGetAllPitsAction extends ActionType { + public static final NodesGetAllPitsAction INSTANCE = new NodesGetAllPitsAction(); + public static final String NAME = "cluster:admin/point_in_time/read_from_nodes"; + + private NodesGetAllPitsAction() { + super(NAME, GetAllPitNodesResponse::new); + } +} diff --git a/server/src/main/java/org/opensearch/action/search/PitService.java b/server/src/main/java/org/opensearch/action/search/PitService.java index 0b79b77fd6014..c1ea69181f18d 100644 --- a/server/src/main/java/org/opensearch/action/search/PitService.java +++ b/server/src/main/java/org/opensearch/action/search/PitService.java @@ -15,6 +15,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.StepListener; import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Strings; @@ -48,11 +49,14 @@ public class PitService { private final SearchTransportService searchTransportService; private final TransportService transportService; + private final NodeClient nodeClient; + @Inject - public PitService(ClusterService clusterService, SearchTransportService searchTransportService, TransportService transportService) { + public PitService(ClusterService clusterService, SearchTransportService searchTransportService, TransportService transportService, NodeClient nodeClient) { this.clusterService = clusterService; this.searchTransportService = searchTransportService; this.transportService = transportService; + this.nodeClient = nodeClient; } /** @@ -144,6 +148,17 @@ public void onFailure(final Exception e) { }, size); } + /** + * This method returns indices associated for each pit + */ + public Map getIndicesForPits(List pitIds) { + Map pitToIndicesMap = new HashMap<>(); + for(String pitId : pitIds) { + pitToIndicesMap.put(pitId, SearchContextId.decode(nodeClient.getNamedWriteableRegistry(), pitId).getActualIndices()); + } + return pitToIndicesMap; + } + /** * Get all active point in time contexts */ @@ -156,7 +171,7 @@ public void getAllPits(ActionListener getAllPitsListener DiscoveryNode[] disNodesArr = nodes.toArray(new DiscoveryNode[nodes.size()]); transportService.sendRequest( transportService.getLocalNode(), - GetAllPitsAction.NAME, + NodesGetAllPitsAction.NAME, new GetAllPitNodesRequest(disNodesArr), new TransportResponseHandler() { @@ -182,4 +197,6 @@ public GetAllPitNodesResponse read(StreamInput in) throws IOException { } ); } + + } diff --git a/server/src/main/java/org/opensearch/action/search/TransportDeletePitAction.java b/server/src/main/java/org/opensearch/action/search/TransportDeletePitAction.java index f9e36c479dd54..5d991a3d92c57 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportDeletePitAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportDeletePitAction.java @@ -43,7 +43,7 @@ public TransportDeletePitAction( SearchTransportService searchTransportService, PitService pitService ) { - super(DeletePitAction.NAME, transportService, actionFilters, DeletePitRequest::new); + super(DeletePitAction.NAME, transportService, actionFilters, in -> new DeletePitRequest(in)); this.namedWriteableRegistry = namedWriteableRegistry; this.transportSearchAction = transportSearchAction; this.clusterService = clusterService; @@ -57,6 +57,7 @@ public TransportDeletePitAction( @Override protected void doExecute(Task task, DeletePitRequest request, ActionListener listener) { List pitIds = request.getPitIds(); + logger.info("pit ids size : " + pitIds.size() + " : " + pitIds.get(0)); if (pitIds.size() == 1 && "_all".equals(pitIds.get(0))) { deleteAllPits(listener); } else { diff --git a/server/src/main/java/org/opensearch/action/search/TransportGetAllPitsAction.java b/server/src/main/java/org/opensearch/action/search/TransportGetAllPitsAction.java index 21a64e388fa7b..f319065997554 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportGetAllPitsAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportGetAllPitsAction.java @@ -8,79 +8,35 @@ package org.opensearch.action.search; -import org.opensearch.action.FailedNodeException; +import org.opensearch.action.ActionListener; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.nodes.TransportNodesAction; -import org.opensearch.cluster.service.ClusterService; +import org.opensearch.action.support.HandledTransportAction; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.search.SearchService; -import org.opensearch.threadpool.ThreadPool; +import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; -import java.io.IOException; -import java.util.List; +import java.util.Collections; /** - * Transport action to get all active PIT contexts across all nodes + * Transport action to get all active PIT contexts in the cluster */ -public class TransportGetAllPitsAction extends TransportNodesAction< - GetAllPitNodesRequest, - GetAllPitNodesResponse, - GetAllPitNodeRequest, - GetAllPitNodeResponse> { - private final SearchService searchService; +public class TransportGetAllPitsAction extends HandledTransportAction { - @Inject - public TransportGetAllPitsAction( - ThreadPool threadPool, - ClusterService clusterService, - TransportService transportService, - ActionFilters actionFilters, - SearchService searchService - ) { - super( - GetAllPitsAction.NAME, - threadPool, - clusterService, - transportService, - actionFilters, - GetAllPitNodesRequest::new, - GetAllPitNodeRequest::new, - ThreadPool.Names.SAME, - GetAllPitNodeResponse.class - ); - this.searchService = searchService; - } - - @Override - protected GetAllPitNodesResponse newResponse( - GetAllPitNodesRequest request, - List getAllPitNodeRespons, - List failures - ) { - return new GetAllPitNodesResponse(clusterService.getClusterName(), getAllPitNodeRespons, failures); - } + private final PitService pitService; - @Override - protected GetAllPitNodeRequest newNodeRequest(GetAllPitNodesRequest request) { - return new GetAllPitNodeRequest(); - } - - @Override - protected GetAllPitNodeResponse newNodeResponse(StreamInput in) throws IOException { - return new GetAllPitNodeResponse(in); + @Inject + public TransportGetAllPitsAction(ActionFilters actionFilters, TransportService transportService, PitService pitService) { + super(GetAllPitsAction.NAME, transportService, actionFilters, in -> new GetAllPitNodesRequest(in)); + this.pitService = pitService; } - /** - * This retrieves all active PITs in the node - */ @Override - protected GetAllPitNodeResponse nodeOperation(GetAllPitNodeRequest request) { - GetAllPitNodeResponse nodeResponse = new GetAllPitNodeResponse( - transportService.getLocalNode(), - searchService.getAllPITReaderContexts() - ); - return nodeResponse; + protected void doExecute(Task task, GetAllPitNodesRequest request, ActionListener listener) { + // If security plugin intercepts the request, it'll replace all PIT IDs with permitted PIT IDs + if(request.getGetAllPitNodesResponse() != null) { + listener.onResponse(request.getGetAllPitNodesResponse()); + } else { + pitService.getAllPits(listener); + } } } diff --git a/server/src/main/java/org/opensearch/action/search/TransportNodesGetAllPitsAction.java b/server/src/main/java/org/opensearch/action/search/TransportNodesGetAllPitsAction.java new file mode 100644 index 0000000000000..520830cd293f0 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/search/TransportNodesGetAllPitsAction.java @@ -0,0 +1,86 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.search; + +import org.opensearch.action.FailedNodeException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.nodes.TransportNodesAction; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.search.SearchService; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.util.List; + +/** + * Transport action to get all active PIT contexts across all nodes + */ +public class TransportNodesGetAllPitsAction extends TransportNodesAction< + GetAllPitNodesRequest, + GetAllPitNodesResponse, + GetAllPitNodeRequest, + GetAllPitNodeResponse> { + private final SearchService searchService; + + @Inject + public TransportNodesGetAllPitsAction( + ThreadPool threadPool, + ClusterService clusterService, + TransportService transportService, + ActionFilters actionFilters, + SearchService searchService + ) { + super( + NodesGetAllPitsAction.NAME, + threadPool, + clusterService, + transportService, + actionFilters, + GetAllPitNodesRequest::new, + GetAllPitNodeRequest::new, + ThreadPool.Names.SAME, + GetAllPitNodeResponse.class + ); + this.searchService = searchService; + } + + @Override + protected GetAllPitNodesResponse newResponse( + GetAllPitNodesRequest request, + List getAllPitNodeRespons, + List failures + ) { + return new GetAllPitNodesResponse(clusterService.getClusterName(), getAllPitNodeRespons, failures); + } + + @Override + protected GetAllPitNodeRequest newNodeRequest(GetAllPitNodesRequest request) { + return new GetAllPitNodeRequest(); + } + + @Override + protected GetAllPitNodeResponse newNodeResponse(StreamInput in) throws IOException { + return new GetAllPitNodeResponse(in); + } + + /** + * This retrieves all active PITs in the node + */ + @Override + protected GetAllPitNodeResponse nodeOperation(GetAllPitNodeRequest request) { + GetAllPitNodeResponse nodeResponse = new GetAllPitNodeResponse( + transportService.getLocalNode(), + searchService.getAllPITReaderContexts() + ); + return nodeResponse; + } +} diff --git a/server/src/main/java/org/opensearch/client/Client.java b/server/src/main/java/org/opensearch/client/Client.java index 94043d5c3c89f..fbdb4c27f5849 100644 --- a/server/src/main/java/org/opensearch/client/Client.java +++ b/server/src/main/java/org/opensearch/client/Client.java @@ -62,8 +62,6 @@ import org.opensearch.action.search.ClearScrollResponse; import org.opensearch.action.search.CreatePitRequest; import org.opensearch.action.search.CreatePitResponse; -import org.opensearch.action.search.DeletePitRequest; -import org.opensearch.action.search.DeletePitResponse; import org.opensearch.action.search.MultiSearchRequest; import org.opensearch.action.search.MultiSearchRequestBuilder; import org.opensearch.action.search.MultiSearchResponse; @@ -336,11 +334,6 @@ public interface Client extends OpenSearchClient, Releasable { */ void createPit(CreatePitRequest createPITRequest, ActionListener listener); - /** - * Delete one or more point in time contexts - */ - void deletePits(DeletePitRequest deletePITRequest, ActionListener listener); - /** * Get information of segments of one or more PITs */ diff --git a/server/src/main/java/org/opensearch/client/ClusterAdminClient.java b/server/src/main/java/org/opensearch/client/ClusterAdminClient.java index 7a7b98bf724f6..91e59e0b8ce7e 100644 --- a/server/src/main/java/org/opensearch/client/ClusterAdminClient.java +++ b/server/src/main/java/org/opensearch/client/ClusterAdminClient.java @@ -132,6 +132,10 @@ import org.opensearch.action.ingest.SimulatePipelineRequest; import org.opensearch.action.ingest.SimulatePipelineRequestBuilder; import org.opensearch.action.ingest.SimulatePipelineResponse; +import org.opensearch.action.search.DeletePitRequest; +import org.opensearch.action.search.DeletePitResponse; +import org.opensearch.action.search.GetAllPitNodesRequest; +import org.opensearch.action.search.GetAllPitNodesResponse; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentType; @@ -791,4 +795,25 @@ public interface ClusterAdminClient extends OpenSearchClient { * Delete specified dangling indices. */ ActionFuture deleteDanglingIndex(DeleteDanglingIndexRequest request); + + /** + * Get all point in time searches present in cluster + */ + void getAllPits(GetAllPitNodesRequest request, ActionListener listener); + + /** + * Get all point in time searches present in cluster + */ + ActionFuture getAllPits(GetAllPitNodesRequest request); + + /** + * Get all point in time searches present in cluster + */ + void deletePits(DeletePitRequest request, ActionListener listener); + + /** + * Get all point in time searches present in cluster + */ + ActionFuture deletePits(DeletePitRequest request); + } diff --git a/server/src/main/java/org/opensearch/client/support/AbstractClient.java b/server/src/main/java/org/opensearch/client/support/AbstractClient.java index bc80a2ba92bf8..f031b5e17cb44 100644 --- a/server/src/main/java/org/opensearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/opensearch/client/support/AbstractClient.java @@ -335,6 +335,9 @@ import org.opensearch.action.search.DeletePitAction; import org.opensearch.action.search.DeletePitRequest; import org.opensearch.action.search.DeletePitResponse; +import org.opensearch.action.search.GetAllPitNodesRequest; +import org.opensearch.action.search.GetAllPitNodesResponse; +import org.opensearch.action.search.GetAllPitsAction; import org.opensearch.action.search.MultiSearchAction; import org.opensearch.action.search.MultiSearchRequest; import org.opensearch.action.search.MultiSearchRequestBuilder; @@ -590,11 +593,6 @@ public void createPit(final CreatePitRequest createPITRequest, final ActionListe execute(CreatePitAction.INSTANCE, createPITRequest, listener); } - @Override - public void deletePits(final DeletePitRequest deletePITRequest, final ActionListener listener) { - execute(DeletePitAction.INSTANCE, deletePITRequest, listener); - } - @Override public void pitSegments(final PitSegmentsRequest request, final ActionListener listener) { execute(PitSegmentsAction.INSTANCE, request, listener); @@ -781,6 +779,16 @@ public ActionFuture reroute(final ClusterRerouteRequest return execute(ClusterRerouteAction.INSTANCE, request); } + @Override + public void deletePits(final DeletePitRequest deletePITRequest, final ActionListener listener) { + execute(DeletePitAction.INSTANCE, deletePITRequest, listener); + } + + @Override + public ActionFuture deletePits(DeletePitRequest request) { + return execute(DeletePitAction.INSTANCE, request); + } + @Override public void reroute(final ClusterRerouteRequest request, final ActionListener listener) { execute(ClusterRerouteAction.INSTANCE, request, listener); @@ -1264,6 +1272,16 @@ public ActionFuture deleteDanglingIndex(DeleteDanglingInde return execute(DeleteDanglingIndexAction.INSTANCE, request); } + @Override + public void getAllPits(GetAllPitNodesRequest request, ActionListener listener) { + execute(GetAllPitsAction.INSTANCE, request, listener); + } + + @Override + public ActionFuture getAllPits(GetAllPitNodesRequest request) { + return execute(GetAllPitsAction.INSTANCE, request); + } + @Override public void deleteDanglingIndex(DeleteDanglingIndexRequest request, ActionListener listener) { execute(DeleteDanglingIndexAction.INSTANCE, request, listener); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 6e39809ac1a34..759891e88039b 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -292,6 +292,32 @@ public Iterator> settings() { public static final Setting INDEX_REMOTE_STORE_ENABLED_SETTING = Setting.boolSetting( SETTING_REMOTE_STORE_ENABLED, false, + new Setting.Validator<>() { + + @Override + public void validate(final Boolean value) {} + + @Override + public void validate(final Boolean value, final Map, Object> settings) { + final Object replicationType = settings.get(INDEX_REPLICATION_TYPE_SETTING); + if (replicationType != ReplicationType.SEGMENT && value == true) { + throw new IllegalArgumentException( + "To enable " + + INDEX_REMOTE_STORE_ENABLED_SETTING.getKey() + + ", " + + INDEX_REPLICATION_TYPE_SETTING.getKey() + + " should be set to " + + ReplicationType.SEGMENT + ); + } + } + + @Override + public Iterator> settings() { + final List> settings = Collections.singletonList(INDEX_REPLICATION_TYPE_SETTING); + return settings.iterator(); + } + }, Property.IndexScope, Property.Final ); diff --git a/server/src/main/java/org/opensearch/common/lucene/Lucene.java b/server/src/main/java/org/opensearch/common/lucene/Lucene.java index 74be762bfbcf9..2692a8fa2b914 100644 --- a/server/src/main/java/org/opensearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/opensearch/common/lucene/Lucene.java @@ -125,7 +125,7 @@ * @opensearch.internal */ public class Lucene { - public static final String LATEST_CODEC = "Lucene92"; + public static final String LATEST_CODEC = "Lucene94"; public static final String SOFT_DELETES_FIELD = "__soft_deletes"; diff --git a/server/src/main/java/org/opensearch/gateway/PrimaryShardAllocator.java b/server/src/main/java/org/opensearch/gateway/PrimaryShardAllocator.java index b9cfebaa98521..4dc9396751fc9 100644 --- a/server/src/main/java/org/opensearch/gateway/PrimaryShardAllocator.java +++ b/server/src/main/java/org/opensearch/gateway/PrimaryShardAllocator.java @@ -313,6 +313,11 @@ private static ShardStoreInfo shardStoreInfo(NodeGatewayStartedShards nodeShardS NodeGatewayStartedShards::primary ).reversed(); + private static final Comparator HIGHEST_REPLICATION_CHECKPOINT_FIRST_COMPARATOR = Comparator.comparing( + NodeGatewayStartedShards::replicationCheckpoint, + Comparator.nullsLast(Comparator.naturalOrder()) + ); + /** * Builds a list of nodes. If matchAnyShard is set to false, only nodes that have an allocation id matching * inSyncAllocationIds are added to the list. Otherwise, any node that has a shard is added to the list, but @@ -381,6 +386,12 @@ protected static NodeShardsResult buildNodeShardsResult( } } + /** + * Orders the active shards copies based on below comparators + * 1. No store exception i.e. shard copy is readable + * 2. Prefer previous primary shard + * 3. Prefer shard copy with the highest replication checkpoint. It is NO-OP for doc rep enabled indices. + */ final Comparator comparator; // allocation preference if (matchAnyShard) { // prefer shards with matching allocation ids @@ -388,9 +399,11 @@ protected static NodeShardsResult buildNodeShardsResult( (NodeGatewayStartedShards state) -> inSyncAllocationIds.contains(state.allocationId()) ).reversed(); comparator = matchingAllocationsFirst.thenComparing(NO_STORE_EXCEPTION_FIRST_COMPARATOR) - .thenComparing(PRIMARY_FIRST_COMPARATOR); + .thenComparing(PRIMARY_FIRST_COMPARATOR) + .thenComparing(HIGHEST_REPLICATION_CHECKPOINT_FIRST_COMPARATOR); } else { - comparator = NO_STORE_EXCEPTION_FIRST_COMPARATOR.thenComparing(PRIMARY_FIRST_COMPARATOR); + comparator = NO_STORE_EXCEPTION_FIRST_COMPARATOR.thenComparing(PRIMARY_FIRST_COMPARATOR) + .thenComparing(HIGHEST_REPLICATION_CHECKPOINT_FIRST_COMPARATOR); } nodeShardStates.sort(comparator); diff --git a/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java b/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java index 78b4fa287ef59..c43f539243d7a 100644 --- a/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java +++ b/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java @@ -35,6 +35,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.LegacyESVersion; import org.opensearch.OpenSearchException; +import org.opensearch.Version; import org.opensearch.action.ActionListener; import org.opensearch.action.ActionType; import org.opensearch.action.FailedNodeException; @@ -56,11 +57,13 @@ import org.opensearch.common.xcontent.NamedXContentRegistry; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.IndexSettings; +import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.shard.ShardStateMetadata; import org.opensearch.index.store.Store; import org.opensearch.indices.IndicesService; +import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -195,6 +198,7 @@ protected NodeGatewayStartedShards nodeOperation(NodeRequest request) { clusterService.localNode(), allocationId, shardStateMetadata.primary, + null, exception ); } @@ -202,10 +206,16 @@ protected NodeGatewayStartedShards nodeOperation(NodeRequest request) { logger.debug("{} shard state info found: [{}]", shardId, shardStateMetadata); String allocationId = shardStateMetadata.allocationId != null ? shardStateMetadata.allocationId.getId() : null; - return new NodeGatewayStartedShards(clusterService.localNode(), allocationId, shardStateMetadata.primary); + final IndexShard shard = indicesService.getShardOrNull(shardId); + return new NodeGatewayStartedShards( + clusterService.localNode(), + allocationId, + shardStateMetadata.primary, + shard != null ? shard.getLatestReplicationCheckpoint() : null + ); } logger.trace("{} no local shard info found", shardId); - return new NodeGatewayStartedShards(clusterService.localNode(), null, false); + return new NodeGatewayStartedShards(clusterService.localNode(), null, false, null); } catch (Exception e) { throw new OpenSearchException("failed to load started shards", e); } @@ -349,10 +359,10 @@ public String getCustomDataPath() { * @opensearch.internal */ public static class NodeGatewayStartedShards extends BaseNodeResponse { - private final String allocationId; private final boolean primary; private final Exception storeException; + private final ReplicationCheckpoint replicationCheckpoint; public NodeGatewayStartedShards(StreamInput in) throws IOException { super(in); @@ -363,16 +373,33 @@ public NodeGatewayStartedShards(StreamInput in) throws IOException { } else { storeException = null; } + if (in.getVersion().onOrAfter(Version.V_2_3_0) && in.readBoolean()) { + replicationCheckpoint = new ReplicationCheckpoint(in); + } else { + replicationCheckpoint = null; + } } - public NodeGatewayStartedShards(DiscoveryNode node, String allocationId, boolean primary) { - this(node, allocationId, primary, null); + public NodeGatewayStartedShards( + DiscoveryNode node, + String allocationId, + boolean primary, + ReplicationCheckpoint replicationCheckpoint + ) { + this(node, allocationId, primary, replicationCheckpoint, null); } - public NodeGatewayStartedShards(DiscoveryNode node, String allocationId, boolean primary, Exception storeException) { + public NodeGatewayStartedShards( + DiscoveryNode node, + String allocationId, + boolean primary, + ReplicationCheckpoint replicationCheckpoint, + Exception storeException + ) { super(node); this.allocationId = allocationId; this.primary = primary; + this.replicationCheckpoint = replicationCheckpoint; this.storeException = storeException; } @@ -384,6 +411,10 @@ public boolean primary() { return this.primary; } + public ReplicationCheckpoint replicationCheckpoint() { + return this.replicationCheckpoint; + } + public Exception storeException() { return this.storeException; } @@ -399,6 +430,14 @@ public void writeTo(StreamOutput out) throws IOException { } else { out.writeBoolean(false); } + if (out.getVersion().onOrAfter(Version.V_2_3_0)) { + if (replicationCheckpoint != null) { + out.writeBoolean(true); + replicationCheckpoint.writeTo(out); + } else { + out.writeBoolean(false); + } + } } @Override @@ -414,7 +453,8 @@ public boolean equals(Object o) { return primary == that.primary && Objects.equals(allocationId, that.allocationId) - && Objects.equals(storeException, that.storeException); + && Objects.equals(storeException, that.storeException) + && Objects.equals(replicationCheckpoint, that.replicationCheckpoint); } @Override @@ -422,6 +462,7 @@ public int hashCode() { int result = (allocationId != null ? allocationId.hashCode() : 0); result = 31 * result + (primary ? 1 : 0); result = 31 * result + (storeException != null ? storeException.hashCode() : 0); + result = 31 * result + (replicationCheckpoint != null ? replicationCheckpoint.hashCode() : 0); return result; } @@ -432,6 +473,9 @@ public String toString() { if (storeException != null) { buf.append(",storeException=").append(storeException); } + if (replicationCheckpoint != null) { + buf.append(",ReplicationCheckpoint=").append(replicationCheckpoint.toString()); + } buf.append("]"); return buf.toString(); } diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 210df9d342cb7..e1427df1c34ab 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -88,6 +88,7 @@ import org.opensearch.index.shard.ShardPath; import org.opensearch.index.similarity.SimilarityService; import org.opensearch.index.store.Store; +import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.Translog; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.indices.cluster.IndicesClusterStateService; @@ -547,7 +548,9 @@ public synchronized IndexShard createShard( () -> globalCheckpointSyncer.accept(shardId), retentionLeaseSyncer, circuitBreakerService, - this.indexSettings.isSegRepEnabled() && routing.primary() ? checkpointPublisher : null, + // TODO Replace with remote translog factory in the follow up PR + this.indexSettings.isRemoteTranslogStoreEnabled() ? null : new InternalTranslogFactory(), + this.indexSettings.isSegRepEnabled() ? checkpointPublisher : null, remoteStore ); eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created"); diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index ff254a63fadb6..b1e73b3855759 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -34,8 +34,8 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene92.Lucene92Codec; -import org.apache.lucene.codecs.lucene92.Lucene92Codec.Mode; +import org.apache.lucene.codecs.lucene94.Lucene94Codec; +import org.apache.lucene.codecs.lucene94.Lucene94Codec.Mode; import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; import org.opensearch.index.mapper.MapperService; @@ -62,8 +62,8 @@ public class CodecService { public CodecService(@Nullable MapperService mapperService, Logger logger) { final MapBuilder codecs = MapBuilder.newMapBuilder(); if (mapperService == null) { - codecs.put(DEFAULT_CODEC, new Lucene92Codec()); - codecs.put(BEST_COMPRESSION_CODEC, new Lucene92Codec(Mode.BEST_COMPRESSION)); + codecs.put(DEFAULT_CODEC, new Lucene94Codec()); + codecs.put(BEST_COMPRESSION_CODEC, new Lucene94Codec(Mode.BEST_COMPRESSION)); } else { codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); diff --git a/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java b/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java index fd0c66983208a..c101321e47350 100644 --- a/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java @@ -36,7 +36,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene92.Lucene92Codec; +import org.apache.lucene.codecs.lucene94.Lucene94Codec; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; import org.opensearch.common.lucene.Lucene; import org.opensearch.index.mapper.CompletionFieldMapper; @@ -53,7 +53,7 @@ * * @opensearch.internal */ -public class PerFieldMappingPostingFormatCodec extends Lucene92Codec { +public class PerFieldMappingPostingFormatCodec extends Lucene94Codec { private final Logger logger; private final MapperService mapperService; private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat(); diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java index 4ae6646ed14f0..ba30103f70269 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java @@ -51,8 +51,10 @@ import org.opensearch.index.seqno.RetentionLeases; import org.opensearch.index.shard.ShardId; import org.opensearch.index.store.Store; +import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.TranslogConfig; import org.opensearch.index.translog.TranslogDeletionPolicyFactory; +import org.opensearch.index.translog.TranslogFactory; import org.opensearch.indices.IndexingMemoryController; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.threadpool.ThreadPool; @@ -150,6 +152,8 @@ public Supplier retentionLeasesSupplier() { private final TranslogConfig translogConfig; + private final TranslogFactory translogFactory; + public EngineConfig( ShardId shardId, ThreadPool threadPool, @@ -253,7 +257,8 @@ public EngineConfig( retentionLeasesSupplier, primaryTermSupplier, tombstoneDocSupplier, - false + false, + new InternalTranslogFactory() ); } @@ -284,7 +289,8 @@ public EngineConfig( Supplier retentionLeasesSupplier, LongSupplier primaryTermSupplier, TombstoneDocSupplier tombstoneDocSupplier, - boolean isReadOnlyReplica + boolean isReadOnlyReplica, + TranslogFactory translogFactory ) { if (isReadOnlyReplica && indexSettings.isSegRepEnabled() == false) { throw new IllegalArgumentException("Shard can only be wired as a read only replica with Segment Replication enabled"); @@ -328,6 +334,7 @@ public EngineConfig( this.primaryTermSupplier = primaryTermSupplier; this.tombstoneDocSupplier = tombstoneDocSupplier; this.isReadOnlyReplica = isReadOnlyReplica; + this.translogFactory = translogFactory; } /** @@ -532,6 +539,14 @@ public boolean isReadOnlyReplica() { return indexSettings.isSegRepEnabled() && isReadOnlyReplica; } + /** + * Returns the underlying translog factory + * @return the translog factory + */ + public TranslogFactory getTranslogFactory() { + return translogFactory; + } + /** * A supplier supplies tombstone documents which will be used in soft-update methods. * The returned document consists only _uid, _seqno, _term and _version fields; other metadata fields are excluded. diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java index c8aec3570f8b5..f0db086e47816 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java @@ -28,6 +28,7 @@ import org.opensearch.index.store.Store; import org.opensearch.index.translog.TranslogConfig; import org.opensearch.index.translog.TranslogDeletionPolicyFactory; +import org.opensearch.index.translog.TranslogFactory; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.plugins.EnginePlugin; import org.opensearch.plugins.PluginsService; @@ -147,7 +148,8 @@ public EngineConfig newEngineConfig( Supplier retentionLeasesSupplier, LongSupplier primaryTermSupplier, EngineConfig.TombstoneDocSupplier tombstoneDocSupplier, - boolean isReadOnlyReplica + boolean isReadOnlyReplica, + TranslogFactory translogFactory ) { CodecService codecServiceToUse = codecService; if (codecService == null && this.codecServiceFactory != null) { @@ -178,7 +180,8 @@ public EngineConfig newEngineConfig( retentionLeasesSupplier, primaryTermSupplier, tombstoneDocSupplier, - isReadOnlyReplica + isReadOnlyReplica, + translogFactory ); } diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java index 7d90c2ad653be..16599141b1345 100644 --- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java @@ -289,7 +289,8 @@ public void onFailure(String reason, Exception ex) { () -> getLocalCheckpointTracker(), translogUUID, new CompositeTranslogEventListener(Arrays.asList(internalTranslogEventListener, translogEventListener), shardId), - this::ensureOpen + this::ensureOpen, + engineConfig.getTranslogFactory() ); this.translogManager = translogManagerRef; this.softDeletesPolicy = newSoftDeletesPolicy(); diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java index 1f9306cf51e1e..6f5b7030ed65f 100644 --- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java @@ -101,7 +101,8 @@ public void onAfterTranslogSync() { } } }, - this + this, + engineConfig.getTranslogFactory() ); this.translogManager = translogManagerRef; } catch (IOException e) { @@ -128,6 +129,23 @@ public synchronized void updateSegments(final SegmentInfos infos, long seqNo) th localCheckpointTracker.fastForwardProcessedSeqNo(seqNo); } + /** + * Persist the latest live SegmentInfos. + * + * This method creates a commit point from the latest SegmentInfos. It is intended to be used when this shard is about to be promoted as the new primary. + * + * TODO: If this method is invoked while the engine is currently updating segments on its reader, wait for that update to complete so the updated segments are used. + * + * + * @throws IOException - When there is an IO error committing the SegmentInfos. + */ + public void commitSegmentInfos() throws IOException { + // TODO: This method should wait for replication events to finalize. + final SegmentInfos latestSegmentInfos = getLatestSegmentInfos(); + store.commitSegmentInfos(latestSegmentInfos, localCheckpointTracker.getMaxSeqNo(), localCheckpointTracker.getProcessedCheckpoint()); + translogManager.syncTranslog(); + } + @Override public String getHistoryUUID() { return loadHistoryUUID(lastCommittedSegmentInfos.userData); diff --git a/server/src/main/java/org/opensearch/index/engine/NoOpEngine.java b/server/src/main/java/org/opensearch/index/engine/NoOpEngine.java index 4bc37084675ea..f6c5bf7640a73 100644 --- a/server/src/main/java/org/opensearch/index/engine/NoOpEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/NoOpEngine.java @@ -195,14 +195,15 @@ public void trimUnreferencedTranslogFiles() throws TranslogException { final TranslogDeletionPolicy translogDeletionPolicy = new DefaultTranslogDeletionPolicy(-1, -1, 0); translogDeletionPolicy.setLocalCheckpointOfSafeCommit(localCheckpoint); try ( - Translog translog = new Translog( - translogConfig, - translogUuid, - translogDeletionPolicy, - engineConfig.getGlobalCheckpointSupplier(), - engineConfig.getPrimaryTermSupplier(), - seqNo -> {} - ) + Translog translog = engineConfig.getTranslogFactory() + .newTranslog( + translogConfig, + translogUuid, + translogDeletionPolicy, + engineConfig.getGlobalCheckpointSupplier(), + engineConfig.getPrimaryTermSupplier(), + seqNo -> {} + ) ) { translog.trimUnreferencedReaders(); // refresh the translog stats diff --git a/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java index cebe262fee5d1..f426768119c1d 100644 --- a/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java @@ -258,14 +258,15 @@ private static TranslogStats translogStats(final EngineConfig config, final Segm final long localCheckpoint = Long.parseLong(infos.getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)); translogDeletionPolicy.setLocalCheckpointOfSafeCommit(localCheckpoint); try ( - Translog translog = new Translog( - translogConfig, - translogUuid, - translogDeletionPolicy, - config.getGlobalCheckpointSupplier(), - config.getPrimaryTermSupplier(), - seqNo -> {} - ) + Translog translog = config.getTranslogFactory() + .newTranslog( + translogConfig, + translogUuid, + translogDeletionPolicy, + config.getGlobalCheckpointSupplier(), + config.getPrimaryTermSupplier(), + seqNo -> {} + ) ) { return translog.stats(); } diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java index e816b366c3153..d7ce6ae8aba3e 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java @@ -40,6 +40,7 @@ import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.Scorable; import org.apache.lucene.search.SortField; +import org.apache.lucene.search.comparators.TermOrdValComparator; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BytesRef; import org.opensearch.common.util.BigArrays; @@ -99,7 +100,7 @@ public FieldComparator newComparator(String fieldname, int numHits, boolean e final boolean sortMissingLast = sortMissingLast(missingValue) ^ reversed; final BytesRef missingBytes = (BytesRef) missingObject(missingValue, reversed); if (indexFieldData instanceof IndexOrdinalsFieldData) { - return new FieldComparator.TermOrdValComparator(numHits, null, sortMissingLast) { + FieldComparator cmp = new TermOrdValComparator(numHits, indexFieldData.getFieldName(), sortMissingLast, reversed) { @Override protected SortedDocValues getSortedDocValues(LeafReaderContext context, String field) throws IOException { @@ -121,13 +122,9 @@ protected SortedDocValues getSortedDocValues(LeafReaderContext context, String f return new ReplaceMissing(selectedValues, missingBytes); } } - - @Override - public void setScorer(Scorable scorer) { - BytesRefFieldComparatorSource.this.setScorer(scorer); - } - }; + cmp.disableSkipping(); + return cmp; } return new FieldComparator.TermValComparator(numHits, null, sortMissingLast) { diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java index fcbca6049ec0b..1702c7700cf60 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java @@ -380,6 +380,10 @@ protected static void parseProperties(ObjectMapper.Builder objBuilder, Map fieldBuilder = typeParser.parse(realFieldName, propNode, parserContext); for (int i = fieldNameParts.length - 2; i >= 0; --i) { diff --git a/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java b/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java index cdb7464ff250a..6d59e861eb32f 100644 --- a/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java +++ b/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java @@ -56,7 +56,6 @@ import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.automaton.RegExp; import org.opensearch.common.lucene.search.Queries; import org.opensearch.common.regex.Regex; import org.opensearch.common.unit.Fuzziness; @@ -565,7 +564,7 @@ private Query getPrefixQuerySingle(String field, String termStr) throws ParseExc if (currentFieldType == null || currentFieldType.getTextSearchInfo() == TextSearchInfo.NONE) { return newUnmappedFieldQuery(field); } - setAnalyzer(forceAnalyzer == null ? queryBuilder.context.getSearchAnalyzer(currentFieldType) : forceAnalyzer); + setAnalyzer(getSearchAnalyzer(currentFieldType)); Query query = null; if (currentFieldType.getTextSearchInfo().isTokenized() == false) { query = currentFieldType.prefixQuery(termStr, getMultiTermRewriteMethod(), context); @@ -741,6 +740,13 @@ private Query getWildcardQuerySingle(String field, String termStr) throws ParseE } } + private Analyzer getSearchAnalyzer(MappedFieldType currentFieldType) { + if (forceAnalyzer == null) { + return queryBuilder.context.getSearchAnalyzer(currentFieldType); + } + return forceAnalyzer; + } + @Override protected Query getRegexpQuery(String field, String termStr) throws ParseException { final int maxAllowedRegexLength = context.getIndexSettings().getMaxRegexLength(); @@ -781,11 +787,8 @@ private Query getRegexpQuerySingle(String field, String termStr) throws ParseExc if (currentFieldType == null) { return newUnmappedFieldQuery(field); } - if (forceAnalyzer != null) { - setAnalyzer(forceAnalyzer); - return super.getRegexpQuery(field, termStr); - } - return currentFieldType.regexpQuery(termStr, RegExp.ALL, 0, getDeterminizeWorkLimit(), getMultiTermRewriteMethod(), context); + setAnalyzer(getSearchAnalyzer(currentFieldType)); + return super.getRegexpQuery(field, termStr); } catch (RuntimeException e) { if (lenient) { return newLenientFieldQuery(field, e); diff --git a/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java index ac6754bf6a74a..fb046e2310d93 100644 --- a/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java @@ -40,7 +40,7 @@ public void beforeRefresh() throws IOException { @Override public void afterRefresh(boolean didRefresh) throws IOException { - if (didRefresh) { + if (didRefresh && shard.state() != IndexShardState.CLOSED && shard.getReplicationTracker().isPrimaryMode()) { publisher.publish(shard); } } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index b05eec4304cd5..67a8e691fda0d 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -151,6 +151,7 @@ import org.opensearch.index.store.StoreStats; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.TranslogConfig; +import org.opensearch.index.translog.TranslogFactory; import org.opensearch.index.translog.TranslogRecoveryRunner; import org.opensearch.index.translog.TranslogStats; import org.opensearch.index.warmer.ShardIndexWarmerService; @@ -241,6 +242,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl private final GlobalCheckpointListeners globalCheckpointListeners; private final PendingReplicationActions pendingReplicationActions; private final ReplicationTracker replicationTracker; + private final SegmentReplicationCheckpointPublisher checkpointPublisher; protected volatile ShardRouting shardRouting; protected volatile IndexShardState state; @@ -305,9 +307,8 @@ Runnable getGlobalCheckpointSyncer() { private final AtomicReference pendingRefreshLocation = new AtomicReference<>(); private final RefreshPendingLocationListener refreshPendingLocationListener; private volatile boolean useRetentionLeasesInPeerRecovery; - private final ReferenceManager.RefreshListener checkpointRefreshListener; - private final Store remoteStore; + private final TranslogFactory translogFactory; public IndexShard( final ShardRouting shardRouting, @@ -330,6 +331,7 @@ public IndexShard( final Runnable globalCheckpointSyncer, final RetentionLeaseSyncer retentionLeaseSyncer, final CircuitBreakerService circuitBreakerService, + final TranslogFactory translogFactory, @Nullable final SegmentReplicationCheckpointPublisher checkpointPublisher, @Nullable final Store remoteStore ) throws IOException { @@ -414,12 +416,9 @@ public boolean shouldCache(Query query) { persistMetadata(path, indexSettings, shardRouting, null, logger); this.useRetentionLeasesInPeerRecovery = replicationTracker.hasAllPeerRecoveryRetentionLeases(); this.refreshPendingLocationListener = new RefreshPendingLocationListener(); - if (checkpointPublisher != null) { - this.checkpointRefreshListener = new CheckpointRefreshListener(this, checkpointPublisher); - } else { - this.checkpointRefreshListener = null; - } + this.checkpointPublisher = checkpointPublisher; this.remoteStore = remoteStore; + this.translogFactory = translogFactory; } public ThreadPool getThreadPool() { @@ -623,6 +622,11 @@ public void updateShardState( + newRouting; assert getOperationPrimaryTerm() == newPrimaryTerm; try { + if (indexSettings.isSegRepEnabled()) { + // this Shard's engine was read only, we need to update its engine before restoring local history from xlog. + assert newRouting.primary() && currentRouting.primary() == false; + promoteNRTReplicaToPrimary(); + } replicationTracker.activatePrimaryMode(getLocalCheckpoint()); ensurePeerRecoveryRetentionLeasesExist(); /* @@ -1396,10 +1400,13 @@ public GatedCloseable acquireSafeIndexCommit() throws EngineExcepti } /** - * Returns the lastest Replication Checkpoint that shard received. Shards will return an EMPTY checkpoint before - * the engine is opened. + * Returns the latest ReplicationCheckpoint that shard received. + * @return EMPTY checkpoint before the engine is opened and null for non-segrep enabled indices */ public ReplicationCheckpoint getLatestReplicationCheckpoint() { + if (indexSettings.isSegRepEnabled() == false) { + return null; + } if (getEngineOrNull() == null) { return ReplicationCheckpoint.empty(shardId); } @@ -1431,6 +1438,10 @@ public final boolean shouldProcessCheckpoint(ReplicationCheckpoint requestCheckp logger.trace(() -> new ParameterizedMessage("Ignoring new replication checkpoint - shard is not started {}", state())); return false; } + if (getReplicationTracker().isPrimaryMode()) { + logger.warn("Ignoring new replication checkpoint - shard is in primaryMode and cannot receive any checkpoints."); + return false; + } ReplicationCheckpoint localCheckpoint = getLatestReplicationCheckpoint(); if (localCheckpoint.isAheadOf(requestCheckpoint)) { logger.trace( @@ -3220,8 +3231,8 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) thro Directory remoteDirectory = ((FilterDirectory) ((FilterDirectory) remoteStore.directory()).getDelegate()).getDelegate(); internalRefreshListener.add(new RemoteStoreRefreshListener(store.directory(), remoteDirectory)); } - if (this.checkpointRefreshListener != null) { - internalRefreshListener.add(checkpointRefreshListener); + if (this.checkpointPublisher != null && indexSettings.isSegRepEnabled() && shardRouting.primary()) { + internalRefreshListener.add(new CheckpointRefreshListener(this, this.checkpointPublisher)); } return this.engineConfigFactory.newEngineConfig( @@ -3247,7 +3258,8 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) thro replicationTracker::getRetentionLeases, () -> getOperationPrimaryTerm(), tombstoneDocSupplier(), - indexSettings.isSegRepEnabled() && shardRouting.primary() == false + indexSettings.isSegRepEnabled() && shardRouting.primary() == false, + translogFactory ); } @@ -4111,4 +4123,26 @@ RetentionLeaseSyncer getRetentionLeaseSyncer() { public GatedCloseable getSegmentInfosSnapshot() { return getEngine().getSegmentInfosSnapshot(); } + + /** + * With segment replication enabled - prepare the shard's engine to be promoted as the new primary. + * + * If this shard is currently using a replication engine, this method: + * 1. Invokes {@link NRTReplicationEngine#commitSegmentInfos()} to ensure the engine can be reopened as writeable from the latest refresh point. + * InternalEngine opens its IndexWriter from an on-disk commit point, but this replica may have recently synced from a primary's refresh point, meaning it has documents searchable in its in-memory SegmentInfos + * that are not part of a commit point. This ensures that those documents are made part of a commit and do not need to be reindexed after promotion. + * 2. Invokes resetEngineToGlobalCheckpoint - This call performs the engine swap, opening up as a writeable engine and replays any operations in the xlog. The operations indexed from xlog here will be + * any ack'd writes that were not copied to this replica before promotion. + */ + private void promoteNRTReplicaToPrimary() { + assert shardRouting.primary() && indexSettings.isSegRepEnabled(); + getReplicationEngine().ifPresentOrElse(engine -> { + try { + engine.commitSegmentInfos(); + resetEngineToGlobalCheckpoint(); + } catch (IOException e) { + throw new EngineException(shardId, "Unable to update replica to writeable engine, failing shard", e); + } + }, () -> { throw new EngineException(shardId, "Expected replica engine to be of type NRTReplicationEngine"); }); + } } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java index 855457f275122..62e2b12896411 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java @@ -33,7 +33,7 @@ * * @opensearch.internal */ -public final class RemoteDirectory extends Directory { +public class RemoteDirectory extends Directory { private final BlobContainer blobContainer; @@ -50,6 +50,16 @@ public String[] listAll() throws IOException { return blobContainer.listBlobs().keySet().stream().sorted().toArray(String[]::new); } + /** + * Returns names of files with given prefix in this directory. + * @param filenamePrefix The prefix to match against file names in the directory + * @return A list of the matching filenames in the directory + * @throws IOException if there were any failures in reading from the blob container + */ + public Collection listFilesByPrefix(String filenamePrefix) throws IOException { + return blobContainer.listBlobsByPrefix(filenamePrefix).keySet(); + } + /** * Removes an existing file in the directory. * diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java new file mode 100644 index 0000000000000..d7d6b29d08bfc --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -0,0 +1,372 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FilterDirectory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.opensearch.common.UUIDs; + +import java.io.IOException; +import java.nio.file.NoSuchFileException; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * A RemoteDirectory extension for remote segment store. We need to make sure we don't overwrite a segment file once uploaded. + * In order to prevent segment overwrite which can occur due to two primary nodes for the same shard at the same time, + * a unique suffix is added to the uploaded segment file. This class keeps track of filename of segments stored + * in remote segment store vs filename in local filesystem and provides the consistent Directory interface so that + * caller will be accessing segment files in the same way as {@code FSDirectory}. Apart from storing actual segment files, + * remote segment store also keeps track of refresh checkpoints as metadata in a separate path which is handled by + * another instance of {@code RemoteDirectory}. + * @opensearch.internal + */ +public final class RemoteSegmentStoreDirectory extends FilterDirectory { + /** + * Each segment file is uploaded with unique suffix. + * For example, _0.cfe in local filesystem will be uploaded to remote segment store as _0.cfe__gX7bNIIBrs0AUNsR2yEG + */ + public static final String SEGMENT_NAME_UUID_SEPARATOR = "__"; + + public static final MetadataFilenameUtils.MetadataFilenameComparator METADATA_FILENAME_COMPARATOR = + new MetadataFilenameUtils.MetadataFilenameComparator(); + + /** + * remoteDataDirectory is used to store segment files at path: cluster_UUID/index_UUID/shardId/segments/data + */ + private final RemoteDirectory remoteDataDirectory; + /** + * remoteMetadataDirectory is used to store metadata files at path: cluster_UUID/index_UUID/shardId/segments/metadata + */ + private final RemoteDirectory remoteMetadataDirectory; + + /** + * To prevent explosion of refresh metadata files, we replace refresh files for the given primary term and generation + * This is achieved by uploading refresh metadata file with the same UUID suffix. + */ + private String metadataFileUniqueSuffix; + + /** + * Keeps track of local segment filename to uploaded filename along with other attributes like checksum. + * This map acts as a cache layer for uploaded segment filenames which helps avoid calling listAll() each time. + * It is important to initialize this map on creation of RemoteSegmentStoreDirectory and update it on each upload and delete. + */ + private Map segmentsUploadedToRemoteStore; + + private static final Logger logger = LogManager.getLogger(RemoteSegmentStoreDirectory.class); + + public RemoteSegmentStoreDirectory(RemoteDirectory remoteDataDirectory, RemoteDirectory remoteMetadataDirectory) throws IOException { + super(remoteDataDirectory); + this.remoteDataDirectory = remoteDataDirectory; + this.remoteMetadataDirectory = remoteMetadataDirectory; + init(); + } + + /** + * Initializes the cache which keeps track of all the segment files uploaded to the remote segment store. + * As this cache is specific to an instance of RemoteSegmentStoreDirectory, it is possible that cache becomes stale + * if another instance of RemoteSegmentStoreDirectory is used to upload/delete segment files. + * It is caller's responsibility to call init() again to ensure that cache is properly updated. + * @throws IOException if there were any failures in reading the metadata file + */ + public void init() throws IOException { + this.metadataFileUniqueSuffix = UUIDs.base64UUID(); + this.segmentsUploadedToRemoteStore = new ConcurrentHashMap<>(readLatestMetadataFile()); + } + + /** + * Read the latest metadata file to get the list of segments uploaded to the remote segment store. + * We upload a metadata file per refresh, but it is not unique per refresh. Refresh metadata file is unique for a given commit. + * The format of refresh metadata filename is: refresh_metadata__PrimaryTerm__Generation__UUID + * Refresh metadata files keep track of active segments for the shard at the time of refresh. + * In order to get the list of segment files uploaded to the remote segment store, we need to read the latest metadata file. + * Each metadata file contains a map where + * Key is - Segment local filename and + * Value is - local filename::uploaded filename::checksum + * @return Map of segment filename to uploaded filename with checksum + * @throws IOException if there were any failures in reading the metadata file + */ + private Map readLatestMetadataFile() throws IOException { + Map segmentMetadataMap = new HashMap<>(); + + Collection metadataFiles = remoteMetadataDirectory.listFilesByPrefix(MetadataFilenameUtils.METADATA_PREFIX); + Optional latestMetadataFile = metadataFiles.stream().max(METADATA_FILENAME_COMPARATOR); + + if (latestMetadataFile.isPresent()) { + logger.info("Reading latest Metadata file {}", latestMetadataFile.get()); + segmentMetadataMap = readMetadataFile(latestMetadataFile.get()); + } else { + logger.info("No metadata file found, this can happen for new index with no data uploaded to remote segment store"); + } + + return segmentMetadataMap; + } + + private Map readMetadataFile(String metadataFilename) throws IOException { + try (IndexInput indexInput = remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)) { + Map segmentMetadata = indexInput.readMapOfStrings(); + return segmentMetadata.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> UploadedSegmentMetadata.fromString(entry.getValue()))); + } + } + + /** + * Metadata of a segment that is uploaded to remote segment store. + */ + static class UploadedSegmentMetadata { + private static final String SEPARATOR = "::"; + private final String originalFilename; + private final String uploadedFilename; + private final String checksum; + + UploadedSegmentMetadata(String originalFilename, String uploadedFilename, String checksum) { + this.originalFilename = originalFilename; + this.uploadedFilename = uploadedFilename; + this.checksum = checksum; + } + + @Override + public String toString() { + return String.join(SEPARATOR, originalFilename, uploadedFilename, checksum); + } + + public static UploadedSegmentMetadata fromString(String uploadedFilename) { + String[] values = uploadedFilename.split(SEPARATOR); + return new UploadedSegmentMetadata(values[0], values[1], values[2]); + } + } + + /** + * Contains utility methods that provide various parts of metadata filename along with comparator + * Each metadata filename is of format: PREFIX__PrimaryTerm__Generation__UUID + */ + static class MetadataFilenameUtils { + public static final String SEPARATOR = "__"; + public static final String METADATA_PREFIX = "metadata"; + + /** + * Comparator to sort the metadata filenames. The order of sorting is: Primary Term, Generation, UUID + * Even though UUID sort does not provide any info on recency, it provides a consistent way to sort the filenames. + */ + static class MetadataFilenameComparator implements Comparator { + @Override + public int compare(String first, String second) { + String[] firstTokens = first.split(SEPARATOR); + String[] secondTokens = second.split(SEPARATOR); + if (!firstTokens[0].equals(secondTokens[0])) { + return firstTokens[0].compareTo(secondTokens[0]); + } + long firstPrimaryTerm = getPrimaryTerm(firstTokens); + long secondPrimaryTerm = getPrimaryTerm(secondTokens); + if (firstPrimaryTerm != secondPrimaryTerm) { + return firstPrimaryTerm > secondPrimaryTerm ? 1 : -1; + } else { + long firstGeneration = getGeneration(firstTokens); + long secondGeneration = getGeneration(secondTokens); + if (firstGeneration != secondGeneration) { + return firstGeneration > secondGeneration ? 1 : -1; + } else { + return getUuid(firstTokens).compareTo(getUuid(secondTokens)); + } + } + } + } + + // Visible for testing + static String getMetadataFilename(long primaryTerm, long generation, String uuid) { + return String.join( + SEPARATOR, + METADATA_PREFIX, + Long.toString(primaryTerm), + Long.toString(generation, Character.MAX_RADIX), + uuid + ); + } + + // Visible for testing + static long getPrimaryTerm(String[] filenameTokens) { + return Long.parseLong(filenameTokens[1]); + } + + // Visible for testing + static long getGeneration(String[] filenameTokens) { + return Long.parseLong(filenameTokens[2], Character.MAX_RADIX); + } + + // Visible for testing + static String getUuid(String[] filenameTokens) { + return filenameTokens[3]; + } + } + + /** + * Returns list of all the segment files uploaded to remote segment store till the last refresh checkpoint. + * Any segment file that is uploaded without corresponding metadata file will not be visible as part of listAll(). + * We chose not to return cache entries for listAll as cache can have entries for stale segments as well. + * Even if we plan to delete stale segments from remote segment store, it will be a periodic operation. + * @return segment filenames stored in remote segment store + * @throws IOException if there were any failures in reading the metadata file + */ + @Override + public String[] listAll() throws IOException { + return readLatestMetadataFile().keySet().toArray(new String[0]); + } + + /** + * Delete segment file from remote segment store. + * @param name the name of an existing segment file in local filesystem. + * @throws IOException if the file exists but could not be deleted. + */ + @Override + public void deleteFile(String name) throws IOException { + String remoteFilename = getExistingRemoteFilename(name); + if (remoteFilename != null) { + remoteDataDirectory.deleteFile(remoteFilename); + segmentsUploadedToRemoteStore.remove(name); + } + } + + /** + * Returns the byte length of a segment file in the remote segment store. + * @param name the name of an existing segment file in local filesystem. + * @throws IOException in case of I/O error + * @throws NoSuchFileException if the file does not exist in the cache or remote segment store + */ + @Override + public long fileLength(String name) throws IOException { + String remoteFilename = getExistingRemoteFilename(name); + if (remoteFilename != null) { + return remoteDataDirectory.fileLength(remoteFilename); + } else { + throw new NoSuchFileException(name); + } + } + + /** + * Creates and returns a new instance of {@link RemoteIndexOutput} which will be used to copy files to the remote + * segment store. + * @param name the name of the file to create. + * @throws IOException in case of I/O error + */ + @Override + public IndexOutput createOutput(String name, IOContext context) throws IOException { + return remoteDataDirectory.createOutput(getNewRemoteSegmentFilename(name), context); + } + + /** + * Opens a stream for reading an existing file and returns {@link RemoteIndexInput} enclosing the stream. + * @param name the name of an existing file. + * @throws IOException in case of I/O error + * @throws NoSuchFileException if the file does not exist either in cache or remote segment store + */ + @Override + public IndexInput openInput(String name, IOContext context) throws IOException { + String remoteFilename = getExistingRemoteFilename(name); + if (remoteFilename != null) { + return remoteDataDirectory.openInput(remoteFilename, context); + } else { + throw new NoSuchFileException(name); + } + } + + /** + * Copies an existing src file from directory from to a non-existent file dest in this directory. + * Once the segment is uploaded to remote segment store, update the cache accordingly. + */ + @Override + public void copyFrom(Directory from, String src, String dest, IOContext context) throws IOException { + String remoteFilename = getNewRemoteSegmentFilename(dest); + remoteDataDirectory.copyFrom(from, src, remoteFilename, context); + String checksum = getChecksumOfLocalFile(from, src); + UploadedSegmentMetadata segmentMetadata = new UploadedSegmentMetadata(src, remoteFilename, checksum); + segmentsUploadedToRemoteStore.put(src, segmentMetadata); + } + + /** + * Checks if the file exists in the uploadedSegments cache and the checksum matches. + * It is important to match the checksum as the same segment filename can be used for different + * segments due to a concurrency issue. + * @param localFilename filename of segment stored in local filesystem + * @param checksum checksum of the segment file + * @return true if file exists in cache and checksum matches. + */ + public boolean containsFile(String localFilename, String checksum) { + return segmentsUploadedToRemoteStore.containsKey(localFilename) + && segmentsUploadedToRemoteStore.get(localFilename).checksum.equals(checksum); + } + + /** + * Upload metadata file + * @param segmentFiles segment files that are part of the shard at the time of the latest refresh + * @param storeDirectory instance of local directory to temporarily create metadata file before upload + * @param primaryTerm primary term to be used in the name of metadata file + * @param generation commit generation + * @throws IOException in case of I/O error while uploading the metadata file + */ + public void uploadMetadata(Collection segmentFiles, Directory storeDirectory, long primaryTerm, long generation) + throws IOException { + synchronized (this) { + String metadataFilename = MetadataFilenameUtils.getMetadataFilename(primaryTerm, generation, this.metadataFileUniqueSuffix); + IndexOutput indexOutput = storeDirectory.createOutput(metadataFilename, IOContext.DEFAULT); + Map uploadedSegments = new HashMap<>(); + for (String file : segmentFiles) { + if (segmentsUploadedToRemoteStore.containsKey(file)) { + uploadedSegments.put(file, segmentsUploadedToRemoteStore.get(file).toString()); + } else { + throw new NoSuchFileException(file); + } + } + indexOutput.writeMapOfStrings(uploadedSegments); + indexOutput.close(); + storeDirectory.sync(Collections.singleton(metadataFilename)); + remoteMetadataDirectory.copyFrom(storeDirectory, metadataFilename, metadataFilename, IOContext.DEFAULT); + storeDirectory.deleteFile(metadataFilename); + } + } + + private String getChecksumOfLocalFile(Directory directory, String file) throws IOException { + try (IndexInput indexInput = directory.openInput(file, IOContext.DEFAULT)) { + return Long.toString(CodecUtil.retrieveChecksum(indexInput)); + } + } + + private String getExistingRemoteFilename(String localFilename) { + if (segmentsUploadedToRemoteStore.containsKey(localFilename)) { + return segmentsUploadedToRemoteStore.get(localFilename).uploadedFilename; + } else { + return null; + } + } + + private String getNewRemoteSegmentFilename(String localFilename) { + return localFilename + SEGMENT_NAME_UUID_SEPARATOR + UUIDs.base64UUID(); + } + + private String getLocalSegmentFilename(String remoteFilename) { + return remoteFilename.split(SEGMENT_NAME_UUID_SEPARATOR)[0]; + } + + // Visible for testing + Map getSegmentsUploadedToRemoteStore() { + return this.segmentsUploadedToRemoteStore; + } +} diff --git a/server/src/main/java/org/opensearch/index/store/Store.java b/server/src/main/java/org/opensearch/index/store/Store.java index 6828ab7d91b2c..58598ab2d08f4 100644 --- a/server/src/main/java/org/opensearch/index/store/Store.java +++ b/server/src/main/java/org/opensearch/index/store/Store.java @@ -110,8 +110,8 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -121,6 +121,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; +import static org.opensearch.index.seqno.SequenceNumbers.LOCAL_CHECKPOINT_KEY; /** * A Store provides plain access to files written by an opensearch index shard. Each shard @@ -799,6 +800,47 @@ public void beforeClose() { shardLock.setDetails("closing shard"); } + /** + * This method should only be used with Segment Replication. + * Perform a commit from a live {@link SegmentInfos}. Replica engines with segrep do not have an IndexWriter and Lucene does not currently + * have the ability to create a writer directly from a SegmentInfos object. To promote the replica as a primary and avoid reindexing, we must first commit + * on the replica so that it can be opened with a writeable engine. Further, InternalEngine currently invokes `trimUnsafeCommits` which reverts the engine to a previous safeCommit where the max seqNo is less than or equal + * to the current global checkpoint. It is likely that the replica has a maxSeqNo that is higher than the global cp and a new commit will be wiped. + * + * To get around these limitations, this method first creates an IndexCommit directly from SegmentInfos, it then + * uses an appending IW to create an IndexCommit from the commit created on SegmentInfos. + * This ensures that 1. All files in the new commit are fsynced and 2. Deletes older commit points so the only commit to start from is our new commit. + * + * @param latestSegmentInfos {@link SegmentInfos} The latest active infos + * @param maxSeqNo The engine's current maxSeqNo + * @param processedCheckpoint The engine's current processed checkpoint. + * @throws IOException when there is an IO error committing. + */ + public void commitSegmentInfos(SegmentInfos latestSegmentInfos, long maxSeqNo, long processedCheckpoint) throws IOException { + assert indexSettings.isSegRepEnabled(); + metadataLock.writeLock().lock(); + try { + final Map userData = new HashMap<>(latestSegmentInfos.getUserData()); + userData.put(LOCAL_CHECKPOINT_KEY, String.valueOf(processedCheckpoint)); + userData.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(maxSeqNo)); + latestSegmentInfos.setUserData(userData, true); + latestSegmentInfos.commit(directory()); + + // similar to TrimUnsafeCommits, create a commit with an appending IW, this will delete old commits and ensure all files + // associated with the SegmentInfos.commit are fsynced. + final List existingCommits = DirectoryReader.listCommits(directory); + assert existingCommits.isEmpty() == false : "Expected at least one commit but none found"; + final IndexCommit lastIndexCommit = existingCommits.get(existingCommits.size() - 1); + assert latestSegmentInfos.getSegmentsFileName().equals(lastIndexCommit.getSegmentsFileName()); + try (IndexWriter writer = newAppendingIndexWriter(directory, lastIndexCommit)) { + writer.setLiveCommitData(lastIndexCommit.getUserData().entrySet()); + writer.commit(); + } + } finally { + metadataLock.writeLock().unlock(); + } + } + /** * A store directory * @@ -1102,6 +1144,30 @@ public Map asMap() { private static final String LIV_FILE_EXTENSION = "liv"; // lucene 5 delete file private static final String SEGMENT_INFO_EXTENSION = "si"; + /** + * Helper method used to group store files according to segment and commit. + * + * @see MetadataSnapshot#recoveryDiff(MetadataSnapshot) + * @see MetadataSnapshot#segmentReplicationDiff(MetadataSnapshot) + */ + private Iterable> getGroupedFilesIterable() { + final Map> perSegment = new HashMap<>(); + final List perCommitStoreFiles = new ArrayList<>(); + for (StoreFileMetadata meta : this) { + final String segmentId = IndexFileNames.parseSegmentName(meta.name()); + final String extension = IndexFileNames.getExtension(meta.name()); + if (IndexFileNames.SEGMENTS.equals(segmentId) + || DEL_FILE_EXTENSION.equals(extension) + || LIV_FILE_EXTENSION.equals(extension)) { + // only treat del files as per-commit files fnm files are generational but only for upgradable DV + perCommitStoreFiles.add(meta); + } else { + perSegment.computeIfAbsent(segmentId, k -> new ArrayList<>()).add(meta); + } + } + return Iterables.concat(perSegment.values(), Collections.singleton(perCommitStoreFiles)); + } + /** * Returns a diff between the two snapshots that can be used for recovery. The given snapshot is treated as the * recovery target and this snapshot as the source. The returned diff will hold a list of files that are: @@ -1139,23 +1205,8 @@ public RecoveryDiff recoveryDiff(MetadataSnapshot recoveryTargetSnapshot) { final List identical = new ArrayList<>(); final List different = new ArrayList<>(); final List missing = new ArrayList<>(); - final Map> perSegment = new HashMap<>(); - final List perCommitStoreFiles = new ArrayList<>(); - - for (StoreFileMetadata meta : this) { - final String segmentId = IndexFileNames.parseSegmentName(meta.name()); - final String extension = IndexFileNames.getExtension(meta.name()); - if (IndexFileNames.SEGMENTS.equals(segmentId) - || DEL_FILE_EXTENSION.equals(extension) - || LIV_FILE_EXTENSION.equals(extension)) { - // only treat del files as per-commit files fnm files are generational but only for upgradable DV - perCommitStoreFiles.add(meta); - } else { - perSegment.computeIfAbsent(segmentId, k -> new ArrayList<>()).add(meta); - } - } final ArrayList identicalFiles = new ArrayList<>(); - for (List segmentFiles : Iterables.concat(perSegment.values(), Collections.singleton(perCommitStoreFiles))) { + for (List segmentFiles : getGroupedFilesIterable()) { identicalFiles.clear(); boolean consistent = true; for (StoreFileMetadata meta : segmentFiles) { @@ -1190,6 +1241,51 @@ public RecoveryDiff recoveryDiff(MetadataSnapshot recoveryTargetSnapshot) { return recoveryDiff; } + /** + * Segment Replication method + * Returns a diff between the two snapshots that can be used for getting list of files to copy over to a replica for segment replication. The given snapshot is treated as the + * target and this snapshot as the source. The returned diff will hold a list of files that are: + *
    + *
  • identical: they exist in both snapshots and they can be considered the same ie. they don't need to be recovered
    • + *
  • different: they exist in both snapshots but their they are not identical
    • + *
  • missing: files that exist in the source but not in the target
    • + *
+ */ + public RecoveryDiff segmentReplicationDiff(MetadataSnapshot recoveryTargetSnapshot) { + final List identical = new ArrayList<>(); + final List different = new ArrayList<>(); + final List missing = new ArrayList<>(); + final ArrayList identicalFiles = new ArrayList<>(); + for (List segmentFiles : getGroupedFilesIterable()) { + identicalFiles.clear(); + boolean consistent = true; + for (StoreFileMetadata meta : segmentFiles) { + StoreFileMetadata storeFileMetadata = recoveryTargetSnapshot.get(meta.name()); + if (storeFileMetadata == null) { + // Do not consider missing files as inconsistent in SegRep as replicas may lag while primary updates + // documents and generate new files specific to a segment + missing.add(meta); + } else if (storeFileMetadata.isSame(meta) == false) { + consistent = false; + different.add(meta); + } else { + identicalFiles.add(meta); + } + } + if (consistent) { + identical.addAll(identicalFiles); + } else { + different.addAll(identicalFiles); + } + } + RecoveryDiff recoveryDiff = new RecoveryDiff( + Collections.unmodifiableList(identical), + Collections.unmodifiableList(different), + Collections.unmodifiableList(missing) + ); + return recoveryDiff; + } + /** * Returns the number of files in this snapshot */ diff --git a/server/src/main/java/org/opensearch/index/translog/InternalTranslogFactory.java b/server/src/main/java/org/opensearch/index/translog/InternalTranslogFactory.java new file mode 100644 index 0000000000000..566eda4fe4a6e --- /dev/null +++ b/server/src/main/java/org/opensearch/index/translog/InternalTranslogFactory.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.translog; + +import java.io.IOException; +import java.util.function.LongConsumer; +import java.util.function.LongSupplier; + +/** + * Translog Factory for the local on-disk {@link Translog} + * + * @opensearch.internal + */ +public class InternalTranslogFactory implements TranslogFactory { + + @Override + public Translog newTranslog( + TranslogConfig translogConfig, + String translogUUID, + TranslogDeletionPolicy translogDeletionPolicy, + LongSupplier globalCheckpointSupplier, + LongSupplier primaryTermSupplier, + LongConsumer persistedSequenceNumberConsumer + ) throws IOException { + + return new Translog( + translogConfig, + translogUUID, + translogDeletionPolicy, + globalCheckpointSupplier, + primaryTermSupplier, + persistedSequenceNumberConsumer + ); + } +} diff --git a/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java b/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java index ac82cf246cc55..fd52e02132006 100644 --- a/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java +++ b/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java @@ -54,7 +54,8 @@ public InternalTranslogManager( Supplier localCheckpointTrackerSupplier, String translogUUID, TranslogEventListener translogEventListener, - LifecycleAware engineLifeCycleAware + LifecycleAware engineLifeCycleAware, + TranslogFactory translogFactory ) throws IOException { this.shardId = shardId; this.readLock = readLock; @@ -67,7 +68,7 @@ public InternalTranslogManager( if (tracker != null) { tracker.markSeqNoAsPersisted(seqNo); } - }, translogUUID); + }, translogUUID, translogFactory); assert translog.getGeneration() != null; this.translog = translog; assert pendingTranslogRecovery.get() == false : "translog recovery can't be pending before we set it"; @@ -333,10 +334,11 @@ protected Translog openTranslog( TranslogDeletionPolicy translogDeletionPolicy, LongSupplier globalCheckpointSupplier, LongConsumer persistedSequenceNumberConsumer, - String translogUUID + String translogUUID, + TranslogFactory translogFactory ) throws IOException { - return new Translog( + return translogFactory.newTranslog( translogConfig, translogUUID, translogDeletionPolicy, diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogFactory.java b/server/src/main/java/org/opensearch/index/translog/TranslogFactory.java new file mode 100644 index 0000000000000..5500bda99808d --- /dev/null +++ b/server/src/main/java/org/opensearch/index/translog/TranslogFactory.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.translog; + +import java.io.IOException; +import java.util.function.LongConsumer; +import java.util.function.LongSupplier; + +/** + * Translog Factory to enable creation of various local on-disk + * and remote store flavors of {@link Translog} + * + * @opensearch.internal + */ +@FunctionalInterface +public interface TranslogFactory { + + Translog newTranslog( + final TranslogConfig config, + final String translogUUID, + final TranslogDeletionPolicy deletionPolicy, + final LongSupplier globalCheckpointSupplier, + final LongSupplier primaryTermSupplier, + final LongConsumer persistedSequenceNumberConsumer + ) throws IOException; +} diff --git a/server/src/main/java/org/opensearch/index/translog/WriteOnlyTranslogManager.java b/server/src/main/java/org/opensearch/index/translog/WriteOnlyTranslogManager.java index 09f5f38a9f6a9..96a2dd05851c0 100644 --- a/server/src/main/java/org/opensearch/index/translog/WriteOnlyTranslogManager.java +++ b/server/src/main/java/org/opensearch/index/translog/WriteOnlyTranslogManager.java @@ -35,7 +35,8 @@ public WriteOnlyTranslogManager( Supplier localCheckpointTrackerSupplier, String translogUUID, TranslogEventListener translogEventListener, - LifecycleAware engineLifecycleAware + LifecycleAware engineLifecycleAware, + TranslogFactory translogFactory ) throws IOException { super( translogConfig, @@ -47,7 +48,8 @@ public WriteOnlyTranslogManager( localCheckpointTrackerSupplier, translogUUID, translogEventListener, - engineLifecycleAware + engineLifecycleAware, + translogFactory ); } diff --git a/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java b/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java index a9b032c98b70f..dfebe5f7cabf2 100644 --- a/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java +++ b/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java @@ -24,7 +24,10 @@ import java.io.IOException; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.function.Predicate; +import java.util.stream.Collectors; /** * Manages references to ongoing segrep events on a node. @@ -38,7 +41,7 @@ class OngoingSegmentReplications { private final RecoverySettings recoverySettings; private final IndicesService indicesService; private final Map copyStateMap; - private final Map nodesToHandlers; + private final Map allocationIdToHandlers; /** * Constructor. @@ -50,7 +53,7 @@ class OngoingSegmentReplications { this.indicesService = indicesService; this.recoverySettings = recoverySettings; this.copyStateMap = Collections.synchronizedMap(new HashMap<>()); - this.nodesToHandlers = ConcurrentCollections.newConcurrentMap(); + this.allocationIdToHandlers = ConcurrentCollections.newConcurrentMap(); } /** @@ -96,8 +99,7 @@ synchronized CopyState getCachedCopyState(ReplicationCheckpoint checkpoint) thro * @param listener {@link ActionListener} that resolves when sending files is complete. */ void startSegmentCopy(GetSegmentFilesRequest request, ActionListener listener) { - final DiscoveryNode node = request.getTargetNode(); - final SegmentReplicationSourceHandler handler = nodesToHandlers.get(node); + final SegmentReplicationSourceHandler handler = allocationIdToHandlers.get(request.getTargetAllocationId()); if (handler != null) { if (handler.isReplicating()) { throw new OpenSearchException( @@ -108,7 +110,7 @@ void startSegmentCopy(GetSegmentFilesRequest request, ActionListener wrappedListener = ActionListener.runBefore(listener, () -> { - final SegmentReplicationSourceHandler sourceHandler = nodesToHandlers.remove(node); + final SegmentReplicationSourceHandler sourceHandler = allocationIdToHandlers.remove(request.getTargetAllocationId()); if (sourceHandler != null) { removeCopyState(sourceHandler.getCopyState()); } @@ -123,19 +125,6 @@ void startSegmentCopy(GetSegmentFilesRequest request, ActionListener handler.getCopyState().getShard().shardId().equals(shard.shardId()), reason); + } + + /** + * Cancel any ongoing replications for a given {@link DiscoveryNode} + * + * @param node {@link DiscoveryNode} node for which to cancel replication events. + */ + void cancelReplication(DiscoveryNode node) { + cancelHandlers(handler -> handler.getTargetNode().equals(node), "Node left"); + } /** @@ -186,19 +180,25 @@ boolean isInCopyStateMap(ReplicationCheckpoint replicationCheckpoint) { } int size() { - return nodesToHandlers.size(); + return allocationIdToHandlers.size(); } int cachedCopyStateSize() { return copyStateMap.size(); } - private SegmentReplicationSourceHandler createTargetHandler(DiscoveryNode node, CopyState copyState, FileChunkWriter fileChunkWriter) { + private SegmentReplicationSourceHandler createTargetHandler( + DiscoveryNode node, + CopyState copyState, + String allocationId, + FileChunkWriter fileChunkWriter + ) { return new SegmentReplicationSourceHandler( node, fileChunkWriter, copyState.getShard().getThreadPool(), copyState, + allocationId, Math.toIntExact(recoverySettings.getChunkSize().getBytes()), recoverySettings.getMaxConcurrentFileChunks() ); @@ -231,4 +231,23 @@ private synchronized void removeCopyState(CopyState copyState) { copyStateMap.remove(copyState.getRequestedReplicationCheckpoint()); } } + + /** + * Remove handlers from allocationIdToHandlers map based on a filter predicate. + * This will also decref the handler's CopyState reference. + */ + private void cancelHandlers(Predicate predicate, String reason) { + final List allocationIds = allocationIdToHandlers.values() + .stream() + .filter(predicate) + .map(SegmentReplicationSourceHandler::getAllocationId) + .collect(Collectors.toList()); + for (String allocationId : allocationIds) { + final SegmentReplicationSourceHandler handler = allocationIdToHandlers.remove(allocationId); + if (handler != null) { + handler.cancel(reason); + removeCopyState(handler.getCopyState()); + } + } + } } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java index 8911302a722f5..2d21653c1924c 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java @@ -27,6 +27,7 @@ import org.opensearch.indices.recovery.FileChunkWriter; import org.opensearch.indices.recovery.MultiChunkTransfer; import org.opensearch.indices.replication.common.CopyState; +import org.opensearch.indices.replication.common.ReplicationTimer; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transports; @@ -54,6 +55,8 @@ class SegmentReplicationSourceHandler { private final List resources = new CopyOnWriteArrayList<>(); private final Logger logger; private final AtomicBoolean isReplicating = new AtomicBoolean(); + private final DiscoveryNode targetNode; + private final String allocationId; /** * Constructor. @@ -70,9 +73,11 @@ class SegmentReplicationSourceHandler { FileChunkWriter writer, ThreadPool threadPool, CopyState copyState, + String allocationId, int fileChunkSizeInBytes, int maxConcurrentFileChunks ) { + this.targetNode = targetNode; this.shard = copyState.getShard(); this.logger = Loggers.getLogger( SegmentReplicationSourceHandler.class, @@ -89,6 +94,7 @@ class SegmentReplicationSourceHandler { fileChunkSizeInBytes, maxConcurrentFileChunks ); + this.allocationId = allocationId; this.copyState = copyState; } @@ -99,16 +105,24 @@ class SegmentReplicationSourceHandler { * @param listener {@link ActionListener} that completes with the list of files sent. */ public synchronized void sendFiles(GetSegmentFilesRequest request, ActionListener listener) { + final ReplicationTimer timer = new ReplicationTimer(); if (isReplicating.compareAndSet(false, true) == false) { throw new OpenSearchException("Replication to {} is already running.", shard.shardId()); } future.addListener(listener, OpenSearchExecutors.newDirectExecutorService()); final Closeable releaseResources = () -> IOUtils.close(resources); try { - + timer.start(); final Consumer onFailure = e -> { assert Transports.assertNotTransportThread(SegmentReplicationSourceHandler.this + "[onFailure]"); IOUtils.closeWhileHandlingException(releaseResources, () -> future.onFailure(e)); + timer.stop(); + logger.trace( + "[replication id {}] Source node failed to send files to target node [{}], timing: {}", + request.getReplicationId(), + request.getTargetNode().getId(), + timer.time() + ); }; RunUnderPrimaryPermit.run(() -> { @@ -118,7 +132,7 @@ public synchronized void sendFiles(GetSegmentFilesRequest request, ActionListene logger.debug( "delaying replication of {} as it is not listed as assigned to target node {}", shard.shardId(), - request.getTargetNode() + targetNode ); throw new DelayRecoveryException("source node does not have the shard listed in its state as allocated on the node"); } @@ -142,18 +156,17 @@ public synchronized void sendFiles(GetSegmentFilesRequest request, ActionListene transfer.start(); sendFileStep.whenComplete(r -> { - final String targetAllocationId = request.getTargetAllocationId(); - RunUnderPrimaryPermit.run( - () -> shard.markAllocationIdAsInSync(targetAllocationId, request.getCheckpoint().getSeqNo()), - shard.shardId() + " marking " + targetAllocationId + " as in sync", - shard, - cancellableThreads, - logger - ); try { future.onResponse(new GetSegmentFilesResponse(List.of(storeFileMetadata))); } finally { IOUtils.close(resources); + timer.stop(); + logger.trace( + "[replication id {}] Source node completed sending files to target node [{}], timing: {}", + request.getReplicationId(), + request.getTargetNode().getId(), + timer.time() + ); } }, onFailure); } catch (Exception e) { @@ -175,4 +188,12 @@ CopyState getCopyState() { public boolean isReplicating() { return isReplicating.get(); } + + public DiscoveryNode getTargetNode() { + return targetNode; + } + + public String getAllocationId() { + return allocationId; + } } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java index d428459884f97..0cee731fde2cb 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.support.ChannelActionListener; import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.cluster.ClusterStateListener; @@ -25,6 +26,7 @@ import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.recovery.RetryableTransportClient; import org.opensearch.indices.replication.common.CopyState; +import org.opensearch.indices.replication.common.ReplicationTimer; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportChannel; @@ -86,6 +88,8 @@ public SegmentReplicationSourceService( private class CheckpointInfoRequestHandler implements TransportRequestHandler { @Override public void messageReceived(CheckpointInfoRequest request, TransportChannel channel, Task task) throws Exception { + final ReplicationTimer timer = new ReplicationTimer(); + timer.start(); final RemoteSegmentFileChunkWriter segmentSegmentFileChunkWriter = new RemoteSegmentFileChunkWriter( request.getReplicationId(), recoverySettings, @@ -109,6 +113,16 @@ public void messageReceived(CheckpointInfoRequest request, TransportChannel chan copyState.getPendingDeleteFiles() ) ); + timer.stop(); + logger.trace( + new ParameterizedMessage( + "[replication id {}] Source node sent checkpoint info [{}] to target node [{}], timing: {}", + request.getReplicationId(), + copyState.getCheckpoint(), + request.getTargetNode().getId(), + timer.time() + ) + ); } } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java index 838c06a4785ef..f865ba1332186 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java @@ -8,10 +8,14 @@ package org.opensearch.indices.replication; +import org.opensearch.common.collect.Tuple; import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.indices.replication.common.ReplicationState; import org.opensearch.indices.replication.common.ReplicationTimer; +import java.util.ArrayList; +import java.util.List; + /** * ReplicationState implementation to track Segment Replication events. * @@ -26,10 +30,12 @@ public class SegmentReplicationState implements ReplicationState { */ public enum Stage { DONE((byte) 0), - INIT((byte) 1), - - REPLICATING((byte) 2); + REPLICATING((byte) 2), + GET_CHECKPOINT_INFO((byte) 3), + FILE_DIFF((byte) 4), + GET_FILES((byte) 5), + FINALIZE_REPLICATION((byte) 6); private static final Stage[] STAGES = new Stage[Stage.values().length]; @@ -60,13 +66,27 @@ public static Stage fromId(byte id) { private Stage stage; private final ReplicationLuceneIndex index; - private final ReplicationTimer timer; + private final ReplicationTimer overallTimer; + private final ReplicationTimer stageTimer; + private final List> timingData; + private long replicationId; public SegmentReplicationState(ReplicationLuceneIndex index) { stage = Stage.INIT; this.index = index; - timer = new ReplicationTimer(); - timer.start(); + // Timing data will have as many entries as stages, plus one + // additional entry for the overall timer + timingData = new ArrayList<>(Stage.values().length + 1); + overallTimer = new ReplicationTimer(); + stageTimer = new ReplicationTimer(); + stageTimer.start(); + // set an invalid value by default + this.replicationId = -1L; + } + + public SegmentReplicationState(ReplicationLuceneIndex index, long replicationId) { + this(index); + this.replicationId = replicationId; } @Override @@ -74,9 +94,17 @@ public ReplicationLuceneIndex getIndex() { return index; } + public long getReplicationId() { + return replicationId; + } + @Override public ReplicationTimer getTimer() { - return timer; + return overallTimer; + } + + public List> getTimingData() { + return timingData; } public Stage getStage() { @@ -90,6 +118,12 @@ protected void validateAndSetStage(Stage expected, Stage next) { "can't move replication to stage [" + next + "]. current stage: [" + stage + "] (expected [" + expected + "])" ); } + // save the timing data for the current step + stageTimer.stop(); + timingData.add(new Tuple<>(stage.name(), stageTimer.time())); + // restart the step timer + stageTimer.reset(); + stageTimer.start(); stage = next; } @@ -97,16 +131,29 @@ public void setStage(Stage stage) { switch (stage) { case INIT: this.stage = Stage.INIT; - getIndex().reset(); break; case REPLICATING: validateAndSetStage(Stage.INIT, stage); - getIndex().start(); + // only start the overall timer once we've started replication + overallTimer.start(); break; - case DONE: + case GET_CHECKPOINT_INFO: validateAndSetStage(Stage.REPLICATING, stage); - getIndex().stop(); - getTimer().stop(); + break; + case FILE_DIFF: + validateAndSetStage(Stage.GET_CHECKPOINT_INFO, stage); + break; + case GET_FILES: + validateAndSetStage(Stage.FILE_DIFF, stage); + break; + case FINALIZE_REPLICATION: + validateAndSetStage(Stage.GET_FILES, stage); + break; + case DONE: + validateAndSetStage(Stage.FINALIZE_REPLICATION, stage); + // add the overall timing data + overallTimer.stop(); + timingData.add(new Tuple<>("OVERALL", overallTimer.time())); break; default: throw new IllegalArgumentException("unknown SegmentReplicationState.Stage [" + stage + "]"); diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index 516cfa91a787b..a658ffc09d590 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -64,7 +64,7 @@ public SegmentReplicationTarget( super("replication_target", indexShard, new ReplicationLuceneIndex(), listener); this.checkpoint = checkpoint; this.source = source; - this.state = new SegmentReplicationState(stateIndex); + this.state = new SegmentReplicationState(stateIndex, getId()); this.multiFileWriter = new MultiFileWriter(indexShard.store(), stateIndex, getPrefix(), logger, this::ensureRefCount); } @@ -139,7 +139,9 @@ public void startReplication(ActionListener listener) { final StepListener getFilesListener = new StepListener<>(); final StepListener finalizeListener = new StepListener<>(); + logger.trace("[shardId {}] Replica starting replication [id {}]", shardId().getId(), getId()); // Get list of files to copy from this checkpoint. + state.setStage(SegmentReplicationState.Stage.GET_CHECKPOINT_INFO); source.getCheckpointMetadata(getId(), checkpoint, checkpointInfoListener); checkpointInfoListener.whenComplete(checkpointInfo -> getFiles(checkpointInfo, getFilesListener), listener::onFailure); @@ -152,14 +154,16 @@ public void startReplication(ActionListener listener) { private void getFiles(CheckpointInfoResponse checkpointInfo, StepListener getFilesListener) throws IOException { + state.setStage(SegmentReplicationState.Stage.FILE_DIFF); final Store.MetadataSnapshot snapshot = checkpointInfo.getSnapshot(); Store.MetadataSnapshot localMetadata = getMetadataSnapshot(); - final Store.RecoveryDiff diff = snapshot.recoveryDiff(localMetadata); - logger.debug("Replication diff {}", diff); - // Segments are immutable. So if the replica has any segments with the same name that differ from the one in the incoming snapshot - // from - // source that means the local copy of the segment has been corrupted/changed in some way and we throw an IllegalStateException to - // fail the shard + final Store.RecoveryDiff diff = snapshot.segmentReplicationDiff(localMetadata); + logger.trace("Replication diff {}", diff); + /* + * Segments are immutable. So if the replica has any segments with the same name that differ from the one in the incoming + * snapshot from source that means the local copy of the segment has been corrupted/changed in some way and we throw an + * IllegalStateException to fail the shard + */ if (diff.different.isEmpty() == false) { getFilesListener.onFailure( new IllegalStateException( @@ -177,15 +181,18 @@ private void getFiles(CheckpointInfoResponse checkpointInfo, StepListener listener) { + state.setStage(SegmentReplicationState.Stage.FINALIZE_REPLICATION); ActionListener.completeWith(listener, () -> { multiFileWriter.renameAllTempFiles(); final Store store = store(); diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java index f699f0edba842..a79ce195ad83b 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java @@ -116,7 +116,7 @@ public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexSh * @param replicaShard replica shard on which checkpoint is received */ public synchronized void onNewCheckpoint(final ReplicationCheckpoint receivedCheckpoint, final IndexShard replicaShard) { - + logger.trace(() -> new ParameterizedMessage("Replica received new replication checkpoint from primary [{}]", receivedCheckpoint)); // Checks if received checkpoint is already present and ahead then it replaces old received checkpoint if (latestReceivedCheckpoint.get(replicaShard.shardId()) != null) { if (receivedCheckpoint.isAheadOf(latestReceivedCheckpoint.get(replicaShard.shardId()))) { @@ -139,6 +139,14 @@ public synchronized void onNewCheckpoint(final ReplicationCheckpoint receivedChe startReplication(receivedCheckpoint, replicaShard, new SegmentReplicationListener() { @Override public void onReplicationDone(SegmentReplicationState state) { + logger.trace( + () -> new ParameterizedMessage( + "[shardId {}] [replication id {}] Replication complete, timing data: {}", + replicaShard.shardId().getId(), + state.getReplicationId(), + state.getTimingData() + ) + ); // if we received a checkpoint during the copy event that is ahead of this // try and process it. if (latestReceivedCheckpoint.get(replicaShard.shardId()).isAheadOf(replicaShard.getLatestReplicationCheckpoint())) { @@ -154,6 +162,14 @@ public void onReplicationDone(SegmentReplicationState state) { @Override public void onReplicationFailure(SegmentReplicationState state, OpenSearchException e, boolean sendShardFailure) { + logger.trace( + () -> new ParameterizedMessage( + "[shardId {}] [replication id {}] Replication failed, timing data: {}", + replicaShard.shardId().getId(), + state.getReplicationId(), + state.getTimingData() + ) + ); if (sendShardFailure == true) { logger.error("replication failure", e); replicaShard.failShard("replication failure", e); @@ -172,9 +188,9 @@ public void startReplication( startReplication(new SegmentReplicationTarget(checkpoint, indexShard, sourceFactory.get(indexShard), listener)); } - public void startReplication(final SegmentReplicationTarget target) { + // pkg-private for integration tests + void startReplication(final SegmentReplicationTarget target) { final long replicationId = onGoingReplications.start(target, recoverySettings.activityTimeout()); - logger.trace(() -> new ParameterizedMessage("Starting replication {}", replicationId)); threadPool.generic().execute(new ReplicationRunner(replicationId)); } diff --git a/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java b/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java index 8093b6aee88f9..cc51082639cdb 100644 --- a/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java +++ b/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java @@ -29,6 +29,7 @@ import org.opensearch.index.shard.IndexShardClosedException; import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.SegmentReplicationTargetService; +import org.opensearch.indices.replication.common.ReplicationTimer; import org.opensearch.node.NodeClosedException; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; @@ -103,7 +104,10 @@ final void publish(IndexShard indexShard) { // we have to execute under the system context so that if security is enabled the sync is authorized threadContext.markAsSystemContext(); PublishCheckpointRequest request = new PublishCheckpointRequest(indexShard.getLatestReplicationCheckpoint()); + final ReplicationCheckpoint checkpoint = request.getCheckpoint(); final ReplicationTask task = (ReplicationTask) taskManager.register("transport", "segrep_publish_checkpoint", request); + final ReplicationTimer timer = new ReplicationTimer(); + timer.start(); transportService.sendChildRequest( clusterService.localNode(), transportPrimaryAction, @@ -123,12 +127,23 @@ public String executor() { @Override public void handleResponse(ReplicationResponse response) { + timer.stop(); + logger.trace( + () -> new ParameterizedMessage( + "[shardId {}] Completed publishing checkpoint [{}], timing: {}", + indexShard.shardId().getId(), + checkpoint, + timer.time() + ) + ); task.setPhase("finished"); taskManager.unregister(task); } @Override public void handleException(TransportException e) { + timer.stop(); + logger.trace("[shardId {}] Failed to publish checkpoint, timing: {}", indexShard.shardId().getId(), timer.time()); task.setPhase("finished"); taskManager.unregister(task); if (ExceptionsHelper.unwrap(e, NodeClosedException.class) != null) { @@ -151,6 +166,13 @@ public void handleException(TransportException e) { } } ); + logger.trace( + () -> new ParameterizedMessage( + "[shardId {}] Publishing replication checkpoint [{}]", + checkpoint.getShardId().getId(), + checkpoint + ) + ); } } @@ -168,7 +190,7 @@ protected void shardOperationOnReplica(PublishCheckpointRequest request, IndexSh Objects.requireNonNull(request); Objects.requireNonNull(replica); ActionListener.completeWith(listener, () -> { - logger.trace("Checkpoint received on replica {}", request); + logger.trace(() -> new ParameterizedMessage("Checkpoint {} received on replica {}", request, replica.shardId())); if (request.getCheckpoint().getShardId().equals(replica.shardId())) { replicationService.onNewCheckpoint(request.getCheckpoint(), replica); } diff --git a/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java b/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java index 8afb5bd055636..6a4e5e449f178 100644 --- a/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java +++ b/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java @@ -23,7 +23,7 @@ * * @opensearch.internal */ -public class ReplicationCheckpoint implements Writeable { +public class ReplicationCheckpoint implements Writeable, Comparable { private final ShardId shardId; private final long primaryTerm; @@ -107,6 +107,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(segmentInfosVersion); } + @Override + public int compareTo(ReplicationCheckpoint other) { + return this.isAheadOf(other) ? -1 : 1; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/server/src/main/java/org/opensearch/plugins/SearchPlugin.java b/server/src/main/java/org/opensearch/plugins/SearchPlugin.java index a743360e1e90c..af7d4fc2e9fe5 100644 --- a/server/src/main/java/org/opensearch/plugins/SearchPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/SearchPlugin.java @@ -55,6 +55,9 @@ import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.PipelineAggregationBuilder; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregation; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationParsingFunction; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; import org.opensearch.search.aggregations.bucket.terms.SignificantTerms; import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic; import org.opensearch.search.aggregations.pipeline.MovAvgModel; @@ -172,6 +175,15 @@ default List> getAggregationExtentions() return emptyList(); } + /** + * Allows plugins to register new Aggregation in the {@link CompositeAggregation}. + * + * @return A {@link List} of {@link CompositeAggregationSpec} + */ + default List getCompositeAggregations() { + return emptyList(); + } + /** * The new {@link PipelineAggregator}s added by this plugin. */ @@ -532,6 +544,76 @@ public AggregationSpec setAggregatorRegistrar(Consumer aggregatorRegistrar; + private final Class valueSourceBuilderClass; + @Deprecated + /** This is added for backward compatibility, you don't need to set it, as we use aggregationType instead of + * byte code + */ + private Byte byteCode; + private final CompositeAggregationParsingFunction parsingFunction; + private final String aggregationType; + private final Writeable.Reader> reader; + + /** + * Specification for registering an aggregation in Composite Aggregation + * + * @param aggregatorRegistrar function to register the + * {@link org.opensearch.search.aggregations.support.ValuesSource} to aggregator mappings for Composite + * aggregation + * @param valueSourceBuilderClass ValueSourceBuilder class name which is building the aggregation + * @param byteCode byte code which is used in serialisation and de-serialisation to indentify which + * aggregation builder to use + * @param reader Typically, a reference to a constructor that takes a {@link StreamInput}, which is + * registered with the aggregation + * @param parsingFunction a reference function which will be used to parse the Aggregation input. + * @param aggregationType a {@link String} defined in the AggregationBuilder as type. + */ + public CompositeAggregationSpec( + final Consumer aggregatorRegistrar, + final Class> valueSourceBuilderClass, + final Byte byteCode, + final Writeable.Reader> reader, + final CompositeAggregationParsingFunction parsingFunction, + final String aggregationType + ) { + this.aggregatorRegistrar = aggregatorRegistrar; + this.valueSourceBuilderClass = valueSourceBuilderClass; + this.byteCode = byteCode; + this.parsingFunction = parsingFunction; + this.aggregationType = aggregationType; + this.reader = reader; + } + + public Consumer getAggregatorRegistrar() { + return aggregatorRegistrar; + } + + public Class getValueSourceBuilderClass() { + return valueSourceBuilderClass; + } + + public Byte getByteCode() { + return byteCode; + } + + public CompositeAggregationParsingFunction getParsingFunction() { + return parsingFunction; + } + + public String getAggregationType() { + return aggregationType; + } + + public Writeable.Reader> getReader() { + return reader; + } + } + /** * Specification for a {@link PipelineAggregator}. */ diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 9cafe4194e578..600e4cc0839e5 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -597,7 +597,10 @@ protected Table getTableWithHeader(final RestRequest request) { ); table.addCell("pri.search.scroll_total", "default:false;text-align:right;desc:completed scroll contexts"); - table.addCell("search.point_in_time_current", "sibling:pri;alias:scc,searchPointInTimeCurrent;default:false;text-align:right;desc:open pit contexts"); + table.addCell( + "search.point_in_time_current", + "sibling:pri;alias:scc,searchPointInTimeCurrent;default:false;text-align:right;desc:open pit contexts" + ); table.addCell("pri.search.point_in_time_current", "default:false;text-align:right;desc:open pit contexts"); table.addCell( diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java index 65b8c73349616..5c468edeccd89 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java @@ -310,9 +310,18 @@ protected Table getTableWithHeader(final RestRequest request) { ); table.addCell("search.scroll_total", "alias:scto,searchScrollTotal;default:false;text-align:right;desc:completed scroll contexts"); - table.addCell("search.point_in_time_current", "alias:scc,searchPointInTimeCurrent;default:false;text-align:right;desc:open pit contexts"); - table.addCell("search.point_in_time_time", "alias:scti,searchPointInTimeTime;default:false;text-align:right;desc:time pit contexts held open"); - table.addCell("search.point_in_time_total", "alias:scto,searchPointInTimeTotal;default:false;text-align:right;desc:completed pit contexts"); + table.addCell( + "search.point_in_time_current", + "alias:scc,searchPointInTimeCurrent;default:false;text-align:right;desc:open pit contexts" + ); + table.addCell( + "search.point_in_time_time", + "alias:scti,searchPointInTimeTime;default:false;text-align:right;desc:time pit contexts held open" + ); + table.addCell( + "search.point_in_time_total", + "alias:scto,searchPointInTimeTotal;default:false;text-align:right;desc:completed pit contexts" + ); table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments"); table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments"); diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestPitSegmentsAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestPitSegmentsAction.java index d7078cf71a344..8325f8861354d 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestPitSegmentsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestPitSegmentsAction.java @@ -162,4 +162,4 @@ private Table buildTable(final RestRequest request, ClusterStateResponse state, } return table; } -} \ No newline at end of file +} diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java index 5f0409a5e3ba9..7205b17252f55 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java @@ -226,9 +226,18 @@ protected Table getTableWithHeader(final RestRequest request) { ); table.addCell("search.scroll_total", "alias:scto,searchScrollTotal;default:false;text-align:right;desc:completed scroll contexts"); - table.addCell("search.point_in_time_current", "alias:spc,searchPointInTimeCurrent;default:false;text-align:right;desc:open pit contexts"); - table.addCell("search.point_in_time_time", "alias:spti,searchPointInTimeTime;default:false;text-align:right;desc:time pit contexts held open"); - table.addCell("search.point_in_time_total", "alias:spto,searchPointInTimeTotal;default:false;text-align:right;desc:completed pit contexts"); + table.addCell( + "search.point_in_time_current", + "alias:spc,searchPointInTimeCurrent;default:false;text-align:right;desc:open pit contexts" + ); + table.addCell( + "search.point_in_time_time", + "alias:spti,searchPointInTimeTime;default:false;text-align:right;desc:time pit contexts held open" + ); + table.addCell( + "search.point_in_time_total", + "alias:spto,searchPointInTimeTotal;default:false;text-align:right;desc:completed pit contexts" + ); table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments"); table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments"); diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestDeletePitAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestDeletePitAction.java index 452e66f8f5018..43b32184c3dc5 100644 --- a/server/src/main/java/org/opensearch/rest/action/search/RestDeletePitAction.java +++ b/server/src/main/java/org/opensearch/rest/action/search/RestDeletePitAction.java @@ -50,7 +50,9 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client } })); } - return channel -> client.deletePits(deletePITRequest, new RestStatusToXContentListener(channel)); + return channel -> client.admin() + .cluster() + .deletePits(deletePITRequest, new RestStatusToXContentListener(channel)); } @Override diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestGetAllPitsAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestGetAllPitsAction.java index d2a788f3ad820..1f2d1e2e69c1a 100644 --- a/server/src/main/java/org/opensearch/rest/action/search/RestGetAllPitsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/search/RestGetAllPitsAction.java @@ -8,26 +8,20 @@ package org.opensearch.rest.action.search; -import com.carrotsearch.hppc.cursors.ObjectCursor; import org.opensearch.action.admin.cluster.state.ClusterStateRequest; -import org.opensearch.action.admin.cluster.state.ClusterStateResponse; import org.opensearch.action.search.GetAllPitNodesRequest; import org.opensearch.action.search.GetAllPitNodesResponse; -import org.opensearch.action.search.GetAllPitsAction; import org.opensearch.client.node.NodeClient; -import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; import org.opensearch.rest.RestStatus; -import org.opensearch.rest.action.RestActionListener; import org.opensearch.rest.action.RestResponseListener; import java.io.IOException; import java.util.Collections; -import java.util.LinkedList; import java.util.List; import static java.util.Collections.unmodifiableList; @@ -48,48 +42,40 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli clusterStateRequest.local(false); boolean includeAll = request.paramAsBoolean("include_all", false); clusterStateRequest.clusterManagerNodeTimeout( - request.paramAsTime("cluster_manager_timeout", clusterStateRequest.clusterManagerNodeTimeout()) + request.paramAsTime("cluster_manager_timeout", clusterStateRequest.clusterManagerNodeTimeout()) ); clusterStateRequest.clear().nodes(true).routingTable(true).indices("*"); - return channel -> client.admin().cluster().state(clusterStateRequest, new RestActionListener(channel) { - @Override - public void processResponse(final ClusterStateResponse clusterStateResponse) { - final List nodes = new LinkedList<>(); - for (ObjectCursor cursor : clusterStateResponse.getState().nodes().getDataNodes().values()) { - DiscoveryNode node = cursor.value; - nodes.add(node); - } - DiscoveryNode[] disNodesArr = new DiscoveryNode[nodes.size()]; - nodes.toArray(disNodesArr); - GetAllPitNodesRequest getAllPITNodesRequest = new GetAllPitNodesRequest(disNodesArr); - client.execute(GetAllPitsAction.INSTANCE, getAllPITNodesRequest, new RestResponseListener(channel) { - @Override - public RestResponse buildResponse(final GetAllPitNodesResponse getAllPITNodesResponse) throws Exception { - try (XContentBuilder builder = channel.newBuilder()) { - builder.startObject(); - if(getAllPITNodesResponse.hasFailures()) { - builder.startArray("failures"); - for (int idx = 0; idx < getAllPITNodesResponse.failures().size(); idx++) { - builder.field(getAllPITNodesResponse.failures().get(idx).nodeId(), - getAllPITNodesResponse.failures().get(idx).getDetailedMessage()); - } - builder.endArray(); + GetAllPitNodesRequest getAllPITNodesRequest = new GetAllPitNodesRequest(); + return channel -> client.admin() + .cluster() + .getAllPits(getAllPITNodesRequest, new RestResponseListener(channel) { + @Override + public RestResponse buildResponse(final GetAllPitNodesResponse getAllPITNodesResponse) throws Exception { + try (XContentBuilder builder = channel.newBuilder()) { + builder.startObject(); + if (getAllPITNodesResponse.hasFailures()) { + builder.startArray("failures"); + for (int idx = 0; idx < getAllPITNodesResponse.failures().size(); idx++) { + builder.field( + getAllPITNodesResponse.failures().get(idx).nodeId(), + getAllPITNodesResponse.failures().get(idx).getDetailedMessage() + ); } - builder.field("pits", getAllPITNodesResponse.getPitInfos()); - if(includeAll) { - builder.startArray("nodeResults"); - for (int idx = 0; idx < getAllPITNodesResponse.getNodes().size(); idx++) { - builder.value(getAllPITNodesResponse.getNodes().get(idx)); - } - builder.endArray(); + builder.endArray(); + } + builder.field("pits", getAllPITNodesResponse.getPitInfos()); + if (includeAll) { + builder.startArray("nodeResults"); + for (int idx = 0; idx < getAllPITNodesResponse.getNodes().size(); idx++) { + builder.value(getAllPITNodesResponse.getNodes().get(idx)); } - builder.endObject(); - return new BytesRestResponse(RestStatus.OK, builder); + builder.endArray(); } + builder.endObject(); + return new BytesRestResponse(RestStatus.OK, builder); } - }); - } - }); + } + }); } @Override @@ -97,4 +83,4 @@ public List routes() { return unmodifiableList(Collections.singletonList(new Route(GET, "/_search/point_in_time/all"))); } -} \ No newline at end of file +} diff --git a/server/src/main/java/org/opensearch/search/DocValueFormat.java b/server/src/main/java/org/opensearch/search/DocValueFormat.java index 7e7e4f83334f5..84c46e400543a 100644 --- a/server/src/main/java/org/opensearch/search/DocValueFormat.java +++ b/server/src/main/java/org/opensearch/search/DocValueFormat.java @@ -47,7 +47,7 @@ import org.opensearch.common.time.DateUtils; import org.opensearch.geometry.utils.Geohash; import org.opensearch.index.mapper.DateFieldMapper; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import java.io.IOException; import java.math.BigInteger; diff --git a/server/src/main/java/org/opensearch/search/SearchModule.java b/server/src/main/java/org/opensearch/search/SearchModule.java index 0995497eba50e..0149f9a025bcd 100644 --- a/server/src/main/java/org/opensearch/search/SearchModule.java +++ b/server/src/main/java/org/opensearch/search/SearchModule.java @@ -126,10 +126,6 @@ import org.opensearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; import org.opensearch.search.aggregations.bucket.filter.InternalFilter; import org.opensearch.search.aggregations.bucket.filter.InternalFilters; -import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid; -import org.opensearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.opensearch.search.aggregations.bucket.global.InternalGlobal; import org.opensearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; @@ -185,12 +181,10 @@ import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder; import org.opensearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.opensearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder; -import org.opensearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; import org.opensearch.search.aggregations.metrics.GeoCentroidAggregationBuilder; import org.opensearch.search.aggregations.metrics.InternalAvg; import org.opensearch.search.aggregations.metrics.InternalCardinality; import org.opensearch.search.aggregations.metrics.InternalExtendedStats; -import org.opensearch.search.aggregations.metrics.InternalGeoBounds; import org.opensearch.search.aggregations.metrics.InternalGeoCentroid; import org.opensearch.search.aggregations.metrics.InternalHDRPercentileRanks; import org.opensearch.search.aggregations.metrics.InternalHDRPercentiles; @@ -630,22 +624,6 @@ private ValuesSourceRegistry registerAggregations(List plugins) { ).addResultReader(InternalGeoDistance::new).setAggregatorRegistrar(GeoDistanceAggregationBuilder::registerAggregators), builder ); - registerAggregation( - new AggregationSpec( - GeoHashGridAggregationBuilder.NAME, - GeoHashGridAggregationBuilder::new, - GeoHashGridAggregationBuilder.PARSER - ).addResultReader(InternalGeoHashGrid::new).setAggregatorRegistrar(GeoHashGridAggregationBuilder::registerAggregators), - builder - ); - registerAggregation( - new AggregationSpec( - GeoTileGridAggregationBuilder.NAME, - GeoTileGridAggregationBuilder::new, - GeoTileGridAggregationBuilder.PARSER - ).addResultReader(InternalGeoTileGrid::new).setAggregatorRegistrar(GeoTileGridAggregationBuilder::registerAggregators), - builder - ); registerAggregation( new AggregationSpec(NestedAggregationBuilder.NAME, NestedAggregationBuilder::new, NestedAggregationBuilder::parse) .addResultReader(InternalNested::new), @@ -664,12 +642,6 @@ private ValuesSourceRegistry registerAggregations(List plugins) { .addResultReader(InternalTopHits::new), builder ); - registerAggregation( - new AggregationSpec(GeoBoundsAggregationBuilder.NAME, GeoBoundsAggregationBuilder::new, GeoBoundsAggregationBuilder.PARSER) - .addResultReader(InternalGeoBounds::new) - .setAggregatorRegistrar(GeoBoundsAggregationBuilder::registerAggregators), - builder - ); registerAggregation( new AggregationSpec( GeoCentroidAggregationBuilder.NAME, @@ -689,7 +661,7 @@ private ValuesSourceRegistry registerAggregations(List plugins) { registerAggregation( new AggregationSpec(CompositeAggregationBuilder.NAME, CompositeAggregationBuilder::new, CompositeAggregationBuilder.PARSER) .addResultReader(InternalComposite::new) - .setAggregatorRegistrar(CompositeAggregationBuilder::registerAggregators), + .setAggregatorRegistrar(reg -> CompositeAggregationBuilder.registerAggregators(reg, plugins)), builder ); registerAggregation( diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java index 4bd95da193668..04fab85c163a9 100644 --- a/server/src/main/java/org/opensearch/search/SearchService.java +++ b/server/src/main/java/org/opensearch/search/SearchService.java @@ -881,6 +881,7 @@ public void createPitReaderContext(ShardId shardId, TimeValue keepAlive, ActionL shard.awaitShardSearchActive(ignored -> { Engine.SearcherSupplier searcherSupplier = null; ReaderContext readerContext = null; + Releasable decreasePitContexts = openPitContexts::decrementAndGet; try { if (openPitContexts.incrementAndGet() > maxOpenPitContext) { throw new OpenSearchRejectedExecutionException( @@ -902,15 +903,16 @@ public void createPitReaderContext(ShardId shardId, TimeValue keepAlive, ActionL searchOperationListener.onNewPitContext(finalReaderContext); readerContext.addOnClose(() -> { - openPitContexts.decrementAndGet(); searchOperationListener.onFreeReaderContext(finalReaderContext); searchOperationListener.onFreePitContext(finalReaderContext); }); + readerContext.addOnClose(decreasePitContexts); // add the newly created pit reader context to active readers putReaderContext(readerContext); readerContext = null; listener.onResponse(finalReaderContext.id()); } catch (Exception exc) { + Releasables.closeWhileHandlingException(decreasePitContexts); Releasables.closeWhileHandlingException(searcherSupplier, readerContext); listener.onFailure(exc); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilders.java b/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilders.java index 01e0f95b0d750..9886e423bbc76 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilders.java +++ b/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilders.java @@ -43,10 +43,6 @@ import org.opensearch.search.aggregations.bucket.filter.Filters; import org.opensearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; import org.opensearch.search.aggregations.bucket.filter.FiltersAggregator.KeyedFilter; -import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid; -import org.opensearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; import org.opensearch.search.aggregations.bucket.global.Global; import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; @@ -78,8 +74,6 @@ import org.opensearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.opensearch.search.aggregations.metrics.ExtendedStats; import org.opensearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder; -import org.opensearch.search.aggregations.metrics.GeoBounds; -import org.opensearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; import org.opensearch.search.aggregations.metrics.GeoCentroid; import org.opensearch.search.aggregations.metrics.GeoCentroidAggregationBuilder; import org.opensearch.search.aggregations.metrics.Max; @@ -263,20 +257,6 @@ public static HistogramAggregationBuilder histogram(String name) { return new HistogramAggregationBuilder(name); } - /** - * Create a new {@link InternalGeoHashGrid} aggregation with the given name. - */ - public static GeoHashGridAggregationBuilder geohashGrid(String name) { - return new GeoHashGridAggregationBuilder(name); - } - - /** - * Create a new {@link InternalGeoTileGrid} aggregation with the given name. - */ - public static GeoTileGridAggregationBuilder geotileGrid(String name) { - return new GeoTileGridAggregationBuilder(name); - } - /** * Create a new {@link SignificantTerms} aggregation with the given name. */ @@ -364,13 +344,6 @@ public static TopHitsAggregationBuilder topHits(String name) { return new TopHitsAggregationBuilder(name); } - /** - * Create a new {@link GeoBounds} aggregation with the given name. - */ - public static GeoBoundsAggregationBuilder geoBounds(String name) { - return new GeoBoundsAggregationBuilder(name); - } - /** * Create a new {@link GeoCentroid} aggregation with the given name. */ diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/GeoTileUtils.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtils.java rename to server/src/main/java/org/opensearch/search/aggregations/bucket/GeoTileUtils.java index 5498b2b1a7109..6cd1823622f01 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/GeoTileUtils.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.search.aggregations.bucket; import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.util.SloppyMath; @@ -104,7 +104,7 @@ private GeoTileUtils() {} * @param parser {@link XContentParser} to parse the value from * @return int representing precision */ - static int parsePrecision(XContentParser parser) throws IOException, OpenSearchParseException { + public static int parsePrecision(XContentParser parser) throws IOException, OpenSearchParseException { final Object node = parser.currentToken().equals(XContentParser.Token.VALUE_NUMBER) ? Integer.valueOf(parser.intValue()) : parser.text(); @@ -252,7 +252,7 @@ public static String stringEncode(long hash) { /** * Decode long hash as a GeoPoint (center of the tile) */ - static GeoPoint hashToGeoPoint(long hash) { + public static GeoPoint hashToGeoPoint(long hash) { int[] res = parseHash(hash); return zxyToGeoPoint(res[0], res[1], res[2]); } @@ -260,7 +260,7 @@ static GeoPoint hashToGeoPoint(long hash) { /** * Decode a string bucket key in "zoom/x/y" format to a GeoPoint (center of the tile) */ - static GeoPoint keyToGeoPoint(String hashAsString) { + public static GeoPoint keyToGeoPoint(String hashAsString) { int[] hashAsInts = parseHash(hashAsString); return zxyToGeoPoint(hashAsInts[0], hashAsInts[1], hashAsInts[2]); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java index 8b07df3f689bf..093c2ad42722e 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java @@ -35,9 +35,11 @@ import org.opensearch.common.ParseField; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.io.stream.Writeable; import org.opensearch.common.xcontent.ConstructingObjectParser; import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.plugins.SearchPlugin; import org.opensearch.search.aggregations.AbstractAggregationBuilder; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregatorFactories; @@ -47,11 +49,14 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Function; import static org.opensearch.common.xcontent.ConstructingObjectParser.constructorArg; @@ -82,14 +87,55 @@ public class CompositeAggregationBuilder extends AbstractAggregationBuilder p.map(), AFTER_FIELD_NAME); } - public static void registerAggregators(ValuesSourceRegistry.Builder builder) { + static final Map, Byte> BUILDER_CLASS_TO_BYTE_CODE = new HashMap<>(); + static final Map BUILDER_TYPE_TO_PARSER = new HashMap<>(); + static final Map>> BYTE_CODE_TO_COMPOSITE_VALUE_SOURCE_READER = + new HashMap<>(); + static final Map< + String, + Writeable.Reader>> AGGREGATION_TYPE_TO_COMPOSITE_VALUE_SOURCE_READER = new HashMap<>(); + static final Map, String> BUILDER_CLASS_TO_AGGREGATION_TYPE = new HashMap<>(); + + public static void registerAggregators(ValuesSourceRegistry.Builder builder, final List plugins) { DateHistogramValuesSourceBuilder.register(builder); HistogramValuesSourceBuilder.register(builder); - GeoTileGridValuesSourceBuilder.register(builder); TermsValuesSourceBuilder.register(builder); + // Register All other aggregations that wants to be part of Composite Aggregation which are provided in + // Plugins along with their parsers and serialisation codes + registerCompositeAggregatorsPlugins(plugins, SearchPlugin::getCompositeAggregations, (compositeAggregationSpec) -> { + compositeAggregationSpec.getAggregatorRegistrar().accept(builder); + BUILDER_TYPE_TO_PARSER.put(compositeAggregationSpec.getAggregationType(), compositeAggregationSpec.getParsingFunction()); + // This is added for backward compatibility, so that we can move away from byte code in the serialisation + if (compositeAggregationSpec.getByteCode() != null) { + BYTE_CODE_TO_COMPOSITE_VALUE_SOURCE_READER.put( + (int) compositeAggregationSpec.getByteCode(), + compositeAggregationSpec.getReader() + ); + BUILDER_CLASS_TO_BYTE_CODE.put( + compositeAggregationSpec.getValueSourceBuilderClass(), + compositeAggregationSpec.getByteCode() + ); + } + AGGREGATION_TYPE_TO_COMPOSITE_VALUE_SOURCE_READER.put( + compositeAggregationSpec.getAggregationType(), + compositeAggregationSpec.getReader() + ); + BUILDER_CLASS_TO_AGGREGATION_TYPE.put( + compositeAggregationSpec.getValueSourceBuilderClass(), + compositeAggregationSpec.getAggregationType() + ); + }); builder.registerUsage(NAME); } + private static void registerCompositeAggregatorsPlugins( + final List plugins, + final Function> producer, + final Consumer consumer + ) { + plugins.forEach(searchPlugin -> producer.apply(searchPlugin).forEach(consumer)); + } + private List> sources; private Map after; private int size = 10; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationParsingFunction.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationParsingFunction.java new file mode 100644 index 0000000000000..344563ad20309 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationParsingFunction.java @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations.bucket.composite; + +import org.opensearch.common.xcontent.XContentParser; + +import java.io.IOException; + +/** + * A functional interface which encapsulates the parsing function to be called for the aggregation which is + * also registered as CompositeAggregation. + */ +@FunctionalInterface +public interface CompositeAggregationParsingFunction { + CompositeValuesSourceBuilder parse(final String name, final XContentParser parser) throws IOException; +} diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java index 7764d367a0cec..26015ae04cf76 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java @@ -69,11 +69,11 @@ public abstract class CompositeValuesSourceBuilder createValuesSource( * @param missingBucket If true an explicit null bucket will represent documents with missing values. * @param hasScript true if the source contains a script that can change the value. */ - CompositeValuesSourceConfig( + public CompositeValuesSourceConfig( String name, @Nullable MappedFieldType fieldType, ValuesSource vs, @@ -113,21 +113,21 @@ SingleDimensionValuesSource createValuesSource( /** * Returns the name associated with this configuration. */ - String name() { + protected String name() { return name; } /** * Returns the {@link MappedFieldType} for this config. */ - MappedFieldType fieldType() { + public MappedFieldType fieldType() { return fieldType; } /** * Returns the {@link ValuesSource} for this configuration. */ - ValuesSource valuesSource() { + public ValuesSource valuesSource() { return vs; } @@ -135,35 +135,35 @@ ValuesSource valuesSource() { * The {@link DocValueFormat} to use for formatting the keys. * {@link DocValueFormat#RAW} means no formatting. */ - DocValueFormat format() { + public DocValueFormat format() { return format; } /** * If true, an explicit `null bucket represents documents with missing values. */ - boolean missingBucket() { + public boolean missingBucket() { return missingBucket; } /** * Return the {@link MissingOrder} for the config. */ - MissingOrder missingOrder() { + public MissingOrder missingOrder() { return missingOrder; } /** * Returns true if the source contains a script that can change the value. */ - boolean hasScript() { + protected boolean hasScript() { return hasScript; } /** * The sort order for the values source (e.g. -1 for descending and 1 for ascending). */ - int reverseMul() { + public int reverseMul() { assert reverseMul == -1 || reverseMul == 1; return reverseMul; } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java index 60d7f277f7650..d8526e684f391 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java @@ -49,6 +49,11 @@ import java.io.IOException; import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder.AGGREGATION_TYPE_TO_COMPOSITE_VALUE_SOURCE_READER; +import static org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder.BUILDER_CLASS_TO_AGGREGATION_TYPE; +import static org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder.BUILDER_CLASS_TO_BYTE_CODE; +import static org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder.BUILDER_TYPE_TO_PARSER; +import static org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder.BYTE_CODE_TO_COMPOSITE_VALUE_SOURCE_READER; /** * Helper class for obtaining values source parsers for different aggs @@ -57,7 +62,11 @@ */ public class CompositeValuesSourceParserHelper { - static , T> void declareValuesSourceFields(AbstractObjectParser objectParser) { + private static final int AGGREGATION_TYPE_REFERENCE = Byte.MAX_VALUE; + + public static , T> void declareValuesSourceFields( + AbstractObjectParser objectParser + ) { objectParser.declareField(VB::field, XContentParser::text, new ParseField("field"), ObjectParser.ValueType.STRING); objectParser.declareBoolean(VB::missingBucket, new ParseField("missing_bucket")); objectParser.declareString(VB::missingOrder, new ParseField(MissingOrder.NAME)); @@ -78,28 +87,45 @@ static , T> void declareValuesSource } public static void writeTo(CompositeValuesSourceBuilder builder, StreamOutput out) throws IOException { - final byte code; + int code = Byte.MIN_VALUE; + String aggregationType = null; if (builder.getClass() == TermsValuesSourceBuilder.class) { code = 0; } else if (builder.getClass() == DateHistogramValuesSourceBuilder.class) { code = 1; } else if (builder.getClass() == HistogramValuesSourceBuilder.class) { code = 2; - } else if (builder.getClass() == GeoTileGridValuesSourceBuilder.class) { - if (out.getVersion().before(LegacyESVersion.V_7_5_0)) { - throw new IOException( - "Attempting to serialize [" - + builder.getClass().getSimpleName() - + "] to a node with unsupported version [" - + out.getVersion() - + "]" - ); - } - code = 3; } else { - throw new IOException("invalid builder type: " + builder.getClass().getSimpleName()); + if (!BUILDER_CLASS_TO_BYTE_CODE.containsKey(builder.getClass()) + && !BUILDER_CLASS_TO_AGGREGATION_TYPE.containsKey(builder.getClass())) { + throw new IOException("invalid builder type: " + builder.getClass().getSimpleName()); + } + aggregationType = BUILDER_CLASS_TO_AGGREGATION_TYPE.get(builder.getClass()); + if (BUILDER_CLASS_TO_BYTE_CODE.containsKey(builder.getClass())) { + code = BUILDER_CLASS_TO_BYTE_CODE.get(builder.getClass()); + if (code == 3 && out.getVersion().before(LegacyESVersion.V_7_5_0)) { + throw new IOException( + "Attempting to serialize [" + + builder.getClass().getSimpleName() + + "] to a node with unsupported version [" + + out.getVersion() + + "]" + ); + } + } + } + + if (code != Byte.MIN_VALUE) { + out.writeByte((byte) code); + } else if (!BUILDER_CLASS_TO_BYTE_CODE.containsKey(builder.getClass())) { + /* + * This is added for backward compatibility when 1 data node is using the new code which is using the + * aggregation type and another is using the only byte code in the serialisation. + */ + out.writeByte((byte) AGGREGATION_TYPE_REFERENCE); + assert aggregationType != null; + out.writeString(aggregationType); } - out.writeByte(code); builder.writeTo(out); } @@ -112,10 +138,17 @@ public static CompositeValuesSourceBuilder readFrom(StreamInput in) throws IO return new DateHistogramValuesSourceBuilder(in); case 2: return new HistogramValuesSourceBuilder(in); - case 3: - return new GeoTileGridValuesSourceBuilder(in); + case AGGREGATION_TYPE_REFERENCE: + final String aggregationType = in.readString(); + if (!AGGREGATION_TYPE_TO_COMPOSITE_VALUE_SOURCE_READER.containsKey(aggregationType)) { + throw new IOException("Invalid aggregation type " + aggregationType); + } + return (CompositeValuesSourceBuilder) AGGREGATION_TYPE_TO_COMPOSITE_VALUE_SOURCE_READER.get(aggregationType).read(in); default: - throw new IOException("Invalid code " + code); + if (!BYTE_CODE_TO_COMPOSITE_VALUE_SOURCE_READER.containsKey(code)) { + throw new IOException("Invalid code " + code); + } + return (CompositeValuesSourceBuilder) BYTE_CODE_TO_COMPOSITE_VALUE_SOURCE_READER.get(code).read(in); } } @@ -143,11 +176,11 @@ public static CompositeValuesSourceBuilder fromXContent(XContentParser parser case HistogramValuesSourceBuilder.TYPE: builder = HistogramValuesSourceBuilder.parse(name, parser); break; - case GeoTileGridValuesSourceBuilder.TYPE: - builder = GeoTileGridValuesSourceBuilder.parse(name, parser); - break; default: - throw new ParsingException(parser.getTokenLocation(), "invalid source type: " + type); + if (!BUILDER_TYPE_TO_PARSER.containsKey(type)) { + throw new ParsingException(parser.getTokenLocation(), "invalid source type: " + type); + } + builder = BUILDER_TYPE_TO_PARSER.get(type).parse(name, parser); } parser.nextToken(); parser.nextToken(); @@ -163,4 +196,5 @@ public static XContentBuilder toXContent(CompositeValuesSourceBuilder source, builder.endObject(); return builder; } + } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/LongValuesSource.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/LongValuesSource.java index a7ed50507288d..ec6410c2a9377 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/LongValuesSource.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/LongValuesSource.java @@ -66,7 +66,7 @@ * * @opensearch.internal */ -class LongValuesSource extends SingleDimensionValuesSource { +public class LongValuesSource extends SingleDimensionValuesSource { private final BigArrays bigArrays; private final CheckedFunction docValuesFunc; private final LongUnaryOperator rounding; @@ -76,7 +76,7 @@ class LongValuesSource extends SingleDimensionValuesSource { private long currentValue; private boolean missingCurrentValue; - LongValuesSource( + public LongValuesSource( BigArrays bigArrays, MappedFieldType fieldType, CheckedFunction docValuesFunc, @@ -165,7 +165,7 @@ private int compareValues(long v1, long v2) { } @Override - void setAfter(Comparable value) { + protected void setAfter(Comparable value) { if (missingBucket && value == null) { afterValue = null; } else { diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SingleDimensionValuesSource.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SingleDimensionValuesSource.java index 747a7017ec872..fe0801d6d230e 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SingleDimensionValuesSource.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SingleDimensionValuesSource.java @@ -53,7 +53,7 @@ * * @opensearch.internal */ -abstract class SingleDimensionValuesSource> implements Releasable { +public abstract class SingleDimensionValuesSource> implements Releasable { protected final BigArrays bigArrays; protected final DocValueFormat format; @Nullable diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java b/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java index 8d036503d1330..b4da1d10b4b68 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java @@ -35,7 +35,6 @@ import org.opensearch.search.aggregations.bucket.composite.InternalComposite; import org.opensearch.search.aggregations.bucket.filter.InternalFilter; import org.opensearch.search.aggregations.bucket.filter.InternalFilters; -import org.opensearch.search.aggregations.bucket.geogrid.InternalGeoGrid; import org.opensearch.search.aggregations.bucket.global.InternalGlobal; import org.opensearch.search.aggregations.bucket.histogram.InternalVariableWidthHistogram; import org.opensearch.search.aggregations.bucket.histogram.InternalAutoDateHistogram; @@ -54,7 +53,6 @@ import org.opensearch.search.aggregations.metrics.InternalAvg; import org.opensearch.search.aggregations.metrics.InternalCardinality; import org.opensearch.search.aggregations.metrics.InternalExtendedStats; -import org.opensearch.search.aggregations.metrics.InternalGeoBounds; import org.opensearch.search.aggregations.metrics.InternalGeoCentroid; import org.opensearch.search.aggregations.metrics.InternalHDRPercentileRanks; import org.opensearch.search.aggregations.metrics.InternalHDRPercentiles; @@ -120,10 +118,6 @@ public static boolean hasValue(InternalFilter agg) { return agg.getDocCount() > 0; } - public static boolean hasValue(InternalGeoGrid agg) { - return agg.getBuckets().stream().anyMatch(bucket -> bucket.getDocCount() > 0); - } - public static boolean hasValue(InternalGlobal agg) { return agg.getDocCount() > 0; } @@ -191,10 +185,6 @@ public static boolean hasValue(InternalExtendedStats agg) { return agg.getCount() > 0; } - public static boolean hasValue(InternalGeoBounds agg) { - return (agg.topLeft() == null && agg.bottomRight() == null) == false; - } - public static boolean hasValue(InternalGeoCentroid agg) { return agg.centroid() != null && agg.count() > 0; } diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/package-info.java b/server/src/main/java/org/opensearch/search/aggregations/support/package-info.java index e16e8c91b3fd0..dd2c16f1daa0e 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/package-info.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/package-info.java @@ -43,7 +43,7 @@ * output). A class hierarchy defines the type of values returned by the source. The top level sub-classes define type-specific behavior, * such as {@link org.opensearch.search.aggregations.support.ValuesSource.Numeric#isFloatingPoint()}. Second level subclasses are * then specialized based on where they read values from, e.g. script or field cases. There are also adapter classes like - * {@link org.opensearch.search.aggregations.bucket.geogrid.CellIdSource} which do run-time conversion from one type to another, often + * org.opensearch.search.aggregations.bucket.geogrid.CellIdSource which do run-time conversion from one type to another, often * dependent on a user specified parameter (precision in that case). *

* diff --git a/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java b/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java index 4a1ecb9661687..3c39ec9f03b2a 100644 --- a/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java +++ b/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java @@ -62,6 +62,7 @@ import org.opensearch.common.util.set.Sets; import org.opensearch.env.ShardLockObtainFailedException; import org.opensearch.index.shard.ShardId; +import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.repositories.IndexId; import org.opensearch.snapshots.Snapshot; import org.opensearch.snapshots.SnapshotId; @@ -205,6 +206,203 @@ public void testShardLockObtainFailedException() { assertClusterHealthStatus(allocation, ClusterHealthStatus.YELLOW); } + /** + * Tests that replica with the highest primary term version will be selected as target + */ + public void testPreferReplicaWithHighestPrimaryTerm() { + String allocId1 = randomAlphaOfLength(10); + String allocId2 = randomAlphaOfLength(10); + String allocId3 = randomAlphaOfLength(10); + final RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas( + yesAllocationDeciders(), + CLUSTER_RECOVERED, + allocId1, + allocId2, + allocId3 + ); + testAllocator.addData(node1, allocId1, false, new ReplicationCheckpoint(shardId, 20, 10, 101, 1)); + testAllocator.addData(node2, allocId2, false, new ReplicationCheckpoint(shardId, 22, 10, 120, 2)); + testAllocator.addData(node3, allocId3, false, new ReplicationCheckpoint(shardId, 20, 10, 120, 2)); + allocateAllUnassigned(allocation); + assertThat(allocation.routingNodesChanged(), equalTo(true)); + assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true)); + assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), + equalTo(node2.getId()) + ); + // Assert node2's allocation id is used + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).allocationId().getId(), + equalTo(allocId2) + ); + assertClusterHealthStatus(allocation, ClusterHealthStatus.YELLOW); + } + + /** + * Tests that replica with highest primary ter version will be selected as target + */ + public void testPreferReplicaWithNullReplicationCheckpoint() { + String allocId1 = randomAlphaOfLength(10); + String allocId2 = randomAlphaOfLength(10); + String allocId3 = randomAlphaOfLength(10); + final RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas( + yesAllocationDeciders(), + CLUSTER_RECOVERED, + allocId1, + allocId2, + allocId3 + ); + testAllocator.addData(node1, allocId1, false, new ReplicationCheckpoint(shardId, 20, 10, 101, 1)); + testAllocator.addData(node2, allocId2, false); + testAllocator.addData(node3, allocId3, false, new ReplicationCheckpoint(shardId, 40, 10, 120, 2)); + allocateAllUnassigned(allocation); + assertThat(allocation.routingNodesChanged(), equalTo(true)); + assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true)); + assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), + equalTo(node3.getId()) + ); + // Assert node3's allocation id should be used as it has highest replication checkpoint + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).allocationId().getId(), + equalTo(allocId3) + ); + assertClusterHealthStatus(allocation, ClusterHealthStatus.YELLOW); + } + + /** + * Tests that null ReplicationCheckpoint are ignored + */ + public void testPreferReplicaWithAllNullReplicationCheckpoint() { + String allocId1 = randomAlphaOfLength(10); + String allocId2 = randomAlphaOfLength(10); + String allocId3 = randomAlphaOfLength(10); + final RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas( + yesAllocationDeciders(), + CLUSTER_RECOVERED, + allocId1, + allocId2, + allocId3 + ); + testAllocator.addData(node1, allocId1, false, null, null); + testAllocator.addData(node2, allocId2, false, null, null); + testAllocator.addData(node3, allocId3, true, null, null); + allocateAllUnassigned(allocation); + assertThat(allocation.routingNodesChanged(), equalTo(true)); + assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true)); + assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), + equalTo(node3.getId()) + ); + // Assert node3's allocation id should be used as it was previous primary + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).allocationId().getId(), + equalTo(allocId3) + ); + assertClusterHealthStatus(allocation, ClusterHealthStatus.YELLOW); + } + + /** + * Tests that replica with highest segment info version will be selected as target on equal primary terms + */ + public void testPreferReplicaWithHighestSegmentInfoVersion() { + String allocId1 = randomAlphaOfLength(10); + String allocId2 = randomAlphaOfLength(10); + String allocId3 = randomAlphaOfLength(10); + final RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas( + yesAllocationDeciders(), + CLUSTER_RECOVERED, + allocId1, + allocId2, + allocId3 + ); + testAllocator.addData(node1, allocId1, false, new ReplicationCheckpoint(shardId, 10, 10, 101, 1)); + testAllocator.addData(node2, allocId2, false, new ReplicationCheckpoint(shardId, 20, 10, 120, 3)); + testAllocator.addData(node3, allocId3, false, new ReplicationCheckpoint(shardId, 20, 10, 120, 2)); + allocateAllUnassigned(allocation); + assertThat(allocation.routingNodesChanged(), equalTo(true)); + assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true)); + assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), + equalTo(node2.getId()) + ); + // Assert node2's allocation id is used + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).allocationId().getId(), + equalTo(allocId2) + ); + assertClusterHealthStatus(allocation, ClusterHealthStatus.YELLOW); + } + + /** + * Tests that prefer allocation of replica at lower checkpoint but in sync set + */ + public void testOutOfSyncHighestRepCheckpointIsIgnored() { + String allocId1 = randomAlphaOfLength(10); + String allocId2 = randomAlphaOfLength(10); + String allocId3 = randomAlphaOfLength(10); + final RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas( + yesAllocationDeciders(), + CLUSTER_RECOVERED, + allocId1, + allocId3 + ); + testAllocator.addData(node1, allocId1, false, new ReplicationCheckpoint(shardId, 10, 10, 101, 1)); + testAllocator.addData(node2, allocId2, false, new ReplicationCheckpoint(shardId, 20, 10, 120, 2)); + testAllocator.addData(node3, allocId3, false, new ReplicationCheckpoint(shardId, 15, 10, 120, 2)); + allocateAllUnassigned(allocation); + assertThat(allocation.routingNodesChanged(), equalTo(true)); + assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true)); + assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), + equalTo(node3.getId()) + ); + // Assert node3's allocation id is used + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).allocationId().getId(), + equalTo(allocId3) + ); + assertClusterHealthStatus(allocation, ClusterHealthStatus.YELLOW); + } + + /** + * Tests that prefer allocation of older primary over replica with higher replication checkpoint + */ + public void testPreferAllocatingPreviousPrimaryWithLowerRepCheckpoint() { + String allocId1 = randomAlphaOfLength(10); + String allocId2 = randomAlphaOfLength(10); + String allocId3 = randomAlphaOfLength(10); + final RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas( + yesAllocationDeciders(), + CLUSTER_RECOVERED, + allocId1, + allocId2, + allocId3 + ); + testAllocator.addData(node1, allocId1, true, new ReplicationCheckpoint(shardId, 10, 10, 101, 1)); + testAllocator.addData(node2, allocId2, false, new ReplicationCheckpoint(shardId, 20, 10, 120, 2)); + testAllocator.addData(node3, allocId3, false, new ReplicationCheckpoint(shardId, 15, 10, 120, 2)); + allocateAllUnassigned(allocation); + assertThat(allocation.routingNodesChanged(), equalTo(true)); + assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true)); + assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), + equalTo(node1.getId()) + ); + // Assert node1's allocation id is used with highest replication checkpoint + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).allocationId().getId(), + equalTo(allocId1) + ); + assertClusterHealthStatus(allocation, ClusterHealthStatus.YELLOW); + } + /** * Tests that when one node returns a ShardLockObtainFailedException and another properly loads the store, it will * select the second node as target @@ -219,7 +417,7 @@ public void testShardLockObtainFailedExceptionPreferOtherValidCopies() { allocId2 ); testAllocator.addData(node1, allocId1, randomBoolean(), new ShardLockObtainFailedException(shardId, "test")); - testAllocator.addData(node2, allocId2, randomBoolean(), null); + testAllocator.addData(node2, allocId2, randomBoolean()); allocateAllUnassigned(allocation); assertThat(allocation.routingNodesChanged(), equalTo(true)); assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true)); @@ -601,17 +799,42 @@ public TestAllocator clear() { return this; } + public TestAllocator addData( + DiscoveryNode node, + String allocationId, + boolean primary, + ReplicationCheckpoint replicationCheckpoint + ) { + return addData(node, allocationId, primary, replicationCheckpoint, null); + } + public TestAllocator addData(DiscoveryNode node, String allocationId, boolean primary) { - return addData(node, allocationId, primary, null); + return addData(node, allocationId, primary, ReplicationCheckpoint.empty(shardId), null); } public TestAllocator addData(DiscoveryNode node, String allocationId, boolean primary, @Nullable Exception storeException) { + return addData(node, allocationId, primary, ReplicationCheckpoint.empty(shardId), storeException); + } + + public TestAllocator addData( + DiscoveryNode node, + String allocationId, + boolean primary, + ReplicationCheckpoint replicationCheckpoint, + @Nullable Exception storeException + ) { if (data == null) { data = new HashMap<>(); } data.put( node, - new TransportNodesListGatewayStartedShards.NodeGatewayStartedShards(node, allocationId, primary, storeException) + new TransportNodesListGatewayStartedShards.NodeGatewayStartedShards( + node, + allocationId, + primary, + replicationCheckpoint, + storeException + ) ); return this; } diff --git a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java index 90aa36253fb7f..e02eac85beafb 100644 --- a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java @@ -42,6 +42,7 @@ import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.translog.Translog; +import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; @@ -854,4 +855,25 @@ public void testEnablingRemoteTranslogStoreFailsWhenRemoteSegmentDisabled() { iae.getMessage() ); } + + public void testEnablingRemoteStoreFailsWhenReplicationTypeIsDocument() { + Settings indexSettings = Settings.builder() + .put("index.replication.type", ReplicationType.DOCUMENT) + .put("index.remote_store.enabled", true) + .build(); + IllegalArgumentException iae = expectThrows( + IllegalArgumentException.class, + () -> IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.get(indexSettings) + ); + assertEquals("To enable index.remote_store.enabled, index.replication.type should be set to SEGMENT", iae.getMessage()); + } + + public void testEnablingRemoteStoreFailsWhenReplicationTypeIsDefault() { + Settings indexSettings = Settings.builder().put("index.remote_store.enabled", true).build(); + IllegalArgumentException iae = expectThrows( + IllegalArgumentException.class, + () -> IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.get(indexSettings) + ); + assertEquals("To enable index.remote_store.enabled, index.replication.type should be set to SEGMENT", iae.getMessage()); + } } diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java index 0275066f9af1b..0a6338333bffc 100644 --- a/server/src/test/java/org/opensearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/opensearch/index/codec/CodecTests.java @@ -34,7 +34,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene92.Lucene92Codec; +import org.apache.lucene.codecs.lucene94.Lucene94Codec; import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; @@ -65,21 +65,21 @@ public class CodecTests extends OpenSearchTestCase { public void testResolveDefaultCodecs() throws Exception { CodecService codecService = createCodecService(); assertThat(codecService.codec("default"), instanceOf(PerFieldMappingPostingFormatCodec.class)); - assertThat(codecService.codec("default"), instanceOf(Lucene92Codec.class)); + assertThat(codecService.codec("default"), instanceOf(Lucene94Codec.class)); } public void testDefault() throws Exception { Codec codec = createCodecService().codec("default"); - assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_SPEED, codec); + assertStoredFieldsCompressionEquals(Lucene94Codec.Mode.BEST_SPEED, codec); } public void testBestCompression() throws Exception { Codec codec = createCodecService().codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene94Codec.Mode.BEST_COMPRESSION, codec); } // write some docs with it, inspect .si to see this was the used compression - private void assertStoredFieldsCompressionEquals(Lucene92Codec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene94Codec.Mode expected, Codec actual) throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(null); iwc.setCodec(actual); @@ -91,7 +91,7 @@ private void assertStoredFieldsCompressionEquals(Lucene92Codec.Mode expected, Co SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader(); String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene92Codec.Mode.valueOf(v)); + assertEquals(expected, Lucene94Codec.Mode.valueOf(v)); ir.close(); dir.close(); } diff --git a/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java b/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java index 340811352a203..575997dc2609e 100644 --- a/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java +++ b/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java @@ -32,7 +32,7 @@ package org.opensearch.index.engine; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene92.Lucene92Codec; +import org.apache.lucene.codecs.lucene94.Lucene94Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -70,7 +70,7 @@ public void testExceptionsAreNotCached() { public void testCompletionStatsCache() throws IOException, InterruptedException { final IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); final PostingsFormat postingsFormat = new Completion90PostingsFormat(); - indexWriterConfig.setCodec(new Lucene92Codec() { + indexWriterConfig.setCodec(new Lucene94Codec() { @Override public PostingsFormat getPostingsFormatForField(String field) { return postingsFormat; // all fields are suggest fields diff --git a/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java b/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java index 7ddd92ea7b36e..269d89352fb18 100644 --- a/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java @@ -16,6 +16,7 @@ import org.opensearch.index.codec.CodecService; import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.seqno.RetentionLeases; +import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.TranslogDeletionPolicy; import org.opensearch.index.translog.TranslogDeletionPolicyFactory; import org.opensearch.index.translog.TranslogReader; @@ -66,7 +67,8 @@ public void testCreateEngineConfigFromFactory() { () -> new RetentionLeases(0, 0, Collections.emptyList()), null, null, - false + false, + new InternalTranslogFactory() ); assertNotNull(config.getCodec()); @@ -143,7 +145,8 @@ public void testCreateCodecServiceFromFactory() { () -> new RetentionLeases(0, 0, Collections.emptyList()), null, null, - false + false, + new InternalTranslogFactory() ); assertNotNull(config.getCodec()); } diff --git a/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java b/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java index 1c6d06e9bcc08..1754d6082b86d 100644 --- a/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java +++ b/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java @@ -13,6 +13,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexSettings; import org.opensearch.index.seqno.RetentionLeases; +import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.OpenSearchTestCase; @@ -102,7 +103,8 @@ private EngineConfig createReadOnlyEngine(IndexSettings indexSettings) { () -> RetentionLeases.EMPTY, null, null, - true + true, + new InternalTranslogFactory() ); } } diff --git a/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java b/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java index 675ff860c3334..1fe1a37dedae0 100644 --- a/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java @@ -12,18 +12,25 @@ import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.SegmentInfos; import org.hamcrest.MatcherAssert; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.search.Queries; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.ParsedDocument; +import org.opensearch.index.seqno.LocalCheckpointTracker; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.store.Store; import org.opensearch.index.translog.TestTranslog; import org.opensearch.index.translog.Translog; +import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.test.IndexSettingsModule; import java.io.IOException; import java.nio.file.Path; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; @@ -31,6 +38,8 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; import static org.opensearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED; +import static org.opensearch.index.seqno.SequenceNumbers.LOCAL_CHECKPOINT_KEY; +import static org.opensearch.index.seqno.SequenceNumbers.MAX_SEQ_NO; public class NRTReplicationEngineTests extends EngineTestCase { @@ -210,6 +219,49 @@ public void testTrimTranslogOps() throws Exception { } } + public void testCommitSegmentInfos() throws Exception { + // This test asserts that NRTReplication#commitSegmentInfos creates a new commit point with the latest checkpoints + // stored in user data. + final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); + final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings( + "index", + Settings.builder().put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT).build() + ); + try ( + final Store nrtEngineStore = createStore(indexSettings, newDirectory()); + final NRTReplicationEngine nrtEngine = buildNrtReplicaEngine(globalCheckpoint, nrtEngineStore) + ) { + List operations = generateHistoryOnReplica(between(1, 500), randomBoolean(), randomBoolean(), randomBoolean()) + .stream() + .filter(op -> op.operationType().equals(Engine.Operation.TYPE.INDEX)) + .collect(Collectors.toList()); + for (Engine.Operation op : operations) { + applyOperation(nrtEngine, op); + } + + final SegmentInfos previousInfos = nrtEngine.getLatestSegmentInfos(); + LocalCheckpointTracker localCheckpointTracker = nrtEngine.getLocalCheckpointTracker(); + final long maxSeqNo = localCheckpointTracker.getMaxSeqNo(); + final long processedCheckpoint = localCheckpointTracker.getProcessedCheckpoint(); + nrtEngine.commitSegmentInfos(); + + // ensure getLatestSegmentInfos returns an updated infos ref with correct userdata. + final SegmentInfos latestSegmentInfos = nrtEngine.getLatestSegmentInfos(); + assertEquals(previousInfos.getGeneration(), latestSegmentInfos.getLastGeneration()); + Map userData = latestSegmentInfos.getUserData(); + assertEquals(processedCheckpoint, localCheckpointTracker.getProcessedCheckpoint()); + assertEquals(maxSeqNo, Long.parseLong(userData.get(MAX_SEQ_NO))); + assertEquals(processedCheckpoint, Long.parseLong(userData.get(LOCAL_CHECKPOINT_KEY))); + + // read infos from store and assert userdata + final String lastCommitSegmentsFileName = SegmentInfos.getLastCommitSegmentsFileName(nrtEngineStore.directory()); + final SegmentInfos committedInfos = SegmentInfos.readCommit(nrtEngineStore.directory(), lastCommitSegmentsFileName); + userData = committedInfos.getUserData(); + assertEquals(processedCheckpoint, Long.parseLong(userData.get(LOCAL_CHECKPOINT_KEY))); + assertEquals(maxSeqNo, Long.parseLong(userData.get(MAX_SEQ_NO))); + } + } + private void assertMatchingSegmentsAndCheckpoints(NRTReplicationEngine nrtEngine, SegmentInfos expectedSegmentInfos) throws IOException { assertEquals(engine.getPersistedLocalCheckpoint(), nrtEngine.getPersistedLocalCheckpoint()); diff --git a/server/src/test/java/org/opensearch/index/fielddata/AbstractStringFieldDataTestCase.java b/server/src/test/java/org/opensearch/index/fielddata/AbstractStringFieldDataTestCase.java index 763ee59a385a2..76496491b3ed4 100644 --- a/server/src/test/java/org/opensearch/index/fielddata/AbstractStringFieldDataTestCase.java +++ b/server/src/test/java/org/opensearch/index/fielddata/AbstractStringFieldDataTestCase.java @@ -52,6 +52,7 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopFieldDocs; +import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.join.QueryBitSetProducer; import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.search.join.ToParentBlockJoinQuery; @@ -340,7 +341,13 @@ public void testSortMissing(boolean first, boolean reverse) throws IOException { randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(sortField) ); - assertEquals(numDocs, topDocs.totalHits.value); + // As of Lucene 9.0.0, totalHits may be a lower bound + if (topDocs.totalHits.relation == TotalHits.Relation.EQUAL_TO) { + assertEquals(numDocs, topDocs.totalHits.value); + } else { + assertTrue(1000 <= topDocs.totalHits.value); + assertTrue(numDocs >= topDocs.totalHits.value); + } BytesRef previousValue = first ? null : reverse ? UnicodeUtil.BIG_TERM : new BytesRef(); for (int i = 0; i < topDocs.scoreDocs.length; ++i) { final String docValue = searcher.doc(topDocs.scoreDocs[i].doc).get("value"); diff --git a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java index 079475d9f3554..d6c89342c9df2 100644 --- a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java @@ -178,6 +178,26 @@ public void testFieldsWithFilledArrayShouldThrowException() throws Exception { } } + public void testDotAsFieldName() throws Exception { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(".") + .field("type", "text") + .endObject() + .endObject() + .endObject() + ); + + try { + createIndex("test").mapperService().documentMapperParser().parse("tweet", new CompressedXContent(mapping)); + fail("Expected MapperParsingException"); + } catch (MapperParsingException e) { + assertThat(e.getMessage(), containsString("Invalid field name")); + } + } + public void testFieldPropertiesArray() throws Exception { String mapping = Strings.toString( XContentFactory.jsonBuilder() diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java index 57c2289c848ef..8c00ab97a46ea 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java @@ -3522,7 +3522,7 @@ public void testCheckpointRefreshListenerWithNull() throws IOException { } /** - * creates a new initializing shard. The shard will will be put in its proper path under the + * creates a new initializing shard. The shard will be put in its proper path under the * current node id the shard is assigned to. * @param checkpointPublisher Segment Replication Checkpoint Publisher to publish checkpoint */ diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index 3fcf6116b11a2..23371a39871c7 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -8,20 +8,108 @@ package org.opensearch.index.shard; +import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.action.index.IndexRequest; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.DocIdSeqNoAndSource; +import org.opensearch.index.engine.InternalEngine; +import org.opensearch.index.engine.NRTReplicationEngine; import org.opensearch.index.engine.NRTReplicationEngineFactory; +import org.opensearch.index.mapper.MapperService; import org.opensearch.index.replication.OpenSearchIndexLevelReplicationTestCase; +import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; +import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.ReplicationType; +import java.io.IOException; +import java.util.List; + +import static java.util.Arrays.asList; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + public class SegmentReplicationIndexShardTests extends OpenSearchIndexLevelReplicationTestCase { private static final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(); + /** + * Test that latestReplicationCheckpoint returns null only for docrep enabled indices + */ + public void testReplicationCheckpointNullForDocRep() throws IOException { + Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_REPLICATION_TYPE, "DOCUMENT").put(Settings.EMPTY).build(); + final IndexShard indexShard = newStartedShard(false, indexSettings); + assertNull(indexShard.getLatestReplicationCheckpoint()); + closeShards(indexShard); + } + + /** + * Test that latestReplicationCheckpoint returns ReplicationCheckpoint for segrep enabled indices + */ + public void testReplicationCheckpointNotNullForSegReb() throws IOException { + Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_REPLICATION_TYPE, "SEGMENT").put(Settings.EMPTY).build(); + final IndexShard indexShard = newStartedShard(indexSettings); + final ReplicationCheckpoint replicationCheckpoint = indexShard.getLatestReplicationCheckpoint(); + assertNotNull(replicationCheckpoint); + closeShards(indexShard); + } + + public void testSegmentReplication_Index_Update_Delete() throws Exception { + String mappings = "{ \"" + MapperService.SINGLE_MAPPING_NAME + "\": { \"properties\": { \"foo\": { \"type\": \"keyword\"} }}}"; + try (ReplicationGroup shards = createGroup(2, settings, mappings, new NRTReplicationEngineFactory())) { + shards.startAll(); + final IndexShard primaryShard = shards.getPrimary(); + + final int numDocs = randomIntBetween(100, 200); + for (int i = 0; i < numDocs; i++) { + shards.index(new IndexRequest(index.getName()).id(String.valueOf(i)).source("{\"foo\": \"bar\"}", XContentType.JSON)); + } + + primaryShard.refresh("Test"); + replicateSegments(primaryShard, shards.getReplicas()); + + shards.assertAllEqual(numDocs); + + for (int i = 0; i < numDocs; i++) { + // randomly update docs. + if (randomBoolean()) { + shards.index( + new IndexRequest(index.getName()).id(String.valueOf(i)).source("{ \"foo\" : \"baz\" }", XContentType.JSON) + ); + } + } + + primaryShard.refresh("Test"); + replicateSegments(primaryShard, shards.getReplicas()); + shards.assertAllEqual(numDocs); + + final List docs = getDocIdAndSeqNos(primaryShard); + for (IndexShard shard : shards.getReplicas()) { + assertEquals(getDocIdAndSeqNos(shard), docs); + } + for (int i = 0; i < numDocs; i++) { + // randomly delete. + if (randomBoolean()) { + shards.delete(new DeleteRequest(index.getName()).id(String.valueOf(i))); + } + } + primaryShard.refresh("Test"); + replicateSegments(primaryShard, shards.getReplicas()); + final List docsAfterDelete = getDocIdAndSeqNos(primaryShard); + for (IndexShard shard : shards.getReplicas()) { + assertEquals(getDocIdAndSeqNos(shard), docsAfterDelete); + } + } + } + public void testIgnoreShardIdle() throws Exception { try (ReplicationGroup shards = createGroup(1, settings, new NRTReplicationEngineFactory())) { shards.startAll(); @@ -56,4 +144,113 @@ public void testIgnoreShardIdle() throws Exception { replica.awaitShardSearchActive(b -> assertFalse("A new RefreshListener should not be registered", b)); } } + + /** + * here we are starting a new primary shard in PrimaryMode and testing if the shard publishes checkpoint after refresh. + */ + public void testPublishCheckpointOnPrimaryMode() throws IOException { + final SegmentReplicationCheckpointPublisher mock = mock(SegmentReplicationCheckpointPublisher.class); + IndexShard shard = newStartedShard(true); + CheckpointRefreshListener refreshListener = new CheckpointRefreshListener(shard, mock); + refreshListener.afterRefresh(true); + + // verify checkpoint is published + verify(mock, times(1)).publish(any()); + closeShards(shard); + } + + /** + * here we are starting a new primary shard in PrimaryMode initially and starting relocation handoff. Later we complete relocation handoff then shard is no longer + * in PrimaryMode, and we test if the shard does not publish checkpoint after refresh. + */ + public void testPublishCheckpointAfterRelocationHandOff() throws IOException { + final SegmentReplicationCheckpointPublisher mock = mock(SegmentReplicationCheckpointPublisher.class); + IndexShard shard = newStartedShard(true); + CheckpointRefreshListener refreshListener = new CheckpointRefreshListener(shard, mock); + String id = shard.routingEntry().allocationId().getId(); + + // Starting relocation handoff + shard.getReplicationTracker().startRelocationHandoff(id); + + // Completing relocation handoff + shard.getReplicationTracker().completeRelocationHandoff(); + refreshListener.afterRefresh(true); + + // verify checkpoint is not published + verify(mock, times(0)).publish(any()); + closeShards(shard); + } + + public void testNRTReplicaPromotedAsPrimary() throws Exception { + try (ReplicationGroup shards = createGroup(2, settings, new NRTReplicationEngineFactory())) { + shards.startAll(); + IndexShard oldPrimary = shards.getPrimary(); + final IndexShard nextPrimary = shards.getReplicas().get(0); + final IndexShard replica = shards.getReplicas().get(1); + + // 1. Create ops that are in the index and xlog of both shards but not yet part of a commit point. + final int numDocs = shards.indexDocs(randomInt(10)); + + // refresh and copy the segments over. + oldPrimary.refresh("Test"); + replicateSegments(oldPrimary, shards.getReplicas()); + + // at this point both shards should have numDocs persisted and searchable. + assertDocCounts(oldPrimary, numDocs, numDocs); + for (IndexShard shard : shards.getReplicas()) { + assertDocCounts(shard, numDocs, numDocs); + } + + // 2. Create ops that are in the replica's xlog, not in the index. + // index some more into both but don't replicate. replica will have only numDocs searchable, but should have totalDocs + // persisted. + final int totalDocs = numDocs + shards.indexDocs(randomInt(10)); + + assertDocCounts(oldPrimary, totalDocs, totalDocs); + for (IndexShard shard : shards.getReplicas()) { + assertDocCounts(shard, totalDocs, numDocs); + } + + // promote the replica + shards.syncGlobalCheckpoint(); + assertEquals(totalDocs, nextPrimary.translogStats().estimatedNumberOfOperations()); + shards.promoteReplicaToPrimary(nextPrimary); + + // close and start the oldPrimary as a replica. + oldPrimary.close("demoted", false); + oldPrimary.store().close(); + oldPrimary = shards.addReplicaWithExistingPath(oldPrimary.shardPath(), oldPrimary.routingEntry().currentNodeId()); + shards.recoverReplica(oldPrimary); + + assertEquals(NRTReplicationEngine.class, oldPrimary.getEngine().getClass()); + assertEquals(InternalEngine.class, nextPrimary.getEngine().getClass()); + assertDocCounts(nextPrimary, totalDocs, totalDocs); + assertEquals(0, nextPrimary.translogStats().estimatedNumberOfOperations()); + + // refresh and push segments to our other replica. + nextPrimary.refresh("test"); + replicateSegments(nextPrimary, asList(replica)); + + for (IndexShard shard : shards) { + assertConsistentHistoryBetweenTranslogAndLucene(shard); + } + final List docsAfterRecovery = getDocIdAndSeqNos(shards.getPrimary()); + for (IndexShard shard : shards.getReplicas()) { + assertThat(shard.routingEntry().toString(), getDocIdAndSeqNos(shard), equalTo(docsAfterRecovery)); + } + } + } + + /** + * Assert persisted and searchable doc counts. This method should not be used while docs are concurrently indexed because + * it asserts point in time seqNos are relative to the doc counts. + */ + private void assertDocCounts(IndexShard indexShard, int expectedPersistedDocCount, int expectedSearchableDocCount) throws IOException { + assertDocCount(indexShard, expectedSearchableDocCount); + // assigned seqNos start at 0, so assert max & local seqNos are 1 less than our persisted doc count. + assertEquals(expectedPersistedDocCount - 1, indexShard.seqNoStats().getMaxSeqNo()); + assertEquals(expectedPersistedDocCount - 1, indexShard.seqNoStats().getLocalCheckpoint()); + // processed cp should be 1 less than our searchable doc count. + assertEquals(expectedSearchableDocCount - 1, indexShard.getProcessedLocalCheckpoint()); + } } diff --git a/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java index 2ded77d2cecfd..97575248b4ad3 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java @@ -15,11 +15,13 @@ import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.support.PlainBlobMetadata; +import org.opensearch.common.collect.Set; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; import java.io.InputStream; import java.nio.file.NoSuchFileException; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -67,6 +69,24 @@ public void testListAllException() throws IOException { assertThrows(IOException.class, () -> remoteDirectory.listAll()); } + public void testListFilesByPrefix() throws IOException { + Map fileNames = Stream.of("abc", "abd", "abe", "abf", "abg") + .collect(Collectors.toMap(filename -> filename, filename -> new PlainBlobMetadata(filename, 100))); + + when(blobContainer.listBlobsByPrefix("ab")).thenReturn(fileNames); + + Collection actualFileNames = remoteDirectory.listFilesByPrefix("ab"); + Collection expectedFileName = Set.of("abc", "abd", "abe", "abf", "abg"); + assertEquals(expectedFileName, actualFileNames); + } + + public void testListFilesByPrefixException() throws IOException { + when(blobContainer.listBlobsByPrefix("abc")).thenThrow(new IOException("Error reading blob store")); + + assertThrows(IOException.class, () -> remoteDirectory.listFilesByPrefix("abc")); + verify(blobContainer).listBlobsByPrefix("abc"); + } + public void testDeleteFile() throws IOException { remoteDirectory.deleteFile("segment_1"); diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java new file mode 100644 index 0000000000000..4eabfa74625f2 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -0,0 +1,339 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.junit.Before; +import org.opensearch.common.collect.Set; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.nio.file.NoSuchFileException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.startsWith; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class RemoteSegmentStoreDirectoryTests extends OpenSearchTestCase { + private RemoteDirectory remoteDataDirectory; + private RemoteDirectory remoteMetadataDirectory; + + private RemoteSegmentStoreDirectory remoteSegmentStoreDirectory; + + @Before + public void setup() throws IOException { + remoteDataDirectory = mock(RemoteDirectory.class); + remoteMetadataDirectory = mock(RemoteDirectory.class); + + remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory(remoteDataDirectory, remoteMetadataDirectory); + } + + public void testUploadedSegmentMetadataToString() { + RemoteSegmentStoreDirectory.UploadedSegmentMetadata metadata = new RemoteSegmentStoreDirectory.UploadedSegmentMetadata( + "abc", + "pqr", + "123456" + ); + assertEquals("abc::pqr::123456", metadata.toString()); + } + + public void testUploadedSegmentMetadataFromString() { + RemoteSegmentStoreDirectory.UploadedSegmentMetadata metadata = RemoteSegmentStoreDirectory.UploadedSegmentMetadata.fromString( + "_0.cfe::_0.cfe__uuidxyz::4567" + ); + assertEquals("_0.cfe::_0.cfe__uuidxyz::4567", metadata.toString()); + } + + public void testGetMetadataFilename() { + // Generation 23 is replaced by n due to radix 32 + assertEquals( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX + "__12__n__uuid1", + RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(12, 23, "uuid1") + ); + } + + public void testGetPrimaryTermGenerationUuid() { + String[] filenameTokens = "abc__12__n__uuid_xyz".split(RemoteSegmentStoreDirectory.MetadataFilenameUtils.SEPARATOR); + assertEquals(12, RemoteSegmentStoreDirectory.MetadataFilenameUtils.getPrimaryTerm(filenameTokens)); + assertEquals(23, RemoteSegmentStoreDirectory.MetadataFilenameUtils.getGeneration(filenameTokens)); + assertEquals("uuid_xyz", RemoteSegmentStoreDirectory.MetadataFilenameUtils.getUuid(filenameTokens)); + } + + public void testMetadataFilenameComparator() { + List metadataFilenames = new ArrayList<>( + List.of( + "abc__10__20__uuid1", + "abc__12__2__uuid2", + "pqr__1__1__uuid0", + "abc__3__n__uuid3", + "abc__10__8__uuid8", + "abc__3__a__uuid4", + "abc__3__a__uuid5" + ) + ); + metadataFilenames.sort(RemoteSegmentStoreDirectory.METADATA_FILENAME_COMPARATOR); + assertEquals( + List.of( + "abc__3__a__uuid4", + "abc__3__a__uuid5", + "abc__3__n__uuid3", + "abc__10__8__uuid8", + "abc__10__20__uuid1", + "abc__12__2__uuid2", + "pqr__1__1__uuid0" + ), + metadataFilenames + ); + } + + public void testInitException() throws IOException { + when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenThrow( + new IOException("Error") + ); + + assertThrows(IOException.class, () -> remoteSegmentStoreDirectory.init()); + } + + public void testInitNoMetadataFile() throws IOException { + when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( + List.of() + ); + + remoteSegmentStoreDirectory.init(); + Map actualCache = remoteSegmentStoreDirectory + .getSegmentsUploadedToRemoteStore(); + + assertEquals(Set.of(), actualCache.keySet()); + } + + private Map getDummyMetadata(String prefix, int commitGeneration) { + Map metadata = new HashMap<>(); + metadata.put(prefix + ".cfe", prefix + ".cfe::" + prefix + ".cfe__qrt::" + randomIntBetween(1000, 5000)); + metadata.put(prefix + ".cfs", prefix + ".cfs::" + prefix + ".cfs__zxd::" + randomIntBetween(1000, 5000)); + metadata.put(prefix + ".si", prefix + ".si::" + prefix + ".si__yui::" + randomIntBetween(1000, 5000)); + metadata.put( + "segments_" + commitGeneration, + "segments_" + commitGeneration + "::segments_" + commitGeneration + "__exv::" + randomIntBetween(1000, 5000) + ); + return metadata; + } + + private void populateMetadata() throws IOException { + List metadataFiles = List.of("metadata__1__5__abc", "metadata__1__6__pqr", "metadata__2__1__zxv"); + when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( + metadataFiles + ); + + IndexInput indexInput = mock(IndexInput.class); + Map dummyMetadata = getDummyMetadata("_0", 1); + when(indexInput.readMapOfStrings()).thenReturn(dummyMetadata); + when(remoteMetadataDirectory.openInput("metadata__2__1__zxv", IOContext.DEFAULT)).thenReturn(indexInput); + } + + public void testInit() throws IOException { + populateMetadata(); + + when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( + List.of("metadata__1__5__abc", "metadata__1__6__pqr", "metadata__2__1__zxv") + ); + + remoteSegmentStoreDirectory.init(); + + Map actualCache = remoteSegmentStoreDirectory + .getSegmentsUploadedToRemoteStore(); + + assertEquals(Set.of("_0.cfe", "_0.cfs", "_0.si", "segments_1"), actualCache.keySet()); + } + + public void testListAll() throws IOException { + populateMetadata(); + + assertEquals(Set.of("_0.cfe", "_0.cfs", "_0.si", "segments_1"), Set.of(remoteSegmentStoreDirectory.listAll())); + } + + public void testDeleteFile() throws IOException { + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + Map uploadedSegments = remoteSegmentStoreDirectory + .getSegmentsUploadedToRemoteStore(); + + assertTrue(uploadedSegments.containsKey("_0.si")); + assertFalse(uploadedSegments.containsKey("_100.si")); + + remoteSegmentStoreDirectory.deleteFile("_0.si"); + remoteSegmentStoreDirectory.deleteFile("_100.si"); + + verify(remoteDataDirectory).deleteFile(startsWith("_0.si")); + verify(remoteDataDirectory, times(0)).deleteFile(startsWith("_100.si")); + assertFalse(uploadedSegments.containsKey("_0.si")); + } + + public void testDeleteFileException() throws IOException { + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + doThrow(new IOException("Error")).when(remoteDataDirectory).deleteFile(any()); + assertThrows(IOException.class, () -> remoteSegmentStoreDirectory.deleteFile("_0.si")); + } + + public void testFileLenght() throws IOException { + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + Map uploadedSegments = remoteSegmentStoreDirectory + .getSegmentsUploadedToRemoteStore(); + + assertTrue(uploadedSegments.containsKey("_0.si")); + + when(remoteDataDirectory.fileLength(startsWith("_0.si"))).thenReturn(1234L); + + assertEquals(1234L, remoteSegmentStoreDirectory.fileLength("_0.si")); + } + + public void testFileLenghtNoSuchFile() throws IOException { + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + Map uploadedSegments = remoteSegmentStoreDirectory + .getSegmentsUploadedToRemoteStore(); + + assertFalse(uploadedSegments.containsKey("_100.si")); + assertThrows(NoSuchFileException.class, () -> remoteSegmentStoreDirectory.fileLength("_100.si")); + } + + public void testCreateOutput() throws IOException { + IndexOutput indexOutput = mock(IndexOutput.class); + when(remoteDataDirectory.createOutput(startsWith("abc"), eq(IOContext.DEFAULT))).thenReturn(indexOutput); + + assertEquals(indexOutput, remoteSegmentStoreDirectory.createOutput("abc", IOContext.DEFAULT)); + } + + public void testCreateOutputException() { + when(remoteDataDirectory.createOutput(startsWith("abc"), eq(IOContext.DEFAULT))).thenThrow(new IOException("Error")); + + assertThrows(IOException.class, () -> remoteSegmentStoreDirectory.createOutput("abc", IOContext.DEFAULT)); + } + + public void testOpenInput() throws IOException { + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + IndexInput indexInput = mock(IndexInput.class); + when(remoteDataDirectory.openInput(startsWith("_0.si"), eq(IOContext.DEFAULT))).thenReturn(indexInput); + + assertEquals(indexInput, remoteSegmentStoreDirectory.openInput("_0.si", IOContext.DEFAULT)); + } + + public void testOpenInputNoSuchFile() { + assertThrows(NoSuchFileException.class, () -> remoteSegmentStoreDirectory.openInput("_0.si", IOContext.DEFAULT)); + } + + public void testOpenInputException() throws IOException { + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + when(remoteDataDirectory.openInput(startsWith("_0.si"), eq(IOContext.DEFAULT))).thenThrow(new IOException("Error")); + + assertThrows(IOException.class, () -> remoteSegmentStoreDirectory.openInput("_0.si", IOContext.DEFAULT)); + } + + public void testCopyFrom() throws IOException { + String filename = "_100.si"; + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + Directory storeDirectory = LuceneTestCase.newDirectory(); + IndexOutput indexOutput = storeDirectory.createOutput(filename, IOContext.DEFAULT); + indexOutput.writeString("Hello World!"); + CodecUtil.writeFooter(indexOutput); + indexOutput.close(); + storeDirectory.sync(List.of(filename)); + + assertFalse(remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().containsKey(filename)); + remoteSegmentStoreDirectory.copyFrom(storeDirectory, filename, filename, IOContext.DEFAULT); + assertTrue(remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().containsKey(filename)); + + storeDirectory.close(); + } + + public void testCopyFromException() throws IOException { + String filename = "_100.si"; + Directory storeDirectory = LuceneTestCase.newDirectory(); + assertFalse(remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().containsKey(filename)); + doThrow(new IOException("Error")).when(remoteDataDirectory).copyFrom(storeDirectory, filename, filename, IOContext.DEFAULT); + + assertThrows(IOException.class, () -> remoteSegmentStoreDirectory.copyFrom(storeDirectory, filename, filename, IOContext.DEFAULT)); + assertFalse(remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().containsKey(filename)); + + storeDirectory.close(); + } + + public void testContainsFile() throws IOException { + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + // This is not the correct way to add files but the other way is to open up access to fields in UploadedSegmentMetadata + Map uploadedSegmentMetadataMap = remoteSegmentStoreDirectory + .getSegmentsUploadedToRemoteStore(); + uploadedSegmentMetadataMap.put( + "_100.si", + new RemoteSegmentStoreDirectory.UploadedSegmentMetadata("_100.si", "_100.si__uuid1", "1234") + ); + + assertTrue(remoteSegmentStoreDirectory.containsFile("_100.si", "1234")); + assertFalse(remoteSegmentStoreDirectory.containsFile("_100.si", "2345")); + assertFalse(remoteSegmentStoreDirectory.containsFile("_200.si", "1234")); + } + + public void testUploadMetadataEmpty() throws IOException { + Directory storeDirectory = mock(Directory.class); + IndexOutput indexOutput = mock(IndexOutput.class); + when(storeDirectory.createOutput(startsWith("metadata__12__o"), eq(IOContext.DEFAULT))).thenReturn(indexOutput); + + Collection segmentFiles = List.of("s1", "s2", "s3"); + assertThrows(NoSuchFileException.class, () -> remoteSegmentStoreDirectory.uploadMetadata(segmentFiles, storeDirectory, 12L, 24L)); + } + + public void testUploadMetadataNonEmpty() throws IOException { + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + Directory storeDirectory = mock(Directory.class); + IndexOutput indexOutput = mock(IndexOutput.class); + when(storeDirectory.createOutput(startsWith("metadata__12__o"), eq(IOContext.DEFAULT))).thenReturn(indexOutput); + + Collection segmentFiles = List.of("_0.si"); + remoteSegmentStoreDirectory.uploadMetadata(segmentFiles, storeDirectory, 12L, 24L); + + verify(remoteMetadataDirectory).copyFrom( + eq(storeDirectory), + startsWith("metadata__12__o"), + startsWith("metadata__12__o"), + eq(IOContext.DEFAULT) + ); + String metadataString = remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().get("_0.si").toString(); + verify(indexOutput).writeMapOfStrings(Map.of("_0.si", metadataString)); + } +} diff --git a/server/src/test/java/org/opensearch/index/translog/InternalTranslogManagerTests.java b/server/src/test/java/org/opensearch/index/translog/InternalTranslogManagerTests.java index 5171f0dfa1d18..234abfba66622 100644 --- a/server/src/test/java/org/opensearch/index/translog/InternalTranslogManagerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/InternalTranslogManagerTests.java @@ -47,7 +47,8 @@ public void testRecoveryFromTranslog() throws IOException { () -> tracker, translogUUID, TranslogEventListener.NOOP_TRANSLOG_EVENT_LISTENER, - () -> {} + () -> {}, + new InternalTranslogFactory() ); final int docs = randomIntBetween(1, 100); for (int i = 0; i < docs; i++) { @@ -85,7 +86,8 @@ public void onBeginTranslogRecovery() { beginTranslogRecoveryInvoked.set(true); } }, - () -> {} + () -> {}, + new InternalTranslogFactory() ); AtomicInteger opsRecovered = new AtomicInteger(); int opsRecoveredFromTranslog = translogManager.recoverFromTranslog((snapshot) -> { @@ -122,7 +124,8 @@ public void testTranslogRollsGeneration() throws IOException { () -> tracker, translogUUID, TranslogEventListener.NOOP_TRANSLOG_EVENT_LISTENER, - () -> {} + () -> {}, + new InternalTranslogFactory() ); final int docs = randomIntBetween(1, 100); for (int i = 0; i < docs; i++) { @@ -150,7 +153,8 @@ public void testTranslogRollsGeneration() throws IOException { () -> new LocalCheckpointTracker(NO_OPS_PERFORMED, NO_OPS_PERFORMED), translogUUID, TranslogEventListener.NOOP_TRANSLOG_EVENT_LISTENER, - () -> {} + () -> {}, + new InternalTranslogFactory() ); AtomicInteger opsRecovered = new AtomicInteger(); int opsRecoveredFromTranslog = translogManager.recoverFromTranslog((snapshot) -> { @@ -183,7 +187,8 @@ public void testTrimOperationsFromTranslog() throws IOException { () -> tracker, translogUUID, TranslogEventListener.NOOP_TRANSLOG_EVENT_LISTENER, - () -> {} + () -> {}, + new InternalTranslogFactory() ); final int docs = randomIntBetween(1, 100); for (int i = 0; i < docs; i++) { @@ -213,7 +218,8 @@ public void testTrimOperationsFromTranslog() throws IOException { () -> new LocalCheckpointTracker(NO_OPS_PERFORMED, NO_OPS_PERFORMED), translogUUID, TranslogEventListener.NOOP_TRANSLOG_EVENT_LISTENER, - () -> {} + () -> {}, + new InternalTranslogFactory() ); AtomicInteger opsRecovered = new AtomicInteger(); int opsRecoveredFromTranslog = translogManager.recoverFromTranslog((snapshot) -> { @@ -260,7 +266,8 @@ public void onAfterTranslogSync() { } } }, - () -> {} + () -> {}, + new InternalTranslogFactory() ); translogManagerAtomicReference.set(translogManager); Engine.Index index = indexForDoc(doc); diff --git a/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java b/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java index d42e75871a45a..38c55620e1223 100644 --- a/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java @@ -155,6 +155,9 @@ public void testCancelReplication() throws IOException { } public void testMultipleReplicasUseSameCheckpoint() throws IOException { + IndexShard secondReplica = newShard(primary.shardId(), false); + recoverReplica(secondReplica, primary, true); + OngoingSegmentReplications replications = new OngoingSegmentReplications(mockIndicesService, recoverySettings); final CheckpointInfoRequest request = new CheckpointInfoRequest( 1L, @@ -172,7 +175,7 @@ public void testMultipleReplicasUseSameCheckpoint() throws IOException { final CheckpointInfoRequest secondRequest = new CheckpointInfoRequest( 1L, - replica.routingEntry().allocationId().getId(), + secondReplica.routingEntry().allocationId().getId(), replicaDiscoveryNode, testCheckpoint ); @@ -187,6 +190,7 @@ public void testMultipleReplicasUseSameCheckpoint() throws IOException { assertEquals(0, copyState.refCount()); assertEquals(0, replications.size()); assertEquals(0, replications.cachedCopyStateSize()); + closeShards(secondReplica); } public void testStartCopyWithoutPrepareStep() { @@ -272,4 +276,40 @@ public void onFailure(Exception e) { } }); } + + public void testCancelAllReplicationsForShard() throws IOException { + // This tests when primary has multiple ongoing replications. + IndexShard replica_2 = newShard(primary.shardId(), false); + recoverReplica(replica_2, primary, true); + + OngoingSegmentReplications replications = new OngoingSegmentReplications(mockIndicesService, recoverySettings); + final CheckpointInfoRequest request = new CheckpointInfoRequest( + 1L, + replica.routingEntry().allocationId().getId(), + primaryDiscoveryNode, + testCheckpoint + ); + + final CopyState copyState = replications.prepareForReplication(request, mock(FileChunkWriter.class)); + assertEquals(1, copyState.refCount()); + + final CheckpointInfoRequest secondRequest = new CheckpointInfoRequest( + 1L, + replica_2.routingEntry().allocationId().getId(), + replicaDiscoveryNode, + testCheckpoint + ); + replications.prepareForReplication(secondRequest, mock(FileChunkWriter.class)); + + assertEquals(2, copyState.refCount()); + assertEquals(2, replications.size()); + assertEquals(1, replications.cachedCopyStateSize()); + + // cancel the primary's ongoing replications. + replications.cancel(primary, "Test"); + assertEquals(0, copyState.refCount()); + assertEquals(0, replications.size()); + assertEquals(0, replications.cachedCopyStateSize()); + closeShards(replica_2); + } } diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java index 70061c54d0da2..2c52772649acc 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java @@ -15,7 +15,9 @@ import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.action.ActionListener; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.settings.Settings; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; import org.opensearch.index.store.StoreFileMetadata; @@ -41,7 +43,8 @@ public class SegmentReplicationSourceHandlerTests extends IndexShardTestCase { @Override public void setUp() throws Exception { super.setUp(); - primary = newStartedShard(true); + final Settings settings = Settings.builder().put(IndexMetadata.SETTING_REPLICATION_TYPE, "SEGMENT").put(Settings.EMPTY).build(); + primary = newStartedShard(true, settings); replica = newShard(primary.shardId(), false); recoverReplica(replica, primary, true); replicaDiscoveryNode = replica.recoveryState().getTargetNode(); @@ -63,6 +66,7 @@ public void testSendFiles() throws IOException { chunkWriter, threadPool, copyState, + primary.routingEntry().allocationId().getId(), 5000, 1 ); @@ -100,6 +104,7 @@ public void testSendFiles_emptyRequest() throws IOException { chunkWriter, threadPool, copyState, + primary.routingEntry().allocationId().getId(), 5000, 1 ); @@ -138,6 +143,7 @@ public void testSendFileFails() throws IOException { chunkWriter, threadPool, copyState, + primary.routingEntry().allocationId().getId(), 5000, 1 ); @@ -175,6 +181,7 @@ public void testReplicationAlreadyRunning() throws IOException { chunkWriter, threadPool, copyState, + primary.routingEntry().allocationId().getId(), 5000, 1 ); diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java index 8b4bda7de50ad..d3a6d1a97dacc 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java @@ -13,6 +13,7 @@ import org.mockito.Mockito; import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.index.shard.IndexShard; @@ -50,7 +51,10 @@ public class SegmentReplicationTargetServiceTests extends IndexShardTestCase { @Override public void setUp() throws Exception { super.setUp(); - final Settings settings = Settings.builder().put("node.name", SegmentReplicationTargetServiceTests.class.getSimpleName()).build(); + final Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, "SEGMENT") + .put("node.name", SegmentReplicationTargetServiceTests.class.getSimpleName()) + .build(); final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings); final TransportService transportService = mock(TransportService.class); @@ -96,8 +100,8 @@ public void onReplicationFailure(SegmentReplicationState state, OpenSearchExcept ); final SegmentReplicationTarget spy = Mockito.spy(target); doAnswer(invocation -> { - // setting stage to REPLICATING so transition in markAsDone succeeds on listener completion - target.state().setStage(SegmentReplicationState.Stage.REPLICATING); + // set up stage correctly so the transition in markAsDone succeeds on listener completion + moveTargetToFinalStage(target); final ActionListener listener = invocation.getArgument(0); listener.onResponse(null); return null; @@ -119,7 +123,7 @@ public void onReplicationDone(SegmentReplicationState state) { @Override public void onReplicationFailure(SegmentReplicationState state, OpenSearchException e, boolean sendShardFailure) { - assertEquals(SegmentReplicationState.Stage.REPLICATING, state.getStage()); + assertEquals(SegmentReplicationState.Stage.INIT, state.getStage()); assertEquals(expectedError, e.getCause()); assertTrue(sendShardFailure); } @@ -127,8 +131,6 @@ public void onReplicationFailure(SegmentReplicationState state, OpenSearchExcept ); final SegmentReplicationTarget spy = Mockito.spy(target); doAnswer(invocation -> { - // setting stage to REPLICATING so transition in markAsDone succeeds on listener completion - target.state().setStage(SegmentReplicationState.Stage.REPLICATING); final ActionListener listener = invocation.getArgument(0); listener.onFailure(expectedError); return null; @@ -204,6 +206,23 @@ public void testNewCheckpoint_validationPassesAndReplicationFails() throws IOExc closeShard(indexShard, false); } + /** + * here we are starting a new shard in PrimaryMode and testing that we don't process a checkpoint on shard when it is in PrimaryMode. + */ + public void testRejectCheckpointOnShardPrimaryMode() throws IOException { + SegmentReplicationTargetService spy = spy(sut); + + // Starting a new shard in PrimaryMode. + IndexShard primaryShard = newStartedShard(true); + IndexShard spyShard = spy(primaryShard); + doNothing().when(spy).startReplication(any(), any(), any()); + spy.onNewCheckpoint(aheadCheckpoint, spyShard); + + // Verify that checkpoint is not processed as shard is in PrimaryMode. + verify(spy, times(0)).startReplication(any(), any(), any()); + closeShards(primaryShard); + } + public void testReplicationOnDone() throws IOException { SegmentReplicationTargetService spy = spy(sut); IndexShard spyShard = spy(indexShard); @@ -250,4 +269,17 @@ public void testBeforeIndexShardClosed_CancelsOngoingReplications() { sut.beforeIndexShardClosed(indexShard.shardId(), indexShard, Settings.EMPTY); verify(spy, times(1)).cancel(any()); } + + /** + * Move the {@link SegmentReplicationTarget} object through its {@link SegmentReplicationState.Stage} values in order + * until the final, non-terminal stage. + */ + private void moveTargetToFinalStage(SegmentReplicationTarget target) { + SegmentReplicationState.Stage[] stageValues = SegmentReplicationState.Stage.values(); + assertEquals(target.state().getStage(), SegmentReplicationState.Stage.INIT); + // Skip the first two stages (DONE and INIT) and iterate until the last value + for (int i = 2; i < stageValues.length; i++) { + target.state().setStage(stageValues[i]); + } + } } diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java index a0944ee249859..11217a46b3c69 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java @@ -8,29 +8,52 @@ package org.opensearch.indices.replication; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.IndexFormatTooNewException; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.IndexFormatTooNewException; +import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.store.ByteBuffersDataOutput; import org.apache.lucene.store.ByteBuffersIndexOutput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.util.TestUtil; import org.apache.lucene.util.Version; import org.junit.Assert; import org.mockito.Mockito; +import org.opensearch.ExceptionsHelper; +import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; +import org.opensearch.index.shard.ShardId; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; +import org.opensearch.index.store.StoreTests; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.test.DummyShardLock; +import org.opensearch.test.IndexSettingsModule; +import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.file.NoSuchFileException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.Random; +import java.util.Arrays; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; @@ -69,7 +92,12 @@ public class SegmentReplicationTargetTests extends IndexShardTestCase { 0 ); - SegmentInfos testSegmentInfos; + private static final IndexSettings INDEX_SETTINGS = IndexSettingsModule.newIndexSettings( + "index", + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, org.opensearch.Version.CURRENT).build() + ); + + private SegmentInfos testSegmentInfos; @Override public void setUp() throws Exception { @@ -135,6 +163,7 @@ public void getSegmentFiles( public void onResponse(Void replicationResponse) { try { verify(spyIndexShard, times(1)).finalizeReplication(any(), anyLong()); + segrepTarget.markAsDone(); } catch (IOException ex) { Assert.fail(); } @@ -142,7 +171,7 @@ public void onResponse(Void replicationResponse) { @Override public void onFailure(Exception e) { - logger.error("Unexpected test error", e); + logger.error("Unexpected onFailure", e); Assert.fail(); } }); @@ -186,6 +215,7 @@ public void onResponse(Void replicationResponse) { @Override public void onFailure(Exception e) { assertEquals(exception, e.getCause().getCause()); + segrepTarget.fail(new OpenSearchException(e), false); } }); } @@ -228,6 +258,7 @@ public void onResponse(Void replicationResponse) { @Override public void onFailure(Exception e) { assertEquals(exception, e.getCause().getCause()); + segrepTarget.fail(new OpenSearchException(e), false); } }); } @@ -272,6 +303,7 @@ public void onResponse(Void replicationResponse) { @Override public void onFailure(Exception e) { assertEquals(exception, e.getCause()); + segrepTarget.fail(new OpenSearchException(e), false); } }); } @@ -316,6 +348,7 @@ public void onResponse(Void replicationResponse) { @Override public void onFailure(Exception e) { assertEquals(exception, e.getCause()); + segrepTarget.fail(new OpenSearchException(e), false); } }); } @@ -357,14 +390,123 @@ public void onResponse(Void replicationResponse) { @Override public void onFailure(Exception e) { assert (e instanceof IllegalStateException); + segrepTarget.fail(new OpenSearchException(e), false); + } + }); + } + + /** + * This tests ensures that new files generated on primary (due to delete operation) are not considered missing on replica + * @throws IOException + */ + public void test_MissingFiles_NotCausingFailure() throws IOException { + int docCount = 1 + random().nextInt(10); + // Generate a list of MetadataSnapshot containing two elements. The second snapshot contains extra files + // generated due to delete operations. These two snapshots can then be used in test to mock the primary shard + // snapshot (2nd element which contains delete operations) and replica's existing snapshot (1st element). + List storeMetadataSnapshots = generateStoreMetadataSnapshot(docCount); + + SegmentReplicationSource segrepSource = new SegmentReplicationSource() { + @Override + public void getCheckpointMetadata( + long replicationId, + ReplicationCheckpoint checkpoint, + ActionListener listener + ) { + listener.onResponse( + new CheckpointInfoResponse(checkpoint, storeMetadataSnapshots.get(1), buffer.toArrayCopy(), Set.of(PENDING_DELETE_FILE)) + ); + } + + @Override + public void getSegmentFiles( + long replicationId, + ReplicationCheckpoint checkpoint, + List filesToFetch, + Store store, + ActionListener listener + ) { + listener.onResponse(new GetSegmentFilesResponse(filesToFetch)); + } + }; + SegmentReplicationTargetService.SegmentReplicationListener segRepListener = mock( + SegmentReplicationTargetService.SegmentReplicationListener.class + ); + + segrepTarget = spy(new SegmentReplicationTarget(repCheckpoint, indexShard, segrepSource, segRepListener)); + when(segrepTarget.getMetadataSnapshot()).thenReturn(storeMetadataSnapshots.get(0)); + segrepTarget.startReplication(new ActionListener() { + @Override + public void onResponse(Void replicationResponse) { + logger.info("No error processing checkpoint info"); + segrepTarget.markAsDone(); + } + + @Override + public void onFailure(Exception e) { + logger.error("Unexpected onFailure", e); + Assert.fail(); } }); } + /** + * Generates a list of Store.MetadataSnapshot with two elements where second snapshot has extra files due to delete + * operation. A list of snapshots is returned so that identical files have same checksum. + * @param docCount + * @return + * @throws IOException + */ + private List generateStoreMetadataSnapshot(int docCount) throws IOException { + List docList = new ArrayList<>(); + for (int i = 0; i < docCount; i++) { + Document document = new Document(); + String text = new String(new char[] { (char) (97 + i), (char) (97 + i) }); + document.add(new StringField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO)); + document.add(new TextField("str", text, Field.Store.YES)); + docList.add(document); + } + long seed = random().nextLong(); + Random random = new Random(seed); + IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random)).setCodec(TestUtil.getDefaultCodec()); + iwc.setMergePolicy(NoMergePolicy.INSTANCE); + iwc.setUseCompoundFile(true); + final ShardId shardId = new ShardId("index", "_na_", 1); + Store store = new Store(shardId, INDEX_SETTINGS, StoreTests.newDirectory(random()), new DummyShardLock(shardId)); + IndexWriter writer = new IndexWriter(store.directory(), iwc); + for (Document d : docList) { + writer.addDocument(d); + } + writer.commit(); + Store.MetadataSnapshot storeMetadata = store.getMetadata(); + // Delete one document to generate .liv file + writer.deleteDocuments(new Term("id", Integer.toString(random().nextInt(docCount)))); + writer.commit(); + Store.MetadataSnapshot storeMetadataWithDeletes = store.getMetadata(); + deleteContent(store.directory()); + writer.close(); + store.close(); + return Arrays.asList(storeMetadata, storeMetadataWithDeletes); + } + + private static void deleteContent(Directory directory) throws IOException { + final String[] files = directory.listAll(); + final List exceptions = new ArrayList<>(); + for (String file : files) { + try { + directory.deleteFile(file); + } catch (NoSuchFileException | FileNotFoundException e) { + // ignore + } catch (IOException e) { + exceptions.add(e); + } + } + ExceptionsHelper.rethrowAndSuppress(exceptions); + } + @Override public void tearDown() throws Exception { super.tearDown(); - segrepTarget.markAsDone(); closeShards(spyIndexShard, indexShard); } } diff --git a/server/src/test/java/org/opensearch/search/CreatePitSingleNodeTests.java b/server/src/test/java/org/opensearch/search/CreatePitSingleNodeTests.java index 1e1720317b7c2..811136d4af973 100644 --- a/server/src/test/java/org/opensearch/search/CreatePitSingleNodeTests.java +++ b/server/src/test/java/org/opensearch/search/CreatePitSingleNodeTests.java @@ -14,6 +14,10 @@ import org.opensearch.action.search.CreatePitController; import org.opensearch.action.search.CreatePitRequest; import org.opensearch.action.search.CreatePitResponse; +import org.opensearch.action.search.DeletePitAction; +import org.opensearch.action.search.DeletePitInfo; +import org.opensearch.action.search.DeletePitRequest; +import org.opensearch.action.search.DeletePitResponse; import org.opensearch.action.search.PitTestsUtil; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; @@ -33,6 +37,8 @@ import java.util.concurrent.ExecutionException; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.blankOrNullString; +import static org.hamcrest.Matchers.not; import static org.opensearch.action.search.PitTestsUtil.assertSegments; import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; @@ -277,6 +283,24 @@ public void testMaxOpenPitContexts() throws Exception { ) ); final int maxPitContexts = SearchService.MAX_OPEN_PIT_CONTEXT.get(Settings.EMPTY); + DeletePitRequest deletePITRequest = new DeletePitRequest("_all"); + + /** + * When we invoke delete again, returns success after clearing the remaining readers. Asserting reader context + * not found exceptions don't result in failures ( as deletion in one node is successful ) + */ + ActionFuture execute1 = client().execute(DeletePitAction.INSTANCE, deletePITRequest); + DeletePitResponse deletePITResponse = execute1.get(); + for (DeletePitInfo deletePitInfo : deletePITResponse.getDeletePitResults()) { + assertThat(deletePitInfo.getPitId(), not(blankOrNullString())); + assertTrue(deletePitInfo.isSuccessful()); + } + assertEquals(0,service.getActiveContexts()); + execute = client().execute(CreatePitAction.INSTANCE, request); + CreatePitResponse pitResponse = execute.get(); + assertEquals(1,service.getActiveContexts()); + + validatePitStats("index", 0, maxPitContexts, 0); validatePitStats("index", maxPitContexts, 0, 0); service.doClose(); validatePitStats("index", 0, maxPitContexts, 0); diff --git a/server/src/test/java/org/opensearch/search/DocValueFormatTests.java b/server/src/test/java/org/opensearch/search/DocValueFormatTests.java index 36a6eb3ae87b0..bd0fbfe69960c 100644 --- a/server/src/test/java/org/opensearch/search/DocValueFormatTests.java +++ b/server/src/test/java/org/opensearch/search/DocValueFormatTests.java @@ -48,7 +48,7 @@ import java.util.ArrayList; import java.util.List; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.longEncode; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.longEncode; public class DocValueFormatTests extends OpenSearchTestCase { diff --git a/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java b/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java index 421865013a28c..94fb6cded637d 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java @@ -48,8 +48,6 @@ import org.opensearch.search.aggregations.bucket.composite.InternalCompositeTests; import org.opensearch.search.aggregations.bucket.filter.InternalFilterTests; import org.opensearch.search.aggregations.bucket.filter.InternalFiltersTests; -import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridTests; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridTests; import org.opensearch.search.aggregations.bucket.global.InternalGlobalTests; import org.opensearch.search.aggregations.bucket.histogram.InternalAutoDateHistogramTests; import org.opensearch.search.aggregations.bucket.histogram.InternalDateHistogramTests; @@ -80,7 +78,6 @@ import org.opensearch.search.aggregations.metrics.InternalSumTests; import org.opensearch.search.aggregations.metrics.InternalAvgTests; import org.opensearch.search.aggregations.metrics.InternalCardinalityTests; -import org.opensearch.search.aggregations.metrics.InternalGeoBoundsTests; import org.opensearch.search.aggregations.metrics.InternalGeoCentroidTests; import org.opensearch.search.aggregations.metrics.InternalHDRPercentilesRanksTests; import org.opensearch.search.aggregations.metrics.InternalHDRPercentilesTests; @@ -142,7 +139,6 @@ private static List> getAggsTests() { aggsTests.add(new InternalStatsBucketTests()); aggsTests.add(new InternalExtendedStatsTests()); aggsTests.add(new InternalExtendedStatsBucketTests()); - aggsTests.add(new InternalGeoBoundsTests()); aggsTests.add(new InternalGeoCentroidTests()); aggsTests.add(new InternalHistogramTests()); aggsTests.add(new InternalDateHistogramTests()); @@ -159,8 +155,6 @@ private static List> getAggsTests() { aggsTests.add(new InternalGlobalTests()); aggsTests.add(new InternalFilterTests()); aggsTests.add(new InternalSamplerTests()); - aggsTests.add(new GeoHashGridTests()); - aggsTests.add(new GeoTileGridTests()); aggsTests.add(new InternalRangeTests()); aggsTests.add(new InternalDateRangeTests()); aggsTests.add(new InternalGeoDistanceTests()); diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilderTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilderTests.java index c4a87f3993bb4..9290183ec7312 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilderTests.java @@ -32,10 +32,8 @@ package org.opensearch.search.aggregations.bucket.composite; -import org.opensearch.common.geo.GeoBoundingBoxTests; import org.opensearch.script.Script; import org.opensearch.search.aggregations.BaseAggregationTestCase; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.opensearch.search.aggregations.bucket.missing.MissingOrder; import org.opensearch.search.sort.SortOrder; @@ -74,17 +72,6 @@ private DateHistogramValuesSourceBuilder randomDateHistogramSourceBuilder() { return histo; } - private GeoTileGridValuesSourceBuilder randomGeoTileGridValuesSourceBuilder() { - GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder(randomAlphaOfLengthBetween(5, 10)); - if (randomBoolean()) { - geoTile.precision(randomIntBetween(0, GeoTileUtils.MAX_ZOOM)); - } - if (randomBoolean()) { - geoTile.geoBoundingBox(GeoBoundingBoxTests.randomBBox()); - } - return geoTile; - } - private TermsValuesSourceBuilder randomTermsSourceBuilder() { TermsValuesSourceBuilder terms = new TermsValuesSourceBuilder(randomAlphaOfLengthBetween(5, 10)); if (randomBoolean()) { @@ -118,11 +105,9 @@ private HistogramValuesSourceBuilder randomHistogramSourceBuilder() { @Override protected CompositeAggregationBuilder createTestAggregatorBuilder() { int numSources = randomIntBetween(1, 10); - numSources = 1; List> sources = new ArrayList<>(); for (int i = 0; i < numSources; i++) { - int type = randomIntBetween(0, 3); - type = 3; + int type = randomIntBetween(0, 2); switch (type) { case 0: sources.add(randomTermsSourceBuilder()); @@ -133,9 +118,6 @@ protected CompositeAggregationBuilder createTestAggregatorBuilder() { case 2: sources.add(randomHistogramSourceBuilder()); break; - case 3: - sources.add(randomGeoTileGridValuesSourceBuilder()); - break; default: throw new AssertionError("wrong branch"); } diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index 88b2323b8adfc..25003e0b84567 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -32,68 +32,24 @@ package org.opensearch.search.aggregations.bucket.composite; -import org.apache.lucene.tests.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.DoublePoint; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.InetAddressPoint; -import org.apache.lucene.document.IntPoint; -import org.apache.lucene.document.LatLonPoint; import org.apache.lucene.document.LongPoint; -import org.apache.lucene.document.SortedNumericDocValuesField; -import org.apache.lucene.document.SortedSetDocValuesField; -import org.apache.lucene.document.StringField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.search.DocValuesFieldExistsQuery; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.SortedNumericSortField; -import org.apache.lucene.search.SortedSetSortField; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.NumericUtils; -import org.apache.lucene.tests.util.TestUtil; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.geo.GeoPoint; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.text.Text; -import org.opensearch.common.time.DateFormatter; -import org.opensearch.common.time.DateFormatters; -import org.opensearch.index.Index; -import org.opensearch.index.IndexSettings; -import org.opensearch.index.mapper.DateFieldMapper; -import org.opensearch.index.mapper.DocumentMapper; -import org.opensearch.index.mapper.GeoPointFieldMapper; -import org.opensearch.index.mapper.IpFieldMapper; -import org.opensearch.index.mapper.KeywordFieldMapper; -import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.search.aggregations.Aggregator; -import org.opensearch.search.aggregations.AggregatorTestCase; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.opensearch.search.aggregations.bucket.missing.MissingOrder; import org.opensearch.search.aggregations.bucket.terms.StringTerms; import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.opensearch.search.aggregations.composite.BaseCompositeAggregatorTestCase; import org.opensearch.search.aggregations.metrics.InternalMax; import org.opensearch.search.aggregations.metrics.MaxAggregationBuilder; import org.opensearch.search.aggregations.metrics.TopHits; import org.opensearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.opensearch.search.aggregations.support.ValueType; import org.opensearch.search.sort.SortOrder; -import org.opensearch.test.IndexSettingsModule; -import org.junit.After; -import org.junit.Before; import java.io.IOException; import java.net.InetAddress; @@ -109,51 +65,14 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; -import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class CompositeAggregatorTests extends AggregatorTestCase { - private static MappedFieldType[] FIELD_TYPES; - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - FIELD_TYPES = new MappedFieldType[8]; - FIELD_TYPES[0] = new KeywordFieldMapper.KeywordFieldType("keyword"); - FIELD_TYPES[1] = new NumberFieldMapper.NumberFieldType("long", NumberFieldMapper.NumberType.LONG); - FIELD_TYPES[2] = new NumberFieldMapper.NumberFieldType("double", NumberFieldMapper.NumberType.DOUBLE); - FIELD_TYPES[3] = new DateFieldMapper.DateFieldType("date", DateFormatter.forPattern("yyyy-MM-dd||epoch_millis")); - FIELD_TYPES[4] = new NumberFieldMapper.NumberFieldType("price", NumberFieldMapper.NumberType.INTEGER); - FIELD_TYPES[5] = new KeywordFieldMapper.KeywordFieldType("terms"); - FIELD_TYPES[6] = new IpFieldMapper.IpFieldType("ip"); - FIELD_TYPES[7] = new GeoPointFieldMapper.GeoPointFieldType("geo_point"); - } - - @Override - @After - public void tearDown() throws Exception { - super.tearDown(); - FIELD_TYPES = null; - } - @Override - protected MapperService mapperServiceMock() { - MapperService mapperService = mock(MapperService.class); - DocumentMapper mapper = mock(DocumentMapper.class); - when(mapper.typeText()).thenReturn(new Text("_doc")); - when(mapper.type()).thenReturn("_doc"); - when(mapperService.documentMapper()).thenReturn(mapper); - return mapperService; - } +public class CompositeAggregatorTests extends BaseCompositeAggregatorTestCase { public void testUnmappedFieldWithTerms() throws Exception { final List>> dataset = new ArrayList<>(); @@ -234,80 +153,6 @@ public void testUnmappedFieldWithTerms() throws Exception { ); } - public void testUnmappedFieldWithGeopoint() throws Exception { - final List>> dataset = new ArrayList<>(); - final String mappedFieldName = "geo_point"; - dataset.addAll( - Arrays.asList( - createDocument(mappedFieldName, new GeoPoint(48.934059, 41.610741)), - createDocument(mappedFieldName, new GeoPoint(-23.065941, 113.610741)), - createDocument(mappedFieldName, new GeoPoint(90.0, 0.0)), - createDocument(mappedFieldName, new GeoPoint(37.2343, -115.8067)), - createDocument(mappedFieldName, new GeoPoint(90.0, 0.0)) - ) - ); - - // just unmapped = no results - testSearchCase( - Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), - dataset, - () -> new CompositeAggregationBuilder("name", Arrays.asList(new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped"))), - (result) -> assertEquals(0, result.getBuckets().size()) - ); - - // unmapped missing bucket = one result - testSearchCase( - Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), - dataset, - () -> new CompositeAggregationBuilder( - "name", - Arrays.asList(new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped").missingBucket(true)) - ), - (result) -> { - assertEquals(1, result.getBuckets().size()); - assertEquals("{unmapped=null}", result.afterKey().toString()); - assertEquals("{unmapped=null}", result.getBuckets().get(0).getKeyAsString()); - assertEquals(5L, result.getBuckets().get(0).getDocCount()); - } - ); - - // field + unmapped, no missing bucket = no results - testSearchCase( - Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), - dataset, - () -> new CompositeAggregationBuilder( - "name", - Arrays.asList( - new GeoTileGridValuesSourceBuilder(mappedFieldName).field(mappedFieldName), - new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped") - ) - ), - (result) -> assertEquals(0, result.getBuckets().size()) - ); - - // field + unmapped with missing bucket = multiple results - testSearchCase( - Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), - dataset, - () -> new CompositeAggregationBuilder( - "name", - Arrays.asList( - new GeoTileGridValuesSourceBuilder(mappedFieldName).field(mappedFieldName), - new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped").missingBucket(true) - ) - ), - (result) -> { - assertEquals(2, result.getBuckets().size()); - assertEquals("{geo_point=7/64/56, unmapped=null}", result.afterKey().toString()); - assertEquals("{geo_point=7/32/56, unmapped=null}", result.getBuckets().get(0).getKeyAsString()); - assertEquals(2L, result.getBuckets().get(0).getDocCount()); - assertEquals("{geo_point=7/64/56, unmapped=null}", result.getBuckets().get(1).getKeyAsString()); - assertEquals(3L, result.getBuckets().get(1).getDocCount()); - } - ); - - } - public void testUnmappedFieldWithHistogram() throws Exception { final List>> dataset = new ArrayList<>(); final String mappedFieldName = "price"; @@ -2483,42 +2328,6 @@ public void testWithIP() throws Exception { }); } - public void testWithGeoPoint() throws Exception { - final List>> dataset = new ArrayList<>(); - dataset.addAll( - Arrays.asList( - createDocument("geo_point", new GeoPoint(48.934059, 41.610741)), - createDocument("geo_point", new GeoPoint(-23.065941, 113.610741)), - createDocument("geo_point", new GeoPoint(90.0, 0.0)), - createDocument("geo_point", new GeoPoint(37.2343, -115.8067)), - createDocument("geo_point", new GeoPoint(90.0, 0.0)) - ) - ); - testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("geo_point")), dataset, () -> { - GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder("geo_point").field("geo_point"); - return new CompositeAggregationBuilder("name", Collections.singletonList(geoTile)); - }, (result) -> { - assertEquals(2, result.getBuckets().size()); - assertEquals("{geo_point=7/64/56}", result.afterKey().toString()); - assertEquals("{geo_point=7/32/56}", result.getBuckets().get(0).getKeyAsString()); - assertEquals(2L, result.getBuckets().get(0).getDocCount()); - assertEquals("{geo_point=7/64/56}", result.getBuckets().get(1).getKeyAsString()); - assertEquals(3L, result.getBuckets().get(1).getDocCount()); - }); - - testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("geo_point")), dataset, () -> { - GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder("geo_point").field("geo_point"); - return new CompositeAggregationBuilder("name", Collections.singletonList(geoTile)).aggregateAfter( - Collections.singletonMap("geo_point", "7/32/56") - ); - }, (result) -> { - assertEquals(1, result.getBuckets().size()); - assertEquals("{geo_point=7/64/56}", result.afterKey().toString()); - assertEquals("{geo_point=7/64/56}", result.getBuckets().get(0).getKeyAsString()); - assertEquals(3L, result.getBuckets().get(0).getDocCount()); - }); - } - public void testEarlyTermination() throws Exception { final List>> dataset = new ArrayList<>(); dataset.addAll( @@ -2648,193 +2457,4 @@ public void testIndexSortWithDuplicate() throws Exception { ); } } - - private void testSearchCase( - List queries, - List>> dataset, - Supplier create, - Consumer verify - ) throws IOException { - for (Query query : queries) { - executeTestCase(false, false, query, dataset, create, verify); - executeTestCase(false, true, query, dataset, create, verify); - } - } - - private void executeTestCase( - boolean forceMerge, - boolean useIndexSort, - Query query, - List>> dataset, - Supplier create, - Consumer verify - ) throws IOException { - Map types = Arrays.stream(FIELD_TYPES) - .collect(Collectors.toMap(MappedFieldType::name, Function.identity())); - CompositeAggregationBuilder aggregationBuilder = create.get(); - Sort indexSort = useIndexSort ? buildIndexSort(aggregationBuilder.sources(), types) : null; - IndexSettings indexSettings = createIndexSettings(indexSort); - try (Directory directory = newDirectory()) { - IndexWriterConfig config = newIndexWriterConfig(random(), new MockAnalyzer(random())); - if (indexSort != null) { - config.setIndexSort(indexSort); - config.setCodec(TestUtil.getDefaultCodec()); - } - try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory, config)) { - Document document = new Document(); - int id = 0; - for (Map> fields : dataset) { - document.clear(); - addToDocument(id, document, fields); - indexWriter.addDocument(document); - id++; - } - if (forceMerge || rarely()) { - // forceMerge randomly or if the collector-per-leaf testing stuff would break the tests. - indexWriter.forceMerge(1); - } else { - if (dataset.size() > 0) { - int numDeletes = randomIntBetween(1, 25); - for (int i = 0; i < numDeletes; i++) { - id = randomIntBetween(0, dataset.size() - 1); - indexWriter.deleteDocuments(new Term("id", Integer.toString(id))); - document.clear(); - addToDocument(id, document, dataset.get(id)); - indexWriter.addDocument(document); - } - } - - } - } - try (IndexReader indexReader = DirectoryReader.open(directory)) { - IndexSearcher indexSearcher = new IndexSearcher(indexReader); - InternalComposite composite = searchAndReduce(indexSettings, indexSearcher, query, aggregationBuilder, FIELD_TYPES); - verify.accept(composite); - } - } - } - - private static IndexSettings createIndexSettings(Sort sort) { - Settings.Builder builder = Settings.builder(); - if (sort != null) { - String[] fields = Arrays.stream(sort.getSort()).map(SortField::getField).toArray(String[]::new); - String[] orders = Arrays.stream(sort.getSort()).map((o) -> o.getReverse() ? "desc" : "asc").toArray(String[]::new); - builder.putList("index.sort.field", fields); - builder.putList("index.sort.order", orders); - } - return IndexSettingsModule.newIndexSettings(new Index("_index", "0"), builder.build()); - } - - private void addToDocument(int id, Document doc, Map> keys) { - doc.add(new StringField("id", Integer.toString(id), Field.Store.NO)); - for (Map.Entry> entry : keys.entrySet()) { - final String name = entry.getKey(); - for (Object value : entry.getValue()) { - if (value instanceof Integer) { - doc.add(new SortedNumericDocValuesField(name, (int) value)); - doc.add(new IntPoint(name, (int) value)); - } else if (value instanceof Long) { - doc.add(new SortedNumericDocValuesField(name, (long) value)); - doc.add(new LongPoint(name, (long) value)); - } else if (value instanceof Double) { - doc.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong((double) value))); - doc.add(new DoublePoint(name, (double) value)); - } else if (value instanceof String) { - doc.add(new SortedSetDocValuesField(name, new BytesRef((String) value))); - doc.add(new StringField(name, new BytesRef((String) value), Field.Store.NO)); - } else if (value instanceof InetAddress) { - doc.add(new SortedSetDocValuesField(name, new BytesRef(InetAddressPoint.encode((InetAddress) value)))); - doc.add(new InetAddressPoint(name, (InetAddress) value)); - } else if (value instanceof GeoPoint) { - GeoPoint point = (GeoPoint) value; - doc.add( - new SortedNumericDocValuesField( - name, - GeoTileUtils.longEncode(point.lon(), point.lat(), GeoTileGridAggregationBuilder.DEFAULT_PRECISION) - ) - ); - doc.add(new LatLonPoint(name, point.lat(), point.lon())); - } else { - throw new AssertionError("invalid object: " + value.getClass().getSimpleName()); - } - } - } - } - - private static Map createAfterKey(Object... fields) { - assert fields.length % 2 == 0; - final Map map = new HashMap<>(); - for (int i = 0; i < fields.length; i += 2) { - String field = (String) fields[i]; - map.put(field, fields[i + 1]); - } - return map; - } - - @SuppressWarnings("unchecked") - private static Map> createDocument(Object... fields) { - assert fields.length % 2 == 0; - final Map> map = new HashMap<>(); - for (int i = 0; i < fields.length; i += 2) { - String field = (String) fields[i]; - if (fields[i + 1] instanceof List) { - map.put(field, (List) fields[i + 1]); - } else { - map.put(field, Collections.singletonList(fields[i + 1])); - } - } - return map; - } - - private static long asLong(String dateTime) { - return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); - } - - private static Sort buildIndexSort(List> sources, Map fieldTypes) { - List sortFields = new ArrayList<>(); - Map remainingFieldTypes = new HashMap<>(fieldTypes); - for (CompositeValuesSourceBuilder source : sources) { - MappedFieldType type = fieldTypes.remove(source.field()); - remainingFieldTypes.remove(source.field()); - SortField sortField = sortFieldFrom(type); - if (sortField == null) { - break; - } - sortFields.add(sortField); - } - while (remainingFieldTypes.size() > 0 && randomBoolean()) { - // Add extra unused sorts - List fields = new ArrayList<>(remainingFieldTypes.keySet()); - Collections.sort(fields); - String field = fields.get(between(0, fields.size() - 1)); - SortField sortField = sortFieldFrom(remainingFieldTypes.remove(field)); - if (sortField != null) { - sortFields.add(sortField); - } - } - return sortFields.size() > 0 ? new Sort(sortFields.toArray(new SortField[0])) : null; - } - - private static SortField sortFieldFrom(MappedFieldType type) { - if (type instanceof KeywordFieldMapper.KeywordFieldType) { - return new SortedSetSortField(type.name(), false); - } else if (type instanceof DateFieldMapper.DateFieldType) { - return new SortedNumericSortField(type.name(), SortField.Type.LONG, false); - } else if (type instanceof NumberFieldMapper.NumberFieldType) { - switch (type.typeName()) { - case "byte": - case "short": - case "integer": - return new SortedNumericSortField(type.name(), SortField.Type.INT, false); - case "long": - return new SortedNumericSortField(type.name(), SortField.Type.LONG, false); - case "float": - case "double": - return new SortedNumericSortField(type.name(), SortField.Type.DOUBLE, false); - default: - return null; - } - } - return null; - } } diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java index dfe4034650594..1443208a1d2fc 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java @@ -34,14 +34,15 @@ import org.opensearch.common.geo.GeoPoint; import org.opensearch.geometry.Rectangle; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.test.OpenSearchTestCase; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.MAX_ZOOM; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.checkPrecisionRange; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.hashToGeoPoint; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.keyToGeoPoint; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.longEncode; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.stringEncode; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.MAX_ZOOM; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.checkPrecisionRange; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.hashToGeoPoint; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.keyToGeoPoint; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.longEncode; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.stringEncode; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.containsString; diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/GeoBoundsTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/GeoBoundsTests.java deleted file mode 100644 index e132426680fc8..0000000000000 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/GeoBoundsTests.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.search.aggregations.metrics; - -import org.opensearch.search.aggregations.BaseAggregationTestCase; - -public class GeoBoundsTests extends BaseAggregationTestCase { - - @Override - protected GeoBoundsAggregationBuilder createTestAggregatorBuilder() { - GeoBoundsAggregationBuilder factory = new GeoBoundsAggregationBuilder(randomAlphaOfLengthBetween(1, 20)); - String field = randomAlphaOfLengthBetween(3, 20); - factory.field(field); - if (randomBoolean()) { - factory.wrapLongitude(randomBoolean()); - } - if (randomBoolean()) { - factory.missing("0,0"); - } - return factory; - } - -} diff --git a/server/src/test/java/org/opensearch/search/pit/RestDeletePitActionTests.java b/server/src/test/java/org/opensearch/search/pit/RestDeletePitActionTests.java index 36a1f91a02887..5df8990ceddf7 100644 --- a/server/src/test/java/org/opensearch/search/pit/RestDeletePitActionTests.java +++ b/server/src/test/java/org/opensearch/search/pit/RestDeletePitActionTests.java @@ -44,7 +44,7 @@ public void testParseDeletePitRequestWithInvalidJsonThrowsException() throws Exc public void testDeletePitWithBody() throws Exception { SetOnce pitCalled = new SetOnce<>(); try (NodeClient nodeClient = new NoOpNodeClient(this.getTestName()) { - @Override + // @Override public void deletePits(DeletePitRequest request, ActionListener listener) { pitCalled.set(true); assertThat(request.getPitIds(), hasSize(1)); @@ -66,7 +66,7 @@ public void deletePits(DeletePitRequest request, ActionListener pitCalled = new SetOnce<>(); try (NodeClient nodeClient = new NoOpNodeClient(this.getTestName()) { - @Override + // @Override public void deletePits(DeletePitRequest request, ActionListener listener) { pitCalled.set(true); assertThat(request.getPitIds(), hasSize(1)); @@ -85,7 +85,7 @@ public void deletePits(DeletePitRequest request, ActionListener pitCalled = new SetOnce<>(); try (NodeClient nodeClient = new NoOpNodeClient(this.getTestName()) { - @Override + // @Override public void deletePits(DeletePitRequest request, ActionListener listener) { pitCalled.set(true); assertThat(request.getPitIds(), hasSize(1)); @@ -110,7 +110,7 @@ public void deletePits(DeletePitRequest request, ActionListener pitCalled = new SetOnce<>(); try (NodeClient nodeClient = new NoOpNodeClient(this.getTestName()) { - @Override + // @Override public void deletePits(DeletePitRequest request, ActionListener listener) { pitCalled.set(true); assertThat(request.getPitIds(), hasSize(2)); diff --git a/settings.gradle b/settings.gradle index 65dc6a13100e2..4c389b5490e7c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -10,7 +10,7 @@ */ plugins { - id "com.gradle.enterprise" version "3.10.3" + id "com.gradle.enterprise" version "3.11.1" } buildCache { diff --git a/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java index 249ffcfd0bf6e..b3f062aef4fbe 100644 --- a/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java @@ -141,7 +141,12 @@ protected ReplicationGroup createGroup(int replicas, Settings settings) throws I } protected ReplicationGroup createGroup(int replicas, Settings settings, EngineFactory engineFactory) throws IOException { - IndexMetadata metadata = buildIndexMetadata(replicas, settings, indexMapping); + return createGroup(replicas, settings, indexMapping, engineFactory); + } + + protected ReplicationGroup createGroup(int replicas, Settings settings, String mappings, EngineFactory engineFactory) + throws IOException { + IndexMetadata metadata = buildIndexMetadata(replicas, settings, mappings); return new ReplicationGroup(metadata) { @Override protected EngineFactory getEngineFactory(ShardRouting routing) { diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 7dedc572ff19b..f446538acccbb 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -90,6 +90,7 @@ import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; +import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.Translog; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; @@ -555,6 +556,7 @@ protected IndexShard newShard( globalCheckpointSyncer, retentionLeaseSyncer, breakerService, + new InternalTranslogFactory(), checkpointPublisher, remoteStore ); diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java new file mode 100644 index 0000000000000..7d00772913d6e --- /dev/null +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java @@ -0,0 +1,310 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations.composite; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedNumericSortField; +import org.apache.lucene.search.SortedSetSortField; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.util.TestUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NumericUtils; +import org.junit.After; +import org.junit.Before; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.text.Text; +import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.time.DateFormatters; +import org.opensearch.index.Index; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.mapper.DateFieldMapper; +import org.opensearch.index.mapper.DocumentMapper; +import org.opensearch.index.mapper.IpFieldMapper; +import org.opensearch.index.mapper.KeywordFieldMapper; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.search.aggregations.AggregatorTestCase; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; +import org.opensearch.search.aggregations.bucket.composite.InternalComposite; +import org.opensearch.test.IndexSettingsModule; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Base class for the Aggregator Tests which are registered under Composite Aggregation. + */ +public class BaseCompositeAggregatorTestCase extends AggregatorTestCase { + + protected static List FIELD_TYPES; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + FIELD_TYPES = new ArrayList<>(); + FIELD_TYPES.add(new KeywordFieldMapper.KeywordFieldType("keyword")); + FIELD_TYPES.add(new NumberFieldMapper.NumberFieldType("long", NumberFieldMapper.NumberType.LONG)); + FIELD_TYPES.add(new NumberFieldMapper.NumberFieldType("double", NumberFieldMapper.NumberType.DOUBLE)); + FIELD_TYPES.add(new DateFieldMapper.DateFieldType("date", DateFormatter.forPattern("yyyy-MM-dd||epoch_millis"))); + FIELD_TYPES.add(new NumberFieldMapper.NumberFieldType("price", NumberFieldMapper.NumberType.INTEGER)); + FIELD_TYPES.add(new KeywordFieldMapper.KeywordFieldType("terms")); + FIELD_TYPES.add(new IpFieldMapper.IpFieldType("ip")); + } + + @Override + @After + public void tearDown() throws Exception { + super.tearDown(); + FIELD_TYPES = null; + } + + @Override + protected MapperService mapperServiceMock() { + MapperService mapperService = mock(MapperService.class); + DocumentMapper mapper = mock(DocumentMapper.class); + when(mapper.typeText()).thenReturn(new Text("_doc")); + when(mapper.type()).thenReturn("_doc"); + when(mapperService.documentMapper()).thenReturn(mapper); + return mapperService; + } + + protected static Map> createDocument(Object... fields) { + assert fields.length % 2 == 0; + final Map> map = new HashMap<>(); + for (int i = 0; i < fields.length; i += 2) { + String field = (String) fields[i]; + if (fields[i + 1] instanceof List) { + map.put(field, (List) fields[i + 1]); + } else { + map.put(field, Collections.singletonList(fields[i + 1])); + } + } + return map; + } + + protected void testSearchCase( + List queries, + List>> dataset, + Supplier create, + Consumer verify + ) throws IOException { + for (Query query : queries) { + executeTestCase(false, false, query, dataset, create, verify); + executeTestCase(false, true, query, dataset, create, verify); + } + } + + protected void executeTestCase( + boolean forceMerge, + boolean useIndexSort, + Query query, + List>> dataset, + Supplier create, + Consumer verify + ) throws IOException { + Map types = FIELD_TYPES.stream().collect(Collectors.toMap(MappedFieldType::name, Function.identity())); + CompositeAggregationBuilder aggregationBuilder = create.get(); + Sort indexSort = useIndexSort ? buildIndexSort(aggregationBuilder.sources(), types) : null; + IndexSettings indexSettings = createIndexSettings(indexSort); + try (Directory directory = newDirectory()) { + IndexWriterConfig config = newIndexWriterConfig(random(), new MockAnalyzer(random())); + if (indexSort != null) { + config.setIndexSort(indexSort); + config.setCodec(TestUtil.getDefaultCodec()); + } + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory, config)) { + Document document = new Document(); + int id = 0; + for (Map> fields : dataset) { + document.clear(); + addToDocument(id, document, fields); + indexWriter.addDocument(document); + id++; + } + if (forceMerge || rarely()) { + // forceMerge randomly or if the collector-per-leaf testing stuff would break the tests. + indexWriter.forceMerge(1); + } else { + if (dataset.size() > 0) { + int numDeletes = randomIntBetween(1, 25); + for (int i = 0; i < numDeletes; i++) { + id = randomIntBetween(0, dataset.size() - 1); + indexWriter.deleteDocuments(new Term("id", Integer.toString(id))); + document.clear(); + addToDocument(id, document, dataset.get(id)); + indexWriter.addDocument(document); + } + } + + } + } + try (IndexReader indexReader = DirectoryReader.open(directory)) { + IndexSearcher indexSearcher = new IndexSearcher(indexReader); + InternalComposite composite = searchAndReduce( + indexSettings, + indexSearcher, + query, + aggregationBuilder, + FIELD_TYPES.toArray(new MappedFieldType[0]) + ); + verify.accept(composite); + } + } + } + + protected void addToDocument(int id, Document doc, Map> keys) { + doc.add(new StringField("id", Integer.toString(id), Field.Store.NO)); + for (Map.Entry> entry : keys.entrySet()) { + final String name = entry.getKey(); + for (Object value : entry.getValue()) { + if (value instanceof Integer) { + doc.add(new SortedNumericDocValuesField(name, (int) value)); + doc.add(new IntPoint(name, (int) value)); + } else if (value instanceof Long) { + doc.add(new SortedNumericDocValuesField(name, (long) value)); + doc.add(new LongPoint(name, (long) value)); + } else if (value instanceof Double) { + doc.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong((double) value))); + doc.add(new DoublePoint(name, (double) value)); + } else if (value instanceof String) { + doc.add(new SortedSetDocValuesField(name, new BytesRef((String) value))); + doc.add(new StringField(name, new BytesRef((String) value), Field.Store.NO)); + } else if (value instanceof InetAddress) { + doc.add(new SortedSetDocValuesField(name, new BytesRef(InetAddressPoint.encode((InetAddress) value)))); + doc.add(new InetAddressPoint(name, (InetAddress) value)); + } else { + if (!addValueToDocument(doc, name, value)) throw new AssertionError( + "invalid object: " + value.getClass().getSimpleName() + ); + } + } + } + } + + /** + * Override this function to handle any specific type of value you want to add in the document for doing the + * composite aggregation. If you have added another Composite Aggregation Type then you must override this + * function so that your field value can be added in the document correctly. + * + * @param doc {@link Document} + * @param name {@link String} Field Name + * @param value {@link Object} Field value + * @return boolean true or false, based on if value is added or not + */ + protected boolean addValueToDocument(final Document doc, final String name, final Object value) { + return false; + } + + protected static Sort buildIndexSort(List> sources, Map fieldTypes) { + List sortFields = new ArrayList<>(); + Map remainingFieldTypes = new HashMap<>(fieldTypes); + for (CompositeValuesSourceBuilder source : sources) { + MappedFieldType type = fieldTypes.remove(source.field()); + remainingFieldTypes.remove(source.field()); + SortField sortField = sortFieldFrom(type); + if (sortField == null) { + break; + } + sortFields.add(sortField); + } + while (remainingFieldTypes.size() > 0 && randomBoolean()) { + // Add extra unused sorts + List fields = new ArrayList<>(remainingFieldTypes.keySet()); + Collections.sort(fields); + String field = fields.get(between(0, fields.size() - 1)); + SortField sortField = sortFieldFrom(remainingFieldTypes.remove(field)); + if (sortField != null) { + sortFields.add(sortField); + } + } + return sortFields.size() > 0 ? new Sort(sortFields.toArray(new SortField[0])) : null; + } + + protected static SortField sortFieldFrom(MappedFieldType type) { + if (type instanceof KeywordFieldMapper.KeywordFieldType) { + return new SortedSetSortField(type.name(), false); + } else if (type instanceof DateFieldMapper.DateFieldType) { + return new SortedNumericSortField(type.name(), SortField.Type.LONG, false); + } else if (type instanceof NumberFieldMapper.NumberFieldType) { + switch (type.typeName()) { + case "byte": + case "short": + case "integer": + return new SortedNumericSortField(type.name(), SortField.Type.INT, false); + case "long": + return new SortedNumericSortField(type.name(), SortField.Type.LONG, false); + case "float": + case "double": + return new SortedNumericSortField(type.name(), SortField.Type.DOUBLE, false); + default: + return null; + } + } + return null; + } + + protected static IndexSettings createIndexSettings(Sort sort) { + Settings.Builder builder = Settings.builder(); + if (sort != null) { + String[] fields = Arrays.stream(sort.getSort()).map(SortField::getField).toArray(String[]::new); + String[] orders = Arrays.stream(sort.getSort()).map((o) -> o.getReverse() ? "desc" : "asc").toArray(String[]::new); + builder.putList("index.sort.field", fields); + builder.putList("index.sort.order", orders); + } + return IndexSettingsModule.newIndexSettings(new Index("_index", "0"), builder.build()); + } + + protected static Map createAfterKey(Object... fields) { + assert fields.length % 2 == 0; + final Map map = new HashMap<>(); + for (int i = 0; i < fields.length; i += 2) { + String field = (String) fields[i]; + map.put(field, fields[i + 1]); + } + return map; + } + + protected static long asLong(String dateTime) { + return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + } +} diff --git a/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java index f138de152a488..5325c48e16913 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java @@ -68,10 +68,6 @@ import org.opensearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; import org.opensearch.search.aggregations.bucket.filter.ParsedFilter; import org.opensearch.search.aggregations.bucket.filter.ParsedFilters; -import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid; -import org.opensearch.search.aggregations.bucket.geogrid.ParsedGeoTileGrid; import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.opensearch.search.aggregations.bucket.global.ParsedGlobal; import org.opensearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; @@ -117,7 +113,6 @@ import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder; import org.opensearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.opensearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder; -import org.opensearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; import org.opensearch.search.aggregations.metrics.GeoCentroidAggregationBuilder; import org.opensearch.search.aggregations.metrics.InternalHDRPercentileRanks; import org.opensearch.search.aggregations.metrics.InternalHDRPercentiles; @@ -129,7 +124,6 @@ import org.opensearch.search.aggregations.metrics.ParsedAvg; import org.opensearch.search.aggregations.metrics.ParsedCardinality; import org.opensearch.search.aggregations.metrics.ParsedExtendedStats; -import org.opensearch.search.aggregations.metrics.ParsedGeoBounds; import org.opensearch.search.aggregations.metrics.ParsedGeoCentroid; import org.opensearch.search.aggregations.metrics.ParsedHDRPercentileRanks; import org.opensearch.search.aggregations.metrics.ParsedHDRPercentiles; @@ -261,7 +255,6 @@ public ReduceContext forFinalReduction() { map.put(StatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedStatsBucket.fromXContent(p, (String) c)); map.put(ExtendedStatsAggregationBuilder.NAME, (p, c) -> ParsedExtendedStats.fromXContent(p, (String) c)); map.put(ExtendedStatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedExtendedStatsBucket.fromXContent(p, (String) c)); - map.put(GeoBoundsAggregationBuilder.NAME, (p, c) -> ParsedGeoBounds.fromXContent(p, (String) c)); map.put(GeoCentroidAggregationBuilder.NAME, (p, c) -> ParsedGeoCentroid.fromXContent(p, (String) c)); map.put(HistogramAggregationBuilder.NAME, (p, c) -> ParsedHistogram.fromXContent(p, (String) c)); map.put(DateHistogramAggregationBuilder.NAME, (p, c) -> ParsedDateHistogram.fromXContent(p, (String) c)); @@ -278,8 +271,6 @@ public ReduceContext forFinalReduction() { map.put(GlobalAggregationBuilder.NAME, (p, c) -> ParsedGlobal.fromXContent(p, (String) c)); map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c)); map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c)); - map.put(GeoHashGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c)); - map.put(GeoTileGridAggregationBuilder.NAME, (p, c) -> ParsedGeoTileGrid.fromXContent(p, (String) c)); map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c)); map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c)); map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c)); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 8d8df2fec39f9..1ab7785b17f5e 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -2097,6 +2097,7 @@ protected Collection> getMockPlugins() { if (addMockGeoShapeFieldMapper()) { mocks.add(TestGeoShapeFieldMapperPlugin.class); } + return Collections.unmodifiableList(mocks); } diff --git a/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java b/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java index 54c92f4d519aa..a36dc26685eb4 100644 --- a/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java +++ b/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java @@ -43,6 +43,7 @@ import org.opensearch.gateway.ReplicaShardAllocator; import org.opensearch.gateway.TransportNodesListGatewayStartedShards.NodeGatewayStartedShards; import org.opensearch.index.shard.ShardId; +import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.store.TransportNodesListShardStoreMetadata.NodeStoreFilesMetadata; import java.util.Collections; @@ -71,6 +72,7 @@ public class TestGatewayAllocator extends GatewayAllocator { Map> knownAllocations = new HashMap<>(); DiscoveryNodes currentNodes = DiscoveryNodes.EMPTY_NODES; + Map shardIdNodeToReplicationCheckPointMap = new HashMap<>(); PrimaryShardAllocator primaryShardAllocator = new PrimaryShardAllocator() { @Override @@ -90,7 +92,8 @@ protected AsyncShardFetch.FetchResult fetchData(ShardR routing -> new NodeGatewayStartedShards( currentNodes.get(routing.currentNodeId()), routing.allocationId().getId(), - routing.primary() + routing.primary(), + getReplicationCheckpoint(shardId, routing.currentNodeId()) ) ) ); @@ -99,6 +102,10 @@ protected AsyncShardFetch.FetchResult fetchData(ShardR } }; + private ReplicationCheckpoint getReplicationCheckpoint(ShardId shardId, String nodeName) { + return shardIdNodeToReplicationCheckPointMap.getOrDefault(getReplicationCheckPointKey(shardId, nodeName), null); + } + ReplicaShardAllocator replicaShardAllocator = new ReplicaShardAllocator() { @Override protected AsyncShardFetch.FetchResult fetchData(ShardRouting shard, RoutingAllocation allocation) { @@ -156,4 +163,12 @@ public void allocateUnassigned( public void addKnownAllocation(ShardRouting shard) { knownAllocations.computeIfAbsent(shard.currentNodeId(), id -> new HashMap<>()).put(shard.shardId(), shard); } + + public String getReplicationCheckPointKey(ShardId shardId, String nodeName) { + return shardId.toString() + "_" + nodeName; + } + + public void addReplicationCheckpoint(ShardId shardId, String nodeName, ReplicationCheckpoint replicationCheckpoint) { + shardIdNodeToReplicationCheckPointMap.putIfAbsent(getReplicationCheckPointKey(shardId, nodeName), replicationCheckpoint); + } }