From 713090232e824d97b9c2cd805160cbe5c59d58a4 Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Wed, 27 Jul 2022 15:08:28 -0700 Subject: [PATCH] Added GeoBounds aggregation on GeoShape field type.(#3980) This change also includes: * Addition of Doc values on the GeoShape Field.(#4049) * Addition of GeoShape ValueSource level code interfaces for accessing the DocValues.(#4049) * Addition of Missing Value feature in the GeoShape Aggregations. Signed-off-by: Navneet Verma --- CHANGELOG.md | 6 +- .../geometry/GeometryCollection.java | 9 + modules/geo/build.gradle | 1 + .../geo/GeoModulePluginIntegTestCase.java | 3 + .../opensearch/geo/search/MissingValueIT.java | 133 ++++++++-- .../aggregations/common/GeoBoundsHelper.java | 187 ++++++++++++++ ...ractGeoAggregatorModulePluginTestCase.java | 54 ++-- .../metrics/GeoBoundsITTestCase.java | 41 ++- .../org/opensearch/geo/GeoModulePlugin.java | 23 ++ .../geo/algorithm/PolygonGenerator.java | 190 ++++++++++++++ .../metrics/GeoBoundsGeoShapeAggregator.java | 116 +++++++++ .../GeoBoundsGeoShapeAggregatorTests.java | 237 +++++++++++++++++ .../common/RandomGeoGeometryGenerator.java | 240 ++++++++++++++++++ .../common/geo/GeoShapeDocValue.java | 175 +++++++++++++ .../opensearch/common/geo/ShapeDocValue.java | 188 ++++++++++++++ .../common/util/CollectionUtils.java | 12 + .../opensearch/index/fielddata/FieldData.java | 47 +++- .../index/fielddata/GeoShapeValue.java | 161 ++++++++++++ .../fielddata/LeafGeoShapeFieldData.java | 24 ++ .../plain/AbstractGeoShapeIndexFieldData.java | 132 ++++++++++ .../plain/AbstractLeafGeoShapeFieldData.java | 41 +++ .../plain/GeoShapeDVLeafFieldData.java | 79 ++++++ .../mapper/AbstractGeometryFieldMapper.java | 9 +- .../index/mapper/GeoShapeFieldMapper.java | 38 ++- .../support/CoreValuesSourceType.java | 103 +++++++- .../aggregations/support/MissingValues.java | 24 ++ .../aggregations/support/ValuesSource.java | 87 +++++++ .../common/util/CollectionUtilsTests.java | 7 + .../index/fielddata/GeoShapeValueTests.java | 61 +++++ .../mapper/GeoShapeFieldMapperTests.java | 6 +- .../support/CoreValuesSourceTypeTests.java | 1 + 31 files changed, 2370 insertions(+), 65 deletions(-) create mode 100644 modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/common/GeoBoundsHelper.java create mode 100644 modules/geo/src/main/java/org/opensearch/geo/algorithm/PolygonGenerator.java create mode 100644 modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsGeoShapeAggregator.java create mode 100644 modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsGeoShapeAggregatorTests.java create mode 100644 modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGeometryGenerator.java create mode 100644 server/src/main/java/org/opensearch/common/geo/GeoShapeDocValue.java create mode 100644 server/src/main/java/org/opensearch/common/geo/ShapeDocValue.java create mode 100644 server/src/main/java/org/opensearch/index/fielddata/GeoShapeValue.java create mode 100644 server/src/main/java/org/opensearch/index/fielddata/LeafGeoShapeFieldData.java create mode 100644 server/src/main/java/org/opensearch/index/fielddata/plain/AbstractGeoShapeIndexFieldData.java create mode 100644 server/src/main/java/org/opensearch/index/fielddata/plain/AbstractLeafGeoShapeFieldData.java create mode 100644 server/src/main/java/org/opensearch/index/fielddata/plain/GeoShapeDVLeafFieldData.java create mode 100644 server/src/test/java/org/opensearch/index/fielddata/GeoShapeValueTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index fb1bfc91c5361..24bf607b50114 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,10 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Github workflow for changelog verification ([#4085](https://github.com/opensearch-project/OpenSearch/pull/4085)) - Label configuration for dependabot PRs ([#4348](https://github.com/opensearch-project/OpenSearch/pull/4348)) - Added RestLayer Changes for PIT stats ([#4217](https://github.com/opensearch-project/OpenSearch/pull/4217)) +- Added GeoBounds aggregation on GeoShape field type.([#4266](https://github.com/opensearch-project/OpenSearch/pull/4266)) + - Addition of Doc values on the GeoShape Field + - Addition of GeoShape ValueSource level code interfaces for accessing the DocValues. + - Addition of Missing Value feature in the GeoShape Aggregations. ### Changed @@ -89,4 +93,4 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) [Unreleased]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...HEAD -[2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...2.x \ No newline at end of file +[2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...2.x diff --git a/libs/geo/src/main/java/org/opensearch/geometry/GeometryCollection.java b/libs/geo/src/main/java/org/opensearch/geometry/GeometryCollection.java index dfadf9269a097..8aca043017e32 100644 --- a/libs/geo/src/main/java/org/opensearch/geometry/GeometryCollection.java +++ b/libs/geo/src/main/java/org/opensearch/geometry/GeometryCollection.java @@ -88,6 +88,15 @@ public G get(int i) { return shapes.get(i); } + /** + * Returns a {@link List} of All {@link Geometry} present in this collection. + * + * @return a {@link List} of All {@link Geometry} + */ + public List getAll() { + return shapes; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/modules/geo/build.gradle b/modules/geo/build.gradle index 7f687a414e566..6b00709f08bf9 100644 --- a/modules/geo/build.gradle +++ b/modules/geo/build.gradle @@ -40,6 +40,7 @@ restResources { includeCore '_common', 'indices', 'index', 'search', 'bulk' } } + artifacts { restTests(project.file('src/yamlRestTest/resources/rest-api-spec/test')) } diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java index 7dc6f2c1b89b7..31ff2ef4689bd 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java @@ -21,6 +21,9 @@ * for the test cluster on which integration tests are running. */ public abstract class GeoModulePluginIntegTestCase extends OpenSearchIntegTestCase { + + protected static final double GEOHASH_TOLERANCE = 1E-5D; + /** * Returns a collection of plugins that should be loaded on each node for doing the integration tests. As this * geo plugin is not getting packaged in a zip, we need to load it before the tests run. diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/MissingValueIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/MissingValueIT.java index 2ac73728b2dab..9bd082a6e1ffe 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/MissingValueIT.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/MissingValueIT.java @@ -8,52 +8,149 @@ package org.opensearch.geo.search; +import org.hamcrest.MatcherAssert; +import org.junit.Before; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.geo.GeoPoint; import org.opensearch.geo.GeoModulePluginIntegTestCase; +import org.opensearch.geo.search.aggregations.common.GeoBoundsHelper; import org.opensearch.geo.search.aggregations.metrics.GeoBounds; import org.opensearch.geo.tests.common.AggregationBuilders; +import org.opensearch.geo.tests.common.RandomGeoGenerator; +import org.opensearch.geo.tests.common.RandomGeoGeometryGenerator; +import org.opensearch.geometry.Geometry; +import org.opensearch.geometry.utils.WellKnownText; import org.opensearch.test.OpenSearchIntegTestCase; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.closeTo; +/** + * Tests to validate if user specified a missingValue in the input while doing the aggregation + */ @OpenSearchIntegTestCase.SuiteScopeTestCase public class MissingValueIT extends GeoModulePluginIntegTestCase { + private static final String INDEX_NAME = "idx"; + private static final String GEO_SHAPE_FIELD_NAME = "myshape"; + private static final String GEO_SHAPE_FIELD_TYPE = "type=geo_shape"; + private static final String AGGREGATION_NAME = "bounds"; + private static final String NON_EXISTENT_FIELD = "non_existing_field"; + private static final WellKnownText WKT = WellKnownText.INSTANCE; + private static Geometry indexedGeometry; + private static GeoPoint indexedGeoPoint; + private GeoPoint bottomRight; + private GeoPoint topLeft; + @Override protected void setupSuiteScopeCluster() throws Exception { - assertAcked(prepareCreate("idx").setMapping("date", "type=date", "location", "type=geo_point", "str", "type=keyword").get()); + assertAcked( + prepareCreate(INDEX_NAME).setMapping( + "date", + "type=date", + "location", + "type=geo_point", + "str", + "type=keyword", + GEO_SHAPE_FIELD_NAME, + GEO_SHAPE_FIELD_TYPE + ).get() + ); + indexedGeometry = RandomGeoGeometryGenerator.randomGeometry(random()); + indexedGeoPoint = RandomGeoGenerator.randomPoint(random()); + assert indexedGeometry != null; indexRandom( true, - client().prepareIndex("idx").setId("1").setSource(), - client().prepareIndex("idx") + client().prepareIndex(INDEX_NAME).setId("1").setSource(), + client().prepareIndex(INDEX_NAME) .setId("2") - .setSource("str", "foo", "long", 3L, "double", 5.5, "date", "2015-05-07", "location", "1,2") + .setSource( + "str", + "foo", + "long", + 3L, + "double", + 5.5, + "date", + "2015-05-07", + "location", + indexedGeoPoint.toString(), + GEO_SHAPE_FIELD_NAME, + WKT.toWKT(indexedGeometry) + ) ); } + @Before + public void runBeforeEachTest() { + bottomRight = new GeoPoint(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY); + topLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); + } + public void testUnmappedGeoBounds() { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(AggregationBuilders.geoBounds("bounds").field("non_existing_field").missing("2,1")) + final GeoPoint missingGeoPoint = RandomGeoGenerator.randomPoint(random()); + GeoBoundsHelper.updateBoundsBottomRight(missingGeoPoint, bottomRight); + GeoBoundsHelper.updateBoundsTopLeft(missingGeoPoint, topLeft); + SearchResponse response = client().prepareSearch(INDEX_NAME) + .addAggregation( + AggregationBuilders.geoBounds(AGGREGATION_NAME) + .field(NON_EXISTENT_FIELD) + .wrapLongitude(false) + .missing(missingGeoPoint.toString()) + ) .get(); assertSearchResponse(response); - GeoBounds bounds = response.getAggregations().get("bounds"); - assertThat(bounds.bottomRight().lat(), closeTo(2.0, 1E-5)); - assertThat(bounds.bottomRight().lon(), closeTo(1.0, 1E-5)); - assertThat(bounds.topLeft().lat(), closeTo(2.0, 1E-5)); - assertThat(bounds.topLeft().lon(), closeTo(1.0, 1E-5)); + validateResult(response.getAggregations().get(AGGREGATION_NAME)); } public void testGeoBounds() { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(AggregationBuilders.geoBounds("bounds").field("location").missing("2,1")) + GeoBoundsHelper.updateBoundsForGeoPoint(indexedGeoPoint, topLeft, bottomRight); + final GeoPoint missingGeoPoint = RandomGeoGenerator.randomPoint(random()); + GeoBoundsHelper.updateBoundsForGeoPoint(missingGeoPoint, topLeft, bottomRight); + SearchResponse response = client().prepareSearch(INDEX_NAME) + .addAggregation( + AggregationBuilders.geoBounds(AGGREGATION_NAME).field("location").wrapLongitude(false).missing(missingGeoPoint.toString()) + ) .get(); assertSearchResponse(response); - GeoBounds bounds = response.getAggregations().get("bounds"); - assertThat(bounds.bottomRight().lat(), closeTo(1.0, 1E-5)); - assertThat(bounds.bottomRight().lon(), closeTo(2.0, 1E-5)); - assertThat(bounds.topLeft().lat(), closeTo(2.0, 1E-5)); - assertThat(bounds.topLeft().lon(), closeTo(1.0, 1E-5)); + validateResult(response.getAggregations().get(AGGREGATION_NAME)); + } + + public void testGeoBoundsWithMissingShape() { + // create GeoBounds for the indexed Field + GeoBoundsHelper.updateBoundsForGeometry(indexedGeometry, topLeft, bottomRight); + final Geometry missingGeometry = RandomGeoGeometryGenerator.randomGeometry(random()); + assert missingGeometry != null; + GeoBoundsHelper.updateBoundsForGeometry(missingGeometry, topLeft, bottomRight); + final SearchResponse response = client().prepareSearch(INDEX_NAME) + .addAggregation( + AggregationBuilders.geoBounds(AGGREGATION_NAME) + .wrapLongitude(false) + .field(GEO_SHAPE_FIELD_NAME) + .missing(WKT.toWKT(missingGeometry)) + ) + .get(); + assertSearchResponse(response); + validateResult(response.getAggregations().get(AGGREGATION_NAME)); + } + + public void testUnmappedGeoBoundsOnGeoShape() { + // We cannot useGeometry other than Point as for GeoBoundsAggregation as the Default Value for the + // CoreValueSourceType is GeoPoint hence we need to use Point here. + final Geometry missingGeometry = RandomGeoGeometryGenerator.randomPoint(random()); + final SearchResponse response = client().prepareSearch(INDEX_NAME) + .addAggregation(AggregationBuilders.geoBounds(AGGREGATION_NAME).field(NON_EXISTENT_FIELD).missing(WKT.toWKT(missingGeometry))) + .get(); + GeoBoundsHelper.updateBoundsForGeometry(missingGeometry, topLeft, bottomRight); + assertSearchResponse(response); + validateResult(response.getAggregations().get(AGGREGATION_NAME)); + } + + private void validateResult(final GeoBounds bounds) { + MatcherAssert.assertThat(bounds.bottomRight().lat(), closeTo(bottomRight.lat(), GEOHASH_TOLERANCE)); + MatcherAssert.assertThat(bounds.bottomRight().lon(), closeTo(bottomRight.lon(), GEOHASH_TOLERANCE)); + MatcherAssert.assertThat(bounds.topLeft().lat(), closeTo(topLeft.lat(), GEOHASH_TOLERANCE)); + MatcherAssert.assertThat(bounds.topLeft().lon(), closeTo(topLeft.lon(), GEOHASH_TOLERANCE)); } } diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/common/GeoBoundsHelper.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/common/GeoBoundsHelper.java new file mode 100644 index 0000000000000..257cc98db69fc --- /dev/null +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/common/GeoBoundsHelper.java @@ -0,0 +1,187 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.common; + +import org.junit.Assert; +import org.opensearch.common.geo.GeoPoint; +import org.opensearch.geometry.Geometry; +import org.opensearch.geometry.GeometryCollection; +import org.opensearch.geometry.Line; +import org.opensearch.geometry.MultiLine; +import org.opensearch.geometry.MultiPoint; +import org.opensearch.geometry.MultiPolygon; +import org.opensearch.geometry.Point; +import org.opensearch.geometry.Polygon; +import org.opensearch.geometry.Rectangle; +import org.opensearch.geometry.ShapeType; + +import java.util.Locale; + +/** + * A helper class for finding the geo bounds for a shape or a point. + */ +public final class GeoBoundsHelper { + + /** + * Updates the GeoBounds for the input GeoPoint in topLeft and bottomRight GeoPoints. + * + * @param geoPoint {@link GeoPoint} + * @param topLeft {@link GeoPoint} + * @param bottomRight {@link GeoPoint} + */ + public static void updateBoundsForGeoPoint(final GeoPoint geoPoint, final GeoPoint topLeft, final GeoPoint bottomRight) { + updateBoundsBottomRight(geoPoint, bottomRight); + updateBoundsTopLeft(geoPoint, topLeft); + } + + /** + * Find the bottom right for a point and put it in the currentBounds param. + * + * @param geoPoint {@link GeoPoint} + * @param currentBound {@link GeoPoint} + */ + public static void updateBoundsBottomRight(final GeoPoint geoPoint, final GeoPoint currentBound) { + if (geoPoint.lat() < currentBound.lat()) { + currentBound.resetLat(geoPoint.lat()); + } + if (geoPoint.lon() > currentBound.lon()) { + currentBound.resetLon(geoPoint.lon()); + } + } + + /** + * Find the top left for a point and put it in the currentBounds param. + * + * @param geoPoint {@link GeoPoint} + * @param currentBound {@link GeoPoint} + */ + public static void updateBoundsTopLeft(final GeoPoint geoPoint, final GeoPoint currentBound) { + if (geoPoint.lat() > currentBound.lat()) { + currentBound.resetLat(geoPoint.lat()); + } + if (geoPoint.lon() < currentBound.lon()) { + currentBound.resetLon(geoPoint.lon()); + } + } + + /** + * Find the bounds for an input shape. + * + * @param geometry {@link Geometry} + * @param geoShapeTopLeft {@link GeoPoint} + * @param geoShapeBottomRight {@link GeoPoint} + */ + public static void updateBoundsForGeometry( + final Geometry geometry, + final GeoPoint geoShapeTopLeft, + final GeoPoint geoShapeBottomRight + ) { + final ShapeType shapeType = geometry.type(); + switch (shapeType) { + case POINT: + updateBoundsTopLeft((Point) geometry, geoShapeTopLeft); + updateBoundsBottomRight((Point) geometry, geoShapeBottomRight); + return; + case MULTIPOINT: + ((MultiPoint) geometry).getAll().forEach(p -> updateBoundsTopLeft(p, geoShapeTopLeft)); + ((MultiPoint) geometry).getAll().forEach(p -> updateBoundsBottomRight(p, geoShapeBottomRight)); + return; + case POLYGON: + updateBoundsTopLeft((Polygon) geometry, geoShapeTopLeft); + updateBoundsBottomRight((Polygon) geometry, geoShapeBottomRight); + return; + case LINESTRING: + updateBoundsTopLeft((Line) geometry, geoShapeTopLeft); + updateBoundsBottomRight((Line) geometry, geoShapeBottomRight); + return; + case MULTIPOLYGON: + ((MultiPolygon) geometry).getAll().forEach(p -> updateBoundsTopLeft(p, geoShapeTopLeft)); + ((MultiPolygon) geometry).getAll().forEach(p -> updateBoundsBottomRight(p, geoShapeBottomRight)); + return; + case GEOMETRYCOLLECTION: + ((GeometryCollection) geometry).getAll() + .forEach(geo -> updateBoundsForGeometry(geo, geoShapeTopLeft, geoShapeBottomRight)); + return; + case MULTILINESTRING: + ((MultiLine) geometry).getAll().forEach(line -> updateBoundsTopLeft(line, geoShapeTopLeft)); + ((MultiLine) geometry).getAll().forEach(line -> updateBoundsBottomRight(line, geoShapeBottomRight)); + return; + case ENVELOPE: + updateBoundsTopLeft((Rectangle) geometry, geoShapeTopLeft); + updateBoundsBottomRight((Rectangle) geometry, geoShapeBottomRight); + return; + default: + Assert.fail(String.format(Locale.ROOT, "The shape type %s is not supported", shapeType)); + } + } + + private static void updateBoundsTopLeft(final Point p, final GeoPoint currentBound) { + final GeoPoint geoPoint = new GeoPoint(p.getLat(), p.getLon()); + updateBoundsTopLeft(geoPoint, currentBound); + } + + private static void updateBoundsTopLeft(final Polygon polygon, final GeoPoint currentBound) { + for (int i = 0; i < polygon.getPolygon().length(); i++) { + double lat = polygon.getPolygon().getLats()[i]; + double lon = polygon.getPolygon().getLons()[i]; + final GeoPoint geoPoint = new GeoPoint(lat, lon); + updateBoundsTopLeft(geoPoint, currentBound); + } + } + + private static void updateBoundsTopLeft(final Line line, final GeoPoint currentBound) { + for (int i = 0; i < line.length(); i++) { + double lat = line.getLats()[i]; + double lon = line.getLons()[i]; + final GeoPoint geoPoint = new GeoPoint(lat, lon); + updateBoundsTopLeft(geoPoint, currentBound); + } + } + + private static void updateBoundsTopLeft(final Rectangle rectangle, final GeoPoint currentBound) { + if (rectangle.getMaxLat() > currentBound.lat()) { + currentBound.resetLat(rectangle.getMaxLat()); + } + if (rectangle.getMinLon() < currentBound.lon()) { + currentBound.resetLon(rectangle.getMinLon()); + } + } + + private static void updateBoundsBottomRight(final Point p, final GeoPoint currentBound) { + final GeoPoint geoPoint = new GeoPoint(p.getLat(), p.getLon()); + updateBoundsBottomRight(geoPoint, currentBound); + } + + private static void updateBoundsBottomRight(final Polygon polygon, final GeoPoint currentBound) { + for (int i = 0; i < polygon.getPolygon().length(); i++) { + double lat = polygon.getPolygon().getLats()[i]; + double lon = polygon.getPolygon().getLons()[i]; + final GeoPoint geoPoint = new GeoPoint(lat, lon); + updateBoundsBottomRight(geoPoint, currentBound); + } + } + + private static void updateBoundsBottomRight(final Line line, final GeoPoint currentBound) { + for (int i = 0; i < line.length(); i++) { + double lat = line.getLats()[i]; + double lon = line.getLons()[i]; + final GeoPoint geoPoint = new GeoPoint(lat, lon); + updateBoundsBottomRight(geoPoint, currentBound); + } + } + + private static void updateBoundsBottomRight(final Rectangle rectangle, final GeoPoint currentBound) { + if (rectangle.getMinLat() < currentBound.lat()) { + currentBound.resetLat(rectangle.getMinLat()); + } + if (rectangle.getMaxLon() > currentBound.lon()) { + currentBound.resetLon(rectangle.getMaxLon()); + } + } +} diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java index 92987d407f51d..b6f33ec2e0cae 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java @@ -22,14 +22,20 @@ import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.geo.GeoModulePluginIntegTestCase; +import org.opensearch.geo.search.aggregations.common.GeoBoundsHelper; import org.opensearch.geo.tests.common.RandomGeoGenerator; +import org.opensearch.geo.tests.common.RandomGeoGeometryGenerator; +import org.opensearch.geometry.Geometry; import org.opensearch.geometry.utils.Geohash; +import org.opensearch.geometry.utils.StandardValidator; +import org.opensearch.geometry.utils.WellKnownText; import org.opensearch.search.SearchHit; import org.opensearch.search.sort.SortBuilders; import org.opensearch.search.sort.SortOrder; import java.util.ArrayList; import java.util.List; +import java.util.stream.IntStream; import static org.hamcrest.Matchers.equalTo; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; @@ -46,6 +52,7 @@ public abstract class AbstractGeoAggregatorModulePluginTestCase extends GeoModul protected static final String SINGLE_VALUED_FIELD_NAME = "geo_value"; protected static final String MULTI_VALUED_FIELD_NAME = "geo_values"; + protected static final String GEO_SHAPE_FIELD_NAME = "shape"; protected static final String NUMBER_FIELD_NAME = "l_values"; protected static final String UNMAPPED_IDX_NAME = "idx_unmapped"; protected static final String IDX_NAME = "idx"; @@ -57,11 +64,13 @@ public abstract class AbstractGeoAggregatorModulePluginTestCase extends GeoModul protected static int numDocs; protected static int numUniqueGeoPoints; protected static GeoPoint[] singleValues, multiValues; + protected static Geometry[] geoShapesValues; protected static GeoPoint singleTopLeft, singleBottomRight, multiTopLeft, multiBottomRight, singleCentroid, multiCentroid, - unmappedCentroid; + unmappedCentroid, geoShapeTopLeft, geoShapeBottomRight; protected static ObjectIntMap expectedDocCountsForGeoHash = null; protected static ObjectObjectMap expectedCentroidsForGeoHash = null; - protected static final double GEOHASH_TOLERANCE = 1E-5D; + + protected static final WellKnownText WKT = new WellKnownText(true, new StandardValidator(true)); @Override public void setupSuiteScopeCluster() throws Exception { @@ -75,7 +84,9 @@ public void setupSuiteScopeCluster() throws Exception { NUMBER_FIELD_NAME, "type=long", "tag", - "type=keyword" + "type=keyword", + GEO_SHAPE_FIELD_NAME, + "type=geo_shape" ) ); @@ -83,6 +94,8 @@ public void setupSuiteScopeCluster() throws Exception { singleBottomRight = new GeoPoint(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY); multiTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); multiBottomRight = new GeoPoint(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY); + geoShapeTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); + geoShapeBottomRight = new GeoPoint(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY); singleCentroid = new GeoPoint(0, 0); multiCentroid = new GeoPoint(0, 0); unmappedCentroid = new GeoPoint(0, 0); @@ -95,17 +108,21 @@ public void setupSuiteScopeCluster() throws Exception { singleValues = new GeoPoint[numUniqueGeoPoints]; for (int i = 0; i < singleValues.length; i++) { singleValues[i] = RandomGeoGenerator.randomPoint(random()); - updateBoundsTopLeft(singleValues[i], singleTopLeft); - updateBoundsBottomRight(singleValues[i], singleBottomRight); + GeoBoundsHelper.updateBoundsForGeoPoint(singleValues[i], singleTopLeft, singleBottomRight); } multiValues = new GeoPoint[numUniqueGeoPoints]; for (int i = 0; i < multiValues.length; i++) { multiValues[i] = RandomGeoGenerator.randomPoint(random()); - updateBoundsTopLeft(multiValues[i], multiTopLeft); - updateBoundsBottomRight(multiValues[i], multiBottomRight); + GeoBoundsHelper.updateBoundsForGeoPoint(multiValues[i], multiTopLeft, multiBottomRight); } + geoShapesValues = new Geometry[numDocs]; + IntStream.range(0, numDocs).forEach(iterator -> { + geoShapesValues[iterator] = RandomGeoGeometryGenerator.randomGeometry(random()); + GeoBoundsHelper.updateBoundsForGeometry(geoShapesValues[iterator], geoShapeTopLeft, geoShapeBottomRight); + }); + List builders = new ArrayList<>(); GeoPoint singleVal; @@ -132,6 +149,7 @@ public void setupSuiteScopeCluster() throws Exception { .endArray() .field(NUMBER_FIELD_NAME, i) .field("tag", "tag" + i) + .field(GEO_SHAPE_FIELD_NAME, WKT.toWKT(geoShapesValues[i])) .endObject() ) ); @@ -147,7 +165,9 @@ public void setupSuiteScopeCluster() throws Exception { ); } - assertAcked(prepareCreate(EMPTY_IDX_NAME).setMapping(SINGLE_VALUED_FIELD_NAME, "type=geo_point")); + assertAcked( + prepareCreate(EMPTY_IDX_NAME).setMapping(SINGLE_VALUED_FIELD_NAME, "type=geo_point", GEO_SHAPE_FIELD_NAME, "type=geo_shape") + ); assertAcked( prepareCreate(DATELINE_IDX_NAME).setMapping( @@ -274,22 +294,4 @@ private GeoPoint updateHashCentroid(String hash, final GeoPoint location) { final double newLat = centroid.lat() + (location.lat() - centroid.lat()) / docCount; return centroid.reset(newLat, newLon); } - - private void updateBoundsBottomRight(GeoPoint geoPoint, GeoPoint currentBound) { - if (geoPoint.lat() < currentBound.lat()) { - currentBound.resetLat(geoPoint.lat()); - } - if (geoPoint.lon() > currentBound.lon()) { - currentBound.resetLon(geoPoint.lon()); - } - } - - private void updateBoundsTopLeft(GeoPoint geoPoint, GeoPoint currentBound) { - if (geoPoint.lat() > currentBound.lat()) { - currentBound.resetLat(geoPoint.lat()); - } - if (geoPoint.lon() < currentBound.lon()) { - currentBound.resetLon(geoPoint.lon()); - } - } } diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsITTestCase.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsITTestCase.java index 8cc82da12d69a..ed3196319faca 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsITTestCase.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsITTestCase.java @@ -32,6 +32,7 @@ package org.opensearch.geo.search.aggregations.metrics; +import org.hamcrest.MatcherAssert; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.util.BigArray; @@ -43,18 +44,18 @@ import java.util.List; -import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.sameInstance; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.closeTo; +import static org.opensearch.geo.tests.common.AggregationBuilders.geoBounds; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.search.aggregations.AggregationBuilders.global; import static org.opensearch.search.aggregations.AggregationBuilders.terms; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; -import static org.opensearch.geo.tests.common.AggregationBuilders.geoBounds; @OpenSearchIntegTestCase.SuiteScopeTestCase public class GeoBoundsITTestCase extends AbstractGeoAggregatorModulePluginTestCase { @@ -275,4 +276,36 @@ public void testSingleValuedFieldWithZeroLon() throws Exception { assertThat(bottomRight.lat(), closeTo(1.0, GEOHASH_TOLERANCE)); assertThat(bottomRight.lon(), closeTo(0.0, GEOHASH_TOLERANCE)); } + + public void testGeoShapeValuedField() { + final SearchResponse response = client().prepareSearch(IDX_NAME) + .addAggregation(geoBounds(aggName).field(GEO_SHAPE_FIELD_NAME).wrapLongitude(false)) + .get(); + assertSearchResponse(response); + final GeoBounds geoBounds = response.getAggregations().get(aggName); + MatcherAssert.assertThat(geoBounds, notNullValue()); + MatcherAssert.assertThat(geoBounds.getName(), equalTo(aggName)); + final GeoPoint topLeft = geoBounds.topLeft(); + final GeoPoint bottomRight = geoBounds.bottomRight(); + MatcherAssert.assertThat(topLeft.lat(), closeTo(geoShapeTopLeft.lat(), GEOHASH_TOLERANCE)); + MatcherAssert.assertThat(topLeft.lon(), closeTo(geoShapeTopLeft.lon(), GEOHASH_TOLERANCE)); + MatcherAssert.assertThat(bottomRight.lat(), closeTo(geoShapeBottomRight.lat(), GEOHASH_TOLERANCE)); + MatcherAssert.assertThat(bottomRight.lon(), closeTo(geoShapeBottomRight.lon(), GEOHASH_TOLERANCE)); + } + + public void testEmptyAggregationOnGeoShapes() { + final SearchResponse searchResponse = client().prepareSearch(EMPTY_IDX_NAME) + .setQuery(matchAllQuery()) + .addAggregation(geoBounds(aggName).field(GEO_SHAPE_FIELD_NAME).wrapLongitude(false)) + .get(); + + MatcherAssert.assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L)); + final GeoBounds geoBounds = searchResponse.getAggregations().get(aggName); + MatcherAssert.assertThat(geoBounds, notNullValue()); + MatcherAssert.assertThat(geoBounds.getName(), equalTo(aggName)); + final GeoPoint topLeft = geoBounds.topLeft(); + final GeoPoint bottomRight = geoBounds.bottomRight(); + MatcherAssert.assertThat(topLeft, equalTo(null)); + MatcherAssert.assertThat(bottomRight, equalTo(null)); + } } diff --git a/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java b/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java index 25dcf8db2c407..77abba7f54677 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java +++ b/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java @@ -40,6 +40,7 @@ import org.opensearch.geo.search.aggregations.bucket.geogrid.InternalGeoTileGrid; import org.opensearch.geo.search.aggregations.metrics.GeoBounds; import org.opensearch.geo.search.aggregations.metrics.GeoBoundsAggregationBuilder; +import org.opensearch.geo.search.aggregations.metrics.GeoBoundsGeoShapeAggregator; import org.opensearch.geo.search.aggregations.metrics.InternalGeoBounds; import org.opensearch.index.mapper.GeoShapeFieldMapper; import org.opensearch.index.mapper.Mapper; @@ -47,10 +48,13 @@ import org.opensearch.plugins.Plugin; import org.opensearch.plugins.SearchPlugin; import org.opensearch.search.aggregations.bucket.composite.CompositeAggregation; +import org.opensearch.search.aggregations.support.CoreValuesSourceType; +import org.opensearch.search.aggregations.support.ValuesSourceRegistry; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Consumer; public class GeoModulePlugin extends Plugin implements MapperPlugin, SearchPlugin { @@ -102,4 +106,23 @@ public List getCompositeAggregations() { ) ); } + + /** + * Registering the GeoBounds Aggregation on the GeoShape Field. This function allows plugins to register new + * aggregations using aggregation names that are already defined in Core, as long as the new aggregations target + * different ValuesSourceTypes. + * + * @return A list of the new registrar functions + */ + @Override + public List> getAggregationExtentions() { + final Consumer geoShapeConsumer = builder -> builder.register( + GeoBoundsAggregationBuilder.REGISTRY_KEY, + CoreValuesSourceType.GEO_SHAPE, + GeoBoundsGeoShapeAggregator::new, + true + ); + return Collections.singletonList(geoShapeConsumer); + } + } diff --git a/modules/geo/src/main/java/org/opensearch/geo/algorithm/PolygonGenerator.java b/modules/geo/src/main/java/org/opensearch/geo/algorithm/PolygonGenerator.java new file mode 100644 index 0000000000000..246ece4342cff --- /dev/null +++ b/modules/geo/src/main/java/org/opensearch/geo/algorithm/PolygonGenerator.java @@ -0,0 +1,190 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.algorithm; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.util.CollectionUtils; + +import java.awt.geom.Point2D; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Random; +import java.util.stream.IntStream; + +/** + * Helper class to generate a polygon. Keeping this in the src folder so that GeoSpatial plugin can take advantage of + * this helper to create the Polygons, rather than hardcoding the values. + */ +public class PolygonGenerator { + + private static final Logger LOG = LogManager.getLogger(PolygonGenerator.class); + + /** + * A helper function to create the Polygons for testing. The returned list of double array where first element + * contains all the X points and second contains all the Y points. + * + * @param xPool a {@link java.util.List} of {@link Double} + * @param yPool a {@link java.util.List} of {@link Double} + * @return a {@link List} of double array. + */ + public static List generatePolygon(final List xPool, final List yPool, final Random random) { + if (CollectionUtils.isEmpty(xPool) || CollectionUtils.isEmpty(yPool)) { + LOG.debug("One of the X or Y list is empty or null. X.size : {} Y.size : {}", xPool, yPool); + return Collections.emptyList(); + } + final List generatedPolygonPointsList = ValtrAlgorithm.generateRandomConvexPolygon(xPool, yPool, random); + final double[] x = new double[generatedPolygonPointsList.size()]; + final double[] y = new double[generatedPolygonPointsList.size()]; + IntStream.range(0, generatedPolygonPointsList.size()).forEach(iterator -> { + x[iterator] = generatedPolygonPointsList.get(iterator).getX(); + y[iterator] = generatedPolygonPointsList.get(iterator).getY(); + }); + final List pointsList = new ArrayList<>(); + pointsList.add(x); + pointsList.add(y); + return pointsList; + } + + /* + * MIT License + * + * Copyright (c) 2017 Sander Verdonschot + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + /** + * Provides a helper function to create a Polygon with a list of points. This source code is used to create the + * polygons in the test cases. + * Reference Link + * Visual Link + */ + private static class ValtrAlgorithm { + /** + * Generates a convex polygon using the points provided as a {@link List} of {@link Double} for both X and Y axis. + * + * @param xPool a {@link List} of {@link Double} + * @param yPool a {@link List} of {@link Double} + * @return a {@link List} of {@link Point2D.Double} + */ + private static List generateRandomConvexPolygon( + final List xPool, + final List yPool, + final Random random + ) { + final int n = xPool.size(); + // Sort them + Collections.sort(xPool); + Collections.sort(yPool); + + // Isolate the extreme points + final Double minX = xPool.get(0); + final Double maxX = xPool.get(n - 1); + final Double minY = yPool.get(0); + final Double maxY = yPool.get(n - 1); + + // Divide the interior points into two chains & Extract the vector components + java.util.List xVec = new ArrayList<>(n); + java.util.List yVec = new ArrayList<>(n); + + double lastTop = minX, lastBot = minX; + + for (int i = 1; i < n - 1; i++) { + double x = xPool.get(i); + + if (random.nextBoolean()) { + xVec.add(x - lastTop); + lastTop = x; + } else { + xVec.add(lastBot - x); + lastBot = x; + } + } + + xVec.add(maxX - lastTop); + xVec.add(lastBot - maxX); + + double lastLeft = minY, lastRight = minY; + + for (int i = 1; i < n - 1; i++) { + double y = yPool.get(i); + + if (random.nextBoolean()) { + yVec.add(y - lastLeft); + lastLeft = y; + } else { + yVec.add(lastRight - y); + lastRight = y; + } + } + + yVec.add(maxY - lastLeft); + yVec.add(lastRight - maxY); + + // Randomly pair up the X- and Y-components + Collections.shuffle(yVec, random); + + // Combine the paired up components into vectors + List vec = new ArrayList<>(n); + + for (int i = 0; i < n; i++) { + vec.add(new Point2D.Double(xVec.get(i), yVec.get(i))); + } + + // Sort the vectors by angle + Collections.sort(vec, Comparator.comparingDouble(v -> Math.atan2(v.getY(), v.getX()))); + + // Lay them end-to-end + double x = 0, y = 0; + double minPolygonX = 0; + double minPolygonY = 0; + List points = new ArrayList<>(n); + + for (int i = 0; i < n; i++) { + points.add(new Point2D.Double(x, y)); + + x += vec.get(i).getX(); + y += vec.get(i).getY(); + + minPolygonX = Math.min(minPolygonX, x); + minPolygonY = Math.min(minPolygonY, y); + } + + // Move the polygon to the original min and max coordinates + double xShift = minX - minPolygonX; + double yShift = minY - minPolygonY; + + for (int i = 0; i < n; i++) { + Point2D.Double p = points.get(i); + points.set(i, new Point2D.Double(p.x + xShift, p.y + yShift)); + } + + return points; + } + } + +} diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsGeoShapeAggregator.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsGeoShapeAggregator.java new file mode 100644 index 0000000000000..918b9a6701490 --- /dev/null +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsGeoShapeAggregator.java @@ -0,0 +1,116 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.metrics; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.index.LeafReaderContext; +import org.opensearch.common.geo.GeoShapeDocValue; +import org.opensearch.common.util.BigArrays; +import org.opensearch.index.fielddata.GeoShapeValue; +import org.opensearch.search.aggregations.Aggregator; +import org.opensearch.search.aggregations.LeafBucketCollector; +import org.opensearch.search.aggregations.LeafBucketCollectorBase; +import org.opensearch.search.aggregations.support.ValuesSource; +import org.opensearch.search.aggregations.support.ValuesSourceConfig; +import org.opensearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Map; + +/** + * Aggregate all docs into a geographic bounds for field geo_shape. + * + * @opensearch.internal + */ +public final class GeoBoundsGeoShapeAggregator extends AbstractGeoBoundsAggregator { + private static final Logger LOGGER = LogManager.getLogger(GeoBoundsGeoShapeAggregator.class); + + public GeoBoundsGeoShapeAggregator( + String name, + SearchContext searchContext, + Aggregator aggregator, + ValuesSourceConfig valuesSourceConfig, + boolean wrapLongitude, + Map metaData + ) throws IOException { + super(name, searchContext, aggregator, valuesSourceConfig, wrapLongitude, metaData); + } + + @Override + protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector leafBucketCollector) { + if (valuesSource == null) { + return LeafBucketCollector.NO_OP_COLLECTOR; + } + final BigArrays bigArrays = context.bigArrays(); + final GeoShapeValue values = valuesSource.getGeoShapeValues(ctx); + return new LeafBucketCollectorBase(leafBucketCollector, values) { + @Override + public void collect(int doc, long bucket) throws IOException { + setBucketSize(bucket, bigArrays); + if (values.advanceExact(doc)) { + final GeoShapeDocValue value = values.nextValue(); + final GeoShapeDocValue.BoundingRectangle boundingBox = value.getBoundingRectangle(); + if (boundingBox != null) { + double top = tops.get(bucket); + if (boundingBox.getMaxLatitude() > top) { + top = boundingBox.getMaxLatitude(); + } + + double bottom = bottoms.get(bucket); + if (boundingBox.getMinLatitude() < bottom) { + bottom = boundingBox.getMinLatitude(); + } + + double posLeft = posLefts.get(bucket); + if (boundingBox.getMinLongitude() >= 0 && boundingBox.getMinLongitude() < posLeft) { + posLeft = boundingBox.getMinLongitude(); + } + if (boundingBox.getMaxLongitude() >= 0 && boundingBox.getMaxLongitude() < posLeft) { + posLeft = boundingBox.getMaxLongitude(); + } + + double posRight = posRights.get(bucket); + if (boundingBox.getMaxLongitude() >= 0 && boundingBox.getMaxLongitude() > posRight) { + posRight = boundingBox.getMaxLongitude(); + } + if (boundingBox.getMinLongitude() >= 0 && boundingBox.getMinLongitude() > posRight) { + posRight = boundingBox.getMinLongitude(); + } + + double negLeft = negLefts.get(bucket); + if (boundingBox.getMinLongitude() < 0 && boundingBox.getMinLongitude() < negLeft) { + negLeft = boundingBox.getMinLongitude(); + } + if (boundingBox.getMaxLongitude() < 0 && boundingBox.getMaxLongitude() < negLeft) { + negLeft = boundingBox.getMaxLongitude(); + } + + double negRight = negRights.get(bucket); + if (boundingBox.getMaxLongitude() < 0 && boundingBox.getMaxLongitude() > negRight) { + negRight = boundingBox.getMaxLongitude(); + } + if (boundingBox.getMinLongitude() < 0 && boundingBox.getMinLongitude() > negRight) { + negRight = boundingBox.getMinLongitude(); + } + + tops.set(bucket, top); + bottoms.set(bucket, bottom); + posLefts.set(bucket, posLeft); + posRights.set(bucket, posRight); + negLefts.set(bucket, negLeft); + negRights.set(bucket, negRight); + } else { + LOGGER.error("The bounding box was null for the Doc id {}", doc); + } + } + } + }; + } +} diff --git a/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsGeoShapeAggregatorTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsGeoShapeAggregatorTests.java new file mode 100644 index 0000000000000..68d9434631364 --- /dev/null +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsGeoShapeAggregatorTests.java @@ -0,0 +1,237 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.metrics; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.LatLonShape; +import org.apache.lucene.document.ShapeDocValuesField; +import org.apache.lucene.geo.LatLonGeometry; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.hamcrest.MatcherAssert; +import org.junit.Assert; +import org.opensearch.common.geo.GeoBoundingBox; +import org.opensearch.common.geo.GeoPoint; +import org.opensearch.common.geo.GeoShapeUtils; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.tests.common.AggregationInspectionHelper; +import org.opensearch.geo.tests.common.RandomGeoGeometryGenerator; +import org.opensearch.geometry.Circle; +import org.opensearch.geometry.Geometry; +import org.opensearch.geometry.Line; +import org.opensearch.geometry.Point; +import org.opensearch.geometry.Polygon; +import org.opensearch.geometry.ShapeType; +import org.opensearch.index.mapper.GeoShapeFieldMapper; +import org.opensearch.index.mapper.GeoShapeIndexer; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.plugins.SearchPlugin; +import org.opensearch.search.aggregations.AggregatorTestCase; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Random; + +import static org.hamcrest.Matchers.closeTo; + +public class GeoBoundsGeoShapeAggregatorTests extends AggregatorTestCase { + private static final Logger LOG = LogManager.getLogger(GeoBoundsGeoShapeAggregatorTests.class); + private static final double GEOHASH_TOLERANCE = 1E-5D; + private static final String AGGREGATION_NAME = "my_agg"; + private static final String FIELD_NAME = "field"; + + /** + * Overriding the Search Plugins list with {@link GeoModulePlugin} so that the testcase will know that this plugin is + * to be loaded during the tests. + * + * @return List of {@link SearchPlugin} + */ + @Override + protected List getSearchPlugins() { + return Collections.singletonList(new GeoModulePlugin()); + } + + /** + * Testing Empty aggregator results. + * + * @throws Exception + */ + public void testEmpty() throws Exception { + try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + final GeoBoundsAggregationBuilder aggBuilder = new GeoBoundsAggregationBuilder(AGGREGATION_NAME).field(FIELD_NAME) + .wrapLongitude(false); + + final MappedFieldType fieldType = new GeoShapeFieldMapper.GeoShapeFieldType(FIELD_NAME); + try (IndexReader reader = w.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + InternalGeoBounds bounds = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + assertTrue(Double.isInfinite(bounds.top)); + assertTrue(Double.isInfinite(bounds.bottom)); + assertTrue(Double.isInfinite(bounds.posLeft)); + assertTrue(Double.isInfinite(bounds.posRight)); + assertTrue(Double.isInfinite(bounds.negLeft)); + assertTrue(Double.isInfinite(bounds.negRight)); + assertFalse(AggregationInspectionHelper.hasValue(bounds)); + } + } + } + + /** + * Testing GeoBoundAggregator for random shapes which are indexed. + * + * @throws Exception + */ + public void testRandom() throws Exception { + final int numDocs = randomIntBetween(50, 100); + final List Y = new ArrayList<>(); + final List X = new ArrayList<>(); + final Random random = random(); + try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random, dir)) { + for (int i = 0; i < numDocs; i++) { + final Document document = new Document(); + final Geometry geometry = randomLuceneGeometry(random); + LOG.debug("Random Geometry created for Indexing : {}", geometry); + document.add(createShapeDocValue(geometry)); + w.addDocument(document); + getAllXAndYPoints(geometry, X, Y); + } + final GeoBoundsAggregationBuilder aggBuilder = new GeoBoundsAggregationBuilder(AGGREGATION_NAME).field(FIELD_NAME) + .wrapLongitude(false); + final MappedFieldType fieldType = new GeoShapeFieldMapper.GeoShapeFieldType(FIELD_NAME); + try (IndexReader reader = w.getReader()) { + final IndexSearcher searcher = new IndexSearcher(reader); + final InternalGeoBounds actualBounds = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + final GeoBoundingBox expectedGeoBounds = getExpectedGeoBounds(X, Y); + MatcherAssert.assertThat( + actualBounds.bottomRight().getLat(), + closeTo(expectedGeoBounds.bottomRight().getLat(), GEOHASH_TOLERANCE) + ); + MatcherAssert.assertThat( + actualBounds.bottomRight().getLon(), + closeTo(expectedGeoBounds.bottomRight().getLon(), GEOHASH_TOLERANCE) + ); + MatcherAssert.assertThat(actualBounds.topLeft().getLat(), closeTo(expectedGeoBounds.topLeft().getLat(), GEOHASH_TOLERANCE)); + MatcherAssert.assertThat(actualBounds.topLeft().getLon(), closeTo(expectedGeoBounds.topLeft().getLon(), GEOHASH_TOLERANCE)); + assertTrue(AggregationInspectionHelper.hasValue(actualBounds)); + } + } + } + + private GeoBoundingBox getExpectedGeoBounds(final List X, final List Y) { + double top = Double.NEGATIVE_INFINITY; + double bottom = Double.POSITIVE_INFINITY; + double posLeft = Double.POSITIVE_INFINITY; + double posRight = Double.NEGATIVE_INFINITY; + double negLeft = Double.POSITIVE_INFINITY; + double negRight = Double.NEGATIVE_INFINITY; + // Finding the bounding box for the shapes. + for (final Double lon : X) { + if (lon >= 0 && lon < posLeft) { + posLeft = lon; + } + if (lon >= 0 && lon > posRight) { + posRight = lon; + } + if (lon < 0 && lon < negLeft) { + negLeft = lon; + } + if (lon < 0 && lon > negRight) { + negRight = lon; + } + } + for (final Double lat : Y) { + if (lat > top) { + top = lat; + } + if (lat < bottom) { + bottom = lat; + } + } + if (Double.isInfinite(posLeft)) { + return new GeoBoundingBox(new GeoPoint(top, negLeft), new GeoPoint(bottom, negRight)); + } else if (Double.isInfinite(negLeft)) { + return new GeoBoundingBox(new GeoPoint(top, posLeft), new GeoPoint(bottom, posRight)); + } else { + return new GeoBoundingBox(new GeoPoint(top, negLeft), new GeoPoint(bottom, posRight)); + } + } + + private void getAllXAndYPoints(final Geometry geometry, final List X, final List Y) { + if (geometry instanceof Point) { + final Point point = (Point) geometry; + X.add(point.getX()); + Y.add(point.getY()); + return; + } else if (geometry instanceof Polygon) { + final Polygon polygon = (Polygon) geometry; + for (int i = 0; i < polygon.getPolygon().getX().length; i++) { + X.add(polygon.getPolygon().getX(i)); + Y.add(polygon.getPolygon().getY(i)); + } + return; + } else if (geometry instanceof Line) { + final Line line = (Line) geometry; + for (int i = 0; i < line.getX().length; i++) { + X.add(line.getX(i)); + Y.add(line.getY(i)); + } + return; + } + Assert.fail( + String.format(Locale.ROOT, "Error cannot convert the %s to a valid indexable format[POINT, POLYGON, LINE]", geometry.getClass()) + ); + } + + private ShapeDocValuesField createShapeDocValue(final Geometry geometry) { + if (geometry instanceof Point) { + final Point point = (Point) geometry; + return LatLonShape.createDocValueField(FIELD_NAME, point.getLat(), point.getLon()); + } else if (geometry instanceof Polygon) { + return LatLonShape.createDocValueField(FIELD_NAME, GeoShapeUtils.toLucenePolygon((Polygon) geometry)); + } else if (geometry instanceof Line) { + return LatLonShape.createDocValueField(FIELD_NAME, GeoShapeUtils.toLuceneLine((Line) geometry)); + } + Assert.fail( + String.format(Locale.ROOT, "Error cannot convert the %s to a valid indexable format[POINT, POLYGON, LINE]", geometry.getClass()) + ); + return null; + } + + /** + * Random function to generate a {@link LatLonGeometry}. Now for indexing of GeoShape field, we index all the + * different Geometry shapes that we support({@link ShapeType}) in OpenSearch are broken down into 3 shapes only. + * Hence, we are generating only 3 shapes : {@link org.apache.lucene.geo.Point}, + * {@link org.apache.lucene.geo.Line}, {@link org.apache.lucene.geo.Polygon}. {@link Circle} is not supported. + * Check {@link GeoShapeIndexer#prepareForIndexing(org.opensearch.geometry.Geometry)} + * + * @return {@link LatLonGeometry} + */ + private static Geometry randomLuceneGeometry(final Random r) { + int shapeNumber = OpenSearchTestCase.randomIntBetween(0, 2); + if (shapeNumber == 0) { + // Point + return RandomGeoGeometryGenerator.randomPoint(r); + } else if (shapeNumber == 1) { + // LineString + return RandomGeoGeometryGenerator.randomLine(r); + } else { + // Polygon + return RandomGeoGeometryGenerator.randomPolygon(r); + } + } + +} diff --git a/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGeometryGenerator.java b/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGeometryGenerator.java new file mode 100644 index 0000000000000..caf15507e08c5 --- /dev/null +++ b/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGeometryGenerator.java @@ -0,0 +1,240 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.tests.common; + +import org.junit.Assert; +import org.opensearch.geo.algorithm.PolygonGenerator; +import org.opensearch.geometry.Geometry; +import org.opensearch.geometry.GeometryCollection; +import org.opensearch.geometry.Line; +import org.opensearch.geometry.LinearRing; +import org.opensearch.geometry.MultiLine; +import org.opensearch.geometry.MultiPoint; +import org.opensearch.geometry.MultiPolygon; +import org.opensearch.geometry.Point; +import org.opensearch.geometry.Polygon; +import org.opensearch.geometry.Rectangle; +import org.opensearch.geometry.ShapeType; +import org.opensearch.index.mapper.GeoShapeIndexer; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Random; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * Random geo generation utilities for randomized geo_shape type testing. + */ +public class RandomGeoGeometryGenerator { + // Just picking a number 10 to be the max edges of a polygon. Don't want to make too large which can impact + // debugging. + private static final int MAX_VERTEXES = 10; + private static final int MAX_MULTIPLE_GEOMETRIES = 10; + + private static final Predicate NOT_SUPPORTED_SHAPES = shapeType -> shapeType != ShapeType.CIRCLE + && shapeType != ShapeType.LINEARRING; + + /** + * Creating list of only supported geometries defined here: {@link GeoShapeIndexer#prepareForIndexing(Geometry)} + */ + private static final List SUPPORTED_SHAPE_TYPES = Arrays.stream(ShapeType.values()) + .filter(NOT_SUPPORTED_SHAPES) + .collect(Collectors.toList()); + + /** + * Returns a random Geometry. It makes sure that only that geometry is returned which is supported by OpenSearch + * while indexing. Check {@link GeoShapeIndexer#prepareForIndexing(Geometry)} + * + * @return {@link Geometry} + */ + public static Geometry randomGeometry(final Random r) { + final ShapeType randomShapeType = SUPPORTED_SHAPE_TYPES.get( + OpenSearchTestCase.randomIntBetween(0, SUPPORTED_SHAPE_TYPES.size() - 1) + ); + switch (randomShapeType) { + case POINT: + return randomPoint(r); + case MULTIPOINT: + return randomMultiPoint(r); + case POLYGON: + return randomPolygon(r); + case LINESTRING: + return randomLine(r); + case MULTIPOLYGON: + return randomMultiPolygon(r); + case GEOMETRYCOLLECTION: + return randomGeometryCollection(r); + case MULTILINESTRING: + return randomMultiLine(r); + case ENVELOPE: + return randomRectangle(r); + default: + Assert.fail(String.format(Locale.ROOT, "Cannot create a geometry of type %s ", randomShapeType)); + } + return null; + } + + /** + * Generate a random point on the Earth Surface. + * + * @param r {@link Random} + * @return {@link Point} + */ + public static Point randomPoint(final Random r) { + double[] pt = getLonAndLatitude(r); + return new Point(pt[0], pt[1]); + } + + /** + * Generate a random polygon on earth surface. + * + * @param r {@link Random} + * @return {@link Polygon} + */ + public static Polygon randomPolygon(final Random r) { + final int vertexCount = OpenSearchTestCase.randomIntBetween(3, MAX_VERTEXES); + return randomPolygonWithFixedVertexCount(r, vertexCount); + } + + /** + * Generate a random line on the earth Surface. + * + * @param r {@link Random} + * @return {@link Line} + */ + public static Line randomLine(final Random r) { + final double[] pt1 = getLonAndLatitude(r); + final double[] pt2 = getLonAndLatitude(r); + final double[] x = { pt1[0], pt2[0] }; + final double[] y = { pt1[1], pt2[1] }; + return new Line(x, y); + } + + /** + * Returns an object of {@link MultiPoint} denoting a list of points on earth surface. + * @param r {@link Random} + * @return {@link MultiPoint} + */ + public static MultiPoint randomMultiPoint(final Random r) { + int multiplePoints = OpenSearchTestCase.randomIntBetween(1, MAX_MULTIPLE_GEOMETRIES); + final List pointsList = new ArrayList<>(); + IntStream.range(0, multiplePoints).forEach(i -> pointsList.add(randomPoint(r))); + return new MultiPoint(pointsList); + } + + /** + * Returns an object of {@link MultiPolygon} denoting various polygons on earth surface. + * + * @param r {@link Random} + * @return {@link MultiPolygon} + */ + public static MultiPolygon randomMultiPolygon(final Random r) { + int multiplePolygons = OpenSearchTestCase.randomIntBetween(1, MAX_MULTIPLE_GEOMETRIES); + final List polygonList = new ArrayList<>(); + IntStream.range(0, multiplePolygons).forEach(i -> polygonList.add(randomPolygon(r))); + return new MultiPolygon(polygonList); + } + + /** + * Returns an object of {@link GeometryCollection} having various shapes on earth surface. + * + * @param r {@link Random} + * @return {@link GeometryCollection} + */ + public static GeometryCollection randomGeometryCollection(final Random r) { + final List geometries = new ArrayList<>(); + geometries.addAll(randomMultiPoint(r).getAll()); + geometries.addAll(randomMultiPolygon(r).getAll()); + geometries.addAll(randomMultiLine(r).getAll()); + geometries.add(randomPoint(r)); + geometries.add(randomLine(r)); + geometries.add(randomPolygon(r)); + geometries.add(randomRectangle(r)); + return new GeometryCollection<>(geometries); + } + + /** + * Returns a {@link MultiLine} object containing multiple lines on earth surface. + * + * @param r {@link Random} + * @return {@link MultiLine} + */ + public static MultiLine randomMultiLine(Random r) { + int multiLines = OpenSearchTestCase.randomIntBetween(1, MAX_MULTIPLE_GEOMETRIES); + final List linesList = new ArrayList<>(); + IntStream.range(0, multiLines).forEach(i -> linesList.add(randomLine(r))); + return new MultiLine(linesList); + } + + /** + * Returns a random {@link Rectangle} created on earth surface. + * + * @param r {@link Random} + * @return {@link Rectangle} + */ + public static Rectangle randomRectangle(final Random r) { + final Polygon polygon = randomPolygonWithFixedVertexCount(r, 4); + double minX = Double.POSITIVE_INFINITY, maxX = Double.NEGATIVE_INFINITY, maxY = Double.NEGATIVE_INFINITY, minY = + Double.POSITIVE_INFINITY; + for (int i = 0; i < polygon.getPolygon().length(); i++) { + double x = polygon.getPolygon().getX()[i]; + double y = polygon.getPolygon().getY()[i]; + + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + } + return new Rectangle(minX, maxX, maxY, minY); + } + + /** + * Returns a double array where pt[0] : longitude and pt[1] : latitude + * + * @param r {@link Random} + * @return double[] + */ + private static double[] getLonAndLatitude(final Random r) { + double[] pt = new double[2]; + RandomGeoGenerator.randomPoint(r, pt); + return pt; + } + + private static Polygon randomPolygonWithFixedVertexCount(final Random r, final int vertexCount) { + final List xPool = new ArrayList<>(vertexCount); + final List yPool = new ArrayList<>(vertexCount); + IntStream.range(0, vertexCount).forEach(iterator -> { + double[] pt = getLonAndLatitude(r); + xPool.add(pt[0]); + yPool.add(pt[1]); + }); + final List pointsList = PolygonGenerator.generatePolygon(xPool, yPool, r); + // Checking the list + assert vertexCount == pointsList.get(0).length; + assert vertexCount == pointsList.get(1).length; + // Create the linearRing, as we need to close the polygon hence increasing vertexes count by 1 + final double[] x = new double[vertexCount + 1]; + final double[] y = new double[vertexCount + 1]; + IntStream.range(0, vertexCount).forEach(iterator -> { + x[iterator] = pointsList.get(0)[iterator]; + y[iterator] = pointsList.get(1)[iterator]; + }); + // making sure to close the polygon + x[vertexCount] = x[0]; + y[vertexCount] = y[0]; + final LinearRing linearRing = new LinearRing(x, y); + return new Polygon(linearRing); + } + +} diff --git a/server/src/main/java/org/opensearch/common/geo/GeoShapeDocValue.java b/server/src/main/java/org/opensearch/common/geo/GeoShapeDocValue.java new file mode 100644 index 0000000000000..9bc28c1f67d47 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/geo/GeoShapeDocValue.java @@ -0,0 +1,175 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.geo; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.LatLonShape; +import org.apache.lucene.document.LatLonShapeDocValuesField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.util.BytesRef; +import org.opensearch.geometry.Geometry; +import org.opensearch.index.mapper.GeoShapeIndexer; + +import java.util.List; + +/** + * This class is an OpenSearch Internal representation of lucene {@link LatLonShapeDocValuesField} for GeoShape. + * + * @opensearch.internal + */ +public class GeoShapeDocValue extends ShapeDocValue { + private static final String FIELD_NAME = "missingField"; + + public GeoShapeDocValue(final String fieldName, final BytesRef bytesRef) { + this(LatLonShape.createDocValueField(fieldName, bytesRef)); + } + + public GeoShapeDocValue(final LatLonShapeDocValuesField shapeDocValuesField) { + centroid = new Centroid(shapeDocValuesField.getCentroid().getLat(), shapeDocValuesField.getCentroid().getLon()); + highestDimensionType = ShapeType.fromShapeFieldType(shapeDocValuesField.getHighestDimensionType()); + boundingRectangle = new BoundingRectangle( + shapeDocValuesField.getBoundingBox().maxLon, + shapeDocValuesField.getBoundingBox().maxLat, + shapeDocValuesField.getBoundingBox().minLon, + shapeDocValuesField.getBoundingBox().minLat + ); + } + + /** + * This function takes a {@link Geometry} and creates the {@link GeoShapeDocValue}. The function uses the + * {@link GeoShapeIndexer} to first convert the {@link Geometry} to {@link IndexableField}s and then convert it + * to the DocValue. This is very expensive function and should not be used on the Geometry Objects which are + * already converted to {@link IndexableField}s as it does the Tessellation internally which is already done on the + * {@link IndexableField}s. + * + * @param geometry {@link Geometry} + * @return {@link GeoShapeDocValue} + */ + public static GeoShapeDocValue createGeometryDocValue(final Geometry geometry) { + // Setting the orientation to CCW, which will make the holes to CW. This is the default value which we will + // be using. This is in conjunction with what we take as a default value for geoshape in WKT format. + final GeoShapeIndexer shapeIndex = new GeoShapeIndexer(true, FIELD_NAME); + final List indexableFields = shapeIndex.indexShape(null, shapeIndex.prepareForIndexing(geometry)); + Field[] fieldsArray = new Field[indexableFields.size()]; + fieldsArray = indexableFields.toArray(fieldsArray); + final LatLonShapeDocValuesField latLonShapeDocValuesField = LatLonShape.createDocValueField(FIELD_NAME, fieldsArray); + return new GeoShapeDocValue(latLonShapeDocValuesField); + } + + public Centroid getCentroid() { + return (Centroid) centroid; + } + + public BoundingRectangle getBoundingRectangle() { + return (BoundingRectangle) boundingRectangle; + } + + @Override + public String toString() { + return "BoundingRectangle(" + boundingRectangle + "), Centroid(" + centroid + "), HighestDimension(" + highestDimensionType + ")"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + GeoShapeDocValue object = (GeoShapeDocValue) o; + boolean isEqual = true; + if (boundingRectangle != null) { + isEqual = boundingRectangle.equals(object.getBoundingRectangle()); + } + if (centroid != null) { + isEqual = isEqual && centroid.equals(object.getCentroid()); + } + if (highestDimensionType != null) { + isEqual = isEqual && highestDimensionType == object.getHighestDimensionType(); + } + return isEqual; + } + + @Override + public int hashCode() { + int result; + long temp; + temp = boundingRectangle != null ? boundingRectangle.hashCode() : 0L; + result = Long.hashCode(temp); + temp = centroid != null ? centroid.hashCode() : 0L; + result = 31 * result + Long.hashCode(temp); + temp = highestDimensionType != null ? highestDimensionType.hashCode() : 0L; + + result = 31 * result + Long.hashCode(temp); + return result; + } + + /** + * An extension for {@link ShapeDocValue.Centroid} which make easy to read the values when centroid is on the + * EarthSurface + */ + public static final class Centroid extends ShapeDocValue.Centroid { + + Centroid(final double lat, final double lon) { + super(lat, lon); + } + + public double getLatitude() { + return getY(); + } + + public double getLongitude() { + return getX(); + } + + @Override + public String toString() { + return getY() + ", " + getX(); + } + + } + + /** + * An extension for {@link ShapeDocValue.BoundingRectangle} which make easy to read the values when BB is on the + * EarthSurface + */ + public static final class BoundingRectangle extends ShapeDocValue.BoundingRectangle { + + BoundingRectangle(final double maxLon, final double maxLat, final double minLon, final double minLat) { + super(maxLon, maxLat, minLon, minLat); + } + + public double getMaxLongitude() { + return getMaxX(); + } + + public double getMaxLatitude() { + return getMaxY(); + } + + public double getMinLatitude() { + return getMinY(); + } + + public double getMinLongitude() { + return getMinX(); + } + + @Override + public String toString() { + return "maxLatitude: " + + getMaxY() + + ", minLatitude: " + + getMinY() + + ", maxLongitude: " + + getMaxX() + + ", minLongitude: " + + getMinX(); + + } + } +} diff --git a/server/src/main/java/org/opensearch/common/geo/ShapeDocValue.java b/server/src/main/java/org/opensearch/common/geo/ShapeDocValue.java new file mode 100644 index 0000000000000..068cab91e53d3 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/geo/ShapeDocValue.java @@ -0,0 +1,188 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.geo; + +import org.apache.lucene.document.ShapeField; +import org.apache.lucene.document.XYShapeDocValuesField; + +import java.util.Arrays; +import java.util.Locale; + +/** + * This class is an OpenSearch Internal representation of lucene {@link XYShapeDocValuesField} for GeoShape. + * + * @opensearch.internal + */ +public class ShapeDocValue { + protected Centroid centroid; + protected BoundingRectangle boundingRectangle; + protected ShapeType highestDimensionType; + + public Centroid getCentroid() { + return centroid; + } + + public BoundingRectangle getBoundingRectangle() { + return boundingRectangle; + } + + public ShapeType getHighestDimensionType() { + return highestDimensionType; + } + + /** + * Provides the centroid of the field(Shape) which has been indexed. + */ + public static class Centroid { + private final double y; + private final double x; + + Centroid(final double y, final double x) { + this.y = y; + this.x = x; + } + + public double getY() { + return y; + } + + public double getX() { + return x; + } + + @Override + public String toString() { + return y + ", " + x; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Centroid centroid = (Centroid) o; + + if (Double.compare(centroid.y, y) != 0) return false; + if (Double.compare(centroid.x, x) != 0) return false; + return true; + } + + @Override + public int hashCode() { + int result; + long temp; + temp = y != +0.0d ? Double.doubleToLongBits(y) : 0L; + result = Long.hashCode(temp); + temp = x != +0.0d ? Double.doubleToLongBits(x) : 0L; + result = 31 * result + Long.hashCode(temp); + return result; + } + } + + /** + * Provides the BoundingBox of the field(Shape) which has been indexed. + */ + public static class BoundingRectangle { + private final double maxX, maxY, minY, minX; + + BoundingRectangle(final double maxLon, final double maxLat, final double minLon, final double minLat) { + maxY = maxLat; + maxX = maxLon; + minY = minLat; + minX = minLon; + } + + public double getMaxX() { + return maxX; + } + + public double getMaxY() { + return maxY; + } + + public double getMinY() { + return minY; + } + + public double getMinX() { + return minX; + } + + @Override + public String toString() { + return "maxY: " + maxY + "minY: " + minY + "maxX: " + maxX + "minX: " + minX; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + BoundingRectangle boundingRectangle = (BoundingRectangle) o; + + if (Double.compare(boundingRectangle.maxY, maxY) != 0) return false; + if (Double.compare(boundingRectangle.maxX, minX) != 0) return false; + if (Double.compare(boundingRectangle.minY, minY) != 0) return false; + if (Double.compare(boundingRectangle.minX, minX) != 0) return false; + return true; + } + + @Override + public int hashCode() { + int result; + long temp; + temp = maxY != +0.0d ? Double.doubleToLongBits(maxY) : 0L; + result = Long.hashCode(temp); + + temp = maxX != +0.0d ? Double.doubleToLongBits(maxX) : 0L; + result = 31 * result + Long.hashCode(temp); + + temp = minY != +0.0d ? Double.doubleToLongBits(minY) : 0L; + result = 31 * result + Long.hashCode(temp); + + temp = minX != +0.0d ? Double.doubleToLongBits(minX) : 0L; + result = 31 * result + Long.hashCode(temp); + + return result; + } + } + + /** + * An Enum class defining the highest type of Geometry present in this doc value. + */ + public enum ShapeType { + POINT, + LINE, + TRIANGLE; + + public static ShapeType fromShapeFieldType(final ShapeField.DecodedTriangle.TYPE type) { + switch (type) { + case POINT: + return POINT; + case LINE: + return LINE; + case TRIANGLE: + return TRIANGLE; + } + throw new IllegalStateException( + String.format( + Locale.ROOT, + "No correct mapped type found for the value %s in the list of values : %s", + type, + Arrays.toString(ShapeType.values()) + ) + ); + } + + @Override + public String toString() { + return name(); + } + } +} diff --git a/server/src/main/java/org/opensearch/common/util/CollectionUtils.java b/server/src/main/java/org/opensearch/common/util/CollectionUtils.java index 622efc2074d0a..d8a86f4878f58 100644 --- a/server/src/main/java/org/opensearch/common/util/CollectionUtils.java +++ b/server/src/main/java/org/opensearch/common/util/CollectionUtils.java @@ -384,4 +384,16 @@ public static List> eagerPartition(List list, int size) { return result; } + + /** + * Check if a collection is empty or not. Empty collection mean either it is null or it has no elements in it. If + * collection contains a null element it means it is not empty. + * + * @param collection {@link Collection} + * @return boolean + * @param Element + */ + public static boolean isEmpty(final Collection collection) { + return collection == null || collection.isEmpty(); + } } diff --git a/server/src/main/java/org/opensearch/index/fielddata/FieldData.java b/server/src/main/java/org/opensearch/index/fielddata/FieldData.java index 0f18944c67ebb..a6c471fa2307f 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/FieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/FieldData.java @@ -99,12 +99,19 @@ public GeoPoint geoPointValue() { } /** - * Return a {@link SortedNumericDoubleValues} that doesn't contain any value. + * Return a {@link MultiGeoPointValues} that doesn't contain any value. */ public static MultiGeoPointValues emptyMultiGeoPoints() { return singleton(emptyGeoPoint()); } + /** + * Return a {@link GeoShapeValue} that doesn't contain any value. + */ + public static GeoShapeValue emptyGeoShape() { + return new GeoShapeValue.EmptyGeoShapeValue(); + } + /** * Returns a {@link DocValueBits} representing all documents from values that have a value. */ @@ -143,6 +150,19 @@ public boolean advanceExact(int doc) throws IOException { }; } + /** + * Returns a {@link DocValueBits} representing all documents from shapeValues that have + * a value. + */ + public static DocValueBits docsWithValue(final GeoShapeValue shapeValues) { + return new DocValueBits() { + @Override + public boolean advanceExact(int doc) throws IOException { + return shapeValues.advanceExact(doc); + } + }; + } + /** * Returns a {@link DocValueBits} representing all documents from doubleValues that have a value. */ @@ -408,6 +428,31 @@ public void get(List list) throws IOException { }); } + /** + * Return a {@link String} representation of the provided values. That is + * typically used for scripts or for the `map` execution mode of terms aggs. + * NOTE: this is very slow! + */ + public static SortedBinaryDocValues toString(final GeoShapeValue geoShapeValue) { + return toString(new ToStringValues() { + + /** + * Advance this instance to the given document id + * @return true if there is a value for this document + */ + @Override + public boolean advanceExact(int doc) throws IOException { + return geoShapeValue.advanceExact(doc); + } + + /** Fill the list of charsequences with the list of values for the current document. */ + @Override + public void get(List list) throws IOException { + list.add(geoShapeValue.nextValue().toString()); + } + }); + } + private static SortedBinaryDocValues toString(final ToStringValues toStringValues) { return new SortingBinaryDocValues() { diff --git a/server/src/main/java/org/opensearch/index/fielddata/GeoShapeValue.java b/server/src/main/java/org/opensearch/index/fielddata/GeoShapeValue.java new file mode 100644 index 0000000000000..5c1d0155c9c56 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/fielddata/GeoShapeValue.java @@ -0,0 +1,161 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.fielddata; + +import org.apache.lucene.document.LatLonShapeDocValuesField; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.util.BytesRef; +import org.opensearch.common.geo.GeoShapeDocValue; +import org.opensearch.geometry.Geometry; +import org.opensearch.index.mapper.GeoShapeFieldMapper; + +import java.io.IOException; + +/** + * A stateful lightweight iterator interface to read the stored form of {@link Geometry} aka + * {@link LatLonShapeDocValuesField} from Lucene per document. Check {@link GeoShapeFieldMapper} for details how we + * converted the {@link Geometry} to {@link LatLonShapeDocValuesField} + * + * @opensearch.internal + */ +public abstract class GeoShapeValue { + /** + * Creates a new {@link GeoShapeValue} instance + */ + protected GeoShapeValue() {} + + /** + * Advance this instance to the given document id + * + * @return true if there is a value for this document + */ + public abstract boolean advanceExact(int doc) throws IOException; + + /** + * Return the next value associated with the current document. + * + * @return the next value for the current docID set to {@link #advanceExact(int)}. + */ + public abstract GeoShapeDocValue nextValue() throws IOException; + + /** + * This is the representation of an EmptyGeoShapeValue + */ + public static class EmptyGeoShapeValue extends GeoShapeValue { + /** + * Advance this instance to the given document id + * + * @param doc int + * @return true if there is a value for this document + */ + @Override + public boolean advanceExact(int doc) throws IOException { + return false; + } + + /** + * Return the next value associated with the current document. + * + * @return the next value for the current docID set to {@link #advanceExact(int)}. + */ + @Override + public GeoShapeDocValue nextValue() throws IOException { + throw new UnsupportedOperationException("This empty geoShape value, hence this operation is not supported"); + } + } + + /** + * The MissingGeoShapeValue is used when on a particular document the GeoShape field is not present and user has + * provided a missing/default GeoShape value in the input which should be used. + */ + public static class MissingGeoShapeValue extends GeoShapeValue { + + private boolean useMissingGeoShapeValue; + private final GeoShapeValue valueSourceData; + private final Geometry missing; + + private GeoShapeDocValue geoShapeDocValue; + + public MissingGeoShapeValue(final GeoShapeValue valueSourceData, final Geometry missing) { + super(); + this.missing = missing; + this.valueSourceData = valueSourceData; + this.useMissingGeoShapeValue = false; + } + + /** + * Advance this instance to the given document id + * + * @param doc int + * @return true if there is a value for this document + */ + @Override + public boolean advanceExact(int doc) throws IOException { + // If we don't have next value for the doc then set useMissingGeoShapeValue = true + useMissingGeoShapeValue = !valueSourceData.advanceExact(doc); + // always return true because we want to return a value even if + // the document does not have a value + return true; + } + + /** + * Return the next value associated with the current document. + * + * @return the next value for the current docID set to {@link #advanceExact(int)}. + */ + @Override + public GeoShapeDocValue nextValue() throws IOException { + if (useMissingGeoShapeValue) { + if (geoShapeDocValue == null) { + // keeping geometryDocValue cache so that it can be reused. + geoShapeDocValue = GeoShapeDocValue.createGeometryDocValue(missing); + } + return geoShapeDocValue; + } + return valueSourceData.nextValue(); + } + } + + /** + * This is the standard implementation of the {@link GeoShapeValue} interface for iterating over the doc values + * for a GeoShape field. + */ + public static class StandardGeoShapeValue extends GeoShapeValue { + + private final BinaryDocValues binaryDocValues; + private final String fieldName; + + public StandardGeoShapeValue(final BinaryDocValues binaryDocValues, final String fieldName) { + this.binaryDocValues = binaryDocValues; + this.fieldName = fieldName; + } + + /** + * Advance this instance to the given document id + * + * @return true if there is a value for this document + */ + @Override + public boolean advanceExact(int doc) throws IOException { + return binaryDocValues.advanceExact(doc); + } + + /** + * Return the next value associated with the current document. + * + * @return the next value for the current docID set to {@link #advanceExact(int)}. + */ + @Override + public GeoShapeDocValue nextValue() throws IOException { + final BytesRef bytesRef = binaryDocValues.binaryValue(); + // Converting the ByteRef to GeometryDocValue. + return new GeoShapeDocValue(fieldName, bytesRef); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/fielddata/LeafGeoShapeFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/LeafGeoShapeFieldData.java new file mode 100644 index 0000000000000..ab4bc5d411852 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/fielddata/LeafGeoShapeFieldData.java @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.fielddata; + +/** + * {@link LeafFieldData} specialization for geo shapes. + * + * @opensearch.internal + */ +public interface LeafGeoShapeFieldData extends LeafFieldData { + + /** + * Return the appropriate instance that can be used to read the Geo shape values from lucene. + * + * @return {@link GeoShapeValue} + */ + GeoShapeValue getGeoShapeValue(); +} diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractGeoShapeIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractGeoShapeIndexFieldData.java new file mode 100644 index 0000000000000..2c6aabf04d4ee --- /dev/null +++ b/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractGeoShapeIndexFieldData.java @@ -0,0 +1,132 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.fielddata.plain; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.SortField; +import org.opensearch.common.Nullable; +import org.opensearch.common.util.BigArrays; +import org.opensearch.index.fielddata.IndexFieldData; +import org.opensearch.index.fielddata.IndexFieldDataCache; +import org.opensearch.index.fielddata.LeafGeoShapeFieldData; +import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.search.DocValueFormat; +import org.opensearch.search.MultiValueMode; +import org.opensearch.search.aggregations.support.ValuesSourceType; +import org.opensearch.search.sort.BucketedSort; +import org.opensearch.search.sort.SortOrder; + +/** + * Base class for retrieving Geometry docvalues + * + * @opensearch.internal + */ +public abstract class AbstractGeoShapeIndexFieldData implements IndexFieldData { + protected final String fieldName; + protected final ValuesSourceType valuesSourceType; + + AbstractGeoShapeIndexFieldData(String fieldName, ValuesSourceType valuesSourceType) { + this.fieldName = fieldName; + this.valuesSourceType = valuesSourceType; + } + + @Override + public final String getFieldName() { + return fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return valuesSourceType; + } + + /** + * Returns the {@link SortField} to use for sorting. + */ + @Override + public SortField sortField( + @Nullable Object missingValue, + MultiValueMode sortMode, + XFieldComparatorSource.Nested nested, + boolean reverse + ) { + throw new IllegalArgumentException("can't sort on geo_shape field without using specific sorting feature, like geo_distance"); + } + + /** + * Build a sort implementation specialized for aggregations. + */ + @Override + public BucketedSort newBucketedSort( + BigArrays bigArrays, + Object missingValue, + MultiValueMode sortMode, + XFieldComparatorSource.Nested nested, + SortOrder sortOrder, + DocValueFormat format, + int bucketSize, + BucketedSort.ExtraData extra + ) { + throw new IllegalArgumentException("can't sort on geo_shape field without using specific sorting feature, like geo_distance"); + } + + /** + * A concrete implementation of {@link AbstractGeoShapeIndexFieldData} which provides how to load the field data + * aka Doc Values from Lucene. + */ + public static class GeoShapeIndexFieldData extends AbstractGeoShapeIndexFieldData { + + public GeoShapeIndexFieldData(String fieldName, ValuesSourceType valuesSourceType) { + super(fieldName, valuesSourceType); + } + + /** + * Loads the atomic field data for the reader, possibly cached. + * + * @param context {@link LeafReaderContext} + */ + @Override + public LeafGeoShapeFieldData load(LeafReaderContext context) { + // do a compatibility check for the fieldName by getting the + // filed info from the context. + return new GeoShapeDVLeafFieldData(context.reader(), fieldName); + } + + /** + * Loads directly the atomic field data for the reader, ignoring any caching involved. + * + * @param context {@link LeafReaderContext} + */ + @Override + public LeafGeoShapeFieldData loadDirect(LeafReaderContext context) throws Exception { + return load(context); + } + } + + /** + * Builder class for creating the GeoShapeIndexFieldData. + * This is required the way the indexfieldData is created via the builder class only. + * @opensearch.internal + */ + public static class Builder implements IndexFieldData.Builder { + private final String name; + private final ValuesSourceType valuesSourceType; + + public Builder(String name, ValuesSourceType valuesSourceType) { + this.name = name; + this.valuesSourceType = valuesSourceType; + } + + @Override + public IndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { + // ignore breaker + return new AbstractGeoShapeIndexFieldData.GeoShapeIndexFieldData(name, valuesSourceType); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractLeafGeoShapeFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractLeafGeoShapeFieldData.java new file mode 100644 index 0000000000000..1c190b4282d67 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractLeafGeoShapeFieldData.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.fielddata.plain; + +import org.apache.lucene.document.LatLonShapeDocValuesField; +import org.opensearch.index.fielddata.FieldData; +import org.opensearch.index.fielddata.LeafGeoShapeFieldData; +import org.opensearch.index.fielddata.ScriptDocValues; +import org.opensearch.index.fielddata.SortedBinaryDocValues; +import org.opensearch.search.aggregations.AggregationExecutionException; + +/** + * Base class for retrieving GeoShape doc values which are added as {@link LatLonShapeDocValuesField} in Lucene + */ +public abstract class AbstractLeafGeoShapeFieldData implements LeafGeoShapeFieldData { + + /** + * Return a String representation of the values. + */ + @Override + public final SortedBinaryDocValues getBytesValues() { + return FieldData.toString(getGeoShapeValue()); + } + + /** + * Returns field values for use in scripting. We don't support Script values in the GeoShape for now. + * Code should not come to this place, as we have added not to support this at: + * CoreValuesSourceTypeGEO_SHAPE + */ + @Override + public final ScriptDocValues getScriptValues() { + // TODO: https://github.com/opensearch-project/geospatial/issues/128 + throw new AggregationExecutionException("Script doc value for the GeoShape field is not supported"); + } +} diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/GeoShapeDVLeafFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/GeoShapeDVLeafFieldData.java new file mode 100644 index 0000000000000..8089abfdd7c00 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/fielddata/plain/GeoShapeDVLeafFieldData.java @@ -0,0 +1,79 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.fielddata.plain; + +import org.apache.lucene.document.LatLonShapeDocValuesField; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.Accountables; +import org.opensearch.common.geo.GeoShapeDocValue; +import org.opensearch.index.fielddata.GeoShapeValue; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +/** + * This is the class that converts the DocValue of GeoShape field which is stored in Binary form using + * {@link LatLonShapeDocValuesField} to {@link GeoShapeDocValue}. + * + * @opensearch.internal + */ +public class GeoShapeDVLeafFieldData extends AbstractLeafGeoShapeFieldData { + + private final LeafReader reader; + private final String fieldName; + + GeoShapeDVLeafFieldData(final LeafReader reader, String fieldName) { + super(); + this.reader = reader; + this.fieldName = fieldName; + } + + /** + * Return the memory usage of this object in bytes. Negative values are illegal. + */ + @Override + public long ramBytesUsed() { + return 0; // not exposed by lucene + } + + @Override + public void close() { + // noop + } + + /** + * Returns nested resources of this class. The result should be a point-in-time snapshot (to avoid + * race conditions). + * + * @see Accountables + */ + @Override + public Collection getChildResources() { + return Collections.emptyList(); + } + + /** + * Reads the binary data from the {@link LeafReader} for a geo shape field and returns + * {@link GeoShapeValue.StandardGeoShapeValue} instance which can be used to get the doc values from Lucene. + * + * @return {@link GeoShapeValue.StandardGeoShapeValue} + */ + @Override + public GeoShapeValue getGeoShapeValue() { + try { + // Using BinaryDocValues as LatLonShapeDocValuesField stores data in binary form. + return new GeoShapeValue.StandardGeoShapeValue(DocValues.getBinary(reader, fieldName), fieldName); + } catch (IOException e) { + throw new IllegalStateException("Cannot load GeoShapeDocValues from lucene", e); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java index a4d8222896bc3..ded21780d3e2d 100644 --- a/server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java @@ -260,10 +260,15 @@ public T parse(String name, Map node, Map params return builder; } + /** + * Parse the node with the field name as name; using various parse methods for different attributes. + */ @Override public T parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - Map params = new HashMap<>(); - return parse(name, node, params, parserContext); + final T builder = parse(name, node, new HashMap<>(), parserContext); + // parse the common attributes(like doc_values, boosts etc.) and set them in the builder. + TypeParsers.parseField(builder, name, node, parserContext); + return builder; } } diff --git a/server/src/main/java/org/opensearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/GeoShapeFieldMapper.java index f0cc6839d6e7c..4a4b2684b5f4c 100644 --- a/server/src/main/java/org/opensearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/GeoShapeFieldMapper.java @@ -31,20 +31,28 @@ package org.opensearch.index.mapper; +import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.LatLonShape; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; import org.opensearch.common.Explicit; import org.opensearch.common.geo.GeometryParser; import org.opensearch.common.geo.ShapeRelation; import org.opensearch.common.geo.builders.ShapeBuilder; import org.opensearch.geometry.Geometry; +import org.opensearch.index.fielddata.IndexFieldData; +import org.opensearch.index.fielddata.plain.AbstractGeoShapeIndexFieldData; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.VectorGeoShapeQueryProcessor; +import org.opensearch.search.aggregations.support.CoreValuesSourceType; +import org.opensearch.search.lookup.SearchLookup; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * FieldMapper for indexing {@link LatLonShape}s. @@ -87,7 +95,8 @@ public static class Builder extends AbstractShapeGeometryFieldMapper.Builder searchLookup) { + failIfNoDocValues(); + return new AbstractGeoShapeIndexFieldData.Builder(name(), CoreValuesSourceType.GEO_SHAPE); + } } /** @@ -180,9 +199,15 @@ protected void addStoredFields(ParseContext context, Geometry geometry) { } @Override - @SuppressWarnings("rawtypes") - protected void addDocValuesFields(String name, Geometry geometry, List fields, ParseContext context) { - // we will throw a mapping exception before we get here + protected void addDocValuesFields( + final String name, + final Geometry geometry, + final List indexableFields, + final ParseContext context + ) { + Field[] fieldsArray = new Field[indexableFields.size()]; + fieldsArray = indexableFields.toArray(fieldsArray); + context.doc().add(LatLonShape.createDocValueField(name, fieldsArray)); } @Override @@ -219,9 +244,4 @@ public GeoShapeFieldType fieldType() { protected String contentType() { return CONTENT_TYPE; } - - @Override - protected boolean docValuesByDefault() { - return false; - } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/CoreValuesSourceType.java b/server/src/main/java/org/opensearch/search/aggregations/support/CoreValuesSourceType.java index 209b996cc5e5e..224f9281705e1 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/CoreValuesSourceType.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/CoreValuesSourceType.java @@ -35,13 +35,17 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.PointValues; import org.apache.lucene.util.BytesRef; +import org.opensearch.OpenSearchParseException; import org.opensearch.common.Rounding; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.time.DateFormatter; +import org.opensearch.geometry.Geometry; +import org.opensearch.geometry.utils.WellKnownText; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexGeoPointFieldData; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.fielddata.IndexOrdinalsFieldData; +import org.opensearch.index.fielddata.plain.AbstractGeoShapeIndexFieldData; import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.mapper.DateFieldMapper.DateFieldType; import org.opensearch.index.mapper.MappedFieldType; @@ -189,6 +193,101 @@ public DocValueFormat getFormatter(String format, ZoneId tz) { return DocValueFormat.GEOHASH; } }, + GEO_SHAPE() { + /** + * Called when an aggregation is operating over a known empty set (usually because the field isn't specified), this method allows for + * returning a no-op implementation. All ValuesSource should implement this method. + * + * @return - Empty specialization of the base ValuesSource + */ + @Override + public ValuesSource getEmpty() { + return ValuesSource.GeoShape.EMPTY; + } + + /** + * Returns the type-specific sub class for a script data source. ValuesSource that do not support scripts should throw + * AggregationExecutionException. Note that this method is called when a script is + * operating without an underlying field. Scripts operating over fields are handled by the script argument to getField below. + * + * @param script - The script being wrapped + * @param scriptValueType - The expected output type of the script + * @return - Script specialization of the base ValuesSource + */ + @Override + public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType scriptValueType) { + throw new AggregationExecutionException( + String.format(Locale.ROOT, "value source of type [%s] is not supported by scripts", this.value()) + ); + } + + /** + * Return a ValuesSource wrapping a field for the given type. All ValuesSource must + * implement this method. + * + * @param fieldContext - The field being wrapped + * @param script - Optional script that might be applied over the field + * @return - Field specialization of the base ValuesSource + */ + @Override + public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { + if (!(fieldContext.indexFieldData() instanceof AbstractGeoShapeIndexFieldData)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Expected geo_shape type on field [%s], but got [%s]", + fieldContext.field(), + fieldContext.fieldType().typeName() + ) + ); + } + return new ValuesSource.GeoShape.FieldData((AbstractGeoShapeIndexFieldData) fieldContext.indexFieldData()); + } + + /** + * Apply the given missing value to an already-constructed ValuesSource. The parameter rawMissing is + * the value which has been supplied by the user. We support String representation only for the + * rawMissing for now. + * + * @param valuesSource - The original ValuesSource + * @param rawMissing - The missing value we got from the parser, typically a string or number + * @param docValueFormat - The format to use for further parsing the user supplied value, e.g. a date format + * @param now - Used in conjunction with the formatter, should return the current time in milliseconds + * @return - Wrapper over the provided ValuesSource to apply the given missing value + */ + @Override + public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { + // TODO: also support the structured formats of geo shapes + try { + final Geometry geometry = WellKnownText.INSTANCE.fromWKT((String) rawMissing); + return MissingValues.replaceMissing((ValuesSource.GeoShape) valuesSource, geometry); + } catch (Exception e) { + throw new OpenSearchParseException( + String.format(Locale.ROOT, "Unable to parse the missing value [%s] provided in the input.", rawMissing), + e + ); + } + } + + /** + * This method provides a hook for specifying a type-specific formatter. When ValuesSourceConfig can resolve a + * MappedFieldType, it prefers to get the formatter from there. Only when a field can't be + * resolved (which is to say script cases and unmapped field cases), it will fall back to calling this method on whatever + * ValuesSourceType it was able to resolve to. + * + * For geoshape field we may never hit this function till we have aggregations which are only geo_shape + * specific and not present on geo_points, as we use default CoreValueSource types for Geo based aggregations + * as GEOPOINT + * + * @param format - User supplied format string (Optional) + * @param tz - User supplied time zone (Optional) + * @return - A formatter object, configured with the passed in settings if appropriate. + */ + @Override + public DocValueFormat getFormatter(String format, ZoneId tz) { + return DocValueFormat.RAW; + } + }, RANGE() { @Override public ValuesSource getEmpty() { @@ -360,6 +459,8 @@ public String typeName() { return value(); } + CoreValuesSourceType() {} + /** List containing all members of the enumeration. */ - public static List ALL_CORE = Arrays.asList(CoreValuesSourceType.values()); + public static final List ALL_CORE = Arrays.asList(CoreValuesSourceType.values()); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/MissingValues.java b/server/src/main/java/org/opensearch/search/aggregations/support/MissingValues.java index 449c961cb393a..4b1c7196be644 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/MissingValues.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/MissingValues.java @@ -37,8 +37,10 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.opensearch.common.geo.GeoPoint; +import org.opensearch.geometry.Geometry; import org.opensearch.index.fielddata.AbstractSortedNumericDocValues; import org.opensearch.index.fielddata.AbstractSortedSetDocValues; +import org.opensearch.index.fielddata.GeoShapeValue; import org.opensearch.index.fielddata.MultiGeoPointValues; import org.opensearch.index.fielddata.SortedBinaryDocValues; import org.opensearch.index.fielddata.SortedNumericDoubleValues; @@ -484,4 +486,26 @@ public String toString() { } }; } + + /** + * Replace the missing value provided in the param while iterating over a ValuesSource which doesn't have the + * value for the field. + * + * @param missing Value to be returned if doc doesn't contain the data for the field. + * @return {@link ValuesSource.GeoShape} + */ + public static ValuesSource.GeoShape replaceMissing(final ValuesSource.GeoShape valuesSource, final Geometry missing) { + return new ValuesSource.GeoShape() { + @Override + public GeoShapeValue getGeoShapeValues(LeafReaderContext context) { + final GeoShapeValue currentValueSourceValue = valuesSource.getGeoShapeValues(context); + return new GeoShapeValue.MissingGeoShapeValue(currentValueSourceValue, missing); + } + + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { + return replaceMissing(valuesSource.bytesValues(context), new BytesRef(missing.toString())); + } + }; + } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSource.java b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSource.java index ba9e1fcb3ccc1..4c2abedd0a006 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSource.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSource.java @@ -47,6 +47,7 @@ import org.opensearch.common.util.CollectionUtils; import org.opensearch.index.fielddata.AbstractSortingNumericDocValues; import org.opensearch.index.fielddata.DocValueBits; +import org.opensearch.index.fielddata.GeoShapeValue; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexGeoPointFieldData; import org.opensearch.index.fielddata.IndexNumericFieldData; @@ -57,6 +58,7 @@ import org.opensearch.index.fielddata.SortedNumericDoubleValues; import org.opensearch.index.fielddata.SortingBinaryDocValues; import org.opensearch.index.fielddata.SortingNumericDoubleValues; +import org.opensearch.index.fielddata.plain.AbstractGeoShapeIndexFieldData; import org.opensearch.index.mapper.RangeType; import org.opensearch.script.AggregationScript; import org.opensearch.search.aggregations.AggregationExecutionException; @@ -694,4 +696,89 @@ public org.opensearch.index.fielddata.MultiGeoPointValues geoPointValues(LeafRea } } } + + /** + * The primitive data type for doing an aggregation on the GeoShape + */ + public abstract static class GeoShape extends ValuesSource { + public static final GeoShape EMPTY = new GeoShape() { + /** + * This provides the {@link GeoShapeValue} after reading from LeafReaderContext + * + * @param context {@link LeafReaderContext} + * @return {@link GeoShapeValue} + */ + @Override + public GeoShapeValue getGeoShapeValues(LeafReaderContext context) { + return org.opensearch.index.fielddata.FieldData.emptyGeoShape(); + } + + /** + * Get the current {@link BytesValues}. + */ + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { + return org.opensearch.index.fielddata.FieldData.emptySortedBinary(); + } + }; + + /** + * This is getting used in the {@link org.opensearch.search.aggregations.bucket.missing.MissingAggregator} + * @param context {@link LeafReaderContext} + * @return DocValueBits + */ + @Override + public DocValueBits docsWithValue(LeafReaderContext context) { + final GeoShapeValue geoShapeValue = getGeoShapeValues(context); + return org.opensearch.index.fielddata.FieldData.docsWithValue(geoShapeValue); + } + + @Override + public Function roundingPreparer(IndexReader reader) { + throw new AggregationExecutionException("can't round a [GEO_SHAPE]"); + } + + /** + * This provides the {@link GeoShapeValue} after reading from LeafReaderContext + * @param context {@link LeafReaderContext} + * @return {@link GeoShapeValue} + */ + public abstract GeoShapeValue getGeoShapeValues(LeafReaderContext context); + + /** + * Field data for geo shape values source + * + * @opensearch.internal + */ + public static class FieldData extends GeoShape { + + protected final AbstractGeoShapeIndexFieldData indexFieldData; + + public FieldData(AbstractGeoShapeIndexFieldData indexFieldData) { + this.indexFieldData = indexFieldData; + } + + /** + * Get the current {@link BytesValues}. + * + * @param context {@link LeafReaderContext} + * @return SortedBinaryDocValues + */ + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { + return indexFieldData.load(context).getBytesValues(); + } + + /** + * This provides the {@link GeoShapeValue} after reading from LeafReaderContext + * + * @param context {@link LeafReaderContext} + * @return {@link GeoShapeValue} + */ + @Override + public GeoShapeValue getGeoShapeValues(LeafReaderContext context) { + return indexFieldData.load(context).getGeoShapeValue(); + } + } + } } diff --git a/server/src/test/java/org/opensearch/common/util/CollectionUtilsTests.java b/server/src/test/java/org/opensearch/common/util/CollectionUtilsTests.java index 381bd1784909d..40b2706d314ce 100644 --- a/server/src/test/java/org/opensearch/common/util/CollectionUtilsTests.java +++ b/server/src/test/java/org/opensearch/common/util/CollectionUtilsTests.java @@ -201,4 +201,11 @@ public void testEnsureNoSelfReferences() { } } + + public void testIsEmpty() { + assertTrue(CollectionUtils.isEmpty(new ArrayList<>())); + final List list = null; + assertTrue(CollectionUtils.isEmpty(list)); + assertFalse(CollectionUtils.isEmpty(Collections.singletonList(5))); + } } diff --git a/server/src/test/java/org/opensearch/index/fielddata/GeoShapeValueTests.java b/server/src/test/java/org/opensearch/index/fielddata/GeoShapeValueTests.java new file mode 100644 index 0000000000000..b2d3729e53553 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/fielddata/GeoShapeValueTests.java @@ -0,0 +1,61 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.fielddata; + +import org.apache.lucene.tests.util.TestUtil; +import org.opensearch.common.geo.GeoShapeDocValue; +import org.opensearch.geometry.Point; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +public class GeoShapeValueTests extends OpenSearchTestCase { + + public void testMissingGeoShapeValue() throws IOException { + final int numDocs = TestUtil.nextInt(random(), 1, 100); + final GeoShapeDocValue[][] values = new GeoShapeDocValue[numDocs][]; + + for (int i = 0; i < numDocs; ++i) { + values[i] = new GeoShapeDocValue[1]; + int number = TestUtil.nextInt(random(), 1, 2); + if (number == 1) { + values[i][0] = GeoShapeDocValue.createGeometryDocValue(new Point(randomDouble() * 90, randomDouble() * 180)); + } else { + values[i][0] = null; + } + + } + final GeoShapeValue asGeoValues = new GeoShapeValue() { + int doc; + + @Override + public boolean advanceExact(int docId) { + doc = docId; + return values[doc][0] != null; + } + + @Override + public GeoShapeDocValue nextValue() { + return values[doc][0]; + } + + }; + final Point missing = new Point(randomDouble() * 90, randomDouble() * 180); + final GeoShapeValue withMissingReplaced = new GeoShapeValue.MissingGeoShapeValue(asGeoValues, missing); + final GeoShapeDocValue missingValue = GeoShapeDocValue.createGeometryDocValue(missing); + for (int i = 0; i < numDocs; i++) { + assertTrue(withMissingReplaced.advanceExact(i)); + if (values[i][0] != null) { + assertEquals(values[i][0], withMissingReplaced.nextValue()); + } else { + assertEquals(missingValue, withMissingReplaced.nextValue()); + } + } + } +} diff --git a/server/src/test/java/org/opensearch/index/mapper/GeoShapeFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/GeoShapeFieldMapperTests.java index 187f17cbfd6ff..dd4f3e2b1ae0a 100644 --- a/server/src/test/java/org/opensearch/index/mapper/GeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/GeoShapeFieldMapperTests.java @@ -55,7 +55,7 @@ public class GeoShapeFieldMapperTests extends FieldMapperTestCase2 unsupportedProperties() { - return org.opensearch.common.collect.Set.of("analyzer", "similarity", "doc_values", "store"); + return org.opensearch.common.collect.Set.of("analyzer", "similarity", "store"); } @Override @@ -117,7 +117,7 @@ public void testDefaultConfiguration() throws IOException { assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(GeoShapeFieldMapper.Defaults.ORIENTATION.value())); - assertThat(geoShapeFieldMapper.fieldType().hasDocValues(), equalTo(false)); + assertThat(geoShapeFieldMapper.fieldType().hasDocValues(), equalTo(true)); } /** @@ -246,7 +246,7 @@ public void testGeoShapeArrayParsing() throws Exception { b.endArray(); })); assertThat(document.docs(), hasSize(1)); - assertThat(document.docs().get(0).getFields("field").length, equalTo(2)); + assertThat(document.docs().get(0).getFields("field").length, equalTo(4)); } @Override diff --git a/server/src/test/java/org/opensearch/search/aggregations/support/CoreValuesSourceTypeTests.java b/server/src/test/java/org/opensearch/search/aggregations/support/CoreValuesSourceTypeTests.java index b6c8f6d8b0d67..2c32d37534b4f 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/support/CoreValuesSourceTypeTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/support/CoreValuesSourceTypeTests.java @@ -43,6 +43,7 @@ public void testFromString() { assertThat(CoreValuesSourceType.fromString("bytes"), equalTo(CoreValuesSourceType.BYTES)); assertThat(CoreValuesSourceType.fromString("geopoint"), equalTo(CoreValuesSourceType.GEOPOINT)); assertThat(CoreValuesSourceType.fromString("range"), equalTo(CoreValuesSourceType.RANGE)); + assertThat(CoreValuesSourceType.fromString("geo_shape"), equalTo(CoreValuesSourceType.GEO_SHAPE)); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> CoreValuesSourceType.fromString("does_not_exist")); assertThat( e.getMessage(),