From 9b014552bf42010a4d1624506345d61ee9a23882 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 9 Sep 2024 10:54:03 -0400 Subject: [PATCH 01/44] Add 2.17 release notes (#15762) (#15768) (#15776) Signed-off-by: Andriy Redko --- CHANGELOG.md | 94 +--------------- .../opensearch.release-notes-2.17.0.md | 104 ++++++++++++++++++ 2 files changed, 105 insertions(+), 93 deletions(-) create mode 100644 release-notes/opensearch.release-notes-2.17.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c95f84d78842..9d1faea226390 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,54 +5,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added -- [Offline Nodes] Adds offline-tasks library containing various interfaces to be used for Offline Background Tasks. ([#13574](https://github.com/opensearch-project/OpenSearch/pull/13574)) -- Fix for hasInitiatedFetching to fix allocation explain and manual reroute APIs (([#14972](https://github.com/opensearch-project/OpenSearch/pull/14972)) -- [Workload Management] Add queryGroupId to Task ([14708](https://github.com/opensearch-project/OpenSearch/pull/14708)) -- Add setting to ignore throttling nodes for allocation of unassigned primaries in remote restore ([#14991](https://github.com/opensearch-project/OpenSearch/pull/14991)) -- [Workload Management] Add Delete QueryGroup API Logic ([#14735](https://github.com/opensearch-project/OpenSearch/pull/14735)) -- [Streaming Indexing] Enhance RestClient with a new streaming API support ([#14437](https://github.com/opensearch-project/OpenSearch/pull/14437)) -- Add basic aggregation support for derived fields ([#14618](https://github.com/opensearch-project/OpenSearch/pull/14618)) -- [Workload Management] Add Create QueryGroup API Logic ([#14680](https://github.com/opensearch-project/OpenSearch/pull/14680))- [Workload Management] Add Create QueryGroup API Logic ([#14680](https://github.com/opensearch-project/OpenSearch/pull/14680)) -- Add ThreadContextPermission for markAsSystemContext and allow core to perform the method ([#15016](https://github.com/opensearch-project/OpenSearch/pull/15016)) -- Add ThreadContextPermission for stashAndMergeHeaders and stashWithOrigin ([#15039](https://github.com/opensearch-project/OpenSearch/pull/15039)) -- [Concurrent Segment Search] Support composite aggregations with scripting ([#15072](https://github.com/opensearch-project/OpenSearch/pull/15072)) -- Add `rangeQuery` and `regexpQuery` for `constant_keyword` field type ([#14711](https://github.com/opensearch-project/OpenSearch/pull/14711)) -- Add took time to request nodes stats ([#15054](https://github.com/opensearch-project/OpenSearch/pull/15054)) -- [Workload Management] Add Get QueryGroup API Logic ([14709](https://github.com/opensearch-project/OpenSearch/pull/14709)) -- [Workload Management] Add Settings for Workload Management feature ([#15028](https://github.com/opensearch-project/OpenSearch/pull/15028)) -- [Workload Management] Add Update QueryGroup API Logic ([#14775](https://github.com/opensearch-project/OpenSearch/pull/14775)) -- [Workload Management] QueryGroup resource tracking framework changes ([#13897](https://github.com/opensearch-project/OpenSearch/pull/13897)) -- Support filtering on a large list encoded by bitmap ([#14774](https://github.com/opensearch-project/OpenSearch/pull/14774)) -- Add slice execution listeners to SearchOperationListener interface ([#15153](https://github.com/opensearch-project/OpenSearch/pull/15153)) -- Make balanced shards allocator timebound ([#15239](https://github.com/opensearch-project/OpenSearch/pull/15239)) -- Add allowlist setting for ingest-geoip and ingest-useragent ([#15325](https://github.com/opensearch-project/OpenSearch/pull/15325)) -- Adding access to noSubMatches and noOverlappingMatches in Hyphenation ([#13895](https://github.com/opensearch-project/OpenSearch/pull/13895)) -- Add support for index level max slice count setting for concurrent segment search ([#15336](https://github.com/opensearch-project/OpenSearch/pull/15336)) -- Support cancellation for cat shards and node stats API.([#13966](https://github.com/opensearch-project/OpenSearch/pull/13966)) -- [Streaming Indexing] Introduce bulk HTTP API streaming flavor ([#15381](https://github.com/opensearch-project/OpenSearch/pull/15381)) -- Add support for centralize snapshot creation with pinned timestamp ([#15124](https://github.com/opensearch-project/OpenSearch/pull/15124)) -- Add concurrent search support for Derived Fields ([#15326](https://github.com/opensearch-project/OpenSearch/pull/15326)) -- [Workload Management] Add query group stats constructs ([#15343](https://github.com/opensearch-project/OpenSearch/pull/15343))) -- Add limit on number of processors for Ingest pipeline([#15460](https://github.com/opensearch-project/OpenSearch/pull/15465)). -- Add runAs to Subject interface and introduce IdentityAwarePlugin extension point ([#14630](https://github.com/opensearch-project/OpenSearch/pull/14630)) -- Optimize NodeIndicesStats output behind flag ([#14454](https://github.com/opensearch-project/OpenSearch/pull/14454)) -- [Workload Management] Add rejection logic for co-ordinator and shard level requests ([#15428](https://github.com/opensearch-project/OpenSearch/pull/15428))) -- Adding translog durability validation in index templates ([#15494](https://github.com/opensearch-project/OpenSearch/pull/15494)) -- Add index creation using the context field ([#15290](https://github.com/opensearch-project/OpenSearch/pull/15290)) -- [Reader Writer Separation] Add experimental search replica shard type to achieve reader writer separation ([#15237](https://github.com/opensearch-project/OpenSearch/pull/15237)) -- [Range Queries] Add new approximateable query framework to short-circuit range queries ([#13788](https://github.com/opensearch-project/OpenSearch/pull/13788)) -- [Workload Management] Add query group level failure tracking ([#15227](https://github.com/opensearch-project/OpenSearch/pull/15527)) -- Add support for pluggable deciders for concurrent search ([#15363](https://github.com/opensearch-project/OpenSearch/pull/15363)) -- Add support to upload snapshot shard blobs with hashed prefix ([#15426](https://github.com/opensearch-project/OpenSearch/pull/15426)) -- [Remote Publication] Add remote download stats ([#15291](https://github.com/opensearch-project/OpenSearch/pull/15291))) -- Add support for comma-separated list of index names to be used with Snapshot Status API ([#15409](https://github.com/opensearch-project/OpenSearch/pull/15409)) -- Add prefix support to hashed prefix & infix path types on remote store ([#15557](https://github.com/opensearch-project/OpenSearch/pull/15557)) -- Add path prefix support to hashed prefix snapshots ([#15664](https://github.com/opensearch-project/OpenSearch/pull/15664)) -- Optimise snapshot deletion to speed up snapshot deletion and creation ([#15568](https://github.com/opensearch-project/OpenSearch/pull/15568)) -- [Remote Publication] Added checksum validation for cluster state behind a cluster setting ([#15218](https://github.com/opensearch-project/OpenSearch/pull/15218)) -- Add canRemain method to TargetPoolAllocationDecider to move shards from local to remote pool for hot to warm tiering ([#15010](https://github.com/opensearch-project/OpenSearch/pull/15010)) -- ClusterManagerTaskThrottler Improvements ([#15508](https://github.com/opensearch-project/OpenSearch/pull/15508)) -- Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131)) - Adding WithFieldName interface for QueryBuilders with fieldName ([#15705](https://github.com/opensearch-project/OpenSearch/pull/15705)) - Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471)) - Static RemotePublication setting added, removed experimental feature flag ([#15478](https://github.com/opensearch-project/OpenSearch/pull/15478)) @@ -62,62 +14,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Memory optimisations in _cluster/health API ([#15492](https://github.com/opensearch-project/OpenSearch/pull/15492)) ### Dependencies -- Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) -- Bump `org.apache.commons:commons-lang3` from 3.14.0 to 3.17.0 ([#14861](https://github.com/opensearch-project/OpenSearch/pull/14861), [#15205](https://github.com/opensearch-project/OpenSearch/pull/15205), [#15580](https://github.com/opensearch-project/OpenSearch/pull/15580)) -- OpenJDK Update (July 2024 Patch releases) ([#14998](https://github.com/opensearch-project/OpenSearch/pull/14998)) -- Bump `com.microsoft.azure:msal4j` from 1.16.1 to 1.17.0 ([#14995](https://github.com/opensearch-project/OpenSearch/pull/14995), [#15420](https://github.com/opensearch-project/OpenSearch/pull/15420)) -- Bump `actions/github-script` from 6 to 7 ([#14997](https://github.com/opensearch-project/OpenSearch/pull/14997)) -- Bump `org.tukaani:xz` from 1.9 to 1.10 ([#15110](https://github.com/opensearch-project/OpenSearch/pull/15110)) -- Bump `actions/setup-java` from 1 to 4 ([#15104](https://github.com/opensearch-project/OpenSearch/pull/15104)) -- Bump `org.apache.avro:avro` from 1.11.3 to 1.12.0 in /plugins/repository-hdfs ([#15119](https://github.com/opensearch-project/OpenSearch/pull/15119)) -- Bump `org.bouncycastle:bcpg-fips` from 1.0.7.1 to 2.0.9 ([#15103](https://github.com/opensearch-project/OpenSearch/pull/15103), [#15299](https://github.com/opensearch-project/OpenSearch/pull/15299)) -- Bump `com.azure:azure-core` from 1.49.1 to 1.51.0 ([#15111](https://github.com/opensearch-project/OpenSearch/pull/15111)) -- Bump `org.xerial.snappy:snappy-java` from 1.1.10.5 to 1.1.10.6 ([#15207](https://github.com/opensearch-project/OpenSearch/pull/15207)) -- Bump `com.azure:azure-xml` from 1.0.0 to 1.1.0 ([#15206](https://github.com/opensearch-project/OpenSearch/pull/15206)) -- Bump `reactor` from 3.5.19 to 3.5.20 ([#15262](https://github.com/opensearch-project/OpenSearch/pull/15262)) -- Bump `reactor-netty` from 1.1.21 to 1.1.22 ([#15262](https://github.com/opensearch-project/OpenSearch/pull/15262)) -- Bump `org.apache.kerby:kerb-admin` from 2.0.3 to 2.1.0 ([#15301](https://github.com/opensearch-project/OpenSearch/pull/15301)) -- Bump `com.azure:azure-core-http-netty` from 1.15.1 to 1.15.3 ([#15300](https://github.com/opensearch-project/OpenSearch/pull/15300)) -- Bump `com.gradle.develocity` from 3.17.6 to 3.18 ([#15297](https://github.com/opensearch-project/OpenSearch/pull/15297)) -- Bump `commons-cli:commons-cli` from 1.8.0 to 1.9.0 ([#15298](https://github.com/opensearch-project/OpenSearch/pull/15298)) -- Bump `opentelemetry` from 1.40.0 to 1.41.0 ([#15361](https://github.com/opensearch-project/OpenSearch/pull/15361)) -- Bump `opentelemetry-semconv` from 1.26.0-alpha to 1.27.0-alpha ([#15361](https://github.com/opensearch-project/OpenSearch/pull/15361)) -- Bump `tj-actions/changed-files` from 44 to 45 ([#15422](https://github.com/opensearch-project/OpenSearch/pull/15422)) -- Bump `dnsjava:dnsjava` from 3.6.0 to 3.6.1 ([#15418](https://github.com/opensearch-project/OpenSearch/pull/15418)) -- Bump `com.netflix.nebula.ospackage-base` from 11.9.1 to 11.10.0 ([#15419](https://github.com/opensearch-project/OpenSearch/pull/15419)) -- Bump `org.roaringbitmap:RoaringBitmap` from 1.1.0 to 1.2.1 ([#15423](https://github.com/opensearch-project/OpenSearch/pull/15423)) -- Bump `icu4j` from 70.1 to 75.1 ([#15469](https://github.com/opensearch-project/OpenSearch/pull/15469)) - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) - Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684)) ### Changed -- Add lower limit for primary and replica batch allocators timeout ([#14979](https://github.com/opensearch-project/OpenSearch/pull/14979)) -- Optimize regexp-based include/exclude on aggregations when pattern matches prefixes ([#14371](https://github.com/opensearch-project/OpenSearch/pull/14371)) -- Replace and block usages of org.apache.logging.log4j.util.Strings ([#15238](https://github.com/opensearch-project/OpenSearch/pull/15238)) -- Remote publication using minimum node version for backward compatibility ([#15216](https://github.com/opensearch-project/OpenSearch/pull/15216)) ### Deprecated ### Removed -- Remove some unused code in the search backpressure package ([#15518](https://github.com/opensearch-project/OpenSearch/pull/15518)) ### Fixed -- Fix constraint bug which allows more primary shards than average primary shards per index ([#14908](https://github.com/opensearch-project/OpenSearch/pull/14908)) -- Fix NPE when bulk ingest with empty pipeline ([#15033](https://github.com/opensearch-project/OpenSearch/pull/15033)) -- Fix missing value of FieldSort for unsigned_long ([#14963](https://github.com/opensearch-project/OpenSearch/pull/14963)) -- Fix delete index template failed when the index template matches a data stream but is unused ([#15080](https://github.com/opensearch-project/OpenSearch/pull/15080)) -- Fix array_index_out_of_bounds_exception when indexing documents with field name containing only dot ([#15126](https://github.com/opensearch-project/OpenSearch/pull/15126)) -- Fixed array field name omission in flat_object function for nested JSON ([#13620](https://github.com/opensearch-project/OpenSearch/pull/13620)) -- Fix range aggregation optimization ignoring top level queries ([#15194](https://github.com/opensearch-project/OpenSearch/pull/15194)) -- Fix incorrect parameter names in MinHash token filter configuration handling ([#15233](https://github.com/opensearch-project/OpenSearch/pull/15233)) -- Fix indexing error when flat_object field is explicitly null ([#15375](https://github.com/opensearch-project/OpenSearch/pull/15375)) -- Fix split response processor not included in allowlist ([#15393](https://github.com/opensearch-project/OpenSearch/pull/15393)) -- Fix unchecked cast in dynamic action map getter ([#15394](https://github.com/opensearch-project/OpenSearch/pull/15394)) -- Fix null values indexed as "null" strings in flat_object field ([#14069](https://github.com/opensearch-project/OpenSearch/pull/14069)) -- Fix terms query on wildcard field returns nothing ([#15607](https://github.com/opensearch-project/OpenSearch/pull/15607)) -- Fix remote snapshot file_cache exceeding capacity ([#15077](https://github.com/opensearch-project/OpenSearch/pull/15077)) ### Security -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.15...2.x +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.17...2.x diff --git a/release-notes/opensearch.release-notes-2.17.0.md b/release-notes/opensearch.release-notes-2.17.0.md new file mode 100644 index 0000000000000..18b7c9ac7cd68 --- /dev/null +++ b/release-notes/opensearch.release-notes-2.17.0.md @@ -0,0 +1,104 @@ +## 2024-09-17 Version 2.17.0 Release Notes + +## [2.17.0] +### Added +- [Workload Management] Add Settings for Workload Management feature ([#15028](https://github.com/opensearch-project/OpenSearch/pull/15028)) +- Fix for hasInitiatedFetching to fix allocation explain and manual reroute APIs (([#14972](https://github.com/opensearch-project/OpenSearch/pull/14972)) +- [Workload Management] Add queryGroupId to Task ([14708](https://github.com/opensearch-project/OpenSearch/pull/14708)) +- Add setting to ignore throttling nodes for allocation of unassigned primaries in remote restore ([#14991](https://github.com/opensearch-project/OpenSearch/pull/14991)) +- [Workload Management] Add Delete QueryGroup API Logic ([#14735](https://github.com/opensearch-project/OpenSearch/pull/14735)) +- [Streaming Indexing] Enhance RestClient with a new streaming API support ([#14437](https://github.com/opensearch-project/OpenSearch/pull/14437)) +- Add basic aggregation support for derived fields ([#14618](https://github.com/opensearch-project/OpenSearch/pull/14618)) +- [Workload Management] Add Create QueryGroup API Logic ([#14680](https://github.com/opensearch-project/OpenSearch/pull/14680))- [Workload Management] Add Create QueryGroup API Logic ([#14680](https://github.com/opensearch-project/OpenSearch/pull/14680)) +- Add ThreadContextPermission for markAsSystemContext and allow core to perform the method ([#15016](https://github.com/opensearch-project/OpenSearch/pull/15016)) +- Add ThreadContextPermission for stashAndMergeHeaders and stashWithOrigin ([#15039](https://github.com/opensearch-project/OpenSearch/pull/15039)) +- [Concurrent Segment Search] Support composite aggregations with scripting ([#15072](https://github.com/opensearch-project/OpenSearch/pull/15072)) +- Add `rangeQuery` and `regexpQuery` for `constant_keyword` field type ([#14711](https://github.com/opensearch-project/OpenSearch/pull/14711)) +- Add took time to request nodes stats ([#15054](https://github.com/opensearch-project/OpenSearch/pull/15054)) +- [Workload Management] Add Get QueryGroup API Logic ([14709](https://github.com/opensearch-project/OpenSearch/pull/14709)) +- [Workload Management] Add Update QueryGroup API Logic ([#14775](https://github.com/opensearch-project/OpenSearch/pull/14775)) +- [Workload Management] QueryGroup resource tracking framework changes ([#13897](https://github.com/opensearch-project/OpenSearch/pull/13897)) +- Support filtering on a large list encoded by bitmap ([#14774](https://github.com/opensearch-project/OpenSearch/pull/14774)) +- Add slice execution listeners to SearchOperationListener interface ([#15153](https://github.com/opensearch-project/OpenSearch/pull/15153)) +- Make balanced shards allocator timebound ([#15239](https://github.com/opensearch-project/OpenSearch/pull/15239)) +- Add allowlist setting for ingest-geoip and ingest-useragent ([#15325](https://github.com/opensearch-project/OpenSearch/pull/15325)) +- Adding access to noSubMatches and noOverlappingMatches in Hyphenation ([#13895](https://github.com/opensearch-project/OpenSearch/pull/13895)) +- Star tree mapping changes ([#14605](https://github.com/opensearch-project/OpenSearch/pull/14605)) +- Add support for index level max slice count setting for concurrent segment search ([#15336](https://github.com/opensearch-project/OpenSearch/pull/15336)) +- Support cancellation for cat shards and node stats API.([#13966](https://github.com/opensearch-project/OpenSearch/pull/13966)) +- [Streaming Indexing] Introduce bulk HTTP API streaming flavor ([#15381](https://github.com/opensearch-project/OpenSearch/pull/15381)) +- Add support for centralize snapshot creation with pinned timestamp ([#15124](https://github.com/opensearch-project/OpenSearch/pull/15124)) +- Add concurrent search support for Derived Fields ([#15326](https://github.com/opensearch-project/OpenSearch/pull/15326)) +- [Workload Management] Add query group stats constructs ([#15343](https://github.com/opensearch-project/OpenSearch/pull/15343))) +- Add limit on number of processors for Ingest pipeline([#15460](https://github.com/opensearch-project/OpenSearch/pull/15465)). +- Add runAs to Subject interface and introduce IdentityAwarePlugin extension point ([#14630](https://github.com/opensearch-project/OpenSearch/pull/14630)) +- [Workload Management] Add rejection logic for co-ordinator and shard level requests ([#15428](https://github.com/opensearch-project/OpenSearch/pull/15428))) +- Adding translog durability validation in index templates ([#15494](https://github.com/opensearch-project/OpenSearch/pull/15494)) +- [Range Queries] Add new approximateable query framework to short-circuit range queries ([#13788](https://github.com/opensearch-project/OpenSearch/pull/13788)) +- [Workload Management] Add query group level failure tracking ([#15227](https://github.com/opensearch-project/OpenSearch/pull/15527)) +- [Reader Writer Separation] Add experimental search replica shard type to achieve reader writer separation ([#15237](https://github.com/opensearch-project/OpenSearch/pull/15237)) +- Add index creation using the context field ([#15290](https://github.com/opensearch-project/OpenSearch/pull/15290)) +- [Remote Publication] Add remote download stats ([#15291](https://github.com/opensearch-project/OpenSearch/pull/15291)) +- Add support to upload snapshot shard blobs with hashed prefix ([#15426](https://github.com/opensearch-project/OpenSearch/pull/15426)) +- Add prefix support to hashed prefix & infix path types on remote store ([#15557](https://github.com/opensearch-project/OpenSearch/pull/15557)) +- Add canRemain method to TargetPoolAllocationDecider to move shards from local to remote pool for hot to warm tiering ([#15010](https://github.com/opensearch-project/OpenSearch/pull/15010)) +- Add support for pluggable deciders for concurrent search ([#15363](https://github.com/opensearch-project/OpenSearch/pull/15363)) +- Optimise snapshot deletion to speed up snapshot deletion and creation ([#15568](https://github.com/opensearch-project/OpenSearch/pull/15568)) +- [Remote Publication] Added checksum validation for cluster state behind a cluster setting ([#15218](https://github.com/opensearch-project/OpenSearch/pull/15218)) +- Optimize NodeIndicesStats output behind flag ([#14454](https://github.com/opensearch-project/OpenSearch/pull/14454)) +- Add support for comma-separated list of index names to be used with Snapshot Status API ([#15409](https://github.com/opensearch-project/OpenSearch/pull/15409))[SnapshotV2] Snapshot Status API changes (#15409)) +- ClusterManagerTaskThrottler Improvements ([#15508](https://github.com/opensearch-project/OpenSearch/pull/15508)) +- Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471)) +- Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131)) + +### Dependencies +- Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) +- Bump `org.apache.commons:commons-lang3` from 3.14.0 to 3.15.0 ([#14861](https://github.com/opensearch-project/OpenSearch/pull/14861)) +- OpenJDK Update (July 2024 Patch releases) ([#14998](https://github.com/opensearch-project/OpenSearch/pull/14998)) +- Bump `com.microsoft.azure:msal4j` from 1.16.1 to 1.17.0 ([#14995](https://github.com/opensearch-project/OpenSearch/pull/14995), [#15420](https://github.com/opensearch-project/OpenSearch/pull/15420)) +- Bump `actions/github-script` from 6 to 7 ([#14997](https://github.com/opensearch-project/OpenSearch/pull/14997)) +- Bump `org.tukaani:xz` from 1.9 to 1.10 ([#15110](https://github.com/opensearch-project/OpenSearch/pull/15110)) +- Bump `org.apache.avro:avro` from 1.11.3 to 1.12.0 in /plugins/repository-hdfs ([#15119](https://github.com/opensearch-project/OpenSearch/pull/15119)) +- Bump `org.bouncycastle:bcpg-fips` from 1.0.7.1 to 2.0.9 ([#15103](https://github.com/opensearch-project/OpenSearch/pull/15103), [#15299](https://github.com/opensearch-project/OpenSearch/pull/15299)) +- Bump `com.azure:azure-core` from 1.49.1 to 1.51.0 ([#15111](https://github.com/opensearch-project/OpenSearch/pull/15111)) +- Bump `org.xerial.snappy:snappy-java` from 1.1.10.5 to 1.1.10.6 ([#15207](https://github.com/opensearch-project/OpenSearch/pull/15207)) +- Bump `com.azure:azure-xml` from 1.0.0 to 1.1.0 ([#15206](https://github.com/opensearch-project/OpenSearch/pull/15206)) +- Bump `reactor` from 3.5.19 to 3.5.20 ([#15262](https://github.com/opensearch-project/OpenSearch/pull/15262)) +- Bump `reactor-netty` from 1.1.21 to 1.1.22 ([#15262](https://github.com/opensearch-project/OpenSearch/pull/15262)) +- Bump `org.apache.kerby:kerb-admin` from 2.0.3 to 2.1.0 ([#15301](https://github.com/opensearch-project/OpenSearch/pull/15301)) +- Bump `com.azure:azure-core-http-netty` from 1.15.1 to 1.15.3 ([#15300](https://github.com/opensearch-project/OpenSearch/pull/15300)) +- Bump `com.gradle.develocity` from 3.17.6 to 3.18 ([#15297](https://github.com/opensearch-project/OpenSearch/pull/15297)) +- Bump `commons-cli:commons-cli` from 1.8.0 to 1.9.0 ([#15298](https://github.com/opensearch-project/OpenSearch/pull/15298)) +- Bump `opentelemetry` from 1.40.0 to 1.41.0 ([#15361](https://github.com/opensearch-project/OpenSearch/pull/15361)) +- Bump `opentelemetry-semconv` from 1.26.0-alpha to 1.27.0-alpha ([#15361](https://github.com/opensearch-project/OpenSearch/pull/15361)) +- Bump `tj-actions/changed-files` from 44 to 45 ([#15422](https://github.com/opensearch-project/OpenSearch/pull/15422)) +- Bump `dnsjava:dnsjava` from 3.6.0 to 3.6.1 ([#15418](https://github.com/opensearch-project/OpenSearch/pull/15418)) +- Bump `com.netflix.nebula.ospackage-base` from 11.9.1 to 11.10.0 ([#15419](https://github.com/opensearch-project/OpenSearch/pull/15419)) +- Bump `org.roaringbitmap:RoaringBitmap` from 1.1.0 to 1.2.1 ([#15423](https://github.com/opensearch-project/OpenSearch/pull/15423)) +- Bump `icu4j` from 70.1 to 75.1 ([#15469](https://github.com/opensearch-project/OpenSearch/pull/15469)) + +### Changed +- Add lower limit for primary and replica batch allocators timeout ([#14979](https://github.com/opensearch-project/OpenSearch/pull/14979)) +- Optimize regexp-based include/exclude on aggregations when pattern matches prefixes ([#14371](https://github.com/opensearch-project/OpenSearch/pull/14371)) +- Replace and block usages of org.apache.logging.log4j.util.Strings ([#15238](https://github.com/opensearch-project/OpenSearch/pull/15238)) +- Remote publication using minimum node version for backward compatibility ([#15216](https://github.com/opensearch-project/OpenSearch/pull/15216)) + +### Deprecated + +### Removed +- Remove some unused code in the search backpressure package ([#15518](https://github.com/opensearch-project/OpenSearch/pull/15518)) + +### Fixed +- Fix constraint bug which allows more primary shards than average primary shards per index ([#14908](https://github.com/opensearch-project/OpenSearch/pull/14908)) +- Fix NPE when bulk ingest with empty pipeline ([#15033](https://github.com/opensearch-project/OpenSearch/pull/15033)) +- Fix missing value of FieldSort for unsigned_long ([#14963](https://github.com/opensearch-project/OpenSearch/pull/14963)) +- Fix delete index template failed when the index template matches a data stream but is unused ([#15080](https://github.com/opensearch-project/OpenSearch/pull/15080)) +- Fix array_index_out_of_bounds_exception when indexing documents with field name containing only dot ([#15126](https://github.com/opensearch-project/OpenSearch/pull/15126)) +- Fixed array field name omission in flat_object function for nested JSON ([#13620](https://github.com/opensearch-project/OpenSearch/pull/13620)) +- Fix incorrect parameter names in MinHash token filter configuration handling ([#15233](https://github.com/opensearch-project/OpenSearch/pull/15233)) +- Fix range aggregation optimization ignoring top level queries ([#15287](https://github.com/opensearch-project/OpenSearch/pull/15287)) +- Fix indexing error when flat_object field is explicitly null ([#15375](https://github.com/opensearch-project/OpenSearch/pull/15375)) +- Fix split response processor not included in allowlist ([#15393](https://github.com/opensearch-project/OpenSearch/pull/15393)) +- Fix unchecked cast in dynamic action map getter ([#15394](https://github.com/opensearch-project/OpenSearch/pull/15394)) +- Fix null values indexed as "null" strings in flat_object field ([#14069](https://github.com/opensearch-project/OpenSearch/pull/14069)) +- Fix terms query on wildcard field returns nothing ([#15607](https://github.com/opensearch-project/OpenSearch/pull/15607)) From 503e6f0b76fb5feb82e0e1af35e14dea02c9fd3b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 10:58:54 -0400 Subject: [PATCH 02/44] Bump org.apache.logging.log4j:log4j-core from 2.23.1 to 2.24.0 in /buildSrc/src/testKit/thirdPartyAudit/sample_jars (#15858) * Bump org.apache.logging.log4j:log4j-core Bumps org.apache.logging.log4j:log4j-core from 2.23.1 to 2.24.0. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d1faea226390..c5f667a175750 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) - Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684)) +- Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858)) ### Changed diff --git a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle index 48dfb206375ca..59063b3789c70 100644 --- a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle +++ b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle @@ -17,7 +17,7 @@ repositories { } dependencies { - implementation "org.apache.logging.log4j:log4j-core:2.23.1" + implementation "org.apache.logging.log4j:log4j-core:2.24.0" } ["0.0.1", "0.0.2"].forEach { v -> From e688d39a5d4e17c241c3415322554363312ad475 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 11:54:55 -0400 Subject: [PATCH 03/44] Bump peter-evans/create-pull-request from 6 to 7 (#15863) * Bump peter-evans/create-pull-request from 6 to 7 Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 6 to 7. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/v6...v7) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- .github/workflows/version.yml | 6 +++--- CHANGELOG.md | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml index 2de54716256ff..959c5c394be7d 100644 --- a/.github/workflows/version.yml +++ b/.github/workflows/version.yml @@ -62,7 +62,7 @@ jobs: - name: Create PR for BASE id: base_pr - uses: peter-evans/create-pull-request@v6 + uses: peter-evans/create-pull-request@v7 with: base: ${{ env.BASE }} branch: 'create-pull-request/patch-${{ env.BASE }}' @@ -88,7 +88,7 @@ jobs: - name: Create PR for BASE_X id: base_x_pr - uses: peter-evans/create-pull-request@v6 + uses: peter-evans/create-pull-request@v7 with: base: ${{ env.BASE_X }} branch: 'create-pull-request/patch-${{ env.BASE_X }}' @@ -114,7 +114,7 @@ jobs: - name: Create PR for main id: main_pr - uses: peter-evans/create-pull-request@v6 + uses: peter-evans/create-pull-request@v7 with: base: main branch: 'create-pull-request/patch-main' diff --git a/CHANGELOG.md b/CHANGELOG.md index c5f667a175750..d70fc315cbf14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) - Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684)) - Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858)) +- Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863)) ### Changed From 7cb2bd09b18da1110c74f6947c2234932a0be675 Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Mon, 9 Sep 2024 11:39:15 -0700 Subject: [PATCH 04/44] Remove awaits fix since tests are now passing (#15727) Signed-off-by: Harsha Vamsi Kalluri --- .../ApproximatePointRangeQueryTests.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java b/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java index 1eaea6a9e9d47..9c022aade5dc6 100644 --- a/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java +++ b/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java @@ -13,9 +13,12 @@ import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHits; import org.apache.lucene.store.Directory; @@ -26,6 +29,7 @@ import java.io.IOException; +import static java.util.Arrays.asList; import static org.apache.lucene.document.LongPoint.pack; import static org.mockito.Mockito.mock; @@ -112,7 +116,6 @@ protected String toString(int dimension, byte[] value) { } } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15600") public void testApproximateRangeWithSizeUnderDefault() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { @@ -151,7 +154,6 @@ protected String toString(int dimension, byte[] value) { } } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15600") public void testApproximateRangeWithSizeOverDefault() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { @@ -196,7 +198,6 @@ protected String toString(int dimension, byte[] value) { } } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15600") public void testApproximateRangeShortCircuit() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { @@ -256,8 +257,7 @@ public void testApproximateRangeShortCircuitAscSort() throws IOException { for (int v = 0; v < dims; v++) { scratch[v] = i; } - doc.add(new LongPoint("point", scratch)); - iw.addDocument(doc); + iw.addDocument(asList(new LongPoint("point", scratch[0]), new NumericDocValuesField("point", scratch[0]))); } iw.flush(); iw.forceMerge(1); @@ -280,8 +280,9 @@ protected String toString(int dimension, byte[] value) { Query query = LongPoint.newRangeQuery("point", lower, upper); ; IndexSearcher searcher = new IndexSearcher(reader); - TopDocs topDocs = searcher.search(approximateQuery, 10); - TopDocs topDocs1 = searcher.search(query, 10); + Sort sort = new Sort(new SortField("point", SortField.Type.LONG)); + TopDocs topDocs = searcher.search(approximateQuery, 10, sort); + TopDocs topDocs1 = searcher.search(query, 10, sort); // since we short-circuit from the approx range at the end of size these will not be equal assertNotEquals(topDocs.totalHits, topDocs1.totalHits); From 62c6d08f62309aed374ceb46d238cab52a492e0e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Sep 2024 08:18:40 -0400 Subject: [PATCH 05/44] Bump com.nimbusds:oauth2-oidc-sdk from 11.9.1 to 11.19.1 in /plugins/repository-azure (#15862) * Bump com.nimbusds:oauth2-oidc-sdk in /plugins/repository-azure Bumps [com.nimbusds:oauth2-oidc-sdk](https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) from 11.9.1 to 11.19.1. - [Changelog](https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions/src/master/CHANGELOG.txt) - [Commits](https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions/branches/compare/11.19.1..11.9.1) --- updated-dependencies: - dependency-name: com.nimbusds:oauth2-oidc-sdk dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- .../repository-azure/licenses/oauth2-oidc-sdk-11.19.1.jar.sha1 | 1 + .../repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 | 1 - 4 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 plugins/repository-azure/licenses/oauth2-oidc-sdk-11.19.1.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index d70fc315cbf14..3a28f3852d162 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684)) - Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858)) - Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863)) +- Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862)) ### Changed diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index c8767478e8dad..5669c5e559ac8 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -62,7 +62,7 @@ dependencies { api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0' api "net.java.dev.jna:jna-platform:${versions.jna}" api 'com.microsoft.azure:msal4j:1.17.0' - api 'com.nimbusds:oauth2-oidc-sdk:11.9.1' + api 'com.nimbusds:oauth2-oidc-sdk:11.19.1' api 'com.nimbusds:nimbus-jose-jwt:9.40' api 'com.nimbusds:content-type:2.3' api 'com.nimbusds:lang-tag:1.7' diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.19.1.jar.sha1 b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.19.1.jar.sha1 new file mode 100644 index 0000000000000..7d83b0e8ca639 --- /dev/null +++ b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.19.1.jar.sha1 @@ -0,0 +1 @@ +58db85a807a56ae76baffa519772271ad5808195 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 deleted file mode 100644 index 96d9a196a172a..0000000000000 --- a/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fa9a2e447e2cef4dfda40a854dd7ec35624a7799 \ No newline at end of file From 19aedcd896c060db9028b2055e03d7f3e239e59c Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 10 Sep 2024 18:06:01 +0530 Subject: [PATCH 06/44] Move changelog entries to release notes which are backported to 2.17 (#15880) Signed-off-by: Shivansh Arora --- CHANGELOG.md | 2 -- release-notes/opensearch.release-notes-2.17.0.md | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a28f3852d162..20e3ef809b648 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - Adding WithFieldName interface for QueryBuilders with fieldName ([#15705](https://github.com/opensearch-project/OpenSearch/pull/15705)) - Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471)) -- Static RemotePublication setting added, removed experimental feature flag ([#15478](https://github.com/opensearch-project/OpenSearch/pull/15478)) - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) -- [Remote Publication] Upload incremental cluster state on master re-election ([#15145](https://github.com/opensearch-project/OpenSearch/pull/15145)) - Making _cat/allocation API use indexLevelStats ([#15292](https://github.com/opensearch-project/OpenSearch/pull/15292)) - Memory optimisations in _cluster/health API ([#15492](https://github.com/opensearch-project/OpenSearch/pull/15492)) diff --git a/release-notes/opensearch.release-notes-2.17.0.md b/release-notes/opensearch.release-notes-2.17.0.md index 18b7c9ac7cd68..c1870cafea411 100644 --- a/release-notes/opensearch.release-notes-2.17.0.md +++ b/release-notes/opensearch.release-notes-2.17.0.md @@ -50,6 +50,8 @@ - ClusterManagerTaskThrottler Improvements ([#15508](https://github.com/opensearch-project/OpenSearch/pull/15508)) - Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471)) - Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131)) +- [Remote Publication] Upload incremental cluster state on master re-election ([#15145](https://github.com/opensearch-project/OpenSearch/pull/15145)) +- Static RemotePublication setting added, removed experimental feature flag ([#15478](https://github.com/opensearch-project/OpenSearch/pull/15478)) ### Dependencies - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) From 107f0ce6a3c04ea4a759a8cf980e4d23c88ab1b8 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 10 Sep 2024 14:52:42 -0400 Subject: [PATCH 07/44] Update to Gradle 8.10.1 (#15867) Signed-off-by: Andriy Redko --- gradle/wrapper/gradle-wrapper.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 39a291b258efb..5c6205ebf24d4 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -11,7 +11,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.1-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703 +distributionSha256Sum=fdfca5dbc2834f0ece5020465737538e5ba679deeff5ab6c09621d67f8bb1a15 From f8515c7bd16feb0a2ae8c6de4e01c2d70c903624 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 11 Sep 2024 00:42:18 +0530 Subject: [PATCH 08/44] [BUG] Bug fix for checksum validation for mapping metadata (#15885) * Bug fix for checksum validation for mapping metadata Signed-off-by: Himshikha Gupta --- .../cluster/metadata/IndexMetadata.java | 40 ++++++++++++++++++- .../cluster/metadata/MappingMetadata.java | 11 ++++- .../common/compress/CompressedXContent.java | 5 +++ .../cluster/metadata/IndexMetadataTests.java | 28 ++++++++++++- 4 files changed, 81 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index bb470ea9e4ab8..713f8c9fc332c 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -1278,7 +1278,7 @@ public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOExcepti out.writeByte(state.id()); writeSettingsToStream(settings, out); out.writeVLongArray(primaryTerms); - out.writeMapValues(mappings, (stream, val) -> val.writeTo(stream)); + out.writeMapValues(mappings, (stream, val) -> val.writeVerifiableTo((BufferedChecksumStreamOutput) stream)); out.writeMapValues(aliases, (stream, val) -> val.writeTo(stream)); out.writeMap(customData, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); out.writeMap( @@ -1293,6 +1293,44 @@ public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOExcepti } } + @Override + public String toString() { + return new StringBuilder().append("IndexMetadata{routingNumShards=") + .append(routingNumShards) + .append(", index=") + .append(index) + .append(", version=") + .append(version) + .append(", state=") + .append(state) + .append(", settingsVersion=") + .append(settingsVersion) + .append(", mappingVersion=") + .append(mappingVersion) + .append(", aliasesVersion=") + .append(aliasesVersion) + .append(", primaryTerms=") + .append(Arrays.toString(primaryTerms)) + .append(", aliases=") + .append(aliases) + .append(", settings=") + .append(settings) + .append(", mappings=") + .append(mappings) + .append(", customData=") + .append(customData) + .append(", inSyncAllocationIds=") + .append(inSyncAllocationIds) + .append(", rolloverInfos=") + .append(rolloverInfos) + .append(", isSystem=") + .append(isSystem) + .append(", context=") + .append(context) + .append("}") + .toString(); + } + public boolean isSystem() { return isSystem; } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MappingMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/MappingMetadata.java index e8180613c0fa3..5856e7861d91c 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MappingMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MappingMetadata.java @@ -40,8 +40,10 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.MapperService; @@ -60,7 +62,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class MappingMetadata extends AbstractDiffable { +public class MappingMetadata extends AbstractDiffable implements VerifiableWriteable { public static final MappingMetadata EMPTY_MAPPINGS = new MappingMetadata(MapperService.SINGLE_MAPPING_NAME, Collections.emptyMap()); private final String type; @@ -164,6 +166,13 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(routingRequired); } + @Override + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { + out.writeString(type()); + source().writeVerifiableTo(out); + out.writeBoolean(routingRequired); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/server/src/main/java/org/opensearch/common/compress/CompressedXContent.java b/server/src/main/java/org/opensearch/common/compress/CompressedXContent.java index 23fc6353dbad3..daf2ffab2d5e2 100644 --- a/server/src/main/java/org/opensearch/common/compress/CompressedXContent.java +++ b/server/src/main/java/org/opensearch/common/compress/CompressedXContent.java @@ -38,6 +38,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.compress.Compressor; @@ -169,6 +170,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeByteArray(bytes); } + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { + out.writeInt(crc32); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java index a92a5b41e8580..92988ab7e9cba 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java @@ -199,7 +199,30 @@ public void testWriteVerifiableTo() throws IOException { ), randomNonNegativeLong() ); - + String mappings = " {\n" + + " \"_doc\": {\n" + + " \"properties\": {\n" + + " \"actiongroups\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"allowlist\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; IndexMetadata metadata1 = IndexMetadata.builder("foo") .settings( Settings.builder() @@ -220,11 +243,13 @@ public void testWriteVerifiableTo() throws IOException { .putRolloverInfo(info1) .putRolloverInfo(info2) .putInSyncAllocationIds(0, Set.of("1", "2", "3")) + .putMapping(mappings) .build(); BytesStreamOutput out = new BytesStreamOutput(); BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out); metadata1.writeVerifiableTo(checksumOut); + assertNotNull(metadata1.toString()); IndexMetadata metadata2 = IndexMetadata.builder(metadata1.getIndex().getName()) .settings( @@ -246,6 +271,7 @@ public void testWriteVerifiableTo() throws IOException { .putRolloverInfo(info2) .putRolloverInfo(info1) .putInSyncAllocationIds(0, Set.of("3", "1", "2")) + .putMapping(mappings) .build(); BytesStreamOutput out2 = new BytesStreamOutput(); From 320611a9bd1db71503ae32e9bcd54b0c54a5f0f3 Mon Sep 17 00:00:00 2001 From: "Samuel.G" <1148690954@qq.com> Date: Wed, 11 Sep 2024 04:53:28 +0800 Subject: [PATCH 09/44] fix escaped wildcard query on wildcard field (#15737) * fix escaped wildcard query on wildcard field Signed-off-by: gesong.samuel * fix format error Signed-off-by: gesong.samuel * add change log Signed-off-by: gesong.samuel --------- Signed-off-by: gesong.samuel Signed-off-by: Michael Froh Co-authored-by: gesong.samuel Co-authored-by: Michael Froh --- CHANGELOG.md | 1 + .../index/mapper/WildcardFieldMapper.java | 54 ++++++++++++++----- .../index/mapper/WildcardFieldTypeTests.java | 26 +++++++++ 3 files changed, 68 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20e3ef809b648..4912d465a87a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Removed ### Fixed +- Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737)) ### Security diff --git a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java index cd95e320209ee..0cb416a9b8370 100644 --- a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java @@ -40,7 +40,6 @@ import org.apache.lucene.util.automaton.RegExp; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.common.lucene.Lucene; -import org.opensearch.common.regex.Regex; import org.opensearch.common.unit.Fuzziness; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.analysis.IndexAnalyzers; @@ -430,22 +429,27 @@ public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, bo finalValue = value; } Predicate matchPredicate; - if (value.contains("?")) { - Automaton automaton = WildcardQuery.toAutomaton(new Term(name(), finalValue)); - CompiledAutomaton compiledAutomaton = new CompiledAutomaton(automaton); + Automaton automaton = WildcardQuery.toAutomaton(new Term(name(), finalValue)); + CompiledAutomaton compiledAutomaton = new CompiledAutomaton(automaton); + if (compiledAutomaton.type == CompiledAutomaton.AUTOMATON_TYPE.SINGLE) { + // when type equals SINGLE, #compiledAutomaton.runAutomaton is null matchPredicate = s -> { if (caseInsensitive) { s = s.toLowerCase(Locale.ROOT); } - BytesRef valueBytes = BytesRefs.toBytesRef(s); - return compiledAutomaton.runAutomaton.run(valueBytes.bytes, valueBytes.offset, valueBytes.length); + return s.equals(finalValue); }; + } else if (compiledAutomaton.type == CompiledAutomaton.AUTOMATON_TYPE.ALL) { + return existsQuery(context); + } else if (compiledAutomaton.type == CompiledAutomaton.AUTOMATON_TYPE.NONE) { + return new MatchNoDocsQuery("Wildcard expression matches nothing"); } else { matchPredicate = s -> { if (caseInsensitive) { s = s.toLowerCase(Locale.ROOT); } - return Regex.simpleMatch(finalValue, s); + BytesRef valueBytes = BytesRefs.toBytesRef(s); + return compiledAutomaton.runAutomaton.run(valueBytes.bytes, valueBytes.offset, valueBytes.length); }; } @@ -468,11 +472,18 @@ public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, bo // Package-private for testing static Set getRequiredNGrams(String value) { Set terms = new HashSet<>(); + + if (value.isEmpty()) { + return terms; + } + int pos = 0; + String rawSequence = null; String currentSequence = null; if (!value.startsWith("?") && !value.startsWith("*")) { // Can add prefix term - currentSequence = getNonWildcardSequence(value, 0); + rawSequence = getNonWildcardSequence(value, 0); + currentSequence = performEscape(rawSequence); if (currentSequence.length() == 1) { terms.add(new String(new char[] { 0, currentSequence.charAt(0) })); } else { @@ -480,10 +491,11 @@ static Set getRequiredNGrams(String value) { } } else { pos = findNonWildcardSequence(value, pos); - currentSequence = getNonWildcardSequence(value, pos); + rawSequence = getNonWildcardSequence(value, pos); } while (pos < value.length()) { - boolean isEndOfValue = pos + currentSequence.length() == value.length(); + boolean isEndOfValue = pos + rawSequence.length() == value.length(); + currentSequence = performEscape(rawSequence); if (!currentSequence.isEmpty() && currentSequence.length() < 3 && !isEndOfValue && pos > 0) { // If this is a prefix or suffix of length < 3, then we already have a longer token including the anchor. terms.add(currentSequence); @@ -502,8 +514,8 @@ static Set getRequiredNGrams(String value) { terms.add(new String(new char[] { a, b, 0 })); } } - pos = findNonWildcardSequence(value, pos + currentSequence.length()); - currentSequence = getNonWildcardSequence(value, pos); + pos = findNonWildcardSequence(value, pos + rawSequence.length()); + rawSequence = getNonWildcardSequence(value, pos); } return terms; } @@ -511,7 +523,7 @@ static Set getRequiredNGrams(String value) { private static String getNonWildcardSequence(String value, int startFrom) { for (int i = startFrom; i < value.length(); i++) { char c = value.charAt(i); - if (c == '?' || c == '*') { + if ((c == '?' || c == '*') && (i == 0 || value.charAt(i - 1) != '\\')) { return value.substring(startFrom, i); } } @@ -529,6 +541,22 @@ private static int findNonWildcardSequence(String value, int startFrom) { return value.length(); } + private static String performEscape(String str) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < str.length(); i++) { + if (str.charAt(i) == '\\' && (i + 1) < str.length()) { + char c = str.charAt(i + 1); + if (c == '*' || c == '?') { + i++; + } + } + sb.append(str.charAt(i)); + } + assert !sb.toString().contains("\\*"); + assert !sb.toString().contains("\\?"); + return sb.toString(); + } + @Override public Query regexpQuery( String value, diff --git a/server/src/test/java/org/opensearch/index/mapper/WildcardFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/WildcardFieldTypeTests.java index cd2a23cf94c37..1a813495e9033 100644 --- a/server/src/test/java/org/opensearch/index/mapper/WildcardFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/WildcardFieldTypeTests.java @@ -88,6 +88,32 @@ public void testWildcardQuery() { ); } + public void testEscapedWildcardQuery() { + MappedFieldType ft = new WildcardFieldMapper.WildcardFieldType("field"); + Set expectedTerms = new HashSet<>(); + expectedTerms.add(prefixAnchored("*")); + expectedTerms.add(suffixAnchored("*")); + + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + for (String term : expectedTerms) { + builder.add(new TermQuery(new Term("field", term)), BooleanClause.Occur.FILTER); + } + + assertEquals( + new WildcardFieldMapper.WildcardMatchingQuery("field", builder.build(), "\\**\\*"), + ft.wildcardQuery("\\**\\*", null, null) + ); + + assertEquals(new WildcardFieldMapper.WildcardMatchingQuery("field", builder.build(), "\\*"), ft.wildcardQuery("\\*", null, null)); + + expectedTerms.remove(suffixAnchored("*")); + builder = new BooleanQuery.Builder(); + for (String term : expectedTerms) { + builder.add(new TermQuery(new Term("field", term)), BooleanClause.Occur.FILTER); + } + assertEquals(new WildcardFieldMapper.WildcardMatchingQuery("field", builder.build(), "\\**"), ft.wildcardQuery("\\**", null, null)); + } + public void testMultipleWildcardsInQuery() { final String pattern = "a?cd*efg?h"; MappedFieldType ft = new WildcardFieldMapper.WildcardFieldType("field"); From 92d7fe89755dc87cbc16e220473c7faae055afd6 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Wed, 11 Sep 2024 11:22:39 +0530 Subject: [PATCH 10/44] Add more wait time to assertBusy to fix RemoteFsTimestampAwareTranslogTests.testSimpleOperationsUpload (#15865) Signed-off-by: Sachin Kale --- .../index/translog/RemoteFsTimestampAwareTranslogTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java index 4c9da7e95dfa7..4ec68f7fb75b4 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java @@ -55,6 +55,7 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -310,7 +311,6 @@ public void testIndexDeletionWithNoPinnedTimestampButRecentFiles() throws Except } @Override - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15731") public void testSimpleOperationsUpload() throws Exception { ArrayList ops = new ArrayList<>(); @@ -366,7 +366,7 @@ public void testSimpleOperationsUpload() throws Exception { 10, blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() ); - }); + }, 60, TimeUnit.SECONDS); } @Override From 98a9161aff122ecd290b3e8053eb271d65251ccd Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 11 Sep 2024 11:51:56 -0400 Subject: [PATCH 11/44] Fixing MacOS 13 assemble workflows (once again) (#15898) Signed-off-by: Andriy Redko --- .github/workflows/assemble.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml index b3838b8e5ae97..d90b05c323cf1 100644 --- a/.github/workflows/assemble.yml +++ b/.github/workflows/assemble.yml @@ -32,7 +32,10 @@ jobs: if: runner.os == 'macos' continue-on-error: true run: | - brew install docker colima coreutils + # Force QEMU 9.0.2 usage + curl https://raw.githubusercontent.com/Homebrew/homebrew-core/f1a9cf104a9a51779c7a532b658c490f69974839/Formula/q/qemu.rb > qemu.rb + brew install qemu.rb + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 HOMEBREW_NO_AUTO_UPDATE=1 brew install docker colima coreutils gtimeout 15m colima start shell: bash - name: Run Gradle (assemble) From 9354dd92db5b2e50e0f9067dc4a3eba59282daf1 Mon Sep 17 00:00:00 2001 From: rajiv-kv <157019998+rajiv-kv@users.noreply.github.com> Date: Wed, 11 Sep 2024 23:02:02 +0530 Subject: [PATCH 12/44] updating release-notes for 2.17 (#15884) Signed-off-by: Rajiv Kumar Vaidyanathan --- CHANGELOG.md | 3 --- release-notes/opensearch.release-notes-2.17.0.md | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4912d465a87a8..43f35a9119c12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added - Adding WithFieldName interface for QueryBuilders with fieldName ([#15705](https://github.com/opensearch-project/OpenSearch/pull/15705)) -- Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471)) - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) -- Making _cat/allocation API use indexLevelStats ([#15292](https://github.com/opensearch-project/OpenSearch/pull/15292)) -- Memory optimisations in _cluster/health API ([#15492](https://github.com/opensearch-project/OpenSearch/pull/15492)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/release-notes/opensearch.release-notes-2.17.0.md b/release-notes/opensearch.release-notes-2.17.0.md index c1870cafea411..e374b222104b0 100644 --- a/release-notes/opensearch.release-notes-2.17.0.md +++ b/release-notes/opensearch.release-notes-2.17.0.md @@ -52,6 +52,8 @@ - Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131)) - [Remote Publication] Upload incremental cluster state on master re-election ([#15145](https://github.com/opensearch-project/OpenSearch/pull/15145)) - Static RemotePublication setting added, removed experimental feature flag ([#15478](https://github.com/opensearch-project/OpenSearch/pull/15478)) +- Making _cat/allocation API use indexLevelStats ([#15292](https://github.com/opensearch-project/OpenSearch/pull/15292)) +- Memory optimisations in _cluster/health API ([#15492](https://github.com/opensearch-project/OpenSearch/pull/15492)) ### Dependencies - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) From 58794ada3d18f1103b3382a8556eadc2a2e0e94b Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Wed, 11 Sep 2024 12:12:04 -0700 Subject: [PATCH 13/44] Add cancellation framework changes in wlm (#15651) * cancellation related Signed-off-by: Kiran Prakash * Update CHANGELOG.md Signed-off-by: Kiran Prakash * add better cancellation reason Signed-off-by: Kiran Prakash * Update DefaultTaskCancellationTests.java Signed-off-by: Kiran Prakash * refactor Signed-off-by: Kiran Prakash * refactor Signed-off-by: Kiran Prakash * Update DefaultTaskCancellation.java Signed-off-by: Kiran Prakash * Update DefaultTaskCancellation.java Signed-off-by: Kiran Prakash * Update DefaultTaskCancellation.java Signed-off-by: Kiran Prakash * Update DefaultTaskSelectionStrategy.java Signed-off-by: Kiran Prakash * refactor Signed-off-by: Kiran Prakash * refactor node level threshold Signed-off-by: Kiran Prakash * use query group task Signed-off-by: Kaushal Kumar * code clean up and refactorings Signed-off-by: Kaushal Kumar * add unit tests and fix existing ones Signed-off-by: Kaushal Kumar * uncomment the test case Signed-off-by: Kaushal Kumar * update CHANGELOG Signed-off-by: Kaushal Kumar * fix imports Signed-off-by: Kaushal Kumar * refactor and add UTs for new constructs Signed-off-by: Kaushal Kumar * fix javadocs Signed-off-by: Kaushal Kumar * remove code clutter Signed-off-by: Kaushal Kumar * change annotation version and task selection strategy Signed-off-by: Kaushal Kumar * rename a util class Signed-off-by: Kaushal Kumar * remove wrappers from resource type Signed-off-by: Kaushal Kumar * apply spotless Signed-off-by: Kaushal Kumar * address comments Signed-off-by: Kaushal Kumar * add rename changes Signed-off-by: Kaushal Kumar * address comments Signed-off-by: Kaushal Kumar * refactor changes and logical bug fix Signed-off-by: Kaushal Kumar * address comments Signed-off-by: Kaushal Kumar --------- Signed-off-by: Kiran Prakash Signed-off-by: Kaushal Kumar Signed-off-by: Ankit Jain Co-authored-by: Kiran Prakash Co-authored-by: Ankit Jain --- CHANGELOG.md | 1 + .../cluster/metadata/QueryGroup.java | 3 +- .../wlm/QueryGroupLevelResourceUsageView.java | 12 +- .../org/opensearch/wlm/QueryGroupTask.java | 24 +- .../java/org/opensearch/wlm/ResourceType.java | 41 +- .../wlm/WorkloadManagementSettings.java | 2 + .../MaximumResourceTaskSelectionStrategy.java | 71 +++ .../QueryGroupTaskCancellationService.java | 205 +++++++ .../cancellation/TaskSelectionStrategy.java | 28 + .../wlm/cancellation/package-info.java | 12 + .../wlm/tracker/CpuUsageCalculator.java | 38 ++ .../wlm/tracker/MemoryUsageCalculator.java | 35 ++ ...QueryGroupResourceUsageTrackerService.java | 21 +- .../wlm/tracker/ResourceUsageCalculator.java | 34 ++ ...QueryGroupLevelResourceUsageViewTests.java | 49 +- .../org/opensearch/wlm/ResourceTypeTests.java | 19 - ...mumResourceTaskSelectionStrategyTests.java | 126 ++++ ...ueryGroupTaskCancellationServiceTests.java | 541 ++++++++++++++++++ .../tracker/ResourceUsageCalculatorTests.java | 70 +++ ...ceUsageCalculatorTrackerServiceTests.java} | 63 +- 20 files changed, 1297 insertions(+), 98 deletions(-) create mode 100644 server/src/main/java/org/opensearch/wlm/cancellation/MaximumResourceTaskSelectionStrategy.java create mode 100644 server/src/main/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationService.java create mode 100644 server/src/main/java/org/opensearch/wlm/cancellation/TaskSelectionStrategy.java create mode 100644 server/src/main/java/org/opensearch/wlm/cancellation/package-info.java create mode 100644 server/src/main/java/org/opensearch/wlm/tracker/CpuUsageCalculator.java create mode 100644 server/src/main/java/org/opensearch/wlm/tracker/MemoryUsageCalculator.java create mode 100644 server/src/main/java/org/opensearch/wlm/tracker/ResourceUsageCalculator.java create mode 100644 server/src/test/java/org/opensearch/wlm/cancellation/MaximumResourceTaskSelectionStrategyTests.java create mode 100644 server/src/test/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationServiceTests.java create mode 100644 server/src/test/java/org/opensearch/wlm/tracker/ResourceUsageCalculatorTests.java rename server/src/test/java/org/opensearch/wlm/tracker/{QueryGroupResourceUsageTrackerServiceTests.java => ResourceUsageCalculatorTrackerServiceTests.java} (69%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43f35a9119c12..610ebb304ba1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - Adding WithFieldName interface for QueryBuilders with fieldName ([#15705](https://github.com/opensearch-project/OpenSearch/pull/15705)) - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) +- [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java b/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java index dcd96dceb4bf1..0eeafdc8f5eed 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java @@ -12,6 +12,7 @@ import org.opensearch.cluster.Diff; import org.opensearch.common.UUIDs; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; @@ -41,7 +42,7 @@ * "updated_at": 4513232415 * } */ -@ExperimentalApi +@PublicApi(since = "2.18.0") public class QueryGroup extends AbstractDiffable implements ToXContentObject { public static final String _ID_STRING = "_id"; diff --git a/server/src/main/java/org/opensearch/wlm/QueryGroupLevelResourceUsageView.java b/server/src/main/java/org/opensearch/wlm/QueryGroupLevelResourceUsageView.java index 7577c8573ec10..de213eaab64a8 100644 --- a/server/src/main/java/org/opensearch/wlm/QueryGroupLevelResourceUsageView.java +++ b/server/src/main/java/org/opensearch/wlm/QueryGroupLevelResourceUsageView.java @@ -8,8 +8,6 @@ package org.opensearch.wlm; -import org.opensearch.tasks.Task; - import java.util.List; import java.util.Map; @@ -20,11 +18,11 @@ */ public class QueryGroupLevelResourceUsageView { // resourceUsage holds the resource usage data for a QueryGroup at a point in time - private final Map resourceUsage; + private final Map resourceUsage; // activeTasks holds the list of active tasks for a QueryGroup at a point in time - private final List activeTasks; + private final List activeTasks; - public QueryGroupLevelResourceUsageView(Map resourceUsage, List activeTasks) { + public QueryGroupLevelResourceUsageView(Map resourceUsage, List activeTasks) { this.resourceUsage = resourceUsage; this.activeTasks = activeTasks; } @@ -34,7 +32,7 @@ public QueryGroupLevelResourceUsageView(Map resourceUsage, L * * @return The map of resource usage data */ - public Map getResourceUsageData() { + public Map getResourceUsageData() { return resourceUsage; } @@ -43,7 +41,7 @@ public Map getResourceUsageData() { * * @return The list of active tasks */ - public List getActiveTasks() { + public List getActiveTasks() { return activeTasks; } } diff --git a/server/src/main/java/org/opensearch/wlm/QueryGroupTask.java b/server/src/main/java/org/opensearch/wlm/QueryGroupTask.java index 4eb413be61b72..a1cb766579d43 100644 --- a/server/src/main/java/org/opensearch/wlm/QueryGroupTask.java +++ b/server/src/main/java/org/opensearch/wlm/QueryGroupTask.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.tasks.TaskId; @@ -17,6 +18,7 @@ import java.util.Map; import java.util.Optional; +import java.util.function.LongSupplier; import java.util.function.Supplier; import static org.opensearch.search.SearchService.NO_TIMEOUT; @@ -24,15 +26,17 @@ /** * Base class to define QueryGroup tasks */ +@PublicApi(since = "2.18.0") public class QueryGroupTask extends CancellableTask { private static final Logger logger = LogManager.getLogger(QueryGroupTask.class); public static final String QUERY_GROUP_ID_HEADER = "queryGroupId"; public static final Supplier DEFAULT_QUERY_GROUP_ID_SUPPLIER = () -> "DEFAULT_QUERY_GROUP"; + private final LongSupplier nanoTimeSupplier; private String queryGroupId; public QueryGroupTask(long id, String type, String action, String description, TaskId parentTaskId, Map headers) { - this(id, type, action, description, parentTaskId, headers, NO_TIMEOUT); + this(id, type, action, description, parentTaskId, headers, NO_TIMEOUT, System::nanoTime); } public QueryGroupTask( @@ -43,8 +47,22 @@ public QueryGroupTask( TaskId parentTaskId, Map headers, TimeValue cancelAfterTimeInterval + ) { + this(id, type, action, description, parentTaskId, headers, cancelAfterTimeInterval, System::nanoTime); + } + + public QueryGroupTask( + long id, + String type, + String action, + String description, + TaskId parentTaskId, + Map headers, + TimeValue cancelAfterTimeInterval, + LongSupplier nanoTimeSupplier ) { super(id, type, action, description, parentTaskId, headers, cancelAfterTimeInterval); + this.nanoTimeSupplier = nanoTimeSupplier; } /** @@ -69,6 +87,10 @@ public final void setQueryGroupId(final ThreadContext threadContext) { .orElse(DEFAULT_QUERY_GROUP_ID_SUPPLIER.get()); } + public long getElapsedTime() { + return nanoTimeSupplier.getAsLong() - getStartTimeNanos(); + } + @Override public boolean shouldCancelChildrenOnCancellation() { return false; diff --git a/server/src/main/java/org/opensearch/wlm/ResourceType.java b/server/src/main/java/org/opensearch/wlm/ResourceType.java index 2e8da4f57f36c..a560268a66853 100644 --- a/server/src/main/java/org/opensearch/wlm/ResourceType.java +++ b/server/src/main/java/org/opensearch/wlm/ResourceType.java @@ -10,8 +10,9 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.tasks.resourcetracker.ResourceStats; -import org.opensearch.tasks.Task; +import org.opensearch.wlm.tracker.CpuUsageCalculator; +import org.opensearch.wlm.tracker.MemoryUsageCalculator; +import org.opensearch.wlm.tracker.ResourceUsageCalculator; import java.io.IOException; import java.util.List; @@ -24,19 +25,25 @@ */ @PublicApi(since = "2.17.0") public enum ResourceType { - CPU("cpu", task -> task.getTotalResourceUtilization(ResourceStats.CPU), true), - MEMORY("memory", task -> task.getTotalResourceUtilization(ResourceStats.MEMORY), true); + CPU("cpu", true, CpuUsageCalculator.INSTANCE, WorkloadManagementSettings::getNodeLevelCpuCancellationThreshold), + MEMORY("memory", true, MemoryUsageCalculator.INSTANCE, WorkloadManagementSettings::getNodeLevelMemoryCancellationThreshold); private final String name; - private final Function getResourceUsage; private final boolean statsEnabled; - + private final ResourceUsageCalculator resourceUsageCalculator; + private final Function nodeLevelThresholdSupplier; private static List sortedValues = List.of(CPU, MEMORY); - ResourceType(String name, Function getResourceUsage, boolean statsEnabled) { + ResourceType( + String name, + boolean statsEnabled, + ResourceUsageCalculator resourceUsageCalculator, + Function nodeLevelThresholdSupplier + ) { this.name = name; - this.getResourceUsage = getResourceUsage; this.statsEnabled = statsEnabled; + this.resourceUsageCalculator = resourceUsageCalculator; + this.nodeLevelThresholdSupplier = nodeLevelThresholdSupplier; } /** @@ -61,20 +68,18 @@ public String getName() { return name; } - /** - * Gets the resource usage for a given resource type and task. - * - * @param task the task for which to calculate resource usage - * @return the resource usage - */ - public long getResourceUsage(Task task) { - return getResourceUsage.apply(task); - } - public boolean hasStatsEnabled() { return statsEnabled; } + public ResourceUsageCalculator getResourceUsageCalculator() { + return resourceUsageCalculator; + } + + public double getNodeLevelThreshold(WorkloadManagementSettings settings) { + return nodeLevelThresholdSupplier.apply(settings); + } + public static List getSortedValues() { return sortedValues; } diff --git a/server/src/main/java/org/opensearch/wlm/WorkloadManagementSettings.java b/server/src/main/java/org/opensearch/wlm/WorkloadManagementSettings.java index b104925df77b3..b3577c1b3219d 100644 --- a/server/src/main/java/org/opensearch/wlm/WorkloadManagementSettings.java +++ b/server/src/main/java/org/opensearch/wlm/WorkloadManagementSettings.java @@ -8,6 +8,7 @@ package org.opensearch.wlm; +import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; @@ -15,6 +16,7 @@ /** * Main class to declare Workload Management related settings */ +@PublicApi(since = "2.18.0") public class WorkloadManagementSettings { private static final Double DEFAULT_NODE_LEVEL_MEMORY_REJECTION_THRESHOLD = 0.8; private static final Double DEFAULT_NODE_LEVEL_MEMORY_CANCELLATION_THRESHOLD = 0.9; diff --git a/server/src/main/java/org/opensearch/wlm/cancellation/MaximumResourceTaskSelectionStrategy.java b/server/src/main/java/org/opensearch/wlm/cancellation/MaximumResourceTaskSelectionStrategy.java new file mode 100644 index 0000000000000..ffb326c07e7ac --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/cancellation/MaximumResourceTaskSelectionStrategy.java @@ -0,0 +1,71 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.cancellation; + +import org.opensearch.wlm.QueryGroupTask; +import org.opensearch.wlm.ResourceType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +import static org.opensearch.wlm.cancellation.QueryGroupTaskCancellationService.MIN_VALUE; + +/** + * Represents the highest resource consuming task first selection strategy. + */ +public class MaximumResourceTaskSelectionStrategy implements TaskSelectionStrategy { + + public MaximumResourceTaskSelectionStrategy() {} + + /** + * Returns a comparator that defines the sorting condition for tasks. + * This is the default implementation since the most resource consuming tasks are the likely to regress the performance. + * from resiliency point of view it makes sense to cancel them first + * + * @return The comparator + */ + private Comparator sortingCondition(ResourceType resourceType) { + return Comparator.comparingDouble(task -> resourceType.getResourceUsageCalculator().calculateTaskResourceUsage(task)); + } + + /** + * Selects tasks for cancellation based on the provided limit and resource type. + * The tasks are sorted based on the sorting condition and then selected until the accumulated resource usage reaches the limit. + * + * @param tasks The list of tasks from which to select + * @param limit The limit on the accumulated resource usage + * @param resourceType The type of resource to consider + * @return The list of selected tasks + * @throws IllegalArgumentException If the limit is less than zero + */ + public List selectTasksForCancellation(List tasks, double limit, ResourceType resourceType) { + if (limit < 0) { + throw new IllegalArgumentException("limit has to be greater than zero"); + } + if (limit < MIN_VALUE) { + return Collections.emptyList(); + } + + List sortedTasks = tasks.stream().sorted(sortingCondition(resourceType).reversed()).collect(Collectors.toList()); + + List selectedTasks = new ArrayList<>(); + double accumulated = 0; + for (QueryGroupTask task : sortedTasks) { + selectedTasks.add(task); + accumulated += resourceType.getResourceUsageCalculator().calculateTaskResourceUsage(task); + if ((accumulated - limit) > MIN_VALUE) { + break; + } + } + return selectedTasks; + } +} diff --git a/server/src/main/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationService.java b/server/src/main/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationService.java new file mode 100644 index 0000000000000..a2c97c8d8635b --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationService.java @@ -0,0 +1,205 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.cancellation; + +import org.opensearch.cluster.metadata.QueryGroup; +import org.opensearch.tasks.CancellableTask; +import org.opensearch.tasks.TaskCancellation; +import org.opensearch.wlm.MutableQueryGroupFragment.ResiliencyMode; +import org.opensearch.wlm.QueryGroupLevelResourceUsageView; +import org.opensearch.wlm.QueryGroupTask; +import org.opensearch.wlm.ResourceType; +import org.opensearch.wlm.WorkloadManagementSettings; +import org.opensearch.wlm.tracker.QueryGroupResourceUsageTrackerService; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.BooleanSupplier; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.opensearch.wlm.tracker.QueryGroupResourceUsageTrackerService.TRACKED_RESOURCES; + +/** + * Manages the cancellation of tasks enforced by QueryGroup thresholds on resource usage criteria. + * This class utilizes a strategy pattern through {@link MaximumResourceTaskSelectionStrategy} to identify tasks that exceed + * predefined resource usage limits and are therefore eligible for cancellation. + * + *

The cancellation process is initiated by evaluating the resource usage of each QueryGroup against its + * resource limits. Tasks that contribute to exceeding these limits are selected for cancellation based on the + * implemented task selection strategy.

+ * + *

Instances of this class are configured with a map linking QueryGroup IDs to their corresponding resource usage + * views, a set of active QueryGroups, and a task selection strategy. These components collectively facilitate the + * identification and cancellation of tasks that threaten to breach QueryGroup resource limits.

+ * + * @see MaximumResourceTaskSelectionStrategy + * @see QueryGroup + * @see ResourceType + */ +public class QueryGroupTaskCancellationService { + public static final double MIN_VALUE = 1e-9; + + private final WorkloadManagementSettings workloadManagementSettings; + private final TaskSelectionStrategy taskSelectionStrategy; + private final QueryGroupResourceUsageTrackerService resourceUsageTrackerService; + // a map of QueryGroupId to its corresponding QueryGroupLevelResourceUsageView object + Map queryGroupLevelResourceUsageViews; + private final Collection activeQueryGroups; + private final Collection deletedQueryGroups; + + public QueryGroupTaskCancellationService( + WorkloadManagementSettings workloadManagementSettings, + TaskSelectionStrategy taskSelectionStrategy, + QueryGroupResourceUsageTrackerService resourceUsageTrackerService, + Collection activeQueryGroups, + Collection deletedQueryGroups + ) { + this.workloadManagementSettings = workloadManagementSettings; + this.taskSelectionStrategy = taskSelectionStrategy; + this.resourceUsageTrackerService = resourceUsageTrackerService; + this.activeQueryGroups = activeQueryGroups; + this.deletedQueryGroups = deletedQueryGroups; + } + + /** + * Cancel tasks based on the implemented strategy. + */ + public final void cancelTasks(BooleanSupplier isNodeInDuress) { + queryGroupLevelResourceUsageViews = resourceUsageTrackerService.constructQueryGroupLevelUsageViews(); + // cancel tasks from QueryGroups that are in Enforced mode that are breaching their resource limits + cancelTasks(ResiliencyMode.ENFORCED); + // if the node is in duress, cancel tasks accordingly. + handleNodeDuress(isNodeInDuress); + } + + private void handleNodeDuress(BooleanSupplier isNodeInDuress) { + if (!isNodeInDuress.getAsBoolean()) { + return; + } + // List of tasks to be executed in order if the node is in duress + List> duressActions = List.of(v -> cancelTasksFromDeletedQueryGroups(), v -> cancelTasks(ResiliencyMode.SOFT)); + + for (Consumer duressAction : duressActions) { + if (!isNodeInDuress.getAsBoolean()) { + break; + } + duressAction.accept(null); + } + } + + private void cancelTasksFromDeletedQueryGroups() { + cancelTasks(getAllCancellableTasks(this.deletedQueryGroups)); + } + + /** + * Get all cancellable tasks from the QueryGroups. + * + * @return List of tasks that can be cancelled + */ + List getAllCancellableTasks(ResiliencyMode resiliencyMode) { + return getAllCancellableTasks( + activeQueryGroups.stream().filter(queryGroup -> queryGroup.getResiliencyMode() == resiliencyMode).collect(Collectors.toList()) + ); + } + + /** + * Get all cancellable tasks from the given QueryGroups. + * + * @return List of tasks that can be cancelled + */ + List getAllCancellableTasks(Collection queryGroups) { + List taskCancellations = new ArrayList<>(); + for (QueryGroup queryGroup : queryGroups) { + final List reasons = new ArrayList<>(); + List selectedTasks = new ArrayList<>(); + for (ResourceType resourceType : TRACKED_RESOURCES) { + // We need to consider the already selected tasks since those tasks also consumed the resources + double excessUsage = getExcessUsage(queryGroup, resourceType) - resourceType.getResourceUsageCalculator() + .calculateResourceUsage(selectedTasks); + if (excessUsage > MIN_VALUE) { + reasons.add(new TaskCancellation.Reason(generateReasonString(queryGroup, resourceType), 1)); + // TODO: We will need to add the cancellation callback for these resources for the queryGroup to reflect stats + + // Only add tasks not already added to avoid double cancellations + selectedTasks.addAll( + taskSelectionStrategy.selectTasksForCancellation(getTasksFor(queryGroup), excessUsage, resourceType) + .stream() + .filter(x -> selectedTasks.stream().noneMatch(y -> x.getId() != y.getId())) + .collect(Collectors.toList()) + ); + } + } + + if (!reasons.isEmpty()) { + taskCancellations.addAll( + selectedTasks.stream().map(task -> createTaskCancellation(task, reasons)).collect(Collectors.toList()) + ); + } + } + return taskCancellations; + } + + private String generateReasonString(QueryGroup queryGroup, ResourceType resourceType) { + final double currentUsage = getCurrentUsage(queryGroup, resourceType); + return "QueryGroup ID : " + + queryGroup.get_id() + + " breached the resource limit: (" + + currentUsage + + " > " + + queryGroup.getResourceLimits().get(resourceType) + + ") for resource type : " + + resourceType.getName(); + } + + private List getTasksFor(QueryGroup queryGroup) { + return queryGroupLevelResourceUsageViews.get(queryGroup.get_id()).getActiveTasks(); + } + + private void cancelTasks(ResiliencyMode resiliencyMode) { + cancelTasks(getAllCancellableTasks(resiliencyMode)); + } + + private void cancelTasks(List cancellableTasks) { + cancellableTasks.forEach(TaskCancellation::cancel); + } + + private TaskCancellation createTaskCancellation(CancellableTask task, List reasons) { + return new TaskCancellation(task, reasons, List.of(this::callbackOnCancel)); + } + + private double getExcessUsage(QueryGroup queryGroup, ResourceType resourceType) { + if (queryGroup.getResourceLimits().get(resourceType) == null + || !queryGroupLevelResourceUsageViews.containsKey(queryGroup.get_id())) { + return 0; + } + return getCurrentUsage(queryGroup, resourceType) - getNormalisedThreshold(queryGroup, resourceType); + } + + private double getCurrentUsage(QueryGroup queryGroup, ResourceType resourceType) { + final QueryGroupLevelResourceUsageView queryGroupResourceUsageView = queryGroupLevelResourceUsageViews.get(queryGroup.get_id()); + return queryGroupResourceUsageView.getResourceUsageData().get(resourceType); + } + + /** + * normalises configured value with respect to node level cancellation thresholds + * @param queryGroup instance + * @return normalised value with respect to node level cancellation thresholds + */ + private double getNormalisedThreshold(QueryGroup queryGroup, ResourceType resourceType) { + double nodeLevelCancellationThreshold = resourceType.getNodeLevelThreshold(workloadManagementSettings); + return queryGroup.getResourceLimits().get(resourceType) * nodeLevelCancellationThreshold; + } + + private void callbackOnCancel() { + // TODO Implement callback logic here mostly used for Stats + } +} diff --git a/server/src/main/java/org/opensearch/wlm/cancellation/TaskSelectionStrategy.java b/server/src/main/java/org/opensearch/wlm/cancellation/TaskSelectionStrategy.java new file mode 100644 index 0000000000000..63fbf9b791a33 --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/cancellation/TaskSelectionStrategy.java @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.cancellation; + +import org.opensearch.wlm.QueryGroupTask; +import org.opensearch.wlm.ResourceType; + +import java.util.List; + +/** + * This interface exposes a method which implementations can use + */ +public interface TaskSelectionStrategy { + /** + * Determines how the tasks are selected from the list of given tasks based on resource type + * @param tasks to select from + * @param limit min cumulative resource usage sum of selected tasks + * @param resourceType + * @return list of tasks + */ + List selectTasksForCancellation(List tasks, double limit, ResourceType resourceType); +} diff --git a/server/src/main/java/org/opensearch/wlm/cancellation/package-info.java b/server/src/main/java/org/opensearch/wlm/cancellation/package-info.java new file mode 100644 index 0000000000000..1ce7b571e9a9c --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/cancellation/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Workload management resource based cancellation artifacts + */ +package org.opensearch.wlm.cancellation; diff --git a/server/src/main/java/org/opensearch/wlm/tracker/CpuUsageCalculator.java b/server/src/main/java/org/opensearch/wlm/tracker/CpuUsageCalculator.java new file mode 100644 index 0000000000000..05c84cd767b1f --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/tracker/CpuUsageCalculator.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.tracker; + +import org.opensearch.core.tasks.resourcetracker.ResourceStats; +import org.opensearch.wlm.QueryGroupTask; + +import java.util.List; + +/** + * class to help make cpu usage calculations for the query group + */ +public class CpuUsageCalculator extends ResourceUsageCalculator { + // This value should be initialised at the start time of the process and be used throughout the codebase + public static final int PROCESSOR_COUNT = Runtime.getRuntime().availableProcessors(); + public static final CpuUsageCalculator INSTANCE = new CpuUsageCalculator(); + + private CpuUsageCalculator() {} + + @Override + public double calculateResourceUsage(List tasks) { + double usage = tasks.stream().mapToDouble(this::calculateTaskResourceUsage).sum(); + + usage /= PROCESSOR_COUNT; + return usage; + } + + @Override + public double calculateTaskResourceUsage(QueryGroupTask task) { + return (1.0f * task.getTotalResourceUtilization(ResourceStats.CPU)) / task.getElapsedTime(); + } +} diff --git a/server/src/main/java/org/opensearch/wlm/tracker/MemoryUsageCalculator.java b/server/src/main/java/org/opensearch/wlm/tracker/MemoryUsageCalculator.java new file mode 100644 index 0000000000000..fb66ff47f58d0 --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/tracker/MemoryUsageCalculator.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.tracker; + +import org.opensearch.core.tasks.resourcetracker.ResourceStats; +import org.opensearch.monitor.jvm.JvmStats; +import org.opensearch.wlm.QueryGroupTask; + +import java.util.List; + +/** + * class to help make memory usage calculations for the query group + */ +public class MemoryUsageCalculator extends ResourceUsageCalculator { + public static final long HEAP_SIZE_BYTES = JvmStats.jvmStats().getMem().getHeapMax().getBytes(); + public static final MemoryUsageCalculator INSTANCE = new MemoryUsageCalculator(); + + private MemoryUsageCalculator() {} + + @Override + public double calculateResourceUsage(List tasks) { + return tasks.stream().mapToDouble(this::calculateTaskResourceUsage).sum(); + } + + @Override + public double calculateTaskResourceUsage(QueryGroupTask task) { + return (1.0f * task.getTotalResourceUtilization(ResourceStats.MEMORY)) / HEAP_SIZE_BYTES; + } +} diff --git a/server/src/main/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerService.java b/server/src/main/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerService.java index 15852b5bbe6a8..b23d9ff342139 100644 --- a/server/src/main/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerService.java +++ b/server/src/main/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerService.java @@ -8,7 +8,6 @@ package org.opensearch.wlm.tracker; -import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskResourceTrackingService; import org.opensearch.wlm.QueryGroupLevelResourceUsageView; import org.opensearch.wlm.QueryGroupTask; @@ -25,7 +24,6 @@ * This class tracks resource usage per QueryGroup */ public class QueryGroupResourceUsageTrackerService { - public static final EnumSet TRACKED_RESOURCES = EnumSet.allOf(ResourceType.class); private final TaskResourceTrackingService taskResourceTrackingService; @@ -44,19 +42,16 @@ public QueryGroupResourceUsageTrackerService(TaskResourceTrackingService taskRes * @return Map of QueryGroup views */ public Map constructQueryGroupLevelUsageViews() { - final Map> tasksByQueryGroup = getTasksGroupedByQueryGroup(); + final Map> tasksByQueryGroup = getTasksGroupedByQueryGroup(); final Map queryGroupViews = new HashMap<>(); // Iterate over each QueryGroup entry - for (Map.Entry> queryGroupEntry : tasksByQueryGroup.entrySet()) { - // Compute the QueryGroup usage - final EnumMap queryGroupUsage = new EnumMap<>(ResourceType.class); + for (Map.Entry> queryGroupEntry : tasksByQueryGroup.entrySet()) { + // Compute the QueryGroup resource usage + final Map queryGroupUsage = new EnumMap<>(ResourceType.class); for (ResourceType resourceType : TRACKED_RESOURCES) { - long queryGroupResourceUsage = 0; - for (Task task : queryGroupEntry.getValue()) { - queryGroupResourceUsage += resourceType.getResourceUsage(task); - } - queryGroupUsage.put(resourceType, queryGroupResourceUsage); + double usage = resourceType.getResourceUsageCalculator().calculateResourceUsage(queryGroupEntry.getValue()); + queryGroupUsage.put(resourceType, usage); } // Add to the QueryGroup View @@ -73,12 +68,12 @@ public Map constructQueryGroupLevelUsa * * @return Map of tasks grouped by QueryGroup */ - private Map> getTasksGroupedByQueryGroup() { + private Map> getTasksGroupedByQueryGroup() { return taskResourceTrackingService.getResourceAwareTasks() .values() .stream() .filter(QueryGroupTask.class::isInstance) .map(QueryGroupTask.class::cast) - .collect(Collectors.groupingBy(QueryGroupTask::getQueryGroupId, Collectors.mapping(task -> (Task) task, Collectors.toList()))); + .collect(Collectors.groupingBy(QueryGroupTask::getQueryGroupId, Collectors.mapping(task -> task, Collectors.toList()))); } } diff --git a/server/src/main/java/org/opensearch/wlm/tracker/ResourceUsageCalculator.java b/server/src/main/java/org/opensearch/wlm/tracker/ResourceUsageCalculator.java new file mode 100644 index 0000000000000..bc8317cbfbf92 --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/tracker/ResourceUsageCalculator.java @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.tracker; + +import org.opensearch.common.annotation.PublicApi; +import org.opensearch.wlm.QueryGroupTask; + +import java.util.List; + +/** + * This class is used to track query group level resource usage + */ +@PublicApi(since = "2.18.0") +public abstract class ResourceUsageCalculator { + /** + * calculates the current resource usage for the query group + * + * @param tasks list of tasks in the query group + */ + public abstract double calculateResourceUsage(List tasks); + + /** + * calculates the task level resource usage + * @param task QueryGroupTask + * @return task level resource usage + */ + public abstract double calculateTaskResourceUsage(QueryGroupTask task); +} diff --git a/server/src/test/java/org/opensearch/wlm/QueryGroupLevelResourceUsageViewTests.java b/server/src/test/java/org/opensearch/wlm/QueryGroupLevelResourceUsageViewTests.java index 532bf3de95bd6..0c7eb721806d5 100644 --- a/server/src/test/java/org/opensearch/wlm/QueryGroupLevelResourceUsageViewTests.java +++ b/server/src/test/java/org/opensearch/wlm/QueryGroupLevelResourceUsageViewTests.java @@ -8,23 +8,34 @@ package org.opensearch.wlm; -import org.opensearch.action.search.SearchAction; -import org.opensearch.core.tasks.TaskId; -import org.opensearch.tasks.Task; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.wlm.tracker.ResourceUsageCalculatorTrackerServiceTests; -import java.util.Collections; import java.util.List; import java.util.Map; +import static org.opensearch.wlm.cancellation.QueryGroupTaskCancellationService.MIN_VALUE; +import static org.opensearch.wlm.tracker.CpuUsageCalculator.PROCESSOR_COUNT; +import static org.opensearch.wlm.tracker.MemoryUsageCalculator.HEAP_SIZE_BYTES; +import static org.opensearch.wlm.tracker.ResourceUsageCalculatorTests.createMockTaskWithResourceStats; +import static org.mockito.Mockito.mock; + public class QueryGroupLevelResourceUsageViewTests extends OpenSearchTestCase { - Map resourceUsage; - List activeTasks; + Map resourceUsage; + List activeTasks; + ResourceUsageCalculatorTrackerServiceTests.TestClock clock; + WorkloadManagementSettings settings; public void setUp() throws Exception { super.setUp(); - resourceUsage = Map.of(ResourceType.fromName("memory"), 34L, ResourceType.fromName("cpu"), 12L); - activeTasks = List.of(getRandomTask(4321)); + settings = mock(WorkloadManagementSettings.class); + clock = new ResourceUsageCalculatorTrackerServiceTests.TestClock(); + activeTasks = List.of(createMockTaskWithResourceStats(QueryGroupTask.class, 100, 200, 0, 1)); + clock.fastForwardBy(300); + double memoryUsage = 200.0 / HEAP_SIZE_BYTES; + double cpuUsage = 100.0 / (PROCESSOR_COUNT * 300.0); + + resourceUsage = Map.of(ResourceType.MEMORY, memoryUsage, ResourceType.CPU, cpuUsage); } public void testGetResourceUsageData() { @@ -32,7 +43,7 @@ public void testGetResourceUsageData() { resourceUsage, activeTasks ); - Map resourceUsageData = queryGroupLevelResourceUsageView.getResourceUsageData(); + Map resourceUsageData = queryGroupLevelResourceUsageView.getResourceUsageData(); assertTrue(assertResourceUsageData(resourceUsageData)); } @@ -41,23 +52,13 @@ public void testGetActiveTasks() { resourceUsage, activeTasks ); - List activeTasks = queryGroupLevelResourceUsageView.getActiveTasks(); + List activeTasks = queryGroupLevelResourceUsageView.getActiveTasks(); assertEquals(1, activeTasks.size()); - assertEquals(4321, activeTasks.get(0).getId()); + assertEquals(1, activeTasks.get(0).getId()); } - private boolean assertResourceUsageData(Map resourceUsageData) { - return resourceUsageData.get(ResourceType.fromName("memory")) == 34L && resourceUsageData.get(ResourceType.fromName("cpu")) == 12L; - } - - private Task getRandomTask(long id) { - return new Task( - id, - "transport", - SearchAction.NAME, - "test description", - new TaskId(randomLong() + ":" + randomLong()), - Collections.emptyMap() - ); + private boolean assertResourceUsageData(Map resourceUsageData) { + return (resourceUsageData.get(ResourceType.MEMORY) - 200.0 / HEAP_SIZE_BYTES) <= MIN_VALUE + && (resourceUsageData.get(ResourceType.CPU) - 100.0 / (300)) < MIN_VALUE; } } diff --git a/server/src/test/java/org/opensearch/wlm/ResourceTypeTests.java b/server/src/test/java/org/opensearch/wlm/ResourceTypeTests.java index 737cbb37b554c..16bd8b7e66266 100644 --- a/server/src/test/java/org/opensearch/wlm/ResourceTypeTests.java +++ b/server/src/test/java/org/opensearch/wlm/ResourceTypeTests.java @@ -8,14 +8,8 @@ package org.opensearch.wlm; -import org.opensearch.action.search.SearchShardTask; -import org.opensearch.core.tasks.resourcetracker.ResourceStats; -import org.opensearch.tasks.CancellableTask; import org.opensearch.test.OpenSearchTestCase; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public class ResourceTypeTests extends OpenSearchTestCase { public void testFromName() { @@ -35,17 +29,4 @@ public void testGetName() { assertEquals("cpu", ResourceType.CPU.getName()); assertEquals("memory", ResourceType.MEMORY.getName()); } - - public void testGetResourceUsage() { - SearchShardTask mockTask = createMockTask(SearchShardTask.class, 100, 200); - assertEquals(100, ResourceType.CPU.getResourceUsage(mockTask)); - assertEquals(200, ResourceType.MEMORY.getResourceUsage(mockTask)); - } - - private T createMockTask(Class type, long cpuUsage, long heapUsage) { - T task = mock(type); - when(task.getTotalResourceUtilization(ResourceStats.CPU)).thenReturn(cpuUsage); - when(task.getTotalResourceUtilization(ResourceStats.MEMORY)).thenReturn(heapUsage); - return task; - } } diff --git a/server/src/test/java/org/opensearch/wlm/cancellation/MaximumResourceTaskSelectionStrategyTests.java b/server/src/test/java/org/opensearch/wlm/cancellation/MaximumResourceTaskSelectionStrategyTests.java new file mode 100644 index 0000000000000..dc79822c59c49 --- /dev/null +++ b/server/src/test/java/org/opensearch/wlm/cancellation/MaximumResourceTaskSelectionStrategyTests.java @@ -0,0 +1,126 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.cancellation; + +import org.opensearch.action.search.SearchAction; +import org.opensearch.action.search.SearchTask; +import org.opensearch.core.tasks.TaskId; +import org.opensearch.core.tasks.resourcetracker.ResourceStats; +import org.opensearch.core.tasks.resourcetracker.ResourceStatsType; +import org.opensearch.core.tasks.resourcetracker.ResourceUsageMetric; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.wlm.QueryGroupTask; +import org.opensearch.wlm.ResourceType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.IntStream; + +import static org.opensearch.wlm.cancellation.QueryGroupTaskCancellationService.MIN_VALUE; +import static org.opensearch.wlm.tracker.MemoryUsageCalculator.HEAP_SIZE_BYTES; + +public class MaximumResourceTaskSelectionStrategyTests extends OpenSearchTestCase { + + public void testSelectTasksToCancelSelectsTasksMeetingThreshold_ifReduceByIsGreaterThanZero() { + MaximumResourceTaskSelectionStrategy testHighestResourceConsumingTaskFirstSelectionStrategy = + new MaximumResourceTaskSelectionStrategy(); + double reduceBy = 50000.0 / HEAP_SIZE_BYTES; + ResourceType resourceType = ResourceType.MEMORY; + List tasks = getListOfTasks(100); + List selectedTasks = testHighestResourceConsumingTaskFirstSelectionStrategy.selectTasksForCancellation( + tasks, + reduceBy, + resourceType + ); + assertFalse(selectedTasks.isEmpty()); + boolean sortedInDescendingResourceUsage = IntStream.range(0, selectedTasks.size() - 1) + .noneMatch( + index -> ResourceType.MEMORY.getResourceUsageCalculator() + .calculateTaskResourceUsage(selectedTasks.get(index)) < ResourceType.MEMORY.getResourceUsageCalculator() + .calculateTaskResourceUsage(selectedTasks.get(index + 1)) + ); + assertTrue(sortedInDescendingResourceUsage); + assertTrue(tasksUsageMeetsThreshold(selectedTasks, reduceBy)); + } + + public void testSelectTasksToCancelSelectsTasksMeetingThreshold_ifReduceByIsLesserThanZero() { + MaximumResourceTaskSelectionStrategy testHighestResourceConsumingTaskFirstSelectionStrategy = + new MaximumResourceTaskSelectionStrategy(); + double reduceBy = -50.0 / HEAP_SIZE_BYTES; + ResourceType resourceType = ResourceType.MEMORY; + List tasks = getListOfTasks(3); + try { + testHighestResourceConsumingTaskFirstSelectionStrategy.selectTasksForCancellation(tasks, reduceBy, resourceType); + } catch (Exception e) { + assertTrue(e instanceof IllegalArgumentException); + assertEquals("limit has to be greater than zero", e.getMessage()); + } + } + + public void testSelectTasksToCancelSelectsTasksMeetingThreshold_ifReduceByIsEqualToZero() { + MaximumResourceTaskSelectionStrategy testHighestResourceConsumingTaskFirstSelectionStrategy = + new MaximumResourceTaskSelectionStrategy(); + double reduceBy = 0.0; + ResourceType resourceType = ResourceType.MEMORY; + List tasks = getListOfTasks(50); + List selectedTasks = testHighestResourceConsumingTaskFirstSelectionStrategy.selectTasksForCancellation( + tasks, + reduceBy, + resourceType + ); + assertTrue(selectedTasks.isEmpty()); + } + + private boolean tasksUsageMeetsThreshold(List selectedTasks, double threshold) { + double memory = 0; + for (QueryGroupTask task : selectedTasks) { + memory += ResourceType.MEMORY.getResourceUsageCalculator().calculateTaskResourceUsage(task); + if ((memory - threshold) > MIN_VALUE) { + return true; + } + } + return false; + } + + private List getListOfTasks(int numberOfTasks) { + List tasks = new ArrayList<>(); + + while (tasks.size() < numberOfTasks) { + long id = randomLong(); + final QueryGroupTask task = getRandomSearchTask(id); + long initial_memory = randomLongBetween(1, 100); + + ResourceUsageMetric[] initialTaskResourceMetrics = new ResourceUsageMetric[] { + new ResourceUsageMetric(ResourceStats.MEMORY, initial_memory) }; + task.startThreadResourceTracking(id, ResourceStatsType.WORKER_STATS, initialTaskResourceMetrics); + + long memory = initial_memory + randomLongBetween(1, 10000); + + ResourceUsageMetric[] taskResourceMetrics = new ResourceUsageMetric[] { + new ResourceUsageMetric(ResourceStats.MEMORY, memory), }; + task.updateThreadResourceStats(id, ResourceStatsType.WORKER_STATS, taskResourceMetrics); + task.stopThreadResourceTracking(id, ResourceStatsType.WORKER_STATS); + tasks.add(task); + } + + return tasks; + } + + private QueryGroupTask getRandomSearchTask(long id) { + return new SearchTask( + id, + "transport", + SearchAction.NAME, + () -> "test description", + new TaskId(randomLong() + ":" + randomLong()), + Collections.emptyMap() + ); + } +} diff --git a/server/src/test/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationServiceTests.java b/server/src/test/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationServiceTests.java new file mode 100644 index 0000000000000..f7a49235efc69 --- /dev/null +++ b/server/src/test/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationServiceTests.java @@ -0,0 +1,541 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.cancellation; + +import org.opensearch.action.search.SearchAction; +import org.opensearch.cluster.metadata.QueryGroup; +import org.opensearch.core.tasks.TaskId; +import org.opensearch.tasks.TaskCancellation; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.wlm.MutableQueryGroupFragment; +import org.opensearch.wlm.MutableQueryGroupFragment.ResiliencyMode; +import org.opensearch.wlm.QueryGroupLevelResourceUsageView; +import org.opensearch.wlm.QueryGroupTask; +import org.opensearch.wlm.ResourceType; +import org.opensearch.wlm.WorkloadManagementSettings; +import org.opensearch.wlm.tracker.QueryGroupResourceUsageTrackerService; +import org.opensearch.wlm.tracker.ResourceUsageCalculatorTrackerServiceTests.TestClock; +import org.junit.Before; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class QueryGroupTaskCancellationServiceTests extends OpenSearchTestCase { + private static final String queryGroupId1 = "queryGroup1"; + private static final String queryGroupId2 = "queryGroup2"; + + private TestClock clock; + + private Map queryGroupLevelViews; + private Set activeQueryGroups; + private Set deletedQueryGroups; + private QueryGroupTaskCancellationService taskCancellation; + private WorkloadManagementSettings workloadManagementSettings; + private QueryGroupResourceUsageTrackerService resourceUsageTrackerService; + + @Before + public void setup() { + workloadManagementSettings = mock(WorkloadManagementSettings.class); + queryGroupLevelViews = new HashMap<>(); + activeQueryGroups = new HashSet<>(); + deletedQueryGroups = new HashSet<>(); + + clock = new TestClock(); + when(workloadManagementSettings.getNodeLevelCpuCancellationThreshold()).thenReturn(0.9); + when(workloadManagementSettings.getNodeLevelMemoryCancellationThreshold()).thenReturn(0.9); + resourceUsageTrackerService = mock(QueryGroupResourceUsageTrackerService.class); + taskCancellation = new QueryGroupTaskCancellationService( + workloadManagementSettings, + new MaximumResourceTaskSelectionStrategy(), + resourceUsageTrackerService, + activeQueryGroups, + deletedQueryGroups + ); + } + + public void testGetCancellableTasksFrom_setupAppropriateCancellationReasonAndScore() { + ResourceType resourceType = ResourceType.CPU; + double cpuUsage = 0.11; + double memoryUsage = 0.0; + Double threshold = 0.1; + + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + clock.fastForwardBy(1000); + + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + when(mockView.getResourceUsageData()).thenReturn(Map.of(resourceType, cpuUsage, ResourceType.MEMORY, memoryUsage)); + queryGroupLevelViews.put(queryGroupId1, mockView); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(List.of(queryGroup1)); + assertEquals(2, cancellableTasksFrom.size()); + assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); + assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); + assertEquals(1, cancellableTasksFrom.get(0).getReasons().get(0).getCancellationScore()); + } + + public void testGetCancellableTasksFrom_returnsTasksWhenBreachingThreshold() { + ResourceType resourceType = ResourceType.CPU; + double cpuUsage = 0.11; + double memoryUsage = 0.0; + Double threshold = 0.1; + + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + when(mockView.getResourceUsageData()).thenReturn(Map.of(resourceType, cpuUsage, ResourceType.MEMORY, memoryUsage)); + queryGroupLevelViews.put(queryGroupId1, mockView); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(List.of(queryGroup1)); + assertEquals(2, cancellableTasksFrom.size()); + assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); + assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); + } + + public void testGetCancellableTasksFrom_returnsTasksWhenBreachingThresholdForMemory() { + ResourceType resourceType = ResourceType.MEMORY; + double cpuUsage = 0.0; + double memoryUsage = 0.11; + Double threshold = 0.1; + + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + when(mockView.getResourceUsageData()).thenReturn(Map.of(ResourceType.CPU, cpuUsage, resourceType, memoryUsage)); + + queryGroupLevelViews.put(queryGroupId1, mockView); + activeQueryGroups.add(queryGroup1); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + assertEquals(2, cancellableTasksFrom.size()); + assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); + assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); + } + + public void testGetCancellableTasksFrom_returnsNoTasksWhenNotBreachingThreshold() { + ResourceType resourceType = ResourceType.CPU; + double cpuUsage = 0.81; + double memoryUsage = 0.0; + Double threshold = 0.9; + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + when(mockView.getResourceUsageData()).thenReturn(Map.of(ResourceType.CPU, cpuUsage, ResourceType.MEMORY, memoryUsage)); + queryGroupLevelViews.put(queryGroupId1, mockView); + activeQueryGroups.add(queryGroup1); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(List.of(queryGroup1)); + assertTrue(cancellableTasksFrom.isEmpty()); + } + + public void testGetCancellableTasksFrom_filtersQueryGroupCorrectly() { + ResourceType resourceType = ResourceType.CPU; + double usage = 0.02; + Double threshold = 0.01; + + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + queryGroupLevelViews.put(queryGroupId1, mockView); + activeQueryGroups.add(queryGroup1); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + QueryGroupTaskCancellationService taskCancellation = new QueryGroupTaskCancellationService( + workloadManagementSettings, + new MaximumResourceTaskSelectionStrategy(), + resourceUsageTrackerService, + activeQueryGroups, + deletedQueryGroups + ); + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.SOFT); + assertEquals(0, cancellableTasksFrom.size()); + } + + public void testCancelTasks_cancelsGivenTasks() { + ResourceType resourceType = ResourceType.CPU; + double cpuUsage = 0.011; + double memoryUsage = 0.011; + + Double threshold = 0.01; + + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold, ResourceType.MEMORY, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + when(mockView.getResourceUsageData()).thenReturn(Map.of(ResourceType.CPU, cpuUsage, ResourceType.MEMORY, memoryUsage)); + + queryGroupLevelViews.put(queryGroupId1, mockView); + activeQueryGroups.add(queryGroup1); + + QueryGroupTaskCancellationService taskCancellation = new QueryGroupTaskCancellationService( + workloadManagementSettings, + new MaximumResourceTaskSelectionStrategy(), + resourceUsageTrackerService, + activeQueryGroups, + deletedQueryGroups + ); + + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + assertEquals(2, cancellableTasksFrom.size()); + assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); + assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); + + when(resourceUsageTrackerService.constructQueryGroupLevelUsageViews()).thenReturn(queryGroupLevelViews); + taskCancellation.cancelTasks(() -> false); + assertTrue(cancellableTasksFrom.get(0).getTask().isCancelled()); + assertTrue(cancellableTasksFrom.get(1).getTask().isCancelled()); + } + + public void testCancelTasks_cancelsTasksFromDeletedQueryGroups() { + ResourceType resourceType = ResourceType.CPU; + double activeQueryGroupCpuUsage = 0.01; + double activeQueryGroupMemoryUsage = 0.0; + double deletedQueryGroupCpuUsage = 0.01; + double deletedQueryGroupMemoryUsage = 0.0; + Double threshold = 0.01; + + QueryGroup activeQueryGroup = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroup deletedQueryGroup = new QueryGroup( + "testQueryGroup", + queryGroupId2, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView1 = createResourceUsageViewMock(); + QueryGroupLevelResourceUsageView mockView2 = createResourceUsageViewMock( + resourceType, + deletedQueryGroupCpuUsage, + List.of(1000, 1001) + ); + + when(mockView1.getResourceUsageData()).thenReturn( + Map.of(ResourceType.CPU, activeQueryGroupCpuUsage, ResourceType.MEMORY, activeQueryGroupMemoryUsage) + ); + when(mockView2.getResourceUsageData()).thenReturn( + Map.of(ResourceType.CPU, deletedQueryGroupCpuUsage, ResourceType.MEMORY, deletedQueryGroupMemoryUsage) + ); + queryGroupLevelViews.put(queryGroupId1, mockView1); + queryGroupLevelViews.put(queryGroupId2, mockView2); + + activeQueryGroups.add(activeQueryGroup); + deletedQueryGroups.add(deletedQueryGroup); + + QueryGroupTaskCancellationService taskCancellation = new QueryGroupTaskCancellationService( + workloadManagementSettings, + new MaximumResourceTaskSelectionStrategy(), + resourceUsageTrackerService, + activeQueryGroups, + deletedQueryGroups + ); + + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + assertEquals(2, cancellableTasksFrom.size()); + assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); + assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); + + List cancellableTasksFromDeletedQueryGroups = taskCancellation.getAllCancellableTasks(List.of(deletedQueryGroup)); + assertEquals(2, cancellableTasksFromDeletedQueryGroups.size()); + assertEquals(1000, cancellableTasksFromDeletedQueryGroups.get(0).getTask().getId()); + assertEquals(1001, cancellableTasksFromDeletedQueryGroups.get(1).getTask().getId()); + + when(resourceUsageTrackerService.constructQueryGroupLevelUsageViews()).thenReturn(queryGroupLevelViews); + taskCancellation.cancelTasks(() -> true); + + assertTrue(cancellableTasksFrom.get(0).getTask().isCancelled()); + assertTrue(cancellableTasksFrom.get(1).getTask().isCancelled()); + assertTrue(cancellableTasksFromDeletedQueryGroups.get(0).getTask().isCancelled()); + assertTrue(cancellableTasksFromDeletedQueryGroups.get(1).getTask().isCancelled()); + } + + public void testCancelTasks_does_not_cancelTasksFromDeletedQueryGroups_whenNodeNotInDuress() { + ResourceType resourceType = ResourceType.CPU; + double activeQueryGroupCpuUsage = 0.11; + double activeQueryGroupMemoryUsage = 0.0; + double deletedQueryGroupCpuUsage = 0.11; + double deletedQueryGroupMemoryUsage = 0.0; + + Double threshold = 0.01; + + QueryGroup activeQueryGroup = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroup deletedQueryGroup = new QueryGroup( + "testQueryGroup", + queryGroupId2, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView1 = createResourceUsageViewMock(); + QueryGroupLevelResourceUsageView mockView2 = createResourceUsageViewMock( + resourceType, + deletedQueryGroupCpuUsage, + List.of(1000, 1001) + ); + + when(mockView1.getResourceUsageData()).thenReturn( + Map.of(ResourceType.CPU, activeQueryGroupCpuUsage, ResourceType.MEMORY, activeQueryGroupMemoryUsage) + ); + when(mockView2.getResourceUsageData()).thenReturn( + Map.of(ResourceType.CPU, deletedQueryGroupCpuUsage, ResourceType.MEMORY, deletedQueryGroupMemoryUsage) + ); + + queryGroupLevelViews.put(queryGroupId1, mockView1); + queryGroupLevelViews.put(queryGroupId2, mockView2); + activeQueryGroups.add(activeQueryGroup); + deletedQueryGroups.add(deletedQueryGroup); + + QueryGroupTaskCancellationService taskCancellation = new QueryGroupTaskCancellationService( + workloadManagementSettings, + new MaximumResourceTaskSelectionStrategy(), + resourceUsageTrackerService, + activeQueryGroups, + deletedQueryGroups + ); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + assertEquals(2, cancellableTasksFrom.size()); + assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); + assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); + + List cancellableTasksFromDeletedQueryGroups = taskCancellation.getAllCancellableTasks(List.of(deletedQueryGroup)); + assertEquals(2, cancellableTasksFromDeletedQueryGroups.size()); + assertEquals(1000, cancellableTasksFromDeletedQueryGroups.get(0).getTask().getId()); + assertEquals(1001, cancellableTasksFromDeletedQueryGroups.get(1).getTask().getId()); + + when(resourceUsageTrackerService.constructQueryGroupLevelUsageViews()).thenReturn(queryGroupLevelViews); + taskCancellation.cancelTasks(() -> false); + + assertTrue(cancellableTasksFrom.get(0).getTask().isCancelled()); + assertTrue(cancellableTasksFrom.get(1).getTask().isCancelled()); + assertFalse(cancellableTasksFromDeletedQueryGroups.get(0).getTask().isCancelled()); + assertFalse(cancellableTasksFromDeletedQueryGroups.get(1).getTask().isCancelled()); + } + + public void testCancelTasks_cancelsGivenTasks_WhenNodeInDuress() { + ResourceType resourceType = ResourceType.CPU; + double cpuUsage1 = 0.11; + double memoryUsage1 = 0.0; + double cpuUsage2 = 0.11; + double memoryUsage2 = 0.0; + Double threshold = 0.01; + + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroup queryGroup2 = new QueryGroup( + "testQueryGroup", + queryGroupId2, + new MutableQueryGroupFragment(ResiliencyMode.SOFT, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView1 = createResourceUsageViewMock(); + when(mockView1.getResourceUsageData()).thenReturn(Map.of(ResourceType.CPU, cpuUsage1, ResourceType.MEMORY, memoryUsage1)); + queryGroupLevelViews.put(queryGroupId1, mockView1); + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + when(mockView.getActiveTasks()).thenReturn(List.of(getRandomSearchTask(5678), getRandomSearchTask(8765))); + when(mockView.getResourceUsageData()).thenReturn(Map.of(ResourceType.CPU, cpuUsage2, ResourceType.MEMORY, memoryUsage2)); + queryGroupLevelViews.put(queryGroupId2, mockView); + Collections.addAll(activeQueryGroups, queryGroup1, queryGroup2); + + QueryGroupTaskCancellationService taskCancellation = new QueryGroupTaskCancellationService( + workloadManagementSettings, + new MaximumResourceTaskSelectionStrategy(), + resourceUsageTrackerService, + activeQueryGroups, + deletedQueryGroups + ); + + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + assertEquals(2, cancellableTasksFrom.size()); + assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); + assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); + + List cancellableTasksFrom1 = taskCancellation.getAllCancellableTasks(ResiliencyMode.SOFT); + assertEquals(2, cancellableTasksFrom1.size()); + assertEquals(5678, cancellableTasksFrom1.get(0).getTask().getId()); + assertEquals(8765, cancellableTasksFrom1.get(1).getTask().getId()); + + when(resourceUsageTrackerService.constructQueryGroupLevelUsageViews()).thenReturn(queryGroupLevelViews); + taskCancellation.cancelTasks(() -> true); + assertTrue(cancellableTasksFrom.get(0).getTask().isCancelled()); + assertTrue(cancellableTasksFrom.get(1).getTask().isCancelled()); + assertTrue(cancellableTasksFrom1.get(0).getTask().isCancelled()); + assertTrue(cancellableTasksFrom1.get(1).getTask().isCancelled()); + } + + public void testGetAllCancellableTasks_ReturnsNoTasksWhenNotBreachingThresholds() { + ResourceType resourceType = ResourceType.CPU; + double queryGroupCpuUsage = 0.09; + double queryGroupMemoryUsage = 0.0; + Double threshold = 0.1; + + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + when(mockView.getResourceUsageData()).thenReturn( + Map.of(ResourceType.CPU, queryGroupCpuUsage, ResourceType.MEMORY, queryGroupMemoryUsage) + ); + queryGroupLevelViews.put(queryGroupId1, mockView); + activeQueryGroups.add(queryGroup1); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List allCancellableTasks = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + assertTrue(allCancellableTasks.isEmpty()); + } + + public void testGetAllCancellableTasks_ReturnsTasksWhenBreachingThresholds() { + ResourceType resourceType = ResourceType.CPU; + double cpuUsage = 0.11; + double memoryUsage = 0.0; + Double threshold = 0.01; + + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + when(mockView.getResourceUsageData()).thenReturn(Map.of(ResourceType.CPU, cpuUsage, ResourceType.MEMORY, memoryUsage)); + queryGroupLevelViews.put(queryGroupId1, mockView); + activeQueryGroups.add(queryGroup1); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List allCancellableTasks = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + assertEquals(2, allCancellableTasks.size()); + assertEquals(1234, allCancellableTasks.get(0).getTask().getId()); + assertEquals(4321, allCancellableTasks.get(1).getTask().getId()); + } + + public void testGetCancellableTasksFrom_doesNotReturnTasksWhenQueryGroupIdNotFound() { + ResourceType resourceType = ResourceType.CPU; + double usage = 0.11; + Double threshold = 0.01; + + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup1", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + QueryGroup queryGroup2 = new QueryGroup( + "testQueryGroup2", + queryGroupId2, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + queryGroupLevelViews.put(queryGroupId1, mockView); + activeQueryGroups.add(queryGroup1); + activeQueryGroups.add(queryGroup2); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(List.of(queryGroup2)); + assertEquals(0, cancellableTasksFrom.size()); + } + + private QueryGroupLevelResourceUsageView createResourceUsageViewMock() { + QueryGroupLevelResourceUsageView mockView = mock(QueryGroupLevelResourceUsageView.class); + when(mockView.getActiveTasks()).thenReturn(List.of(getRandomSearchTask(1234), getRandomSearchTask(4321))); + return mockView; + } + + private QueryGroupLevelResourceUsageView createResourceUsageViewMock(ResourceType resourceType, double usage, Collection ids) { + QueryGroupLevelResourceUsageView mockView = mock(QueryGroupLevelResourceUsageView.class); + when(mockView.getResourceUsageData()).thenReturn(Collections.singletonMap(resourceType, usage)); + when(mockView.getActiveTasks()).thenReturn(ids.stream().map(this::getRandomSearchTask).collect(Collectors.toList())); + return mockView; + } + + private QueryGroupTask getRandomSearchTask(long id) { + return new QueryGroupTask( + id, + "transport", + SearchAction.NAME, + "test description", + new TaskId(randomLong() + ":" + randomLong()), + Collections.emptyMap(), + null, + clock::getTime + ); + } +} diff --git a/server/src/test/java/org/opensearch/wlm/tracker/ResourceUsageCalculatorTests.java b/server/src/test/java/org/opensearch/wlm/tracker/ResourceUsageCalculatorTests.java new file mode 100644 index 0000000000000..21d9717a1aaca --- /dev/null +++ b/server/src/test/java/org/opensearch/wlm/tracker/ResourceUsageCalculatorTests.java @@ -0,0 +1,70 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.tracker; + +import org.opensearch.core.tasks.resourcetracker.ResourceStats; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.wlm.QueryGroupTask; +import org.opensearch.wlm.ResourceType; +import org.opensearch.wlm.tracker.ResourceUsageCalculatorTrackerServiceTests.TestClock; + +import java.util.List; + +import static org.opensearch.wlm.cancellation.QueryGroupTaskCancellationService.MIN_VALUE; +import static org.opensearch.wlm.tracker.CpuUsageCalculator.PROCESSOR_COUNT; +import static org.opensearch.wlm.tracker.MemoryUsageCalculator.HEAP_SIZE_BYTES; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ResourceUsageCalculatorTests extends OpenSearchTestCase { + + public void testQueryGroupCpuUsage() { + TestClock clock = new TestClock(); + long fastForwardTime = PROCESSOR_COUNT * 200L; + clock.fastForwardBy(fastForwardTime); + + double expectedQueryGroupCpuUsage = 1.0 / PROCESSOR_COUNT; + + QueryGroupTask mockTask = createMockTaskWithResourceStats(QueryGroupTask.class, fastForwardTime, 200, 0, 123); + when(mockTask.getElapsedTime()).thenReturn(fastForwardTime); + double actualUsage = ResourceType.CPU.getResourceUsageCalculator().calculateResourceUsage(List.of(mockTask)); + assertEquals(expectedQueryGroupCpuUsage, actualUsage, MIN_VALUE); + + double taskResourceUsage = ResourceType.CPU.getResourceUsageCalculator().calculateTaskResourceUsage(mockTask); + assertEquals(1.0, taskResourceUsage, MIN_VALUE); + } + + public void testQueryGroupMemoryUsage() { + QueryGroupTask mockTask = createMockTaskWithResourceStats(QueryGroupTask.class, 100, 200, 0, 123); + double actualMemoryUsage = ResourceType.MEMORY.getResourceUsageCalculator().calculateResourceUsage(List.of(mockTask)); + double expectedMemoryUsage = 200.0 / HEAP_SIZE_BYTES; + + assertEquals(expectedMemoryUsage, actualMemoryUsage, MIN_VALUE); + assertEquals( + 200.0 / HEAP_SIZE_BYTES, + ResourceType.MEMORY.getResourceUsageCalculator().calculateTaskResourceUsage(mockTask), + MIN_VALUE + ); + } + + public static T createMockTaskWithResourceStats( + Class type, + long cpuUsage, + long heapUsage, + long startTimeNanos, + long taskId + ) { + T task = mock(type); + when(task.getTotalResourceUtilization(ResourceStats.CPU)).thenReturn(cpuUsage); + when(task.getTotalResourceUtilization(ResourceStats.MEMORY)).thenReturn(heapUsage); + when(task.getStartTimeNanos()).thenReturn(startTimeNanos); + when(task.getId()).thenReturn(taskId); + return task; + } +} diff --git a/server/src/test/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerServiceTests.java b/server/src/test/java/org/opensearch/wlm/tracker/ResourceUsageCalculatorTrackerServiceTests.java similarity index 69% rename from server/src/test/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerServiceTests.java rename to server/src/test/java/org/opensearch/wlm/tracker/ResourceUsageCalculatorTrackerServiceTests.java index ca2891cb532f2..fe72bd6e710c8 100644 --- a/server/src/test/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerServiceTests.java +++ b/server/src/test/java/org/opensearch/wlm/tracker/ResourceUsageCalculatorTrackerServiceTests.java @@ -9,10 +9,8 @@ package org.opensearch.wlm.tracker; import org.opensearch.action.search.SearchShardTask; -import org.opensearch.action.search.SearchTask; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.tasks.resourcetracker.ResourceStats; -import org.opensearch.tasks.CancellableTask; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskResourceTrackingService; import org.opensearch.test.OpenSearchTestCase; @@ -21,6 +19,7 @@ import org.opensearch.wlm.QueryGroupLevelResourceUsageView; import org.opensearch.wlm.QueryGroupTask; import org.opensearch.wlm.ResourceType; +import org.opensearch.wlm.WorkloadManagementSettings; import org.junit.After; import org.junit.Before; @@ -31,18 +30,38 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.opensearch.wlm.QueryGroupTask.QUERY_GROUP_ID_HEADER; +import static org.opensearch.wlm.cancellation.QueryGroupTaskCancellationService.MIN_VALUE; +import static org.opensearch.wlm.tracker.CpuUsageCalculator.PROCESSOR_COUNT; +import static org.opensearch.wlm.tracker.MemoryUsageCalculator.HEAP_SIZE_BYTES; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class QueryGroupResourceUsageTrackerServiceTests extends OpenSearchTestCase { +public class ResourceUsageCalculatorTrackerServiceTests extends OpenSearchTestCase { TestThreadPool threadPool; TaskResourceTrackingService mockTaskResourceTrackingService; QueryGroupResourceUsageTrackerService queryGroupResourceUsageTrackerService; + WorkloadManagementSettings settings; + + public static class TestClock { + long time; + + public void fastForwardBy(long nanos) { + time += nanos; + } + + public long getTime() { + return time; + } + } + + TestClock clock; @Before public void setup() { + clock = new TestClock(); + settings = mock(WorkloadManagementSettings.class); threadPool = new TestThreadPool(getTestName()); mockTaskResourceTrackingService = mock(TaskResourceTrackingService.class); queryGroupResourceUsageTrackerService = new QueryGroupResourceUsageTrackerService(mockTaskResourceTrackingService); @@ -55,16 +74,24 @@ public void cleanup() { public void testConstructQueryGroupLevelViews_CreatesQueryGroupLevelUsageView_WhenTasksArePresent() { List queryGroupIds = List.of("queryGroup1", "queryGroup2", "queryGroup3"); + clock.fastForwardBy(2000); Map activeSearchShardTasks = createActiveSearchShardTasks(queryGroupIds); when(mockTaskResourceTrackingService.getResourceAwareTasks()).thenReturn(activeSearchShardTasks); + Map stringQueryGroupLevelResourceUsageViewMap = queryGroupResourceUsageTrackerService .constructQueryGroupLevelUsageViews(); for (String queryGroupId : queryGroupIds) { assertEquals( - 400, - (long) stringQueryGroupLevelResourceUsageViewMap.get(queryGroupId).getResourceUsageData().get(ResourceType.MEMORY) + (400 * 1.0f) / HEAP_SIZE_BYTES, + stringQueryGroupLevelResourceUsageViewMap.get(queryGroupId).getResourceUsageData().get(ResourceType.MEMORY), + MIN_VALUE + ); + assertEquals( + (200 * 1.0f) / (PROCESSOR_COUNT * 2000), + stringQueryGroupLevelResourceUsageViewMap.get(queryGroupId).getResourceUsageData().get(ResourceType.CPU), + MIN_VALUE ); assertEquals(2, stringQueryGroupLevelResourceUsageViewMap.get(queryGroupId).getActiveTasks().size()); } @@ -78,14 +105,23 @@ public void testConstructQueryGroupLevelViews_CreatesQueryGroupLevelUsageView_Wh public void testConstructQueryGroupLevelUsageViews_WithTasksHavingDifferentResourceUsage() { Map activeSearchShardTasks = new HashMap<>(); + clock.fastForwardBy(2000); activeSearchShardTasks.put(1L, createMockTask(SearchShardTask.class, 100, 200, "queryGroup1")); activeSearchShardTasks.put(2L, createMockTask(SearchShardTask.class, 200, 400, "queryGroup1")); when(mockTaskResourceTrackingService.getResourceAwareTasks()).thenReturn(activeSearchShardTasks); - Map queryGroupViews = queryGroupResourceUsageTrackerService .constructQueryGroupLevelUsageViews(); - assertEquals(600, (long) queryGroupViews.get("queryGroup1").getResourceUsageData().get(ResourceType.MEMORY)); + assertEquals( + (double) 600 / HEAP_SIZE_BYTES, + queryGroupViews.get("queryGroup1").getResourceUsageData().get(ResourceType.MEMORY), + MIN_VALUE + ); + assertEquals( + ((double) 300) / (PROCESSOR_COUNT * 2000), + queryGroupViews.get("queryGroup1").getResourceUsageData().get(ResourceType.CPU), + MIN_VALUE + ); assertEquals(2, queryGroupViews.get("queryGroup1").getActiveTasks().size()); } @@ -100,19 +136,16 @@ private Map createActiveSearchShardTasks(List queryGroupIds) return activeSearchShardTasks; } - private T createMockTask(Class type, long cpuUsage, long heapUsage, String queryGroupId) { + private T createMockTask(Class type, long cpuUsage, long heapUsage, String queryGroupId) { T task = mock(type); - if (task instanceof SearchTask || task instanceof SearchShardTask) { - // Stash the current thread context to ensure that any existing context is preserved and restored after setting the query group - // ID. - try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) { - threadPool.getThreadContext().putHeader(QUERY_GROUP_ID_HEADER, queryGroupId); - ((QueryGroupTask) task).setQueryGroupId(threadPool.getThreadContext()); - } + try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) { + threadPool.getThreadContext().putHeader(QUERY_GROUP_ID_HEADER, queryGroupId); + task.setQueryGroupId(threadPool.getThreadContext()); } when(task.getTotalResourceUtilization(ResourceStats.CPU)).thenReturn(cpuUsage); when(task.getTotalResourceUtilization(ResourceStats.MEMORY)).thenReturn(heapUsage); when(task.getStartTimeNanos()).thenReturn((long) 0); + when(task.getElapsedTime()).thenReturn(clock.getTime()); AtomicBoolean isCancelled = new AtomicBoolean(false); doAnswer(invocation -> { From 2e98df39c07bb816b50032f84ded2f62cf7fd09c Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Thu, 12 Sep 2024 01:28:56 +0530 Subject: [PATCH 14/44] Fixing the CHANGELOG.md by removing change published in 2.17 (#15903) Signed-off-by: Ankit Jain --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 610ebb304ba1b..add9fab2092bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added -- Adding WithFieldName interface for QueryBuilders with fieldName ([#15705](https://github.com/opensearch-project/OpenSearch/pull/15705)) - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) From 12ff5ed9c79aa66c8824e1df7e124756746c8c94 Mon Sep 17 00:00:00 2001 From: rajiv-kv <157019998+rajiv-kv@users.noreply.github.com> Date: Thu, 12 Sep 2024 11:54:18 +0530 Subject: [PATCH 15/44] using remote cluster-state as fallback (#15424) Signed-off-by: Rajiv Kumar Vaidyanathan --- CHANGELOG.md | 1 + .../RemoteClusterStateTermVersionIT.java | 212 ++++++++++++++++++ .../state/TransportClusterStateAction.java | 6 +- .../TransportClusterManagerNodeAction.java | 59 ++++- .../term/GetTermVersionResponse.java | 20 ++ .../term/TransportGetTermVersionAction.java | 28 ++- .../cluster/coordination/Coordinator.java | 9 + .../cluster/service/ClusterApplier.java | 6 + .../service/ClusterApplierService.java | 16 +- .../cluster/service/ClusterService.java | 8 + .../common/settings/FeatureFlagSettings.java | 3 +- .../opensearch/common/util/FeatureFlags.java | 9 +- .../remote/RemoteClusterStateCache.java | 58 +++++ .../remote/RemoteClusterStateService.java | 36 ++- .../gateway/remote/RemoteManifestManager.java | 23 +- ...ransportClusterManagerNodeActionTests.java | 92 ++++++++ .../term/ClusterTermVersionIT.java | 108 ++++++++- .../coordination/NoOpClusterApplier.java | 5 + .../RemoteClusterStateServiceTests.java | 169 +++++++++++++- .../snapshots/SnapshotResiliencyTests.java | 6 +- 20 files changed, 853 insertions(+), 21 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateTermVersionIT.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCache.java diff --git a/CHANGELOG.md b/CHANGELOG.md index add9fab2092bd..3268852cc99f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) +- Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateTermVersionIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateTermVersionIT.java new file mode 100644 index 0000000000000..256c2ef44b078 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateTermVersionIT.java @@ -0,0 +1,212 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.action.admin.cluster.state.ClusterStateAction; +import org.opensearch.action.admin.cluster.state.ClusterStateRequest; +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.action.support.clustermanager.term.GetTermVersionAction; +import org.opensearch.action.support.clustermanager.term.GetTermVersionResponse; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.coordination.ClusterStateTermVersion; +import org.opensearch.cluster.coordination.PublicationTransportHandler; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.transport.TransportResponse; +import org.opensearch.gateway.remote.model.RemoteRoutingTableBlobStore; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.remote.RemoteStoreEnums; +import org.opensearch.plugins.Plugin; +import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.transport.MockTransportService; +import org.opensearch.transport.TransportService; +import org.junit.Before; + +import java.nio.file.Path; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.hamcrest.Matchers.is; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class RemoteClusterStateTermVersionIT extends RemoteStoreBaseIntegTestCase { + private static final String INDEX_NAME = "test-index"; + private static final String INDEX_NAME_1 = "test-index-1"; + List indexRoutingPaths; + AtomicInteger indexRoutingFiles = new AtomicInteger(); + private final RemoteStoreEnums.PathType pathType = RemoteStoreEnums.PathType.HASHED_PREFIX; + + @Before + public void setup() { + asyncUploadMockFsRepo = false; + } + + protected Collection> nodePlugins() { + return List.of(MockTransportService.TestPlugin.class); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .put( + RemoteRoutingTableBlobStore.REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING.getKey(), + RemoteStoreEnums.PathType.HASHED_PREFIX.toString() + ) + .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, REMOTE_ROUTING_TABLE_REPO) + .put(REMOTE_PUBLICATION_SETTING_KEY, true) + .build(); + } + + public void testRemoteClusterStateFallback() throws Exception { + BlobStoreRepository repository = prepareClusterAndVerifyRepository(); + + RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance( + RemoteClusterStateService.class + ); + + RemoteManifestManager remoteManifestManager = remoteClusterStateService.getRemoteManifestManager(); + Optional latestManifest = remoteManifestManager.getLatestClusterMetadataManifest( + getClusterState().getClusterName().value(), + getClusterState().getMetadata().clusterUUID() + ); + + String[] dataNodes = internalCluster().getDataNodeNames().toArray(String[]::new); + MockTransportService primaryService = (MockTransportService) internalCluster().getInstance(TransportService.class, dataNodes[0]); + + String cm = internalCluster().getClusterManagerName(); + primaryService.addRequestHandlingBehavior( + PublicationTransportHandler.COMMIT_STATE_ACTION_NAME, + (handler, request, channel, task) -> { + // not committing the state + logger.info("ignoring the commit from cluster-manager {}", request); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + ); + + String index = "index_1"; + createIndex( + index, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), Long.MAX_VALUE) + .build() + ); + logger.info("created index {}", index); + Map callCounters = Map.ofEntries( + Map.entry(ClusterStateAction.NAME, new AtomicInteger()), + Map.entry(GetTermVersionAction.NAME, new AtomicInteger()) + ); + + addCallCountInterceptor(cm, callCounters); + + ClusterStateResponse stateResponseM = client(cm).admin().cluster().state(new ClusterStateRequest()).actionGet(); + + ClusterStateResponse stateResponseD = client(dataNodes[0]).admin().cluster().state(new ClusterStateRequest()).actionGet(); + assertEquals(stateResponseM, stateResponseD); + assertThat(callCounters.get(ClusterStateAction.NAME).get(), is(0)); + assertThat(callCounters.get(GetTermVersionAction.NAME).get(), is(1)); + + } + + public void testNoRemoteClusterStateFound() throws Exception { + BlobStoreRepository repository = prepareClusterAndVerifyRepository(); + + RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance( + RemoteClusterStateService.class + ); + + RemoteManifestManager remoteManifestManager = remoteClusterStateService.getRemoteManifestManager(); + Optional latestManifest = remoteManifestManager.getLatestClusterMetadataManifest( + getClusterState().getClusterName().value(), + getClusterState().getMetadata().clusterUUID() + ); + + String[] dataNodes = internalCluster().getDataNodeNames().toArray(String[]::new); + MockTransportService primaryService = (MockTransportService) internalCluster().getInstance(TransportService.class, dataNodes[0]); + primaryService.addRequestHandlingBehavior( + PublicationTransportHandler.COMMIT_STATE_ACTION_NAME, + (handler, request, channel, task) -> { + // not committing the state + logger.info("ignoring the commit from cluster-manager {}", request); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + ); + + ClusterState state = internalCluster().clusterService().state(); + String cm = internalCluster().getClusterManagerName(); + MockTransportService cmservice = (MockTransportService) internalCluster().getInstance(TransportService.class, cm); + cmservice.addRequestHandlingBehavior(GetTermVersionAction.NAME, (handler, request, channel, task) -> { + channel.sendResponse( + new GetTermVersionResponse(new ClusterStateTermVersion(state.getClusterName(), state.stateUUID(), -1, -1), true) + ); + }); + + Map callCounters = Map.ofEntries( + Map.entry(ClusterStateAction.NAME, new AtomicInteger()), + Map.entry(GetTermVersionAction.NAME, new AtomicInteger()) + ); + + addCallCountInterceptor(cm, callCounters); + + ClusterStateResponse stateResponseM = client(cm).admin().cluster().state(new ClusterStateRequest()).actionGet(); + ClusterStateResponse stateResponseD = client(dataNodes[0]).admin().cluster().state(new ClusterStateRequest()).actionGet(); + assertEquals(stateResponseM, stateResponseD); + assertThat(callCounters.get(ClusterStateAction.NAME).get(), is(1)); + assertThat(callCounters.get(GetTermVersionAction.NAME).get(), is(1)); + + } + + private void addCallCountInterceptor(String nodeName, Map callCounters) { + MockTransportService primaryService = (MockTransportService) internalCluster().getInstance(TransportService.class, nodeName); + for (var ctrEnty : callCounters.entrySet()) { + primaryService.addRequestHandlingBehavior(ctrEnty.getKey(), (handler, request, channel, task) -> { + ctrEnty.getValue().incrementAndGet(); + logger.info("--> {} response redirect", ctrEnty.getKey()); + handler.messageReceived(request, channel, task); + }); + } + } + + private BlobStoreRepository prepareClusterAndVerifyRepository() throws Exception { + clusterSettingsSuppliedByTest = true; + Path segmentRepoPath = randomRepoPath(); + Path translogRepoPath = randomRepoPath(); + Path remoteRoutingTableRepoPath = randomRepoPath(); + Settings settings = buildRemoteStoreNodeAttributes( + REPOSITORY_NAME, + segmentRepoPath, + REPOSITORY_2_NAME, + translogRepoPath, + REMOTE_ROUTING_TABLE_REPO, + remoteRoutingTableRepoPath, + false + ); + prepareCluster(1, 3, INDEX_NAME, 1, 5, settings); + ensureGreen(INDEX_NAME); + + RepositoriesService repositoriesService = internalCluster().getClusterManagerNodeInstance(RepositoriesService.class); + BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(REMOTE_ROUTING_TABLE_REPO); + + return repository; + } + +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/state/TransportClusterStateAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/state/TransportClusterStateAction.java index 13ea7eaa43bf8..da91d273f3bce 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -46,10 +46,12 @@ import org.opensearch.cluster.metadata.Metadata.Custom; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.Nullable; import org.opensearch.common.inject.Inject; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.node.NodeClosedException; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -80,7 +82,8 @@ public TransportClusterStateAction( ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver + IndexNameExpressionResolver indexNameExpressionResolver, + @Nullable RemoteClusterStateService remoteClusterStateService ) { super( ClusterStateAction.NAME, @@ -93,6 +96,7 @@ public TransportClusterStateAction( indexNameExpressionResolver ); this.localExecuteSupported = true; + this.remoteClusterStateService = remoteClusterStateService; } @Override diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java b/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java index 4e869f29878cd..819e09312a0df 100644 --- a/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java @@ -49,6 +49,7 @@ import org.opensearch.cluster.ClusterStateObserver; import org.opensearch.cluster.NotClusterManagerException; import org.opensearch.cluster.block.ClusterBlockException; +import org.opensearch.cluster.coordination.ClusterStateTermVersion; import org.opensearch.cluster.coordination.FailedToCommitClusterStateException; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.ProcessClusterEventTimeoutException; @@ -63,6 +64,8 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.discovery.ClusterManagerNotDiscoveredException; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.node.NodeClosedException; import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlActionType; import org.opensearch.tasks.Task; @@ -74,6 +77,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.util.Optional; import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Predicate; @@ -95,6 +99,8 @@ public abstract class TransportClusterManagerNodeAction clusterMetadataManifest = remoteClusterStateService + .getClusterMetadataManifestByTermVersion( + clusterStateTermVersion.getClusterName().value(), + clusterStateTermVersion.getClusterUUID(), + clusterStateTermVersion.getTerm(), + clusterStateTermVersion.getVersion() + ); + if (clusterMetadataManifest.isEmpty()) { + logger.trace("could not find manifest in remote-store for ClusterStateTermVersion {}", termVersion); + return null; + } + ClusterState clusterStateFromRemote = remoteClusterStateService.getClusterStateForManifest( + appliedState.getClusterName().value(), + clusterMetadataManifest.get(), + appliedState.nodes().getLocalNode().getId(), + true + ); + + if (clusterStateFromRemote != null) { + logger.trace("Using the remote cluster-state fetched from local node, ClusterStateTermVersion {}", termVersion); + return clusterStateFromRemote; + } + } catch (Exception e) { + logger.trace("Error while fetching from remote cluster state", e); + } + } + return null; + } + private boolean checkForBlock(Request request, ClusterState localClusterState) { final ClusterBlockException blockException = checkBlock(request, localClusterState); if (blockException != null) { diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java b/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java index 0906abe57d547..cf8e6085d4b32 100644 --- a/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java @@ -8,6 +8,7 @@ package org.opensearch.action.support.clustermanager.term; +import org.opensearch.Version; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.coordination.ClusterStateTermVersion; import org.opensearch.core.action.ActionResponse; @@ -25,18 +26,34 @@ public class GetTermVersionResponse extends ActionResponse { private final ClusterStateTermVersion clusterStateTermVersion; + private final boolean isStatePresentInRemote; + public GetTermVersionResponse(ClusterStateTermVersion clusterStateTermVersion) { this.clusterStateTermVersion = clusterStateTermVersion; + this.isStatePresentInRemote = false; + } + + public GetTermVersionResponse(ClusterStateTermVersion clusterStateTermVersion, boolean canDownloadFromRemote) { + this.clusterStateTermVersion = clusterStateTermVersion; + this.isStatePresentInRemote = canDownloadFromRemote; } public GetTermVersionResponse(StreamInput in) throws IOException { super(in); this.clusterStateTermVersion = new ClusterStateTermVersion(in); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + this.isStatePresentInRemote = in.readOptionalBoolean(); + } else { + this.isStatePresentInRemote = false; + } } @Override public void writeTo(StreamOutput out) throws IOException { clusterStateTermVersion.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalBoolean(isStatePresentInRemote); + } } public ClusterStateTermVersion getClusterStateTermVersion() { @@ -47,4 +64,7 @@ public boolean matches(ClusterState clusterState) { return clusterStateTermVersion != null && clusterStateTermVersion.equals(new ClusterStateTermVersion(clusterState)); } + public boolean isStatePresentInRemote() { + return isStatePresentInRemote; + } } diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/term/TransportGetTermVersionAction.java b/server/src/main/java/org/opensearch/action/support/clustermanager/term/TransportGetTermVersionAction.java index 4752a99c910e4..1cab739a20838 100644 --- a/server/src/main/java/org/opensearch/action/support/clustermanager/term/TransportGetTermVersionAction.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/term/TransportGetTermVersionAction.java @@ -15,11 +15,14 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.coordination.ClusterStateTermVersion; +import org.opensearch.cluster.coordination.Coordinator; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.discovery.Discovery; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -34,13 +37,18 @@ public class TransportGetTermVersionAction extends TransportClusterManagerNodeRe private final Logger logger = LogManager.getLogger(getClass()); + private final Discovery discovery; + + private boolean usePreCommitState = false; + @Inject public TransportGetTermVersionAction( TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver + IndexNameExpressionResolver indexNameExpressionResolver, + Discovery discovery ) { super( GetTermVersionAction.NAME, @@ -52,6 +60,8 @@ public TransportGetTermVersionAction( GetTermVersionRequest::new, indexNameExpressionResolver ); + this.usePreCommitState = FeatureFlags.isEnabled(FeatureFlags.TERM_VERSION_PRECOMMIT_ENABLE_SETTING); + this.discovery = discovery; } @Override @@ -76,10 +86,22 @@ protected void clusterManagerOperation( ClusterState state, ActionListener listener ) throws Exception { - ActionListener.completeWith(listener, () -> buildResponse(request, state)); + if (usePreCommitState) { + ActionListener.completeWith(listener, () -> buildResponse(request, clusterService.preCommitState())); + } else { + ActionListener.completeWith(listener, () -> buildResponse(request, state)); + } + } private GetTermVersionResponse buildResponse(GetTermVersionRequest request, ClusterState state) { - return new GetTermVersionResponse(new ClusterStateTermVersion(state)); + ClusterStateTermVersion termVersion = new ClusterStateTermVersion(state); + if (discovery instanceof Coordinator) { + Coordinator coordinator = (Coordinator) discovery; + if (coordinator.isRemotePublicationEnabled()) { + return new GetTermVersionResponse(termVersion, coordinator.isRemotePublicationEnabled()); + } + } + return new GetTermVersionResponse(termVersion); } } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 9aaaa77bcbb23..13a57d93f03f0 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -386,6 +386,8 @@ private void handleApplyCommit(ApplyCommitRequest applyCommitRequest, ActionList coordinationState.get().handleCommit(applyCommitRequest); final ClusterState committedState = hideStateIfNotRecovered(coordinationState.get().getLastAcceptedState()); applierState = mode == Mode.CANDIDATE ? clusterStateWithNoClusterManagerBlock(committedState) : committedState; + clusterApplier.setPreCommitState(applierState); + if (applyCommitRequest.getSourceNode().equals(getLocalNode())) { // cluster-manager node applies the committed state at the end of the publication process, not here. applyListener.onResponse(null); @@ -1862,4 +1864,11 @@ protected void sendApplyCommit( public static boolean isZen1Node(DiscoveryNode discoveryNode) { return Booleans.isTrue(discoveryNode.getAttributes().getOrDefault("zen1", "false")); } + + public boolean isRemotePublicationEnabled() { + if (coordinationState.get() != null) { + return coordinationState.get().isRemotePublicationEnabled(); + } + return false; + } } diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterApplier.java b/server/src/main/java/org/opensearch/cluster/service/ClusterApplier.java index 5b3f7f1001779..df31b0e94d734 100644 --- a/server/src/main/java/org/opensearch/cluster/service/ClusterApplier.java +++ b/server/src/main/java/org/opensearch/cluster/service/ClusterApplier.java @@ -49,6 +49,12 @@ public interface ClusterApplier { */ void setInitialState(ClusterState initialState); + /** + * Sets the pre-commit state for the applier. + * @param clusterState state that has been committed by coordinator to store + */ + void setPreCommitState(ClusterState clusterState); + /** * Method to invoke when a new cluster state is available to be applied * diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java index b2548a8976c73..47080cfbde692 100644 --- a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java +++ b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java @@ -119,7 +119,7 @@ public class ClusterApplierService extends AbstractLifecycleComponent implements private final Collection clusterStateListeners = new CopyOnWriteArrayList<>(); private final Map timeoutClusterStateListeners = new ConcurrentHashMap<>(); - + private final AtomicReference preCommitState = new AtomicReference<>(); // last state which is yet to be applied private final AtomicReference state; // last applied state private final String nodeName; @@ -750,4 +750,18 @@ protected long currentTimeInMillis() { protected boolean applicationMayFail() { return false; } + + /** + * Pre-commit State of the cluster-applier + * @return ClusterState + */ + public ClusterState preCommitState() { + return preCommitState.get(); + } + + @Override + public void setPreCommitState(ClusterState clusterState) { + preCommitState.set(clusterState); + } + } diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterService.java index c3c48dd8b87ef..1a79161d223e2 100644 --- a/server/src/main/java/org/opensearch/cluster/service/ClusterService.java +++ b/server/src/main/java/org/opensearch/cluster/service/ClusterService.java @@ -183,6 +183,14 @@ public ClusterState state() { return clusterApplierService.state(); } + /** + * The state that is persisted to store but may not be applied to cluster. + * @return ClusterState + */ + public ClusterState preCommitState() { + return clusterApplierService.preCommitState(); + } + /** * Adds a high priority applier of updated cluster states. */ diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index c8d00f65bda10..73375f9eaa413 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -39,6 +39,7 @@ protected FeatureFlagSettings( FeatureFlags.PLUGGABLE_CACHE_SETTING, FeatureFlags.STAR_TREE_INDEX_SETTING, FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING, - FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL_SETTING + FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL_SETTING, + FeatureFlags.TERM_VERSION_PRECOMMIT_ENABLE_SETTING ); } diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 49ecbb0a7069d..a5acea004c3b2 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -128,6 +128,12 @@ public class FeatureFlags { false, Property.NodeScope ); + public static final String TERM_VERSION_PRECOMMIT_ENABLE = "opensearch.experimental.optimization.termversion.precommit.enabled"; + public static final Setting TERM_VERSION_PRECOMMIT_ENABLE_SETTING = Setting.boolSetting( + TERM_VERSION_PRECOMMIT_ENABLE, + false, + Property.NodeScope + ); private static final List> ALL_FEATURE_FLAG_SETTINGS = List.of( REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, @@ -139,7 +145,8 @@ public class FeatureFlags { PLUGGABLE_CACHE_SETTING, STAR_TREE_INDEX_SETTING, APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING, - READER_WRITER_SPLIT_EXPERIMENTAL_SETTING + READER_WRITER_SPLIT_EXPERIMENTAL_SETTING, + TERM_VERSION_PRECOMMIT_ENABLE_SETTING ); /** diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCache.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCache.java new file mode 100644 index 0000000000000..de36ac4429302 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCache.java @@ -0,0 +1,58 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.coordination.ClusterStateTermVersion; +import org.opensearch.common.collect.Tuple; + +import java.util.concurrent.atomic.AtomicReference; + +/** + * Cache to Remote Cluster State based on term-version check. The current implementation + * caches the last highest version of cluster-state that was downloaded from cache. + * + * @opensearch.internal + */ +public class RemoteClusterStateCache { + + private final AtomicReference> clusterStateFromCache = new AtomicReference<>(); + + public ClusterState getState(String clusterName, ClusterMetadataManifest manifest) { + Tuple cache = clusterStateFromCache.get(); + if (cache != null) { + ClusterStateTermVersion manifestStateTermVersion = new ClusterStateTermVersion( + new ClusterName(clusterName), + manifest.getClusterUUID(), + manifest.getClusterTerm(), + manifest.getStateVersion() + ); + if (cache.v1().equals(manifestStateTermVersion)) { + return cache.v2(); + } + } + return null; + } + + public void putState(final ClusterState newState) { + if (newState.metadata() == null || newState.coordinationMetadata() == null) { + // ensure the remote cluster state has coordination metadata set + return; + } + + ClusterStateTermVersion cacheStateTermVersion = new ClusterStateTermVersion( + new ClusterName(newState.getClusterName().value()), + newState.metadata().clusterUUID(), + newState.term(), + newState.version() + ); + clusterStateFromCache.set(new Tuple<>(cacheStateTermVersion, newState)); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index e504c5abb46d3..12d10fd908b44 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -33,6 +33,7 @@ import org.opensearch.cluster.routing.remote.RemoteRoutingTableServiceFactory; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; +import org.opensearch.common.annotation.InternalApi; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.settings.ClusterSettings; @@ -117,6 +118,7 @@ * * @opensearch.internal */ +@InternalApi public class RemoteClusterStateService implements Closeable { private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class); @@ -233,6 +235,7 @@ public static RemoteClusterStateValidationMode parseString(String mode) { private final boolean isPublicationEnabled; private final String remotePathPrefix; + private final RemoteClusterStateCache remoteClusterStateCache; // ToXContent Params with gateway mode. // We are using gateway context mode to persist all custom metadata. public static final ToXContent.Params FORMAT_PARAMS; @@ -282,6 +285,7 @@ public RemoteClusterStateService( ClusterName.CLUSTER_NAME_SETTING.get(settings).value() ); remoteClusterStateCleanupManager = new RemoteClusterStateCleanupManager(this, clusterService, remoteRoutingTableService); + remoteClusterStateCache = new RemoteClusterStateCache(); } /** @@ -1032,6 +1036,15 @@ public ClusterMetadataManifest getClusterMetadataManifestByFileName(String clust return remoteManifestManager.getRemoteClusterMetadataManifestByFileName(clusterUUID, fileName); } + public Optional getClusterMetadataManifestByTermVersion( + String clusterName, + String clusterUUID, + long term, + long version + ) { + return remoteManifestManager.getClusterMetadataManifestByTermVersion(clusterName, clusterUUID, term, version); + } + @Override public void close() throws IOException { remoteClusterStateCleanupManager.close(); @@ -1442,6 +1455,11 @@ public ClusterState getClusterStateForManifest( String localNodeId, boolean includeEphemeral ) throws IOException { + ClusterState stateFromCache = remoteClusterStateCache.getState(clusterName, manifest); + if (stateFromCache != null) { + return stateFromCache; + } + final ClusterState clusterState; final long startTimeNanos = relativeTimeNanosSupplier.getAsLong(); if (manifest.onOrAfterCodecVersion(CODEC_V2)) { @@ -1498,7 +1516,10 @@ public ClusterState getClusterStateForManifest( final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); remoteStateStats.stateFullDownloadSucceeded(); remoteStateStats.stateFullDownloadTook(durationMillis); - + if (includeEphemeral) { + // cache only if the entire cluster-state is present + remoteClusterStateCache.putState(clusterState); + } return clusterState; } @@ -1506,6 +1527,8 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C assert manifest.getDiffManifest() != null : "Diff manifest null which is required for downloading cluster state"; final long startTimeNanos = relativeTimeNanosSupplier.getAsLong(); ClusterStateDiffManifest diff = manifest.getDiffManifest(); + boolean includeEphemeral = true; + List updatedIndices = diff.getIndicesUpdated().stream().map(idx -> { Optional uploadedIndexMetadataOptional = manifest.getIndices() .stream() @@ -1554,7 +1577,7 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C manifest.getDiffManifest() != null && manifest.getDiffManifest().getIndicesRoutingDiffPath() != null && !manifest.getDiffManifest().getIndicesRoutingDiffPath().isEmpty(), - true + includeEphemeral ); ClusterState.Builder clusterStateBuilder = ClusterState.builder(updatedClusterState); Metadata.Builder metadataBuilder = Metadata.builder(updatedClusterState.metadata()); @@ -1588,7 +1611,6 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C .metadata(metadataBuilder) .routingTable(new RoutingTable(manifest.getRoutingTableVersion(), indexRoutingTables)) .build(); - if (!remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) && manifest.getClusterStateChecksum() != null) { validateClusterStateFromChecksum(manifest, clusterState, previousState.getClusterName().value(), localNodeId, false); } @@ -1596,6 +1618,9 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C remoteStateStats.stateDiffDownloadSucceeded(); remoteStateStats.stateDiffDownloadTook(durationMillis); + assert includeEphemeral == true; + // newState includes all the fields of cluster-state (includeEphemeral=true always) + remoteClusterStateCache.putState(clusterState); return clusterState; } @@ -1989,4 +2014,9 @@ public void fullDownloadFailed() { public void diffDownloadFailed() { remoteStateStats.stateDiffDownloadFailed(); } + + RemoteClusterStateCache getRemoteClusterStateCache() { + return remoteClusterStateCache; + } + } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index 47c847b5dc32a..b243269fe323e 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -194,6 +194,17 @@ public Optional getLatestClusterMetadataManifest(String return latestManifestFileName.map(s -> fetchRemoteClusterMetadataManifest(clusterName, clusterUUID, s)); } + public Optional getClusterMetadataManifestByTermVersion( + String clusterName, + String clusterUUID, + long term, + long version + ) { + String prefix = RemoteManifestManager.getManifestFilePrefixForTermVersion(term, version); + Optional latestManifestFileName = getManifestFileNameByPrefix(clusterName, clusterUUID, prefix); + return latestManifestFileName.map(s -> fetchRemoteClusterMetadataManifest(clusterName, clusterUUID, s)); + } + public ClusterMetadataManifest getRemoteClusterMetadataManifestByFileName(String clusterUUID, String filename) throws IllegalStateException { try { @@ -293,7 +304,7 @@ private List getManifestFileNames(String clusterName, String clust } } - static String getManifestFilePrefixForTermVersion(long term, long version) { + public static String getManifestFilePrefixForTermVersion(long term, long version) { return String.join( DELIMITER, RemoteClusterMetadataManifest.MANIFEST, @@ -322,4 +333,14 @@ private Optional getLatestManifestFileName(String clusterName, String cl logger.info("No manifest file present in remote store for cluster name: {}, cluster UUID: {}", clusterName, clusterUUID); return Optional.empty(); } + + private Optional getManifestFileNameByPrefix(String clusterName, String clusterUUID, String filePrefix) + throws IllegalStateException { + List manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, filePrefix, 1); + if (manifestFilesMetadata != null && !manifestFilesMetadata.isEmpty()) { + return Optional.of(manifestFilesMetadata.get(0).name()); + } + logger.info("No manifest file present in remote store for cluster name: {}, cluster UUID: {}", clusterName, clusterUUID); + return Optional.empty(); + } } diff --git a/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java b/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java index 3c1c84653b384..00198364fc8d7 100644 --- a/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java @@ -21,6 +21,7 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.support.ThreadedActionListener; +import org.opensearch.action.support.clustermanager.term.GetTermVersionResponse; import org.opensearch.action.support.replication.ClusterStateCreationUtils; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -30,6 +31,8 @@ import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.cluster.coordination.ClusterStateTermVersion; +import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.coordination.FailedToCommitClusterStateException; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.Metadata; @@ -54,6 +57,8 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.rest.RestStatus; import org.opensearch.discovery.ClusterManagerNotDiscoveredException; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.node.NodeClosedException; import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.snapshots.EmptySnapshotsInfoService; @@ -77,6 +82,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; @@ -84,8 +90,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import org.mockito.Mockito; + import static org.opensearch.index.remote.RemoteMigrationIndexMetadataUpdaterTests.createIndexMetadataWithRemoteStoreSettings; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; @@ -94,6 +103,8 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; public class TransportClusterManagerNodeActionTests extends OpenSearchTestCase { private static ThreadPool threadPool; @@ -209,6 +220,8 @@ public void writeTo(StreamOutput out) throws IOException { } class Action extends TransportClusterManagerNodeAction { + private boolean localExecuteSupported = false; + Action(String actionName, TransportService transportService, ClusterService clusterService, ThreadPool threadPool) { super( actionName, @@ -221,6 +234,18 @@ class Action extends TransportClusterManagerNodeAction { ); } + Action( + String actionName, + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + RemoteClusterStateService clusterStateService + ) { + this(actionName, transportService, clusterService, threadPool); + this.remoteClusterStateService = clusterStateService; + this.localExecuteSupported = true; + } + @Override protected void doExecute(Task task, final Request request, ActionListener listener) { // remove unneeded threading by wrapping listener with SAME to prevent super.doExecute from wrapping it with LISTENER @@ -247,6 +272,10 @@ protected void clusterManagerOperation(Request request, ClusterState state, Acti protected ClusterBlockException checkBlock(Request request, ClusterState state) { return null; // default implementation, overridden in specific tests } + + public boolean localExecuteSupportedByAction() { + return localExecuteSupported; + } } public void testLocalOperationWithoutBlocks() throws ExecutionException, InterruptedException { @@ -715,6 +744,69 @@ protected void masterOperation(Task task, Request request, ClusterState state, A assertFalse(exception.get()); } + public void testFetchFromRemoteStore() throws InterruptedException, BrokenBarrierException, ExecutionException, IOException { + Map attributes = new HashMap<>(); + attributes.put(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "repo1"); + attributes.put(REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "repo2"); + + localNode = new DiscoveryNode( + "local_node", + buildNewFakeTransportAddress(), + attributes, + Collections.singleton(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE), + Version.CURRENT + ); + remoteNode = new DiscoveryNode( + "remote_node", + buildNewFakeTransportAddress(), + attributes, + Collections.singleton(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE), + Version.CURRENT + ); + allNodes = new DiscoveryNode[] { localNode, remoteNode }; + + setState(clusterService, ClusterStateCreationUtils.state(localNode, remoteNode, allNodes)); + + ClusterState state = clusterService.state(); + RemoteClusterStateService remoteClusterStateService = Mockito.mock(RemoteClusterStateService.class); + ClusterMetadataManifest manifest = ClusterMetadataManifest.builder() + .clusterTerm(state.term() + 1) + .stateVersion(state.version() + 1) + .build(); + when( + remoteClusterStateService.getClusterMetadataManifestByTermVersion( + eq(state.getClusterName().value()), + eq(state.metadata().clusterUUID()), + eq(state.term() + 1), + eq(state.version() + 1) + ) + ).thenReturn(Optional.of(manifest)); + when(remoteClusterStateService.getClusterStateForManifest(state.getClusterName().value(), manifest, localNode.getId(), true)) + .thenReturn(buildClusterState(state, state.term() + 1, state.version() + 1)); + + PlainActionFuture listener = new PlainActionFuture<>(); + Request request = new Request(); + Action action = new Action("internal:testAction", transportService, clusterService, threadPool, remoteClusterStateService); + action.execute(request, listener); + + CapturingTransport.CapturedRequest capturedRequest = transport.capturedRequests()[0]; + // mismatch term and version + GetTermVersionResponse termResp = new GetTermVersionResponse( + new ClusterStateTermVersion(state.getClusterName(), state.metadata().clusterUUID(), state.term() + 1, state.version() + 1), + true + ); + transport.handleResponse(capturedRequest.requestId, termResp); + // no more transport calls + assertThat(transport.capturedRequests().length, equalTo(1)); + assertTrue(listener.isDone()); + } + + private ClusterState buildClusterState(ClusterState state, long term, long version) { + CoordinationMetadata.Builder coordMetadataBuilder = CoordinationMetadata.builder().term(term); + Metadata newMetadata = Metadata.builder().coordinationMetadata(coordMetadataBuilder.build()).build(); + return ClusterState.builder(state).version(version).metadata(newMetadata).build(); + } + public void testDontAllowSwitchingToStrictCompatibilityModeForMixedCluster() { // request to change cluster compatibility mode to STRICT Settings currentCompatibilityModeSettings = Settings.builder() diff --git a/server/src/test/java/org/opensearch/action/support/clustermanager/term/ClusterTermVersionIT.java b/server/src/test/java/org/opensearch/action/support/clustermanager/term/ClusterTermVersionIT.java index 7b783e025a575..7ab9da231896c 100644 --- a/server/src/test/java/org/opensearch/action/support/clustermanager/term/ClusterTermVersionIT.java +++ b/server/src/test/java/org/opensearch/action/support/clustermanager/term/ClusterTermVersionIT.java @@ -11,9 +11,19 @@ import org.opensearch.action.admin.cluster.state.ClusterStateAction; import org.opensearch.action.admin.cluster.state.ClusterStateRequest; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterStateApplier; import org.opensearch.cluster.coordination.ClusterStateTermVersion; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.action.ActionFuture; +import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.index.mapper.MapperService; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; @@ -22,6 +32,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import static org.hamcrest.Matchers.is; @@ -100,12 +111,107 @@ public void testDatanodeOutOfSync() throws Exception { assertThat(stateResponse.getState().nodes().getSize(), is(internalCluster().getNodeNames().length)); } + public void testDatanodeWithSlowClusterApplierFallbackToPublish() throws Exception { + List masters = internalCluster().startClusterManagerOnlyNodes( + 3, + Settings.builder().put(FeatureFlags.TERM_VERSION_PRECOMMIT_ENABLE, "true").build() + ); + List datas = internalCluster().startDataOnlyNodes(3); + + Map callCounters = Map.ofEntries( + Map.entry(ClusterStateAction.NAME, new AtomicInteger()), + Map.entry(GetTermVersionAction.NAME, new AtomicInteger()) + ); + ensureGreen(); + + String master = internalCluster().getClusterManagerName(); + + AtomicBoolean processState = new AtomicBoolean(); + ClusterService cmClsService = internalCluster().getInstance(ClusterService.class, datas.get(0)); + cmClsService.addStateApplier(new ClusterStateApplier() { + @Override + public void applyClusterState(ClusterChangedEvent event) { + logger.info("Slow applier started"); + while (processState.get()) { + try { + logger.info("Sleeping for 1s"); + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + logger.info("Slow applier ended"); + } + }); + + ensureGreen(); + + GetTermVersionResponse respBeforeUpdate = internalCluster().getInstance(Client.class, master) + .execute(GetTermVersionAction.INSTANCE, new GetTermVersionRequest()) + .get(); + + processState.set(true); + String index = "index_1"; + ActionFuture startCreateIndex1 = prepareCreate(index).setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), Long.MAX_VALUE) + .build() + ).execute(); + + ActionFuture startCreateIndex2 = prepareCreate("index_2").setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), Long.MAX_VALUE) + .build() + ).execute(); + + // wait for cluster-manager to publish new state + waitUntil(() -> { + try { + // node is yet to ack commit to cluster-manager , only the state-update corresponding to index_1 should have been published + GetTermVersionResponse respAfterUpdate = internalCluster().getInstance(Client.class, master) + .execute(GetTermVersionAction.INSTANCE, new GetTermVersionRequest()) + .get(); + logger.info( + "data has latest , {} , {}", + respAfterUpdate.getClusterStateTermVersion().getTerm(), + respAfterUpdate.getClusterStateTermVersion().getVersion() + ); + return respBeforeUpdate.getClusterStateTermVersion().getVersion() + 1 == respAfterUpdate.getClusterStateTermVersion() + .getVersion(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + addCallCountInterceptor(master, callCounters); + ClusterStateResponse stateResponseD = internalCluster().getInstance(Client.class, datas.get(0)) + .admin() + .cluster() + .state(new ClusterStateRequest()) + .actionGet(); + logger.info("data has the version , {} , {}", stateResponseD.getState().term(), stateResponseD.getState().version()); + assertTrue(respBeforeUpdate.getClusterStateTermVersion().getVersion() + 1 == stateResponseD.getState().version()); + + processState.set(false); + + AtomicInteger clusterStateCallsOnMaster = callCounters.get(ClusterStateAction.NAME); + AtomicInteger termCallsOnMaster = callCounters.get(GetTermVersionAction.NAME); + startCreateIndex1.get(); + startCreateIndex2.get(); + assertThat(clusterStateCallsOnMaster.get(), is(0)); + assertThat(termCallsOnMaster.get(), is(1)); + } + private void addCallCountInterceptor(String nodeName, Map callCounters) { MockTransportService primaryService = (MockTransportService) internalCluster().getInstance(TransportService.class, nodeName); for (var ctrEnty : callCounters.entrySet()) { primaryService.addRequestHandlingBehavior(ctrEnty.getKey(), (handler, request, channel, task) -> { ctrEnty.getValue().incrementAndGet(); - logger.info("--> {} response redirect", ClusterStateAction.NAME); + logger.info("--> {} response redirect", ctrEnty.getKey()); handler.messageReceived(request, channel, task); }); } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/NoOpClusterApplier.java b/server/src/test/java/org/opensearch/cluster/coordination/NoOpClusterApplier.java index 9b865ace3b082..6e0d86540cf99 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/NoOpClusterApplier.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/NoOpClusterApplier.java @@ -42,6 +42,11 @@ public void setInitialState(ClusterState initialState) { } + @Override + public void setPreCommitState(ClusterState clusterState) { + + } + @Override public void onNewClusterState(String source, Supplier clusterStateSupplier, ClusterApplyListener listener) { listener.onSuccess(source); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index e875b1c5dc64e..b11d5e48bec06 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -46,7 +46,6 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; -import org.opensearch.core.ParseField; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; @@ -62,6 +61,7 @@ import org.opensearch.gateway.remote.model.RemoteReadResult; import org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata; import org.opensearch.index.remote.RemoteIndexPathUploader; +import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.indices.DefaultRemoteStoreSettings; import org.opensearch.indices.IndicesModule; import org.opensearch.repositories.FilterRepository; @@ -2322,6 +2322,73 @@ public void testReadLatestMetadataManifestSuccessButNoIndexMetadata() throws IOE ); } + public void testReadLatestClusterStateFromCache() throws IOException { + final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + remoteClusterStateService.start(); + final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() + .indices(List.of()) + .clusterTerm(1L) + .stateVersion(12) + .stateUUID("state-uuid") + .clusterUUID("cluster-uuid") + .codecVersion(MANIFEST_CURRENT_CODEC_VERSION) + .coordinationMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(COORDINATION_METADATA, "mock-coordination-file")) + .settingMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(SETTING_METADATA, "mock-setting-file")) + .templatesMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(TEMPLATES_METADATA, "mock-templates-file")) + .put( + IndexGraveyard.TYPE, + new ClusterMetadataManifest.UploadedMetadataAttribute(IndexGraveyard.TYPE, "mock-custom-" + IndexGraveyard.TYPE + "-file") + ) + .nodeId("nodeA") + .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) + .previousClusterUUID("prev-cluster-uuid") + .routingTableVersion(1) + .indicesRouting(List.of()) + .build(); + + Metadata expectedMetadata = Metadata.builder() + .clusterUUID("cluster-uuid") + .version(12) + .coordinationMetadata(CoordinationMetadata.builder().term(1).build()) + .persistentSettings(Settings.builder().put("readonly", true).build()) + .build(); + mockBlobContainerForGlobalMetadata(mockBlobStoreObjects(), expectedManifest, expectedMetadata); + + ClusterState state = remoteClusterStateService.getLatestClusterState( + clusterState.getClusterName().value(), + clusterState.metadata().clusterUUID(), + true + ); + + ClusterState stateFromCache = remoteClusterStateService.getRemoteClusterStateCache() + .getState(clusterState.getClusterName().value(), expectedManifest); + assertEquals(stateFromCache.getMetadata(), state.getMetadata()); + + final ClusterMetadataManifest notExistMetadata = ClusterMetadataManifest.builder() + .indices(List.of()) + .clusterTerm(1L) + .stateVersion(13) + .stateUUID("state-uuid") + .clusterUUID("cluster-uuid") + .codecVersion(MANIFEST_CURRENT_CODEC_VERSION) + .coordinationMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(COORDINATION_METADATA, "mock-coordination-file")) + .settingMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(SETTING_METADATA, "mock-setting-file")) + .templatesMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(TEMPLATES_METADATA, "mock-templates-file")) + .put( + IndexGraveyard.TYPE, + new ClusterMetadataManifest.UploadedMetadataAttribute(IndexGraveyard.TYPE, "mock-custom-" + IndexGraveyard.TYPE + "-file") + ) + .nodeId("nodeA") + .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) + .previousClusterUUID("prev-cluster-uuid") + .routingTableVersion(1) + .indicesRouting(List.of()) + .build(); + ClusterState notInCacheState = remoteClusterStateService.getRemoteClusterStateCache() + .getState(clusterState.getClusterName().value(), notExistMetadata); + assertNull(notInCacheState); + } + public void testReadLatestMetadataManifestSuccessButIndexMetadataFetchIOException() throws IOException { final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename__2"); @@ -2388,9 +2455,46 @@ public void testReadLatestMetadataManifestSuccess() throws IOException { assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); } + public void testReadLatestMetadataManifestSuccessByTermVersion() throws IOException { + final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); + final List indices = List.of(uploadedIndexMetadata); + + final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() + .indices(indices) + .clusterTerm(1L) + .stateVersion(1L) + .stateUUID("state-uuid") + .clusterUUID("cluster-uuid") + .nodeId("nodeA") + .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) + .codecVersion(CODEC_V2) + .previousClusterUUID("prev-cluster-uuid") + .build(); + + mockBlobContainer(mockBlobStoreObjects(), expectedManifest, new HashMap<>(), CODEC_V2, 1, 1); + remoteClusterStateService.start(); + final ClusterMetadataManifest manifest = remoteClusterStateService.getClusterMetadataManifestByTermVersion( + clusterState.getClusterName().value(), + clusterState.metadata().clusterUUID(), + 1, + 1 + ).get(); + + assertThat(manifest.getIndices().size(), is(1)); + assertThat(manifest.getIndices().get(0).getIndexName(), is(uploadedIndexMetadata.getIndexName())); + assertThat(manifest.getIndices().get(0).getIndexUUID(), is(uploadedIndexMetadata.getIndexUUID())); + assertThat(manifest.getIndices().get(0).getUploadedFilename(), notNullValue()); + assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm())); + assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion())); + assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); + assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); + } + public void testReadGlobalMetadata() throws IOException { - when(blobStoreRepository.getNamedXContentRegistry()).thenReturn(new NamedXContentRegistry( - List.of(new NamedXContentRegistry.Entry(Metadata.Custom.class, new ParseField(IndexGraveyard.TYPE), IndexGraveyard::fromXContent)))); + // when(blobStoreRepository.getNamedXContentRegistry()).thenReturn(new NamedXContentRegistry( + // List.of(new NamedXContentRegistry.Entry(Metadata.Custom.class, new ParseField(IndexGraveyard.TYPE), + // IndexGraveyard::fromXContent)))); final ClusterState clusterState = generateClusterStateWithGlobalMetadata().nodes(nodesWithLocalNodeClusterManager()).build(); remoteClusterStateService.start(); @@ -2405,7 +2509,10 @@ public void testReadGlobalMetadata() throws IOException { .coordinationMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(COORDINATION_METADATA, "mock-coordination-file")) .settingMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(SETTING_METADATA, "mock-setting-file")) .templatesMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(TEMPLATES_METADATA, "mock-templates-file")) - .put(IndexGraveyard.TYPE, new ClusterMetadataManifest.UploadedMetadataAttribute(IndexGraveyard.TYPE, "mock-custom-" +IndexGraveyard.TYPE+ "-file")) + .put( + IndexGraveyard.TYPE, + new ClusterMetadataManifest.UploadedMetadataAttribute(IndexGraveyard.TYPE, "mock-custom-" + IndexGraveyard.TYPE + "-file") + ) .nodeId("nodeA") .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) .previousClusterUUID("prev-cluster-uuid") @@ -2413,7 +2520,10 @@ public void testReadGlobalMetadata() throws IOException { .indicesRouting(List.of()) .build(); - Metadata expectedMetadata = Metadata.builder().clusterUUID("cluster-uuid").persistentSettings(Settings.builder().put("readonly", true).build()).build(); + Metadata expectedMetadata = Metadata.builder() + .clusterUUID("cluster-uuid") + .persistentSettings(Settings.builder().put("readonly", true).build()) + .build(); mockBlobContainerForGlobalMetadata(mockBlobStoreObjects(), expectedManifest, expectedMetadata); ClusterState newClusterState = remoteClusterStateService.getLatestClusterState( @@ -3939,6 +4049,55 @@ private void mockBlobContainer( }); } + private void mockBlobContainer( + BlobContainer blobContainer, + ClusterMetadataManifest clusterMetadataManifest, + Map indexMetadataMap, + int codecVersion, + long term, + long version + ) throws IOException { + String manifestFileName = codecVersion >= CODEC_V1 + ? "manifest__manifestFileName__abcd__abcd__abcd__" + codecVersion + : "manifestFileName"; + BlobMetadata blobMetadata = new PlainBlobMetadata(manifestFileName, 1); + + String manifestPrefix = String.join(DELIMITER, "manifest", RemoteStoreUtils.invertLong(term), RemoteStoreUtils.invertLong(version)) + + DELIMITER; + when(blobContainer.listBlobsByPrefixInSortedOrder(manifestPrefix, 1, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC)).thenReturn( + Arrays.asList(blobMetadata) + ); + + BytesReference bytes = RemoteClusterMetadataManifest.CLUSTER_METADATA_MANIFEST_FORMAT.serialize( + clusterMetadataManifest, + manifestFileName, + blobStoreRepository.getCompressor(), + FORMAT_PARAMS + ); + when(blobContainer.readBlob(manifestFileName)).thenReturn(new ByteArrayInputStream(bytes.streamInput().readAllBytes())); + + clusterMetadataManifest.getIndices().forEach(uploadedIndexMetadata -> { + try { + IndexMetadata indexMetadata = indexMetadataMap.get(uploadedIndexMetadata.getIndexUUID()); + if (indexMetadata == null) { + return; + } + String fileName = uploadedIndexMetadata.getUploadedFilename(); + when(blobContainer.readBlob(getFormattedIndexFileName(fileName))).thenAnswer((invocationOnMock) -> { + BytesReference bytesIndexMetadata = INDEX_METADATA_FORMAT.serialize( + indexMetadata, + fileName, + blobStoreRepository.getCompressor(), + FORMAT_PARAMS + ); + return new ByteArrayInputStream(bytesIndexMetadata.streamInput().readAllBytes()); + }); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + private void mockBlobContainerForGlobalMetadata( BlobContainer blobContainer, ClusterMetadataManifest clusterMetadataManifest, diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index c439ef209e049..350c6f9ae8f6b 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -2413,7 +2413,8 @@ public void onFailure(final Exception e) { clusterService, threadPool, actionFilters, - indexNameExpressionResolver + indexNameExpressionResolver, + null ) ); actions.put( @@ -2455,7 +2456,8 @@ public void onFailure(final Exception e) { clusterService, threadPool, actionFilters, - indexNameExpressionResolver + indexNameExpressionResolver, + null ) ); From 330b2497a9380fa73ee548e61f0c85196eea7acf Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Thu, 12 Sep 2024 14:49:45 +0530 Subject: [PATCH 16/44] Add restriction to have a single repository with shallow snapshot v2 setting (#15901) * Add restriction to have a single repository with shallow snapshot v2 setting Signed-off-by: Sachin Kale * Do not allow shallow snapshot v2 repo name to contain SNAPSHOT_PINNED_TIMESTAMP_DELIMITER Signed-off-by: Sachin Kale --------- Signed-off-by: Sachin Kale --- .../RemoteStorePinnedTimestampService.java | 31 ++++- .../repositories/RepositoriesService.java | 79 ++++++++++- .../BlobStoreRepositoryRemoteIndexTests.java | 124 ++++++++++++++++++ 3 files changed, 228 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java index 3a7734fc0538f..71133615ed056 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java @@ -30,6 +30,8 @@ import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -75,25 +77,46 @@ public RemoteStorePinnedTimestampService( * and starts the asynchronous update task. */ public void start() { - validateRemoteStoreConfiguration(); + blobContainer = validateAndCreateBlobContainer(settings, repositoriesService.get()); startAsyncUpdateTask(RemoteStoreSettings.getPinnedTimestampsSchedulerInterval()); } - private void validateRemoteStoreConfiguration() { + private static BlobContainer validateAndCreateBlobContainer(Settings settings, RepositoriesService repositoriesService) { final String remoteStoreRepo = settings.get( Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY ); assert remoteStoreRepo != null : "Remote Segment Store repository is not configured"; - final Repository repository = repositoriesService.get().repository(remoteStoreRepo); + final Repository repository = repositoriesService.repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository; - blobContainer = blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath().add(PINNED_TIMESTAMPS_PATH_TOKEN)); + return blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath().add(PINNED_TIMESTAMPS_PATH_TOKEN)); } private void startAsyncUpdateTask(TimeValue pinnedTimestampsSchedulerInterval) { asyncUpdatePinnedTimestampTask = new AsyncUpdatePinnedTimestampTask(logger, threadPool, pinnedTimestampsSchedulerInterval, true); } + public static Map> fetchPinnedTimestamps(Settings settings, RepositoriesService repositoriesService) + throws IOException { + BlobContainer blobContainer = validateAndCreateBlobContainer(settings, repositoriesService); + Set pinnedTimestamps = blobContainer.listBlobs().keySet(); + Map> pinningEntityTimestampMap = new HashMap<>(); + for (String pinnedTimestamp : pinnedTimestamps) { + try { + String[] tokens = pinnedTimestamp.split(PINNED_TIMESTAMPS_FILENAME_SEPARATOR); + Long timestamp = Long.parseLong(tokens[tokens.length - 1]); + String pinningEntity = pinnedTimestamp.substring(0, pinnedTimestamp.lastIndexOf(PINNED_TIMESTAMPS_FILENAME_SEPARATOR)); + if (pinningEntityTimestampMap.containsKey(pinningEntity) == false) { + pinningEntityTimestampMap.put(pinningEntity, new HashSet<>()); + } + pinningEntityTimestampMap.get(pinningEntity).add(timestamp); + } catch (NumberFormatException e) { + logger.error("Exception while parsing pinned timestamp from {}, skipping this entry", pinnedTimestamp); + } + } + return pinningEntityTimestampMap; + } + /** * Pins a timestamp in the remote store. * diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java index 68669feb16abc..7da52147661dc 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java @@ -68,7 +68,9 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; +import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; import org.opensearch.repositories.blobstore.MeteredBlobStoreRepository; +import org.opensearch.snapshots.SnapshotsService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -84,6 +86,7 @@ import java.util.stream.Stream; import static org.opensearch.repositories.blobstore.BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY; +import static org.opensearch.repositories.blobstore.BlobStoreRepository.SHALLOW_SNAPSHOT_V2; import static org.opensearch.repositories.blobstore.BlobStoreRepository.SYSTEM_REPOSITORY_SETTING; /** @@ -123,6 +126,7 @@ public class RepositoriesService extends AbstractLifecycleComponent implements C private final RepositoriesStatsArchive repositoriesStatsArchive; private final ClusterManagerTaskThrottler.ThrottlingKey putRepositoryTaskKey; private final ClusterManagerTaskThrottler.ThrottlingKey deleteRepositoryTaskKey; + private final Settings settings; public RepositoriesService( Settings settings, @@ -132,6 +136,7 @@ public RepositoriesService( Map internalTypesRegistry, ThreadPool threadPool ) { + this.settings = settings; this.typesRegistry = typesRegistry; this.internalTypesRegistry = internalTypesRegistry; this.clusterService = clusterService; @@ -173,7 +178,7 @@ public void registerOrUpdateRepository(final PutRepositoryRequest request, final CryptoMetadata.fromRequest(request.cryptoSettings()) ); validate(request.name()); - validateRepositoryMetadataSettings(clusterService, request.name(), request.settings()); + validateRepositoryMetadataSettings(clusterService, request.name(), request.settings(), repositories, settings, this); if (newRepositoryMetadata.cryptoMetadata() != null) { validate(newRepositoryMetadata.cryptoMetadata().keyProviderName()); } @@ -684,7 +689,10 @@ public static void validate(final String identifier) { public static void validateRepositoryMetadataSettings( ClusterService clusterService, final String repositoryName, - final Settings repositoryMetadataSettings + final Settings repositoryMetadataSettings, + Map repositories, + Settings settings, + RepositoriesService repositoriesService ) { // We can add more validations here for repository settings in the future. Version minVersionInCluster = clusterService.state().getNodes().getMinNodeVersion(); @@ -699,6 +707,51 @@ public static void validateRepositoryMetadataSettings( + minVersionInCluster ); } + if (SHALLOW_SNAPSHOT_V2.get(repositoryMetadataSettings)) { + if (minVersionInCluster.onOrAfter(Version.V_2_17_0) == false) { + throw new RepositoryException( + repositoryName, + "setting " + + SHALLOW_SNAPSHOT_V2.getKey() + + " cannot be enabled as some of the nodes in cluster are on version older than " + + Version.V_2_17_0 + + ". Minimum node version in cluster is: " + + minVersionInCluster + ); + } + if (repositoryName.contains(SnapshotsService.SNAPSHOT_PINNED_TIMESTAMP_DELIMITER)) { + throw new RepositoryException( + repositoryName, + "setting " + + SHALLOW_SNAPSHOT_V2.getKey() + + " cannot be enabled for repository with " + + SnapshotsService.SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + + " in the name as this delimiter is used to create pinning entity" + ); + } + if (repositoryWithShallowV2Exists(repositories)) { + throw new RepositoryException( + repositoryName, + "setting " + + SHALLOW_SNAPSHOT_V2.getKey() + + " cannot be enabled as this setting can be enabled only on one repository " + + " and one or more repositories in the cluster have the setting as enabled" + ); + } + try { + if (pinnedTimestampExistsWithDifferentRepository(repositoryName, settings, repositoriesService)) { + throw new RepositoryException( + repositoryName, + "setting " + + SHALLOW_SNAPSHOT_V2.getKey() + + " cannot be enabled if there are existing snapshots created with shallow V2 " + + "setting using different repository." + ); + } + } catch (IOException e) { + throw new RepositoryException(repositoryName, "Exception while fetching pinned timestamp details"); + } + } // Validation to not allow users to create system repository via put repository call. if (isSystemRepositorySettingPresent(repositoryMetadataSettings)) { throw new RepositoryException( @@ -710,6 +763,28 @@ public static void validateRepositoryMetadataSettings( } } + private static boolean repositoryWithShallowV2Exists(Map repositories) { + return repositories.values().stream().anyMatch(repo -> SHALLOW_SNAPSHOT_V2.get(repo.getMetadata().settings())); + } + + private static boolean pinnedTimestampExistsWithDifferentRepository( + String newRepoName, + Settings settings, + RepositoriesService repositoriesService + ) throws IOException { + Map> pinningEntityTimestampMap = RemoteStorePinnedTimestampService.fetchPinnedTimestamps( + settings, + repositoriesService + ); + for (String pinningEntity : pinningEntityTimestampMap.keySet()) { + String repoNameWithPinnedTimestamps = pinningEntity.split(SnapshotsService.SNAPSHOT_PINNED_TIMESTAMP_DELIMITER)[0]; + if (repoNameWithPinnedTimestamps.equals(newRepoName) == false) { + return true; + } + } + return false; + } + private static void ensureRepositoryNotInUse(ClusterState clusterState, String repository) { if (isRepositoryInUse(clusterState, repository)) { throw new IllegalStateException("trying to modify or unregister repository that is currently used"); diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java index 9cca495cced72..e280ab8c7a73c 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java @@ -33,6 +33,8 @@ package org.opensearch.repositories.blobstore; import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; +import org.opensearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse; +import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.common.settings.Settings; @@ -41,13 +43,16 @@ import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.index.IndexSettings; import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; +import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.RepositoryData; +import org.opensearch.repositories.RepositoryException; import org.opensearch.repositories.fs.FsRepository; import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotInfo; +import org.opensearch.snapshots.SnapshotsService; import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; @@ -64,6 +69,9 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.repositories.blobstore.BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY; +import static org.opensearch.repositories.blobstore.BlobStoreRepository.SHALLOW_SNAPSHOT_V2; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; /** @@ -81,6 +89,7 @@ protected Settings nodeSettings() { .put(Environment.PATH_HOME_SETTING.getKey(), tempDir) .put(Environment.PATH_REPO_SETTING.getKey(), tempDir.resolve("repo")) .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), tempDir.getParent()) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) .build(); } @@ -373,4 +382,119 @@ public void testRetrieveShallowCopySnapshotCase2() throws IOException { assertThat(snapshotIds, equalTo(originalSnapshots)); } + public void testRepositoryCreationShallowV2() throws Exception { + Client client = client(); + + Settings snapshotRepoSettings1 = Settings.builder() + .put(node().settings()) + .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) + .put(REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(SHALLOW_SNAPSHOT_V2.getKey(), true) + .build(); + + String invalidRepoName = "test" + SnapshotsService.SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + "repo-1"; + try { + createRepository(client, invalidRepoName, snapshotRepoSettings1); + } catch (RepositoryException e) { + assertEquals( + "[" + + invalidRepoName + + "] setting shallow_snapshot_v2 cannot be enabled for repository with __ in the name as this delimiter is used to create pinning entity", + e.getMessage() + ); + } + + // Create repo with shallow snapshot V2 enabled + createRepository(client, "test-repo-1", snapshotRepoSettings1); + + logger.info("--> verify the repository"); + VerifyRepositoryResponse verifyRepositoryResponse = client.admin().cluster().prepareVerifyRepository("test-repo-1").get(); + assertNotNull(verifyRepositoryResponse.getNodes()); + + GetRepositoriesResponse getRepositoriesResponse = client.admin().cluster().prepareGetRepositories("test-repo-1").get(); + assertTrue(SHALLOW_SNAPSHOT_V2.get(getRepositoriesResponse.repositories().get(0).settings())); + + Settings snapshotRepoSettings2 = Settings.builder() + .put(node().settings()) + .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) + .put(REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(SHALLOW_SNAPSHOT_V2.getKey(), true) + .build(); + + // Create another repo with shallow snapshot V2 enabled, this should fail. + try { + createRepository(client, "test-repo-2", snapshotRepoSettings2); + } catch (RepositoryException e) { + assertEquals( + "[test-repo-2] setting shallow_snapshot_v2 cannot be enabled as this setting can be enabled only on one repository and one or more repositories in the cluster have the setting as enabled", + e.getMessage() + ); + } + + // Disable shallow snapshot V2 setting on test-repo-1 + updateRepository( + client, + "test-repo-1", + Settings.builder().put(snapshotRepoSettings1).put(SHALLOW_SNAPSHOT_V2.getKey(), false).build() + ); + getRepositoriesResponse = client.admin().cluster().prepareGetRepositories("test-repo-1").get(); + assertFalse(SHALLOW_SNAPSHOT_V2.get(getRepositoriesResponse.repositories().get(0).settings())); + + // Create test-repo-2 with shallow snapshot V2 enabled, this should pass now. + createRepository(client, "test-repo-2", snapshotRepoSettings2); + getRepositoriesResponse = client.admin().cluster().prepareGetRepositories("test-repo-2").get(); + assertTrue(SHALLOW_SNAPSHOT_V2.get(getRepositoriesResponse.repositories().get(0).settings())); + + final String indexName = "test-idx"; + createIndex(indexName); + ensureGreen(); + indexDocuments(client, indexName); + + // Create pinned timestamp snapshot in test-repo-2 + SnapshotInfo snapshotInfo = createSnapshot("test-repo-2", "test-snap-2", new ArrayList<>()); + assertNotNull(snapshotInfo.snapshotId()); + + // As snapshot is present, even after disabling shallow snapshot setting in test-repo-2, we will not be able to + // enable shallow snapshot v2 setting in test-repo-1 + updateRepository( + client, + "test-repo-2", + Settings.builder().put(snapshotRepoSettings2).put(SHALLOW_SNAPSHOT_V2.getKey(), false).build() + ); + getRepositoriesResponse = client.admin().cluster().prepareGetRepositories("test-repo-2").get(); + assertFalse(SHALLOW_SNAPSHOT_V2.get(getRepositoriesResponse.repositories().get(0).settings())); + + try { + updateRepository(client, "test-repo-1", snapshotRepoSettings1); + } catch (RepositoryException e) { + assertEquals( + "[test-repo-1] setting shallow_snapshot_v2 cannot be enabled if there are existing snapshots created with shallow V2 setting using different repository.", + e.getMessage() + ); + } + + // After deleting the snapshot, we will be able to enable shallow snapshot v2 setting in test-repo-1 + AcknowledgedResponse deleteSnapshotResponse = client().admin().cluster().prepareDeleteSnapshot("test-repo-2", "test-snap-2").get(); + + assertAcked(deleteSnapshotResponse); + + updateRepository(client, "test-repo-1", snapshotRepoSettings1); + getRepositoriesResponse = client.admin().cluster().prepareGetRepositories("test-repo-1").get(); + assertTrue(SHALLOW_SNAPSHOT_V2.get(getRepositoriesResponse.repositories().get(0).settings())); + + // Having a snapshot in the same repo should allow disabling and re-enabling shallow snapshot v2 setting + snapshotInfo = createSnapshot("test-repo-1", "test-snap-1", new ArrayList<>()); + assertNotNull(snapshotInfo.snapshotId()); + updateRepository( + client, + "test-repo-1", + Settings.builder().put(snapshotRepoSettings1).put(SHALLOW_SNAPSHOT_V2.getKey(), false).build() + ); + getRepositoriesResponse = client.admin().cluster().prepareGetRepositories("test-repo-1").get(); + assertFalse(SHALLOW_SNAPSHOT_V2.get(getRepositoriesResponse.repositories().get(0).settings())); + + updateRepository(client, "test-repo-1", snapshotRepoSettings1); + getRepositoriesResponse = client.admin().cluster().prepareGetRepositories("test-repo-1").get(); + assertTrue(SHALLOW_SNAPSHOT_V2.get(getRepositoriesResponse.repositories().get(0).settings())); + } } From 4223fab40bedd0571324f76f8838ccea4a8b1b8a Mon Sep 17 00:00:00 2001 From: Sagar <99425694+sgup432@users.noreply.github.com> Date: Thu, 12 Sep 2024 15:31:46 -0700 Subject: [PATCH 17/44] Removing task Id from map using remove() (#15918) Signed-off-by: Sagar Upadhyaya --- .../org/opensearch/tasks/TaskCancellationMonitoringService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java b/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java index 343d4571593a7..2040703d88c38 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java +++ b/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java @@ -125,7 +125,7 @@ public void onTaskCompleted(Task task) { if (!TASKS_TO_TRACK.contains(task.getClass())) { return; } - this.cancelledTaskTracker.entrySet().removeIf(entry -> entry.getKey() == task.getId()); + this.cancelledTaskTracker.remove(task.getId()); } /** From 07029b2513f0cffbd788d95836dbc6cff0b4dbf4 Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Fri, 13 Sep 2024 05:53:57 -0700 Subject: [PATCH 18/44] Add the capability to override indices.breaker.total.use_real_memory setting for test clusters (#15906) Signed-off-by: Navneet Verma --- .../org/opensearch/gradle/testclusters/OpenSearchNode.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java index 268de50340cbf..cd22560af9a96 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java @@ -116,7 +116,12 @@ public class OpenSearchNode implements TestClusterConfiguration { private static final TimeUnit NODE_UP_TIMEOUT_UNIT = TimeUnit.MINUTES; private static final int ADDITIONAL_CONFIG_TIMEOUT = 15; private static final TimeUnit ADDITIONAL_CONFIG_TIMEOUT_UNIT = TimeUnit.SECONDS; - private static final List OVERRIDABLE_SETTINGS = Arrays.asList("path.repo", "discovery.seed_providers", "discovery.seed_hosts"); + private static final List OVERRIDABLE_SETTINGS = Arrays.asList( + "path.repo", + "discovery.seed_providers", + "discovery.seed_hosts", + "indices.breaker.total.use_real_memory" + ); private static final int TAIL_LOG_MESSAGES_COUNT = 40; private static final List MESSAGES_WE_DONT_CARE_ABOUT = Arrays.asList( From 260edc5f13debc04858da4e73dfc396eeb459ef9 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 13 Sep 2024 10:44:16 -0400 Subject: [PATCH 19/44] Fix flaky org.opensearch.rest.ReactorNetty4StreamingStressIT.testCloseClientStreamingRequest test case (#15859) Signed-off-by: Andriy Redko --- .../rest/ReactorNetty4StreamingStressIT.java | 56 +++++++------------ 1 file changed, 21 insertions(+), 35 deletions(-) diff --git a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java index a978af1b11db4..a853ed56fad32 100644 --- a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java +++ b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java @@ -8,7 +8,6 @@ package org.opensearch.rest; -import org.apache.hc.core5.http.ConnectionClosedException; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.StreamingRequest; @@ -16,24 +15,20 @@ import org.opensearch.test.rest.OpenSearchRestTestCase; import org.junit.After; +import java.io.IOException; import java.io.InterruptedIOException; +import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.Duration; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; import reactor.core.publisher.Flux; -import reactor.test.subscriber.TestSubscriber; +import reactor.test.StepVerifier; +import reactor.test.scheduler.VirtualTimeScheduler; -import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.collection.IsEmptyCollection.empty; public class ReactorNetty4StreamingStressIT extends OpenSearchRestTestCase { @After @@ -49,6 +44,8 @@ public void tearDown() throws Exception { } public void testCloseClientStreamingRequest() throws Exception { + final VirtualTimeScheduler scheduler = VirtualTimeScheduler.create(true); + final AtomicInteger id = new AtomicInteger(0); final Stream stream = Stream.generate( () -> "{ \"index\": { \"_index\": \"test-stress-streaming\", \"_id\": \"" @@ -57,39 +54,28 @@ public void testCloseClientStreamingRequest() throws Exception { + "{ \"name\": \"josh\" }\n" ); + final Duration delay = Duration.ofMillis(1); final StreamingRequest streamingRequest = new StreamingRequest<>( "POST", "/_bulk/stream", - Flux.fromStream(stream).delayElements(Duration.ofMillis(500)).map(s -> ByteBuffer.wrap(s.getBytes(StandardCharsets.UTF_8))) + Flux.fromStream(stream).delayElements(delay, scheduler).map(s -> ByteBuffer.wrap(s.getBytes(StandardCharsets.UTF_8))) ); streamingRequest.addParameter("refresh", "true"); final StreamingResponse streamingResponse = client().streamRequest(streamingRequest); - TestSubscriber subscriber = TestSubscriber.create(); - streamingResponse.getBody().subscribe(subscriber); - - final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); - try { - // Await for subscriber to receive at least one chunk - assertBusy(() -> assertThat(subscriber.getReceivedOnNext(), not(empty()))); - - // Close client forceably - executor.schedule(() -> { - client().close(); - return null; - }, 2, TimeUnit.SECONDS); + scheduler.advanceTimeBy(delay); /* emit first element */ - // Await for subscriber to terminate - subscriber.block(Duration.ofSeconds(10)); - assertThat( - subscriber.expectTerminalError(), - anyOf(instanceOf(InterruptedIOException.class), instanceOf(ConnectionClosedException.class)) - ); - } finally { - executor.shutdown(); - if (executor.awaitTermination(1, TimeUnit.SECONDS) == false) { - executor.shutdownNow(); - } - } + StepVerifier.create(Flux.from(streamingResponse.getBody()).map(b -> new String(b.array(), StandardCharsets.UTF_8))) + .expectNextMatches(s -> s.contains("\"result\":\"created\"") && s.contains("\"_id\":\"1\"")) + .then(() -> { + try { + client().close(); + } catch (final IOException ex) { + throw new UncheckedIOException(ex); + } + }) + .then(() -> scheduler.advanceTimeBy(delay)) + .expectErrorMatches(t -> t instanceof InterruptedIOException) + .verify(); } } From 36c89bf4f975bd2750423d1308d2cd3408d8553f Mon Sep 17 00:00:00 2001 From: Finn Date: Fri, 13 Sep 2024 08:00:37 -0700 Subject: [PATCH 20/44] Update TESTING.md 'Building with extra plugins' documentation (#15893) * Update TESTING.md 'Building with extra plugins' documentation Signed-off-by: Finn Carroll * Typo Signed-off-by: Finn Carroll * Fix TOC Signed-off-by: Finn Carroll --------- Signed-off-by: Finn Carroll --- TESTING.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/TESTING.md b/TESTING.md index de7ab3eefe2f8..9127f005ced75 100644 --- a/TESTING.md +++ b/TESTING.md @@ -39,7 +39,7 @@ OpenSearch uses [jUnit](https://junit.org/junit5/) for testing, it also uses ran - [Expect a specific segment topology](#expect-a-specific-segment-topology) - [Leave environment in an unstable state after test](#leave-environment-in-an-unstable-state-after-test) - [Test coverage analysis](#test-coverage-analysis) -- [Building with extra plugins](#building-with-extra-plugins) +- [Testing with plugins](#testing-with-plugins) - [Environment misc](#environment-misc) # Requirements @@ -552,11 +552,17 @@ Apart from using Gradle, it is also possible to gain insight in code coverage us Please read your IDE documentation for how to attach a debugger to a JVM process. -# Building with extra plugins +# Testing with plugins -Additional plugins may be built alongside OpenSearch, where their dependency on OpenSearch will be substituted with the local OpenSearch build. To add your plugin, create a directory called `opensearch-extra` as a sibling of OpenSearch. Checkout your plugin underneath `opensearch-extra` and the build will automatically pick it up. You can verify the plugin is included as part of the build by checking the projects of the build. +To test a plugin with a custom build of OpenSearch, build OpenSearch and use the `customDistributionUrl` setting supported by each plugin to override the OpenSearch distribution. - ./gradlew projects +For example, in your OpenSearch repository assemble a custom distribution. + + ./gradlew :distribution:archives:linux-tar:assemble + +Then in your plugin repository, substitute in your OpenSearch build + + ./gradlew run -PcustomDistributionUrl="/distribution/archives/linux-tar/build/distributions/opensearch-min-3.0.0-SNAPSHOT-linux-x64.tar.gz" # Environment misc From aaa92aeeb75a745ee7b058cc4b30b734e861cfc4 Mon Sep 17 00:00:00 2001 From: "Samuel.G" <1148690954@qq.com> Date: Sat, 14 Sep 2024 04:54:02 +0800 Subject: [PATCH 21/44] Fix case insensitive query on wildcard field (#15882) * fix case insensitive query on wildcard field Signed-off-by: gesong.samuel * fix YAML test Signed-off-by: gesong.samuel * add change log Signed-off-by: gesong.samuel --------- Signed-off-by: gesong.samuel Co-authored-by: gesong.samuel --- CHANGELOG.md | 2 +- .../search/270_wildcard_fieldtype_queries.yml | 41 +++++++++++++++++-- .../index/mapper/WildcardFieldMapper.java | 15 +++++-- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3268852cc99f9..ec4ac7504b9c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed - Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737)) - +- Fix case-insensitive query on wildcard field ([#15882](https://github.com/opensearch-project/OpenSearch/pull/15882)) ### Security [Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.17...2.x diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml index c449a06cf633d..d92538824232d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml @@ -56,6 +56,12 @@ setup: id: 6 body: other_field: "test" + - do: + index: + index: test + id: 7 + body: + my_field: "ABCD" - do: indices.refresh: {} @@ -90,8 +96,9 @@ setup: query: term: my_field.lower: "abcd" - - match: { hits.total.value: 1 } + - match: { hits.total.value: 2 } - match: { hits.hits.0._id: "5" } + - match: { hits.hits.1._id: "7" } - do: search: @@ -100,8 +107,9 @@ setup: query: term: my_field.lower: "ABCD" - - match: { hits.total.value: 1 } + - match: { hits.total.value: 2 } - match: { hits.hits.0._id: "5" } + - match: { hits.hits.1._id: "7" } - do: search: @@ -215,7 +223,7 @@ setup: wildcard: my_field: value: "*" - - match: { hits.total.value: 5 } + - match: { hits.total.value: 6 } --- "regexp match-all works": - do: @@ -226,7 +234,7 @@ setup: regexp: my_field: value: ".*" - - match: { hits.total.value: 5 } + - match: { hits.total.value: 6 } --- "terms query on wildcard field matches": - do: @@ -237,3 +245,28 @@ setup: terms: { my_field: ["AbCd"] } - match: { hits.total.value: 1 } - match: { hits.hits.0._id: "5" } +--- +"case insensitive query on wildcard field": + - do: + search: + index: test + body: + query: + wildcard: + my_field: + value: "AbCd" + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "5" } + + - do: + search: + index: test + body: + query: + wildcard: + my_field: + value: "AbCd" + case_insensitive: true + - match: { hits.total.value: 2 } + - match: { hits.hits.0._id: "5" } + - match: { hits.hits.1._id: "7" } diff --git a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java index 0cb416a9b8370..e43e3bda692e7 100644 --- a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java @@ -40,6 +40,7 @@ import org.apache.lucene.util.automaton.RegExp; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.common.lucene.Lucene; +import org.opensearch.common.lucene.search.AutomatonQueries; import org.opensearch.common.unit.Fuzziness; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.analysis.IndexAnalyzers; @@ -464,7 +465,7 @@ public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, bo return existsQuery(context); } } else { - approximation = matchAllTermsQuery(name(), requiredNGrams); + approximation = matchAllTermsQuery(name(), requiredNGrams, caseInsensitive); } return new WildcardMatchingQuery(name(), approximation, matchPredicate, value, context, this); } @@ -678,7 +679,7 @@ public Query termsQuery(List values, QueryShardContext context) { StringBuilder pattern = new StringBuilder(); for (Object value : values) { String stringVal = BytesRefs.toString(value); - builder.add(matchAllTermsQuery(name(), getRequiredNGrams(stringVal)), BooleanClause.Occur.SHOULD); + builder.add(matchAllTermsQuery(name(), getRequiredNGrams(stringVal), false), BooleanClause.Occur.SHOULD); expectedValues.add(stringVal); if (pattern.length() > 0) { pattern.append('|'); @@ -688,10 +689,16 @@ public Query termsQuery(List values, QueryShardContext context) { return new WildcardMatchingQuery(name(), builder.build(), expectedValues::contains, pattern.toString(), context, this); } - private static BooleanQuery matchAllTermsQuery(String fieldName, Set terms) { + private static BooleanQuery matchAllTermsQuery(String fieldName, Set terms, boolean caseInsensitive) { BooleanQuery.Builder matchAllTermsBuilder = new BooleanQuery.Builder(); + Query query; for (String term : terms) { - matchAllTermsBuilder.add(new TermQuery(new Term(fieldName, term)), BooleanClause.Occur.FILTER); + if (caseInsensitive) { + query = AutomatonQueries.caseInsensitiveTermQuery(new Term(fieldName, term)); + } else { + query = new TermQuery(new Term(fieldName, term)); + } + matchAllTermsBuilder.add(query, BooleanClause.Occur.FILTER); } return matchAllTermsBuilder.build(); } From bd26056669f41826a500f944869b198149c476a2 Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Mon, 16 Sep 2024 14:58:07 +0530 Subject: [PATCH 22/44] Remove TooManyShardsInSnapshotsStatusException (#15877) Signed-off-by: Lakshya Taragi --- .../opensearch/OpenSearchServerException.java | 8 --- ...oManyShardsInSnapshotsStatusException.java | 69 ------------------- .../ExceptionSerializationTests.java | 2 - 3 files changed, 79 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java diff --git a/server/src/main/java/org/opensearch/OpenSearchServerException.java b/server/src/main/java/org/opensearch/OpenSearchServerException.java index a1fc61834f69b..b0ab62259d5c0 100644 --- a/server/src/main/java/org/opensearch/OpenSearchServerException.java +++ b/server/src/main/java/org/opensearch/OpenSearchServerException.java @@ -1210,14 +1210,6 @@ public static void registerExceptions() { V_2_17_0 ) ); - registerExceptionHandle( - new OpenSearchExceptionHandle( - org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException.class, - org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException::new, - 175, - V_2_17_0 - ) - ); registerExceptionHandle( new OpenSearchExceptionHandle( org.opensearch.cluster.block.IndexCreateBlockException.class, diff --git a/server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java b/server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java deleted file mode 100644 index 1689b3e4941ec..0000000000000 --- a/server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.snapshots; - -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.rest.RestStatus; - -import java.io.IOException; - -/** - * Thrown if the number of shards across the requested resources (snapshot(s) or the index/indices of a particular snapshot) - * breaches the limit of snapshot.max_shards_allowed_in_status_api cluster setting - * - * @opensearch.internal - */ -public class TooManyShardsInSnapshotsStatusException extends SnapshotException { - - public TooManyShardsInSnapshotsStatusException( - final String repositoryName, - final SnapshotId snapshotId, - final String message, - final Throwable cause - ) { - super(repositoryName, snapshotId, message, cause); - } - - public TooManyShardsInSnapshotsStatusException(final String repositoryName, final String message, String... snapshotName) { - super(repositoryName, String.join(", ", snapshotName), message); - } - - public TooManyShardsInSnapshotsStatusException(StreamInput in) throws IOException { - super(in); - } - - @Override - public RestStatus status() { - return RestStatus.REQUEST_ENTITY_TOO_LARGE; - } -} diff --git a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java index eff312a36dbc6..2e4a2d7bdd59c 100644 --- a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java @@ -119,7 +119,6 @@ import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotInProgressException; import org.opensearch.snapshots.SnapshotInUseDeletionException; -import org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; import org.opensearch.transport.ActionNotFoundTransportException; @@ -899,7 +898,6 @@ public void testIds() { ids.put(172, ViewNotFoundException.class); ids.put(173, ViewAlreadyExistsException.class); ids.put(174, InvalidIndexContextException.class); - ids.put(175, TooManyShardsInSnapshotsStatusException.class); ids.put(10001, IndexCreateBlockException.class); Map, Integer> reverse = new HashMap<>(); From c709400543a16d7f7d6d644f26bb2f7a10b90caf Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Mon, 16 Sep 2024 15:28:29 +0530 Subject: [PATCH 23/44] Revert "Mute failing snapshot status tests (#15652)" (#15876) This reverts commit 48cf5f061094d89aaa564b97667f4965556a8e49. Signed-off-by: Lakshya Taragi --- .../test/snapshot.status/10_basic.yml | 70 +++++++++---------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml index 819f04407b219..c35f2419bdc91 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml @@ -25,40 +25,38 @@ setup: snapshot: test_snapshot wait_for_completion: true -# TODO: fix and unmute tests + - do: + snapshot.status: + repository: test_repo_status_1 + snapshot: test_snapshot + + - is_true: snapshots + - match: { snapshots.0.snapshot: test_snapshot } + - match: { snapshots.0.state: SUCCESS } + - gt: { snapshots.0.stats.incremental.file_count: 0 } + - gt: { snapshots.0.stats.incremental.size_in_bytes: 0 } + - gt: { snapshots.0.stats.total.file_count: 0 } + - gt: { snapshots.0.stats.total.size_in_bytes: 0 } + - is_true: snapshots.0.stats.start_time_in_millis +## fast in memory snapshots can take less than one millisecond to complete. + - gte: { snapshots.0.stats.time_in_millis: 0 } + +--- +"Get missing snapshot status throws an exception": + + - do: + catch: /snapshot_missing_exception.+ is missing/ + snapshot.status: + repository: test_repo_status_1 + snapshot: test_nonexistent_snapshot + +--- +"Get missing snapshot status succeeds when ignoreUnavailable is true": + + - do: + snapshot.status: + repository: test_repo_status_1 + snapshot: test_nonexistent_snapshot + ignore_unavailable: true -# - do: -# snapshot.status: -# repository: test_repo_status_1 -# snapshot: test_snapshot -# -# - is_true: snapshots -# - match: { snapshots.0.snapshot: test_snapshot } -# - match: { snapshots.0.state: SUCCESS } -# - gt: { snapshots.0.stats.incremental.file_count: 0 } -# - gt: { snapshots.0.stats.incremental.size_in_bytes: 0 } -# - gt: { snapshots.0.stats.total.file_count: 0 } -# - gt: { snapshots.0.stats.total.size_in_bytes: 0 } -# - is_true: snapshots.0.stats.start_time_in_millis -### fast in memory snapshots can take less than one millisecond to complete. -# - gte: { snapshots.0.stats.time_in_millis: 0 } -# -#--- -#"Get missing snapshot status throws an exception": -# -# - do: -# catch: /snapshot_missing_exception.+ is missing/ -# snapshot.status: -# repository: test_repo_status_1 -# snapshot: test_nonexistent_snapshot -# -#--- -#"Get missing snapshot status succeeds when ignoreUnavailable is true": -# -# - do: -# snapshot.status: -# repository: test_repo_status_1 -# snapshot: test_nonexistent_snapshot -# ignore_unavailable: true -# -# - is_true: snapshots + - is_true: snapshots From 45a8ed41f0c67f8853e36d95c1cc05d921b92739 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 11:09:17 -0400 Subject: [PATCH 24/44] Bump com.gradle.develocity from 3.18 to 3.18.1 (#15947) Bumps com.gradle.develocity from 3.18 to 3.18.1. --- updated-dependencies: - dependency-name: com.gradle.develocity dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- settings.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle b/settings.gradle index b79c2aee135fc..8412d198a2a29 100644 --- a/settings.gradle +++ b/settings.gradle @@ -10,7 +10,7 @@ */ plugins { - id "com.gradle.develocity" version "3.18" + id "com.gradle.develocity" version "3.18.1" } ext.disableBuildCache = hasProperty('DISABLE_BUILD_CACHE') || System.getenv().containsKey('DISABLE_BUILD_CACHE') From e3bbc74868dcba47b4231c8d6e095c2721d019e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 11:40:39 -0400 Subject: [PATCH 25/44] Bump ch.qos.logback:logback-core from 1.5.6 to 1.5.8 in /test/fixtures/hdfs-fixture (#15946) * Bump ch.qos.logback:logback-core in /test/fixtures/hdfs-fixture Bumps [ch.qos.logback:logback-core](https://github.com/qos-ch/logback) from 1.5.6 to 1.5.8. - [Commits](https://github.com/qos-ch/logback/compare/v_1.5.6...v_1.5.8) --- updated-dependencies: - dependency-name: ch.qos.logback:logback-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec4ac7504b9c2..cc313fdcdb849 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858)) - Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863)) - Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862)) +- Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946)) ### Changed diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index b5cd12ef0c11f..20fd1058a181d 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -74,7 +74,7 @@ dependencies { api 'org.apache.zookeeper:zookeeper:3.9.2' api "org.apache.commons:commons-text:1.12.0" api "commons-net:commons-net:3.11.1" - api "ch.qos.logback:logback-core:1.5.6" + api "ch.qos.logback:logback-core:1.5.8" api "ch.qos.logback:logback-classic:1.2.13" api "org.jboss.xnio:xnio-nio:3.8.16.Final" api 'org.jline:jline:3.26.3' From 81288b1044348d99cce6d23770755a464121f7cc Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 18 Sep 2024 00:28:59 +0530 Subject: [PATCH 26/44] remote publication checksum stats (#15957) * Remote publication checksum stats Signed-off-by: Himshikha Gupta Co-authored-by: Himshikha Gupta --- .../remote/RemoteStatePublicationIT.java | 19 ++++++++++ .../remote/RemoteClusterStateService.java | 6 ++++ .../gateway/remote/RemoteDownloadStats.java | 36 +++++++++++++++++++ .../remote/RemotePersistenceStats.java | 24 ++++++++++--- .../RemoteClusterStateServiceTests.java | 10 ++++++ 5 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index faab3645ae894..ffb9352e8ba47 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -56,6 +56,7 @@ import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING_KEY; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteDownloadStats.CHECKSUM_VALIDATION_FAILED_COUNT; import static org.opensearch.gateway.remote.model.RemoteClusterBlocks.CLUSTER_BLOCKS; import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadata.COORDINATION_METADATA; import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_METADATA; @@ -405,10 +406,28 @@ private void assertDataNodeDownloadStats(NodesStatsResponse nodesStatsResponse) assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(0).getSuccessCount() > 0); assertEquals(0, dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(0).getFailedCount()); assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(0).getTotalTimeInMillis() > 0); + assertEquals( + 0, + dataNodeDiscoveryStats.getClusterStateStats() + .getPersistenceStats() + .get(0) + .getExtendedFields() + .get(CHECKSUM_VALIDATION_FAILED_COUNT) + .get() + ); assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(1).getSuccessCount() > 0); assertEquals(0, dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(1).getFailedCount()); assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(1).getTotalTimeInMillis() > 0); + assertEquals( + 0, + dataNodeDiscoveryStats.getClusterStateStats() + .getPersistenceStats() + .get(1) + .getExtendedFields() + .get(CHECKSUM_VALIDATION_FAILED_COUNT) + .get() + ); } private Map getMetadataFiles(BlobStoreRepository repository, String subDirectory) throws IOException { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 12d10fd908b44..ece29180f9cf5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -1644,6 +1644,12 @@ void validateClusterStateFromChecksum( failedValidation ) ); + if (isFullStateDownload) { + remoteStateStats.stateFullDownloadValidationFailed(); + } else { + remoteStateStats.stateDiffDownloadValidationFailed(); + } + if (isFullStateDownload && remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.FAILURE)) { throw new IllegalStateException( "Cluster state checksums do not match during full state read. Validation failed for " + failedValidation diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java new file mode 100644 index 0000000000000..a8f4b33a19c37 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.cluster.coordination.PersistedStateStats; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * Download stats for remote state + * + * @opensearch.internal + */ +public class RemoteDownloadStats extends PersistedStateStats { + static final String CHECKSUM_VALIDATION_FAILED_COUNT = "checksum_validation_failed_count"; + private AtomicLong checksumValidationFailedCount = new AtomicLong(0); + + public RemoteDownloadStats(String statsName) { + super(statsName); + addToExtendedFields(CHECKSUM_VALIDATION_FAILED_COUNT, checksumValidationFailedCount); + } + + public void checksumValidationFailedCount() { + checksumValidationFailedCount.incrementAndGet(); + } + + public long getChecksumValidationFailedCount() { + return checksumValidationFailedCount.get(); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java index 417ebdafd3ba7..11f26ac8b3ed9 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java @@ -18,16 +18,16 @@ public class RemotePersistenceStats { RemoteUploadStats remoteUploadStats; - PersistedStateStats remoteDiffDownloadStats; - PersistedStateStats remoteFullDownloadStats; + RemoteDownloadStats remoteDiffDownloadStats; + RemoteDownloadStats remoteFullDownloadStats; final String FULL_DOWNLOAD_STATS = "remote_full_download"; final String DIFF_DOWNLOAD_STATS = "remote_diff_download"; public RemotePersistenceStats() { remoteUploadStats = new RemoteUploadStats(); - remoteDiffDownloadStats = new PersistedStateStats(DIFF_DOWNLOAD_STATS); - remoteFullDownloadStats = new PersistedStateStats(FULL_DOWNLOAD_STATS); + remoteDiffDownloadStats = new RemoteDownloadStats(DIFF_DOWNLOAD_STATS); + remoteFullDownloadStats = new RemoteDownloadStats(FULL_DOWNLOAD_STATS); } public void cleanUpAttemptFailed() { @@ -90,6 +90,22 @@ public void stateDiffDownloadFailed() { remoteDiffDownloadStats.stateFailed(); } + public void stateDiffDownloadValidationFailed() { + remoteDiffDownloadStats.checksumValidationFailedCount(); + } + + public void stateFullDownloadValidationFailed() { + remoteFullDownloadStats.checksumValidationFailedCount(); + } + + public long getStateDiffDownloadValidationFailed() { + return remoteDiffDownloadStats.getChecksumValidationFailedCount(); + } + + public long getStateFullDownloadValidationFailed() { + return remoteFullDownloadStats.getChecksumValidationFailedCount(); + } + public PersistedStateStats getUploadStats() { return remoteUploadStats; } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index b11d5e48bec06..56857285fa8d3 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -3342,6 +3342,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithNullC anyString(), anyBoolean() ); + assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed()); } public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws IOException { @@ -3374,6 +3375,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws ); mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true); verify(mockService, times(1)).validateClusterStateFromChecksum(manifest, clusterState, ClusterName.DEFAULT.value(), NODE_ID, true); + assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed()); } public void testGetClusterStateForManifestWithChecksumValidationModeNone() throws IOException { @@ -3406,6 +3408,7 @@ public void testGetClusterStateForManifestWithChecksumValidationModeNone() throw ); mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true); verify(mockService, times(0)).validateClusterStateFromChecksum(any(), any(), anyString(), anyString(), anyBoolean()); + assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed()); } public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMismatch() throws IOException { @@ -3448,6 +3451,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMisma NODE_ID, true ); + assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed()); } public void testGetClusterStateForManifestWithChecksumValidationDebugWithMismatch() throws IOException { @@ -3494,6 +3498,7 @@ public void testGetClusterStateForManifestWithChecksumValidationDebugWithMismatc NODE_ID, true ); + assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed()); } public void testGetClusterStateUsingDiffWithChecksum() throws IOException { @@ -3535,6 +3540,7 @@ public void testGetClusterStateUsingDiffWithChecksum() throws IOException { eq(NODE_ID), eq(false) ); + assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed()); } public void testGetClusterStateUsingDiffWithChecksumModeNone() throws IOException { @@ -3576,6 +3582,7 @@ public void testGetClusterStateUsingDiffWithChecksumModeNone() throws IOExceptio eq(NODE_ID), eq(false) ); + assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed()); } public void testGetClusterStateUsingDiffWithChecksumModeDebugMismatch() throws IOException { @@ -3616,6 +3623,7 @@ public void testGetClusterStateUsingDiffWithChecksumModeDebugMismatch() throws I eq(NODE_ID), eq(false) ); + assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed()); } public void testGetClusterStateUsingDiffWithChecksumModeTraceMismatch() throws IOException { @@ -3677,6 +3685,7 @@ public void testGetClusterStateUsingDiffWithChecksumModeTraceMismatch() throws I eq(NODE_ID), eq(false) ); + assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed()); } public void testGetClusterStateUsingDiffWithChecksumMismatch() throws IOException { @@ -3738,6 +3747,7 @@ public void testGetClusterStateUsingDiffWithChecksumMismatch() throws IOExceptio eq(NODE_ID), eq(false) ); + assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed()); } private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { From 8347d0ec22aa865dea3bc4b2027fb7bdf486fab2 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Wed, 18 Sep 2024 03:12:05 +0800 Subject: [PATCH 27/44] Add validation for the search backpressure cancellation settings (#15501) * Fix updating search backpressure settings crashing the cluster Signed-off-by: Gao Binlong * Modify change log Signed-off-by: Gao Binlong * Fix version check Signed-off-by: Gao Binlong * Increase test coverage Signed-off-by: Gao Binlong * Format the code Signed-off-by: Gao Binlong * Optimize some code Signed-off-by: Gao Binlong * Fix typo Signed-off-by: Gao Binlong --------- Signed-off-by: Gao Binlong --- CHANGELOG.md | 2 + .../test/cluster.put_settings/10_basic.yml | 113 ++++++++++++++++++ .../opensearch/common/settings/Setting.java | 13 ++ .../SearchBackpressureService.java | 6 +- .../backpressure/SearchBackpressureState.java | 5 +- .../settings/SearchBackpressureSettings.java | 16 ++- .../settings/SearchShardTaskSettings.java | 16 ++- .../settings/SearchTaskSettings.java | 16 ++- .../common/settings/SettingTests.java | 28 +++++ .../SearchBackpressureSettingsTests.java | 28 +++++ 10 files changed, 231 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc313fdcdb849..db41e2eee16f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed - Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737)) - Fix case-insensitive query on wildcard field ([#15882](https://github.com/opensearch-project/OpenSearch/pull/15882)) +- Add validation for the search backpressure cancellation settings ([#15501](https://github.com/opensearch-project/OpenSearch/pull/15501)) + ### Security [Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.17...2.x diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml index 107d298b597d3..0f4a45e4591a3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml @@ -98,3 +98,116 @@ - match: {error.root_cause.0.type: "illegal_argument_exception"} - match: { error.root_cause.0.reason: "Invalid SearchBackpressureMode: monitor-only" } - match: { status: 400 } + + +--- +"Test setting search backpressure cancellation settings": + - skip: + version: "- 2.99.99" + reason: "Fixed in 3.0.0" + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_burst: 11 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.cancellation_burst: "11"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_rate: 0.1 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.cancellation_rate: "0.1"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_ratio: 0.2 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.cancellation_ratio: "0.2"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_burst: 12 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.cancellation_burst: "12"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_rate: 0.3 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.cancellation_rate: "0.3"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_ratio: 0.4 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.cancellation_ratio: "0.4"} + +--- +"Test setting invalid search backpressure cancellation_rate and cancellation_ratio": + - skip: + version: "- 2.99.99" + reason: "Fixed in 3.0.0" + + - do: + catch: /search_backpressure.search_task.cancellation_rate must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_rate: 0.0 + + - do: + catch: /search_backpressure.search_task.cancellation_ratio must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_ratio: 0.0 + + - do: + catch: /search_backpressure.search_shard_task.cancellation_rate must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_rate: 0.0 + + - do: + catch: /search_backpressure.search_shard_task.cancellation_ratio must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_ratio: 0.0 diff --git a/server/src/main/java/org/opensearch/common/settings/Setting.java b/server/src/main/java/org/opensearch/common/settings/Setting.java index fea4c165809ba..081029c1c106c 100644 --- a/server/src/main/java/org/opensearch/common/settings/Setting.java +++ b/server/src/main/java/org/opensearch/common/settings/Setting.java @@ -1855,6 +1855,10 @@ public static Setting doubleSetting( ); } + public static Setting doubleSetting(String key, double defaultValue, Validator validator, Property... properties) { + return new Setting<>(key, Double.toString(defaultValue), Double::parseDouble, validator, properties); + } + /** * A writeable parser for double * @@ -1961,6 +1965,15 @@ public static Setting doubleSetting( ); } + public static Setting doubleSetting( + String key, + Setting fallbackSetting, + Validator validator, + Property... properties + ) { + return new Setting<>(new SimpleKey(key), fallbackSetting, fallbackSetting::getRaw, Double::parseDouble, validator, properties); + } + /// simpleString public static Setting simpleString(String key, Property... properties) { diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java index 43b9f8ae87529..e98046ba1dede 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java +++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java @@ -158,14 +158,16 @@ public SearchBackpressureService( timeNanosSupplier, getSettings().getSearchTaskSettings().getCancellationRateNanos(), getSettings().getSearchTaskSettings().getCancellationBurst(), - getSettings().getSearchTaskSettings().getCancellationRatio() + getSettings().getSearchTaskSettings().getCancellationRatio(), + getSettings().getSearchTaskSettings().getCancellationRate() ), SearchShardTask.class, new SearchBackpressureState( timeNanosSupplier, getSettings().getSearchShardTaskSettings().getCancellationRateNanos(), getSettings().getSearchShardTaskSettings().getCancellationBurst(), - getSettings().getSearchShardTaskSettings().getCancellationRatio() + getSettings().getSearchShardTaskSettings().getCancellationRatio(), + getSettings().getSearchShardTaskSettings().getCancellationRate() ) ); this.settings.getSearchTaskSettings().addListener(searchBackpressureStates.get(SearchTask.class)); diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureState.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureState.java index 5f086bd498036..36f5b25e002c3 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureState.java +++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureState.java @@ -43,12 +43,15 @@ public class SearchBackpressureState implements CancellationSettingsListener { LongSupplier timeNanosSupplier, double cancellationRateNanos, double cancellationBurst, - double cancellationRatio + double cancellationRatio, + double cancellationRate ) { rateLimiter = new AtomicReference<>(new TokenBucket(timeNanosSupplier, cancellationRateNanos, cancellationBurst)); ratioLimiter = new AtomicReference<>(new TokenBucket(this::getCompletionCount, cancellationRatio, cancellationBurst)); this.timeNanosSupplier = timeNanosSupplier; this.cancellationBurst = cancellationBurst; + this.cancellationRatio = cancellationRatio; + this.cancellationRate = cancellationRate; } public long getCompletionCount() { diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java index 79494eb0d3c24..55a031382f282 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java +++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java @@ -61,8 +61,14 @@ private static class Defaults { public static final Setting SETTING_CANCELLATION_RATIO = Setting.doubleSetting( "search_backpressure.cancellation_ratio", Defaults.CANCELLATION_RATIO, - 0.0, - 1.0, + value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("search_backpressure.cancellation_ratio must be > 0"); + } + if (value > 1.0) { + throw new IllegalArgumentException("search_backpressure.cancellation_ratio must be <= 1.0"); + } + }, Setting.Property.Deprecated, Setting.Property.Dynamic, Setting.Property.NodeScope @@ -78,7 +84,11 @@ private static class Defaults { public static final Setting SETTING_CANCELLATION_RATE = Setting.doubleSetting( "search_backpressure.cancellation_rate", Defaults.CANCELLATION_RATE, - 0.0, + value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("search_backpressure.cancellation_rate must be > 0"); + } + }, Setting.Property.Deprecated, Setting.Property.Dynamic, Setting.Property.NodeScope diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java index 6d016c7466362..38213506c55b7 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java +++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java @@ -44,8 +44,14 @@ private static class Defaults { public static final Setting SETTING_CANCELLATION_RATIO = Setting.doubleSetting( "search_backpressure.search_shard_task.cancellation_ratio", SearchBackpressureSettings.SETTING_CANCELLATION_RATIO, - 0.0, - 1.0, + value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("search_backpressure.search_shard_task.cancellation_ratio must be > 0"); + } + if (value > 1.0) { + throw new IllegalArgumentException("search_backpressure.search_shard_task.cancellation_ratio must be <= 1.0"); + } + }, Setting.Property.Dynamic, Setting.Property.NodeScope ); @@ -58,7 +64,11 @@ private static class Defaults { public static final Setting SETTING_CANCELLATION_RATE = Setting.doubleSetting( "search_backpressure.search_shard_task.cancellation_rate", SearchBackpressureSettings.SETTING_CANCELLATION_RATE, - 0.0, + value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("search_backpressure.search_shard_task.cancellation_rate must be > 0"); + } + }, Setting.Property.Dynamic, Setting.Property.NodeScope ); diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java index 4b34323b1ddc6..f9af7f9b59fdb 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java +++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java @@ -48,8 +48,14 @@ private static class Defaults { public static final Setting SETTING_CANCELLATION_RATIO = Setting.doubleSetting( "search_backpressure.search_task.cancellation_ratio", Defaults.CANCELLATION_RATIO, - 0.0, - 1.0, + value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("search_backpressure.search_task.cancellation_ratio must be > 0"); + } + if (value > 1.0) { + throw new IllegalArgumentException("search_backpressure.search_task.cancellation_ratio must be <= 1.0"); + } + }, Setting.Property.Dynamic, Setting.Property.NodeScope ); @@ -62,7 +68,11 @@ private static class Defaults { public static final Setting SETTING_CANCELLATION_RATE = Setting.doubleSetting( "search_backpressure.search_task.cancellation_rate", Defaults.CANCELLATION_RATE, - 0.0, + value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("search_backpressure.search_task.cancellation_rate must be > 0"); + } + }, Setting.Property.Dynamic, Setting.Property.NodeScope ); diff --git a/server/src/test/java/org/opensearch/common/settings/SettingTests.java b/server/src/test/java/org/opensearch/common/settings/SettingTests.java index 7ebee680e8e52..c3c399a9d88b2 100644 --- a/server/src/test/java/org/opensearch/common/settings/SettingTests.java +++ b/server/src/test/java/org/opensearch/common/settings/SettingTests.java @@ -1274,6 +1274,20 @@ public void testFloatParser() throws Exception { public void testDoubleWithDefaultValue() { Setting doubleSetting = Setting.doubleSetting("foo.bar", 42.1); assertEquals(doubleSetting.get(Settings.EMPTY), Double.valueOf(42.1)); + + Setting doubleSettingWithValidator = Setting.doubleSetting("foo.bar", 42.1, value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("The setting foo.bar must be >0"); + } + }); + try { + assertThrows( + IllegalArgumentException.class, + () -> doubleSettingWithValidator.get(Settings.builder().put("foo.bar", randomFrom(-1, 0)).build()) + ); + } catch (IllegalArgumentException ex) { + assertEquals("The setting foo.bar must be >0", ex.getMessage()); + } } public void testDoubleWithFallbackValue() { @@ -1282,6 +1296,20 @@ public void testDoubleWithFallbackValue() { assertEquals(doubleSetting.get(Settings.EMPTY), Double.valueOf(2.1)); assertEquals(doubleSetting.get(Settings.builder().put("foo.bar", 3.2).build()), Double.valueOf(3.2)); assertEquals(doubleSetting.get(Settings.builder().put("foo.baz", 3.2).build()), Double.valueOf(3.2)); + + Setting doubleSettingWithValidator = Setting.doubleSetting("foo.bar", fallbackSetting, value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("The setting foo.bar must be >0"); + } + }); + try { + assertThrows( + IllegalArgumentException.class, + () -> doubleSettingWithValidator.get(Settings.builder().put("foo.bar", randomFrom(-1, 0)).build()) + ); + } catch (IllegalArgumentException ex) { + assertEquals("The setting foo.bar must be >0", ex.getMessage()); + } } public void testDoubleWithMinMax() throws Exception { diff --git a/server/src/test/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettingsTests.java b/server/src/test/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettingsTests.java index a02ca3cf877ad..683ada76c7683 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettingsTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettingsTests.java @@ -37,4 +37,32 @@ public void testSearchBackpressureSettingValidateInvalidMode() { () -> new SearchBackpressureSettings(settings, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) ); } + + public void testInvalidCancellationRate() { + Settings settings1 = Settings.builder().put("search_backpressure.search_task.cancellation_rate", randomFrom(-1, 0)).build(); + assertThrows( + IllegalArgumentException.class, + () -> new SearchBackpressureSettings(settings1, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + ); + + Settings settings2 = Settings.builder().put("search_backpressure.search_shard_task.cancellation_rate", randomFrom(-1, 0)).build(); + assertThrows( + IllegalArgumentException.class, + () -> new SearchBackpressureSettings(settings2, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + ); + } + + public void testInvalidCancellationRatio() { + Settings settings1 = Settings.builder().put("search_backpressure.search_task.cancellation_ratio", randomFrom(-1, 0)).build(); + assertThrows( + IllegalArgumentException.class, + () -> new SearchBackpressureSettings(settings1, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + ); + + Settings settings2 = Settings.builder().put("search_backpressure.search_shard_task.cancellation_ratio", randomFrom(-1, 0)).build(); + assertThrows( + IllegalArgumentException.class, + () -> new SearchBackpressureSettings(settings2, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + ); + } } From eb5b70370e548cd0f3b51721397df818b3050daa Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 17 Sep 2024 15:44:32 -0400 Subject: [PATCH 28/44] Fix flaky terminaton conditions for org.opensearch.rest.ReactorNetty4StreamingStressIT.testCloseClientStreamingRequest test case (#15959) Signed-off-by: Andriy Redko --- .../org/opensearch/rest/ReactorNetty4StreamingStressIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java index a853ed56fad32..9da456f618ffc 100644 --- a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java +++ b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java @@ -8,6 +8,7 @@ package org.opensearch.rest; +import org.apache.hc.core5.http.ConnectionClosedException; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.StreamingRequest; @@ -75,7 +76,7 @@ public void testCloseClientStreamingRequest() throws Exception { } }) .then(() -> scheduler.advanceTimeBy(delay)) - .expectErrorMatches(t -> t instanceof InterruptedIOException) + .expectErrorMatches(t -> t instanceof InterruptedIOException || t instanceof ConnectionClosedException) .verify(); } } From 7c427d9fd588597343654d2f534952f7116d1df4 Mon Sep 17 00:00:00 2001 From: David Zane <38449481+dzane17@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:29:12 -0700 Subject: [PATCH 29/44] Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder (#15916) Signed-off-by: David Zane --- CHANGELOG.md | 1 + .../support/ValuesSourceAggregationBuilder.java | 10 +++++++++- .../org/opensearch/search/sort/FieldSortBuilder.java | 8 +++++++- .../bucket/range/RangeAggregationBuilderTests.java | 1 + .../opensearch/search/sort/FieldSortBuilderTests.java | 1 + 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db41e2eee16f5..309d9cb06cc94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) +- Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java index 7a73fafb4a809..1ccceb1d77dcb 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java @@ -40,6 +40,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.WithFieldName; import org.opensearch.script.Script; import org.opensearch.search.aggregations.AbstractAggregationBuilder; import org.opensearch.search.aggregations.AggregationInitializationException; @@ -57,7 +58,9 @@ * * @opensearch.internal */ -public abstract class ValuesSourceAggregationBuilder> extends AbstractAggregationBuilder { +public abstract class ValuesSourceAggregationBuilder> extends AbstractAggregationBuilder + implements + WithFieldName { public static void declareFields( AbstractObjectParser, T> objectParser, @@ -292,6 +295,11 @@ public String field() { return field; } + @Override + public String fieldName() { + return field(); + } + /** * Sets the script to use for this aggregation. */ diff --git a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java index 5cecda1346b90..9825b2cbbe08e 100644 --- a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java @@ -65,6 +65,7 @@ import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryShardException; +import org.opensearch.index.query.WithFieldName; import org.opensearch.search.DocValueFormat; import org.opensearch.search.MultiValueMode; import org.opensearch.search.SearchSortValuesAndFormats; @@ -86,7 +87,7 @@ * * @opensearch.internal */ -public class FieldSortBuilder extends SortBuilder { +public class FieldSortBuilder extends SortBuilder implements WithFieldName { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(FieldSortBuilder.class); public static final String NAME = "field_sort"; @@ -184,6 +185,11 @@ public String getFieldName() { return this.fieldName; } + @Override + public String fieldName() { + return getFieldName(); + } + /** * Sets the value when a field is missing in a doc. Can also be set to {@code _last} or * {@code _first} to sort missing last or first respectively. diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java index 4362ce48003cc..14532e30f8984 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java @@ -128,6 +128,7 @@ public void testNumericKeys() throws IOException { ); assertThat(builder.getName(), equalTo("test")); assertThat(builder.field(), equalTo("f")); + assertThat(builder.fieldName(), equalTo("f")); assertThat(builder.ranges, equalTo(List.of(new RangeAggregator.Range("1", null, 0d)))); } } diff --git a/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java b/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java index 9b8cd1b5f1ce0..ced952db555aa 100644 --- a/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java @@ -196,6 +196,7 @@ protected void sortFieldAssertions(FieldSortBuilder builder, SortField sortField assertEquals(builder.order() == SortOrder.ASC ? false : true, sortField.getReverse()); if (expectedType == SortField.Type.CUSTOM) { assertEquals(builder.getFieldName(), sortField.getField()); + assertEquals(builder.fieldName(), sortField.getField()); } assertEquals(DocValueFormat.RAW, format); } From b2a7136722e9972edcf14ff03abd4f86ae4923b9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:58:08 +0800 Subject: [PATCH 30/44] Bump com.microsoft.azure:msal4j from 1.17.0 to 1.17.1 in /plugins/repository-azure (#15945) * Bump com.microsoft.azure:msal4j in /plugins/repository-azure Bumps [com.microsoft.azure:msal4j](https://github.com/AzureAD/microsoft-authentication-library-for-java) from 1.17.0 to 1.17.1. - [Release notes](https://github.com/AzureAD/microsoft-authentication-library-for-java/releases) - [Changelog](https://github.com/AzureAD/microsoft-authentication-library-for-java/blob/dev/changelog.txt) - [Commits](https://github.com/AzureAD/microsoft-authentication-library-for-java/commits) --- updated-dependencies: - dependency-name: com.microsoft.azure:msal4j dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Signed-off-by: gaobinlong Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: gaobinlong --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1 | 1 - plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1 create mode 100644 plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 309d9cb06cc94..4ed85b630bbd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858)) - Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863)) - Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862)) +- Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945)) - Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946)) ### Changed diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 5669c5e559ac8..616da0be62c30 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -61,7 +61,7 @@ dependencies { // Start of transitive dependencies for azure-identity api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0' api "net.java.dev.jna:jna-platform:${versions.jna}" - api 'com.microsoft.azure:msal4j:1.17.0' + api 'com.microsoft.azure:msal4j:1.17.1' api 'com.nimbusds:oauth2-oidc-sdk:11.19.1' api 'com.nimbusds:nimbus-jose-jwt:9.40' api 'com.nimbusds:content-type:2.3' diff --git a/plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1 deleted file mode 100644 index 34101c989eecd..0000000000000 --- a/plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7d37157da92b719f250b0023234ac9dda922a2a5 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 new file mode 100644 index 0000000000000..46c14e819b630 --- /dev/null +++ b/plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 @@ -0,0 +1 @@ +4eb31a9919d9b103c548af7e37e6f9d9f6e46dbc \ No newline at end of file From 4802d0d744cfd19827ab61c0807f557b768bd1aa Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 18 Sep 2024 07:57:38 -0400 Subject: [PATCH 31/44] Update version in README to 2.18.0 (#15968) Signed-off-by: Craig Perkins --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5d4a9a671c013..9294d739a0103 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![Security Vulnerabilities](https://img.shields.io/github/issues/opensearch-project/OpenSearch/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability") [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues) [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls) -[![2.17.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.17.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.17.0") +[![2.18.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.18.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.18.0") [![3.0.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0") [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml) [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml) From 3937ccb123aa5ced3c963ec2eef5e07c2085b004 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 15:47:49 -0700 Subject: [PATCH 32/44] [AUTO] [main] Add bwc version 2.17.1. (#15966) * Resolve readme conflicts Signed-off-by: Kunal Kotwani * Fix version identifier Signed-off-by: Kunal Kotwani --------- Signed-off-by: Kunal Kotwani Co-authored-by: Kunal Kotwani --- .ci/bwcVersions | 1 + README.md | 1 + libs/core/src/main/java/org/opensearch/Version.java | 1 + 3 files changed, 3 insertions(+) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 0f59fec3d824a..ccb47c1a3b724 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -38,4 +38,5 @@ BWC_VERSION: - "2.16.0" - "2.16.1" - "2.17.0" + - "2.17.1" - "2.18.0" diff --git a/README.md b/README.md index 9294d739a0103..03728aee0135c 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues) [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls) [![2.18.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.18.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.18.0") +[![2.17.1 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.17.1)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.17.1") [![3.0.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0") [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml) [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml) diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index ca5dd306cf907..b86b8459fb8a8 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -109,6 +109,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_16_0 = new Version(2160099, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_16_1 = new Version(2160199, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_17_0 = new Version(2170099, org.apache.lucene.util.Version.LUCENE_9_11_1); + public static final Version V_2_17_1 = new Version(2170199, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_12_0); public static final Version CURRENT = V_3_0_0; From 3a1b6d1795f23f2d14c4f2174867ce70ed88525e Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad <61760125+gaiksaya@users.noreply.github.com> Date: Wed, 18 Sep 2024 16:14:00 -0700 Subject: [PATCH 33/44] Refactor branch deletion for backports (#15983) Signed-off-by: Sayali Gaikawad --- .github/workflows/delete_backport_branch.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml index 387a124b8cb6a..b60c2225db26a 100644 --- a/.github/workflows/delete_backport_branch.yml +++ b/.github/workflows/delete_backport_branch.yml @@ -7,9 +7,16 @@ on: jobs: delete-branch: runs-on: ubuntu-latest + permissions: + contents: write if: startsWith(github.event.pull_request.head.ref,'backport/') steps: - name: Delete merged branch - uses: SvanBoxel/delete-merged-branch@main - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: actions/github-script@v5 + with: + script: | + github.rest.git.deleteRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `heads/${context.payload.pull_request.head.ref}`, + }) From 80ff07eb0c5485832e6fcbc46a9d100435ed3051 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Thu, 19 Sep 2024 12:50:22 +0530 Subject: [PATCH 34/44] star tree file formats refactoring and fixing offset bug (#15975) Signed-off-by: Bharathwaj G --- .../startree/builder/BaseStarTreeBuilder.java | 31 +++--- .../builder/OffHeapStarTreeBuilder.java | 4 +- .../builder/OnHeapStarTreeBuilder.java | 4 +- .../builder/StarTreeDocsFileManager.java | 43 +++----- .../startree/index/StarTreeValues.java | 57 +++++----- .../utils/SequentialDocValuesIterator.java | 84 ++++++--------- .../SortedNumericStarTreeValuesIterator.java | 32 ++++++ .../iterator/StarTreeValuesIterator.java | 48 +++++++++ .../startree/utils/iterator/package-info.java | 14 +++ .../datacube/startree/StarTreeTestUtils.java | 8 +- .../builder/AbstractStarTreeBuilderTests.java | 101 +++++++++++------- .../SequentialDocValuesIteratorTests.java | 64 +++++------ 12 files changed, 281 insertions(+), 209 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/package-info.java diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index a1d638616f2aa..5e5814528fcd2 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -34,6 +34,7 @@ import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; import org.opensearch.index.mapper.DocCountFieldMapper; import org.opensearch.index.mapper.FieldMapper; import org.opensearch.index.mapper.FieldValueConverter; @@ -193,7 +194,9 @@ public List getMetricReaders(SegmentWriteState stat metricFieldInfo = getFieldInfo(metric.getField(), DocValuesType.SORTED_NUMERIC); } metricReader = new SequentialDocValuesIterator( - fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo) + new SortedNumericStarTreeValuesIterator( + fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo) + ) ); } metricReaders.add(metricReader); @@ -228,7 +231,7 @@ public void build( dimensionFieldInfo = getFieldInfo(dimension, DocValuesType.SORTED_NUMERIC); } dimensionReaders[i] = new SequentialDocValuesIterator( - fieldProducerMap.get(dimensionFieldInfo.name).getSortedNumeric(dimensionFieldInfo) + new SortedNumericStarTreeValuesIterator(fieldProducerMap.get(dimensionFieldInfo.name).getSortedNumeric(dimensionFieldInfo)) ); } Iterator starTreeDocumentIterator = sortAndAggregateSegmentDocuments(dimensionReaders, metricReaders); @@ -287,7 +290,7 @@ void appendDocumentsToStarTree(Iterator starTreeDocumentIterat } } - private void serializeStarTree(int numSegmentStarTreeDocument, int numStarTreeDocs) throws IOException { + private void serializeStarTree(int numSegmentStarTreeDocuments, int numStarTreeDocs) throws IOException { // serialize the star tree data long dataFilePointer = dataOut.getFilePointer(); StarTreeWriter starTreeWriter = new StarTreeWriter(); @@ -299,7 +302,7 @@ private void serializeStarTree(int numSegmentStarTreeDocument, int numStarTreeDo starTreeField, metricAggregatorInfos, numStarTreeNodes, - numSegmentStarTreeDocument, + numSegmentStarTreeDocuments, numStarTreeDocs, dataFilePointer, totalStarTreeDataLength @@ -400,22 +403,20 @@ protected StarTreeDocument getStarTreeDocument( ) throws IOException { Long[] dims = new Long[numDimensions]; int i = 0; - for (SequentialDocValuesIterator dimensionDocValueIterator : dimensionReaders) { - dimensionDocValueIterator.nextDoc(currentDocId); - Long val = dimensionDocValueIterator.value(currentDocId); + for (SequentialDocValuesIterator dimensionValueIterator : dimensionReaders) { + dimensionValueIterator.nextEntry(currentDocId); + Long val = dimensionValueIterator.value(currentDocId); dims[i] = val; i++; } i = 0; Object[] metrics = new Object[metricReaders.size()]; - for (SequentialDocValuesIterator metricDocValuesIterator : metricReaders) { - metricDocValuesIterator.nextDoc(currentDocId); + for (SequentialDocValuesIterator metricValuesIterator : metricReaders) { + metricValuesIterator.nextEntry(currentDocId); // As part of merge, we traverse the star tree doc values // The type of data stored in metric fields is different from the // actual indexing field they're based on - metrics[i] = metricAggregatorInfos.get(i) - .getValueAggregators() - .toAggregatedValueType(metricDocValuesIterator.value(currentDocId)); + metrics[i] = metricAggregatorInfos.get(i).getValueAggregators().toAggregatedValueType(metricValuesIterator.value(currentDocId)); i++; } return new StarTreeDocument(dims, metrics); @@ -502,7 +503,7 @@ Long[] getStarTreeDimensionsFromSegment(int currentDocId, SequentialDocValuesIte for (int i = 0; i < numDimensions; i++) { if (dimensionReaders[i] != null) { try { - dimensionReaders[i].nextDoc(currentDocId); + dimensionReaders[i].nextEntry(currentDocId); } catch (IOException e) { logger.error("unable to iterate to next doc", e); throw new RuntimeException("unable to iterate to next doc", e); @@ -530,7 +531,7 @@ private Object[] getStarTreeMetricsFromSegment(int currentDocId, List mergeStarTrees(List starTreeValuesSub .size()]; for (int i = 0; i < dimensionsSplitOrder.size(); i++) { String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension)); + dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionValuesIterator(dimension)); } List metricReaders = new ArrayList<>(); // get doc id set iterators for metrics @@ -164,7 +164,7 @@ Iterator mergeStarTrees(List starTreeValuesSub metric.getField(), metricStat.getTypeName() ); - metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName))); + metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName))); } } int currentDocId = 0; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index 1a5c906ad413b..13c6d03c4dc3d 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -138,7 +138,7 @@ StarTreeDocument[] getSegmentsStarTreeDocuments(List starTreeVal for (int i = 0; i < dimensionsSplitOrder.size(); i++) { String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension)); + dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionValuesIterator(dimension)); } List metricReaders = new ArrayList<>(); @@ -150,7 +150,7 @@ StarTreeDocument[] getSegmentsStarTreeDocuments(List starTreeVal metric.getField(), metricStat.getTypeName() ); - metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName))); + metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName))); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java index 779ed77b0540a..15ed153249243 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java @@ -21,7 +21,6 @@ import java.io.Closeable; import java.io.IOException; -import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -55,11 +54,9 @@ public class StarTreeDocsFileManager extends AbstractDocumentsFileManager implem private RandomAccessInput starTreeDocsFileRandomInput; private IndexOutput starTreeDocsFileOutput; private final Map fileToEndDocIdMap; - private final List starTreeDocumentOffsets = new ArrayList<>(); private int currentFileStartDocId; private int numReadableStarTreeDocuments; private int starTreeFileCount = -1; - private int currBytes = 0; private final int fileCountMergeThreshold; private int numStarTreeDocs = 0; @@ -98,7 +95,11 @@ IndexOutput createStarTreeDocumentsFileOutput() throws IOException { public void writeStarTreeDocument(StarTreeDocument starTreeDocument, boolean isAggregatedDoc) throws IOException { assert isAggregatedDoc == true; int numBytes = writeStarTreeDocument(starTreeDocument, starTreeDocsFileOutput, true); - addStarTreeDocumentOffset(numBytes); + if (docSizeInBytes == -1) { + docSizeInBytes = numBytes; + } else { + assert docSizeInBytes == numBytes; + } numStarTreeDocs++; } @@ -106,7 +107,14 @@ public void writeStarTreeDocument(StarTreeDocument starTreeDocument, boolean isA public StarTreeDocument readStarTreeDocument(int docId, boolean isAggregatedDoc) throws IOException { assert isAggregatedDoc == true; ensureDocumentReadable(docId); - return readStarTreeDocument(starTreeDocsFileRandomInput, starTreeDocumentOffsets.get(docId), true); + return readStarTreeDocument(starTreeDocsFileRandomInput, getOffset(docId), true); + } + + /** + * Returns offset for the docId based on the current file start id + */ + private long getOffset(int docId) { + return (long) (docId - currentFileStartDocId) * docSizeInBytes; } @Override @@ -119,19 +127,10 @@ public Long getDimensionValue(int docId, int dimensionId) throws IOException { public Long[] readDimensions(int docId) throws IOException { ensureDocumentReadable(docId); Long[] dims = new Long[starTreeField.getDimensionsOrder().size()]; - readDimensions(dims, starTreeDocsFileRandomInput, starTreeDocumentOffsets.get(docId)); + readDimensions(dims, starTreeDocsFileRandomInput, getOffset(docId)); return dims; } - private void addStarTreeDocumentOffset(int bytes) { - starTreeDocumentOffsets.add(currBytes); - currBytes += bytes; - if (docSizeInBytes == -1) { - docSizeInBytes = bytes; - } - assert docSizeInBytes == bytes; - } - /** * Load the correct StarTreeDocuments file based on the docId */ @@ -199,7 +198,6 @@ private void loadStarTreeDocumentFile(int docId) throws IOException { * If the operation is only for reading existing documents, a new file is not created. */ private void closeAndMaybeCreateNewFile(boolean shouldCreateFileForAppend, int numStarTreeDocs) throws IOException { - currBytes = 0; if (starTreeDocsFileOutput != null) { fileToEndDocIdMap.put(starTreeDocsFileOutput.getName(), numStarTreeDocs); IOUtils.close(starTreeDocsFileOutput); @@ -232,7 +230,6 @@ private void mergeFiles(int numStarTreeDocs) throws IOException { deleteOldFiles(); fileToEndDocIdMap.clear(); fileToEndDocIdMap.put(mergedOutput.getName(), numStarTreeDocs); - resetStarTreeDocumentOffsets(); } } @@ -259,17 +256,6 @@ private void deleteOldFiles() throws IOException { } } - /** - * Reset the star tree document offsets based on the merged file - */ - private void resetStarTreeDocumentOffsets() { - int curr = 0; - for (int i = 0; i < starTreeDocumentOffsets.size(); i++) { - starTreeDocumentOffsets.set(i, curr); - curr += docSizeInBytes; - } - } - @Override public void close() { try { @@ -288,7 +274,6 @@ public void close() { tmpDirectory.deleteFile(file); } catch (IOException ignored) {} // similar to IOUtils.deleteFilesIgnoringExceptions } - starTreeDocumentOffsets.clear(); fileToEndDocIdMap.clear(); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java index a34bbbe9ee738..255ad343cde32 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.IndexInput; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.compositeindex.CompositeIndexMetadata; @@ -25,6 +24,8 @@ import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeFactory; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; import java.io.IOException; import java.util.ArrayList; @@ -59,14 +60,14 @@ public class StarTreeValues implements CompositeIndexValues { private final StarTreeNode root; /** - * A map containing suppliers for DocIdSetIterators for dimensions. + * A map containing suppliers for StarTreeValues iterators for dimensions. */ - private final Map> dimensionDocValuesIteratorMap; + private final Map> dimensionValuesIteratorMap; /** - * A map containing suppliers for DocIdSetIterators for metrics. + * A map containing suppliers for StarTreeValues iterators for metrics. */ - private final Map> metricDocValuesIteratorMap; + private final Map> metricValuesIteratorMap; /** * A map containing attributes associated with the star tree values. @@ -84,22 +85,22 @@ public class StarTreeValues implements CompositeIndexValues { * * @param starTreeField The StarTreeField object representing the star tree field configuration. * @param root The root node of the star tree. - * @param dimensionDocValuesIteratorMap A map containing suppliers for DocIdSetIterators for dimensions. - * @param metricDocValuesIteratorMap A map containing suppliers for DocIdSetIterators for metrics. + * @param dimensionValuesIteratorMap A map containing suppliers for StarTreeValues iterators for dimensions. + * @param metricValuesIteratorMap A map containing suppliers for StarTreeValues iterators for metrics. * @param attributes A map containing attributes associated with the star tree values. */ public StarTreeValues( StarTreeField starTreeField, StarTreeNode root, - Map> dimensionDocValuesIteratorMap, - Map> metricDocValuesIteratorMap, + Map> dimensionValuesIteratorMap, + Map> metricValuesIteratorMap, Map attributes, StarTreeMetadata compositeIndexMetadata ) { this.starTreeField = starTreeField; this.root = root; - this.dimensionDocValuesIteratorMap = dimensionDocValuesIteratorMap; - this.metricDocValuesIteratorMap = metricDocValuesIteratorMap; + this.dimensionValuesIteratorMap = dimensionValuesIteratorMap; + this.metricValuesIteratorMap = metricValuesIteratorMap; this.attributes = attributes; this.starTreeMetadata = compositeIndexMetadata; } @@ -146,12 +147,12 @@ public StarTreeValues( this.root = StarTreeFactory.createStarTree(compositeIndexDataIn, starTreeMetadata); // get doc id set iterators for metrics and dimensions - dimensionDocValuesIteratorMap = new LinkedHashMap<>(); - metricDocValuesIteratorMap = new LinkedHashMap<>(); + dimensionValuesIteratorMap = new LinkedHashMap<>(); + metricValuesIteratorMap = new LinkedHashMap<>(); // get doc id set iterators for dimensions for (String dimension : starTreeMetadata.getDimensionFields()) { - dimensionDocValuesIteratorMap.put(dimension, () -> { + dimensionValuesIteratorMap.put(dimension, () -> { try { SortedNumericDocValues dimensionSortedNumericDocValues = null; if (readState != null) { @@ -162,9 +163,9 @@ public StarTreeValues( dimensionSortedNumericDocValues = compositeDocValuesProducer.getSortedNumeric(dimensionfieldInfo); } } - return getSortedNumericDocValues(dimensionSortedNumericDocValues); + return new SortedNumericStarTreeValuesIterator(getSortedNumericDocValues(dimensionSortedNumericDocValues)); } catch (IOException e) { - throw new RuntimeException("Error loading dimension DocIdSetIterator", e); + throw new RuntimeException("Error loading dimension StarTreeValuesIterator", e); } }); } @@ -177,7 +178,7 @@ public StarTreeValues( metric.getField(), metricStat.getTypeName() ); - metricDocValuesIteratorMap.put(metricFullName, () -> { + metricValuesIteratorMap.put(metricFullName, () -> { try { SortedNumericDocValues metricSortedNumericDocValues = null; if (readState != null) { @@ -186,7 +187,7 @@ public StarTreeValues( metricSortedNumericDocValues = compositeDocValuesProducer.getSortedNumeric(metricFieldInfo); } } - return getSortedNumericDocValues(metricSortedNumericDocValues); + return new SortedNumericStarTreeValuesIterator(getSortedNumericDocValues(metricSortedNumericDocValues)); } catch (IOException e) { throw new RuntimeException("Error loading metric DocIdSetIterator", e); } @@ -239,30 +240,30 @@ public Map getAttributes() { } /** - * Returns the DocIdSetIterator for the specified dimension. + * Returns the StarTreeValues iterator for the specified dimension. * * @param dimension The name of the dimension. - * @return The DocIdSetIterator for the specified dimension. + * @return The StarTreeValuesIterator for the specified dimension. */ - public DocIdSetIterator getDimensionDocIdSetIterator(String dimension) { + public StarTreeValuesIterator getDimensionValuesIterator(String dimension) { - if (dimensionDocValuesIteratorMap.containsKey(dimension)) { - return dimensionDocValuesIteratorMap.get(dimension).get(); + if (dimensionValuesIteratorMap.containsKey(dimension)) { + return dimensionValuesIteratorMap.get(dimension).get(); } throw new IllegalArgumentException("dimension [" + dimension + "] does not exist in the segment."); } /** - * Returns the DocIdSetIterator for the specified fully qualified metric name. + * Returns the StarTreeValues iterator for the specified fully qualified metric name. * * @param fullyQualifiedMetricName The fully qualified name of the metric. - * @return The DocIdSetIterator for the specified fully qualified metric name. + * @return The StarTreeValuesIterator for the specified fully qualified metric name. */ - public DocIdSetIterator getMetricDocIdSetIterator(String fullyQualifiedMetricName) { + public StarTreeValuesIterator getMetricValuesIterator(String fullyQualifiedMetricName) { - if (metricDocValuesIteratorMap.containsKey(fullyQualifiedMetricName)) { - return metricDocValuesIteratorMap.get(fullyQualifiedMetricName).get(); + if (metricValuesIteratorMap.containsKey(fullyQualifiedMetricName)) { + return metricValuesIteratorMap.get(fullyQualifiedMetricName).get(); } throw new IllegalArgumentException("metric [" + fullyQualifiedMetricName + "] does not exist in the segment."); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java index 061841d3e140a..9029a451ca4d9 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java @@ -9,15 +9,19 @@ package org.opensearch.index.compositeindex.datacube.startree.utils; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; import java.io.IOException; /** - * Coordinates the reading of documents across multiple DocIdSetIterators. - * It encapsulates a single DocIdSetIterator and maintains the latest document ID and its associated value. + * Coordinates the reading of documents across multiple StarTreeValuesIterator. + * It encapsulates a single StarTreeValuesIterator and maintains the latest document ID and its associated value. + * + * In case of merge , this will be reading the entries of star tree values and in case of flush this will go through + * the actual segment documents. + * * @opensearch.experimental */ @ExperimentalApi @@ -26,76 +30,56 @@ public class SequentialDocValuesIterator { /** * The doc id set iterator associated for each field. */ - private final DocIdSetIterator docIdSetIterator; + private final StarTreeValuesIterator starTreeValuesIterator; /** - * The value associated with the latest document. + * The id of the latest record/entry. */ - private Long docValue; + private int entryId = -1; - /** - * The id of the latest document. - */ - private int docId = -1; - - /** - * Constructs a new SequentialDocValuesIterator instance with the given DocIdSetIterator. - * - * @param docIdSetIterator the DocIdSetIterator to be associated with this instance - */ - public SequentialDocValuesIterator(DocIdSetIterator docIdSetIterator) { - this.docIdSetIterator = docIdSetIterator; - } - - /** - * Returns the id of the latest document. - * - * @return the id of the latest document - */ - public int getDocId() { - return docId; + public SequentialDocValuesIterator(StarTreeValuesIterator starTreeValuesIterator) { + this.starTreeValuesIterator = starTreeValuesIterator; } /** - * Sets the id of the latest document. + * Returns the ID of the star tree record/entry or the segment document id * - * @param docId the ID of the latest document + * @return the ID of the star tree record/entry or the segment document id */ - private void setDocId(int docId) { - this.docId = docId; + public int getEntryId() { + return entryId; } /** - * Returns the DocIdSetIterator associated with this instance. + * Sets the id of the latest entry. * - * @return the DocIdSetIterator associated with this instance + * @param entryId the ID of the star tree record/entry or the segment document id */ - public DocIdSetIterator getDocIdSetIterator() { - return docIdSetIterator; + private void setEntryId(int entryId) { + this.entryId = entryId; } - public int nextDoc(int currentDocId) throws IOException { + public int nextEntry(int currentEntryId) throws IOException { // if doc id stored is less than or equal to the requested doc id , return the stored doc id - if (docId >= currentDocId) { - return docId; + if (entryId >= currentEntryId) { + return entryId; } - setDocId(this.docIdSetIterator.nextDoc()); - return docId; + setEntryId(this.starTreeValuesIterator.nextEntry()); + return entryId; } - public Long value(int currentDocId) throws IOException { - if (this.getDocIdSetIterator() instanceof SortedNumericDocValues) { - SortedNumericDocValues sortedNumericDocValues = (SortedNumericDocValues) this.getDocIdSetIterator(); - if (currentDocId < 0) { - throw new IllegalStateException("invalid doc id to fetch the next value"); + public Long value(int currentEntryId) throws IOException { + if (starTreeValuesIterator instanceof SortedNumericStarTreeValuesIterator) { + if (currentEntryId < 0) { + throw new IllegalStateException("invalid entry id to fetch the next value"); } - if (currentDocId == DocIdSetIterator.NO_MORE_DOCS) { - throw new IllegalStateException("DocValuesIterator is already exhausted"); + if (currentEntryId == StarTreeValuesIterator.NO_MORE_ENTRIES) { + throw new IllegalStateException("StarTreeValuesIterator is already exhausted"); } - if (docId == DocIdSetIterator.NO_MORE_DOCS || docId != currentDocId) { + if (entryId == StarTreeValuesIterator.NO_MORE_ENTRIES || entryId != currentEntryId) { return null; } - return sortedNumericDocValues.nextValue(); + return ((SortedNumericStarTreeValuesIterator) starTreeValuesIterator).nextValue(); } else { throw new IllegalStateException("Unsupported Iterator requested for SequentialDocValuesIterator"); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java new file mode 100644 index 0000000000000..27afdf1479b4e --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils.iterator; + +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.common.annotation.ExperimentalApi; + +import java.io.IOException; + +/** + * Wrapper iterator class for StarTree index to traverse through SortedNumericDocValues + * + * @opensearch.experimental + */ +@ExperimentalApi +public class SortedNumericStarTreeValuesIterator extends StarTreeValuesIterator { + + public SortedNumericStarTreeValuesIterator(DocIdSetIterator docIdSetIterator) { + super(docIdSetIterator); + } + + public long nextValue() throws IOException { + return ((SortedNumericDocValues) docIdSetIterator).nextValue(); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java new file mode 100644 index 0000000000000..32866f3e50092 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils.iterator; + +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.common.annotation.ExperimentalApi; + +import java.io.IOException; + +/** + * Wrapper iterator class for StarTree index in place of DocIdSetIterator to read / traverse the docValues formats. + * This is needed since star tree values are different from segment documents and number of star tree values + * can even exceed segment docs in the worst cases. + * + * @opensearch.experimental + */ +@ExperimentalApi +public abstract class StarTreeValuesIterator { + + public static final int NO_MORE_ENTRIES = Integer.MAX_VALUE; + protected final DocIdSetIterator docIdSetIterator; + + public StarTreeValuesIterator(DocIdSetIterator docIdSetIterator) { + this.docIdSetIterator = docIdSetIterator; + } + + public int entryId() { + return docIdSetIterator.docID(); + } + + public int nextEntry() throws IOException { + return docIdSetIterator.nextDoc(); + } + + public int advance(int target) throws IOException { + return docIdSetIterator.advance(target); + } + + public long cost() { + return docIdSetIterator.cost(); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/package-info.java new file mode 100644 index 0000000000000..3c6444a4a5cac --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/package-info.java @@ -0,0 +1,14 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * This contains classes for StarTreeValues iterators + * + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.datacube.startree.utils.iterator; diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java index b7395b993f67b..7cae1cd25ee93 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -54,7 +54,7 @@ public static StarTreeDocument[] getSegmentsStarTreeDocuments( for (int i = 0; i < dimensionsSplitOrder.size(); i++) { String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension)); + dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionValuesIterator(dimension)); } List metricReaders = new ArrayList<>(); @@ -69,7 +69,7 @@ public static StarTreeDocument[] getSegmentsStarTreeDocuments( metric.getField(), metricStat.getTypeName() ); - metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName))); + metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName))); } } @@ -92,7 +92,7 @@ public static StarTreeDocument getStarTreeDocument( Long[] dims = new Long[dimensionReaders.length]; int i = 0; for (SequentialDocValuesIterator dimensionDocValueIterator : dimensionReaders) { - dimensionDocValueIterator.nextDoc(currentDocId); + dimensionDocValueIterator.nextEntry(currentDocId); Long val = dimensionDocValueIterator.value(currentDocId); dims[i] = val; i++; @@ -100,7 +100,7 @@ public static StarTreeDocument getStarTreeDocument( i = 0; Object[] metrics = new Object[metricReaders.size()]; for (SequentialDocValuesIterator metricDocValuesIterator : metricReaders) { - metricDocValuesIterator.nextDoc(currentDocId); + metricDocValuesIterator.nextEntry(currentDocId); metrics[i] = toAggregatorValueType(metricDocValuesIterator.value(currentDocId), fieldValueConverters.get(i)); i++; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java index 65adc43ea8bea..b77200f173e71 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java @@ -53,6 +53,8 @@ import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.FieldValueConverter; @@ -395,7 +397,9 @@ SequentialDocValuesIterator[] getDimensionIterators(StarTreeDocument[] starTreeD docsWithField.add(i); } } - sequentialDocValuesIterators[j] = new SequentialDocValuesIterator(getSortedNumericMock(dimList, docsWithField)); + sequentialDocValuesIterators[j] = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)) + ); } return sequentialDocValuesIterators; } @@ -412,7 +416,9 @@ List getMetricIterators(StarTreeDocument[] starTree docsWithField.add(i); } } - sequentialDocValuesIterators.add(new SequentialDocValuesIterator(getSortedNumericMock(metricslist, docsWithField))); + sequentialDocValuesIterators.add( + new SequentialDocValuesIterator(new SortedNumericStarTreeValuesIterator(getSortedNumericMock(metricslist, docsWithField))) + ); } return sequentialDocValuesIterators; } @@ -1985,10 +1991,16 @@ public void testFlushFlow() throws IOException { List metricsWithField = List.of(0, 1, 2, 3, 4, 5); compositeField = getStarTreeFieldWithMultipleMetrics(); - SortedNumericDocValues d1sndv = getSortedNumericMock(dimList, docsWithField); - SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); - SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); - SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList, metricsWithField); + SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); + SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(dimList2, docsWithField2) + ); + SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); writeState = getWriteState(6, writeState.segmentInfo.getId()); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); @@ -2081,10 +2093,16 @@ public void testFlushFlowDimsReverse() throws IOException { List metricsWithField = List.of(0, 1, 2, 3, 4, 5); compositeField = getStarTreeFieldWithMultipleMetrics(); - SortedNumericDocValues d1sndv = getSortedNumericMock(dimList, docsWithField); - SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); - SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); - SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList, metricsWithField); + SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); + SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(dimList2, docsWithField2) + ); + SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); writeState = getWriteState(6, writeState.segmentInfo.getId()); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( @@ -2508,9 +2526,14 @@ private StarTreeValues getStarTreeValues( SortedNumericDocValues d1sndv = dimList; SortedNumericDocValues d2sndv = dimList2; SortedNumericDocValues m1sndv = metricsList; - Map> dimDocIdSetIterators = Map.of("field1", () -> d1sndv, "field3", () -> d2sndv); + Map> dimDocIdSetIterators = Map.of( + "field1", + () -> new SortedNumericStarTreeValuesIterator(d1sndv), + "field3", + () -> new SortedNumericStarTreeValuesIterator(d2sndv) + ); - Map> metricDocIdSetIterators = new LinkedHashMap<>(); + Map> metricDocIdSetIterators = new LinkedHashMap<>(); for (Metric metric : sf.getMetrics()) { for (MetricStat metricStat : metric.getMetrics()) { String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( @@ -2518,7 +2541,7 @@ private StarTreeValues getStarTreeValues( metric.getField(), metricStat.getTypeName() ); - metricDocIdSetIterators.put(metricFullName, () -> m1sndv); + metricDocIdSetIterators.put(metricFullName, () -> new SortedNumericStarTreeValuesIterator(m1sndv)); } } @@ -3648,18 +3671,18 @@ private StarTreeValues getStarTreeValues( SortedNumericDocValues d4sndv = getSortedNumericMock(dimList4, docsWithField4); SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList1, metricsWithField1); - Map> dimDocIdSetIterators = Map.of( + Map> dimDocIdSetIterators = Map.of( "field1", - () -> d1sndv, + () -> new SortedNumericStarTreeValuesIterator(d1sndv), "field3", - () -> d2sndv, + () -> new SortedNumericStarTreeValuesIterator(d2sndv), "field5", - () -> d3sndv, + () -> new SortedNumericStarTreeValuesIterator(d3sndv), "field8", - () -> d4sndv + () -> new SortedNumericStarTreeValuesIterator(d4sndv) ); - Map> metricDocIdSetIterators = new LinkedHashMap<>(); + Map> metricDocIdSetIterators = new LinkedHashMap<>(); metricDocIdSetIterators.put( fullyQualifiedFieldNameForStarTreeMetricsDocValues( @@ -3667,7 +3690,7 @@ private StarTreeValues getStarTreeValues( "field2", sf.getMetrics().get(0).getMetrics().get(0).getTypeName() ), - () -> m1sndv + () -> new SortedNumericStarTreeValuesIterator(m1sndv) ); metricDocIdSetIterators.put( fullyQualifiedFieldNameForStarTreeMetricsDocValues( @@ -3675,7 +3698,7 @@ private StarTreeValues getStarTreeValues( "_doc_count", sf.getMetrics().get(1).getMetrics().get(0).getTypeName() ), - () -> m2sndv + () -> new SortedNumericStarTreeValuesIterator(m2sndv) ); // metricDocIdSetIterators.put("field2", () -> m1sndv); // metricDocIdSetIterators.put("_doc_count", () -> m2sndv); @@ -4093,24 +4116,24 @@ public void testMergeFlow() throws IOException { SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); SortedNumericDocValues valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount); SortedNumericDocValues m2sndv = DocValues.emptySortedNumeric(); - Map> dimDocIdSetIterators = Map.of( + Map> dimDocIdSetIterators = Map.of( "field1", - () -> d1sndv, + () -> new SortedNumericStarTreeValuesIterator(d1sndv), "field3", - () -> d2sndv, + () -> new SortedNumericStarTreeValuesIterator(d2sndv), "field5", - () -> d3sndv, + () -> new SortedNumericStarTreeValuesIterator(d3sndv), "field8", - () -> d4sndv + () -> new SortedNumericStarTreeValuesIterator(d4sndv) ); - Map> metricDocIdSetIterators = Map.of( + Map> metricDocIdSetIterators = Map.of( "sf_field2_sum_metric", - () -> m1sndv, + () -> new SortedNumericStarTreeValuesIterator(m1sndv), "sf_field2_value_count_metric", - () -> valucountsndv, + () -> new SortedNumericStarTreeValuesIterator(valucountsndv), "sf__doc_count_doc_count_metric", - () -> m2sndv + () -> new SortedNumericStarTreeValuesIterator(m2sndv) ); StarTreeValues starTreeValues = new StarTreeValues( @@ -4129,24 +4152,24 @@ public void testMergeFlow() throws IOException { SortedNumericDocValues f2m1sndv = getSortedNumericMock(metricsList, metricsWithField); SortedNumericDocValues f2valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount); SortedNumericDocValues f2m2sndv = DocValues.emptySortedNumeric(); - Map> f2dimDocIdSetIterators = Map.of( + Map> f2dimDocIdSetIterators = Map.of( "field1", - () -> f2d1sndv, + () -> new SortedNumericStarTreeValuesIterator(f2d1sndv), "field3", - () -> f2d2sndv, + () -> new SortedNumericStarTreeValuesIterator(f2d2sndv), "field5", - () -> f2d3sndv, + () -> new SortedNumericStarTreeValuesIterator(f2d3sndv), "field8", - () -> f2d4sndv + () -> new SortedNumericStarTreeValuesIterator(f2d4sndv) ); - Map> f2metricDocIdSetIterators = Map.of( + Map> f2metricDocIdSetIterators = Map.of( "sf_field2_sum_metric", - () -> f2m1sndv, + () -> new SortedNumericStarTreeValuesIterator(f2m1sndv), "sf_field2_value_count_metric", - () -> f2valucountsndv, + () -> new SortedNumericStarTreeValuesIterator(f2valucountsndv), "sf__doc_count_doc_count_metric", - () -> f2m2sndv + () -> new SortedNumericStarTreeValuesIterator(f2m2sndv) ); StarTreeValues starTreeValues2 = new StarTreeValues( compositeField, diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java index f56f7d9906ae1..78d63800abd16 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java @@ -9,15 +9,13 @@ package org.opensearch.index.compositeindex.datacube.startree.utils; import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.BytesRef; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; import org.opensearch.test.OpenSearchTestCase; import org.junit.BeforeClass; @@ -59,50 +57,32 @@ public void testCreateIterator_SortedNumeric() throws IOException { DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); when(producer.getSortedNumeric(mockFieldInfo)).thenReturn(iterator); - SequentialDocValuesIterator result = new SequentialDocValuesIterator(producer.getSortedNumeric(mockFieldInfo)); - assertEquals(iterator.getClass(), result.getDocIdSetIterator().getClass()); - } - - public void testCreateIterator_UnsupportedType() throws IOException { - DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); - BinaryDocValues iterator = Mockito.mock(BinaryDocValues.class); - when(producer.getBinary(mockFieldInfo)).thenReturn(iterator); - SequentialDocValuesIterator result = new SequentialDocValuesIterator(producer.getBinary(mockFieldInfo)); - assertEquals(iterator.getClass(), result.getDocIdSetIterator().getClass()); - when(iterator.nextDoc()).thenReturn(0); - when(iterator.binaryValue()).thenReturn(new BytesRef("123")); + SequentialDocValuesIterator result = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(producer.getSortedNumeric(mockFieldInfo)) + ); - IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { - result.nextDoc(0); - result.value(0); - }); - assertEquals("Unsupported Iterator requested for SequentialDocValuesIterator", exception.getMessage()); } public void testGetNextValue_SortedNumeric() throws IOException { SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); when(iterator.nextDoc()).thenReturn(0); when(iterator.nextValue()).thenReturn(123L); - SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator); - sequentialDocValuesIterator.nextDoc(0); + SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(iterator) + ); + sequentialDocValuesIterator.nextEntry(0); long result = sequentialDocValuesIterator.value(0); assertEquals(123L, result); } - public void testGetNextValue_UnsupportedIterator() { - DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class); - SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator); - - IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { sequentialDocValuesIterator.value(0); }); - assertEquals("Unsupported Iterator requested for SequentialDocValuesIterator", exception.getMessage()); - } - - public void testNextDoc() throws IOException { + public void testNextEntry() throws IOException { SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); - SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator); + SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(iterator) + ); when(iterator.nextDoc()).thenReturn(5); - int result = sequentialDocValuesIterator.nextDoc(5); + int result = sequentialDocValuesIterator.nextEntry(5); assertEquals(5, result); } @@ -110,8 +90,12 @@ public void test_multipleCoordinatedDocumentReader() throws IOException { SortedNumericDocValues iterator1 = Mockito.mock(SortedNumericDocValues.class); SortedNumericDocValues iterator2 = Mockito.mock(SortedNumericDocValues.class); - SequentialDocValuesIterator sequentialDocValuesIterator1 = new SequentialDocValuesIterator(iterator1); - SequentialDocValuesIterator sequentialDocValuesIterator2 = new SequentialDocValuesIterator(iterator2); + SequentialDocValuesIterator sequentialDocValuesIterator1 = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(iterator1) + ); + SequentialDocValuesIterator sequentialDocValuesIterator2 = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(iterator2) + ); when(iterator1.nextDoc()).thenReturn(0); when(iterator2.nextDoc()).thenReturn(1); @@ -119,13 +103,13 @@ public void test_multipleCoordinatedDocumentReader() throws IOException { when(iterator1.nextValue()).thenReturn(9L); when(iterator2.nextValue()).thenReturn(9L); - sequentialDocValuesIterator1.nextDoc(0); - sequentialDocValuesIterator2.nextDoc(0); - assertEquals(0, sequentialDocValuesIterator1.getDocId()); + sequentialDocValuesIterator1.nextEntry(0); + sequentialDocValuesIterator2.nextEntry(0); + assertEquals(0, sequentialDocValuesIterator1.getEntryId()); assertEquals(9L, (long) sequentialDocValuesIterator1.value(0)); assertNull(sequentialDocValuesIterator2.value(0)); - assertNotEquals(0, sequentialDocValuesIterator2.getDocId()); - assertEquals(1, sequentialDocValuesIterator2.getDocId()); + assertNotEquals(0, sequentialDocValuesIterator2.getEntryId()); + assertEquals(1, sequentialDocValuesIterator2.getEntryId()); assertEquals(9L, (long) sequentialDocValuesIterator2.value(1)); } } From 6bef1e1d8e2dda8d88c2c8f2a7875b492239a480 Mon Sep 17 00:00:00 2001 From: kkewwei Date: Thu, 19 Sep 2024 16:48:52 +0800 Subject: [PATCH 35/44] Fix infinite loop in nested agg (#15931) Signed-off-by: kkewwei --- CHANGELOG.md | 1 + .../search.aggregation/410_nested_aggs.yml | 111 ++++++++++++++++++ .../opensearch/index/mapper/ObjectMapper.java | 13 ++ .../bucket/nested/NestedAggregator.java | 14 +-- .../index/mapper/ObjectMapperTests.java | 47 ++++++++ 5 files changed, 174 insertions(+), 12 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/410_nested_aggs.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ed85b630bbd4..636684af939bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737)) - Fix case-insensitive query on wildcard field ([#15882](https://github.com/opensearch-project/OpenSearch/pull/15882)) - Add validation for the search backpressure cancellation settings ([#15501](https://github.com/opensearch-project/OpenSearch/pull/15501)) +- Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931)) ### Security diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/410_nested_aggs.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/410_nested_aggs.yml new file mode 100644 index 0000000000000..c115dd4751f8f --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/410_nested_aggs.yml @@ -0,0 +1,111 @@ +--- +# The test setup includes: +# - Create nested mapping for test_nested_agg_index index +# - Index two example documents +# - nested agg + +setup: + - do: + indices.create: + index: test_nested_agg_index + body: + mappings: + properties: + a: + type: nested + properties: + b1: + type: keyword + b2: + type: nested + properties: + c: + type: nested + properties: + d: + type: keyword + + - do: + bulk: + refresh: true + body: | + {"index": {"_index": "test_nested_agg_index", "_id": "0"}} + {"a": { "b1": "b11", "b2": { "c": { "d": "d1" } }}} + {"index": {"_index": "test_nested_agg_index", "_id": "1"}} + {"a": { "b1": "b12", "b2": { "c": { "d": "d2" } }}} + +--- +# Delete Index when connection is teardown +teardown: + - do: + indices.delete: + index: test_nested_agg_index + +--- +"Supported queries": + - skip: + version: " - 2.17.99" + reason: "fixed in 2.18.0" + + # Verify Document Count + - do: + search: + body: { + query: { + match_all: { } + } + } + + - length: { hits.hits: 2 } + + # Verify nested aggregation + - do: + search: + body: { + aggs: { + nested_agg: { + nested: { + path: "a" + }, + aggs: { + a_b1: { + terms: { + field: "a.b1" + }, + aggs: { + "c": { + nested: { + path: "a.b2.c" + }, + aggs: { + "d": { + terms: { + field: "a.b2.c.d" + } + } + } + } + } + } + } + } + } + } + + - length: { hits.hits: 2 } + - match: { aggregations.nested_agg.doc_count: 2 } + - length: { aggregations.nested_agg.a_b1.buckets: 2 } + + - match: { aggregations.nested_agg.a_b1.buckets.0.key: "b11" } + - match: { aggregations.nested_agg.a_b1.buckets.0.doc_count: 1 } + - match: { aggregations.nested_agg.a_b1.buckets.0.c.doc_count: 1 } + - length: { aggregations.nested_agg.a_b1.buckets.0.c.d.buckets: "1" } + - match: { aggregations.nested_agg.a_b1.buckets.0.c.d.buckets.0.key: "d1" } + - match: { aggregations.nested_agg.a_b1.buckets.0.c.d.buckets.0.doc_count: 1 } + + - match: { aggregations.nested_agg.a_b1.buckets.1.key: "b12" } + - match: { aggregations.nested_agg.a_b1.buckets.1.doc_count: 1 } + - match: { aggregations.nested_agg.a_b1.buckets.1.c.doc_count: 1 } + - length: { aggregations.nested_agg.a_b1.buckets.1.c.d.buckets: "1" } + - match: { aggregations.nested_agg.a_b1.buckets.1.c.d.buckets.0.key: "d2" } + - match: { aggregations.nested_agg.a_b1.buckets.1.c.d.buckets.0.doc_count: 1 } diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java index dd984373fc9df..b93c82d7a5c7c 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java @@ -171,6 +171,19 @@ public void setIncludeInParent(boolean value) { public void setIncludeInRoot(boolean value) { includeInRoot = new Explicit<>(value, true); } + + public static boolean isParent(ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper, MapperService mapperService) { + if (parentObjectMapper == null || childObjectMapper == null) { + return false; + } + + ObjectMapper parent = childObjectMapper.getParentObjectMapper(mapperService); + while (parent != null && parent != parentObjectMapper) { + childObjectMapper = parent; + parent = childObjectMapper.getParentObjectMapper(mapperService); + } + return parentObjectMapper == parent; + } } /** diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java index 150efa878f866..db8979d611b4f 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java @@ -46,7 +46,6 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.lucene.search.Queries; import org.opensearch.core.ParseField; -import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.ObjectMapper; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; @@ -63,6 +62,8 @@ import java.util.List; import java.util.Map; +import static org.opensearch.index.mapper.ObjectMapper.Nested.isParent; + /** * Aggregate all docs that match a nested path * @@ -98,17 +99,6 @@ public class NestedAggregator extends BucketsAggregator implements SingleBucketA this.collectsFromSingleBucket = cardinality.map(estimate -> estimate < 2); } - private boolean isParent(ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper, MapperService mapperService) { - if (parentObjectMapper == null) { - return false; - } - ObjectMapper parent; - do { - parent = childObjectMapper.getParentObjectMapper(mapperService); - } while (parent != null && parent != parentObjectMapper); - return parentObjectMapper == parent; - } - @Override public LeafBucketCollector getLeafCollector(final LeafReaderContext ctx, final LeafBucketCollector sub) throws IOException { IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(ctx); diff --git a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java index cb06bf23d9cbe..b415e1e657f7f 100644 --- a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java @@ -51,8 +51,12 @@ import java.io.IOException; import java.util.Collection; +import java.util.Collections; + +import org.mockito.Mockito; import static org.opensearch.common.util.FeatureFlags.STAR_TREE_INDEX; +import static org.opensearch.index.mapper.ObjectMapper.Nested.isParent; import static org.hamcrest.Matchers.containsString; public class ObjectMapperTests extends OpenSearchSingleNodeTestCase { @@ -568,6 +572,49 @@ public void testCompositeFields() throws Exception { FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } + public void testNestedIsParent() throws Exception { + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("a") + .field("type", "nested") + .startObject("properties") + .field("b1", Collections.singletonMap("type", "keyword")) + .startObject("b2") + .field("type", "nested") + .startObject("properties") + .startObject("c") + .field("type", "nested") + .startObject("properties") + .field("d", Collections.singletonMap("type", "keyword")) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); + + DocumentMapper documentMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + MapperService mapperService = Mockito.mock(MapperService.class); + Mockito.when(mapperService.getObjectMapper(("a"))).thenReturn(documentMapper.objectMappers().get("a")); + Mockito.when(mapperService.getObjectMapper(("a.b2"))).thenReturn(documentMapper.objectMappers().get("a.b2")); + Mockito.when(mapperService.getObjectMapper(("a.b2.c"))).thenReturn(documentMapper.objectMappers().get("a.b2.c")); + + assertTrue(isParent(documentMapper.objectMappers().get("a"), documentMapper.objectMappers().get("a.b2.c"), mapperService)); + assertTrue(isParent(documentMapper.objectMappers().get("a"), documentMapper.objectMappers().get("a.b2"), mapperService)); + assertTrue(isParent(documentMapper.objectMappers().get("a.b2"), documentMapper.objectMappers().get("a.b2.c"), mapperService)); + + assertFalse(isParent(documentMapper.objectMappers().get("a.b2.c"), documentMapper.objectMappers().get("a"), mapperService)); + assertFalse(isParent(documentMapper.objectMappers().get("a.b2"), documentMapper.objectMappers().get("a"), mapperService)); + assertFalse(isParent(documentMapper.objectMappers().get("a.b2.c"), documentMapper.objectMappers().get("a.b2"), mapperService)); + } + @Override protected Collection> getPlugins() { return pluginList(InternalSettingsPlugin.class); From 620db0a64950cbd6a38642f0be1f592060dac1ee Mon Sep 17 00:00:00 2001 From: SwethaGuptha <156877431+SwethaGuptha@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:03:02 +0530 Subject: [PATCH 36/44] Explicitly set include discovery nodes flag in reloadSecureSetting request. (#15972) Signed-off-by: Swetha Guptha --- .../cluster/node/reload/NodesReloadSecureSettingsRequest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java index aab7ea54f87c2..d694721feda38 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java @@ -60,7 +60,7 @@ public class NodesReloadSecureSettingsRequest extends BaseNodesRequest Date: Thu, 19 Sep 2024 20:48:36 +0800 Subject: [PATCH 37/44] Update version check in yaml test file for bitmap filtering (#15993) Signed-off-by: Gao Binlong --- .../rest-api-spec/test/search/370_bitmap_filtering.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml index d728070adb188..c885e3fbc2446 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml @@ -1,7 +1,7 @@ --- setup: - skip: - version: " - 2.99.99" + version: " - 2.16.99" reason: The bitmap filtering feature is available in 2.17 and later. - do: indices.create: From 77ddfd640cfde672d103c7ff9f25521362cea14e Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad <61760125+gaiksaya@users.noreply.github.com> Date: Thu, 19 Sep 2024 13:50:47 -0700 Subject: [PATCH 38/44] Fix deleting backport branches workflow (#16003) Signed-off-by: Sayali Gaikawad --- .github/workflows/delete_backport_branch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml index b60c2225db26a..91626a0bc5092 100644 --- a/.github/workflows/delete_backport_branch.yml +++ b/.github/workflows/delete_backport_branch.yml @@ -11,7 +11,7 @@ jobs: contents: write if: startsWith(github.event.pull_request.head.ref,'backport/') steps: - - name: Delete merged branch + - name: Delete merged branch uses: actions/github-script@v5 with: script: | From 1bc81d3eee07e96b6c6393510b41d194276e204c Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 19 Sep 2024 16:55:21 -0400 Subject: [PATCH 39/44] Remove identity-related feature flagged code from the RestController (#15430) * Add authenticate to IdentityPlugin interface Signed-off-by: Craig Perkins * Handle null Signed-off-by: Craig Perkins * Fix tests Signed-off-by: Craig Perkins * Fix ActionModuleTests Signed-off-by: Craig Perkins * Add to CHANGELOG Signed-off-by: Craig Perkins * Add DelegatingRestHandlerTests Signed-off-by: Craig Perkins * Address forbiddenApi Signed-off-by: Craig Perkins * Remove authenticate from IdentityPlugin and keep RestController feature flagged code removed Signed-off-by: Craig Perkins * Move RestTokenExtractor to identity-shiro plugin Signed-off-by: Craig Perkins * Remove change in IdentityService Signed-off-by: Craig Perkins * Remove changes in ActionModuleTests Signed-off-by: Craig Perkins * Add tests for RestTokenExtractor Signed-off-by: Craig Perkins * Remove DelegatingRestHandler Signed-off-by: Craig Perkins * Call super instead of keeping a reference to the delegated restHandler Signed-off-by: Craig Perkins * Address code review comments Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins Signed-off-by: Craig Perkins --- CHANGELOG.md | 1 + .../identity/shiro/ShiroIdentityPlugin.java | 47 ++++++++++++++-- .../identity/shiro/ShiroSecurityManager.java | 2 - .../identity/shiro/ShiroSubject.java | 2 - .../identity/shiro/ShiroTokenExtractor.java | 8 +-- .../identity/shiro/ShiroTokenManager.java | 2 - .../shiro/realm/BCryptPasswordMatcher.java | 2 - .../identity/shiro/realm/OpenSearchRealm.java | 6 +-- .../opensearch/identity/shiro/realm/User.java | 2 - .../shiro/ShiroIdentityPluginTests.java | 14 ++--- .../shiro/ShiroTokenExtractorTests.java | 45 ++++++++++++++++ .../org/opensearch/action/ActionModule.java | 2 +- .../org/opensearch/rest/RestController.java | 54 +------------------ .../extensions/ExtensionsManagerTests.java | 3 +- .../opensearch/rest/RestControllerTests.java | 30 ++++------- .../rest/RestHttpResponseHeadersTests.java | 14 +---- .../indices/RestValidateQueryActionTests.java | 12 +---- .../test/rest/RestActionTestCase.java | 15 +----- 18 files changed, 118 insertions(+), 143 deletions(-) rename server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java => plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenExtractor.java (86%) create mode 100644 plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroTokenExtractorTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 636684af939bf..b5aef4bb551d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) - Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) +- Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroIdentityPlugin.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroIdentityPlugin.java index af802596ebaa7..2da788242a745 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroIdentityPlugin.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroIdentityPlugin.java @@ -13,20 +13,28 @@ import org.apache.shiro.SecurityUtils; import org.apache.shiro.mgt.SecurityManager; import org.opensearch.client.Client; +import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.identity.PluginSubject; import org.opensearch.identity.Subject; +import org.opensearch.identity.tokens.AuthToken; import org.opensearch.identity.tokens.TokenManager; +import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.IdentityPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.repositories.RepositoriesService; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestHandler; +import org.opensearch.rest.RestRequest; import org.opensearch.script.ScriptService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; @@ -34,14 +42,12 @@ import java.util.Collection; import java.util.Collections; import java.util.function.Supplier; +import java.util.function.UnaryOperator; /** * Identity implementation with Shiro - * - * @opensearch.experimental */ -@ExperimentalApi -public final class ShiroIdentityPlugin extends Plugin implements IdentityPlugin { +public final class ShiroIdentityPlugin extends Plugin implements IdentityPlugin, ActionPlugin { private Logger log = LogManager.getLogger(this.getClass()); private final Settings settings; @@ -101,6 +107,37 @@ public TokenManager getTokenManager() { } @Override + public UnaryOperator getRestHandlerWrapper(ThreadContext threadContext) { + return AuthcRestHandler::new; + } + + class AuthcRestHandler extends RestHandler.Wrapper { + + public AuthcRestHandler(RestHandler original) { + super(original); + } + + @Override + public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception { + try { + final AuthToken token = ShiroTokenExtractor.extractToken(request); + // If no token was found, continue executing the request + if (token == null) { + // Authentication did not fail so return true. Authorization is handled at the action level. + super.handleRequest(request, channel, client); + return; + } + ShiroSubject shiroSubject = (ShiroSubject) getCurrentSubject(); + shiroSubject.authenticate(token); + // Caller was authorized, forward the request to the handler + super.handleRequest(request, channel, client); + } catch (final Exception e) { + final BytesRestResponse bytesRestResponse = new BytesRestResponse(RestStatus.UNAUTHORIZED, e.getMessage()); + channel.sendResponse(bytesRestResponse); + } + } + } + public PluginSubject getPluginSubject(Plugin plugin) { return new ShiroPluginSubject(threadPool); } diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSecurityManager.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSecurityManager.java index 96cf05ac53a1a..0a809dd6c9071 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSecurityManager.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSecurityManager.java @@ -15,8 +15,6 @@ /** * OpenSearch specific security manager implementation - * - * @opensearch.experimental */ public class ShiroSecurityManager extends DefaultSecurityManager { diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSubject.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSubject.java index 72a168f23c5cd..73ce3f835fc9b 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSubject.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSubject.java @@ -17,8 +17,6 @@ /** * Subject backed by Shiro - * - * @opensearch.experimental */ public class ShiroSubject implements UserSubject { private final ShiroTokenManager authTokenHandler; diff --git a/server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenExtractor.java similarity index 86% rename from server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java rename to plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenExtractor.java index 4bd3ebdded588..86be5ca664daa 100644 --- a/server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenExtractor.java @@ -5,11 +5,13 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ -package org.opensearch.identity.tokens; +package org.opensearch.identity.shiro; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.core.common.Strings; +import org.opensearch.identity.tokens.AuthToken; +import org.opensearch.identity.tokens.BasicAuthToken; import org.opensearch.rest.RestRequest; import java.util.Collections; @@ -18,9 +20,9 @@ /** * Extracts tokens from RestRequests used for authentication */ -public class RestTokenExtractor { +public class ShiroTokenExtractor { - private static final Logger logger = LogManager.getLogger(RestTokenExtractor.class); + private static final Logger logger = LogManager.getLogger(ShiroTokenExtractor.class); public final static String AUTH_HEADER_NAME = "Authorization"; diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java index a14215aa7655b..cd54bbf9b3124 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java @@ -36,8 +36,6 @@ /** * Extracts Shiro's {@link AuthenticationToken} from different types of auth headers - * - * @opensearch.experimental */ class ShiroTokenManager implements TokenManager { diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/BCryptPasswordMatcher.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/BCryptPasswordMatcher.java index a2cb78425929e..f8113101deb70 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/BCryptPasswordMatcher.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/BCryptPasswordMatcher.java @@ -16,8 +16,6 @@ /** * Password matcher for BCrypt - * - * @opensearch.experimental */ public class BCryptPasswordMatcher implements CredentialsMatcher { diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/OpenSearchRealm.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/OpenSearchRealm.java index ef405a5637ae7..1fc9a1f437a42 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/OpenSearchRealm.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/OpenSearchRealm.java @@ -25,8 +25,6 @@ /** * Internal Realm is a custom realm using the internal OpenSearch IdP - * - * @opensearch.experimental */ public class OpenSearchRealm extends AuthenticatingRealm { private static final String DEFAULT_REALM_NAME = "internal"; @@ -93,7 +91,7 @@ public OpenSearchRealm build() { public User getInternalUser(final String principalIdentifier) throws UnknownAccountException { final User userRecord = internalUsers.get(principalIdentifier); if (userRecord == null) { - throw new UnknownAccountException(); + throw new UnknownAccountException("Incorrect credentials"); } return userRecord; } @@ -131,7 +129,7 @@ protected AuthenticationInfo doGetAuthenticationInfo(final AuthenticationToken t return sai; } else { // Bad password - throw new IncorrectCredentialsException(); + throw new IncorrectCredentialsException("Incorrect credentials"); } } diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/User.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/User.java index 35b3348a955d7..1d2d0fed800e2 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/User.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/User.java @@ -12,8 +12,6 @@ /** * A non-volatile and immutable object in the storage. - * - * @opensearch.experimental */ public class User { diff --git a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroIdentityPluginTests.java b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroIdentityPluginTests.java index bc14410d81de0..a15538e48bd66 100644 --- a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroIdentityPluginTests.java +++ b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroIdentityPluginTests.java @@ -13,7 +13,7 @@ import org.opensearch.identity.IdentityService; import org.opensearch.plugins.IdentityPlugin; import org.opensearch.test.OpenSearchTestCase; -import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; import java.util.List; @@ -21,27 +21,27 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.mock; public class ShiroIdentityPluginTests extends OpenSearchTestCase { public void testSingleIdentityPluginSucceeds() { - TestThreadPool threadPool = new TestThreadPool(getTestName()); IdentityPlugin identityPlugin1 = new ShiroIdentityPlugin(Settings.EMPTY); List pluginList1 = List.of(identityPlugin1); - IdentityService identityService1 = new IdentityService(Settings.EMPTY, threadPool, pluginList1); + IdentityService identityService1 = new IdentityService(Settings.EMPTY, mock(ThreadPool.class), pluginList1); assertThat(identityService1.getTokenManager(), is(instanceOf(ShiroTokenManager.class))); - terminate(threadPool); } public void testMultipleIdentityPluginsFail() { - TestThreadPool threadPool = new TestThreadPool(getTestName()); IdentityPlugin identityPlugin1 = new ShiroIdentityPlugin(Settings.EMPTY); IdentityPlugin identityPlugin2 = new ShiroIdentityPlugin(Settings.EMPTY); IdentityPlugin identityPlugin3 = new ShiroIdentityPlugin(Settings.EMPTY); List pluginList = List.of(identityPlugin1, identityPlugin2, identityPlugin3); - Exception ex = assertThrows(OpenSearchException.class, () -> new IdentityService(Settings.EMPTY, threadPool, pluginList)); + Exception ex = assertThrows( + OpenSearchException.class, + () -> new IdentityService(Settings.EMPTY, mock(ThreadPool.class), pluginList) + ); assert (ex.getMessage().contains("Multiple identity plugins are not supported,")); - terminate(threadPool); } } diff --git a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroTokenExtractorTests.java b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroTokenExtractorTests.java new file mode 100644 index 0000000000000..4dc398bacb707 --- /dev/null +++ b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroTokenExtractorTests.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.identity.shiro; + +import org.opensearch.identity.tokens.AuthToken; +import org.opensearch.identity.tokens.BasicAuthToken; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.rest.FakeRestRequest; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class ShiroTokenExtractorTests extends OpenSearchTestCase { + + public void testAuthorizationHeaderExtractionWithBasicAuthToken() { + String basicAuthHeader = Base64.getEncoder().encodeToString("foo:bar".getBytes(StandardCharsets.UTF_8)); + RestRequest fakeRequest = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( + Map.of(ShiroTokenExtractor.AUTH_HEADER_NAME, List.of(BasicAuthToken.TOKEN_IDENTIFIER + " " + basicAuthHeader)) + ).build(); + AuthToken extractedToken = ShiroTokenExtractor.extractToken(fakeRequest); + assertThat(extractedToken, instanceOf(BasicAuthToken.class)); + assertThat(extractedToken.asAuthHeaderValue(), equalTo(basicAuthHeader)); + } + + public void testAuthorizationHeaderExtractionWithUnknownToken() { + String authHeader = "foo"; + RestRequest fakeRequest = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( + Map.of(ShiroTokenExtractor.AUTH_HEADER_NAME, List.of(authHeader)) + ).build(); + AuthToken extractedToken = ShiroTokenExtractor.extractToken(fakeRequest); + assertNull(extractedToken); + } +} diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index fbf90b97d1e8f..3fe0f1dc7cb83 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -579,7 +579,7 @@ public ActionModule( actionPlugins.stream().flatMap(p -> p.indicesAliasesRequestValidators().stream()).collect(Collectors.toList()) ); - restController = new RestController(headers, restWrapper, nodeClient, circuitBreakerService, usageService, identityService); + restController = new RestController(headers, restWrapper, nodeClient, circuitBreakerService, usageService); } public Map> getActions() { diff --git a/server/src/main/java/org/opensearch/rest/RestController.java b/server/src/main/java/org/opensearch/rest/RestController.java index 0d6f965c7033f..4f87c01258396 100644 --- a/server/src/main/java/org/opensearch/rest/RestController.java +++ b/server/src/main/java/org/opensearch/rest/RestController.java @@ -41,7 +41,6 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.path.PathTrie; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.io.Streams; import org.opensearch.common.xcontent.XContentType; @@ -56,11 +55,6 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.http.HttpChunk; import org.opensearch.http.HttpServerTransport; -import org.opensearch.identity.IdentityService; -import org.opensearch.identity.Subject; -import org.opensearch.identity.UserSubject; -import org.opensearch.identity.tokens.AuthToken; -import org.opensearch.identity.tokens.RestTokenExtractor; import org.opensearch.usage.UsageService; import java.io.ByteArrayOutputStream; @@ -125,25 +119,23 @@ public class RestController implements HttpServerTransport.Dispatcher { /** Rest headers that are copied to internal requests made during a rest request. */ private final Set headersToCopy; private final UsageService usageService; - private final IdentityService identityService; public RestController( Set headersToCopy, UnaryOperator handlerWrapper, NodeClient client, CircuitBreakerService circuitBreakerService, - UsageService usageService, - IdentityService identityService + UsageService usageService ) { this.headersToCopy = headersToCopy; this.usageService = usageService; if (handlerWrapper == null) { handlerWrapper = h -> h; // passthrough if no wrapper set } + this.handlerWrapper = handlerWrapper; this.client = client; this.circuitBreakerService = circuitBreakerService; - this.identityService = identityService; registerHandlerNoWrap( RestRequest.Method.GET, "/favicon.ico", @@ -472,11 +464,6 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel return; } } else { - if (FeatureFlags.isEnabled(FeatureFlags.IDENTITY)) { - if (!handleAuthenticateUser(request, channel)) { - return; - } - } dispatchRequest(request, channel, handler); return; } @@ -587,43 +574,6 @@ private void handleBadRequest(String uri, RestRequest.Method method, RestChannel } } - /** - * Attempts to extract auth token and login. - * - * @return false if there was an error and the request should not continue being dispatched - * */ - private boolean handleAuthenticateUser(final RestRequest request, final RestChannel channel) { - try { - final AuthToken token = RestTokenExtractor.extractToken(request); - // If no token was found, continue executing the request - if (token == null) { - // Authentication did not fail so return true. Authorization is handled at the action level. - return true; - } - final Subject currentSubject = identityService.getCurrentSubject(); - if (currentSubject instanceof UserSubject) { - ((UserSubject) currentSubject).authenticate(token); - logger.debug("Logged in as user " + currentSubject); - } - } catch (final Exception e) { - try { - final BytesRestResponse bytesRestResponse = BytesRestResponse.createSimpleErrorResponse( - channel, - RestStatus.UNAUTHORIZED, - e.getMessage() - ); - channel.sendResponse(bytesRestResponse); - } catch (final Exception ex) { - final BytesRestResponse bytesRestResponse = new BytesRestResponse(RestStatus.UNAUTHORIZED, ex.getMessage()); - channel.sendResponse(bytesRestResponse); - } - return false; - } - - // Authentication did not fail so return true. Authorization is handled at the action level. - return true; - } - /** * Get the valid set of HTTP methods for a REST request. */ diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index 5ae1bdce48cd5..bf1d52b49cb1f 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -153,8 +153,7 @@ public List> getExtensionSettings() { null, new NodeClient(Settings.EMPTY, threadPool), new NoneCircuitBreakerService(), - new UsageService(), - new IdentityService(Settings.EMPTY, threadPool, List.of()) + new UsageService() ); when(actionModule.getDynamicActionRegistry()).thenReturn(mock(DynamicActionRegistry.class)); when(actionModule.getRestController()).thenReturn(restController); diff --git a/server/src/test/java/org/opensearch/rest/RestControllerTests.java b/server/src/test/java/org/opensearch/rest/RestControllerTests.java index ef9257d746573..f7f1b02847854 100644 --- a/server/src/test/java/org/opensearch/rest/RestControllerTests.java +++ b/server/src/test/java/org/opensearch/rest/RestControllerTests.java @@ -55,13 +55,11 @@ import org.opensearch.http.HttpResponse; import org.opensearch.http.HttpServerTransport; import org.opensearch.http.HttpStats; -import org.opensearch.identity.IdentityService; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.rest.action.admin.indices.RestCreateIndexAction; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.client.NoOpNodeClient; import org.opensearch.test.rest.FakeRestRequest; -import org.opensearch.threadpool.ThreadPool; import org.opensearch.usage.UsageService; import org.junit.After; import org.junit.Before; @@ -97,7 +95,6 @@ public class RestControllerTests extends OpenSearchTestCase { private RestController restController; private HierarchyCircuitBreakerService circuitBreakerService; private UsageService usageService; - private IdentityService identityService; private NodeClient client; @Before @@ -115,11 +112,9 @@ public void setup() { // we can do this here only because we know that we don't adjust breaker settings dynamically in the test inFlightRequestsBreaker = circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS); - identityService = new IdentityService(Settings.EMPTY, mock(ThreadPool.class), List.of()); - HttpServerTransport httpServerTransport = new TestHttpServerTransport(); client = new NoOpNodeClient(this.getTestName()); - restController = new RestController(Collections.emptySet(), null, client, circuitBreakerService, usageService, identityService); + restController = new RestController(Collections.emptySet(), null, client, circuitBreakerService, usageService); restController.registerHandler( RestRequest.Method.GET, "/", @@ -140,7 +135,7 @@ public void teardown() throws IOException { } public void testDefaultRestControllerGetAllHandlersContainsFavicon() { - final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService, identityService); + final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService); Iterator handlers = restController.getAllHandlers(); assertTrue(handlers.hasNext()); MethodHandlers faviconHandler = handlers.next(); @@ -150,7 +145,7 @@ public void testDefaultRestControllerGetAllHandlersContainsFavicon() { } public void testRestControllerGetAllHandlers() { - final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService, identityService); + final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService); restController.registerHandler(RestRequest.Method.PATCH, "/foo", mock(RestHandler.class)); restController.registerHandler(RestRequest.Method.GET, "/foo", mock(RestHandler.class)); @@ -175,7 +170,7 @@ public void testApplyRelevantHeaders() throws Exception { Set headers = new HashSet<>( Arrays.asList(new RestHeaderDefinition("header.1", true), new RestHeaderDefinition("header.2", true)) ); - final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService, identityService); + final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService); Map> restHeaders = new HashMap<>(); restHeaders.put("header.1", Collections.singletonList("true")); restHeaders.put("header.2", Collections.singletonList("true")); @@ -211,7 +206,7 @@ public void testRequestWithDisallowedMultiValuedHeader() { Set headers = new HashSet<>( Arrays.asList(new RestHeaderDefinition("header.1", true), new RestHeaderDefinition("header.2", false)) ); - final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService, identityService); + final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService); Map> restHeaders = new HashMap<>(); restHeaders.put("header.1", Collections.singletonList("boo")); restHeaders.put("header.2", Arrays.asList("foo", "bar")); @@ -226,14 +221,7 @@ public void testRequestWithDisallowedMultiValuedHeaderButSameValues() { Set headers = new HashSet<>( Arrays.asList(new RestHeaderDefinition("header.1", true), new RestHeaderDefinition("header.2", false)) ); - final RestController restController = new RestController( - headers, - null, - client, - circuitBreakerService, - usageService, - identityService - ); + final RestController restController = new RestController(headers, null, client, circuitBreakerService, usageService); Map> restHeaders = new HashMap<>(); restHeaders.put("header.1", Collections.singletonList("boo")); restHeaders.put("header.2", Arrays.asList("foo", "foo")); @@ -294,7 +282,7 @@ public void testRegisterWithDeprecatedHandler() { } public void testRegisterSecondMethodWithDifferentNamedWildcard() { - final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService, identityService); + final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService); RestRequest.Method firstMethod = randomFrom(RestRequest.Method.values()); RestRequest.Method secondMethod = randomFrom( @@ -322,7 +310,7 @@ public void testRestHandlerWrapper() throws Exception { final RestController restController = new RestController(Collections.emptySet(), h -> { assertSame(handler, h); return (RestRequest request, RestChannel channel, NodeClient client) -> wrapperCalled.set(true); - }, client, circuitBreakerService, usageService, identityService); + }, client, circuitBreakerService, usageService); restController.registerHandler(RestRequest.Method.GET, "/wrapped", handler); RestRequest request = testRestRequest("/wrapped", "{}", MediaTypeRegistry.JSON); AssertingChannel channel = new AssertingChannel(request, true, RestStatus.BAD_REQUEST); @@ -385,7 +373,7 @@ public void testDispatchRequiresContentTypeForRequestsWithContent() { String content = randomAlphaOfLength((int) Math.round(BREAKER_LIMIT.getBytes() / inFlightRequestsBreaker.getOverhead())); RestRequest request = testRestRequest("/", content, null); AssertingChannel channel = new AssertingChannel(request, true, RestStatus.NOT_ACCEPTABLE); - restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService, identityService); + restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService); restController.registerHandler( RestRequest.Method.GET, "/", diff --git a/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java b/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java index 983121a4f481d..b8602cdc20e6a 100644 --- a/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java +++ b/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java @@ -39,12 +39,10 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.core.rest.RestStatus; -import org.opensearch.identity.IdentityService; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestChannel; import org.opensearch.test.rest.FakeRestRequest; -import org.opensearch.threadpool.ThreadPool; import org.opensearch.usage.UsageService; import java.util.ArrayList; @@ -56,7 +54,6 @@ import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.is; -import static org.mockito.Mockito.mock; public class RestHttpResponseHeadersTests extends OpenSearchTestCase { @@ -106,17 +103,8 @@ public void testUnsupportedMethodResponseHttpHeader() throws Exception { new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) ); - final Settings settings = Settings.EMPTY; UsageService usageService = new UsageService(); - final IdentityService identityService = new IdentityService(settings, mock(ThreadPool.class), List.of()); - RestController restController = new RestController( - Collections.emptySet(), - null, - null, - circuitBreakerService, - usageService, - identityService - ); + RestController restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService); // A basic RestHandler handles requests to the endpoint RestHandler restHandler = new RestHandler() { diff --git a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java index c3cf33f4e9034..6aa1d10d71e50 100644 --- a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java @@ -44,7 +44,6 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.core.xcontent.MediaTypeRegistry; -import org.opensearch.identity.IdentityService; import org.opensearch.rest.RestController; import org.opensearch.rest.RestRequest; import org.opensearch.search.AbstractSearchTestCase; @@ -61,7 +60,6 @@ import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import static java.util.Collections.emptyMap; @@ -75,15 +73,7 @@ public class RestValidateQueryActionTests extends AbstractSearchTestCase { private static NodeClient client = new NodeClient(Settings.EMPTY, threadPool); private static UsageService usageService = new UsageService(); - private static IdentityService identityService = new IdentityService(Settings.EMPTY, threadPool, List.of()); - private static RestController controller = new RestController( - emptySet(), - null, - client, - new NoneCircuitBreakerService(), - usageService, - identityService - ); + private static RestController controller = new RestController(emptySet(), null, client, new NoneCircuitBreakerService(), usageService); private static RestValidateQueryAction action = new RestValidateQueryAction(); /** diff --git a/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java b/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java index c7a0fe35b0237..fec1699c9ef64 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java @@ -40,25 +40,20 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.core.action.ActionResponse; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; -import org.opensearch.identity.IdentityService; import org.opensearch.rest.RestController; import org.opensearch.rest.RestRequest; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskListener; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.client.NoOpNodeClient; -import org.opensearch.threadpool.ThreadPool; import org.opensearch.usage.UsageService; import org.junit.After; import org.junit.Before; import java.util.Collections; -import java.util.List; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; -import static org.mockito.Mockito.mock; - /** * A common base class for Rest*ActionTests. Provides access to a {@link RestController} * that can be used to register individual REST actions, and test request handling. @@ -70,15 +65,7 @@ public abstract class RestActionTestCase extends OpenSearchTestCase { @Before public void setUpController() { verifyingClient = new VerifyingClient(this.getTestName()); - final IdentityService identityService = new IdentityService(Settings.EMPTY, mock(ThreadPool.class), List.of()); - controller = new RestController( - Collections.emptySet(), - null, - verifyingClient, - new NoneCircuitBreakerService(), - new UsageService(), - identityService - ); + controller = new RestController(Collections.emptySet(), null, verifyingClient, new NoneCircuitBreakerService(), new UsageService()); } @After From 19d8df02444204e2098e012e553066489bb9b679 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 19 Sep 2024 17:22:45 -0400 Subject: [PATCH 40/44] Update version check in yml test file (#15999) Signed-off-by: Gao Binlong Co-authored-by: Gao Binlong --- .../rest-api-spec/test/cluster.put_settings/10_basic.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml index 0f4a45e4591a3..ce78709f6ed5e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml @@ -103,8 +103,8 @@ --- "Test setting search backpressure cancellation settings": - skip: - version: "- 2.99.99" - reason: "Fixed in 3.0.0" + version: "- 2.17.99" + reason: "Fixed in 2.18.0" - do: cluster.put_settings: @@ -181,8 +181,8 @@ --- "Test setting invalid search backpressure cancellation_rate and cancellation_ratio": - skip: - version: "- 2.99.99" - reason: "Fixed in 3.0.0" + version: "- 2.17.99" + reason: "Fixed in 2.18.0" - do: catch: /search_backpressure.search_task.cancellation_rate must be > 0/ From 94222f1dfaa2fd07598141fb9a894ebca881e201 Mon Sep 17 00:00:00 2001 From: Rishabh Singh Date: Thu, 19 Sep 2024 15:08:45 -0700 Subject: [PATCH 41/44] Update big5 workload config to use the ordered documents snapshot (#16005) Signed-off-by: Rishabh Singh --- .github/benchmark-configs.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/benchmark-configs.json b/.github/benchmark-configs.json index 4ada715d21495..27b7228e1203a 100644 --- a/.github/benchmark-configs.json +++ b/.github/benchmark-configs.json @@ -75,7 +75,7 @@ "SINGLE_NODE_CLUSTER": "true", "MIN_DISTRIBUTION": "true", "TEST_WORKLOAD": "big5", - "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}", + "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}", "CAPTURE_NODE_STAT": "true", "TEST_PROCEDURE": "restore-from-snapshot" }, @@ -126,7 +126,7 @@ "SINGLE_NODE_CLUSTER": "true", "MIN_DISTRIBUTION": "true", "TEST_WORKLOAD": "big5", - "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots\",\"snapshot_name\":\"big5_1_shard\"}", + "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots\",\"snapshot_name\":\"big5_1_shard_ordered\"}", "CAPTURE_NODE_STAT": "true", "TEST_PROCEDURE": "restore-from-snapshot" }, @@ -176,7 +176,7 @@ "MIN_DISTRIBUTION": "true", "TEST_WORKLOAD": "big5", "ADDITIONAL_CONFIG": "search.concurrent_segment_search.enabled:true", - "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}", + "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}", "CAPTURE_NODE_STAT": "true", "TEST_PROCEDURE": "restore-from-snapshot" }, @@ -194,7 +194,7 @@ "MIN_DISTRIBUTION": "true", "TEST_WORKLOAD": "big5", "ADDITIONAL_CONFIG": "search.concurrent_segment_search.mode:all", - "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}", + "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}", "CAPTURE_NODE_STAT": "true", "TEST_PROCEDURE": "restore-from-snapshot" }, @@ -212,7 +212,7 @@ "MIN_DISTRIBUTION": "true", "TEST_WORKLOAD": "big5", "ADDITIONAL_CONFIG": "search.concurrent_segment_search.mode:auto", - "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}", + "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}", "CAPTURE_NODE_STAT": "true", "TEST_PROCEDURE": "restore-from-snapshot" }, From b3696116965ab45b675731af04e6defc34602f83 Mon Sep 17 00:00:00 2001 From: David Zane <38449481+dzane17@users.noreply.github.com> Date: Thu, 19 Sep 2024 16:22:48 -0700 Subject: [PATCH 42/44] Add successfulSearchShardIndices in searchRequestContext before onRequestEnd (#15967) Signed-off-by: David Zane Signed-off-by: Ankit Jain Co-authored-by: Ankit Jain --- CHANGELOG.md | 1 + .../action/search/AbstractSearchAsyncAction.java | 3 +++ .../action/search/SearchRequestContext.java | 14 ++++++++++++++ 3 files changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5aef4bb551d3..eebebec672058 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) - Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) +- Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967)) - Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) ### Dependencies diff --git a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java index f0fc05c595d6f..836083f91b995 100644 --- a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java @@ -754,6 +754,9 @@ public void sendSearchResponse(InternalSearchResponse internalSearchResponse, At } searchRequestContext.setTotalHits(internalSearchResponse.hits().getTotalHits()); searchRequestContext.setShardStats(results.getNumShards(), successfulOps.get(), skippedOps.get(), failures.length); + searchRequestContext.setSuccessfulSearchShardIndices( + results.getSuccessfulResults().map(result -> result.getSearchShardTarget().getIndex()).collect(Collectors.toSet()) + ); onPhaseEnd(searchRequestContext); onRequestEnd(searchRequestContext); listener.onResponse(buildSearchResponse(internalSearchResponse, failures, scrollId, searchContextId)); diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java b/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java index 111d9c64550b3..376cf71448d5c 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java +++ b/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; import java.util.function.Supplier; @@ -36,6 +37,7 @@ public class SearchRequestContext { private final Map phaseTookMap; private TotalHits totalHits; private final EnumMap shardStats; + private Set successfulSearchShardIndices; private final SearchRequest searchRequest; private final LinkedBlockingQueue phaseResourceUsage; @@ -141,6 +143,18 @@ public List getPhaseResourceUsage() { public SearchRequest getRequest() { return searchRequest; } + + void setSuccessfulSearchShardIndices(Set successfulSearchShardIndices) { + this.successfulSearchShardIndices = successfulSearchShardIndices; + } + + /** + * @return A {@link List} of {@link String} representing the names of the indices that were + * successfully queried at the shard level. + */ + public Set getSuccessfulSearchShardIndices() { + return successfulSearchShardIndices; + } } enum ShardStatsFieldNames { From 9537d39cc7f4a55c20b7e6f3f3e1a8cec5cb5ec7 Mon Sep 17 00:00:00 2001 From: Rishabh Singh Date: Thu, 19 Sep 2024 18:11:35 -0700 Subject: [PATCH 43/44] add concurrency check to gradle-check workflow (#16007) Signed-off-by: Rishabh Singh --- .github/workflows/gradle-check.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml index 1b9b30625eb83..1421eeb7f7576 100644 --- a/.github/workflows/gradle-check.yml +++ b/.github/workflows/gradle-check.yml @@ -8,6 +8,10 @@ on: pull_request_target: types: [opened, synchronize, reopened] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + permissions: contents: read # to fetch code (actions/checkout) From 2b670cc11f52ada701cbf867ce8ad4527d8b6072 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Fri, 20 Sep 2024 09:36:00 +0530 Subject: [PATCH 44/44] Add support for async deletion in S3BlobContainer (#15621) * Add support for async deletion in S3BlobContainer Signed-off-by: Ashish Singh * Move helper methods to helper class Signed-off-by: Ashish Singh * Minor refactor Signed-off-by: Ashish Singh * Add UTs Signed-off-by: Ashish Singh * Add more tests Signed-off-by: Ashish Singh * Integrate async deletion in the snapshot interactions Signed-off-by: Ashish Singh --------- Signed-off-by: Ashish Singh --- CHANGELOG.md | 1 + .../s3/S3BlobStoreRepositoryTests.java | 1 + .../s3/S3RepositoryThirdPartyTests.java | 8 + .../repositories/s3/S3BlobContainer.java | 124 +++++ .../repositories/s3/S3BlobStore.java | 2 +- .../repositories/s3/StatsMetricPublisher.java | 4 + .../s3/async/S3AsyncDeleteHelper.java | 95 ++++ .../s3/S3BlobStoreContainerTests.java | 507 +++++++++++++++++- .../s3/async/S3AsyncDeleteHelperTests.java | 236 ++++++++ .../mocks/MockFsAsyncBlobContainer.java | 12 + .../AsyncMultiStreamBlobContainer.java | 5 + ...syncMultiStreamEncryptedBlobContainer.java | 10 + .../common/settings/ClusterSettings.java | 3 + .../blobstore/BlobStoreRepository.java | 60 ++- .../BlobStoreTransferServiceTests.java | 12 + .../snapshots/BlobStoreFormatTests.java | 12 + 16 files changed, 1086 insertions(+), 6 deletions(-) create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelper.java create mode 100644 plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelperTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index eebebec672058..abcadde83f000 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added +- Add support for async deletion in S3BlobContainer ([#15621](https://github.com/opensearch-project/OpenSearch/pull/15621)) - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java index c5438d58e679d..944de326d144c 100644 --- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -153,6 +153,7 @@ protected Settings nodeSettings(int nodeOrdinal) { // Disable request throttling because some random values in tests might generate too many failures for the S3 client .put(S3ClientSettings.USE_THROTTLE_RETRIES_SETTING.getConcreteSettingForNamespace("test").getKey(), false) .put(S3ClientSettings.PROXY_TYPE_SETTING.getConcreteSettingForNamespace("test").getKey(), ProxySettings.ProxyType.DIRECT) + .put(BlobStoreRepository.SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING.getKey(), false) .put(super.nodeSettings(nodeOrdinal)) .setSecureSettings(secureSettings); diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java index 7db9a0d3ba790..f0e40db965646 100644 --- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java +++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java @@ -55,6 +55,14 @@ public class S3RepositoryThirdPartyTests extends AbstractThirdPartyRepositoryTestCase { + @Override + protected Settings nodeSettings() { + return Settings.builder() + .put(super.nodeSettings()) + .put(BlobStoreRepository.SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING.getKey(), false) + .build(); + } + @Override @Before @SuppressForbidden(reason = "Need to set system property here for AWS SDK v2") diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java index b489a3cc85037..1a402e8431e25 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java @@ -62,6 +62,7 @@ import software.amazon.awssdk.services.s3.model.UploadPartRequest; import software.amazon.awssdk.services.s3.model.UploadPartResponse; import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; +import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Publisher; import software.amazon.awssdk.utils.CollectionUtils; import org.apache.logging.log4j.LogManager; @@ -90,6 +91,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.repositories.s3.async.S3AsyncDeleteHelper; import org.opensearch.repositories.s3.async.SizeBasedBlockingQ; import org.opensearch.repositories.s3.async.UploadRequest; import org.opensearch.repositories.s3.utils.HttpRangeUtils; @@ -109,6 +111,9 @@ import java.util.function.Function; import java.util.stream.Collectors; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; + import static org.opensearch.repositories.s3.S3Repository.MAX_FILE_SIZE; import static org.opensearch.repositories.s3.S3Repository.MAX_FILE_SIZE_USING_MULTIPART; import static org.opensearch.repositories.s3.S3Repository.MIN_PART_SIZE_USING_MULTIPART; @@ -875,4 +880,123 @@ CompletableFuture getBlobMetadata(S3AsyncClient s3A return SocketAccess.doPrivileged(() -> s3AsyncClient.getObjectAttributes(getObjectAttributesRequest)); } + + @Override + public void deleteAsync(ActionListener completionListener) { + try (AmazonAsyncS3Reference asyncClientReference = blobStore.asyncClientReference()) { + S3AsyncClient s3AsyncClient = asyncClientReference.get().client(); + + ListObjectsV2Request listRequest = ListObjectsV2Request.builder().bucket(blobStore.bucket()).prefix(keyPath).build(); + ListObjectsV2Publisher listPublisher = s3AsyncClient.listObjectsV2Paginator(listRequest); + + AtomicLong deletedBlobs = new AtomicLong(); + AtomicLong deletedBytes = new AtomicLong(); + + CompletableFuture listingFuture = new CompletableFuture<>(); + + listPublisher.subscribe(new Subscriber<>() { + private Subscription subscription; + private final List objectsToDelete = new ArrayList<>(); + private CompletableFuture deletionChain = CompletableFuture.completedFuture(null); + + @Override + public void onSubscribe(Subscription s) { + this.subscription = s; + subscription.request(1); + } + + @Override + public void onNext(ListObjectsV2Response response) { + response.contents().forEach(s3Object -> { + deletedBlobs.incrementAndGet(); + deletedBytes.addAndGet(s3Object.size()); + objectsToDelete.add(s3Object.key()); + }); + + int bulkDeleteSize = blobStore.getBulkDeletesSize(); + if (objectsToDelete.size() >= bulkDeleteSize) { + int fullBatchesCount = objectsToDelete.size() / bulkDeleteSize; + int itemsToDelete = fullBatchesCount * bulkDeleteSize; + + List batchToDelete = new ArrayList<>(objectsToDelete.subList(0, itemsToDelete)); + objectsToDelete.subList(0, itemsToDelete).clear(); + + deletionChain = S3AsyncDeleteHelper.executeDeleteChain( + s3AsyncClient, + blobStore, + batchToDelete, + deletionChain, + () -> subscription.request(1) + ); + } else { + subscription.request(1); + } + } + + @Override + public void onError(Throwable t) { + listingFuture.completeExceptionally(new IOException("Failed to list objects for deletion", t)); + } + + @Override + public void onComplete() { + if (!objectsToDelete.isEmpty()) { + deletionChain = S3AsyncDeleteHelper.executeDeleteChain( + s3AsyncClient, + blobStore, + objectsToDelete, + deletionChain, + null + ); + } + deletionChain.whenComplete((v, throwable) -> { + if (throwable != null) { + listingFuture.completeExceptionally(throwable); + } else { + listingFuture.complete(null); + } + }); + } + }); + + listingFuture.whenComplete((v, throwable) -> { + if (throwable != null) { + completionListener.onFailure( + throwable instanceof Exception + ? (Exception) throwable + : new IOException("Unexpected error during async deletion", throwable) + ); + } else { + completionListener.onResponse(new DeleteResult(deletedBlobs.get(), deletedBytes.get())); + } + }); + } catch (Exception e) { + completionListener.onFailure(new IOException("Failed to initiate async deletion", e)); + } + } + + @Override + public void deleteBlobsAsyncIgnoringIfNotExists(List blobNames, ActionListener completionListener) { + if (blobNames.isEmpty()) { + completionListener.onResponse(null); + return; + } + + try (AmazonAsyncS3Reference asyncClientReference = blobStore.asyncClientReference()) { + S3AsyncClient s3AsyncClient = asyncClientReference.get().client(); + + List keysToDelete = blobNames.stream().map(this::buildKey).collect(Collectors.toList()); + + S3AsyncDeleteHelper.executeDeleteChain(s3AsyncClient, blobStore, keysToDelete, CompletableFuture.completedFuture(null), null) + .whenComplete((v, throwable) -> { + if (throwable != null) { + completionListener.onFailure(new IOException("Failed to delete blobs " + blobNames, throwable)); + } else { + completionListener.onResponse(null); + } + }); + } catch (Exception e) { + completionListener.onFailure(new IOException("Failed to initiate async blob deletion", e)); + } + } } diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java index f688be9216b8f..90bfa11e18481 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java @@ -63,7 +63,7 @@ import static org.opensearch.repositories.s3.S3Repository.STORAGE_CLASS_SETTING; import static org.opensearch.repositories.s3.S3Repository.UPLOAD_RETRY_ENABLED; -class S3BlobStore implements BlobStore { +public class S3BlobStore implements BlobStore { private static final Logger logger = LogManager.getLogger(S3BlobStore.class); diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java index 8d2772d42ebca..9f73c67df3b18 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java @@ -95,6 +95,10 @@ public void publish(MetricCollection metricCollection) { public void close() {} }; + public MetricPublisher getDeleteObjectsMetricPublisher() { + return deleteObjectsMetricPublisher; + } + public MetricPublisher getObjectMetricPublisher = new MetricPublisher() { @Override public void publish(MetricCollection metricCollection) { diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelper.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelper.java new file mode 100644 index 0000000000000..eed95c0e68ef1 --- /dev/null +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelper.java @@ -0,0 +1,95 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3.async; + +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.Delete; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.repositories.s3.S3BlobStore; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +public class S3AsyncDeleteHelper { + private static final Logger logger = LogManager.getLogger(S3AsyncDeleteHelper.class); + + public static CompletableFuture executeDeleteChain( + S3AsyncClient s3AsyncClient, + S3BlobStore blobStore, + List objectsToDelete, + CompletableFuture currentChain, + Runnable afterDeleteAction + ) { + List> batches = createDeleteBatches(objectsToDelete, blobStore.getBulkDeletesSize()); + CompletableFuture newChain = currentChain.thenCompose(v -> executeDeleteBatches(s3AsyncClient, blobStore, batches)); + if (afterDeleteAction != null) { + newChain = newChain.thenRun(afterDeleteAction); + } + return newChain; + } + + static List> createDeleteBatches(List keys, int bulkDeleteSize) { + List> batches = new ArrayList<>(); + for (int i = 0; i < keys.size(); i += bulkDeleteSize) { + batches.add(keys.subList(i, Math.min(keys.size(), i + bulkDeleteSize))); + } + return batches; + } + + static CompletableFuture executeDeleteBatches(S3AsyncClient s3AsyncClient, S3BlobStore blobStore, List> batches) { + CompletableFuture allDeletesFuture = CompletableFuture.completedFuture(null); + + for (List batch : batches) { + allDeletesFuture = allDeletesFuture.thenCompose(v -> executeSingleDeleteBatch(s3AsyncClient, blobStore, batch)); + } + + return allDeletesFuture; + } + + static CompletableFuture executeSingleDeleteBatch(S3AsyncClient s3AsyncClient, S3BlobStore blobStore, List batch) { + DeleteObjectsRequest deleteRequest = bulkDelete(blobStore.bucket(), batch, blobStore); + return s3AsyncClient.deleteObjects(deleteRequest).thenApply(S3AsyncDeleteHelper::processDeleteResponse); + } + + static Void processDeleteResponse(DeleteObjectsResponse deleteObjectsResponse) { + if (!deleteObjectsResponse.errors().isEmpty()) { + logger.warn( + () -> new ParameterizedMessage( + "Failed to delete some blobs {}", + deleteObjectsResponse.errors() + .stream() + .map(s3Error -> "[" + s3Error.key() + "][" + s3Error.code() + "][" + s3Error.message() + "]") + .collect(Collectors.toList()) + ) + ); + } + return null; + } + + static DeleteObjectsRequest bulkDelete(String bucket, List blobs, S3BlobStore blobStore) { + return DeleteObjectsRequest.builder() + .bucket(bucket) + .delete( + Delete.builder() + .objects(blobs.stream().map(blob -> ObjectIdentifier.builder().key(blob).build()).collect(Collectors.toList())) + .quiet(true) + .build() + ) + .overrideConfiguration(o -> o.addMetricPublisher(blobStore.getStatsMetricPublisher().getDeleteObjectsMetricPublisher())) + .build(); + } +} diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java index 654d8a72690c4..2cb11541d924f 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java @@ -69,6 +69,7 @@ import software.amazon.awssdk.services.s3.model.UploadPartRequest; import software.amazon.awssdk.services.s3.model.UploadPartResponse; import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; +import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Publisher; import org.opensearch.action.LatchedActionListener; import org.opensearch.common.blobstore.BlobContainer; @@ -99,22 +100,28 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; public class S3BlobStoreContainerTests extends OpenSearchTestCase { @@ -1275,6 +1282,504 @@ public void testTransformResponseToInputStreamContainer() throws Exception { assertEquals(inputStream.available(), inputStreamContainer.getInputStream().available()); } + public void testDeleteAsync() throws Exception { + for (int i = 0; i < 100; i++) { + testDeleteAsync(i + 1); + } + } + + private void testDeleteAsync(int bulkDeleteSize) throws InterruptedException { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(bulkDeleteSize); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + final List s3Objects = new ArrayList<>(); + int numObjects = randomIntBetween(20, 100); + long totalSize = 0; + for (int i = 0; i < numObjects; i++) { + long size = randomIntBetween(1, 100); + s3Objects.add(S3Object.builder().key(randomAlphaOfLength(10)).size(size).build()); + totalSize += size; + } + + final List responseList = new ArrayList<>(); + int size = 0; + while (size < numObjects) { + int toAdd = randomIntBetween(10, 20); + int endIndex = Math.min(numObjects, size + toAdd); + responseList.add(ListObjectsV2Response.builder().contents(s3Objects.subList(size, endIndex)).build()); + size = endIndex; + } + int expectedDeletedObjectsCall = numObjects / bulkDeleteSize + (numObjects % bulkDeleteSize > 0 ? 1 : 0); + + final ListObjectsV2Publisher listPublisher = mock(ListObjectsV2Publisher.class); + AtomicInteger counter = new AtomicInteger(); + doAnswer(invocation -> { + Subscriber subscriber = invocation.getArgument(0); + subscriber.onSubscribe(new Subscription() { + @Override + public void request(long n) { + int currentCounter = counter.getAndIncrement(); + if (currentCounter < responseList.size()) { + subscriber.onNext(responseList.get(currentCounter)); + } + if (currentCounter == responseList.size()) { + subscriber.onComplete(); + } + } + + @Override + public void cancel() {} + }); + return null; + }).when(listPublisher).subscribe(ArgumentMatchers.>any()); + when(s3AsyncClient.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenReturn(listPublisher); + + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn( + CompletableFuture.completedFuture(DeleteObjectsResponse.builder().build()) + ); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference deleteResultRef = new AtomicReference<>(); + blobContainer.deleteAsync(new ActionListener<>() { + @Override + public void onResponse(DeleteResult deleteResult) { + deleteResultRef.set(deleteResult); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + logger.error("exception during deleteAsync", e); + fail("Unexpected failure: " + e.getMessage()); + } + }); + + latch.await(); + + DeleteResult deleteResult = deleteResultRef.get(); + assertEquals(numObjects, deleteResult.blobsDeleted()); + assertEquals(totalSize, deleteResult.bytesDeleted()); + + verify(s3AsyncClient, times(1)).listObjectsV2Paginator(any(ListObjectsV2Request.class)); + verify(s3AsyncClient, times(expectedDeletedObjectsCall)).deleteObjects(any(DeleteObjectsRequest.class)); + } + + public void testDeleteAsyncFailure() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(1000); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + // Simulate a failure in listObjectsV2Paginator + RuntimeException simulatedFailure = new RuntimeException("Simulated failure"); + when(s3AsyncClient.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenThrow(simulatedFailure); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + blobContainer.deleteAsync(new ActionListener<>() { + @Override + public void onResponse(DeleteResult deleteResult) { + fail("Expected a failure, but got a success response"); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertNotNull(exceptionRef.get()); + assertEquals(IOException.class, exceptionRef.get().getClass()); + assertEquals("Failed to initiate async deletion", exceptionRef.get().getMessage()); + assertEquals(simulatedFailure, exceptionRef.get().getCause()); + } + + public void testDeleteAsyncListingError() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(1000); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + final ListObjectsV2Publisher listPublisher = mock(ListObjectsV2Publisher.class); + doAnswer(invocation -> { + Subscriber subscriber = invocation.getArgument(0); + subscriber.onSubscribe(new Subscription() { + @Override + public void request(long n) { + subscriber.onError(new RuntimeException("Simulated listing error")); + } + + @Override + public void cancel() {} + }); + return null; + }).when(listPublisher).subscribe(ArgumentMatchers.>any()); + when(s3AsyncClient.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenReturn(listPublisher); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + blobContainer.deleteAsync(new ActionListener<>() { + @Override + public void onResponse(DeleteResult deleteResult) { + fail("Expected a failure, but got a success response"); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertNotNull(exceptionRef.get()); + assertEquals(IOException.class, exceptionRef.get().getClass()); + assertEquals("Failed to list objects for deletion", exceptionRef.get().getMessage()); + assertNotNull(exceptionRef.get().getCause()); + assertEquals("Simulated listing error", exceptionRef.get().getCause().getMessage()); + } + + public void testDeleteAsyncDeletionError() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(1000); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + final ListObjectsV2Publisher listPublisher = mock(ListObjectsV2Publisher.class); + doAnswer(invocation -> { + Subscriber subscriber = invocation.getArgument(0); + subscriber.onSubscribe(new Subscription() { + @Override + public void request(long n) { + subscriber.onNext( + ListObjectsV2Response.builder().contents(S3Object.builder().key("test-key").size(100L).build()).build() + ); + subscriber.onComplete(); + } + + @Override + public void cancel() {} + }); + return null; + }).when(listPublisher).subscribe(ArgumentMatchers.>any()); + when(s3AsyncClient.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenReturn(listPublisher); + + // Simulate a failure in deleteObjects + CompletableFuture failedFuture = new CompletableFuture<>(); + failedFuture.completeExceptionally(new RuntimeException("Simulated delete error")); + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(failedFuture); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + blobContainer.deleteAsync(new ActionListener<>() { + @Override + public void onResponse(DeleteResult deleteResult) { + fail("Expected a failure, but got a success response"); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertNotNull(exceptionRef.get()); + assertEquals(CompletionException.class, exceptionRef.get().getClass()); + assertEquals("java.lang.RuntimeException: Simulated delete error", exceptionRef.get().getMessage()); + assertNotNull(exceptionRef.get().getCause()); + assertEquals("Simulated delete error", exceptionRef.get().getCause().getMessage()); + } + + public void testDeleteBlobsAsyncIgnoringIfNotExists() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + int bulkDeleteSize = randomIntBetween(1, 10); + when(blobStore.getBulkDeletesSize()).thenReturn(bulkDeleteSize); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + final List blobNames = new ArrayList<>(); + int size = randomIntBetween(10, 100); + for (int i = 0; i < size; i++) { + blobNames.add(randomAlphaOfLength(10)); + } + int expectedDeleteCalls = size / bulkDeleteSize + (size % bulkDeleteSize > 0 ? 1 : 0); + + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn( + CompletableFuture.completedFuture(DeleteObjectsResponse.builder().build()) + ); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + blobContainer.deleteBlobsAsyncIgnoringIfNotExists(blobNames, new ActionListener() { + @Override + public void onResponse(Void aVoid) { + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertNull(exceptionRef.get()); + + ArgumentCaptor deleteRequestCaptor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3AsyncClient, times(expectedDeleteCalls)).deleteObjects(deleteRequestCaptor.capture()); + + DeleteObjectsRequest capturedRequest = deleteRequestCaptor.getAllValues().stream().findAny().get(); + assertEquals(bucketName, capturedRequest.bucket()); + int totalBlobsDeleted = deleteRequestCaptor.getAllValues() + .stream() + .map(r -> r.delete().objects().size()) + .reduce(Integer::sum) + .get(); + assertEquals(blobNames.size(), totalBlobsDeleted); + List deletedKeys = deleteRequestCaptor.getAllValues() + .stream() + .flatMap(r -> r.delete().objects().stream()) + .map(ObjectIdentifier::key) + .collect(Collectors.toList()); + assertTrue(deletedKeys.containsAll(blobNames)); + } + + public void testDeleteBlobsAsyncIgnoringIfNotExistsFailure() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(1000); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + // Simulate a failure in deleteObjects + CompletableFuture failedFuture = new CompletableFuture<>(); + failedFuture.completeExceptionally(new RuntimeException("Simulated delete failure")); + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(failedFuture); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + List blobNames = Arrays.asList("blob1", "blob2", "blob3"); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + blobContainer.deleteBlobsAsyncIgnoringIfNotExists(blobNames, new ActionListener() { + @Override + public void onResponse(Void aVoid) { + fail("Expected a failure, but got a success response"); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertNotNull(exceptionRef.get()); + assertEquals(IOException.class, exceptionRef.get().getClass()); + assertEquals("Failed to delete blobs " + blobNames, exceptionRef.get().getMessage()); + assertNotNull(exceptionRef.get().getCause()); + assertEquals("java.lang.RuntimeException: Simulated delete failure", exceptionRef.get().getCause().getMessage()); + } + + public void testDeleteBlobsAsyncIgnoringIfNotExistsWithEmptyList() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(1000); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + List emptyBlobNames = Collections.emptyList(); + + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean onResponseCalled = new AtomicBoolean(false); + AtomicReference exceptionRef = new AtomicReference<>(); + + blobContainer.deleteBlobsAsyncIgnoringIfNotExists(emptyBlobNames, new ActionListener() { + @Override + public void onResponse(Void aVoid) { + onResponseCalled.set(true); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertTrue("onResponse should have been called", onResponseCalled.get()); + assertNull("No exception should have been thrown", exceptionRef.get()); + + // Verify that no interactions with S3AsyncClient occurred + verifyNoInteractions(s3AsyncClient); + } + + public void testDeleteBlobsAsyncIgnoringIfNotExistsInitializationFailure() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(1000); + + // Simulate a failure when getting the asyncClientReference + RuntimeException simulatedFailure = new RuntimeException("Simulated initialization failure"); + when(blobStore.asyncClientReference()).thenThrow(simulatedFailure); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + List blobNames = Arrays.asList("blob1", "blob2", "blob3"); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + + blobContainer.deleteBlobsAsyncIgnoringIfNotExists(blobNames, new ActionListener() { + @Override + public void onResponse(Void aVoid) { + fail("Expected a failure, but got a success response"); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertNotNull("An exception should have been thrown", exceptionRef.get()); + assertTrue("Exception should be an IOException", exceptionRef.get() instanceof IOException); + assertEquals("Failed to initiate async blob deletion", exceptionRef.get().getMessage()); + assertEquals(simulatedFailure, exceptionRef.get().getCause()); + } + private void mockObjectResponse(S3AsyncClient s3AsyncClient, String bucketName, String blobName, int objectSize) { final InputStream inputStream = new ByteArrayInputStream(randomByteArrayOfLength(objectSize)); diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelperTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelperTests.java new file mode 100644 index 0000000000000..d7f924e05cc70 --- /dev/null +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelperTests.java @@ -0,0 +1,236 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3.async; + +import software.amazon.awssdk.metrics.MetricPublisher; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.S3Error; + +import org.opensearch.repositories.s3.S3BlobStore; +import org.opensearch.repositories.s3.StatsMetricPublisher; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class S3AsyncDeleteHelperTests extends OpenSearchTestCase { + + @Mock + private S3AsyncClient s3AsyncClient; + + @Mock + private S3BlobStore blobStore; + + @Override + public void setUp() throws Exception { + super.setUp(); + MockitoAnnotations.openMocks(this); + } + + public void testExecuteDeleteChain() { + List objectsToDelete = Arrays.asList("key1", "key2", "key3"); + CompletableFuture currentChain = CompletableFuture.completedFuture(null); + + // Mock the deleteObjects method of S3AsyncClient + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn( + CompletableFuture.completedFuture(DeleteObjectsResponse.builder().build()) + ); + + // Mock the getBulkDeletesSize method of S3BlobStore + when(blobStore.getBulkDeletesSize()).thenReturn(2); + + // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value + StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class); + MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class); + when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher); + when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher); + + CompletableFuture newChain = S3AsyncDeleteHelper.executeDeleteChain( + s3AsyncClient, + blobStore, + objectsToDelete, + currentChain, + null + ); + + // Verify that the newChain is completed without any exceptions + assertNotNull(newChain); + assertTrue(newChain.isDone()); + assertFalse(newChain.isCompletedExceptionally()); + + // Verify that the deleteObjects method of S3AsyncClient was called with the expected request + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3AsyncClient, times(2)).deleteObjects(requestCaptor.capture()); + + List capturedRequests = requestCaptor.getAllValues(); + assertEquals(2, capturedRequests.size()); + + // Verify that the requests have the expected metric publisher added + for (DeleteObjectsRequest request : capturedRequests) { + assertNotNull(request.overrideConfiguration()); + assertTrue(request.overrideConfiguration().get().metricPublishers().contains(mockDeleteObjectsMetricPublisher)); + } + } + + public void testCreateDeleteBatches() { + List keys = Arrays.asList("key1", "key2", "key3", "key4", "key5", "key6"); + int bulkDeleteSize = 3; + + List> batches = S3AsyncDeleteHelper.createDeleteBatches(keys, bulkDeleteSize); + + assertEquals(2, batches.size()); + assertEquals(Arrays.asList("key1", "key2", "key3"), batches.get(0)); + assertEquals(Arrays.asList("key4", "key5", "key6"), batches.get(1)); + } + + public void testExecuteSingleDeleteBatch() throws Exception { + List batch = Arrays.asList("key1", "key2"); + DeleteObjectsResponse expectedResponse = DeleteObjectsResponse.builder().build(); + + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(CompletableFuture.completedFuture(expectedResponse)); + + // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value + StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class); + MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class); + when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher); + when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher); + + CompletableFuture future = S3AsyncDeleteHelper.executeSingleDeleteBatch(s3AsyncClient, blobStore, batch); + + assertNotNull(future); + assertTrue(future.isDone()); + assertFalse(future.isCompletedExceptionally()); + future.join(); // Wait for the CompletableFuture to complete + + // Verify that the deleteObjects method of S3AsyncClient was called with the expected request + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3AsyncClient).deleteObjects(requestCaptor.capture()); + + DeleteObjectsRequest capturedRequest = requestCaptor.getValue(); + assertEquals(blobStore.bucket(), capturedRequest.bucket()); + assertEquals(batch.size(), capturedRequest.delete().objects().size()); + assertTrue(capturedRequest.delete().objects().stream().map(ObjectIdentifier::key).collect(Collectors.toList()).containsAll(batch)); + } + + public void testProcessDeleteResponse() { + DeleteObjectsResponse response = DeleteObjectsResponse.builder() + .errors( + Arrays.asList( + S3Error.builder().key("key1").code("Code1").message("Message1").build(), + S3Error.builder().key("key2").code("Code2").message("Message2").build() + ) + ) + .build(); + + // Call the processDeleteResponse method + S3AsyncDeleteHelper.processDeleteResponse(response); + } + + public void testBulkDelete() { + List blobs = Arrays.asList("key1", "key2", "key3"); + String bucketName = "my-bucket"; + + // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value + StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class); + MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class); + when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher); + when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher); + + DeleteObjectsRequest request = S3AsyncDeleteHelper.bulkDelete(bucketName, blobs, blobStore); + + assertEquals(bucketName, request.bucket()); + assertEquals(blobs.size(), request.delete().objects().size()); + assertTrue(request.delete().objects().stream().map(ObjectIdentifier::key).collect(Collectors.toList()).containsAll(blobs)); + assertNotNull(request.overrideConfiguration()); + assertTrue(request.overrideConfiguration().get().metricPublishers().contains(mockDeleteObjectsMetricPublisher)); + } + + public void testExecuteDeleteBatches() { + List> batches = Arrays.asList(Arrays.asList("key1", "key2"), Arrays.asList("key3", "key4")); + DeleteObjectsResponse expectedResponse = DeleteObjectsResponse.builder().build(); + + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(CompletableFuture.completedFuture(expectedResponse)); + + // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value + StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class); + MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class); + when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher); + when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher); + + CompletableFuture future = S3AsyncDeleteHelper.executeDeleteBatches(s3AsyncClient, blobStore, batches); + + assertNotNull(future); + assertTrue(future.isDone()); + assertFalse(future.isCompletedExceptionally()); + future.join(); // Wait for the CompletableFuture to complete + + // Verify that the deleteObjects method of S3AsyncClient was called with the expected requests + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3AsyncClient, times(2)).deleteObjects(requestCaptor.capture()); + + List capturedRequests = requestCaptor.getAllValues(); + assertEquals(2, capturedRequests.size()); + for (DeleteObjectsRequest request : capturedRequests) { + assertNotNull(request.overrideConfiguration()); + assertTrue(request.overrideConfiguration().get().metricPublishers().contains(mockDeleteObjectsMetricPublisher)); + } + } + + public void testExecuteDeleteChainWithAfterDeleteAction() { + List objectsToDelete = Arrays.asList("key1", "key2", "key3"); + CompletableFuture currentChain = CompletableFuture.completedFuture(null); + Runnable afterDeleteAction = mock(Runnable.class); + + // Mock the deleteObjects method of S3AsyncClient + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn( + CompletableFuture.completedFuture(DeleteObjectsResponse.builder().build()) + ); + + // Mock the getBulkDeletesSize method of S3BlobStore + when(blobStore.getBulkDeletesSize()).thenReturn(2); + + // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value + StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class); + MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class); + when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher); + when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher); + + CompletableFuture newChain = S3AsyncDeleteHelper.executeDeleteChain( + s3AsyncClient, + blobStore, + objectsToDelete, + currentChain, + afterDeleteAction + ); + + // Verify that the newChain is completed without any exceptions + assertNotNull(newChain); + assertTrue(newChain.isDone()); + assertFalse(newChain.isCompletedExceptionally()); + + // Verify that the afterDeleteAction was called + verify(afterDeleteAction).run(); + } + +} diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java index d45b4e3deb798..875f11203281a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java @@ -1,3 +1,4 @@ + /* * SPDX-License-Identifier: Apache-2.0 * @@ -12,6 +13,7 @@ import org.opensearch.common.StreamContext; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.DeleteResult; import org.opensearch.common.blobstore.fs.FsBlobContainer; import org.opensearch.common.blobstore.fs.FsBlobStore; import org.opensearch.common.blobstore.stream.read.ReadContext; @@ -146,4 +148,14 @@ public boolean remoteIntegrityCheckSupported() { private boolean isSegmentFile(String filename) { return !filename.endsWith(".tlog") && !filename.endsWith(".ckp"); } + + @Override + public void deleteAsync(ActionListener completionListener) { + throw new UnsupportedOperationException("deleteAsync"); + } + + @Override + public void deleteBlobsAsyncIgnoringIfNotExists(List blobNames, ActionListener completionListener) { + throw new UnsupportedOperationException("deleteBlobsAsyncIgnoringIfNotExists"); + } } diff --git a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java index 97f304d776f5c..b769cdc2fe7ab 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java +++ b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java @@ -14,6 +14,7 @@ import org.opensearch.core.action.ActionListener; import java.io.IOException; +import java.util.List; /** * An extension of {@link BlobContainer} that adds {@link AsyncMultiStreamBlobContainer#asyncBlobUpload} to allow @@ -48,4 +49,8 @@ public interface AsyncMultiStreamBlobContainer extends BlobContainer { * by underlying blobContainer. In this case, caller doesn't need to ensure integrity of data. */ boolean remoteIntegrityCheckSupported(); + + void deleteAsync(ActionListener completionListener); + + void deleteBlobsAsyncIgnoringIfNotExists(List blobNames, ActionListener completionListener); } diff --git a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java index 82bc7a0baed50..286c01f9dca44 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java +++ b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java @@ -171,4 +171,14 @@ private InputStreamContainer decryptInputStreamContainer(InputStreamContainer in return new InputStreamContainer(decryptedStream, adjustedLength, adjustedPos); } } + + @Override + public void deleteAsync(ActionListener completionListener) { + blobContainer.deleteAsync(completionListener); + } + + @Override + public void deleteBlobsAsyncIgnoringIfNotExists(List blobNames, ActionListener completionListener) { + blobContainer.deleteBlobsAsyncIgnoringIfNotExists(blobNames, completionListener); + } } diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 09832e2b41b6d..ecdd23530c648 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -779,7 +779,10 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED, RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX, RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX, + + // Snapshot related Settings BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING, + BlobStoreRepository.SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING, SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING, diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index b954560c1bc94..0292cecc36a81 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -50,6 +50,7 @@ import org.opensearch.action.ActionRunnable; import org.opensearch.action.StepListener; import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateUpdateTask; import org.opensearch.cluster.RepositoryCleanupInProgress; @@ -69,6 +70,7 @@ import org.opensearch.common.Randomness; import org.opensearch.common.SetOnce; import org.opensearch.common.UUIDs; +import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; @@ -180,6 +182,7 @@ import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -353,6 +356,16 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp Setting.Property.Final ); + /** + * Controls the fixed prefix for the snapshot shard blob path. cluster.snapshot.async-deletion.enable + */ + public static final Setting SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING = Setting.boolSetting( + "cluster.snapshot.async-deletion.enable", + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + protected volatile boolean supportURLRepo; private volatile int maxShardBlobDeleteBatch; @@ -446,6 +459,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp private final String snapshotShardPathPrefix; + private volatile boolean enableAsyncDeletion; + /** * Flag that is set to {@code true} if this instance is started with {@link #metadata} that has a higher value for * {@link RepositoryMetadata#pendingGeneration()} than for {@link RepositoryMetadata#generation()} indicating a full cluster restart @@ -498,6 +513,8 @@ protected BlobStoreRepository( this.recoverySettings = recoverySettings; this.remoteStoreSettings = new RemoteStoreSettings(clusterService.getSettings(), clusterService.getClusterSettings()); this.snapshotShardPathPrefix = SNAPSHOT_SHARD_PATH_PREFIX_SETTING.get(clusterService.getSettings()); + this.enableAsyncDeletion = SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING.get(clusterService.getSettings()); + clusterService.getClusterSettings().addSettingsUpdateConsumer(SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING, this::setEnableAsyncDeletion); } @Override @@ -2082,7 +2099,7 @@ private void executeOneStaleIndexDelete( } // Finally, we delete the [base_path]/indexId folder - deleteResult = deleteResult.add(indexEntry.getValue().delete()); // Deleting the index folder + deleteResult = deleteResult.add(deleteContainer(indexEntry.getValue())); // Deleting the index folder logger.debug("[{}] Cleaned up stale index [{}]", metadata.name(), indexSnId); return deleteResult; } catch (IOException e) { @@ -2115,6 +2132,21 @@ private void executeOneStaleIndexDelete( })); } + private DeleteResult deleteContainer(BlobContainer container) throws IOException { + long startTime = System.nanoTime(); + DeleteResult deleteResult; + if (enableAsyncDeletion && container instanceof AsyncMultiStreamBlobContainer) { + // Use deleteAsync and wait for the result + PlainActionFuture future = new PlainActionFuture<>(); + ((AsyncMultiStreamBlobContainer) container).deleteAsync(future); + deleteResult = future.actionGet(); + } else { + deleteResult = container.delete(); + } + logger.debug(new ParameterizedMessage("[{}] Deleted {} in {}ns", metadata.name(), container.path(), startTime - System.nanoTime())); + return deleteResult; + } + /** * Cleans up the remote store directory if needed. *

This method cleans up segments in the remote store directory for deleted indices. @@ -2318,7 +2350,7 @@ void releaseRemoteStoreLocksAndCleanup( * @return A DeleteResult object representing the result of the deletion operation. * @throws IOException If an I/O error occurs during the deletion process. */ - private DeleteResult deleteShardData(ShardInfo shardInfo) throws IOException { + private DeleteResult deleteShardData(ShardInfo shardInfo) throws IOException, ExecutionException, InterruptedException { // If the provided ShardInfo is null, return a zero DeleteResult if (shardInfo == null) { return DeleteResult.ZERO; @@ -2330,7 +2362,7 @@ private DeleteResult deleteShardData(ShardInfo shardInfo) throws IOException { // Iterate over the shards and delete each shard's data for (int i = 0; i < shardInfo.getShardCount(); i++) { // Call the delete method on the shardContainer and accumulate the result - deleteResult = deleteResult.add(shardContainer(shardInfo.getIndexId(), i).delete()); + deleteResult = deleteResult.add(deleteContainer(shardContainer(shardInfo.getIndexId(), i))); } // Return the accumulated DeleteResult @@ -2714,7 +2746,23 @@ public IndexMetadata getSnapshotIndexMetaData(RepositoryData repositoryData, Sna private void deleteFromContainer(BlobContainer container, List blobs) throws IOException { logger.trace(() -> new ParameterizedMessage("[{}] Deleting {} from [{}]", metadata.name(), blobs, container.path())); - container.deleteBlobsIgnoringIfNotExists(blobs); + long startTime = System.nanoTime(); + if (enableAsyncDeletion && container instanceof AsyncMultiStreamBlobContainer) { + PlainActionFuture future = new PlainActionFuture<>(); + ((AsyncMultiStreamBlobContainer) container).deleteBlobsAsyncIgnoringIfNotExists(blobs, future); + future.actionGet(); + } else { + container.deleteBlobsIgnoringIfNotExists(blobs); + } + logger.debug( + () -> new ParameterizedMessage( + "[{}] Deletion {} from [{}] took {}ns", + metadata.name(), + blobs, + container.path(), + System.nanoTime() - startTime + ) + ); } private BlobPath indicesPath() { @@ -4565,4 +4613,8 @@ public String toString() { return name; } } + + public void setEnableAsyncDeletion(boolean enableAsyncDeletion) { + this.enableAsyncDeletion = enableAsyncDeletion; + } } diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java index 10e4cc6cfb1ef..fddc6c0c94005 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java @@ -17,6 +17,7 @@ import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.blobstore.DeleteResult; import org.opensearch.common.blobstore.fs.FsBlobContainer; import org.opensearch.common.blobstore.fs.FsBlobStore; import org.opensearch.common.blobstore.stream.read.ReadContext; @@ -51,6 +52,7 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Base64; +import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -340,5 +342,15 @@ public boolean remoteIntegrityCheckSupported() { public BlobContainer getDelegate() { return delegate; } + + @Override + public void deleteBlobsAsyncIgnoringIfNotExists(List blobNames, ActionListener completionListener) { + throw new RuntimeException("deleteBlobsAsyncIgnoringIfNotExists not supported"); + } + + @Override + public void deleteAsync(ActionListener completionListener) { + throw new RuntimeException("deleteAsync not supported"); + } } } diff --git a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java index 95a8267734a07..72798f7691eb1 100644 --- a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java +++ b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java @@ -39,6 +39,7 @@ import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.blobstore.DeleteResult; import org.opensearch.common.blobstore.fs.FsBlobContainer; import org.opensearch.common.blobstore.fs.FsBlobStore; import org.opensearch.common.blobstore.stream.read.ReadContext; @@ -63,6 +64,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; +import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; @@ -335,5 +337,15 @@ public boolean remoteIntegrityCheckSupported() { public BlobContainer getDelegate() { return delegate; } + + @Override + public void deleteAsync(ActionListener completionListener) { + throw new RuntimeException("deleteAsync not supported"); + } + + @Override + public void deleteBlobsAsyncIgnoringIfNotExists(List blobNames, ActionListener completionListener) { + throw new RuntimeException("deleteBlobsAsyncIgnoringIfNotExists not supported"); + } } }