Update Apache Lucene to 9.7.0-snapshot-204acc3 (opensearch-project#7887)

Signed-off-by: Andriy Redko <[email protected]>
sandeshkr419 · Jun 8, 2023 · 3fa9e5c · 3fa9e5c
1 parent 3d4062c
commit 3fa9e5c
Show file tree

Hide file tree

Showing 52 changed files with 55 additions and 31 deletions.
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.7.0-snapshot-4d1ed9e
+lucene            = 9.7.0-snapshot-204acc3
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 20.0.1+9

diff --git a/libs/core/build.gradle b/libs/core/build.gradle
@@ -103,13 +103,23 @@ tasks.named('forbiddenApisMain').configure {
 tasks.named("thirdPartyAudit").configure {
   ignoreMissingClasses(
     // from log4j
-    'org.osgi.framework.Bundle',
-    'org.osgi.framework.BundleActivator',
-    'org.osgi.framework.BundleContext',
-    'org.osgi.framework.BundleEvent',
-    'org.osgi.framework.SynchronousBundleListener',
-    'org.osgi.framework.wiring.BundleWire',
-    'org.osgi.framework.wiring.BundleWiring'
+    *[
+        'org.osgi.framework.Bundle',
+        'org.osgi.framework.BundleActivator',
+        'org.osgi.framework.BundleContext',
+        'org.osgi.framework.BundleEvent',
+        'org.osgi.framework.SynchronousBundleListener',
+        'org.osgi.framework.wiring.BundleWire',
+        'org.osgi.framework.wiring.BundleWiring',
+    ]  + (BuildParams.runtimeJavaVersion < JavaVersion.VERSION_20) ? [] : [
+       'jdk.incubator.vector.ByteVector',
+       'jdk.incubator.vector.FloatVector',
+       'jdk.incubator.vector.IntVector',
+       'jdk.incubator.vector.ShortVector',
+       'jdk.incubator.vector.Vector',
+       'jdk.incubator.vector.VectorOperators',
+       'jdk.incubator.vector.VectorSpecies'
+    ]
   )
 }
 

diff --git a/libs/core/licenses/lucene-core-9.7.0-snapshot-204acc3.jar.sha1 b/libs/core/licenses/lucene-core-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+3aa698cf90f074cbf24acfd7feaaad84c5a6f829
diff --git a/libs/core/licenses/lucene-core-9.7.0-snapshot-4d1ed9e.jar.sha1 b/libs/core/licenses/lucene-core-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.7.0-snapshot-204acc3.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+c31671978777d97c026b13decfbef1d7eeed4410
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.7.0-snapshot-4d1ed9e.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0-snapshot-204acc3.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+e31f7c161ad2b7652b1ac9100b4fa67d52be86eb
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0-snapshot-4d1ed9e.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0-snapshot-204acc3.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+ed13aaa5843c76ba8bb9fcd1cac6de8855ea7f1c
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0-snapshot-4d1ed9e.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0-snapshot-204acc3.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+f8673a2b86d3868e9ef2745d1d3fa9ad5a63e84f
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0-snapshot-4d1ed9e.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0-snapshot-204acc3.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+f7a585d5f62033f19a5ec79b66a74cc24c9e9d10
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0-snapshot-4d1ed9e.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0-snapshot-204acc3.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+40ba622c3aa91a0a8f2747617fef8f59bf212345
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0-snapshot-4d1ed9e.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0-snapshot-204acc3.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+e08432a3db0bde29f425812896bd695c5477b05f
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0-snapshot-4d1ed9e.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/...ns/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0-snapshot-204acc3.jar.sha1 b/...ns/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+a44008e83286060f0265a32f9a9b586e86ec03b1
diff --git a/...ns/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0-snapshot-4d1ed9e.jar.sha1 b/...ns/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/server/build.gradle b/server/build.gradle
@@ -244,6 +244,7 @@ tasks.named("processResources").configure {
 
 tasks.named("thirdPartyAudit").configure {
     ignoreMissingClasses(
+        *[
             // from com.fasterxml.jackson.dataformat.yaml.YAMLMapper (jackson-dataformat-yaml)
             'com.fasterxml.jackson.databind.ObjectMapper',
 
@@ -337,6 +338,15 @@ tasks.named("thirdPartyAudit").configure {
             'com.google.common.geometry.S2Point',
             'com.google.common.geometry.S2$Metric',
             'com.google.common.geometry.S2LatLng'
+        ] + (BuildParams.runtimeJavaVersion < JavaVersion.VERSION_20) ? [] : [
+            'jdk.incubator.vector.ByteVector',
+            'jdk.incubator.vector.FloatVector',
+            'jdk.incubator.vector.IntVector',
+            'jdk.incubator.vector.ShortVector',
+            'jdk.incubator.vector.Vector',
+            'jdk.incubator.vector.VectorOperators',
+            'jdk.incubator.vector.VectorSpecies'
+        ]
     )
     ignoreViolations(
             'com.google.protobuf.MessageSchema',

diff --git a/server/licenses/lucene-analysis-common-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-analysis-common-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+5aa5989b931c68eee90b22bca3f1f280a7d5c1ee
diff --git a/server/licenses/lucene-analysis-common-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-analysis-common-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/server/licenses/lucene-backward-codecs-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-backward-codecs-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+2326fb4762cd14eff6bea770d5c85a848e7a2482
diff --git a/server/licenses/lucene-backward-codecs-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-backward-codecs-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/server/licenses/lucene-core-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-core-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+3aa698cf90f074cbf24acfd7feaaad84c5a6f829
diff --git a/server/licenses/lucene-core-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-core-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/server/licenses/lucene-grouping-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-grouping-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+b3251c58b21c5c205c63bbe618c0d3c1393908e1
diff --git a/server/licenses/lucene-grouping-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-grouping-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/server/licenses/lucene-highlighter-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-highlighter-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+2313a0755a5774eb5dfcdc116dc15bd9367c8cc1
diff --git a/server/licenses/lucene-highlighter-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-highlighter-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/server/licenses/lucene-join-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-join-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+25fddcaf47d5614f48dc264f79151eb87990abd9
diff --git a/server/licenses/lucene-join-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-join-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/server/licenses/lucene-memory-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-memory-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+7c2aad4c1baa818a94bd787615e70824e6ae3a9d
diff --git a/server/licenses/lucene-memory-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-memory-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/server/licenses/lucene-misc-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-misc-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+56211acb2a31b1967568eba9953db502a53f9010
diff --git a/server/licenses/lucene-misc-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-misc-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/server/licenses/lucene-queries-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-queries-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+c45b1847150e2e2e6302177b41faf95df2ca4fbf
diff --git a/server/licenses/lucene-queries-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-queries-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/server/licenses/lucene-queryparser-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-queryparser-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+d40fe6400f21564ccb50b6716ec42d8dcd526b9d
diff --git a/server/licenses/lucene-queryparser-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-queryparser-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/server/licenses/lucene-sandbox-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-sandbox-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+e9347773d0b269768c58dd0b438bdc4b450f9185
diff --git a/server/licenses/lucene-sandbox-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-sandbox-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/server/licenses/lucene-spatial-extras-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-spatial-extras-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+7055502c9a77b9a93d1b96c99397fd765dd7891f
diff --git a/server/licenses/lucene-spatial-extras-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-spatial-extras-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/server/licenses/lucene-spatial3d-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-spatial3d-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+a2c5a7be5887dacf89c251936c3f3388ca20d28e
diff --git a/server/licenses/lucene-spatial3d-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-spatial3d-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/server/licenses/lucene-suggest-9.7.0-snapshot-204acc3.jar.sha1 b/server/licenses/lucene-suggest-9.7.0-snapshot-204acc3.jar.sha1
@@ -0,0 +1 @@
+1a076a8db84fa85634cc4894ee4fc78b5e69525c
diff --git a/server/licenses/lucene-suggest-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-suggest-9.7.0-snapshot-4d1ed9e.jar.sha1
diff --git a/server/src/main/java/org/opensearch/index/search/MatchQuery.java b/server/src/main/java/org/opensearch/index/search/MatchQuery.java
@@ -56,6 +56,7 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.QueryBuilder;
+import org.apache.lucene.util.TermAndBoost;
 import org.apache.lucene.util.graph.GraphTokenStreamFiniteStrings;
 import org.opensearch.OpenSearchException;
 import org.opensearch.common.io.stream.StreamInput;

diff --git a/server/src/main/java/org/opensearch/index/search/MultiMatchQuery.java b/server/src/main/java/org/opensearch/index/search/MultiMatchQuery.java
@@ -42,6 +42,7 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.TermAndBoost;
 import org.opensearch.common.lucene.search.Queries;
 import org.opensearch.index.mapper.MappedFieldType;
 import org.opensearch.index.query.AbstractQueryBuilder;

diff --git a/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java
@@ -222,6 +222,8 @@ public abstract class AnalysisFactoryTestCase extends OpenSearchTestCase {
         // LUCENE-10352
         .put("daitchmokotoffsoundex", Void.class)
         .put("persianstem", Void.class)
+        // https://github.com/apache/lucene/pull/12169
+        .put("word2vecsynonym", Void.class)
         .immutableMap();
 
     static final Map<String, Class<?>> KNOWN_CHARFILTERS = new MapBuilder<String, Class<?>>()