Update lucene version to 8.8.2 (opensearch-project#557)

This commit updates the codebase to the latest released version of Lucene. Signed-off-by: Nicholas Walter Knize <[email protected]>
bharath-techie · Apr 23, 2021 · c5a3c3c · c5a3c3c
1 parent 821c44c
commit c5a3c3c
Show file tree

Hide file tree

Showing 51 changed files with 69 additions and 33 deletions.
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 opensearch        = 1.0.0
-lucene            = 8.7.0
+lucene            = 8.8.2
 
 bundled_jdk_vendor = adoptopenjdk
 bundled_jdk = 15.0.1+9

diff --git a/modules/lang-expression/licenses/lucene-expressions-8.7.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.7.0.jar.sha1
diff --git a/modules/lang-expression/licenses/lucene-expressions-8.8.2.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.8.2.jar.sha1
@@ -0,0 +1 @@
+85c06b0d5b966e6d8a75dd568a6eb1a7c3b34bdd
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.7.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.7.0.jar.sha1
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.8.2.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.8.2.jar.sha1
@@ -0,0 +1 @@
+dcda3ccd38ca68e23ec57c8ca16d13553066672b
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.7.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.7.0.jar.sha1
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.8.2.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.8.2.jar.sha1
@@ -0,0 +1 @@
+829b06a76ffaf0e551c5cda4a1e56990080ea6de
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.7.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.7.0.jar.sha1
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.8.2.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.8.2.jar.sha1
@@ -0,0 +1 @@
+2c23e33c49fb255d96e0bf42ca7c58f030011727
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.7.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.7.0.jar.sha1
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.8.2.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.8.2.jar.sha1
@@ -0,0 +1 @@
+b0ccf32c2baa88815b89c157f19b264a24958180
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.7.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.7.0.jar.sha1
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.8.2.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.8.2.jar.sha1
@@ -0,0 +1 @@
+d7fb0c151dccc1954547acd3289cdf1369cda969
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.7.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.7.0.jar.sha1
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.8.2.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.8.2.jar.sha1
@@ -0,0 +1 @@
+ca1b3379c99553c64d6b76271a5701f2bc2a2ce4
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.7.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.7.0.jar.sha1
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.8.2.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.8.2.jar.sha1
@@ -0,0 +1 @@
+2070b7dd21682b4b9bd23f77146a73d0e3f3b05f
diff --git a/server/licenses/lucene-analyzers-common-8.7.0.jar.sha1 b/server/licenses/lucene-analyzers-common-8.7.0.jar.sha1
diff --git a/server/licenses/lucene-analyzers-common-8.8.2.jar.sha1 b/server/licenses/lucene-analyzers-common-8.8.2.jar.sha1
@@ -0,0 +1 @@
+fa4b3c1d847f4bb91a3afaa990889dd164883d86
diff --git a/server/licenses/lucene-backward-codecs-8.7.0.jar.sha1 b/server/licenses/lucene-backward-codecs-8.7.0.jar.sha1
diff --git a/server/licenses/lucene-backward-codecs-8.8.2.jar.sha1 b/server/licenses/lucene-backward-codecs-8.8.2.jar.sha1
@@ -0,0 +1 @@
+f4a0475a1404d7be87e911f5c85f8b913ec9770b
diff --git a/server/licenses/lucene-core-8.7.0.jar.sha1 b/server/licenses/lucene-core-8.7.0.jar.sha1
diff --git a/server/licenses/lucene-core-8.8.2.jar.sha1 b/server/licenses/lucene-core-8.8.2.jar.sha1
@@ -0,0 +1 @@
+e039fa90ee900c475bbc0943989ed5dccc556d3e
diff --git a/server/licenses/lucene-grouping-8.7.0.jar.sha1 b/server/licenses/lucene-grouping-8.7.0.jar.sha1
diff --git a/server/licenses/lucene-grouping-8.8.2.jar.sha1 b/server/licenses/lucene-grouping-8.8.2.jar.sha1
@@ -0,0 +1 @@
+207ac15287656b6ce61ea921fefec3070a8dd99d
diff --git a/server/licenses/lucene-highlighter-8.7.0.jar.sha1 b/server/licenses/lucene-highlighter-8.7.0.jar.sha1
diff --git a/server/licenses/lucene-highlighter-8.8.2.jar.sha1 b/server/licenses/lucene-highlighter-8.8.2.jar.sha1
@@ -0,0 +1 @@
+2674644580108335c08a25fd2e0f0c7dee010700
diff --git a/server/licenses/lucene-join-8.7.0.jar.sha1 b/server/licenses/lucene-join-8.7.0.jar.sha1
diff --git a/server/licenses/lucene-join-8.8.2.jar.sha1 b/server/licenses/lucene-join-8.8.2.jar.sha1
@@ -0,0 +1 @@
+6eb956741169fe8a5643fa39b14aef2fc067d865
diff --git a/server/licenses/lucene-memory-8.7.0.jar.sha1 b/server/licenses/lucene-memory-8.7.0.jar.sha1
diff --git a/server/licenses/lucene-memory-8.8.2.jar.sha1 b/server/licenses/lucene-memory-8.8.2.jar.sha1
@@ -0,0 +1 @@
+63b6e8e862c898d263b941ffa55fe6c85bcdf2df
diff --git a/server/licenses/lucene-misc-8.7.0.jar.sha1 b/server/licenses/lucene-misc-8.7.0.jar.sha1
diff --git a/server/licenses/lucene-misc-8.8.2.jar.sha1 b/server/licenses/lucene-misc-8.8.2.jar.sha1
@@ -0,0 +1 @@
+a3a916c27c42ff6773c52d0be4045d4e5c00f7f8
diff --git a/server/licenses/lucene-queries-8.7.0.jar.sha1 b/server/licenses/lucene-queries-8.7.0.jar.sha1
diff --git a/server/licenses/lucene-queries-8.8.2.jar.sha1 b/server/licenses/lucene-queries-8.8.2.jar.sha1
@@ -0,0 +1 @@
+1f324c3a6e1a7634bbada993b2c014d12d09cb83
diff --git a/server/licenses/lucene-queryparser-8.7.0.jar.sha1 b/server/licenses/lucene-queryparser-8.7.0.jar.sha1
diff --git a/server/licenses/lucene-queryparser-8.8.2.jar.sha1 b/server/licenses/lucene-queryparser-8.8.2.jar.sha1
@@ -0,0 +1 @@
+67c6580f8a5187fdb7f9eb2404bdecc4635b3328
diff --git a/server/licenses/lucene-sandbox-8.7.0.jar.sha1 b/server/licenses/lucene-sandbox-8.7.0.jar.sha1
diff --git a/server/licenses/lucene-sandbox-8.8.2.jar.sha1 b/server/licenses/lucene-sandbox-8.8.2.jar.sha1
@@ -0,0 +1 @@
+b674deb4957a64db5c8f4afb9127400f4e65c334
diff --git a/server/licenses/lucene-spatial-extras-8.7.0.jar.sha1 b/server/licenses/lucene-spatial-extras-8.7.0.jar.sha1
diff --git a/server/licenses/lucene-spatial-extras-8.8.2.jar.sha1 b/server/licenses/lucene-spatial-extras-8.8.2.jar.sha1
@@ -0,0 +1 @@
+227672b3b88c877b2df27608850ca15d834ec35c
diff --git a/server/licenses/lucene-spatial3d-8.7.0.jar.sha1 b/server/licenses/lucene-spatial3d-8.7.0.jar.sha1
diff --git a/server/licenses/lucene-spatial3d-8.8.2.jar.sha1 b/server/licenses/lucene-spatial3d-8.8.2.jar.sha1
@@ -0,0 +1 @@
+eb77b665a3b3e9ca6b3bb82c3342b843d9729d0d
diff --git a/server/licenses/lucene-suggest-8.7.0.jar.sha1 b/server/licenses/lucene-suggest-8.7.0.jar.sha1
diff --git a/server/licenses/lucene-suggest-8.8.2.jar.sha1 b/server/licenses/lucene-suggest-8.8.2.jar.sha1
@@ -0,0 +1 @@
+36fd5301e7143f0fcb0ba933f96148dc1e132b66
diff --git a/server/src/main/java/org/opensearch/Version.java b/server/src/main/java/org/opensearch/Version.java
@@ -70,7 +70,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final int V_EMPTY_ID = 0;
     public static final Version V_EMPTY = new Version(V_EMPTY_ID, org.apache.lucene.util.Version.LATEST);
 
-    public static final Version V_1_0_0 = new Version(1000099, org.apache.lucene.util.Version.LUCENE_8_7_0);
+    public static final Version V_1_0_0 = new Version(1000099, org.apache.lucene.util.Version.LUCENE_8_8_2);
     public static final Version CURRENT = V_1_0_0;
 
     public static Version readVersion(StreamInput in) throws IOException {

diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java
@@ -35,7 +35,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.lucene87.Lucene87Codec;
-import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat.Mode;
+import org.apache.lucene.codecs.lucene87.Lucene87Codec.Mode;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.collect.MapBuilder;
 import org.opensearch.index.mapper.MapperService;

diff --git a/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java b/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java
@@ -34,8 +34,9 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat;
+import org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat;
 import org.apache.lucene.codecs.lucene87.Lucene87Codec;
 import org.opensearch.common.lucene.Lucene;
 import org.opensearch.index.mapper.CompletionFieldMapper;
@@ -53,13 +54,14 @@
 public class PerFieldMappingPostingFormatCodec extends Lucene87Codec {
     private final Logger logger;
     private final MapperService mapperService;
+    private final DocValuesFormat dvFormat = new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION);
 
     static {
         assert Codec.forName(Lucene.LATEST_CODEC).getClass().isAssignableFrom(PerFieldMappingPostingFormatCodec.class) :
             "PerFieldMappingPostingFormatCodec must subclass the latest " + "lucene codec: " + Lucene.LATEST_CODEC;
     }
 
-    public PerFieldMappingPostingFormatCodec(Lucene87StoredFieldsFormat.Mode compressionMode, MapperService mapperService, Logger logger) {
+    public PerFieldMappingPostingFormatCodec(Mode compressionMode, MapperService mapperService, Logger logger) {
         super(compressionMode);
         this.mapperService = mapperService;
         this.logger = logger;
@@ -76,4 +78,8 @@ public PostingsFormat getPostingsFormatForField(String field) {
         return super.getPostingsFormatForField(field);
     }
 
+    @Override
+    public DocValuesFormat getDocValuesFormatForField(String field) {
+        return dvFormat;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/query/VectorGeoShapeQueryProcessor.java b/server/src/main/java/org/opensearch/index/query/VectorGeoShapeQueryProcessor.java
@@ -33,6 +33,7 @@
 package org.opensearch.index.query;
 
 import org.apache.lucene.document.LatLonShape;
+import org.apache.lucene.geo.GeoEncodingUtils;
 import org.apache.lucene.geo.LatLonGeometry;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
@@ -155,7 +156,12 @@ public Void visit(MultiPolygon multiPolygon) {
         @Override
         public Void visit(Point point) {
             if (point.isEmpty() == false) {
-                geometries.add(GeoShapeUtils.toLucenePoint(point));
+                // points are a special "shape" case: for queries we need to quantize since the Lucene
+                // tessellator doesn't do anything with them.
+                // todo this is a sandy lucene experience so we should investigate fixing this upstream
+                double quantizedLat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(point.getLat()));
+                double quantizedLon = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(point.getLon()));
+                geometries.add(new org.apache.lucene.geo.Point(quantizedLat, quantizedLon));
             }
             return null;
 

diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java
@@ -34,15 +34,17 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat;
 import org.apache.lucene.codecs.lucene87.Lucene87Codec;
 import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat;
-import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat.Mode;
+import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.SegmentReader;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.env.Environment;
@@ -72,16 +74,37 @@ public void testResolveDefaultCodecs() throws Exception {
 
     public void testDefault() throws Exception {
         Codec codec = createCodecService().codec("default");
-        assertCompressionEquals(Mode.BEST_SPEED, codec);
+        assertDVCompressionEquals(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION, codec);
+        assertStoredFieldsFormatCompressionEquals(Lucene87StoredFieldsFormat.Mode.BEST_SPEED, codec);
     }
 
     public void testBestCompression() throws Exception {
         Codec codec = createCodecService().codec("best_compression");
-        assertCompressionEquals(Mode.BEST_COMPRESSION, codec);
+        assertDVCompressionEquals(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION, codec);
+        assertStoredFieldsFormatCompressionEquals(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION, codec);
+    }
+
+    private void assertDVCompressionEquals(Lucene80DocValuesFormat.Mode expected, Codec actual) throws Exception {
+        Directory dir = newDirectory();
+        IndexWriterConfig iwc = newIndexWriterConfig(null);
+        iwc.setCodec(actual);
+        IndexWriter iw = new IndexWriter(dir, iwc);
+        Document doc = new Document();
+        doc.add(new BinaryDocValuesField("foo", new BytesRef("aaa")));
+        iw.addDocument(doc);
+        iw.commit();
+        iw.close();
+        DirectoryReader ir = DirectoryReader.open(dir);
+        SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
+        String v = sr.getFieldInfos().fieldInfo("foo").getAttribute(Lucene80DocValuesFormat.MODE_KEY);
+        assertNotNull(v);
+        assertEquals(expected, Lucene80DocValuesFormat.Mode.valueOf(v));
+        ir.close();
+        dir.close();
     }
 
     // write some docs with it, inspect .si to see this was the used compression
-    private void assertCompressionEquals(Mode expected, Codec actual) throws Exception {
+    private void assertStoredFieldsFormatCompressionEquals(Lucene87StoredFieldsFormat.Mode expected, Codec actual) throws Exception {
         Directory dir = newDirectory();
         IndexWriterConfig iwc = newIndexWriterConfig(null);
         iwc.setCodec(actual);
@@ -93,7 +116,7 @@ private void assertCompressionEquals(Mode expected, Codec actual) throws Excepti
         SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
         String v = sr.getSegmentInfo().info.getAttribute(Lucene87StoredFieldsFormat.MODE_KEY);
         assertNotNull(v);
-        assertEquals(expected, Mode.valueOf(v));
+        assertEquals(expected, Lucene87StoredFieldsFormat.Mode.valueOf(v));
         ir.close();
         dir.close();
     }

diff --git a/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java
@@ -219,6 +219,7 @@ public abstract class AnalysisFactoryTestCase extends OpenSearchTestCase {
         // TODO: these charfilters are not yet exposed: useful?
         // handling of zwnj for persian
         .put("persian",        Void.class)
+        .put("cjkwidth",       Void.class)
         .immutableMap();
 
     /**