opensearch-project · reta · Jun 12, 2024 · Jun 7, 2024
@@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Added
 
 ### Dependencies
+- Update to Apache Lucene 9.11.0 ([#14042](https://github.com/opensearch-project/OpenSearch/pull/14042))
 
 ### Changed
 

@@ -1,5 +1,5 @@
 opensearch        = 2.16.0
-lucene            = 9.10.0
+lucene            = 9.11.0
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.3+9

@@ -0,0 +1 @@
+2e487755a6814b2a1bc770c26569dcba86873dcf
@@ -133,7 +133,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_14_0 = new Version(2140099, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_14_1 = new Version(2140199, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_15_0 = new Version(2150099, org.apache.lucene.util.Version.LUCENE_9_10_0);
-    public static final Version V_2_16_0 = new Version(2160099, org.apache.lucene.util.Version.LUCENE_9_10_0);
+    public static final Version V_2_16_0 = new Version(2160099, org.apache.lucene.util.Version.LUCENE_9_11_0);
     public static final Version CURRENT = V_2_16_0;
 
     public static Version fromId(int id) {

@@ -0,0 +1 @@
+5e21d20edee0712472e7c6f605c9d97aeecf16c0
@@ -0,0 +1 @@
+5c7f2d8eab0fca3fdc3d3e57a7f48a335dc7ac33
@@ -0,0 +1 @@
+efcf65dda1b4e9d7e83926fd5895a47e491cbf29
@@ -0,0 +1 @@
+59599d7b8bed2e6bd27d0dad7935c078b98c39cc
@@ -0,0 +1 @@
+e55f83bb373ac139e313f64e80afe1eb0a75b8c0
@@ -0,0 +1 @@
+1be59d91c45a4de069611fb7f8aa3e8fd26020ec
@@ -0,0 +1 @@
+d5b5922acf3743b5a0c542959dd93fca8be333a7
@@ -0,0 +1 @@
+50fd7b471cbdd6648c4972169f3fc67fae9db7f6
@@ -0,0 +1 @@
+75a0a333cf1e043102743066c929e65fe51cbcda
@@ -0,0 +1 @@
+db385446bc3fd70e7c6a744276c0a157bd60ee0a
@@ -0,0 +1 @@
+2e487755a6814b2a1bc770c26569dcba86873dcf
@@ -0,0 +1 @@
+882bdaf209b0acb332aa34836616424bcbecf462
@@ -0,0 +1 @@
+44accdc03c5482e602718f7bf91e5940ba4e4870
@@ -0,0 +1 @@
+32a30ee03ed4f3e43bf63250270b2d4d53050045
@@ -0,0 +1 @@
+b3e80aa6aa3299118e76a23edc23b58f3ba5a515
@@ -0,0 +1 @@
+54fe308908194e1b0697a1157a45c5998c9e1083
@@ -0,0 +1 @@
+987d1286949ddf514b8405fd453ed47bebdfb12d
@@ -0,0 +1 @@
+e97fe1c0d102edb8d6e1c01454992fd2b8d80ae0
@@ -0,0 +1 @@
+5e46b790744bd9118ccc053f70235364213312a5
@@ -0,0 +1 @@
+079ca5aaf544a3acde84b8b88423ace6dedc23eb
@@ -0,0 +1 @@
+564558818d70fc384db5b36fbc8a0ab27b107609
@@ -0,0 +1 @@
+aa345db9b6caaf881e7890ea5b8911357d592167
@@ -51,6 +51,8 @@
 class CustomFieldHighlighter extends FieldHighlighter {
     private static final Passage[] EMPTY_PASSAGE = new Passage[0];
 
+    private static final Comparator<Passage> DEFAULT_PASSAGE_SORT_COMPARATOR = Comparator.comparingInt(Passage::getStartOffset);
+
     private final Locale breakIteratorLocale;
     private final int noMatchSize;
     private String fieldValue;
@@ -66,7 +68,16 @@ class CustomFieldHighlighter extends FieldHighlighter {
         PassageFormatter passageFormatter,
         int noMatchSize
     ) {
-        super(field, fieldOffsetStrategy, breakIterator, passageScorer, maxPassages, maxNoHighlightPassages, passageFormatter);
+        super(
+            field,
+            fieldOffsetStrategy,
+            breakIterator,
+            passageScorer,
+            maxPassages,
+            maxNoHighlightPassages,
+            passageFormatter,
+            DEFAULT_PASSAGE_SORT_COMPARATOR
+        );
         this.breakIteratorLocale = breakIteratorLocale;
         this.noMatchSize = noMatchSize;
     }

@@ -36,6 +36,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.util.SPIClassIterator;
 import org.opensearch.Build;
@@ -762,6 +763,7 @@ static void reloadLuceneSPI(ClassLoader loader) {
         // Codecs:
         PostingsFormat.reloadPostingsFormats(loader);
         DocValuesFormat.reloadDocValuesFormats(loader);
+        KnnVectorsFormat.reloadKnnVectorsFormat(loader);
         Codec.reloadCodecs(loader);
     }
 

@@ -304,7 +304,7 @@ private static BoundaryScanner getBoundaryScanner(Field field) {
                 return DEFAULT_WORD_BOUNDARY_SCANNER;
             case CHARS:
                 if (fieldOptions.boundaryMaxScan() != SimpleBoundaryScanner.DEFAULT_MAX_SCAN
-                    || fieldOptions.boundaryChars() != SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS) {
+                    || fieldOptions.boundaryChars() != HighlightBuilder.DEFAULT_BOUNDARY_CHARS) {
                     return new SimpleBoundaryScanner(fieldOptions.boundaryMaxScan(), fieldOptions.boundaryChars());
                 }
                 return DEFAULT_SIMPLE_BOUNDARY_SCANNER;

@@ -111,6 +111,8 @@ public class HighlightBuilder extends AbstractHighlighterBuilder<HighlightBuilde
     /** the default closing tags when {@code tag_schema = "styled"}  */
     public static final String[] DEFAULT_STYLED_POST_TAGS = { "</em>" };
 
+    static final Character[] DEFAULT_BOUNDARY_CHARS = HighlightBuilder.convertCharArray(SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS);
+
     /**
      * a {@link FieldOptions} with default settings
      */
@@ -124,7 +126,7 @@ public class HighlightBuilder extends AbstractHighlighterBuilder<HighlightBuilde
         .numberOfFragments(DEFAULT_NUMBER_OF_FRAGMENTS)
         .encoder(DEFAULT_ENCODER)
         .boundaryMaxScan(SimpleBoundaryScanner.DEFAULT_MAX_SCAN)
-        .boundaryChars(SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS)
+        .boundaryChars(DEFAULT_BOUNDARY_CHARS)
         .boundaryScannerLocale(Locale.ROOT)
         .noMatchSize(DEFAULT_NO_MATCH_SIZE)
         .phraseLimit(DEFAULT_PHRASE_LIMIT)

@@ -222,6 +222,9 @@ public abstract class AnalysisFactoryTestCase extends OpenSearchTestCase {
         .put("persianstem", Void.class)
         // https://github.com/apache/lucene/pull/12169
         .put("word2vecsynonym", Void.class)
+        // https://github.com/apache/lucene/pull/12915
+        .put("japanesehiraganauppercase", Void.class)
+        .put("japanesekatakanauppercase", Void.class)
         .immutableMap();
 
     static final Map<String, Class<?>> KNOWN_CHARFILTERS = new MapBuilder<String, Class<?>>()