Remove normalizer support from wildcard field while we decide on appr…

…oach for handling case insensitvity (elastic#55294) Closes elastic#55288
markharwood · Apr 17, 2020 · 94f0dd7 · 94f0dd7
1 parent 389b649
commit 94f0dd7
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 123 deletions.
diff --git a/docs/reference/mapping/types/wildcard.asciidoc b/docs/reference/mapping/types/wildcard.asciidoc
@@ -62,11 +62,6 @@ The following parameters are accepted by `wildcard` fields:
     Do not index any string longer than this value.  Defaults to `2147483647`
     so that all values would be accepted.
 
-<<normalizer,`normalizer`>>::
-
-    How to pre-process the value prior to indexing. Defaults to `null`,
-    meaning the value is kept as-is.
-
 ==== Limitations
 
 * `wildcard` fields are untokenized like keyword fields, so do not support queries that rely on word positions such as phrase queries.

diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml
@@ -1,29 +1,19 @@
 setup:
   - skip:
       features: headers
-      version: " - 7.6.99"
-      reason: "wildcard fields were added from 7.7"
+      version: " - 7.7.99"
+      reason: "wildcard fields were added from 7.8"
 
   - do:
       indices.create:
         index: test-index
         body:
           settings:
             number_of_replicas: 0
-            analysis:
-              normalizer:
-                lowercase:
-                  type: custom
-                  char_filter: []
-                  filter: ["lowercase"]
           mappings:
             properties:
               my_wildcard:
                  type: wildcard
-                 normalizer: lowercase
-                 fields:
-                   case_sensitive:
-                     type: wildcard
   - do:
       index:
         index: test-index
@@ -97,18 +87,6 @@ setup:
 
 
   - match: {hits.total.value: 1}
----
-"Case insensitive query":
-  - do:
-      search:
-        body:
-          track_total_hits: true
-          query:
-            wildcard:
-              my_wildcard: {value: "*Worl*" }
-
-
-  - match: {hits.total.value: 3}
 
 ---
 "Case sensitive query":
@@ -118,7 +96,7 @@ setup:
           track_total_hits: true
           query:
             wildcard:
-              my_wildcard.case_sensitive: {value: "*Worl*" }
+              my_wildcard: {value: "*Worl*" }
 
 
   - match: {hits.total.value: 1}
@@ -229,8 +207,8 @@ setup:
               terms: {field: "my_wildcard" }
 
 
-  - match: {hits.total.value: 3}
-  - length: { aggregations.top_vals.buckets: 3 }
+  - match: {hits.total.value: 2}
+  - length: { aggregations.top_vals.buckets: 2 }
 
 ---
 "Sort works":

diff --git a/...n/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/...n/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
@@ -39,7 +39,6 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalyzerScope;
-import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
@@ -54,7 +53,6 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParseContext.Document;
-import org.elasticsearch.index.mapper.StringFieldType;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.similarity.SimilarityProvider;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
@@ -68,7 +66,6 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 
 import static org.elasticsearch.index.mapper.TypeParsers.parseField;
 
@@ -105,8 +102,6 @@ public static class Defaults {
 
     public static class Builder extends FieldMapper.Builder<Builder, WildcardFieldMapper> {
         protected int ignoreAbove = Defaults.IGNORE_ABOVE;
-        private IndexAnalyzers indexAnalyzers;
-        private String normalizerName;
 
 
         public Builder(String name) {
@@ -173,22 +168,9 @@ public WildcardFieldType fieldType() {
             return (WildcardFieldType) super.fieldType();
         }
 
-        public Builder normalizer(IndexAnalyzers indexAnalyzers, String name) {
-            this.indexAnalyzers = indexAnalyzers;
-            this.normalizerName = name;
-            return builder;
-        }
-
         @Override
         public WildcardFieldMapper build(BuilderContext context) {
             setupFieldType(context);
-            if (normalizerName != null) {
-                NamedAnalyzer normalizer = indexAnalyzers.getNormalizer(normalizerName);
-                if (normalizer == null) {
-                    throw new MapperParsingException("normalizer [" + normalizerName + "] not found for field [" + name + "]");
-                }
-                fieldType().setNormalizer(normalizer);
-            }
             return new WildcardFieldMapper(
                     name, fieldType, defaultFieldType, ignoreAbove,
                     context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
@@ -209,11 +191,6 @@ public static class TypeParser implements Mapper.TypeParser {
                 if (propName.equals("ignore_above")) {
                     builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
                     iterator.remove();
-                } else if (propName.equals("normalizer")) {
-                    if (propNode != null) {
-                        builder.normalizer(parserContext.getIndexAnalyzers(), propNode.toString());
-                    }
-                    iterator.remove();
                 }
             }
 
@@ -225,56 +202,20 @@ public static class TypeParser implements Mapper.TypeParser {
 
      public static final class WildcardFieldType extends MappedFieldType {
 
-        private NamedAnalyzer normalizer = null;
-
         public WildcardFieldType() {
             setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
             setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
         }
 
         protected WildcardFieldType(WildcardFieldType ref) {
             super(ref);
-            this.normalizer = ref.normalizer;
         }
 
         public WildcardFieldType clone() {
             WildcardFieldType result = new WildcardFieldType(this);
             return result;
         }
 
-
-        @Override
-        public boolean equals(Object o) {
-            if (super.equals(o) == false) {
-                return false;
-            }
-            WildcardFieldType other = (WildcardFieldType) o;
-            return Objects.equals(normalizer, other.normalizer);
-        }
-
-        @Override
-        public int hashCode() {
-            return 31 * super.hashCode() + Objects.hash(normalizer);
-        }
-
-        private NamedAnalyzer normalizer() {
-            return normalizer;
-        }
-
-        public void setNormalizer(NamedAnalyzer normalizer) {
-            checkIfFrozen();
-            this.normalizer = normalizer;
-        }
-
-        @Override
-        public void checkCompatibility(MappedFieldType otherFT, List<String> conflicts) {
-            super.checkCompatibility(otherFT, conflicts);
-            WildcardFieldType other = (WildcardFieldType) otherFT;
-            if (Objects.equals(normalizer, other.normalizer) == false) {
-                conflicts.add("mapper [" + name() + "] has different [normalizer]");
-            }
-        }
-
         // Holds parsed information about the wildcard pattern
         static class PatternStructure {
             boolean openStart, openEnd, hasSymbols;
@@ -388,9 +329,6 @@ public boolean equals(Object obj) {
 
         @Override
         public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QueryShardContext context) {
-            if (normalizer != null) {
-                wildcardPattern = StringFieldType.normalizeWildcardPattern(name(), wildcardPattern, normalizer);
-            }
             PatternStructure patternStructure = new PatternStructure(wildcardPattern);
             ArrayList<String> tokens = new ArrayList<>();
 
@@ -538,29 +476,6 @@ public ValuesSourceType getValuesSourceType() {
              return CoreValuesSourceType.BYTES;
          }
 
-         String normalize(String value) throws IOException {
-            if (normalizer != null) {
-                try (TokenStream ts = normalizer.tokenStream(name(), value)) {
-                    final CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
-                    ts.reset();
-                    if (ts.incrementToken() == false) {
-                      throw new IllegalStateException("The normalization token stream is "
-                          + "expected to produce exactly 1 token, but got 0 for analyzer "
-                          + normalizer + " and input \"" + value + "\"");
-                    }
-                    final String newValue = termAtt.toString();
-                    if (ts.incrementToken()) {
-                      throw new IllegalStateException("The normalization token stream is "
-                          + "expected to produce exactly 1 token, but got 2+ for analyzer "
-                          + normalizer + " and input \"" + value + "\"");
-                    }
-                    ts.end();
-                    return newValue;
-                }
-            }
-            return value;
-        }
-
     }
 
     static class  WildcardBytesBinaryDVIndexFieldData extends BytesBinaryDVIndexFieldData{
@@ -614,11 +529,6 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
         if (includeDefaults || ignoreAbove != Defaults.IGNORE_ABOVE) {
             builder.field("ignore_above", ignoreAbove);
         }
-        if (fieldType().normalizer() != null) {
-            builder.field("normalizer", fieldType().normalizer().name());
-        } else if (includeDefaults) {
-            builder.nullField("normalizer");
-        }
     }
 
     @Override
@@ -646,7 +556,6 @@ void createFields(String value, Document parseDoc, List<IndexableField>fields) t
         if (value == null || value.length() > ignoreAbove) {
             return;
         }
-        value = fieldType().normalize(value);
         String ngramValue = TOKEN_START_OR_END_CHAR + value + TOKEN_START_OR_END_CHAR + TOKEN_START_OR_END_CHAR;
         Field ngramField = new Field(fieldType().name(), ngramValue, ngramFieldType);
         fields.add(ngramField);