From bcd70163031fe5681a6ffba4be62f5c91a350f45 Mon Sep 17 00:00:00 2001 From: Hauck Date: Tue, 13 Sep 2022 11:15:28 -0300 Subject: [PATCH 1/3] Solves #4002, ignoring all malformed objects when ignore_malformed is set to true; Signed-off-by: Hauck --- .../opensearch/index/mapper/FieldMapper.java | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java index 137ca4be1ca87..3088bd0a4c94e 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java @@ -34,6 +34,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; + import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; @@ -265,6 +266,7 @@ public boolean parsesArrayValue() { * Parse the field value using the provided {@link ParseContext}. */ public void parse(ParseContext context) throws IOException { + boolean ignore_malformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings()); try { parseCreateField(context); } catch (Exception e) { @@ -278,23 +280,27 @@ public void parse(ParseContext context) throws IOException { valuePreview = complexValue.toString(); } } catch (Exception innerException) { + if (!ignore_malformed) { + throw new MapperParsingException( + "failed to parse field [{}] of type [{}] in document with id '{}'. " + "Could not parse field value preview,", + e, + fieldType().name(), + fieldType().typeName(), + context.sourceToParse().id() + ); + } + } + + if (!ignore_malformed) { throw new MapperParsingException( - "failed to parse field [{}] of type [{}] in document with id '{}'. " + "Could not parse field value preview,", + "failed to parse field [{}] of type [{}] in document with id '{}'. " + "Preview of field's value: '{}'", e, fieldType().name(), fieldType().typeName(), - context.sourceToParse().id() + context.sourceToParse().id(), + valuePreview ); } - - throw new MapperParsingException( - "failed to parse field [{}] of type [{}] in document with id '{}'. " + "Preview of field's value: '{}'", - e, - fieldType().name(), - fieldType().typeName(), - context.sourceToParse().id(), - valuePreview - ); } multiFields.parse(this, context); } From 0d2f3fd3fe9215bf4aea05dfe6d703504a9a318a Mon Sep 17 00:00:00 2001 From: Hauck Date: Tue, 13 Sep 2022 11:57:05 -0300 Subject: [PATCH 2/3] Fix null pointers by checking the IGNORE_MALFORMED_SETTING; Update the CHANGELOG.md; Signed-off-by: Hauck --- CHANGELOG.md | 1 + .../src/main/java/org/opensearch/index/mapper/FieldMapper.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cbfc56ed776c..be111a8dec3ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - [Bug]: gradle check failing with java heap OutOfMemoryError (([#4328](https://github.com/opensearch-project/OpenSearch/ - `opensearch.bat` fails to execute when install path includes spaces ([#4362](https://github.com/opensearch-project/OpenSearch/pull/4362)) - Fixed flaky test `ResourceAwareTasksTests.testTaskIdPersistsInThreadContext` ([#4484](https://github.com/opensearch-project/OpenSearch/pull/4484)) +- Fixed the ignore_malformed setting to also ignore objects ([#4494](https://github.com/opensearch-project/OpenSearch/pull/4494)) ### Security - CVE-2022-25857 org.yaml:snakeyaml DOS vulnerability ([#4341](https://github.com/opensearch-project/OpenSearch/pull/4341)) diff --git a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java index 3088bd0a4c94e..ca118d479eab1 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java @@ -266,10 +266,11 @@ public boolean parsesArrayValue() { * Parse the field value using the provided {@link ParseContext}. */ public void parse(ParseContext context) throws IOException { - boolean ignore_malformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings()); try { parseCreateField(context); } catch (Exception e) { + boolean ignore_malformed = false; + if (context.indexSettings() != null) ignore_malformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings()); String valuePreview = ""; try { XContentParser parser = context.parser(); From 884688c6c3b7f7a97e1a6e93860ec212279f2b61 Mon Sep 17 00:00:00 2001 From: Hauck Date: Wed, 14 Sep 2022 13:22:05 -0300 Subject: [PATCH 3/3] As @nknize pointed contributors must use explicit logic instead of negations; Signed-off-by: Hauck --- .../main/java/org/opensearch/index/mapper/FieldMapper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java index ca118d479eab1..3acf5d4ea85ee 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java @@ -281,7 +281,7 @@ public void parse(ParseContext context) throws IOException { valuePreview = complexValue.toString(); } } catch (Exception innerException) { - if (!ignore_malformed) { + if (ignore_malformed == false) { throw new MapperParsingException( "failed to parse field [{}] of type [{}] in document with id '{}'. " + "Could not parse field value preview,", e, @@ -292,7 +292,7 @@ public void parse(ParseContext context) throws IOException { } } - if (!ignore_malformed) { + if (ignore_malformed == false) { throw new MapperParsingException( "failed to parse field [{}] of type [{}] in document with id '{}'. " + "Preview of field's value: '{}'", e,