From d559a6c7705f633df0b4e542b4eac4d3f6d2bb4d Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 29 Jun 2021 16:25:53 +0200 Subject: [PATCH 01/16] xpack enable all --- x-pack/plugin/build.gradle | 118 ++++++++++++++++++++----------------- 1 file changed, 63 insertions(+), 55 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index ee208ae2f867b..cf2299ffd265f 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -113,73 +113,81 @@ tasks.named("transformV7RestTests").configure({ task -> task.replaceKeyInDo("rollup.stop_job", "xpack-rollup.stop_job") task.addAllowedWarningRegex(".*_xpack/rollup.* is deprecated.*") }) - -tasks.named("yamlRestCompatTest").configure { - /* - * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each - * other if we allow them to set the number of available processors as it's set-once in Netty. - */ - systemProperty 'es.set.netty.runtime.available.processors', 'false' - - //TODO: blacklist specific to REST API compatibility - restTestBlacklist.addAll([ - 'indices.freeze/10_basic/Basic', - 'indices.freeze/10_basic/Test index options', - 'indices.freeze/20_stats/Translog stats on frozen indices', - 'indices.freeze/30_usage/Usage stats on frozen indices', - 'license/30_enterprise_license/Installing enterprise license', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', - 'ml/datafeed_cat_apis/Test cat datafeeds', - 'ml/datafeeds_crud/Test update datafeed to point to different job', - 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', - 'ml/datafeeds_crud/Test update datafeed to point to missing job', - 'ml/job_cat_apis/Test cat anomaly detector jobs', - 'ml/jobs_crud/Test create job with delimited format', - 'ml/jobs_crud/Test update job', - 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', - 'ml/jobs_get_stats/Test get job stats for closed job', - 'ml/jobs_get_stats/Test no exception on get job stats with missing index', - 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', - 'ml/post_data/Test flush with skip_time', - 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', - 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', - 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', - 'ml/trained_model_cat_apis/Test cat trained models', - 'roles/11_idx_arrays/Test put role api using as array of index names', - 'roles/30_prohibited_role_query/Test use prohibited query inside role query', - 'rollup/delete_job/Test basic delete_job', - 'rollup/delete_job/Test delete job twice', - 'rollup/delete_job/Test delete running job', - 'rollup/get_jobs/Test basic get_jobs', - 'rollup/put_job/Test basic put_job', - 'rollup/put_job/Test put_job in non-rollup index', - 'rollup/start_job/Test start job twice', - 'security/authz/14_cat_indices/Test empty request while no-authorized index', - 'security/authz/14_cat_indices/Test empty request while single authorized index', - 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', - 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', - 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', - 'security/authz/50_data_streams/Test that requests not supporting data streams do not include data streams among authorized indices', - 'transform/transforms_cat_apis/Test cat transform stats hiding headers', - 'transform/transforms_cat_apis/Test cat transform stats with column selection', - 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', +def v7compatiblityNotSupportedTests = { + return [ + // to support it, it would require to almost revert back the #48725 and complicate the code 'vectors/10_dense_vector_basic/Deprecated function signature', + // not going to be supported 'vectors/30_sparse_vector_basic/Cosine Similarity', 'vectors/30_sparse_vector_basic/Deprecated function signature', 'vectors/30_sparse_vector_basic/Dot Product', 'vectors/35_sparse_vector_l1l2/L1 norm', 'vectors/35_sparse_vector_l1l2/L2 norm', - 'privileges/40_get_user_privs/Test get_user_privileges for merged roles', // temporary disabled till #70191 gets backported 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', + // the test uses sparse vector - not supported 'vectors/50_vector_stats/Usage stats on vector fields' - ]) + + ] +} + +tasks.named("yamlRestCompatTest").configure { + /* + * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each + * other if we allow them to set the number of available processors as it's set-once in Netty. + */ + systemProperty 'es.set.netty.runtime.available.processors', 'false' + + //TODO: blacklist specific to REST API compatibility + restTestBlacklist.addAll([ +// 'indices.freeze/10_basic/Basic', +// 'indices.freeze/10_basic/Test index options', +// 'indices.freeze/20_stats/Translog stats on frozen indices', +// 'indices.freeze/30_usage/Usage stats on frozen indices', +// 'license/30_enterprise_license/Installing enterprise license', +// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', +// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', +// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', +// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', +// 'ml/datafeed_cat_apis/Test cat datafeeds', +// 'ml/datafeeds_crud/Test update datafeed to point to different job', +// 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', +// 'ml/datafeeds_crud/Test update datafeed to point to missing job', +// 'ml/job_cat_apis/Test cat anomaly detector jobs', +// 'ml/jobs_crud/Test create job with delimited format', +// 'ml/jobs_crud/Test update job', +// 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', +// 'ml/jobs_get_stats/Test get job stats for closed job', +// 'ml/jobs_get_stats/Test no exception on get job stats with missing index', +// 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', +// 'ml/post_data/Test flush with skip_time', +// 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', +// 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', +// 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', +// 'ml/trained_model_cat_apis/Test cat trained models', +// 'roles/11_idx_arrays/Test put role api using as array of index names', +// 'roles/30_prohibited_role_query/Test use prohibited query inside role query', +// 'rollup/delete_job/Test basic delete_job', +// 'rollup/delete_job/Test delete job twice', +// 'rollup/delete_job/Test delete running job', +// 'rollup/get_jobs/Test basic get_jobs', +// 'rollup/put_job/Test basic put_job', +// 'rollup/put_job/Test put_job in non-rollup index', +// 'rollup/start_job/Test start job twice', +// 'security/authz/14_cat_indices/Test empty request while no-authorized index', +// 'security/authz/14_cat_indices/Test empty request while single authorized index', +// 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', +// 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', +// 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', +// 'security/authz/50_data_streams/Test that requests not supporting data streams do not include data streams among authorized indices', +// 'transform/transforms_cat_apis/Test cat transform stats hiding headers', +// 'transform/transforms_cat_apis/Test cat transform stats with column selection', +// 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', + ]+ v7compatiblityNotSupportedTests()) + systemProperty 'tests.rest.blacklist', restTestBlacklist.join(',') dependsOn "copyExtraResources" From bdfec6a86e00081b32788ba9ae293ba65e6bdf88 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 30 Jun 2021 10:01:38 +0200 Subject: [PATCH 02/16] ml json cannonical --- x-pack/plugin/build.gradle | 24 +++++++++++++------ .../writer/JsonDataToProcessWriter.java | 4 ++-- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 0757806309330..66150512b50c0 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -129,7 +129,13 @@ def v7compatiblityNotSupportedTests = { 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', // the test uses sparse vector - not supported - 'vectors/50_vector_stats/Usage stats on vector fields' + 'vectors/50_vector_stats/Usage stats on vector fields', + + // ML - not needing compatible api + // `Remove the ability to update datafeed's job_id` #44752 not applicable + 'ml/datafeeds_crud/Test update datafeed to point to different job', + 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', + 'ml/datafeeds_crud/Test update datafeed to point to missing job' ] } @@ -155,21 +161,25 @@ tasks.named("yamlRestCompatTest").configure { // 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', // 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', // 'ml/datafeed_cat_apis/Test cat datafeeds', -// 'ml/datafeeds_crud/Test update datafeed to point to different job', -// 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', -// 'ml/datafeeds_crud/Test update datafeed to point to missing job', + + // 'ml/job_cat_apis/Test cat anomaly detector jobs', // 'ml/jobs_crud/Test create job with delimited format', // 'ml/jobs_crud/Test update job', -// 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', -// 'ml/jobs_get_stats/Test get job stats for closed job', -// 'ml/jobs_get_stats/Test no exception on get job stats with missing index', + // 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', // 'ml/post_data/Test flush with skip_time', // 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', // 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', // 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', // 'ml/trained_model_cat_apis/Test cat trained models', + + // fixed by using cannonical in JsonDataToProcessWriter +// 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', +// 'ml/jobs_get_stats/Test get job stats for closed job', +// 'ml/jobs_get_stats/Test no exception on get job stats with missing index', + + // 'roles/11_idx_arrays/Test put role api using as array of index names', // 'roles/30_prohibited_role_query/Test use prohibited query inside role query', // 'rollup/delete_job/Test basic delete_job', diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java index 43f0d6558b7c7..d9979c2fb3e69 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java @@ -61,9 +61,9 @@ public void write(InputStream inputStream, CategorizationAnalyzer categorization throws IOException { dataCountsReporter.startNewIncrementalCount(); - if (xContentType.equals(XContentType.JSON)) { + if (xContentType.canonical() == XContentType.JSON) { writeJsonXContent(categorizationAnalyzer, inputStream); - } else if (xContentType.equals(XContentType.SMILE)) { + } else if (xContentType.canonical() == XContentType.SMILE) { writeSmileXContent(categorizationAnalyzer, inputStream); } else { throw new RuntimeException("XContentType [" + xContentType From 48decdedb3679a5798ceca29d8738ee41b806ea1 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 1 Jul 2021 09:54:10 +0200 Subject: [PATCH 03/16] investigating copying regex --- .../rest/YamlRestCompatTestPluginFuncTest.groovy | 5 +++++ x-pack/plugin/build.gradle | 16 +++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy index be0fc2420168a..b022872829af6 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy @@ -250,6 +250,11 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest { - is_false: "value_to_replace" - is_true: "value_not_to_replace" - is_false: "value_not_to_replace" + - match: + $body: | + / #id type create_time state + ^ (dfa\\-outlier\\-detection\\-job \\s+ outlier_detection \\s+ [^\\s]+ \\s+ stopped \\n)+ \$/ + """.stripIndent() when: diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 66150512b50c0..a0ee3cf6f3b91 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -182,13 +182,15 @@ tasks.named("yamlRestCompatTest").configure { // 'roles/11_idx_arrays/Test put role api using as array of index names', // 'roles/30_prohibited_role_query/Test use prohibited query inside role query', -// 'rollup/delete_job/Test basic delete_job', -// 'rollup/delete_job/Test delete job twice', -// 'rollup/delete_job/Test delete running job', -// 'rollup/get_jobs/Test basic get_jobs', -// 'rollup/put_job/Test basic put_job', -// 'rollup/put_job/Test put_job in non-rollup index', -// 'rollup/start_job/Test start job twice', +//https://github.com/elastic/elasticsearch/pull/41227. + 'rollup/delete_job/Test basic delete_job', + 'rollup/delete_job/Test delete job twice', + 'rollup/delete_job/Test delete running job', + 'rollup/get_jobs/Test basic get_jobs', + 'rollup/put_job/Test basic put_job', + 'rollup/put_job/Test put_job in non-rollup index', + //https://github.com/elastic/elasticsearch/pull/41502 + 'rollup/start_job/Test start job twice', // 'security/authz/14_cat_indices/Test empty request while no-authorized index', // 'security/authz/14_cat_indices/Test empty request while single authorized index', // 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', From d4276cb5685243e63508b7c2b8caf3654f5b08c1 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 5 Jul 2021 12:36:36 +0200 Subject: [PATCH 04/16] ml cleanup --- x-pack/plugin/build.gradle | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index a0ee3cf6f3b91..f010b9e9902e0 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -132,11 +132,9 @@ def v7compatiblityNotSupportedTests = { 'vectors/50_vector_stats/Usage stats on vector fields', // ML - not needing compatible api - // `Remove the ability to update datafeed's job_id` #44752 not applicable - 'ml/datafeeds_crud/Test update datafeed to point to different job', - 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', - 'ml/datafeeds_crud/Test update datafeed to point to missing job' - + // some are failing due to cat api being called with JSON accept header + 'ml/*', + '_apis/Test cat trained models' ] } @@ -156,29 +154,6 @@ tasks.named("yamlRestCompatTest").configure { // 'indices.freeze/20_stats/Translog stats on frozen indices', // 'indices.freeze/30_usage/Usage stats on frozen indices', // 'license/30_enterprise_license/Installing enterprise license', -// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', -// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', -// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', -// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', -// 'ml/datafeed_cat_apis/Test cat datafeeds', - - -// 'ml/job_cat_apis/Test cat anomaly detector jobs', -// 'ml/jobs_crud/Test create job with delimited format', -// 'ml/jobs_crud/Test update job', - -// 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', -// 'ml/post_data/Test flush with skip_time', -// 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', -// 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', -// 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', -// 'ml/trained_model_cat_apis/Test cat trained models', - - // fixed by using cannonical in JsonDataToProcessWriter -// 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', -// 'ml/jobs_get_stats/Test get job stats for closed job', -// 'ml/jobs_get_stats/Test no exception on get job stats with missing index', - // 'roles/11_idx_arrays/Test put role api using as array of index names', // 'roles/30_prohibited_role_query/Test use prohibited query inside role query', From 4a32d3dda8dcb7c159f32aabc93a0321b6e74ba0 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 5 Jul 2021 13:53:56 +0200 Subject: [PATCH 05/16] revert groovy test --- .../test/rest/YamlRestCompatTestPluginFuncTest.groovy | 5 ----- 1 file changed, 5 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy index b022872829af6..be0fc2420168a 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy @@ -250,11 +250,6 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest { - is_false: "value_to_replace" - is_true: "value_not_to_replace" - is_false: "value_not_to_replace" - - match: - $body: | - / #id type create_time state - ^ (dfa\\-outlier\\-detection\\-job \\s+ outlier_detection \\s+ [^\\s]+ \\s+ stopped \\n)+ \$/ - """.stripIndent() when: From c1163a57b92d144ec7bd627686bc1bdef46a6c6e Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 5 Jul 2021 14:27:52 +0200 Subject: [PATCH 06/16] do not modify cat headers --- .../YamlRestCompatTestPluginFuncTest.groovy | 17 ++++++++++++++++- .../rest/transform/headers/InjectHeaders.java | 16 ++++++++++++++++ x-pack/plugin/build.gradle | 2 +- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy index be0fc2420168a..31bb124c8a9cd 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy @@ -250,7 +250,12 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest { - is_false: "value_to_replace" - is_true: "value_not_to_replace" - is_false: "value_not_to_replace" - + --- + "use cat with no header": + - do: + cat.indices: + {} + - match: {} """.stripIndent() when: def result = gradleRunner("yamlRestCompatTest").build() @@ -337,6 +342,16 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest { - is_false: "replaced_value" - is_true: "value_not_to_replace" - is_false: "value_not_to_replace" + --- + "use cat with no header": + - do: + cat.indices: + {} + allowed_warnings: + - "added allowed warning" + allowed_warnings_regex: + - "added allowed warning regex .* [0-9]" + - match: {} """.stripIndent()).readAll() expectedAll.eachWithIndex{ ObjectNode expected, int i -> diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java index 27c54a5426cbd..a8b04f02a37c1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java @@ -17,6 +17,7 @@ import org.gradle.api.tasks.Input; import org.gradle.api.tasks.Internal; +import java.util.Iterator; import java.util.Map; /** @@ -39,6 +40,10 @@ public InjectHeaders(Map headers) { @Override public void transformTest(ObjectNode doNodeParent) { ObjectNode doNodeValue = (ObjectNode) doNodeParent.get(getKeyToFind()); + if(isCatNode(doNodeValue)){ + return; + } + ObjectNode headersNode = (ObjectNode) doNodeValue.get("headers"); if (headersNode == null) { headersNode = new ObjectNode(jsonNodeFactory); @@ -49,6 +54,17 @@ public void transformTest(ObjectNode doNodeParent) { doNodeValue.set("headers", headersNode); } + private boolean isCatNode(ObjectNode doNodeValue) { + final Iterator stringIterator = doNodeValue.fieldNames(); + while (stringIterator.hasNext()) { + final String fieldName = stringIterator.next(); + if(fieldName.startsWith("cat")) { + return true; + } + } + return false; + } + @Override @Internal public String getKeyToFind() { diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index f010b9e9902e0..813028919aa2d 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -133,7 +133,7 @@ def v7compatiblityNotSupportedTests = { // ML - not needing compatible api // some are failing due to cat api being called with JSON accept header - 'ml/*', + 'ml*/*/*', '_apis/Test cat trained models' ] } From 9314dd772258ae297958774e6b8144bc043ce62e Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 5 Jul 2021 15:17:30 +0200 Subject: [PATCH 07/16] filter cat. --- .../internal/test/rest/transform/headers/InjectHeaders.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java index a8b04f02a37c1..c1d99a074835c 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java @@ -58,7 +58,7 @@ private boolean isCatNode(ObjectNode doNodeValue) { final Iterator stringIterator = doNodeValue.fieldNames(); while (stringIterator.hasNext()) { final String fieldName = stringIterator.next(); - if(fieldName.startsWith("cat")) { + if(fieldName.startsWith("cat.")) { return true; } } From b41d308a55f59762a06be29cb6873e0b0f3a8291 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 5 Jul 2021 15:45:12 +0200 Subject: [PATCH 08/16] remove comments --- x-pack/plugin/build.gradle | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 813028919aa2d..3388d24f718af 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -147,16 +147,10 @@ tasks.named("yamlRestCompatTest").configure { //TODO: blacklist specific to REST API compatibility restTestBlacklist.addAll([ -// 'analytics/histogram/Ranges over histogram', + 'license/30_enterprise_license/Installing enterprise license', -// 'indices.freeze/10_basic/Basic', -// 'indices.freeze/10_basic/Test index options', -// 'indices.freeze/20_stats/Translog stats on frozen indices', -// 'indices.freeze/30_usage/Usage stats on frozen indices', -// 'license/30_enterprise_license/Installing enterprise license', - -// 'roles/11_idx_arrays/Test put role api using as array of index names', -// 'roles/30_prohibited_role_query/Test use prohibited query inside role query', + // put role request with a term lookup (deprecated) and type. + 'roles/30_prohibited_role_query/Test use prohibited query inside role query', //https://github.com/elastic/elasticsearch/pull/41227. 'rollup/delete_job/Test basic delete_job', 'rollup/delete_job/Test delete job twice', @@ -165,17 +159,8 @@ tasks.named("yamlRestCompatTest").configure { 'rollup/put_job/Test basic put_job', 'rollup/put_job/Test put_job in non-rollup index', //https://github.com/elastic/elasticsearch/pull/41502 - 'rollup/start_job/Test start job twice', -// 'security/authz/14_cat_indices/Test empty request while no-authorized index', -// 'security/authz/14_cat_indices/Test empty request while single authorized index', -// 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', -// 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', -// 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', -// 'security/authz/50_data_streams/Test that requests not supporting data streams do not include data streams among authorized indices', -// 'transform/transforms_cat_apis/Test cat transform stats hiding headers', -// 'transform/transforms_cat_apis/Test cat transform stats with column selection', -// 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', -//'privileges/40_get_user_privs/Test get_user_privileges for merged roles', // temporary disabled till #70191 gets backported + 'rollup/start_job/Test start job twice' + ]+ v7compatiblityNotSupportedTests()) From fe46612bd41b76dd0961d088065683419d2377e0 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 5 Jul 2021 17:49:47 +0200 Subject: [PATCH 09/16] move roles to ignore --- x-pack/plugin/build.gradle | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 3388d24f718af..472c17f3ce6f2 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -134,7 +134,9 @@ def v7compatiblityNotSupportedTests = { // ML - not needing compatible api // some are failing due to cat api being called with JSON accept header 'ml*/*/*', - '_apis/Test cat trained models' + '_apis/Test cat trained models', + // put role request with a term lookup (deprecated) and type. Requires validation in REST layer + 'roles/30_prohibited_role_query/Test use prohibited query inside role query', ] } @@ -148,10 +150,7 @@ tasks.named("yamlRestCompatTest").configure { //TODO: blacklist specific to REST API compatibility restTestBlacklist.addAll([ 'license/30_enterprise_license/Installing enterprise license', - - // put role request with a term lookup (deprecated) and type. - 'roles/30_prohibited_role_query/Test use prohibited query inside role query', -//https://github.com/elastic/elasticsearch/pull/41227. + //https://github.com/elastic/elasticsearch/pull/41227. 'rollup/delete_job/Test basic delete_job', 'rollup/delete_job/Test delete job twice', 'rollup/delete_job/Test delete running job', @@ -160,8 +159,6 @@ tasks.named("yamlRestCompatTest").configure { 'rollup/put_job/Test put_job in non-rollup index', //https://github.com/elastic/elasticsearch/pull/41502 'rollup/start_job/Test start job twice' - - ]+ v7compatiblityNotSupportedTests()) From 5fb01c3d5e7cdb585d3b0cd209f1d9734ad997a9 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 6 Jul 2021 09:46:46 +0200 Subject: [PATCH 10/16] rename --- .../test/rest/transform/headers/InjectHeaders.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java index c1d99a074835c..d399fe214b1d0 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java @@ -40,7 +40,7 @@ public InjectHeaders(Map headers) { @Override public void transformTest(ObjectNode doNodeParent) { ObjectNode doNodeValue = (ObjectNode) doNodeParent.get(getKeyToFind()); - if(isCatNode(doNodeValue)){ + if(isCatOperation(doNodeValue)){ return; } @@ -54,11 +54,11 @@ public void transformTest(ObjectNode doNodeParent) { doNodeValue.set("headers", headersNode); } - private boolean isCatNode(ObjectNode doNodeValue) { - final Iterator stringIterator = doNodeValue.fieldNames(); - while (stringIterator.hasNext()) { - final String fieldName = stringIterator.next(); - if(fieldName.startsWith("cat.")) { + private boolean isCatOperation(ObjectNode doNodeValue) { + final Iterator fieldNamesIterator = doNodeValue.fieldNames(); + while (fieldNamesIterator.hasNext()) { + final String fieldName = fieldNamesIterator.next(); + if (fieldName.startsWith("cat.")) { return true; } } From c3b9f305fa02081f5bdfea76d3647e3f037e572f Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 6 Jul 2021 15:48:35 +0200 Subject: [PATCH 11/16] enable ml related --- x-pack/plugin/build.gradle | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index bddf7a9eb2e15..f1de4d5188faf 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -131,10 +131,6 @@ def v7compatiblityNotSupportedTests = { // the test uses sparse vector - not supported 'vectors/50_vector_stats/Usage stats on vector fields', - // ML - not needing compatible api - // some are failing due to cat api being called with JSON accept header - 'ml*/*/*', - '_apis/Test cat trained models', // put role request with a term lookup (deprecated) and type. Requires validation in REST layer 'roles/30_prohibited_role_query/Test use prohibited query inside role query', ] From 08db9a17f7ef76843d3f3818bd40aab12a9c0449 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 6 Jul 2021 16:19:30 +0200 Subject: [PATCH 12/16] disable ml tests which won't be worked on --- x-pack/plugin/build.gradle | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index f1de4d5188faf..0e1fa6ef1bb0e 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -133,6 +133,12 @@ def v7compatiblityNotSupportedTests = { // put role request with a term lookup (deprecated) and type. Requires validation in REST layer 'roles/30_prohibited_role_query/Test use prohibited query inside role query', + //removing undocumented functionality + 'ml/jobs_crud/Test create job with delimited format', + // behaviour change #44752 - not allowing to update datafeed job_id + 'ml/datafeeds_crud/Test update datafeed to point to missing job', + 'ml/datafeeds_crud/Test update datafeed to point to different job', + 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', ] } @@ -155,7 +161,9 @@ tasks.named("yamlRestCompatTest").configure { 'rollup/put_job/Test put_job in non-rollup index', //https://github.com/elastic/elasticsearch/pull/41502 'rollup/start_job/Test start job twice', - ]+ v7compatiblityNotSupportedTests()) + // a type field was added to cat.ml_trained_models #73660, this might need some work + 'ml/trained_model_cat_apis/Test cat trained models' + ]+ v7compatiblityNotSupportedTests()) systemProperty 'tests.rest.blacklist', restTestBlacklist.join(',') From 7ad7093637609fc420bf04e1483c6320788dabd6 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 6 Jul 2021 18:35:05 +0200 Subject: [PATCH 13/16] apply condition to inject headers --- .../compat/RestCompatTestTransformTask.java | 14 ++++++- .../rest/transform/headers/InjectHeaders.java | 40 +++++++++---------- .../test/rest/transform/TransformTests.java | 7 ++-- .../transform/header/InjectHeaderTests.java | 33 ++++++++++++++- .../with_operation_to_skip_adding_headers.yml | 13 ++++++ 5 files changed, 81 insertions(+), 26 deletions(-) create mode 100644 build-tools-internal/src/test/resources/rest/transform/header/with_operation_to_skip_adding_headers.yml diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java index 74a870cbdd1fa..69733f7fd9514 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java @@ -53,6 +53,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; @@ -94,7 +95,18 @@ public RestCompatTestTransformTask( // always inject compat headers headers.put("Content-Type", "application/vnd.elasticsearch+json;compatible-with=" + compatibleVersion); headers.put("Accept", "application/vnd.elasticsearch+json;compatible-with=" + compatibleVersion); - transformations.add(new InjectHeaders(headers)); + transformations.add(new InjectHeaders(headers, Set.of(RestCompatTestTransformTask::doesNotHaveCatOperation))); + } + + private static boolean doesNotHaveCatOperation(ObjectNode doNodeValue) { + final Iterator fieldNamesIterator = doNodeValue.fieldNames(); + while (fieldNamesIterator.hasNext()) { + final String fieldName = fieldNamesIterator.next(); + if (fieldName.startsWith("cat.")) { + return false; + } + } + return true; } /** diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java index d399fe214b1d0..8c0fcd2a817d5 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java @@ -17,8 +17,10 @@ import org.gradle.api.tasks.Input; import org.gradle.api.tasks.Internal; -import java.util.Iterator; import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Function; /** * A {@link RestTestTransform} that injects HTTP headers into a REST test. This includes adding the necessary values to the "do" section @@ -29,40 +31,36 @@ public class InjectHeaders extends FeatureInjector implements RestTestTransformB private static JsonNodeFactory jsonNodeFactory = JsonNodeFactory.withExactBigDecimals(false); private final Map headers; + private final Set> applyConditions; /** * @param headers The headers to inject + * @param applyConditions a set of conditions that has to be satisfied in order to apply headers */ - public InjectHeaders(Map headers) { + public InjectHeaders(Map headers, Set> applyConditions) { this.headers = headers; + this.applyConditions = applyConditions; } @Override public void transformTest(ObjectNode doNodeParent) { ObjectNode doNodeValue = (ObjectNode) doNodeParent.get(getKeyToFind()); - if(isCatOperation(doNodeValue)){ - return; - } - ObjectNode headersNode = (ObjectNode) doNodeValue.get("headers"); - if (headersNode == null) { - headersNode = new ObjectNode(jsonNodeFactory); - } - for (Map.Entry entry : headers.entrySet()) { - headersNode.set(entry.getKey(), TextNode.valueOf(entry.getValue())); - } - doNodeValue.set("headers", headersNode); - } + if (shouldApplyHeaders(doNodeValue)) { + ObjectNode headersNode = (ObjectNode) doNodeValue.get("headers"); + if (headersNode == null) { + headersNode = new ObjectNode(jsonNodeFactory); + } - private boolean isCatOperation(ObjectNode doNodeValue) { - final Iterator fieldNamesIterator = doNodeValue.fieldNames(); - while (fieldNamesIterator.hasNext()) { - final String fieldName = fieldNamesIterator.next(); - if (fieldName.startsWith("cat.")) { - return true; + for (Map.Entry entry : headers.entrySet()) { + headersNode.set(entry.getKey(), TextNode.valueOf(entry.getValue())); } + doNodeValue.set("headers", headersNode); } - return false; + } + + private boolean shouldApplyHeaders(ObjectNode doNodeValue) { + return applyConditions.stream().allMatch(f -> f.apply(doNodeValue)); } @Override diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/TransformTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/TransformTests.java index c3f13b1a02c4c..2f195a04eb519 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/TransformTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/TransformTests.java @@ -34,6 +34,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.LongAdder; +import java.util.function.Consumer; import java.util.stream.Collectors; public abstract class TransformTests extends GradleUnitTestCase { @@ -125,8 +126,8 @@ protected List getKnownFeatures() { protected List> getTransformations() { List> transformations = new ArrayList<>(); - transformations.add(new InjectHeaders(headers1)); - transformations.add(new InjectHeaders(headers2)); + transformations.add(new InjectHeaders(headers1, Collections.emptySet())); + transformations.add(new InjectHeaders(headers2, Collections.emptySet())); return transformations; } @@ -337,7 +338,7 @@ protected void validateSetupAndTearDownForMatchTests(List tests) { } protected boolean getHumanDebug() { - return false; + return true; } // only to help manually debug diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java index 34d1273a086a2..07881a0f48678 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java @@ -14,9 +14,12 @@ import org.elasticsearch.gradle.internal.test.rest.transform.headers.InjectHeaders; import org.junit.Test; +import java.util.ArrayList; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; public class InjectHeaderTests extends InjectFeatureTests { @@ -57,6 +60,34 @@ public void testInjectHeadersWithPreExisting() throws Exception { validateBodyHasHeaders(transformedTests, headers); } + + @Test + public void testNotInjectingHeaders() throws Exception { + String testName = "/rest/transform/header/with_operation_to_skip_adding_headers.yml"; + List tests = getTests(testName); + validateSetupExist(tests); + validateBodyHasHeaders(tests, Map.of("foo", "bar")); + + List> transformations = + Collections.singletonList(new InjectHeaders(headers, Set.of(InjectHeaderTests::applyCondition))); + List transformedTests = transformTests(tests, transformations); + printTest(testName, transformedTests); + validateSetupAndTearDown(transformedTests); + validateBodyHasHeaders(tests, Map.of("foo", "bar")); + validateBodyHasHeaders(transformedTests, Map.of("foo", "bar")); + } + + private static boolean applyCondition(ObjectNode doNodeValue) { + final Iterator fieldNamesIterator = doNodeValue.fieldNames(); + while (fieldNamesIterator.hasNext()) { + final String fieldName = fieldNamesIterator.next(); + if (fieldName.startsWith("something_to_skip")) { + return false; + } + } + return true; + } + @Override protected List getKnownFeatures() { return Collections.singletonList("headers"); @@ -64,7 +95,7 @@ protected List getKnownFeatures() { @Override protected List> getTransformations() { - return Collections.singletonList(new InjectHeaders(headers)); + return Collections.singletonList(new InjectHeaders(headers, Collections.emptySet())); } @Override diff --git a/build-tools-internal/src/test/resources/rest/transform/header/with_operation_to_skip_adding_headers.yml b/build-tools-internal/src/test/resources/rest/transform/header/with_operation_to_skip_adding_headers.yml new file mode 100644 index 0000000000000..9387dbe2d6b31 --- /dev/null +++ b/build-tools-internal/src/test/resources/rest/transform/header/with_operation_to_skip_adding_headers.yml @@ -0,0 +1,13 @@ +--- +setup: + - skip: + features: headers +--- +"Test without a setup": + - do: + headers: + foo: "bar" + something_to_skip: + id: "something" + - match: { acknowledged: true } + From 4c539f58540ab1307cbd1b40f90d2a5a4bfbf670 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 6 Jul 2021 18:44:43 +0200 Subject: [PATCH 14/16] human debug --- .../gradle/internal/test/rest/transform/TransformTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/TransformTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/TransformTests.java index 2f195a04eb519..62ed726ae949d 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/TransformTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/TransformTests.java @@ -338,7 +338,7 @@ protected void validateSetupAndTearDownForMatchTests(List tests) { } protected boolean getHumanDebug() { - return true; + return false; } // only to help manually debug From d9a264d3e2a3e38bb88997b84ec855b1b02eb3d4 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 6 Jul 2021 18:46:31 +0200 Subject: [PATCH 15/16] revert enable_all changes --- x-pack/plugin/build.gradle | 84 +++++++++++-------- .../writer/JsonDataToProcessWriter.java | 4 +- 2 files changed, 52 insertions(+), 36 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 0e1fa6ef1bb0e..7448fb38fa5e6 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -113,34 +113,6 @@ tasks.named("transformV7RestTests").configure({ task -> task.replaceKeyInDo("rollup.stop_job", "xpack-rollup.stop_job") task.addAllowedWarningRegex(".*_xpack/rollup.* is deprecated.*") }) -def v7compatiblityNotSupportedTests = { - return [ - // to support it, it would require to almost revert back the #48725 and complicate the code - 'vectors/10_dense_vector_basic/Deprecated function signature', - // not going to be supported - 'vectors/30_sparse_vector_basic/Cosine Similarity', - 'vectors/30_sparse_vector_basic/Deprecated function signature', - 'vectors/30_sparse_vector_basic/Dot Product', - 'vectors/35_sparse_vector_l1l2/L1 norm', - 'vectors/35_sparse_vector_l1l2/L2 norm', - 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', - 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', - 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', - 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', - 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', - // the test uses sparse vector - not supported - 'vectors/50_vector_stats/Usage stats on vector fields', - - // put role request with a term lookup (deprecated) and type. Requires validation in REST layer - 'roles/30_prohibited_role_query/Test use prohibited query inside role query', - //removing undocumented functionality - 'ml/jobs_crud/Test create job with delimited format', - // behaviour change #44752 - not allowing to update datafeed job_id - 'ml/datafeeds_crud/Test update datafeed to point to missing job', - 'ml/datafeeds_crud/Test update datafeed to point to different job', - 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', - ] -} tasks.named("yamlRestCompatTest").configure { /* @@ -151,20 +123,64 @@ tasks.named("yamlRestCompatTest").configure { //TODO: blacklist specific to REST API compatibility restTestBlacklist.addAll([ + 'analytics/histogram/Ranges over histogram', + 'indices.freeze/10_basic/Basic', + 'indices.freeze/10_basic/Test index options', + 'indices.freeze/20_stats/Translog stats on frozen indices', + 'indices.freeze/30_usage/Usage stats on frozen indices', 'license/30_enterprise_license/Installing enterprise license', - //https://github.com/elastic/elasticsearch/pull/41227. + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', + 'ml/datafeed_cat_apis/Test cat datafeeds', + 'ml/datafeeds_crud/Test update datafeed to point to different job', + 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', + 'ml/datafeeds_crud/Test update datafeed to point to missing job', + 'ml/job_cat_apis/Test cat anomaly detector jobs', + 'ml/jobs_crud/Test create job with delimited format', + 'ml/jobs_crud/Test update job', + 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', + 'ml/jobs_get_stats/Test get job stats for closed job', + 'ml/jobs_get_stats/Test no exception on get job stats with missing index', + 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', + 'ml/post_data/Test flush with skip_time', + 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', + 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', + 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', + 'ml/trained_model_cat_apis/Test cat trained models', + 'roles/11_idx_arrays/Test put role api using as array of index names', + 'roles/30_prohibited_role_query/Test use prohibited query inside role query', 'rollup/delete_job/Test basic delete_job', 'rollup/delete_job/Test delete job twice', 'rollup/delete_job/Test delete running job', 'rollup/get_jobs/Test basic get_jobs', 'rollup/put_job/Test basic put_job', 'rollup/put_job/Test put_job in non-rollup index', - //https://github.com/elastic/elasticsearch/pull/41502 'rollup/start_job/Test start job twice', - // a type field was added to cat.ml_trained_models #73660, this might need some work - 'ml/trained_model_cat_apis/Test cat trained models' - ]+ v7compatiblityNotSupportedTests()) - + 'security/authz/14_cat_indices/Test empty request while no-authorized index', + 'security/authz/14_cat_indices/Test empty request while single authorized index', + 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', + 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', + 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', + 'security/authz/50_data_streams/Test that requests not supporting data streams do not include data streams among authorized indices', + 'transform/transforms_cat_apis/Test cat transform stats hiding headers', + 'transform/transforms_cat_apis/Test cat transform stats with column selection', + 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', + 'vectors/10_dense_vector_basic/Deprecated function signature', + 'vectors/30_sparse_vector_basic/Cosine Similarity', + 'vectors/30_sparse_vector_basic/Deprecated function signature', + 'vectors/30_sparse_vector_basic/Dot Product', + 'vectors/35_sparse_vector_l1l2/L1 norm', + 'vectors/35_sparse_vector_l1l2/L2 norm', + 'privileges/40_get_user_privs/Test get_user_privileges for merged roles', // temporary disabled till #70191 gets backported + 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', + 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', + 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', + 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', + 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', + 'vectors/50_vector_stats/Usage stats on vector fields' + ]) systemProperty 'tests.rest.blacklist', restTestBlacklist.join(',') dependsOn "copyExtraResources" diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java index d9979c2fb3e69..43f0d6558b7c7 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java @@ -61,9 +61,9 @@ public void write(InputStream inputStream, CategorizationAnalyzer categorization throws IOException { dataCountsReporter.startNewIncrementalCount(); - if (xContentType.canonical() == XContentType.JSON) { + if (xContentType.equals(XContentType.JSON)) { writeJsonXContent(categorizationAnalyzer, inputStream); - } else if (xContentType.canonical() == XContentType.SMILE) { + } else if (xContentType.equals(XContentType.SMILE)) { writeSmileXContent(categorizationAnalyzer, inputStream); } else { throw new RuntimeException("XContentType [" + xContentType From 2b2a342bca1cebce232a614498984e3ec2fb6566 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 7 Jul 2021 16:43:28 +0200 Subject: [PATCH 16/16] javadoc --- .../internal/test/rest/transform/headers/InjectHeaders.java | 1 + 1 file changed, 1 insertion(+) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java index 8c0fcd2a817d5..c61a0a860d1ec 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java @@ -36,6 +36,7 @@ public class InjectHeaders extends FeatureInjector implements RestTestTransformB /** * @param headers The headers to inject * @param applyConditions a set of conditions that has to be satisfied in order to apply headers + * If the Set is empty then headers are always applied. */ public InjectHeaders(Map headers, Set> applyConditions) { this.headers = headers;