[Rest Api Compatibility] Clean up x-pack/plugin rest compat tests

modules/reindex/build.gradle

@@ -168,15 +168,15 @@ tasks.named("transformV7RestTests").configure({ task ->
   task.addAllowedWarningRegex("\\[types removal\\].*")
 })
 
-def v7compatiblityNotSupportedTests = {
+def v7compatibilityNotSupportedTests = {
   return [
     'reindex/20_validation/reindex without source gives useful error message', // exception with a type. Not much benefit adding _doc there.
     'update_by_query/20_validation/update_by_query without source gives useful error message' // exception with a type. Not much benefit adding _doc there.
   ]
 }
 
 tasks.named("yamlRestCompatTest").configure {
-      systemProperty 'tests.rest.blacklist', v7compatiblityNotSupportedTests().join(',')
+      systemProperty 'tests.rest.blacklist', v7compatibilityNotSupportedTests().join(',')
 }
 
 tasks.named('splitPackagesAudit').configure {

rest-api-spec/build.gradle

@@ -28,7 +28,7 @@ testClusters.all {
 
 tasks.named("test").configure { enabled = false }
 tasks.named("jarHell").configure { enabled = false }
-def v7compatiblityNotSupportedTests = {
+def v7compatibilityNotSupportedTests = {
   return [
           // Cat API are meant to be consumed by humans, so will not be supported by Compatible REST API
           'cat*/*/*',
@@ -110,7 +110,7 @@ tasks.named("yamlRestCompatTest").configure {
     'search/310_match_bool_prefix/multi_match multiple fields with cutoff_frequency throws exception', //cutoff_frequency
     'search/340_type_query/type query', // type_query - probably should behave like match_all
     'search_shards/10_basic/Search shards aliases with and without filters',
-   ] + v7compatiblityNotSupportedTests())
+   ] + v7compatibilityNotSupportedTests())
     .join(',')
 }
 

x-pack/plugin/build.gradle

@@ -113,6 +113,34 @@ tasks.named("transformV7RestTests").configure({ task ->
   task.replaceKeyInDo("rollup.stop_job", "xpack-rollup.stop_job")
   task.addAllowedWarningRegex(".*_xpack/rollup.* is deprecated.*")
 })
+def v7compatibilityNotSupportedTests = {
+  return [
+    // to support it, it would require to almost revert back the #48725 and complicate the code
+    'vectors/10_dense_vector_basic/Deprecated function signature',
+    // not going to be supported
+    'vectors/30_sparse_vector_basic/Cosine Similarity',
+    'vectors/30_sparse_vector_basic/Deprecated function signature',
+    'vectors/30_sparse_vector_basic/Dot Product',
+    'vectors/35_sparse_vector_l1l2/L1 norm',
+    'vectors/35_sparse_vector_l1l2/L2 norm',
+    'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently',
+    'vectors/40_sparse_vector_special_cases/Documents missing a vector field',
+    'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors',
+    'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions',
+    'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types',
+    // the test uses sparse vector - not supported
+    'vectors/50_vector_stats/Usage stats on vector fields',
+
+    // put role request with a term lookup (deprecated) and type. Requires validation in REST layer
+    'roles/30_prohibited_role_query/Test use prohibited query inside role query',
+    //removing undocumented functionality
+    'ml/jobs_crud/Test create job with delimited format',
+    // behaviour change #44752 - not allowing to update datafeed job_id
+    'ml/datafeeds_crud/Test update datafeed to point to missing job',
+    'ml/datafeeds_crud/Test update datafeed to point to different job',
+    'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed',
+  ]
+}
 
 tasks.named("yamlRestCompatTest").configure {
   /*
@@ -123,64 +151,19 @@ tasks.named("yamlRestCompatTest").configure {
 
   //TODO: blacklist specific to REST API compatibility
   restTestBlacklist.addAll([
-    'analytics/histogram/Ranges over histogram',
-    'indices.freeze/10_basic/Basic',
-    'indices.freeze/10_basic/Test index options',
-    'indices.freeze/20_stats/Translog stats on frozen indices',
-    'indices.freeze/30_usage/Usage stats on frozen indices',
     'license/30_enterprise_license/Installing enterprise license',
-    'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection',
-    'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header',
-    'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header',
-    'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job',
-    'ml/datafeed_cat_apis/Test cat datafeeds',
-    'ml/datafeeds_crud/Test update datafeed to point to different job',
-    'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed',
-    'ml/datafeeds_crud/Test update datafeed to point to missing job',
-    'ml/job_cat_apis/Test cat anomaly detector jobs',
-    'ml/jobs_crud/Test create job with delimited format',
-    'ml/jobs_crud/Test update job',
-    'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats',
-    'ml/jobs_get_stats/Test get job stats for closed job',
-    'ml/jobs_get_stats/Test no exception on get job stats with missing index',
-    'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc',
-    'ml/post_data/Test flush with skip_time',
-    'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled',
-    'ml/set_upgrade_mode/Setting upgrade_mode to enabled',
-    'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false',
-    'ml/trained_model_cat_apis/Test cat trained models',
-    'roles/11_idx_arrays/Test put role api using as array of index names',
-    'roles/30_prohibited_role_query/Test use prohibited query inside role query',
+    //https://github.com/elastic/elasticsearch/pull/41227.
     'rollup/delete_job/Test basic delete_job',
     'rollup/delete_job/Test delete job twice',
     'rollup/delete_job/Test delete running job',
     'rollup/get_jobs/Test basic get_jobs',
     'rollup/put_job/Test basic put_job',
-    'rollup/put_job/Test put_job in non-rollup index',
+    //https://github.com/elastic/elasticsearch/pull/41502
     'rollup/start_job/Test start job twice',
-    'security/authz/14_cat_indices/Test empty request while no-authorized index',
-    'security/authz/14_cat_indices/Test empty request while single authorized index',
-    'security/authz/14_cat_indices/Test explicit request while multiple authorized indices',
-    'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices',
-    'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices',
-    'security/authz/50_data_streams/Test that requests not supporting data streams do not include data streams among authorized indices',
-    'transform/transforms_cat_apis/Test cat transform stats hiding headers',
-    'transform/transforms_cat_apis/Test cat transform stats with column selection',
-    'transform/transforms_cat_apis/Test cat transform stats with continuous transform',
-    'vectors/10_dense_vector_basic/Deprecated function signature',
-    'vectors/30_sparse_vector_basic/Cosine Similarity',
-    'vectors/30_sparse_vector_basic/Deprecated function signature',
-    'vectors/30_sparse_vector_basic/Dot Product',
-    'vectors/35_sparse_vector_l1l2/L1 norm',
-    'vectors/35_sparse_vector_l1l2/L2 norm',
-    'privileges/40_get_user_privs/Test get_user_privileges for merged roles',  // temporary disabled till #70191 gets backported
-    'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently',
-    'vectors/40_sparse_vector_special_cases/Documents missing a vector field',
-    'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors',
-    'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions',
-    'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types',
-    'vectors/50_vector_stats/Usage stats on vector fields'
-  ])
+    // a type field was added to cat.ml_trained_models #73660, this might need some work
+    'ml/trained_model_cat_apis/Test cat trained models'
+    ] + v7compatibilityNotSupportedTests())
+
 
   systemProperty 'tests.rest.blacklist', restTestBlacklist.join(',')
   dependsOn "copyExtraResources"

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java

@@ -61,9 +61,9 @@ public void write(InputStream inputStream, CategorizationAnalyzer categorization
             throws IOException {
         dataCountsReporter.startNewIncrementalCount();
 
-        if (xContentType.equals(XContentType.JSON)) {
+        if (xContentType.canonical() == XContentType.JSON) {
             writeJsonXContent(categorizationAnalyzer, inputStream);
-        } else if (xContentType.equals(XContentType.SMILE)) {
+        } else if (xContentType.canonical() == XContentType.SMILE) {
             writeSmileXContent(categorizationAnalyzer, inputStream);
         } else {
             throw new RuntimeException("XContentType [" + xContentType