Validate Accept and Content-Type header for compatible API (#54103)

Adding validation of Accept And Content-Type headers. The idea is to verify combinations of presence and values of both headers depending if a request has a content, headers are versioned (type/vnd.elasticsearch+subtype;compatible-with) .
It also changes the way media type is parsed (previously always assuming application as a type i.e application/json) It should expect different types like text - used in SQL
Not adding a compatible headers for _cat api. These in order to return a default text do not expect Accept header (Content-type is not needed as content is not present)

See #53228 (comment) for more context

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java

@@ -117,7 +117,7 @@ public XContent xContent() {
     };
 
     private static final Pattern COMPATIBLE_API_HEADER_PATTERN = Pattern.compile(
-        "application/(vnd.elasticsearch\\+)?([^;]+)(\\s*;\\s*compatible-with=(\\d+))?",
+        "(application|text)/(vnd.elasticsearch\\+)?([^;]+)(\\s*;\\s*compatible-with=(\\d+))?",
         Pattern.CASE_INSENSITIVE);
 
     /**
@@ -126,7 +126,9 @@ public XContent xContent() {
      * also supports a wildcard accept for {@code application/*}. This method can be used to parse the {@code Accept} HTTP header or a
      * format query string parameter. This method will return {@code null} if no match is found
      */
-    public static XContentType fromMediaTypeOrFormat(String mediaType) {
+    public static XContentType fromMediaTypeOrFormat(String mediaTypeHeaderValue) {
+        String mediaType = parseMediaType(mediaTypeHeaderValue);
+
         if (mediaType == null) {
             return null;
         }
@@ -136,7 +138,7 @@ public static XContentType fromMediaTypeOrFormat(String mediaType) {
             }
         }
         final String lowercaseMediaType = mediaType.toLowerCase(Locale.ROOT);
-        if (lowercaseMediaType.startsWith("application/*")) {
+        if (lowercaseMediaType.startsWith("application/*") || lowercaseMediaType.equals("*/*")) {
             return JSON;
         }
 
@@ -165,11 +167,12 @@ public static XContentType fromMediaType(String mediaTypeHeaderValue) {
         return null;
     }
 
+    //public scope needed for text formats hack
     public static String parseMediaType(String mediaType) {
         if (mediaType != null) {
             Matcher matcher = COMPATIBLE_API_HEADER_PATTERN.matcher(mediaType);
             if (matcher.find()) {
-                return "application/" + matcher.group(2).toLowerCase(Locale.ROOT);
+                return (matcher.group(1) + "/" + matcher.group(3)).toLowerCase(Locale.ROOT);
             }
         }
 
@@ -179,9 +182,9 @@ public static String parseMediaType(String mediaType) {
     public static String parseVersion(String mediaType){
         if(mediaType != null){
             Matcher matcher = COMPATIBLE_API_HEADER_PATTERN.matcher(mediaType);
-            if (matcher.find() && "vnd.elasticsearch+".equalsIgnoreCase(matcher.group(1))) {
+            if (matcher.find() && "vnd.elasticsearch+".equalsIgnoreCase(matcher.group(2))) {
 
-                return matcher.group(4);
+                return matcher.group(5);
             }
         }
         return null;

modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java

@@ -37,12 +37,12 @@ public class RestGetActionV7 extends RestGetAction {
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGetAction.class));
     static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in "
         + "document get requests is deprecated, use the /{index}/_doc/{id} endpoint instead.";
+
     @Override
     public String getName() {
         return "document_get_action_v7";
     }
 
-
     @Override
     public List<Route> routes() {
         assert Version.CURRENT.major == 8 : "REST API compatibility for version 7 is only supported on version 8";

modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java

@@ -74,6 +74,7 @@ public boolean compatibilityRequired() {
     }
 
     public static class CompatibleCreateHandler extends RestIndexAction.CreateHandler {
+
         @Override
         public String getName() {
             return "document_create_action_v7";
@@ -100,15 +101,16 @@ public boolean compatibilityRequired() {
     }
 
     public static final class CompatibleAutoIdHandler extends RestIndexAction.AutoIdHandler {
-        @Override
-        public String getName() {
-            return "document_create_action_auto_id_v7";
-        }
 
         public CompatibleAutoIdHandler(Supplier<DiscoveryNodes> nodesInCluster) {
             super(nodesInCluster);
         }
 
+        @Override
+        public String getName() {
+            return "document_create_action_auto_id_v7";
+        }
+
         @Override
         public List<Route> routes() {
             return singletonList(new Route(POST, "/{index}/{type}"));

modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java

@@ -101,4 +101,27 @@ public void testInvalidHeaderValue() throws IOException {
         assertThat(map.get("type"), equalTo("content_type_header_exception"));
         assertThat(map.get("reason"), equalTo("java.lang.IllegalArgumentException: invalid Content-Type header []"));
     }
+
+    public void testInvalidHeaderCombinations() throws IOException {
+        final Request request = new Request("GET", "/_cluster/settings");
+        final RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.addHeader("Content-Type", "application/vnd.elasticsearch+json;compatible-with=7");
+        options.addHeader("Accept", "application/vnd.elasticsearch+json;compatible-with=8");
+        request.setOptions(options);
+        request.setJsonEntity("{\"transient\":{\"search.*\":null}}");
+
+        final ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request));
+        final Response response = e.getResponse();
+        assertThat(response.getStatusLine().getStatusCode(), equalTo(400));
+        final ObjectPath objectPath = ObjectPath.createFromResponse(response);
+        final Map<String, Object> map = objectPath.evaluate("error");
+        assertThat(map.get("type"), equalTo("compatible_api_headers_combination_exception"));
+        assertThat(map.get("reason"), equalTo("Content-Type and Accept headers have to match when content is present. " +
+            "Accept=application/vnd.elasticsearch+json;compatible-with=8 " +
+            "Content-Type=application/vnd.elasticsearch+json;compatible-with=7 " +
+            "hasContent=true " +
+            "path=/_cluster/settings " +
+            "params={} " +
+            "method=GET"));
+    }
 }

qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java

@@ -33,6 +33,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Consumer;
 import java.util.stream.StreamSupport;
 
 /**
@@ -43,7 +44,6 @@ protected AbstractCompatRestTest(@Name("yaml") ClientYamlTestCandidate testCandi
         super(testCandidate);
     }
 
-
     private static final Logger staticLogger = LogManager.getLogger(AbstractCompatRestTest.class);
 
     public static final String COMPAT_TESTS_PATH = "/rest-api-spec/test-compat";
@@ -66,38 +66,48 @@ public static Iterable<Object[]> createParameters() throws Exception {
             });
             finalTestCandidates.add(testCandidates.toArray());
         }
-        localCandidates.keySet().forEach(lc -> finalTestCandidates.add(new Object[]{lc}));
+        localCandidates.keySet().forEach(lc -> finalTestCandidates.add(new Object[] { lc }));
         return finalTestCandidates;
     }
 
-
     private static void mutateTestCandidate(ClientYamlTestCandidate testCandidate) {
-        testCandidate.getTestSection().getExecutableSections().stream().filter(s -> s instanceof DoSection).forEach(ds -> {
+        testCandidate.getSetupSection().getExecutableSections().stream().filter(s -> s instanceof DoSection).forEach(updateDoSection());
+        testCandidate.getTestSection().getExecutableSections().stream().filter(s -> s instanceof DoSection).forEach(updateDoSection());
+    }
+
+    private static Consumer<? super ExecutableSection> updateDoSection() {
+        return ds -> {
             DoSection doSection = (DoSection) ds;
-            //TODO: be more selective here
+            // TODO: be more selective here
             doSection.setIgnoreWarnings(true);
 
             String compatibleHeader = createCompatibleHeader();
-            //TODO decide which one to use - Accept or Content-Type
-            doSection.getApiCallSection()
-                     .addHeaders(Map.of(
-                         CompatibleConstants.COMPATIBLE_HEADER, compatibleHeader,
-                         "Content-Type", compatibleHeader
-                         ));
-        });
+            // for cat apis accept headers would break tests which expect txt response
+            if (doSection.getApiCallSection().getApi().startsWith("cat") == false) {
+                doSection.getApiCallSection()
+                    .addHeaders(
+                        Map.of(
+                            CompatibleConstants.COMPATIBLE_ACCEPT_HEADER,
+                            compatibleHeader,
+                            CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER,
+                            compatibleHeader
+                        )
+                    );
+            }
+
+        };
     }
 
     private static String createCompatibleHeader() {
         return "application/vnd.elasticsearch+json;compatible-with=" + CompatibleConstants.COMPATIBLE_VERSION;
     }
 
-
     private static Map<ClientYamlTestCandidate, ClientYamlTestCandidate> getLocalCompatibilityTests() throws Exception {
-        Iterable<Object[]> candidates =
-            ESClientYamlSuiteTestCase.createParameters(ExecutableSection.XCONTENT_REGISTRY, COMPAT_TESTS_PATH);
+        Iterable<Object[]> candidates = ESClientYamlSuiteTestCase.createParameters(ExecutableSection.XCONTENT_REGISTRY, COMPAT_TESTS_PATH);
         Map<ClientYamlTestCandidate, ClientYamlTestCandidate> localCompatibilityTests = new HashMap<>();
         StreamSupport.stream(candidates.spliterator(), false)
-            .flatMap(Arrays::stream).forEach(o -> localCompatibilityTests.put((ClientYamlTestCandidate) o, (ClientYamlTestCandidate) o));
+            .flatMap(Arrays::stream)
+            .forEach(o -> localCompatibilityTests.put((ClientYamlTestCandidate) o, (ClientYamlTestCandidate) o));
         return localCompatibilityTests;
     }
 }

server/src/main/java/org/elasticsearch/Version.java

@@ -342,6 +342,10 @@ public boolean isCompatible(Version version) {
         return compatible;
     }
 
+    public static Version minimumRestCompatibilityVersion(){
+        return Version.V_7_0_0;
+    }
+
     @SuppressForbidden(reason = "System.out.*")
     public static void main(String[] args) {
         final String versionOutput = String.format(

server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java

@@ -352,6 +352,9 @@ private void handleIncomingRequest(final HttpRequest httpRequest, final HttpChan
             } catch (final RestRequest.BadParameterException e) {
                 badRequestCause = ExceptionsHelper.useOrSuppress(badRequestCause, e);
                 innerRestRequest = RestRequest.requestWithoutParameters(xContentRegistry, httpRequest, httpChannel);
+            } catch (final RestRequest.CompatibleApiHeadersCombinationException e){
+                badRequestCause = ExceptionsHelper.useOrSuppress(badRequestCause, e);
+                innerRestRequest = RestRequest.requestNoValidation(xContentRegistry, httpRequest, httpChannel);
             }
             restRequest = innerRestRequest;
         }
@@ -384,12 +387,11 @@ private void handleIncomingRequest(final HttpRequest httpRequest, final HttpChan
     }
 
     private RestRequest requestWithoutContentTypeHeader(HttpRequest httpRequest, HttpChannel httpChannel, Exception badRequestCause) {
-        HttpRequest httpRequestWithoutContentType = httpRequest.removeHeader("Content-Type");
         try {
-            return RestRequest.request(xContentRegistry, httpRequestWithoutContentType, httpChannel);
+            return RestRequest.requestWithoutContentType(xContentRegistry, httpRequest, httpChannel);
         } catch (final RestRequest.BadParameterException e) {
             badRequestCause.addSuppressed(e);
-            return RestRequest.requestWithoutParameters(xContentRegistry, httpRequestWithoutContentType, httpChannel);
+            return RestRequest.requestNoValidation(xContentRegistry, httpRequest, httpChannel);
         }
     }
 }

server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java

@@ -26,8 +26,9 @@ public class CompatibleConstants {
     /**
      * TODO revisit when https://github.com/elastic/elasticsearch/issues/52370 is resolved
      */
-    public static final String COMPATIBLE_HEADER = "Accept";
+    public static final String COMPATIBLE_ACCEPT_HEADER = "Accept";
+    public static final String COMPATIBLE_CONTENT_TYPE_HEADER = "Content-Type";
     public static final String COMPATIBLE_PARAMS_KEY = "Compatible-With";
-    public static final String COMPATIBLE_VERSION = "" + (Version.CURRENT.major - 1);
+    public static final String COMPATIBLE_VERSION = String.valueOf(Version.minimumRestCompatibilityVersion().major);
 
 }