diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaType.java index 2d61120288706..2221f1bc97614 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaType.java @@ -19,34 +19,45 @@ package org.elasticsearch.common.xcontent; +import org.elasticsearch.common.collect.Tuple; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; + /** - * Abstracts a Media Type and a format parameter. + * Abstracts a Media Type and a query parameter format. * Media types are used as values on Content-Type and Accept headers * format is an URL parameter, specifies response media type. */ public interface MediaType { - /** - * Returns a type part of a MediaType - * i.e. application for application/json - */ - String type(); + String COMPATIBLE_WITH_PARAMETER_NAME = "compatible-with"; + String VERSION_PATTERN = "\\d+"; /** - * Returns a subtype part of a MediaType. - * i.e. json for application/json + * Returns a corresponding format path parameter for a MediaType. + * i.e. ?format=txt for plain/text media type */ - String subtype(); + String queryParameter(); /** - * Returns a corresponding format for a MediaType. i.e. json for application/json media type - * Can differ from the MediaType's subtype i.e plain/text has a subtype of text but format is txt + * Returns a set of HeaderValues - allowed media type values on Accept or Content-Type headers + * Also defines media type parameters for validation. */ - String format(); + Set headerValues(); /** - * returns a string representation of a media type. + * A class to represent supported mediaType values i.e. application/json and parameters to be validated. + * Parameters for validation is a map where a key is a parameter name, value is a parameter regex which is used for validation. + * Regex will be applied with case insensitivity. */ - default String typeWithSubtype(){ - return type() + "/" + subtype(); + class HeaderValue extends Tuple> { + public HeaderValue(String headerValue, Map parametersForValidation) { + super(headerValue, parametersForValidation); + } + + public HeaderValue(String headerValue) { + this(headerValue, Collections.emptyMap()); + } } } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaTypeParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaTypeParser.java deleted file mode 100644 index 62a3f3fd915d0..0000000000000 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaTypeParser.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.xcontent; - -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; -import java.util.regex.Pattern; - -public class MediaTypeParser { - private final Map formatToMediaType; - private final Map typeWithSubtypeToMediaType; - private final Map> parametersMap; - - public MediaTypeParser(Map formatToMediaType, Map typeWithSubtypeToMediaType, - Map> parametersMap) { - this.formatToMediaType = Map.copyOf(formatToMediaType); - this.typeWithSubtypeToMediaType = Map.copyOf(typeWithSubtypeToMediaType); - this.parametersMap = Map.copyOf(parametersMap); - } - - public T fromMediaType(String mediaType) { - ParsedMediaType parsedMediaType = parseMediaType(mediaType); - return parsedMediaType != null ? parsedMediaType.getMediaType() : null; - } - - public T fromFormat(String format) { - if (format == null) { - return null; - } - return formatToMediaType.get(format.toLowerCase(Locale.ROOT)); - } - - /** - * parsing media type that follows https://tools.ietf.org/html/rfc7231#section-3.1.1.1 - * - * @param headerValue a header value from Accept or Content-Type - * @return a parsed media-type - */ - public ParsedMediaType parseMediaType(String headerValue) { - if (headerValue != null) { - String[] split = headerValue.toLowerCase(Locale.ROOT).split(";"); - - String[] typeSubtype = split[0].trim().toLowerCase(Locale.ROOT) - .split("/"); - if (typeSubtype.length == 2) { - - String type = typeSubtype[0]; - String subtype = typeSubtype[1]; - String typeWithSubtype = type + "/" + subtype; - T xContentType = typeWithSubtypeToMediaType.get(typeWithSubtype); - if (xContentType != null) { - Map parameters = new HashMap<>(); - for (int i = 1; i < split.length; i++) { - //spaces are allowed between parameters, but not between '=' sign - String[] keyValueParam = split[i].trim().split("="); - if (keyValueParam.length != 2 || hasSpaces(keyValueParam[0]) || hasSpaces(keyValueParam[1])) { - return null; - } - String parameterName = keyValueParam[0].toLowerCase(Locale.ROOT); - String parameterValue = keyValueParam[1].toLowerCase(Locale.ROOT); - if (isValidParameter(typeWithSubtype, parameterName, parameterValue) == false) { - return null; - } - parameters.put(parameterName, parameterValue); - } - return new ParsedMediaType(xContentType, parameters); - } - } - - } - return null; - } - - private boolean isValidParameter(String typeWithSubtype, String parameterName, String parameterValue) { - if (parametersMap.containsKey(typeWithSubtype)) { - Map parameters = parametersMap.get(typeWithSubtype); - if (parameters.containsKey(parameterName)) { - Pattern regex = parameters.get(parameterName); - return regex.matcher(parameterValue).matches(); - } - } - return false; - } - - private boolean hasSpaces(String s) { - return s.trim().equals(s) == false; - } - - /** - * A media type object that contains all the information provided on a Content-Type or Accept header - */ - public class ParsedMediaType { - private final Map parameters; - private final T mediaType; - - public ParsedMediaType(T mediaType, Map parameters) { - this.parameters = parameters; - this.mediaType = mediaType; - } - - public T getMediaType() { - return mediaType; - } - - public Map getParameters() { - return parameters; - } - } - - public static class Builder { - private final Map formatToMediaType = new HashMap<>(); - private final Map typeWithSubtypeToMediaType = new HashMap<>(); - private final Map> parametersMap = new HashMap<>(); - - public Builder withMediaTypeAndParams(String alternativeMediaType, T mediaType, Map paramNameAndValueRegex) { - typeWithSubtypeToMediaType.put(alternativeMediaType.toLowerCase(Locale.ROOT), mediaType); - formatToMediaType.put(mediaType.format(), mediaType); - - Map parametersForMediaType = new HashMap<>(paramNameAndValueRegex.size()); - for (Map.Entry params : paramNameAndValueRegex.entrySet()) { - String parameterName = params.getKey().toLowerCase(Locale.ROOT); - String parameterRegex = params.getValue(); - Pattern pattern = Pattern.compile(parameterRegex, Pattern.CASE_INSENSITIVE); - parametersForMediaType.put(parameterName, pattern); - } - parametersMap.put(alternativeMediaType, parametersForMediaType); - - return this; - } - - public Builder copyFromMediaTypeParser(MediaTypeParser mediaTypeParser) { - formatToMediaType.putAll(mediaTypeParser.formatToMediaType); - typeWithSubtypeToMediaType.putAll(mediaTypeParser.typeWithSubtypeToMediaType); - parametersMap.putAll(mediaTypeParser.parametersMap); - return this; - } - - public MediaTypeParser build() { - return new MediaTypeParser<>(formatToMediaType, typeWithSubtypeToMediaType, parametersMap); - } - } -} diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaTypeRegistry.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaTypeRegistry.java new file mode 100644 index 0000000000000..8324adfa25384 --- /dev/null +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaTypeRegistry.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * A registry for quick media type lookup. + * It allows to find media type by a header value - typeWithSubtype aka mediaType without parameters. + * I.e. application/json will return XContentType.JSON + * Also allows to find media type by a path parameter format. + * I.e. txt used in path _sql?format=txt will return TextFormat.PLAIN_TEXT + * + * Multiple header representations may map to a single {@link MediaType} for example, "application/json" + * and "application/vnd.elasticsearch+json" both represent a JSON MediaType. + * A MediaType can have only one query parameter representation. + * For example "json" (case insensitive) maps back to a JSON media type. + * + * Additionally, a http header may optionally have parameters. For example "application/json; charset=utf-8". + * This class also allows to define a regular expression for valid values of charset. + */ +public class MediaTypeRegistry { + + private Map queryParamToMediaType = new HashMap<>(); + private Map typeWithSubtypeToMediaType = new HashMap<>(); + private Map> parametersMap = new HashMap<>(); + + public T queryParamToMediaType(String format) { + if (format == null) { + return null; + } + return queryParamToMediaType.get(format.toLowerCase(Locale.ROOT)); + } + + public T typeWithSubtypeToMediaType(String typeWithSubtype) { + return typeWithSubtypeToMediaType.get(typeWithSubtype.toLowerCase(Locale.ROOT)); + } + + public Map parametersFor(String typeWithSubtype) { + return parametersMap.get(typeWithSubtype); + } + + public MediaTypeRegistry register(T[] mediaTypes ) { + for (T mediaType : mediaTypes) { + Set tuples = mediaType.headerValues(); + for (MediaType.HeaderValue headerValue : tuples) { + queryParamToMediaType.put(mediaType.queryParameter(), mediaType); + typeWithSubtypeToMediaType.put(headerValue.v1(), mediaType); + parametersMap.put(headerValue.v1(), convertPatterns(headerValue.v2())); + } + } + return this; + } + + private Map convertPatterns(Map paramNameAndValueRegex) { + Map parametersForMediaType = new HashMap<>(paramNameAndValueRegex.size()); + for (Map.Entry params : paramNameAndValueRegex.entrySet()) { + String parameterName = params.getKey().toLowerCase(Locale.ROOT); + String parameterRegex = params.getValue(); + Pattern pattern = Pattern.compile(parameterRegex, Pattern.CASE_INSENSITIVE); + parametersForMediaType.put(parameterName, pattern); + } + return parametersForMediaType; + } +} diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java new file mode 100644 index 0000000000000..83676cbfa1902 --- /dev/null +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java @@ -0,0 +1,140 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * A raw result of parsing media types from Accept or Content-Type headers. + * It follow parsing and validates as per rules defined in https://tools.ietf.org/html/rfc7231#section-3.1.1.1 + * Can be resolved to MediaType + * @see MediaType + * @see MediaTypeRegistry + */ +public class ParsedMediaType { + // TODO this should be removed once strict parsing is implemented https://github.com/elastic/elasticsearch/issues/63080 + // sun.net.www.protocol.http.HttpURLConnection sets a default Accept header if it was not provided on a request. + // For this value Parsing returns null. + public static final String DEFAULT_ACCEPT_STRING = "text/html, image/gif, image/jpeg, *; q=.2, */*; q=.2"; + + private final String type; + private final String subType; + private final Map parameters; + // tchar pattern as defined by RFC7230 section 3.2.6 + private static final Pattern TCHAR_PATTERN = Pattern.compile("[a-zA-z0-9!#$%&'*+\\-.\\^_`|~]+"); + + private ParsedMediaType(String type, String subType, Map parameters) { + this.type = type; + this.subType = subType; + this.parameters = Collections.unmodifiableMap(parameters); + } + + /** + * The parsed mime type without the associated parameters. Will always return lowercase. + */ + public String mediaTypeWithoutParameters() { + return type + "/" + subType; + } + + public Map getParameters() { + return parameters; + } + + /** + * Parses a header value into it's parts. + * Note: parsing can return null, but it will throw exceptions once https://github.com/elastic/elasticsearch/issues/63080 is done + * Do not rely on nulls + * + * @return a {@link ParsedMediaType} if the header could be parsed. + * @throws IllegalArgumentException if the header is malformed + */ + public static ParsedMediaType parseMediaType(String headerValue) { + if (DEFAULT_ACCEPT_STRING.equals(headerValue) || "*/*".equals(headerValue)) { + return null; + } + if (headerValue != null) { + final String[] elements = headerValue.toLowerCase(Locale.ROOT).split("[\\s\\t]*;"); + + final String[] splitMediaType = elements[0].split("/"); + if ((splitMediaType.length == 2 && TCHAR_PATTERN.matcher(splitMediaType[0].trim()).matches() + && TCHAR_PATTERN.matcher(splitMediaType[1].trim()).matches()) == false) { + throw new IllegalArgumentException("invalid media type [" + headerValue + "]"); + } + if (elements.length == 1) { + return new ParsedMediaType(splitMediaType[0].trim(), splitMediaType[1].trim(), Collections.emptyMap()); + } else { + Map parameters = new HashMap<>(); + for (int i = 1; i < elements.length; i++) { + String paramsAsString = elements[i].trim(); + if (paramsAsString.isEmpty()) { + continue; + } + // intentionally allowing to have spaces around `=` + // https://tools.ietf.org/html/rfc7231#section-3.1.1.1 disallows this + String[] keyValueParam = elements[i].trim().split("="); + if (keyValueParam.length == 2) { + String parameterName = keyValueParam[0].toLowerCase(Locale.ROOT).trim(); + String parameterValue = keyValueParam[1].toLowerCase(Locale.ROOT).trim(); + parameters.put(parameterName, parameterValue); + } else { + throw new IllegalArgumentException("invalid parameters for header [" + headerValue + "]"); + } + } + return new ParsedMediaType(splitMediaType[0].trim().toLowerCase(Locale.ROOT), + splitMediaType[1].trim().toLowerCase(Locale.ROOT), parameters); + } + } + return null; + } + + /** + * Resolves this instance to a MediaType instance defined in given MediaTypeRegistry. + * Performs validation against parameters. + * @param mediaTypeRegistry a registry where a mapping between a raw media type to an instance MediaType is defined + * @return a MediaType instance or null if no media type could be found or if a known parameter do not passes validation + */ + public T toMediaType(MediaTypeRegistry mediaTypeRegistry) { + T type = mediaTypeRegistry.typeWithSubtypeToMediaType(mediaTypeWithoutParameters()); + if (type != null) { + + Map registeredParams = mediaTypeRegistry.parametersFor(mediaTypeWithoutParameters()); + for (Map.Entry givenParamEntry : parameters.entrySet()) { + if (isValidParameter(givenParamEntry.getKey(), givenParamEntry.getValue(), registeredParams) == false) { + return null; + } + } + return type; + } + return null; + } + + private boolean isValidParameter(String paramName, String value, Map registeredParams) { + if (registeredParams.containsKey(paramName)) { + Pattern regex = registeredParams.get(paramName); + return regex.matcher(value).matches(); + } + //TODO undefined parameters are allowed until https://github.com/elastic/elasticsearch/issues/63080 + return true; + } +} diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index 076a20bad006a..c1cd11aef5f19 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -24,8 +24,8 @@ import org.elasticsearch.common.xcontent.smile.SmileXContent; import org.elasticsearch.common.xcontent.yaml.YamlXContent; -import java.util.Collections; import java.util.Map; +import java.util.Set; /** * The content type of {@link org.elasticsearch.common.xcontent.XContent}. @@ -47,7 +47,7 @@ public String mediaType() { } @Override - public String subtype() { + public String queryParameter() { return "json"; } @@ -55,6 +55,18 @@ public String subtype() { public XContent xContent() { return JsonXContent.jsonXContent; } + + @Override + public Set headerValues() { + return Set.of( + new HeaderValue("application/json"), + new HeaderValue("application/x-ndjson"), + new HeaderValue("application/*"), + new HeaderValue(VENDOR_APPLICATION_PREFIX + "json", + Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN)), + new HeaderValue(VENDOR_APPLICATION_PREFIX + "x-ndjson", + Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN))); + } }, /** * The jackson based smile binary format. Fast and compact binary format. @@ -66,7 +78,7 @@ public String mediaTypeWithoutParameters() { } @Override - public String subtype() { + public String queryParameter() { return "smile"; } @@ -74,6 +86,14 @@ public String subtype() { public XContent xContent() { return SmileXContent.smileXContent; } + + @Override + public Set headerValues() { + return Set.of( + new HeaderValue("application/smile"), + new HeaderValue(VENDOR_APPLICATION_PREFIX + "smile", + Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN))); + } }, /** * A YAML based content type. @@ -85,7 +105,7 @@ public String mediaTypeWithoutParameters() { } @Override - public String subtype() { + public String queryParameter() { return "yaml"; } @@ -93,6 +113,14 @@ public String subtype() { public XContent xContent() { return YamlXContent.yamlXContent; } + + @Override + public Set headerValues() { + return Set.of( + new HeaderValue("application/yaml"), + new HeaderValue(VENDOR_APPLICATION_PREFIX + "yaml", + Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN))); + } }, /** * A CBOR based content type. @@ -104,7 +132,7 @@ public String mediaTypeWithoutParameters() { } @Override - public String subtype() { + public String queryParameter() { return "cbor"; } @@ -112,37 +140,28 @@ public String subtype() { public XContent xContent() { return CborXContent.cborXContent; } + + @Override + public Set headerValues() { + return Set.of( + new HeaderValue("application/cbor"), + new HeaderValue(VENDOR_APPLICATION_PREFIX + "cbor", + Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN))); + } }; - private static final String COMPATIBLE_WITH_PARAMETER_NAME = "compatible-with"; - private static final String VERSION_PATTERN = "\\d+"; - public static final MediaTypeParser mediaTypeParser = new MediaTypeParser.Builder() - .withMediaTypeAndParams("application/smile", SMILE, Collections.emptyMap()) - .withMediaTypeAndParams("application/cbor", CBOR, Collections.emptyMap()) - .withMediaTypeAndParams("application/json", JSON, Map.of("charset", "UTF-8")) - .withMediaTypeAndParams("application/yaml", YAML, Map.of("charset", "UTF-8")) - .withMediaTypeAndParams("application/*", JSON, Map.of("charset", "UTF-8")) - .withMediaTypeAndParams("application/x-ndjson", JSON, Map.of("charset", "UTF-8")) - .withMediaTypeAndParams("application/vnd.elasticsearch+json", JSON, - Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN, "charset", "UTF-8")) - .withMediaTypeAndParams("application/vnd.elasticsearch+smile", SMILE, - Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN, "charset", "UTF-8")) - .withMediaTypeAndParams("application/vnd.elasticsearch+yaml", YAML, - Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN, "charset", "UTF-8")) - .withMediaTypeAndParams("application/vnd.elasticsearch+cbor", CBOR, - Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN, "charset", "UTF-8")) - .withMediaTypeAndParams("application/vnd.elasticsearch+x-ndjson", JSON, - Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN, "charset", "UTF-8")) - .build(); + public static final MediaTypeRegistry MEDIA_TYPE_REGISTRY = new MediaTypeRegistry() + .register(XContentType.values()); + public static final String VENDOR_APPLICATION_PREFIX = "application/vnd.elasticsearch+"; /** - * Accepts a format string, which is most of the time is equivalent to {@link XContentType#subtype()} + * Accepts a format string, which is most of the time is equivalent to MediaType's subtype i.e. application/json * and attempts to match the value to an {@link XContentType}. * The comparisons are done in lower case format. * This method will return {@code null} if no match is found */ - public static XContentType fromFormat(String mediaType) { - return mediaTypeParser.fromFormat(mediaType); + public static XContentType fromFormat(String format) { + return MEDIA_TYPE_REGISTRY.queryParamToMediaType(format); } /** @@ -151,8 +170,13 @@ public static XContentType fromFormat(String mediaType) { * This method is suitable for parsing of the {@code Content-Type} and {@code Accept} HTTP headers. * This method will return {@code null} if no match is found */ - public static XContentType fromMediaType(String mediaTypeHeaderValue) { - return mediaTypeParser.fromMediaType(mediaTypeHeaderValue); + public static XContentType fromMediaType(String mediaTypeHeaderValue) throws IllegalArgumentException { + ParsedMediaType parsedMediaType = ParsedMediaType.parseMediaType(mediaTypeHeaderValue); + if (parsedMediaType != null) { + return parsedMediaType + .toMediaType(MEDIA_TYPE_REGISTRY); + } + return null; } private int index; @@ -162,7 +186,7 @@ public static XContentType fromMediaType(String mediaTypeHeaderValue) { } public static Byte parseVersion(String mediaType) { - MediaTypeParser.ParsedMediaType parsedMediaType = mediaTypeParser.parseMediaType(mediaType); + ParsedMediaType parsedMediaType = ParsedMediaType.parseMediaType(mediaType); if (parsedMediaType != null) { String version = parsedMediaType .getParameters() @@ -180,19 +204,7 @@ public String mediaType() { return mediaTypeWithoutParameters(); } - public abstract XContent xContent(); public abstract String mediaTypeWithoutParameters(); - - - @Override - public String type() { - return "application"; - } - - @Override - public String format() { - return subtype(); - } } diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MediaTypeParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MediaTypeParserTests.java deleted file mode 100644 index 08ca08a3d2240..0000000000000 --- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MediaTypeParserTests.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.xcontent; - -import org.elasticsearch.test.ESTestCase; - -import java.util.Collections; -import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; - -public class MediaTypeParserTests extends ESTestCase { - - MediaTypeParser mediaTypeParser = new MediaTypeParser.Builder() - .withMediaTypeAndParams("application/vnd.elasticsearch+json", - XContentType.JSON, Map.of("compatible-with", "\\d+", - "charset", "UTF-8")) - .build(); - - public void testJsonWithParameters() throws Exception { - String mediaType = "application/vnd.elasticsearch+json"; - assertThat(mediaTypeParser.parseMediaType(mediaType).getParameters(), - equalTo(Collections.emptyMap())); - assertThat(mediaTypeParser.parseMediaType(mediaType + ";").getParameters(), - equalTo(Collections.emptyMap())); - assertThat(mediaTypeParser.parseMediaType(mediaType + "; charset=UTF-8").getParameters(), - equalTo(Map.of("charset", "utf-8"))); - assertThat(mediaTypeParser.parseMediaType(mediaType + "; compatible-with=123;charset=UTF-8").getParameters(), - equalTo(Map.of("charset", "utf-8", "compatible-with", "123"))); - } - - public void testWhiteSpaceInTypeSubtype() { - String mediaType = " application/vnd.elasticsearch+json "; - assertThat(mediaTypeParser.parseMediaType(mediaType).getMediaType(), - equalTo(XContentType.JSON)); - - assertThat(mediaTypeParser.parseMediaType(mediaType + "; compatible-with=123; charset=UTF-8").getParameters(), - equalTo(Map.of("charset", "utf-8", "compatible-with", "123"))); - assertThat(mediaTypeParser.parseMediaType(mediaType + "; compatible-with=123;\n charset=UTF-8").getParameters(), - equalTo(Map.of("charset", "utf-8", "compatible-with", "123"))); - - mediaType = " application / json "; - assertThat(mediaTypeParser.parseMediaType(mediaType), - is(nullValue())); - } - - public void testInvalidParameters() { - String mediaType = "application/vnd.elasticsearch+json"; - assertThat(mediaTypeParser.parseMediaType(mediaType + "; charset=unknown") , - is(nullValue())); - assertThat(mediaTypeParser.parseMediaType(mediaType + "; keyvalueNoEqualsSign"), - is(nullValue())); - assertThat(mediaTypeParser.parseMediaType(mediaType + "; key = value"), - is(nullValue())); - assertThat(mediaTypeParser.parseMediaType(mediaType + "; key=") , - is(nullValue())); - } -} diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ParsedMediaTypeTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ParsedMediaTypeTests.java new file mode 100644 index 0000000000000..15b01014830ba --- /dev/null +++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ParsedMediaTypeTests.java @@ -0,0 +1,120 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent; + +import org.elasticsearch.test.ESTestCase; + +import java.util.Collections; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +public class ParsedMediaTypeTests extends ESTestCase { + + MediaTypeRegistry mediaTypeRegistry = new MediaTypeRegistry() + .register(XContentType.values()); + + public void testJsonWithParameters() throws Exception { + String mediaType = "application/vnd.elasticsearch+json"; + assertThat(ParsedMediaType.parseMediaType(mediaType).getParameters(), + equalTo(Collections.emptyMap())); + assertThat(ParsedMediaType.parseMediaType(mediaType + ";").getParameters(), + equalTo(Collections.emptyMap())); + assertThat(ParsedMediaType.parseMediaType(mediaType + "; charset=UTF-8").getParameters(), + equalTo(Map.of("charset", "utf-8"))); + assertThat(ParsedMediaType.parseMediaType(mediaType + "; compatible-with=123;charset=UTF-8").getParameters(), + equalTo(Map.of("charset", "utf-8", "compatible-with", "123"))); + } + + public void testWhiteSpaceInTypeSubtype() { + String mediaType = " application/vnd.elasticsearch+json "; + assertThat(ParsedMediaType.parseMediaType(mediaType).toMediaType(mediaTypeRegistry), + equalTo(XContentType.JSON)); + + assertThat(ParsedMediaType.parseMediaType(mediaType + "; compatible-with=123; charset=UTF-8").getParameters(), + equalTo(Map.of("charset", "utf-8", "compatible-with", "123"))); + assertThat(ParsedMediaType.parseMediaType(mediaType + "; compatible-with=123;\n charset=UTF-8").getParameters(), + equalTo(Map.of("charset", "utf-8", "compatible-with", "123"))); + } + + public void testInvalidParameters() { + String mediaType = "application/vnd.elasticsearch+json"; + expectThrows(IllegalArgumentException.class, () -> ParsedMediaType.parseMediaType(mediaType + "; keyvalueNoEqualsSign") + .toMediaType(mediaTypeRegistry)); + + expectThrows(IllegalArgumentException.class, () -> ParsedMediaType.parseMediaType(mediaType + "; key=") + .toMediaType(mediaTypeRegistry)); + } + + public void testXContentTypes() { + for (XContentType xContentType : XContentType.values()) { + ParsedMediaType parsedMediaType = ParsedMediaType.parseMediaType(xContentType.mediaTypeWithoutParameters()); + assertEquals(xContentType.mediaTypeWithoutParameters(), parsedMediaType.mediaTypeWithoutParameters()); + } + } + + public void testWithParameters() { + String mediaType = "application/foo"; + assertEquals(Collections.emptyMap(), ParsedMediaType.parseMediaType(mediaType).getParameters()); + assertEquals(Collections.emptyMap(), ParsedMediaType.parseMediaType(mediaType + ";").getParameters()); + assertEquals(Map.of("charset", "utf-8"), ParsedMediaType.parseMediaType(mediaType + "; charset=UTF-8").getParameters()); + assertEquals(Map.of("charset", "utf-8", "compatible-with", "123"), + ParsedMediaType.parseMediaType(mediaType + "; compatible-with=123;charset=UTF-8").getParameters()); + } + + public void testWhiteSpaces() { + //be lenient with white space since it can be really hard to troubleshoot + String mediaType = " application/foo "; + ParsedMediaType parsedMediaType = ParsedMediaType.parseMediaType(mediaType + " ; compatible-with = 123 ; charset=UTF-8"); + assertEquals("application/foo", parsedMediaType.mediaTypeWithoutParameters()); + assertEquals((Map.of("charset", "utf-8", "compatible-with", "123")), parsedMediaType.getParameters()); + } + + public void testEmptyParams() { + String mediaType = "application/foo"; + ParsedMediaType parsedMediaType = ParsedMediaType.parseMediaType(mediaType + randomFrom("", " ", ";", ";;", ";;;")); + assertEquals("application/foo", parsedMediaType.mediaTypeWithoutParameters()); + assertEquals(Collections.emptyMap(), parsedMediaType.getParameters()); + } + + public void testMalformedParameters() { + String mediaType = "application/foo"; + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> ParsedMediaType.parseMediaType(mediaType + "; charsetunknown")); + assertThat(exception.getMessage(), equalTo("invalid parameters for header [application/foo; charsetunknown]")); + + exception = expectThrows(IllegalArgumentException.class, + () -> ParsedMediaType.parseMediaType(mediaType + "; char=set=unknown")); + assertThat(exception.getMessage(), equalTo("invalid parameters for header [application/foo; char=set=unknown]")); + } + + public void testDefaultAcceptHeader() { + // This media type is defined in sun.net.www.protocol.http.HttpURLConnection as a default Accept header + // and used when a header was not set on a request + // It should be treated as if a user did not specify a header value + String mediaType = "text/html, image/gif, image/jpeg, *; q=.2, */*; q=.2"; + assertThat(ParsedMediaType.parseMediaType(mediaType), is(nullValue())); + + // When using curl */* is used a default Accept header when not specified by a user + assertThat(ParsedMediaType.parseMediaType("*/*"), is(nullValue())); + } +} diff --git a/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4BadRequestIT.java b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4BadRequestIT.java index cfda71f10096e..6afee02d9ebb8 100644 --- a/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4BadRequestIT.java +++ b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4BadRequestIT.java @@ -98,7 +98,7 @@ public void testInvalidHeaderValue() throws IOException { assertThat(response.getStatusLine().getStatusCode(), equalTo(400)); final ObjectPath objectPath = ObjectPath.createFromResponse(response); final Map map = objectPath.evaluate("error"); - assertThat(map.get("type"), equalTo("content_type_header_exception")); - assertThat(map.get("reason"), equalTo("java.lang.IllegalArgumentException: invalid Content-Type header []")); + assertThat(map.get("type"), equalTo("media_type_header_exception")); + assertThat(map.get("reason"), equalTo("java.lang.IllegalArgumentException: Header [Content-Type] cannot be empty.")); } } diff --git a/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java index af8095d6dece1..c033ac9bfc62c 100644 --- a/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java +++ b/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java @@ -345,7 +345,7 @@ private void handleIncomingRequest(final HttpRequest httpRequest, final HttpChan RestRequest innerRestRequest; try { innerRestRequest = RestRequest.request(xContentRegistry, httpRequest, httpChannel); - } catch (final RestRequest.ContentTypeHeaderException e) { + } catch (final RestRequest.MediaTypeHeaderException e) { badRequestCause = ExceptionsHelper.useOrSuppress(badRequestCause, e); innerRestRequest = requestWithoutContentTypeHeader(httpRequest, httpChannel, badRequestCause); } catch (final RestRequest.BadParameterException e) { diff --git a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index 6f5aa618ae4d0..f08bf5b58e8b4 100644 --- a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -104,7 +104,7 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nu if (Strings.hasText(format)) { responseContentType = XContentType.fromFormat(format); } - if (responseContentType == null) { + if (responseContentType == null && Strings.hasText(acceptHeader)) { responseContentType = XContentType.fromMediaType(acceptHeader); } } diff --git a/server/src/main/java/org/elasticsearch/rest/RestHandler.java b/server/src/main/java/org/elasticsearch/rest/RestHandler.java index 054c618876314..aa9393d7aa834 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/RestHandler.java @@ -20,7 +20,10 @@ package org.elasticsearch.rest; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.xcontent.MediaType; +import org.elasticsearch.common.xcontent.MediaTypeRegistry; import org.elasticsearch.common.xcontent.XContent; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestRequest.Method; import java.util.Collections; @@ -99,6 +102,10 @@ default boolean allowSystemIndexAccessByDefault() { return false; } + default MediaTypeRegistry validAcceptMediaTypes() { + return XContentType.MEDIA_TYPE_REGISTRY; + } + class Route { private final String path; diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java index 512bf72e9c0d3..49a8eabef5f46 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ParsedMediaType; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; @@ -68,7 +69,8 @@ public class RestRequest implements ToXContent.Params { private final Set consumedParams = new HashSet<>(); private final SetOnce xContentType = new SetOnce<>(); private final HttpChannel httpChannel; - + private final ParsedMediaType parsedAccept; + private final ParsedMediaType parsedContentType; private HttpRequest httpRequest; private boolean contentConsumed = false; @@ -86,14 +88,14 @@ protected RestRequest(NamedXContentRegistry xContentRegistry, Map params, String path, Map> headers, HttpRequest httpRequest, HttpChannel httpChannel, long requestId) { - final XContentType xContentType; try { - xContentType = parseContentType(headers.get("Content-Type")); - } catch (final IllegalArgumentException e) { - throw new ContentTypeHeaderException(e); - } - if (xContentType != null) { - this.xContentType.set(xContentType); + this.parsedAccept = parseHeaderWithMediaType(httpRequest.getHeaders(), "Accept"); + this.parsedContentType = parseHeaderWithMediaType(httpRequest.getHeaders(), "Content-Type"); + if (parsedContentType != null) { + this.xContentType.set(parsedContentType.toMediaType(XContentType.MEDIA_TYPE_REGISTRY)); + } + } catch (IllegalArgumentException e) { + throw new MediaTypeHeaderException(e); } this.xContentRegistry = xContentRegistry; this.httpRequest = httpRequest; @@ -104,6 +106,23 @@ private RestRequest(NamedXContentRegistry xContentRegistry, Map this.requestId = requestId; } + private static @Nullable ParsedMediaType parseHeaderWithMediaType(Map> headers, String headerName) { + //TODO: make all usages of headers case-insensitive + List header = headers.get(headerName); + if (header == null || header.isEmpty()) { + return null; + } else if (header.size() > 1) { + throw new IllegalArgumentException("Incorrect header [" + headerName + "]. " + + "Only one value should be provided"); + } + String rawContentType = header.get(0); + if (Strings.hasText(rawContentType)) { + return ParsedMediaType.parseMediaType(rawContentType); + } else { + throw new IllegalArgumentException("Header [" + headerName + "] cannot be empty."); + } + } + protected RestRequest(RestRequest restRequest) { this(restRequest.getXContentRegistry(), restRequest.params(), restRequest.path(), restRequest.getHeaders(), restRequest.getHttpRequest(), restRequest.getHttpChannel(), restRequest.getRequestId()); @@ -126,7 +145,7 @@ void ensureSafeBuffers() { * @param httpRequest the http request * @param httpChannel the http channel * @throws BadParameterException if the parameters can not be decoded - * @throws ContentTypeHeaderException if the Content-Type header can not be parsed + * @throws MediaTypeHeaderException if the Content-Type or Accept header can not be parsed */ public static RestRequest request(NamedXContentRegistry xContentRegistry, HttpRequest httpRequest, HttpChannel httpChannel) { Map params = params(httpRequest.uri()); @@ -164,7 +183,7 @@ private static String path(final String uri) { * @param xContentRegistry the content registry * @param httpRequest the http request * @param httpChannel the http channel - * @throws ContentTypeHeaderException if the Content-Type header can not be parsed + * @throws MediaTypeHeaderException if the Content-Type or Accept header can not be parsed */ public static RestRequest requestWithoutParameters(NamedXContentRegistry xContentRegistry, HttpRequest httpRequest, HttpChannel httpChannel) { @@ -497,6 +516,14 @@ public final Tuple contentOrSourceParam() { return new Tuple<>(xContentType, bytes); } + public ParsedMediaType getParsedAccept() { + return parsedAccept; + } + + public ParsedMediaType getParsedContentType() { + return parsedContentType; + } + /** * Parses the given content type string for the media type. This method currently ignores parameters. */ @@ -522,9 +549,9 @@ public static XContentType parseContentType(List header) { throw new IllegalArgumentException("empty Content-Type header"); } - public static class ContentTypeHeaderException extends RuntimeException { + public static class MediaTypeHeaderException extends RuntimeException { - ContentTypeHeaderException(final IllegalArgumentException cause) { + MediaTypeHeaderException(final IllegalArgumentException cause) { super(cause); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java index bbb746baafe92..645d3afdcd18b 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java @@ -51,18 +51,21 @@ public class RestTable { public static RestResponse buildResponse(Table table, RestChannel channel) throws Exception { RestRequest request = channel.request(); - XContentType xContentType = getXContentType(request); + XContentType xContentType = getResponseContentType(request); if (xContentType != null) { return buildXContentBuilder(table, channel); } return buildTextPlainResponse(table, channel); } - private static XContentType getXContentType(RestRequest request) { + private static XContentType getResponseContentType(RestRequest request) { if (request.hasParam("format")) { return XContentType.fromFormat(request.param("format")); } - return XContentType.fromMediaType(request.header("Accept")); + if (request.getParsedAccept() != null) { + return request.getParsedAccept().toMediaType(XContentType.MEDIA_TYPE_REGISTRY); + } + return null; } public static RestResponse buildXContentBuilder(Table table, RestChannel channel) throws Exception { diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java index 7b18eaa2bad38..72bdf549b8d40 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java @@ -91,9 +91,9 @@ public void testFromWildcardUppercase() throws Exception { public void testFromRubbish() throws Exception { assertThat(XContentType.fromMediaType(null), nullValue()); - assertThat(XContentType.fromMediaType(""), nullValue()); + expectThrows(IllegalArgumentException.class, ()->XContentType.fromMediaType("")); + expectThrows(IllegalArgumentException.class, ()->XContentType.fromMediaType("gobbly;goop")); assertThat(XContentType.fromMediaType("text/plain"), nullValue()); - assertThat(XContentType.fromMediaType("gobbly;goop"), nullValue()); } public void testVersionedMediaType() { @@ -137,23 +137,21 @@ public void testVersionParsing() { assertThat(XContentType.parseVersion("APPLICATION/JSON"), nullValue()); - assertThat(XContentType.parseVersion("application/json;compatible-with=" + version + ".0"), + //validation is done when parsing a MediaType + assertThat(XContentType.fromMediaType("application/vnd.elasticsearch+json;compatible-with=" + version + ".0"), is(nullValue())); + assertThat(XContentType.fromMediaType("application/vnd.elasticsearch+json;compatible-with=" + version + "_sth"), + nullValue()); } - public void testUnrecognizedParameter() { - assertThat(XContentType.parseVersion("application/json; sth=123"), - is(nullValue())); } - - public void testMediaTypeWithoutESSubtype() { + public void testUnrecognizedParameters() { + //unrecognised parameters are ignored String version = String.valueOf(randomNonNegativeByte()); - assertThat(XContentType.fromMediaType("application/json;compatible-with=" + version), nullValue()); - } - public void testAnchoring() { - String version = String.valueOf(randomNonNegativeByte()); - assertThat(XContentType.fromMediaType("sth_application/json;compatible-with=" + version + ".0"), nullValue()); - assertThat(XContentType.fromMediaType("sth_application/json;compatible-with=" + version + "_sth"), nullValue()); - assertThat(XContentType.fromMediaType("application/json;compatible-with=" + version + "_sth"), nullValue()); + assertThat(XContentType.fromMediaType("application/json;compatible-with=" + version), + is(XContentType.JSON)); + // TODO do not allow parsing unrecognized parameter value https://github.com/elastic/elasticsearch/issues/63080 + // assertThat(XContentType.parseVersion("application/json;compatible-with=123"), + // is(nullValue())); } } diff --git a/server/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java b/server/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java index 8eff3b23dc5cb..e5ed811643d5a 100644 --- a/server/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java +++ b/server/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java @@ -299,7 +299,7 @@ public void testErrorToAndFromXContent() throws IOException { final XContentType xContentType = randomFrom(XContentType.values()); - Map params = Collections.singletonMap("format", xContentType.format()); + Map params = Collections.singletonMap("format", xContentType.queryParameter()); RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build(); RestChannel channel = detailed ? new DetailedExceptionRestChannel(request) : new SimpleExceptionRestChannel(request); diff --git a/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java b/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java index 487bbed5a5999..20fa04b72573b 100644 --- a/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java +++ b/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java @@ -199,15 +199,15 @@ public void testPlainTextSupport() { public void testMalformedContentTypeHeader() { final String type = randomFrom("text", "text/:ain; charset=utf-8", "text/plain\";charset=utf-8", ":", "/", "t:/plain"); - final RestRequest.ContentTypeHeaderException e = expectThrows( - RestRequest.ContentTypeHeaderException.class, + final RestRequest.MediaTypeHeaderException e = expectThrows( + RestRequest.MediaTypeHeaderException.class, () -> { final Map> headers = Collections.singletonMap("Content-Type", Collections.singletonList(type)); contentRestRequest("", Collections.emptyMap(), headers); }); assertNotNull(e.getCause()); assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); - assertThat(e.getMessage(), equalTo("java.lang.IllegalArgumentException: invalid Content-Type header [" + type + "]")); + assertThat(e.getMessage(), equalTo("java.lang.IllegalArgumentException: invalid media type [" + type + "]")); } public void testNoContentTypeHeader() { @@ -217,12 +217,13 @@ public void testNoContentTypeHeader() { public void testMultipleContentTypeHeaders() { List headers = new ArrayList<>(randomUnique(() -> randomAlphaOfLengthBetween(1, 16), randomIntBetween(2, 10))); - final RestRequest.ContentTypeHeaderException e = expectThrows( - RestRequest.ContentTypeHeaderException.class, + final RestRequest.MediaTypeHeaderException e = expectThrows( + RestRequest.MediaTypeHeaderException.class, () -> contentRestRequest("", Collections.emptyMap(), Collections.singletonMap("Content-Type", headers))); assertNotNull(e.getCause()); assertThat(e.getCause(), instanceOf((IllegalArgumentException.class))); - assertThat(e.getMessage(), equalTo("java.lang.IllegalArgumentException: only one Content-Type header should be provided")); + assertThat(e.getMessage(), equalTo("java.lang.IllegalArgumentException: Incorrect header [Content-Type]." + + " Only one value should be provided")); } public void testRequiredContent() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java index 9f5d0dfe9dcd7..5e000d38c0531 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java @@ -53,7 +53,8 @@ public ClientYamlTestResponse(Response response) throws IOException { this.response = response; if (response.getEntity() != null) { String contentType = response.getHeader("Content-Type"); - this.bodyContentType = XContentType.fromMediaType(contentType); + + this.bodyContentType = getContentTypeIgnoreExceptions(contentType); try { byte[] bytes = EntityUtils.toByteArray(response.getEntity()); //skip parsing if we got text back (e.g. if we called _cat apis) @@ -71,6 +72,20 @@ public ClientYamlTestResponse(Response response) throws IOException { } } + /** + * A content type returned on a response can be a media type defined outside XContentType (for instance plain/text, plain/csv etc). + * This means that the response cannot be parsed.DefaultHttpHeaders + * Also in testing there is no access to media types defined outside of XContentType. + * Therefore a null has to be returned if a response content-type has a mediatype not defined in XContentType. + */ + private XContentType getContentTypeIgnoreExceptions(String contentType) { + try { + return XContentType.fromMediaType(contentType); + } catch (IllegalArgumentException e) { + return null; + } + } + public int getStatusCode() { return response.getStatusLine().getStatusCode(); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java index 69c444aa9bf4d..516c1e800e8b7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java @@ -13,6 +13,8 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.MediaType; +import org.elasticsearch.common.xcontent.MediaTypeRegistry; import org.elasticsearch.http.HttpChannel; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.rest.BytesRestResponse; @@ -142,4 +144,9 @@ private RestRequest maybeWrapRestRequest(RestRequest restRequest) throws IOExcep } return restRequest; } + + @Override + public MediaTypeRegistry validAcceptMediaTypes() { + return restHandler.validAcceptMediaTypes(); + } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java index 763e460170cf0..7243b8b8ad49e 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java @@ -8,6 +8,7 @@ import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.xcontent.MediaType; +import org.elasticsearch.common.xcontent.MediaTypeRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; @@ -44,6 +45,10 @@ public List routes() { new Route(POST, Protocol.SQL_QUERY_REST_ENDPOINT)); } + public MediaTypeRegistry validAcceptMediaTypes() { + return SqlMediaTypeParser.MEDIA_TYPE_REGISTRY; + } + @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { @@ -84,9 +89,6 @@ public RestResponse buildResponse(SqlQueryResponse response) throws Exception { }); } - - - @Override protected Set responseParams() { return responseMediaType == TextFormat.CSV ? Collections.singleton(URL_PARAM_DELIMITER) : Collections.emptySet(); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlMediaTypeParser.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlMediaTypeParser.java index 189dc137b654c..560ad710dfd94 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlMediaTypeParser.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlMediaTypeParser.java @@ -7,27 +7,19 @@ package org.elasticsearch.xpack.sql.plugin; import org.elasticsearch.common.xcontent.MediaType; -import org.elasticsearch.common.xcontent.MediaTypeParser; +import org.elasticsearch.common.xcontent.MediaTypeRegistry; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.xpack.sql.action.SqlQueryRequest; import org.elasticsearch.xpack.sql.proto.Mode; -import java.util.Map; import static org.elasticsearch.xpack.sql.proto.Protocol.URL_PARAM_FORMAT; public class SqlMediaTypeParser { - private static final MediaTypeParser parser = new MediaTypeParser.Builder<>() - .copyFromMediaTypeParser(XContentType.mediaTypeParser) - .withMediaTypeAndParams(TextFormat.PLAIN_TEXT.typeWithSubtype(), TextFormat.PLAIN_TEXT, - Map.of("header", "present|absent", "charset", "utf-8")) - .withMediaTypeAndParams(TextFormat.CSV.typeWithSubtype(), TextFormat.CSV, - Map.of("header", "present|absent", "charset", "utf-8", - "delimiter", ".+"))// more detailed parsing is in TextFormat.CSV#delimiter - .withMediaTypeAndParams(TextFormat.TSV.typeWithSubtype(), TextFormat.TSV, - Map.of("header", "present|absent", "charset", "utf-8")) - .build(); + public static final MediaTypeRegistry MEDIA_TYPE_REGISTRY = new MediaTypeRegistry<>() + .register(XContentType.values()) + .register(TextFormat.values()); /* * Since we support {@link TextFormat} and @@ -42,25 +34,19 @@ public class SqlMediaTypeParser { * isn't then we use the {@code Content-Type} header which is required. */ public MediaType getMediaType(RestRequest request, SqlQueryRequest sqlRequest) { - if (Mode.isDedicatedClient(sqlRequest.requestInfo().mode()) && (sqlRequest.binaryCommunication() == null || sqlRequest.binaryCommunication())) { // enforce CBOR response for drivers and CLI (unless instructed differently through the config param) return XContentType.CBOR; } else if (request.hasParam(URL_PARAM_FORMAT)) { - return validateColumnarRequest(sqlRequest.columnar(), parser.fromFormat(request.param(URL_PARAM_FORMAT))); - } - if (request.getHeaders().containsKey("Accept")) { - String accept = request.header("Accept"); - // */* means "I don't care" which we should treat like not specifying the header - if ("*/*".equals(accept) == false) { - return validateColumnarRequest(sqlRequest.columnar(), parser.fromMediaType(accept)); - } + return validateColumnarRequest(sqlRequest.columnar(), + MEDIA_TYPE_REGISTRY.queryParamToMediaType(request.param(URL_PARAM_FORMAT))); } - String contentType = request.header("Content-Type"); - assert contentType != null : "The Content-Type header is required"; - return validateColumnarRequest(sqlRequest.columnar(), parser.fromMediaType(contentType)); + if (request.getParsedAccept() != null) { + return request.getParsedAccept().toMediaType(MEDIA_TYPE_REGISTRY); + } + return request.getXContentType(); } private static MediaType validateColumnarRequest(boolean requestIsColumnar, MediaType fromMediaType) { @@ -70,5 +56,4 @@ private static MediaType validateColumnarRequest(boolean requestIsColumnar, Medi } return fromMediaType; } - } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TextFormat.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TextFormat.java index d397d2316959c..6b771dcfd7ca3 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TextFormat.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TextFormat.java @@ -24,7 +24,9 @@ import java.time.ZonedDateTime; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.Function; import static org.elasticsearch.xpack.sql.action.BasicFormatter.FormatOption.TEXT; @@ -83,7 +85,7 @@ String format(RestRequest request, SqlQueryResponse response) { } @Override - public String format() { + public String queryParameter() { return FORMAT_TEXT; } @@ -103,11 +105,14 @@ protected String eol() { } @Override - public String subtype() { - return "plain"; + public Set headerValues() { + return Set.of( + new HeaderValue(CONTENT_TYPE_TXT, + Map.of("header", "present|absent")), + new HeaderValue(VENDOR_CONTENT_TYPE_TXT, + Map.of("header", "present|absent", COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN))); } - }, /** @@ -132,7 +137,7 @@ protected String eol() { } @Override - public String format() { + public String queryParameter() { return FORMAT_CSV; } @@ -224,11 +229,16 @@ boolean hasHeader(RestRequest request) { } @Override - public String subtype() { - return "csv"; + public Set headerValues() { + return Set.of( + new HeaderValue(CONTENT_TYPE_CSV, + Map.of("header", "present|absent","delimiter", ".+")),// more detailed parsing is in TextFormat.CSV#delimiter + new HeaderValue(VENDOR_CONTENT_TYPE_CSV, + Map.of("header", "present|absent","delimiter", ".+", COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN))); } }, + TSV() { @Override protected Character delimiter() { @@ -242,7 +252,7 @@ protected String eol() { } @Override - public String format() { + public String queryParameter() { return FORMAT_TSV; } @@ -278,8 +288,11 @@ String maybeEscape(String value, Character __) { } @Override - public String subtype() { - return "tab-separated-values"; + public Set headerValues() { + return Set.of( + new HeaderValue(CONTENT_TYPE_TSV, Map.of("header", "present|absent")), + new HeaderValue(VENDOR_CONTENT_TYPE_TSV, + Map.of("header", "present|absent", COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN))); } }; @@ -287,8 +300,11 @@ public String subtype() { private static final String FORMAT_CSV = "csv"; private static final String FORMAT_TSV = "tsv"; private static final String CONTENT_TYPE_TXT = "text/plain"; + private static final String VENDOR_CONTENT_TYPE_TXT = "text/vnd.elasticsearch+plain"; private static final String CONTENT_TYPE_CSV = "text/csv"; + private static final String VENDOR_CONTENT_TYPE_CSV = "text/vnd.elasticsearch+csv"; private static final String CONTENT_TYPE_TSV = "text/tab-separated-values"; + private static final String VENDOR_CONTENT_TYPE_TSV = "text/vnd.elasticsearch+tab-separated-values"; private static final String URL_PARAM_HEADER = "header"; private static final String PARAM_HEADER_ABSENT = "absent"; private static final String PARAM_HEADER_PRESENT = "present"; @@ -358,15 +374,4 @@ protected Character delimiter(RestRequest request) { String maybeEscape(String value, Character delimiter) { return value; } - - - @Override - public String type() { - return "text"; - } - - @Override - public String typeWithSubtype() { - return contentType(); - } } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpResponse.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpResponse.java index 6fc19feb8ffd4..e89e049ffe482 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpResponse.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpResponse.java @@ -105,7 +105,12 @@ public XContentType xContentType() { if (values == null || values.length == 0) { return null; } - return XContentType.fromMediaType(values[0]); + try { + return XContentType.fromMediaType(values[0]); + } catch (IllegalArgumentException e) { + //HttpInputTests - content-type being unrecognized_content_type + return null; + } } @Override diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/ExecutableHttpInput.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/ExecutableHttpInput.java index d484ce1e93272..39d5b2a1cd7d3 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/ExecutableHttpInput.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/ExecutableHttpInput.java @@ -99,7 +99,7 @@ HttpInput.Result doExecute(WatchExecutionContext ctx, HttpRequest request) throw } } catch (Exception e) { throw new ElasticsearchParseException("could not parse response body [{}] it does not appear to be [{}]", type(), ctx.id(), - response.body().utf8ToString(), contentType.format()); + response.body().utf8ToString(), contentType.queryParameter()); } } else { payloadMap.put("_value", response.body().utf8ToString()); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java index 1b27e8151ecb2..bf0f3d4d82dfb 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java @@ -160,7 +160,7 @@ static String prepareTemplate(String template, @Nullable XContentType contentTyp return template; } return new StringBuilder("__") - .append(contentType.format().toLowerCase(Locale.ROOT)) + .append(contentType.queryParameter().toLowerCase(Locale.ROOT)) .append("__::") .append(template) .toString();