Skip to content

Commit

Permalink
Core: Backport java time date formatters (#31997)
Browse files Browse the repository at this point in the history
* Date: Add DateFormatters class that uses java.time (#31856)

A newly added class called DateFormatters now contains java.time based
builders for dates, which also intends to be fully backwards compatible,
when the name based date formatters are picked. Also a new class named
CompoundDateTimeFormatter for being able to parse multiple different
formats has been added.

A duelling test class has been added that ensures the same dates when
parsing java or joda time formatted dates for the name based dates.

Note, that java.time and joda time are not fully backwards compatible,
which also means that old formats will currently not work with this
setup.

* Tests: Remove use of joda time in some tests (#31922)

This also extends the dateformatters test to ensure that the printers
are acting the same in java time and joda time.

* Tests: Fix SearchFieldsIT.testDocValueFields

This test produced different implementations of joda time classes,
depending on if the data was serialized or not (DateTime vs
MutableDateTime). This now uses a common base class to extract the
milliseconds from the data.

Closes #31992
  • Loading branch information
spinscale authored and rjernst committed Jul 16, 2018
1 parent 0382984 commit a87ae91
Show file tree
Hide file tree
Showing 20 changed files with 1,767 additions and 99 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,10 @@
import org.elasticsearch.ingest.TestTemplateService;
import org.elasticsearch.script.TemplateScript;
import org.elasticsearch.test.ESTestCase;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;

import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
Expand All @@ -36,19 +37,21 @@

import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.joda.time.DateTimeZone.UTC;

public class DateProcessorTests extends ESTestCase {

private TemplateScript.Factory templatize(Locale locale) {
return new TestTemplateService.MockTemplateScript.Factory(locale.getLanguage());
}

private TemplateScript.Factory templatize(DateTimeZone timezone) {
return new TestTemplateService.MockTemplateScript.Factory(timezone.getID());
private TemplateScript.Factory templatize(ZoneId timezone) {
// prevent writing "UTC" as string, as joda time does not parse it
String id = timezone.equals(ZoneOffset.UTC) ? "UTC" : timezone.getId();
return new TestTemplateService.MockTemplateScript.Factory(id);
}
public void testJodaPattern() {
DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10),
templatize(DateTimeZone.forID("Europe/Amsterdam")), templatize(Locale.ENGLISH),
templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH),
"date_as_string", Collections.singletonList("yyyy dd MM hh:mm:ss"), "date_as_date");
Map<String, Object> document = new HashMap<>();
document.put("date_as_string", "2010 12 06 11:05:15");
Expand All @@ -63,7 +66,7 @@ public void testJodaPatternMultipleFormats() {
matchFormats.add("dd/MM/yyyy");
matchFormats.add("dd-MM-yyyy");
DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10),
templatize(DateTimeZone.forID("Europe/Amsterdam")), templatize(Locale.ENGLISH),
templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH),
"date_as_string", matchFormats, "date_as_date");

Map<String, Object> document = new HashMap<>();
Expand Down Expand Up @@ -98,7 +101,7 @@ public void testJodaPatternMultipleFormats() {
public void testInvalidJodaPattern() {
try {
DateProcessor processor = new DateProcessor(randomAlphaOfLength(10),
templatize(UTC), templatize(randomLocale(random())),
templatize(ZoneOffset.UTC), templatize(randomLocale(random())),
"date_as_string", Collections.singletonList("invalid pattern"), "date_as_date");
Map<String, Object> document = new HashMap<>();
document.put("date_as_string", "2010");
Expand All @@ -112,7 +115,7 @@ public void testInvalidJodaPattern() {

public void testJodaPatternLocale() {
DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10),
templatize(DateTimeZone.forID("Europe/Amsterdam")), templatize(Locale.ITALIAN),
templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ITALIAN),
"date_as_string", Collections.singletonList("yyyy dd MMM"), "date_as_date");
Map<String, Object> document = new HashMap<>();
document.put("date_as_string", "2010 12 giugno");
Expand All @@ -123,18 +126,18 @@ public void testJodaPatternLocale() {

public void testJodaPatternDefaultYear() {
DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10),
templatize(DateTimeZone.forID("Europe/Amsterdam")), templatize(Locale.ENGLISH),
templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH),
"date_as_string", Collections.singletonList("dd/MM"), "date_as_date");
Map<String, Object> document = new HashMap<>();
document.put("date_as_string", "12/06");
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
dateProcessor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue("date_as_date", String.class),
equalTo(DateTime.now().getYear() + "-06-12T00:00:00.000+02:00"));
equalTo(ZonedDateTime.now().getYear() + "-06-12T00:00:00.000+02:00"));
}

public void testTAI64N() {
DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(DateTimeZone.forOffsetHours(2)),
DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(ZoneOffset.ofHours(2)),
templatize(randomLocale(random())),
"date_as_string", Collections.singletonList("TAI64N"), "date_as_date");
Map<String, Object> document = new HashMap<>();
Expand All @@ -146,8 +149,8 @@ public void testTAI64N() {
}

public void testUnixMs() {
DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(UTC), templatize(randomLocale(random())),
"date_as_string", Collections.singletonList("UNIX_MS"), "date_as_date");
DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(ZoneOffset.UTC),
templatize(randomLocale(random())), "date_as_string", Collections.singletonList("UNIX_MS"), "date_as_date");
Map<String, Object> document = new HashMap<>();
document.put("date_as_string", "1000500");
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
Expand All @@ -162,7 +165,7 @@ public void testUnixMs() {
}

public void testUnix() {
DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(UTC),
DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(ZoneOffset.UTC),
templatize(randomLocale(random())),
"date_as_string", Collections.singletonList("UNIX"), "date_as_date");
Map<String, Object> document = new HashMap<>();
Expand All @@ -186,7 +189,7 @@ public void testInvalidTimezone() {

public void testInvalidLocale() {
DateProcessor processor = new DateProcessor(randomAlphaOfLength(10),
templatize(UTC), new TestTemplateService.MockTemplateScript.Factory("invalid_locale"),
templatize(ZoneOffset.UTC), new TestTemplateService.MockTemplateScript.Factory("invalid_locale"),
"date_as_string", Collections.singletonList("yyyy"), "date_as_date");
Map<String, Object> document = new HashMap<>();
document.put("date_as_string", "2010");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import com.carrotsearch.hppc.cursors.IntObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;

import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.rollover.RolloverInfo;
import org.elasticsearch.action.support.ActiveShardCount;
Expand Down Expand Up @@ -56,10 +55,11 @@
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.rest.RestStatus;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;

import java.io.IOException;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
Expand Down Expand Up @@ -1346,7 +1346,7 @@ public static Settings addHumanReadableSettings(Settings settings) {
}
Long creationDate = settings.getAsLong(SETTING_CREATION_DATE, null);
if (creationDate != null) {
DateTime creationDateTime = new DateTime(creationDate, DateTimeZone.UTC);
ZonedDateTime creationDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(creationDate), ZoneOffset.UTC);
builder.put(SETTING_CREATION_DATE_STRING, creationDateTime.toString());
}
return builder.build();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,11 +75,10 @@
import org.elasticsearch.indices.InvalidIndexNameException;
import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
import org.elasticsearch.threadpool.ThreadPool;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;

import java.io.UnsupportedEncodingException;
import java.nio.file.Path;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
Expand Down Expand Up @@ -390,7 +389,7 @@ public ClusterState execute(ClusterState currentState) throws Exception {
}

if (indexSettingsBuilder.get(SETTING_CREATION_DATE) == null) {
indexSettingsBuilder.put(SETTING_CREATION_DATE, new DateTime(DateTimeZone.UTC).getMillis());
indexSettingsBuilder.put(SETTING_CREATION_DATE, Instant.now().toEpochMilli());
}
indexSettingsBuilder.put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, request.getProvidedName());
indexSettingsBuilder.put(SETTING_INDEX_UUID, UUIDs.randomBase64UUID());
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.time;

import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.temporal.TemporalAccessor;

/**
* wrapper class around java.time.DateTimeFormatter that supports multiple formats for easier parsing,
* and one specific format for printing
*/
public class CompoundDateTimeFormatter {

final DateTimeFormatter printer;
final DateTimeFormatter[] parsers;

CompoundDateTimeFormatter(DateTimeFormatter ... parsers) {
if (parsers.length == 0) {
throw new IllegalArgumentException("at least one date time formatter is required");
}
this.printer = parsers[0];
this.parsers = parsers;
}

public TemporalAccessor parse(String input) {
DateTimeParseException failure = null;
for (int i = 0; i < parsers.length; i++) {
try {
return parsers[i].parse(input);
} catch (DateTimeParseException e) {
if (failure == null) {
failure = e;
} else {
failure.addSuppressed(e);
}
}
}

// ensure that all parsers exceptions are returned instead of only the last one
throw failure;
}

public CompoundDateTimeFormatter withZone(ZoneId zoneId) {
final DateTimeFormatter[] parsersWithZone = new DateTimeFormatter[parsers.length];
for (int i = 0; i < parsers.length; i++) {
parsersWithZone[i] = parsers[i].withZone(zoneId);
}

return new CompoundDateTimeFormatter(parsersWithZone);
}

public String format(TemporalAccessor accessor) {
return printer.format(accessor);
}
}
Loading

0 comments on commit a87ae91

Please sign in to comment.