-
Notifications
You must be signed in to change notification settings - Fork 4.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
BigQuery destination : Enable DAT tests (#13155)
* Enable basic DAT tests * adopt basic tests + avoid DateTime issue #13123 * disable array tests due to issue #13154 * format * DON"T MERGE * fix emitted_at Signed-off-by: Sergey Chvalyuk <[email protected]> * Revert "fix emitted_at" This reverts commit e752a24. * Revert "DON"T MERGE" This reverts commit dc2806b. * google format Co-authored-by: Sergey Chvalyuk <[email protected]>
- Loading branch information
1 parent
3dcda7a
commit 87847ba
Showing
4 changed files
with
142 additions
and
52 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
95 changes: 95 additions & 0 deletions
95
...gration/java/io/airbyte/integrations/destination/bigquery/BigQueryTestDataComparator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| /* | ||
| * Copyright (c) 2021 Airbyte, Inc., all rights reserved. | ||
| */ | ||
|
|
||
| package io.airbyte.integrations.destination.bigquery; | ||
|
|
||
| import io.airbyte.integrations.destination.StandardNameTransformer; | ||
| import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; | ||
| import java.time.LocalDate; | ||
| import java.time.LocalDateTime; | ||
| import java.time.ZoneOffset; | ||
| import java.time.ZonedDateTime; | ||
| import java.time.format.DateTimeFormatter; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| public class BigQueryTestDataComparator extends AdvancedTestDataComparator { | ||
|
|
||
| private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryTestDataComparator.class); | ||
| private final StandardNameTransformer namingResolver = new StandardNameTransformer(); | ||
|
|
||
| private static final String BIGQUERY_DATETIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"; | ||
|
|
||
| @Override | ||
| protected List<String> resolveIdentifier(final String identifier) { | ||
| final List<String> result = new ArrayList<>(); | ||
| result.add(identifier); | ||
| result.add(namingResolver.getIdentifier(identifier)); | ||
| return result; | ||
| } | ||
|
|
||
| private LocalDate parseDate(String dateValue) { | ||
| if (dateValue != null) { | ||
| var format = (dateValue.matches(".+Z") ? BIGQUERY_DATETIME_FORMAT : AIRBYTE_DATE_FORMAT); | ||
| return LocalDate.parse(dateValue, DateTimeFormatter.ofPattern(format)); | ||
| } else { | ||
| return null; | ||
| } | ||
| } | ||
|
|
||
| private LocalDateTime parseDateTime(String dateTimeValue) { | ||
| if (dateTimeValue != null) { | ||
| var format = (dateTimeValue.matches(".+Z") ? BIGQUERY_DATETIME_FORMAT : AIRBYTE_DATETIME_FORMAT); | ||
| return LocalDateTime.parse(dateTimeValue, DateTimeFormatter.ofPattern(format)); | ||
| } else { | ||
| return null; | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| protected boolean compareDateTimeValues(String expectedValue, String actualValue) { | ||
| var destinationDate = parseDateTime(actualValue); | ||
| var expectedDate = LocalDateTime.parse(expectedValue, DateTimeFormatter.ofPattern(AIRBYTE_DATETIME_FORMAT)); | ||
| // #13123 Normalization issue | ||
| if (expectedDate.isBefore(getBrokenDate().toLocalDateTime())) { | ||
| LOGGER | ||
| .warn("Validation is skipped due to known Normalization issue. Values older then 1583 year and with time zone stored wrongly(lose days)."); | ||
| return true; | ||
| } else { | ||
| return expectedDate.equals(destinationDate); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| protected boolean compareDateValues(String expectedValue, String actualValue) { | ||
| var destinationDate = parseDate(actualValue); | ||
| var expectedDate = LocalDate.parse(expectedValue, DateTimeFormatter.ofPattern(AIRBYTE_DATE_FORMAT)); | ||
| return expectedDate.equals(destinationDate); | ||
| } | ||
|
|
||
| @Override | ||
| protected ZonedDateTime parseDestinationDateWithTz(String destinationValue) { | ||
| return ZonedDateTime.of(LocalDateTime.parse(destinationValue, DateTimeFormatter.ofPattern(BIGQUERY_DATETIME_FORMAT)), ZoneOffset.UTC); | ||
| } | ||
|
|
||
| @Override | ||
| protected boolean compareDateTimeWithTzValues(String airbyteMessageValue, String destinationValue) { | ||
| // #13123 Normalization issue | ||
| if (parseDestinationDateWithTz(destinationValue).isBefore(getBrokenDate())) { | ||
| LOGGER | ||
| .warn("Validation is skipped due to known Normalization issue. Values older then 1583 year and with time zone stored wrongly(lose days)."); | ||
| return true; | ||
| } else { | ||
| return super.compareDateTimeWithTzValues(airbyteMessageValue, destinationValue); | ||
| } | ||
| } | ||
|
|
||
| // #13123 Normalization issue | ||
| private ZonedDateTime getBrokenDate() { | ||
| return ZonedDateTime.of(1583, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC); | ||
| } | ||
|
|
||
| } |