diff --git a/metadata-ingestion/integration_docs/great-expectations.md b/metadata-ingestion/integration_docs/great-expectations.md index 0bb76357978bf..80f5bedf42661 100644 --- a/metadata-ingestion/integration_docs/great-expectations.md +++ b/metadata-ingestion/integration_docs/great-expectations.md @@ -49,6 +49,7 @@ This integration does not support - `retry_max_times` (optional): Maximum times to retry if HTTP request fails. The delay between retries is increased exponentially. - `extra_headers` (optional): Extra headers which will be added to the datahub request. - `parse_table_names_from_sql` (defaults to false): The integration can use an SQL parser to try to parse the datasets being asserted. This parsing is disabled by default, but can be enabled by setting `parse_table_names_from_sql: True`. The parser is based on the [`sqllineage`](https://pypi.org/project/sqllineage/) package. + - `convert_urns_to_lowercase` (optional): Whether to convert dataset urns to lowercase. ## Debugging Set environment variable `DATAHUB_DEBUG` (default `false`) to `true` to enable debug logging for `DataHubValidationAction`. diff --git a/metadata-ingestion/src/datahub/integrations/great_expectations/action.py b/metadata-ingestion/src/datahub/integrations/great_expectations/action.py index f033dbae8f1a1..36b85c306a86d 100644 --- a/metadata-ingestion/src/datahub/integrations/great_expectations/action.py +++ b/metadata-ingestion/src/datahub/integrations/great_expectations/action.py @@ -87,6 +87,7 @@ def __init__( extra_headers: Optional[Dict[str, str]] = None, exclude_dbname: Optional[bool] = None, parse_table_names_from_sql: bool = False, + convert_urns_to_lowercase: bool = False, ): super().__init__(data_context) self.server_url = server_url @@ -101,6 +102,7 @@ def __init__( self.extra_headers = extra_headers self.exclude_dbname = exclude_dbname self.parse_table_names_from_sql = parse_table_names_from_sql + self.convert_urns_to_lowercase = convert_urns_to_lowercase def _run( self, @@ -593,6 +595,7 @@ def get_dataset_partitions(self, batch_identifier, data_asset): ), self.exclude_dbname, self.platform_alias, + self.convert_urns_to_lowercase, ) batchSpec = BatchSpec( nativeBatchId=batch_identifier, @@ -661,6 +664,7 @@ def get_dataset_partitions(self, batch_identifier, data_asset): ), self.exclude_dbname, self.platform_alias, + self.convert_urns_to_lowercase, ) dataset_partitions.append( { @@ -703,6 +707,7 @@ def make_dataset_urn_from_sqlalchemy_uri( platform_instance=None, exclude_dbname=None, platform_alias=None, + convert_urns_to_lowercase=False, ): data_platform = get_platform_from_sqlalchemy_uri(str(sqlalchemy_uri)) url_instance = make_url(sqlalchemy_uri) @@ -777,6 +782,9 @@ def make_dataset_urn_from_sqlalchemy_uri( dataset_name = f"{schema_name}.{table_name}" + if convert_urns_to_lowercase: + dataset_name = dataset_name.lower() + dataset_urn = builder.make_dataset_urn_with_platform_instance( platform=data_platform if platform_alias is None else platform_alias, name=dataset_name, diff --git a/metadata-ingestion/tests/integration/great-expectations/ge_mcps_golden.json b/metadata-ingestion/tests/integration/great-expectations/ge_mcps_golden.json index 78fa4051a9b77..38b81d84d432b 100644 --- a/metadata-ingestion/tests/integration/great-expectations/ge_mcps_golden.json +++ b/metadata-ingestion/tests/integration/great-expectations/ge_mcps_golden.json @@ -1,782 +1,1929 @@ [ { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD),score)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"8\", \"type\": \"UNKNOWN\"}, \"minValue\": {\"value\": \"0\", \"type\": \"UNKNOWN\"}}, \"nativeType\": \"expect_column_values_to_be_between\", \"nativeParameters\": {\"column\": \"score\", \"max_value\": \"8\", \"min_value\": \"0\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD),score)" + ], + "aggregation": "IDENTITY", + "operator": "BETWEEN", + "parameters": { + "maxValue": { + "value": "8", + "type": "UNKNOWN" + }, + "minValue": { + "value": "0", + "type": "UNKNOWN" + } + }, + "nativeType": "expect_column_values_to_be_between", + "nativeParameters": { + "column": "score", + "max_value": "8", + "min_value": "0" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catA\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_postgresql_datasource\"}, \"nativeBatchId\": \"c8f12129f2e57412eee5fb8656154d05\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResults\": {}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catA\"}" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "data__foo1__asset", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "c8f12129f2e57412eee5fb8656154d05", + "limit": 10 + }, + "status": "COMPLETE", + "result": { + "type": "FAILURE", + "nativeResults": {} + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:3d1699164901675df774ab34fd16f4f3", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"scope\": \"DATASET_SCHEMA\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD),id)\"], \"aggregation\": \"_NATIVE_\", \"operator\": \"_NATIVE_\", \"nativeType\": \"expect_column_to_exist\", \"nativeParameters\": {\"column\": \"id\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "scope": "DATASET_SCHEMA", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD),id)" + ], + "aggregation": "_NATIVE_", + "operator": "_NATIVE_", + "nativeType": "expect_column_to_exist", + "nativeParameters": { + "column": "id" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:3d1699164901675df774ab34fd16f4f3", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:3d1699164901675df774ab34fd16f4f3", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catA\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:3d1699164901675df774ab34fd16f4f3\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_postgresql_datasource\"}, \"nativeBatchId\": \"c8f12129f2e57412eee5fb8656154d05\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catA\"}" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:3d1699164901675df774ab34fd16f4f3", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "data__foo1__asset", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "c8f12129f2e57412eee5fb8656154d05", + "limit": 10 + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:16d6f586b2febda7f2b53faec6bb9035", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD),id)\"], \"aggregation\": \"MIN\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"5\", \"type\": \"UNKNOWN\"}, \"minValue\": {\"value\": \"1\", \"type\": \"UNKNOWN\"}}, \"nativeType\": \"expect_column_min_to_be_between\", \"nativeParameters\": {\"column\": \"id\", \"max_value\": \"5\", \"min_value\": \"1\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD),id)" + ], + "aggregation": "MIN", + "operator": "BETWEEN", + "parameters": { + "maxValue": { + "value": "5", + "type": "UNKNOWN" + }, + "minValue": { + "value": "1", + "type": "UNKNOWN" + } + }, + "nativeType": "expect_column_min_to_be_between", + "nativeParameters": { + "column": "id", + "max_value": "5", + "min_value": "1" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:16d6f586b2febda7f2b53faec6bb9035", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:16d6f586b2febda7f2b53faec6bb9035", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catA\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:16d6f586b2febda7f2b53faec6bb9035\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_postgresql_datasource\"}, \"nativeBatchId\": \"c8f12129f2e57412eee5fb8656154d05\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"actualAggValue\": 1, \"nativeResults\": {\"observed_value\": \"1\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catA\"}" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:16d6f586b2febda7f2b53faec6bb9035", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "data__foo1__asset", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "c8f12129f2e57412eee5fb8656154d05", + "limit": 10 + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "actualAggValue": 1, + "nativeResults": { + "observed_value": "1" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:4cf76385ccf614cc6cbb9daa551c3c74", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"scope\": \"DATASET_ROWS\", \"aggregation\": \"ROW_COUNT\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"5\", \"type\": \"NUMBER\"}, \"minValue\": {\"value\": \"1\", \"type\": \"NUMBER\"}}, \"nativeType\": \"expect_table_row_count_to_be_between\", \"nativeParameters\": {\"max_value\": \"5\", \"min_value\": \"1\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "scope": "DATASET_ROWS", + "aggregation": "ROW_COUNT", + "operator": "BETWEEN", + "parameters": { + "maxValue": { + "value": "5", + "type": "NUMBER" + }, + "minValue": { + "value": "1", + "type": "NUMBER" + } + }, + "nativeType": "expect_table_row_count_to_be_between", + "nativeParameters": { + "max_value": "5", + "min_value": "1" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:4cf76385ccf614cc6cbb9daa551c3c74", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:4cf76385ccf614cc6cbb9daa551c3c74", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catA\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:4cf76385ccf614cc6cbb9daa551c3c74\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_postgresql_datasource\"}, \"nativeBatchId\": \"c8f12129f2e57412eee5fb8656154d05\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"actualAggValue\": 3, \"nativeResults\": {\"observed_value\": \"3\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catA\"}" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:4cf76385ccf614cc6cbb9daa551c3c74", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "data__foo1__asset", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "c8f12129f2e57412eee5fb8656154d05", + "limit": 10 + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "actualAggValue": 3, + "nativeResults": { + "observed_value": "3" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:9729dfafea4bb2c2f114bc80e513a7ec", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD),name)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"expect_column_values_to_not_be_null\", \"nativeParameters\": {\"column\": \"name\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD),name)" + ], + "aggregation": "IDENTITY", + "operator": "NOT_NULL", + "nativeType": "expect_column_values_to_not_be_null", + "nativeParameters": { + "column": "name" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:9729dfafea4bb2c2f114bc80e513a7ec", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:9729dfafea4bb2c2f114bc80e513a7ec", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catA\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:9729dfafea4bb2c2f114bc80e513a7ec\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_postgresql_datasource\"}, \"nativeBatchId\": \"c8f12129f2e57412eee5fb8656154d05\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"rowCount\": 3, \"unexpectedCount\": 0, \"nativeResults\": {}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catA\"}" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:9729dfafea4bb2c2f114bc80e513a7ec", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "data__foo1__asset", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "c8f12129f2e57412eee5fb8656154d05", + "limit": 10 + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "rowCount": 3, + "unexpectedCount": 0, + "nativeResults": {} + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD),score)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"8\", \"type\": \"UNKNOWN\"}, \"minValue\": {\"value\": \"0\", \"type\": \"UNKNOWN\"}}, \"nativeType\": \"expect_column_values_to_be_between\", \"nativeParameters\": {\"column\": \"score\", \"max_value\": \"8\", \"min_value\": \"0\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD),score)" + ], + "aggregation": "IDENTITY", + "operator": "BETWEEN", + "parameters": { + "maxValue": { + "value": "8", + "type": "UNKNOWN" + }, + "minValue": { + "value": "0", + "type": "UNKNOWN" + } + }, + "nativeType": "expect_column_values_to_be_between", + "nativeParameters": { + "column": "score", + "max_value": "8", + "min_value": "0" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catB\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_postgresql_datasource\"}, \"nativeBatchId\": \"aaee0c1f4ae6bba9bcb50c5f20277d0a\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResults\": {}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catB\"}" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "data__foo1__asset", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "aaee0c1f4ae6bba9bcb50c5f20277d0a", + "limit": 10 + }, + "status": "COMPLETE", + "result": { + "type": "FAILURE", + "nativeResults": {} + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:3d1699164901675df774ab34fd16f4f3", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"scope\": \"DATASET_SCHEMA\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD),id)\"], \"aggregation\": \"_NATIVE_\", \"operator\": \"_NATIVE_\", \"nativeType\": \"expect_column_to_exist\", \"nativeParameters\": {\"column\": \"id\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "scope": "DATASET_SCHEMA", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD),id)" + ], + "aggregation": "_NATIVE_", + "operator": "_NATIVE_", + "nativeType": "expect_column_to_exist", + "nativeParameters": { + "column": "id" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:3d1699164901675df774ab34fd16f4f3", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:3d1699164901675df774ab34fd16f4f3", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catB\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:3d1699164901675df774ab34fd16f4f3\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_postgresql_datasource\"}, \"nativeBatchId\": \"aaee0c1f4ae6bba9bcb50c5f20277d0a\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catB\"}" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:3d1699164901675df774ab34fd16f4f3", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "data__foo1__asset", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "aaee0c1f4ae6bba9bcb50c5f20277d0a", + "limit": 10 + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:16d6f586b2febda7f2b53faec6bb9035", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD),id)\"], \"aggregation\": \"MIN\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"5\", \"type\": \"UNKNOWN\"}, \"minValue\": {\"value\": \"1\", \"type\": \"UNKNOWN\"}}, \"nativeType\": \"expect_column_min_to_be_between\", \"nativeParameters\": {\"column\": \"id\", \"max_value\": \"5\", \"min_value\": \"1\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD),id)" + ], + "aggregation": "MIN", + "operator": "BETWEEN", + "parameters": { + "maxValue": { + "value": "5", + "type": "UNKNOWN" + }, + "minValue": { + "value": "1", + "type": "UNKNOWN" + } + }, + "nativeType": "expect_column_min_to_be_between", + "nativeParameters": { + "column": "id", + "max_value": "5", + "min_value": "1" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:16d6f586b2febda7f2b53faec6bb9035", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:16d6f586b2febda7f2b53faec6bb9035", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catB\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:16d6f586b2febda7f2b53faec6bb9035\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_postgresql_datasource\"}, \"nativeBatchId\": \"aaee0c1f4ae6bba9bcb50c5f20277d0a\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"actualAggValue\": 3, \"nativeResults\": {\"observed_value\": \"3\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catB\"}" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:16d6f586b2febda7f2b53faec6bb9035", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "data__foo1__asset", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "aaee0c1f4ae6bba9bcb50c5f20277d0a", + "limit": 10 + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "actualAggValue": 3, + "nativeResults": { + "observed_value": "3" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:4cf76385ccf614cc6cbb9daa551c3c74", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"scope\": \"DATASET_ROWS\", \"aggregation\": \"ROW_COUNT\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"5\", \"type\": \"NUMBER\"}, \"minValue\": {\"value\": \"1\", \"type\": \"NUMBER\"}}, \"nativeType\": \"expect_table_row_count_to_be_between\", \"nativeParameters\": {\"max_value\": \"5\", \"min_value\": \"1\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "scope": "DATASET_ROWS", + "aggregation": "ROW_COUNT", + "operator": "BETWEEN", + "parameters": { + "maxValue": { + "value": "5", + "type": "NUMBER" + }, + "minValue": { + "value": "1", + "type": "NUMBER" + } + }, + "nativeType": "expect_table_row_count_to_be_between", + "nativeParameters": { + "max_value": "5", + "min_value": "1" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:4cf76385ccf614cc6cbb9daa551c3c74", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:4cf76385ccf614cc6cbb9daa551c3c74", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catB\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:4cf76385ccf614cc6cbb9daa551c3c74\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_postgresql_datasource\"}, \"nativeBatchId\": \"aaee0c1f4ae6bba9bcb50c5f20277d0a\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"actualAggValue\": 2, \"nativeResults\": {\"observed_value\": \"2\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catB\"}" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:4cf76385ccf614cc6cbb9daa551c3c74", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "data__foo1__asset", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "aaee0c1f4ae6bba9bcb50c5f20277d0a", + "limit": 10 + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "actualAggValue": 2, + "nativeResults": { + "observed_value": "2" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:9729dfafea4bb2c2f114bc80e513a7ec", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD),name)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"expect_column_values_to_not_be_null\", \"nativeParameters\": {\"column\": \"name\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD),name)" + ], + "aggregation": "IDENTITY", + "operator": "NOT_NULL", + "nativeType": "expect_column_values_to_not_be_null", + "nativeParameters": { + "column": "name" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:9729dfafea4bb2c2f114bc80e513a7ec", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:9729dfafea4bb2c2f114bc80e513a7ec", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catB\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:9729dfafea4bb2c2f114bc80e513a7ec\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_postgresql_datasource\"}, \"nativeBatchId\": \"aaee0c1f4ae6bba9bcb50c5f20277d0a\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"rowCount\": 2, \"unexpectedCount\": 0, \"nativeResults\": {}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catB\"}" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:9729dfafea4bb2c2f114bc80e513a7ec", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "data__foo1__asset", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "aaee0c1f4ae6bba9bcb50c5f20277d0a", + "limit": 10 + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "rowCount": 2, + "unexpectedCount": 0, + "nativeResults": {} + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:2e3e33eaa148f19f6a6e37ba0c6fdb2f", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"scope\": \"DATASET_SCHEMA\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD),id)\"], \"aggregation\": \"_NATIVE_\", \"operator\": \"_NATIVE_\", \"nativeType\": \"expect_column_to_exist\", \"nativeParameters\": {\"column\": \"id\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "scope": "DATASET_SCHEMA", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD),id)" + ], + "aggregation": "_NATIVE_", + "operator": "_NATIVE_", + "nativeType": "expect_column_to_exist", + "nativeParameters": { + "column": "id" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:2e3e33eaa148f19f6a6e37ba0c6fdb2f", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:2e3e33eaa148f19f6a6e37ba0c6fdb2f", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:2e3e33eaa148f19f6a6e37ba0c6fdb2f\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"foo2\", \"datasource_name\": \"my_postgresql_datasource\"}, \"nativeBatchId\": \"010ef8c1cd417910b971f4468f024ec5\"}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:2e3e33eaa148f19f6a6e37ba0c6fdb2f", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "foo2", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "010ef8c1cd417910b971f4468f024ec5" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:ab05b249c82cb5cdeb19743462125fae", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD),id)\"], \"aggregation\": \"MIN\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"5\", \"type\": \"UNKNOWN\"}, \"minValue\": {\"value\": \"1\", \"type\": \"UNKNOWN\"}}, \"nativeType\": \"expect_column_min_to_be_between\", \"nativeParameters\": {\"column\": \"id\", \"max_value\": \"5\", \"min_value\": \"1\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD),id)" + ], + "aggregation": "MIN", + "operator": "BETWEEN", + "parameters": { + "maxValue": { + "value": "5", + "type": "UNKNOWN" + }, + "minValue": { + "value": "1", + "type": "UNKNOWN" + } + }, + "nativeType": "expect_column_min_to_be_between", + "nativeParameters": { + "column": "id", + "max_value": "5", + "min_value": "1" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:ab05b249c82cb5cdeb19743462125fae", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:ab05b249c82cb5cdeb19743462125fae", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:ab05b249c82cb5cdeb19743462125fae\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"foo2\", \"datasource_name\": \"my_postgresql_datasource\"}, \"nativeBatchId\": \"010ef8c1cd417910b971f4468f024ec5\"}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"actualAggValue\": 1, \"nativeResults\": {\"observed_value\": \"1\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:ab05b249c82cb5cdeb19743462125fae", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "foo2", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "010ef8c1cd417910b971f4468f024ec5" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "actualAggValue": 1, + "nativeResults": { + "observed_value": "1" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:e650fdf72c472afccfda2eb6e2e0152e", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"scope\": \"DATASET_ROWS\", \"aggregation\": \"ROW_COUNT\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"5\", \"type\": \"NUMBER\"}, \"minValue\": {\"value\": \"1\", \"type\": \"NUMBER\"}}, \"nativeType\": \"expect_table_row_count_to_be_between\", \"nativeParameters\": {\"max_value\": \"5\", \"min_value\": \"1\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "scope": "DATASET_ROWS", + "aggregation": "ROW_COUNT", + "operator": "BETWEEN", + "parameters": { + "maxValue": { + "value": "5", + "type": "NUMBER" + }, + "minValue": { + "value": "1", + "type": "NUMBER" + } + }, + "nativeType": "expect_table_row_count_to_be_between", + "nativeParameters": { + "max_value": "5", + "min_value": "1" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:e650fdf72c472afccfda2eb6e2e0152e", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:e650fdf72c472afccfda2eb6e2e0152e", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:e650fdf72c472afccfda2eb6e2e0152e\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"foo2\", \"datasource_name\": \"my_postgresql_datasource\"}, \"nativeBatchId\": \"010ef8c1cd417910b971f4468f024ec5\"}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"actualAggValue\": 5, \"nativeResults\": {\"observed_value\": \"5\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:e650fdf72c472afccfda2eb6e2e0152e", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "foo2", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "010ef8c1cd417910b971f4468f024ec5" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "actualAggValue": 5, + "nativeResults": { + "observed_value": "5" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:0d2562733ca1096b6a89287e37b87e9e", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD),score)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"8\", \"type\": \"UNKNOWN\"}, \"minValue\": {\"value\": \"0\", \"type\": \"UNKNOWN\"}}, \"nativeType\": \"expect_column_values_to_be_between\", \"nativeParameters\": {\"column\": \"score\", \"max_value\": \"8\", \"min_value\": \"0\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD),score)" + ], + "aggregation": "IDENTITY", + "operator": "BETWEEN", + "parameters": { + "maxValue": { + "value": "8", + "type": "UNKNOWN" + }, + "minValue": { + "value": "0", + "type": "UNKNOWN" + } + }, + "nativeType": "expect_column_values_to_be_between", + "nativeParameters": { + "column": "score", + "max_value": "8", + "min_value": "0" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:0d2562733ca1096b6a89287e37b87e9e", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:0d2562733ca1096b6a89287e37b87e9e", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:0d2562733ca1096b6a89287e37b87e9e\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"foo2\", \"datasource_name\": \"my_postgresql_datasource\"}, \"nativeBatchId\": \"010ef8c1cd417910b971f4468f024ec5\"}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"rowCount\": 5, \"missingCount\": 0, \"unexpectedCount\": 2, \"nativeResults\": {\"partial_unexpected_list\": \"[\\\"10\\\", \\\"9\\\"]\", \"partial_unexpected_counts\": \"[{\\\"value\\\": \\\"9\\\", \\\"count\\\": 1}, {\\\"value\\\": \\\"10\\\", \\\"count\\\": 1}]\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:0d2562733ca1096b6a89287e37b87e9e", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "foo2", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "010ef8c1cd417910b971f4468f024ec5" + }, + "status": "COMPLETE", + "result": { + "type": "FAILURE", + "rowCount": 5, + "missingCount": 0, + "unexpectedCount": 2, + "nativeResults": { + "partial_unexpected_list": "[\"10\", \"9\"]", + "partial_unexpected_counts": "[{\"value\": \"9\", \"count\": 1}, {\"value\": \"10\", \"count\": 1}]" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:6a54a953afa6403d7b6dffbf02ff3de4", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD),name)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"expect_column_values_to_not_be_null\", \"nativeParameters\": {\"column\": \"name\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD),name)" + ], + "aggregation": "IDENTITY", + "operator": "NOT_NULL", + "nativeType": "expect_column_values_to_not_be_null", + "nativeParameters": { + "column": "name" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:6a54a953afa6403d7b6dffbf02ff3de4", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:6a54a953afa6403d7b6dffbf02ff3de4", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:6a54a953afa6403d7b6dffbf02ff3de4\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"foo2\", \"datasource_name\": \"my_postgresql_datasource\"}, \"nativeBatchId\": \"010ef8c1cd417910b971f4468f024ec5\"}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"rowCount\": 5, \"unexpectedCount\": 0, \"nativeResults\": {}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:6a54a953afa6403d7b6dffbf02ff3de4", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "foo2", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "010ef8c1cd417910b971f4468f024ec5" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "rowCount": 5, + "unexpectedCount": 0, + "nativeResults": {} + } + } + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:fe1fad936d3dcb98a501525c66682253", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo3,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo3,PROD),score)" + ], + "aggregation": "IDENTITY", + "operator": "BETWEEN", + "parameters": { + "maxValue": { + "value": "8", + "type": "UNKNOWN" + }, + "minValue": { + "value": "0", + "type": "UNKNOWN" + } + }, + "nativeType": "expect_column_values_to_be_between", + "nativeParameters": { + "column": "score", + "max_value": "8", + "min_value": "0" + } + } + } + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:fe1fad936d3dcb98a501525c66682253", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:fe1fad936d3dcb98a501525c66682253", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:fe1fad936d3dcb98a501525c66682253", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo3,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "FOO3", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "297f599e52ae5029ad9fab6214412ab8" + }, + "status": "COMPLETE", + "result": { + "type": "FAILURE", + "nativeResults": {} + } + } + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b23c851ecc41bb74edc39b7b8cce570b", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo3,PROD)", + "scope": "DATASET_SCHEMA", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo3,PROD),id)" + ], + "aggregation": "_NATIVE_", + "operator": "_NATIVE_", + "nativeType": "expect_column_to_exist", + "nativeParameters": { + "column": "id" + } + } + } + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b23c851ecc41bb74edc39b7b8cce570b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b23c851ecc41bb74edc39b7b8cce570b", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:b23c851ecc41bb74edc39b7b8cce570b", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo3,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "FOO3", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "297f599e52ae5029ad9fab6214412ab8" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + } + } + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:38caf10bb6fb6ef52a6b0f5827bcd3aa", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo3,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo3,PROD),id)" + ], + "aggregation": "MIN", + "operator": "BETWEEN", + "parameters": { + "maxValue": { + "value": "5", + "type": "UNKNOWN" + }, + "minValue": { + "value": "1", + "type": "UNKNOWN" + } + }, + "nativeType": "expect_column_min_to_be_between", + "nativeParameters": { + "column": "id", + "max_value": "5", + "min_value": "1" + } + } + } + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:38caf10bb6fb6ef52a6b0f5827bcd3aa", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:38caf10bb6fb6ef52a6b0f5827bcd3aa", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:38caf10bb6fb6ef52a6b0f5827bcd3aa", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo3,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "FOO3", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "297f599e52ae5029ad9fab6214412ab8" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "actualAggValue": 1, + "nativeResults": { + "observed_value": "1" + } + } + } + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:fa39b9ea8bf83bb6af5635d878e1f07d", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo3,PROD)", + "scope": "DATASET_ROWS", + "aggregation": "ROW_COUNT", + "operator": "BETWEEN", + "parameters": { + "maxValue": { + "value": "5", + "type": "NUMBER" + }, + "minValue": { + "value": "1", + "type": "NUMBER" + } + }, + "nativeType": "expect_table_row_count_to_be_between", + "nativeParameters": { + "max_value": "5", + "min_value": "1" + } + } + } + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:fa39b9ea8bf83bb6af5635d878e1f07d", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:fa39b9ea8bf83bb6af5635d878e1f07d", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:fa39b9ea8bf83bb6af5635d878e1f07d", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo3,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "FOO3", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "297f599e52ae5029ad9fab6214412ab8" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "actualAggValue": 1, + "nativeResults": { + "observed_value": "1" + } + } + } + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f8e511d9e9d65987b56f0f1495a3bd00", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo3,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo3,PROD),name)" + ], + "aggregation": "IDENTITY", + "operator": "NOT_NULL", + "nativeType": "expect_column_values_to_not_be_null", + "nativeParameters": { + "column": "name" + } + } + } + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f8e511d9e9d65987b56f0f1495a3bd00", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f8e511d9e9d65987b56f0f1495a3bd00", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:f8e511d9e9d65987b56f0f1495a3bd00", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo3,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "FOO3", + "datasource_name": "my_postgresql_datasource" + }, + "nativeBatchId": "297f599e52ae5029ad9fab6214412ab8" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "rowCount": 1, + "unexpectedCount": 0, + "nativeResults": {} + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:2e3e33eaa148f19f6a6e37ba0c6fdb2f", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"scope\": \"DATASET_SCHEMA\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD),id)\"], \"aggregation\": \"_NATIVE_\", \"operator\": \"_NATIVE_\", \"nativeType\": \"expect_column_to_exist\", \"nativeParameters\": {\"column\": \"id\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "scope": "DATASET_SCHEMA", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD),id)" + ], + "aggregation": "_NATIVE_", + "operator": "_NATIVE_", + "nativeType": "expect_column_to_exist", + "nativeParameters": { + "column": "id" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:2e3e33eaa148f19f6a6e37ba0c6fdb2f", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:2e3e33eaa148f19f6a6e37ba0c6fdb2f", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"QUERY\", \"partition\": \"Query_5c069023db2845910198ff14ab86ac27\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:2e3e33eaa148f19f6a6e37ba0c6fdb2f\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"custom_data_asset\", \"datasource_name\": \"runtime_postgres_datasource\"}, \"nativeBatchId\": \"60f7ffb6e021abc8c6333d9040f97140\", \"query\": \"select * from foo2 where score>5\"}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "QUERY", + "partition": "Query_5c069023db2845910198ff14ab86ac27" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:2e3e33eaa148f19f6a6e37ba0c6fdb2f", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "custom_data_asset", + "datasource_name": "runtime_postgres_datasource" + }, + "nativeBatchId": "60f7ffb6e021abc8c6333d9040f97140", + "query": "select * from foo2 where score>5" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:ab05b249c82cb5cdeb19743462125fae", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD),id)\"], \"aggregation\": \"MIN\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"5\", \"type\": \"UNKNOWN\"}, \"minValue\": {\"value\": \"1\", \"type\": \"UNKNOWN\"}}, \"nativeType\": \"expect_column_min_to_be_between\", \"nativeParameters\": {\"column\": \"id\", \"max_value\": \"5\", \"min_value\": \"1\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD),id)" + ], + "aggregation": "MIN", + "operator": "BETWEEN", + "parameters": { + "maxValue": { + "value": "5", + "type": "UNKNOWN" + }, + "minValue": { + "value": "1", + "type": "UNKNOWN" + } + }, + "nativeType": "expect_column_min_to_be_between", + "nativeParameters": { + "column": "id", + "max_value": "5", + "min_value": "1" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:ab05b249c82cb5cdeb19743462125fae", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:ab05b249c82cb5cdeb19743462125fae", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"QUERY\", \"partition\": \"Query_5c069023db2845910198ff14ab86ac27\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:ab05b249c82cb5cdeb19743462125fae\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"custom_data_asset\", \"datasource_name\": \"runtime_postgres_datasource\"}, \"nativeBatchId\": \"60f7ffb6e021abc8c6333d9040f97140\", \"query\": \"select * from foo2 where score>5\"}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"actualAggValue\": 1, \"nativeResults\": {\"observed_value\": \"1\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "QUERY", + "partition": "Query_5c069023db2845910198ff14ab86ac27" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:ab05b249c82cb5cdeb19743462125fae", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "custom_data_asset", + "datasource_name": "runtime_postgres_datasource" + }, + "nativeBatchId": "60f7ffb6e021abc8c6333d9040f97140", + "query": "select * from foo2 where score>5" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "actualAggValue": 1, + "nativeResults": { + "observed_value": "1" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:e650fdf72c472afccfda2eb6e2e0152e", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"scope\": \"DATASET_ROWS\", \"aggregation\": \"ROW_COUNT\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"5\", \"type\": \"NUMBER\"}, \"minValue\": {\"value\": \"1\", \"type\": \"NUMBER\"}}, \"nativeType\": \"expect_table_row_count_to_be_between\", \"nativeParameters\": {\"max_value\": \"5\", \"min_value\": \"1\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "scope": "DATASET_ROWS", + "aggregation": "ROW_COUNT", + "operator": "BETWEEN", + "parameters": { + "maxValue": { + "value": "5", + "type": "NUMBER" + }, + "minValue": { + "value": "1", + "type": "NUMBER" + } + }, + "nativeType": "expect_table_row_count_to_be_between", + "nativeParameters": { + "max_value": "5", + "min_value": "1" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:e650fdf72c472afccfda2eb6e2e0152e", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:e650fdf72c472afccfda2eb6e2e0152e", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"QUERY\", \"partition\": \"Query_5c069023db2845910198ff14ab86ac27\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:e650fdf72c472afccfda2eb6e2e0152e\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"custom_data_asset\", \"datasource_name\": \"runtime_postgres_datasource\"}, \"nativeBatchId\": \"60f7ffb6e021abc8c6333d9040f97140\", \"query\": \"select * from foo2 where score>5\"}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"actualAggValue\": 5, \"nativeResults\": {\"observed_value\": \"5\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "QUERY", + "partition": "Query_5c069023db2845910198ff14ab86ac27" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:e650fdf72c472afccfda2eb6e2e0152e", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "custom_data_asset", + "datasource_name": "runtime_postgres_datasource" + }, + "nativeBatchId": "60f7ffb6e021abc8c6333d9040f97140", + "query": "select * from foo2 where score>5" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "actualAggValue": 5, + "nativeResults": { + "observed_value": "5" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:0d2562733ca1096b6a89287e37b87e9e", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD),score)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"8\", \"type\": \"UNKNOWN\"}, \"minValue\": {\"value\": \"0\", \"type\": \"UNKNOWN\"}}, \"nativeType\": \"expect_column_values_to_be_between\", \"nativeParameters\": {\"column\": \"score\", \"max_value\": \"8\", \"min_value\": \"0\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD),score)" + ], + "aggregation": "IDENTITY", + "operator": "BETWEEN", + "parameters": { + "maxValue": { + "value": "8", + "type": "UNKNOWN" + }, + "minValue": { + "value": "0", + "type": "UNKNOWN" + } + }, + "nativeType": "expect_column_values_to_be_between", + "nativeParameters": { + "column": "score", + "max_value": "8", + "min_value": "0" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:0d2562733ca1096b6a89287e37b87e9e", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:0d2562733ca1096b6a89287e37b87e9e", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"QUERY\", \"partition\": \"Query_5c069023db2845910198ff14ab86ac27\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:0d2562733ca1096b6a89287e37b87e9e\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"custom_data_asset\", \"datasource_name\": \"runtime_postgres_datasource\"}, \"nativeBatchId\": \"60f7ffb6e021abc8c6333d9040f97140\", \"query\": \"select * from foo2 where score>5\"}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"rowCount\": 5, \"missingCount\": 0, \"unexpectedCount\": 2, \"nativeResults\": {\"partial_unexpected_list\": \"[\\\"10\\\", \\\"9\\\"]\", \"partial_unexpected_counts\": \"[{\\\"value\\\": \\\"9\\\", \\\"count\\\": 1}, {\\\"value\\\": \\\"10\\\", \\\"count\\\": 1}]\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "QUERY", + "partition": "Query_5c069023db2845910198ff14ab86ac27" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:0d2562733ca1096b6a89287e37b87e9e", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "custom_data_asset", + "datasource_name": "runtime_postgres_datasource" + }, + "nativeBatchId": "60f7ffb6e021abc8c6333d9040f97140", + "query": "select * from foo2 where score>5" + }, + "status": "COMPLETE", + "result": { + "type": "FAILURE", + "rowCount": 5, + "missingCount": 0, + "unexpectedCount": 2, + "nativeResults": { + "partial_unexpected_list": "[\"10\", \"9\"]", + "partial_unexpected_counts": "[{\"value\": \"9\", \"count\": 1}, {\"value\": \"10\", \"count\": 1}]" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:6a54a953afa6403d7b6dffbf02ff3de4", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD),name)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"expect_column_values_to_not_be_null\", \"nativeParameters\": {\"column\": \"name\"}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD),name)" + ], + "aggregation": "IDENTITY", + "operator": "NOT_NULL", + "nativeType": "expect_column_values_to_not_be_null", + "nativeParameters": { + "column": "name" + } + } + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:6a54a953afa6403d7b6dffbf02ff3de4", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:6a54a953afa6403d7b6dffbf02ff3de4", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1640692800000, \"partitionSpec\": {\"type\": \"QUERY\", \"partition\": \"Query_5c069023db2845910198ff14ab86ac27\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:6a54a953afa6403d7b6dffbf02ff3de4\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"custom_data_asset\", \"datasource_name\": \"runtime_postgres_datasource\"}, \"nativeBatchId\": \"60f7ffb6e021abc8c6333d9040f97140\", \"query\": \"select * from foo2 where score>5\"}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"rowCount\": 5, \"unexpectedCount\": 0, \"nativeResults\": {}}}", - "contentType": "application/json" - }, - "systemMetadata": null + "json": { + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "QUERY", + "partition": "Query_5c069023db2845910198ff14ab86ac27" + }, + "runId": "2021-12-28T12:00:00Z", + "assertionUrn": "urn:li:assertion:6a54a953afa6403d7b6dffbf02ff3de4", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo2,PROD)", + "batchSpec": { + "customProperties": { + "data_asset_name": "custom_data_asset", + "datasource_name": "runtime_postgres_datasource" + }, + "nativeBatchId": "60f7ffb6e021abc8c6333d9040f97140", + "query": "select * from foo2 where score>5" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "rowCount": 5, + "unexpectedCount": 0, + "nativeResults": {} + } + } + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/great-expectations/setup/great_expectations/checkpoints/test_checkpoint.yml b/metadata-ingestion/tests/integration/great-expectations/setup/great_expectations/checkpoints/test_checkpoint.yml index 7b12c5f2cf987..466cbfe39a4ab 100644 --- a/metadata-ingestion/tests/integration/great-expectations/setup/great_expectations/checkpoints/test_checkpoint.yml +++ b/metadata-ingestion/tests/integration/great-expectations/setup/great_expectations/checkpoints/test_checkpoint.yml @@ -27,6 +27,7 @@ action_list: platform_instance_map: my_postgresql_datasource: postgres1 runtime_postgres_datasource: postgres1 + convert_urns_to_lowercase: True evaluation_parameters: {} runtime_configuration: result_format: @@ -53,6 +54,12 @@ validations: data_asset_name: foo2 data_connector_query: index: -1 + - batch_request: + datasource_name: my_postgresql_datasource + data_connector_name: whole_table # InferredAssetSqlDataConnector + data_asset_name: "FOO3" + data_connector_query: + index: -1 - batch_request: datasource_name: runtime_postgres_datasource data_connector_name: default_runtime_data_connector_name #RuntimeDataConnector diff --git a/metadata-ingestion/tests/integration/great-expectations/setup/setup.sql b/metadata-ingestion/tests/integration/great-expectations/setup/setup.sql index da958e43b18d0..029d650590e50 100644 --- a/metadata-ingestion/tests/integration/great-expectations/setup/setup.sql +++ b/metadata-ingestion/tests/integration/great-expectations/setup/setup.sql @@ -18,6 +18,12 @@ CREATE TABLE foo2 ( PRIMARY KEY (id) ); +CREATE TABLE "FOO3" ( + id INTEGER NOT NULL, + name VARCHAR (50) NOT NULL, + PRIMARY KEY (id) +); + INSERT INTO foo1 (id, name, category) VALUES (1, 'name1', 'catA'); INSERT INTO foo1 (id, name, category) VALUES (2, 'name2', 'catA'); INSERT INTO foo1 (id, name, category) VALUES (3, 'name3', 'catB'); @@ -29,4 +35,6 @@ INSERT INTO foo2 (id, name, start_date, start_year, score) VALUES (1, 'name1', T INSERT INTO foo2 (id, name, start_date, start_year, score) VALUES (2, 'name2', TO_DATE('2021-12-22','YYYY-MM-DD'), 2021, 7); INSERT INTO foo2 (id, name, start_date, start_year, score) VALUES (3, 'name3', TO_DATE('2021-12-21','YYYY-MM-DD'), 2021, 8); INSERT INTO foo2 (id, name, start_date, start_year, score) VALUES (4, 'name4', TO_DATE('2021-12-22','YYYY-MM-DD'), 2021, 9); -INSERT INTO foo2 (id, name, start_date, start_year, score) VALUES (5, 'name5', TO_DATE('2021-12-23','YYYY-MM-DD'), 2021, 6); \ No newline at end of file +INSERT INTO foo2 (id, name, start_date, start_year, score) VALUES (5, 'name5', TO_DATE('2021-12-23','YYYY-MM-DD'), 2021, 6); + +INSERT INTO "FOO3" (id, name) VALUES (1, 'name'); \ No newline at end of file