From feb9d59b4d9c284bf4d8eaf83dd243d0bc8deeca Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 5 Oct 2022 03:26:51 +0000 Subject: [PATCH] fix(ingest): remove unused mysql golden file (#6106) --- .../source/sql/two_tier_sql_source.py | 7 +- .../integration/mysql/mysql_mces_golden.json | 2037 ----------------- 2 files changed, 5 insertions(+), 2039 deletions(-) delete mode 100644 metadata-ingestion/tests/integration/mysql/mysql_mces_golden.json diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py b/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py index b9c6d18dac74b..c62c9c88f8854 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py @@ -51,12 +51,12 @@ def get_sql_alchemy_url( class TwoTierSQLAlchemySource(SQLAlchemySource): def __init__(self, config, ctx, platform): super().__init__(config, ctx, platform) - self.current_database = None self.config: TwoTierSQLAlchemyConfig = config def get_parent_container_key(self, db_name: str, schema: str) -> PlatformKey: # Because our overridden get_allowed_schemas method returns db_name as the schema name, # the db_name and schema here will be the same. Hence, we just ignore the schema parameter. + assert db_name == schema return self.gen_database_key(db_name) def get_allowed_schemas( @@ -66,6 +66,10 @@ def get_allowed_schemas( # dbName itself as an allowed schema yield db_name + def gen_schema_key(self, db_name: str, schema: str) -> PlatformKey: + # Sanity check that we don't try to generate schema containers for 2 tier databases. + raise NotImplementedError + def get_inspectors(self): # This method can be overridden in the case that you want to dynamically # run on multiple databases. @@ -84,7 +88,6 @@ def get_inspectors(self): inspector = inspect( create_engine(url, **self.config.options).connect() ) - self.current_database = db yield inspector def gen_schema_containers( diff --git a/metadata-ingestion/tests/integration/mysql/mysql_mces_golden.json b/metadata-ingestion/tests/integration/mysql/mysql_mces_golden.json deleted file mode 100644 index 5732e82e1eb7c..0000000000000 --- a/metadata-ingestion/tests/integration/mysql/mysql_mces_golden.json +++ /dev/null @@ -1,2037 +0,0 @@ -[ -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:17751259af32dd0385cad799df608c40", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\"}, \"name\": \"metagalaxy\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:17751259af32dd0385cad799df608c40", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:17751259af32dd0385cad799df608c40", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Database\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:17751259af32dd0385cad799df608c40", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "domains", - "aspect": { - "value": "{\"domains\": [\"urn:li:domain:sales\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:ba408413d97771e6470c16f9869f2e0d", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\", \"schema\": \"datacharmer\"}, \"name\": \"datacharmer\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:ba408413d97771e6470c16f9869f2e0d", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:ba408413d97771e6470c16f9869f2e0d", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:ba408413d97771e6470c16f9869f2e0d", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:17751259af32dd0385cad799df608c40\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.employees,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:ba408413d97771e6470c16f9869f2e0d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.employees,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": {}, - "externalUrl": null, - "name": "employees", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "datacharmer.employees", - "platform": "urn:li:dataPlatform:mysql", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "emp_no", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "jsonProps": null - }, - { - "fieldPath": "birth_date", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=14)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=16)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "gender", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.EnumType": {} - } - }, - "nativeDataType": "ENUM('M', 'F')", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "hire_date", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.employees,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.salaries,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:ba408413d97771e6470c16f9869f2e0d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.salaries,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": {}, - "externalUrl": null, - "name": "salaries", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "datacharmer.salaries", - "platform": "urn:li:dataPlatform:mysql", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "emp_no", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "jsonProps": null - }, - { - "fieldPath": "salary", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "from_date", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "jsonProps": null - }, - { - "fieldPath": "to_date", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.salaries,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:593ea3998729fdae4bdfb42206561a3a", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\", \"schema\": \"metagalaxy\"}, \"name\": \"metagalaxy\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:593ea3998729fdae4bdfb42206561a3a", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:593ea3998729fdae4bdfb42206561a3a", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:593ea3998729fdae4bdfb42206561a3a", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:17751259af32dd0385cad799df608c40\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:593ea3998729fdae4bdfb42206561a3a\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": {}, - "externalUrl": null, - "name": "metadata_aspect", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "metagalaxy.metadata_aspect", - "platform": "urn:li:dataPlatform:mysql", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "urn", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=500)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "jsonProps": null - }, - { - "fieldPath": "aspect", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=200)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "jsonProps": null - }, - { - "fieldPath": "version", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "BIGINT()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "jsonProps": null - }, - { - "fieldPath": "metadata", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "LONGTEXT()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "createdon", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME(fsp=6)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "createdby", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=255)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "createdfor", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=255)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "domains", - "aspect": { - "value": "{\"domains\": [\"urn:li:domain:sales\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:593ea3998729fdae4bdfb42206561a3a\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": {}, - "externalUrl": null, - "name": "metadata_index", - "qualifiedName": null, - "description": "This is a table comment", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "metagalaxy.metadata_index", - "platform": "urn:li:dataPlatform:mysql", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "BIGINT()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "jsonProps": null - }, - { - "fieldPath": "urn", - "jsonPath": null, - "nullable": false, - "description": "This is a column comment about URNs", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=200)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "aspect", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=150)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "path", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=150)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "longVal", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "BIGINT()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "stringVal", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=200)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "doubleVal", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "DOUBLE(asdecimal=True)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "domains", - "aspect": { - "value": "{\"domains\": [\"urn:li:domain:sales\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:593ea3998729fdae4bdfb42206561a3a\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "view_definition": "CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `metadata_index_view` AS select `metadata_index`.`id` AS `id`,`metadata_index`.`urn` AS `urn`,`metadata_index`.`path` AS `path`,`metadata_index`.`doubleVal` AS `doubleVal` from `metadata_index`", - "is_view": "True" - }, - "externalUrl": null, - "name": "metadata_index_view", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "metagalaxy.metadata_index_view", - "platform": "urn:li:dataPlatform:mysql", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "id", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "BIGINT()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "urn", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=200)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "path", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=150)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "doubleVal", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "DOUBLE(asdecimal=True)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "value": "{\"materialized\": false, \"viewLogic\": \"CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `metadata_index_view` AS select `metadata_index`.`id` AS `id`,`metadata_index`.`urn` AS `urn`,`metadata_index`.`path` AS `path`,`metadata_index`.`doubleVal` AS `doubleVal` from `metadata_index`\", \"viewLanguage\": \"SQL\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "domains", - "aspect": { - "value": "{\"domains\": [\"urn:li:domain:sales\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\", \"schema\": \"northwind\"}, \"name\": \"northwind\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:17751259af32dd0385cad799df608c40\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": {}, - "externalUrl": null, - "name": "customers", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "northwind.customers", - "platform": "urn:li:dataPlatform:mysql", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "jsonProps": null - }, - { - "fieldPath": "company", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=50)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=50)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=50)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "email_address", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=50)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "priority", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "FLOAT()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": {}, - "externalUrl": null, - "name": "orders", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "northwind.orders", - "platform": "urn:li:dataPlatform:mysql", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=50)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": [ - { - "name": "fk_order_customer", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),id)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),customer_id)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)" - } - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:989c003cbe689094c2b5c340a67f62be", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\", \"schema\": \"test_cases\"}, \"name\": \"test_cases\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:989c003cbe689094c2b5c340a67f62be", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:989c003cbe689094c2b5c340a67f62be", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:989c003cbe689094c2b5c340a67f62be", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:17751259af32dd0385cad799df608c40\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,test_cases.test_empty,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:989c003cbe689094c2b5c340a67f62be\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,test_cases.test_empty,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": {}, - "externalUrl": null, - "name": "test_empty", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test_cases.test_empty", - "platform": "urn:li:dataPlatform:mysql", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "dummy", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=50)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,test_cases.test_empty,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.employees,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 10, \"columnCount\": 6, \"fieldProfiles\": [{\"fieldPath\": \"emp_no\", \"uniqueCount\": 10, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"10001\", \"10002\", \"10003\", \"10004\", \"10005\", \"10006\", \"10007\", \"10008\", \"10009\", \"10010\"]}, {\"fieldPath\": \"birth_date\", \"uniqueCount\": 10, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"1952-04-19\", \"max\": \"1964-06-02\", \"sampleValues\": [\"1953-09-02\", \"1964-06-02\", \"1959-12-03\", \"1954-05-01\", \"1955-01-21\", \"1953-04-20\", \"1957-05-23\", \"1958-02-19\", \"1952-04-19\", \"1963-06-01\"]}, {\"fieldPath\": \"first_name\", \"uniqueCount\": 10, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Georgi\", \"Bezalel\", \"Parto\", \"Chirstian\", \"Kyoichi\", \"Anneke\", \"Tzvetan\", \"Saniya\", \"Sumant\", \"Duangkaew\"]}, {\"fieldPath\": \"last_name\", \"uniqueCount\": 10, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Facello\", \"Simmel\", \"Bamford\", \"Koblick\", \"Maliniak\", \"Preusig\", \"Zielinski\", \"Kalloufi\", \"Peac\", \"Piveteau\"]}, {\"fieldPath\": \"gender\", \"uniqueCount\": 2, \"uniqueProportion\": 0.2, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"M\", \"frequency\": 5}, {\"value\": \"F\", \"frequency\": 5}], \"sampleValues\": [\"M\", \"F\", \"M\", \"M\", \"M\", \"F\", \"F\", \"M\", \"F\", \"F\"]}, {\"fieldPath\": \"hire_date\", \"uniqueCount\": 10, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"1985-02-18\", \"max\": \"1994-09-15\", \"sampleValues\": [\"1986-06-26\", \"1985-11-21\", \"1986-08-28\", \"1986-12-01\", \"1989-09-12\", \"1989-06-02\", \"1989-02-10\", \"1994-09-15\", \"1985-02-18\", \"1989-08-24\"]}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.salaries,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 112, \"columnCount\": 4, \"fieldProfiles\": [{\"fieldPath\": \"emp_no\", \"uniqueCount\": 10, \"uniqueProportion\": 0.08928571428571429, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"10001\", \"max\": \"10010\", \"mean\": \"10005.3125\", \"median\": \"10005.0\", \"stdev\": \"2.834889609688869\", \"distinctValueFrequencies\": [{\"value\": \"10001\", \"frequency\": 17}, {\"value\": \"10002\", \"frequency\": 6}, {\"value\": \"10003\", \"frequency\": 7}, {\"value\": \"10004\", \"frequency\": 16}, {\"value\": \"10005\", \"frequency\": 13}, {\"value\": \"10006\", \"frequency\": 12}, {\"value\": \"10007\", \"frequency\": 14}, {\"value\": \"10008\", \"frequency\": 3}, {\"value\": \"10009\", \"frequency\": 18}, {\"value\": \"10010\", \"frequency\": 6}], \"sampleValues\": [\"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10002\", \"10002\", \"10002\"]}, {\"fieldPath\": \"salary\", \"uniqueCount\": 111, \"uniqueProportion\": 0.9910714285714286, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"40000\", \"max\": \"94692\", \"mean\": \"68303.11607142857\", \"median\": \"69544.0\", \"stdev\": \"15505.291475014095\", \"sampleValues\": [\"60117\", \"62102\", \"66074\", \"66596\", \"66961\", \"71046\", \"74333\", \"75286\", \"75994\", \"76884\", \"80013\", \"81025\", \"81097\", \"84917\", \"85112\", \"85097\", \"88958\", \"65909\", \"65909\", \"67534\"]}, {\"fieldPath\": \"from_date\", \"uniqueCount\": 106, \"uniqueProportion\": 0.9464285714285714, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"1985-02-18\", \"max\": \"2002-06-22\", \"sampleValues\": [\"1986-06-26\", \"1987-06-26\", \"1988-06-25\", \"1989-06-25\", \"1990-06-25\", \"1991-06-25\", \"1992-06-24\", \"1993-06-24\", \"1994-06-24\", \"1995-06-24\", \"1996-06-23\", \"1997-06-23\", \"1998-06-23\", \"1999-06-23\", \"2000-06-22\", \"2001-06-22\", \"2002-06-22\", \"1996-08-03\", \"1997-08-03\", \"1998-08-03\"]}, {\"fieldPath\": \"to_date\", \"uniqueCount\": 99, \"uniqueProportion\": 0.8839285714285714, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"1986-02-18\", \"max\": \"9999-01-01\", \"sampleValues\": [\"1987-06-26\", \"1988-06-25\", \"1989-06-25\", \"1990-06-25\", \"1991-06-25\", \"1992-06-24\", \"1993-06-24\", \"1994-06-24\", \"1995-06-24\", \"1996-06-23\", \"1997-06-23\", \"1998-06-23\", \"1999-06-23\", \"2000-06-22\", \"2001-06-22\", \"2002-06-22\", \"9999-01-01\", \"1997-08-03\", \"1998-08-03\", \"1999-08-03\"]}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 5, \"columnCount\": 6, \"fieldProfiles\": [{\"fieldPath\": \"id\", \"uniqueCount\": 5, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"1\", \"2\", \"3\", \"4\", \"5\"]}, {\"fieldPath\": \"company\", \"uniqueCount\": 5, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Company A\", \"Company B\", \"Company C\", \"Company D\", \"Company E\"]}, {\"fieldPath\": \"last_name\", \"uniqueCount\": 5, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Axen\", \"Bedecs\", \"Donnell\", \"Gratacos Solsona\", \"Lee\"]}, {\"fieldPath\": \"first_name\", \"uniqueCount\": 5, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Anna\", \"Antonio\", \"Christina\", \"Martin\", \"Thomas\"]}, {\"fieldPath\": \"email_address\", \"uniqueCount\": 0, \"nullCount\": 5, \"nullProportion\": 1.0, \"sampleValues\": []}, {\"fieldPath\": \"priority\", \"uniqueCount\": 3, \"uniqueProportion\": 0.75, \"nullCount\": 1, \"nullProportion\": 0.2, \"min\": \"3.8\", \"max\": \"4.9\", \"mean\": \"4.175000011920929\", \"median\": \"4.0\", \"distinctValueFrequencies\": [{\"value\": \"3.8\", \"frequency\": 1}, {\"value\": \"4.0\", \"frequency\": 2}, {\"value\": \"4.9\", \"frequency\": 1}], \"sampleValues\": [\"4.0\", \"4.9\", \"4.0\", \"3.8\"]}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 0, \"columnCount\": 3, \"fieldProfiles\": [{\"fieldPath\": \"id\", \"uniqueCount\": 0, \"nullCount\": 0, \"sampleValues\": []}, {\"fieldPath\": \"description\", \"uniqueCount\": 0, \"nullCount\": 0, \"sampleValues\": []}, {\"fieldPath\": \"customer_id\", \"uniqueCount\": 0, \"nullCount\": 0, \"sampleValues\": []}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,test_cases.test_empty,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 0, \"columnCount\": 1, \"fieldProfiles\": [{\"fieldPath\": \"dummy\", \"uniqueCount\": 0, \"nullCount\": 0, \"sampleValues\": []}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -} -] \ No newline at end of file