fix(ingestion/glue): ensure date formatting works on all platforms fo…

…r aws glue (#10836)
datahub-project · Jul 3, 2024 · b8af2b9 · b8af2b9
1 parent e45f7a4
commit b8af2b9
Show file tree

Hide file tree

Showing 5 changed files with 115 additions and 115 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
@@ -901,7 +901,7 @@ def gen_database_containers(
             parameters["LocationUri"] = database["LocationUri"]
         if database.get("CreateTime") is not None:
             create_time: datetime.datetime = database["CreateTime"]
-            parameters["CreateTime"] = create_time.strftime("%B %-d, %Y at %H:%M:%S")
+            parameters["CreateTime"] = create_time.strftime("%B %d, %Y at %H:%M:%S")
         yield from gen_containers(
             container_key=database_container_key,
             name=database["Name"],

diff --git a/metadata-ingestion/tests/unit/glue/glue_delta_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_delta_mces_golden.json
@@ -11,7 +11,7 @@
                 "instance": "delta_platform_instance",
                 "env": "PROD",
                 "database": "delta-database",
-                "CreateTime": "June 9, 2021 at 14:14:19"
+                "CreateTime": "June 09, 2021 at 14:14:19"
             },
             "name": "delta-database",
             "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/delta-database"

diff --git a/metadata-ingestion/tests/unit/glue/glue_malformed_delta_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_malformed_delta_mces_golden.json
@@ -11,7 +11,7 @@
                 "instance": "delta_platform_instance",
                 "env": "PROD",
                 "database": "delta-database",
-                "CreateTime": "June 9, 2021 at 14:14:19"
+                "CreateTime": "June 09, 2021 at 14:14:19"
             },
             "name": "delta-database",
             "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/delta-database"

diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json
@@ -13,7 +13,7 @@
                 "param1": "value1",
                 "param2": "value2",
                 "LocationUri": "s3://test-bucket/test-prefix",
-                "CreateTime": "June 9, 2021 at 14:14:19"
+                "CreateTime": "June 09, 2021 at 14:14:19"
             },
             "name": "flights-database",
             "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database"
@@ -55,6 +55,59 @@
         }
     }
 },
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "glue",
+                "env": "PROD",
+                "database": "test-database",
+                "CreateTime": "June 01, 2021 at 14:55:02"
+            },
+            "name": "test-database",
+            "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database"
+        }
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:glue"
+        }
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Database"
+            ]
+        }
+    }
+},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7",
@@ -66,7 +119,7 @@
                 "platform": "glue",
                 "env": "PROD",
                 "database": "empty-database",
-                "CreateTime": "June 1, 2021 at 14:55:13"
+                "CreateTime": "June 01, 2021 at 14:55:13"
             },
             "name": "empty-database",
             "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database"
@@ -336,59 +389,6 @@
         }
     }
 },
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
-    "changeType": "UPSERT",
-    "aspectName": "containerProperties",
-    "aspect": {
-        "json": {
-            "customProperties": {
-                "platform": "glue",
-                "env": "PROD",
-                "database": "test-database",
-                "CreateTime": "June 1, 2021 at 14:55:02"
-            },
-            "name": "test-database",
-            "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database"
-        }
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
-    "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
-    "aspect": {
-        "json": {
-            "platform": "urn:li:dataPlatform:glue"
-        }
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "Database"
-            ]
-        }
-    }
-},
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {

diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json
@@ -14,7 +14,7 @@
                 "param1": "value1",
                 "param2": "value2",
                 "LocationUri": "s3://test-bucket/test-prefix",
-                "CreateTime": "June 9, 2021 at 14:14:19"
+                "CreateTime": "June 09, 2021 at 14:14:19"
             },
             "name": "flights-database",
             "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database"
@@ -57,6 +57,61 @@
         }
     }
 },
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "glue",
+                "instance": "some_instance_name",
+                "env": "PROD",
+                "database": "test-database",
+                "CreateTime": "June 01, 2021 at 14:55:02"
+            },
+            "name": "test-database",
+            "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database"
+        }
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:glue",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)"
+        }
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Database"
+            ]
+        }
+    }
+},
 {
     "entityType": "container",
     "entityUrn": "urn:li:container:ac4381240e82d55400c22e4392e744a4",
@@ -69,7 +124,7 @@
                 "instance": "some_instance_name",
                 "env": "PROD",
                 "database": "empty-database",
-                "CreateTime": "June 1, 2021 at 14:55:13"
+                "CreateTime": "June 01, 2021 at 14:55:13"
             },
             "name": "empty-database",
             "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database"
@@ -341,61 +396,6 @@
         }
     }
 },
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
-    "changeType": "UPSERT",
-    "aspectName": "containerProperties",
-    "aspect": {
-        "json": {
-            "customProperties": {
-                "platform": "glue",
-                "instance": "some_instance_name",
-                "env": "PROD",
-                "database": "test-database",
-                "CreateTime": "June 1, 2021 at 14:55:02"
-            },
-            "name": "test-database",
-            "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database"
-        }
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
-    "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
-    "aspect": {
-        "json": {
-            "platform": "urn:li:dataPlatform:glue",
-            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)"
-        }
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "Database"
-            ]
-        }
-    }
-},
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {