also deleted the old data now

motherduckdb · Mar 19, 2024 · 615d75e · 615d75e
1 parent c4f66f4
commit 615d75e
Show file tree

Hide file tree

Showing 98 changed files with 888 additions and 1,203 deletions.
diff --git a/data/iceberg/generated_spec1_0_01/base_file/file.parquet b/data/iceberg/generated_spec1_0_01/base_file/file.parquet
diff --git a/..._01/pyspark_iceberg_table/data/00000-1-ac638421-63d3-4413-a336-5446b5a8f47d-00001.parquet b/..._01/pyspark_iceberg_table/data/00000-1-ac638421-63d3-4413-a336-5446b5a8f47d-00001.parquet
diff --git a/...01/pyspark_iceberg_table/data/00000-10-54c9efc6-78d2-4d63-a78c-2c89408a829e-00001.parquet b/...01/pyspark_iceberg_table/data/00000-10-54c9efc6-78d2-4d63-a78c-2c89408a829e-00001.parquet
diff --git a/...01/pyspark_iceberg_table/data/00000-15-a7086310-03d4-43c8-a8cc-95de0c413675-00001.parquet b/...01/pyspark_iceberg_table/data/00000-15-a7086310-03d4-43c8-a8cc-95de0c413675-00001.parquet
diff --git a/...01/pyspark_iceberg_table/data/00000-28-de3e6ed7-06c5-458a-b97d-362d51ab8a42-00001.parquet b/...01/pyspark_iceberg_table/data/00000-28-de3e6ed7-06c5-458a-b97d-362d51ab8a42-00001.parquet
diff --git a/...01/pyspark_iceberg_table/data/00000-39-31cbd415-c80c-499d-bae0-d6666e2d2ce0-00001.parquet b/...01/pyspark_iceberg_table/data/00000-39-31cbd415-c80c-499d-bae0-d6666e2d2ce0-00001.parquet
diff --git a/..._01/pyspark_iceberg_table/data/00000-6-05eccbf6-a7b2-4ae3-9e06-4423a2c078a4-00001.parquet b/..._01/pyspark_iceberg_table/data/00000-6-05eccbf6-a7b2-4ae3-9e06-4423a2c078a4-00001.parquet
diff --git a/...01/pyspark_iceberg_table/data/00001-16-a7086310-03d4-43c8-a8cc-95de0c413675-00001.parquet b/...01/pyspark_iceberg_table/data/00001-16-a7086310-03d4-43c8-a8cc-95de0c413675-00001.parquet
diff --git a/...01/pyspark_iceberg_table/data/00001-29-de3e6ed7-06c5-458a-b97d-362d51ab8a42-00001.parquet b/...01/pyspark_iceberg_table/data/00001-29-de3e6ed7-06c5-458a-b97d-362d51ab8a42-00001.parquet
diff --git a/...01/pyspark_iceberg_table/data/00001-40-31cbd415-c80c-499d-bae0-d6666e2d2ce0-00001.parquet b/...01/pyspark_iceberg_table/data/00001-40-31cbd415-c80c-499d-bae0-d6666e2d2ce0-00001.parquet
diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v1.metadata.json.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v1.metadata.json.crc
diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v2.metadata.json.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v2.metadata.json.crc
diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v3.metadata.json.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v3.metadata.json.crc
diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v4.metadata.json.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v4.metadata.json.crc
diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v5.metadata.json.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v5.metadata.json.crc
diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v6.metadata.json.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v6.metadata.json.crc
diff --git a/...ed_spec1_0_01/pyspark_iceberg_table/metadata/15b81261-3cde-4c38-bcd2-351b5ca7794b-m0.avro b/...ed_spec1_0_01/pyspark_iceberg_table/metadata/15b81261-3cde-4c38-bcd2-351b5ca7794b-m0.avro
diff --git a/...ed_spec1_0_01/pyspark_iceberg_table/metadata/15b81261-3cde-4c38-bcd2-351b5ca7794b-m1.avro b/...ed_spec1_0_01/pyspark_iceberg_table/metadata/15b81261-3cde-4c38-bcd2-351b5ca7794b-m1.avro
diff --git a/...ed_spec1_0_01/pyspark_iceberg_table/metadata/3bf998f2-bd4d-49b6-ae6c-acb57adb4cfc-m0.avro b/...ed_spec1_0_01/pyspark_iceberg_table/metadata/3bf998f2-bd4d-49b6-ae6c-acb57adb4cfc-m0.avro
diff --git a/...ed_spec1_0_01/pyspark_iceberg_table/metadata/3bf998f2-bd4d-49b6-ae6c-acb57adb4cfc-m1.avro b/...ed_spec1_0_01/pyspark_iceberg_table/metadata/3bf998f2-bd4d-49b6-ae6c-acb57adb4cfc-m1.avro
diff --git a/...ed_spec1_0_01/pyspark_iceberg_table/metadata/3bf998f2-bd4d-49b6-ae6c-acb57adb4cfc-m2.avro b/...ed_spec1_0_01/pyspark_iceberg_table/metadata/3bf998f2-bd4d-49b6-ae6c-acb57adb4cfc-m2.avro
diff --git a/...ed_spec1_0_01/pyspark_iceberg_table/metadata/7ad6dba9-e1ab-40ee-8cda-b5c8a47d5e9d-m0.avro b/...ed_spec1_0_01/pyspark_iceberg_table/metadata/7ad6dba9-e1ab-40ee-8cda-b5c8a47d5e9d-m0.avro
diff --git a/...ed_spec1_0_01/pyspark_iceberg_table/metadata/c538d9f5-2e54-49f3-b5ca-f8e0ba7dc7dc-m0.avro b/...ed_spec1_0_01/pyspark_iceberg_table/metadata/c538d9f5-2e54-49f3-b5ca-f8e0ba7dc7dc-m0.avro
diff --git a/...ed_spec1_0_01/pyspark_iceberg_table/metadata/c538d9f5-2e54-49f3-b5ca-f8e0ba7dc7dc-m1.avro b/...ed_spec1_0_01/pyspark_iceberg_table/metadata/c538d9f5-2e54-49f3-b5ca-f8e0ba7dc7dc-m1.avro
diff --git a/...ed_spec1_0_01/pyspark_iceberg_table/metadata/d9dc56ec-773c-4da0-8361-515a11238cb7-m0.avro b/...ed_spec1_0_01/pyspark_iceberg_table/metadata/d9dc56ec-773c-4da0-8361-515a11238cb7-m0.avro
diff --git a/...ed_spec1_0_01/pyspark_iceberg_table/metadata/d9dc56ec-773c-4da0-8361-515a11238cb7-m1.avro b/...ed_spec1_0_01/pyspark_iceberg_table/metadata/d9dc56ec-773c-4da0-8361-515a11238cb7-m1.avro
diff --git a/...ed_spec1_0_01/pyspark_iceberg_table/metadata/ec20b9ca-c83b-4355-9f87-76cf737475aa-m0.avro b/...ed_spec1_0_01/pyspark_iceberg_table/metadata/ec20b9ca-c83b-4355-9f87-76cf737475aa-m0.avro
diff --git a/...ceberg_table/metadata/snap-106676854894038149-1-c538d9f5-2e54-49f3-b5ca-f8e0ba7dc7dc.avro b/...ceberg_table/metadata/snap-106676854894038149-1-c538d9f5-2e54-49f3-b5ca-f8e0ba7dc7dc.avro
diff --git a/...eberg_table/metadata/snap-1249147167734741613-1-3bf998f2-bd4d-49b6-ae6c-acb57adb4cfc.avro b/...eberg_table/metadata/snap-1249147167734741613-1-3bf998f2-bd4d-49b6-ae6c-acb57adb4cfc.avro
diff --git a/...eberg_table/metadata/snap-1253180028603952488-1-ec20b9ca-c83b-4355-9f87-76cf737475aa.avro b/...eberg_table/metadata/snap-1253180028603952488-1-ec20b9ca-c83b-4355-9f87-76cf737475aa.avro
diff --git a/...ceberg_table/metadata/snap-518151069684244348-1-f10347bc-2dce-4f91-85e3-3e54e8d85335.avro b/...ceberg_table/metadata/snap-518151069684244348-1-f10347bc-2dce-4f91-85e3-3e54e8d85335.avro
diff --git a/...eberg_table/metadata/snap-5804373497358040224-1-15b81261-3cde-4c38-bcd2-351b5ca7794b.avro b/...eberg_table/metadata/snap-5804373497358040224-1-15b81261-3cde-4c38-bcd2-351b5ca7794b.avro
diff --git a/...ceberg_table/metadata/snap-642225674660117625-1-d9dc56ec-773c-4da0-8361-515a11238cb7.avro b/...ceberg_table/metadata/snap-642225674660117625-1-d9dc56ec-773c-4da0-8361-515a11238cb7.avro
diff --git a/...eberg_table/metadata/snap-8945194127981408691-1-7ad6dba9-e1ab-40ee-8cda-b5c8a47d5e9d.avro b/...eberg_table/metadata/snap-8945194127981408691-1-7ad6dba9-e1ab-40ee-8cda-b5c8a47d5e9d.avro
diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json
@@ -1,8 +1,8 @@
 {
   "format-version" : 1,
-  "table-uuid" : "67928bdc-2a90-4fa8-abc0-cea23e04a108",
+  "table-uuid" : "f424ef42-5477-4841-b127-2f18e7dad530",
   "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table",
-  "last-updated-ms" : 1710802866455,
+  "last-updated-ms" : 1710862058098,
   "last-column-id" : 15,
   "schema" : {
     "type" : "struct",
@@ -181,37 +181,37 @@
     "owner" : "peter",
     "write.parquet.compression-codec" : "zstd"
   },
-  "current-snapshot-id" : 1253180028603952488,
+  "current-snapshot-id" : 1427538264954246454,
   "refs" : {
     "main" : {
-      "snapshot-id" : 1253180028603952488,
+      "snapshot-id" : 1427538264954246454,
       "type" : "branch"
     }
   },
   "snapshots" : [ {
-    "snapshot-id" : 1253180028603952488,
-    "timestamp-ms" : 1710802866455,
+    "snapshot-id" : 1427538264954246454,
+    "timestamp-ms" : 1710862058098,
     "summary" : {
       "operation" : "append",
-      "spark.app.id" : "local-1710802863322",
+      "spark.app.id" : "local-1710862055070",
       "added-data-files" : "1",
       "added-records" : "60175",
-      "added-files-size" : "4115168",
+      "added-files-size" : "4115457",
       "changed-partition-count" : "1",
       "total-records" : "60175",
-      "total-files-size" : "4115168",
+      "total-files-size" : "4115457",
       "total-data-files" : "1",
       "total-delete-files" : "0",
       "total-position-deletes" : "0",
       "total-equality-deletes" : "0"
     },
-    "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-1253180028603952488-1-ec20b9ca-c83b-4355-9f87-76cf737475aa.avro",
+    "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-1427538264954246454-1-bbab5b8f-dd0a-40e7-8ae6-af228cd96267.avro",
     "schema-id" : 0
   } ],
   "statistics" : [ ],
   "snapshot-log" : [ {
-    "timestamp-ms" : 1710802866455,
-    "snapshot-id" : 1253180028603952488
+    "timestamp-ms" : 1710862058098,
+    "snapshot-id" : 1427538264954246454
   } ],
   "metadata-log" : [ ]
 }
diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json
@@ -1,8 +1,8 @@
 {
   "format-version" : 1,
-  "table-uuid" : "67928bdc-2a90-4fa8-abc0-cea23e04a108",
+  "table-uuid" : "f424ef42-5477-4841-b127-2f18e7dad530",
   "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table",
-  "last-updated-ms" : 1710802867611,
+  "last-updated-ms" : 1710862059247,
   "last-column-id" : 15,
   "schema" : {
     "type" : "struct",
@@ -181,66 +181,66 @@
     "owner" : "peter",
     "write.parquet.compression-codec" : "zstd"
   },
-  "current-snapshot-id" : 106676854894038149,
+  "current-snapshot-id" : 9177163199645670044,
   "refs" : {
     "main" : {
-      "snapshot-id" : 106676854894038149,
+      "snapshot-id" : 9177163199645670044,
       "type" : "branch"
     }
   },
   "snapshots" : [ {
-    "snapshot-id" : 1253180028603952488,
-    "timestamp-ms" : 1710802866455,
+    "snapshot-id" : 1427538264954246454,
+    "timestamp-ms" : 1710862058098,
     "summary" : {
       "operation" : "append",
-      "spark.app.id" : "local-1710802863322",
+      "spark.app.id" : "local-1710862055070",
       "added-data-files" : "1",
       "added-records" : "60175",
-      "added-files-size" : "4115168",
+      "added-files-size" : "4115457",
       "changed-partition-count" : "1",
       "total-records" : "60175",
-      "total-files-size" : "4115168",
+      "total-files-size" : "4115457",
       "total-data-files" : "1",
       "total-delete-files" : "0",
       "total-position-deletes" : "0",
       "total-equality-deletes" : "0"
     },
-    "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-1253180028603952488-1-ec20b9ca-c83b-4355-9f87-76cf737475aa.avro",
+    "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-1427538264954246454-1-bbab5b8f-dd0a-40e7-8ae6-af228cd96267.avro",
     "schema-id" : 0
   }, {
-    "snapshot-id" : 106676854894038149,
-    "parent-snapshot-id" : 1253180028603952488,
-    "timestamp-ms" : 1710802867611,
+    "snapshot-id" : 9177163199645670044,
+    "parent-snapshot-id" : 1427538264954246454,
+    "timestamp-ms" : 1710862059247,
     "summary" : {
       "operation" : "overwrite",
-      "spark.app.id" : "local-1710802863322",
+      "spark.app.id" : "local-1710862055070",
       "added-data-files" : "1",
       "deleted-data-files" : "1",
       "added-records" : "60175",
       "deleted-records" : "60175",
-      "added-files-size" : "3207662",
-      "removed-files-size" : "4115168",
+      "added-files-size" : "3207951",
+      "removed-files-size" : "4115457",
       "changed-partition-count" : "1",
       "total-records" : "60175",
-      "total-files-size" : "3207662",
+      "total-files-size" : "3207951",
       "total-data-files" : "1",
       "total-delete-files" : "0",
       "total-position-deletes" : "0",
       "total-equality-deletes" : "0"
     },
-    "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-106676854894038149-1-c538d9f5-2e54-49f3-b5ca-f8e0ba7dc7dc.avro",
+    "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-9177163199645670044-1-6b161bb4-5fb9-4f06-8ccf-e1b9e0236893.avro",
     "schema-id" : 0
   } ],
   "statistics" : [ ],
   "snapshot-log" : [ {
-    "timestamp-ms" : 1710802866455,
-    "snapshot-id" : 1253180028603952488
+    "timestamp-ms" : 1710862058098,
+    "snapshot-id" : 1427538264954246454
   }, {
-    "timestamp-ms" : 1710802867611,
-    "snapshot-id" : 106676854894038149
+    "timestamp-ms" : 1710862059247,
+    "snapshot-id" : 9177163199645670044
   } ],
   "metadata-log" : [ {
-    "timestamp-ms" : 1710802866455,
+    "timestamp-ms" : 1710862058098,
     "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json"
   } ]
 }
diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json
@@ -1,8 +1,8 @@
 {
   "format-version" : 1,
-  "table-uuid" : "67928bdc-2a90-4fa8-abc0-cea23e04a108",
+  "table-uuid" : "f424ef42-5477-4841-b127-2f18e7dad530",
   "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table",
-  "last-updated-ms" : 1710802868418,
+  "last-updated-ms" : 1710862060042,
   "last-column-id" : 15,
   "schema" : {
     "type" : "struct",
@@ -181,92 +181,92 @@
     "owner" : "peter",
     "write.parquet.compression-codec" : "zstd"
   },
-  "current-snapshot-id" : 8945194127981408691,
+  "current-snapshot-id" : 7383008947754115722,
   "refs" : {
     "main" : {
-      "snapshot-id" : 8945194127981408691,
+      "snapshot-id" : 7383008947754115722,
       "type" : "branch"
     }
   },
   "snapshots" : [ {
-    "snapshot-id" : 1253180028603952488,
-    "timestamp-ms" : 1710802866455,
+    "snapshot-id" : 1427538264954246454,
+    "timestamp-ms" : 1710862058098,
     "summary" : {
       "operation" : "append",
-      "spark.app.id" : "local-1710802863322",
+      "spark.app.id" : "local-1710862055070",
       "added-data-files" : "1",
       "added-records" : "60175",
-      "added-files-size" : "4115168",
+      "added-files-size" : "4115457",
       "changed-partition-count" : "1",
       "total-records" : "60175",
-      "total-files-size" : "4115168",
+      "total-files-size" : "4115457",
       "total-data-files" : "1",
       "total-delete-files" : "0",
       "total-position-deletes" : "0",
       "total-equality-deletes" : "0"
     },
-    "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-1253180028603952488-1-ec20b9ca-c83b-4355-9f87-76cf737475aa.avro",
+    "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-1427538264954246454-1-bbab5b8f-dd0a-40e7-8ae6-af228cd96267.avro",
     "schema-id" : 0
   }, {
-    "snapshot-id" : 106676854894038149,
-    "parent-snapshot-id" : 1253180028603952488,
-    "timestamp-ms" : 1710802867611,
+    "snapshot-id" : 9177163199645670044,
+    "parent-snapshot-id" : 1427538264954246454,
+    "timestamp-ms" : 1710862059247,
     "summary" : {
       "operation" : "overwrite",
-      "spark.app.id" : "local-1710802863322",
+      "spark.app.id" : "local-1710862055070",
       "added-data-files" : "1",
       "deleted-data-files" : "1",
       "added-records" : "60175",
       "deleted-records" : "60175",
-      "added-files-size" : "3207662",
-      "removed-files-size" : "4115168",
+      "added-files-size" : "3207951",
+      "removed-files-size" : "4115457",
       "changed-partition-count" : "1",
       "total-records" : "60175",
-      "total-files-size" : "3207662",
+      "total-files-size" : "3207951",
       "total-data-files" : "1",
       "total-delete-files" : "0",
       "total-position-deletes" : "0",
       "total-equality-deletes" : "0"
     },
-    "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-106676854894038149-1-c538d9f5-2e54-49f3-b5ca-f8e0ba7dc7dc.avro",
+    "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-9177163199645670044-1-6b161bb4-5fb9-4f06-8ccf-e1b9e0236893.avro",
     "schema-id" : 0
   }, {
-    "snapshot-id" : 8945194127981408691,
-    "parent-snapshot-id" : 106676854894038149,
-    "timestamp-ms" : 1710802868418,
+    "snapshot-id" : 7383008947754115722,
+    "parent-snapshot-id" : 9177163199645670044,
+    "timestamp-ms" : 1710862060042,
     "summary" : {
       "operation" : "append",
-      "spark.app.id" : "local-1710802863322",
+      "spark.app.id" : "local-1710862055070",
       "added-data-files" : "1",
       "added-records" : "13126",
-      "added-files-size" : "933306",
+      "added-files-size" : "933466",
       "changed-partition-count" : "1",
       "total-records" : "73301",
-      "total-files-size" : "4140968",
+      "total-files-size" : "4141417",
       "total-data-files" : "2",
       "total-delete-files" : "0",
       "total-position-deletes" : "0",
       "total-equality-deletes" : "0"
     },
-    "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-8945194127981408691-1-7ad6dba9-e1ab-40ee-8cda-b5c8a47d5e9d.avro",
+    "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7383008947754115722-1-cf96b501-e4d6-43df-9bab-3f4ef5769510.avro",
     "schema-id" : 0
   } ],
   "statistics" : [ ],
   "snapshot-log" : [ {
-    "timestamp-ms" : 1710802866455,
-    "snapshot-id" : 1253180028603952488
+    "timestamp-ms" : 1710862058098,
+    "snapshot-id" : 1427538264954246454
   }, {
-    "timestamp-ms" : 1710802867611,
-    "snapshot-id" : 106676854894038149
+    "timestamp-ms" : 1710862059247,
+    "snapshot-id" : 9177163199645670044
   }, {
-    "timestamp-ms" : 1710802868418,
-    "snapshot-id" : 8945194127981408691
+    "timestamp-ms" : 1710862060042,
+    "snapshot-id" : 7383008947754115722
   } ],
   "metadata-log" : [ {
-    "timestamp-ms" : 1710802866455,
+    "timestamp-ms" : 1710862058098,
     "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json"
   }, {
-    "timestamp-ms" : 1710802867611,
+    "timestamp-ms" : 1710862059247,
     "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json"
   } ]
 }