diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00000-1-0eed84dc-1041-4abd-b9d0-e962a205c9c7-00001.parquet.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00000-1-0eed84dc-1041-4abd-b9d0-e962a205c9c7-00001.parquet.crc new file mode 100644 index 0000000..c6e9037 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00000-1-0eed84dc-1041-4abd-b9d0-e962a205c9c7-00001.parquet.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00000-10-ff6b10ea-872b-45cf-afe3-b72a51dc50b6-00001.parquet.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00000-10-ff6b10ea-872b-45cf-afe3-b72a51dc50b6-00001.parquet.crc new file mode 100644 index 0000000..4493794 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00000-10-ff6b10ea-872b-45cf-afe3-b72a51dc50b6-00001.parquet.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00000-15-081e45d9-4d32-40fe-960e-d04b49236ff2-00001.parquet.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00000-15-081e45d9-4d32-40fe-960e-d04b49236ff2-00001.parquet.crc new file mode 100644 index 0000000..c0e0bdc Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00000-15-081e45d9-4d32-40fe-960e-d04b49236ff2-00001.parquet.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00000-28-41f607a2-a570-438f-8c8e-d03cefe4d340-00001.parquet.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00000-28-41f607a2-a570-438f-8c8e-d03cefe4d340-00001.parquet.crc new file mode 100644 index 0000000..3be2d5e Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00000-28-41f607a2-a570-438f-8c8e-d03cefe4d340-00001.parquet.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00000-39-88bcb0b8-8f76-4663-9b64-e49735600635-00001.parquet.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00000-39-88bcb0b8-8f76-4663-9b64-e49735600635-00001.parquet.crc new file mode 100644 index 0000000..562a4a4 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00000-39-88bcb0b8-8f76-4663-9b64-e49735600635-00001.parquet.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00000-6-0d140de0-7c9a-460b-8e08-1ee4284451ef-00001.parquet.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00000-6-0d140de0-7c9a-460b-8e08-1ee4284451ef-00001.parquet.crc new file mode 100644 index 0000000..715739b Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00000-6-0d140de0-7c9a-460b-8e08-1ee4284451ef-00001.parquet.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00001-16-081e45d9-4d32-40fe-960e-d04b49236ff2-00001.parquet.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00001-16-081e45d9-4d32-40fe-960e-d04b49236ff2-00001.parquet.crc new file mode 100644 index 0000000..9f9477f Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00001-16-081e45d9-4d32-40fe-960e-d04b49236ff2-00001.parquet.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00001-29-41f607a2-a570-438f-8c8e-d03cefe4d340-00001.parquet.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00001-29-41f607a2-a570-438f-8c8e-d03cefe4d340-00001.parquet.crc new file mode 100644 index 0000000..1d5cbff Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00001-29-41f607a2-a570-438f-8c8e-d03cefe4d340-00001.parquet.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00001-40-88bcb0b8-8f76-4663-9b64-e49735600635-00001.parquet.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00001-40-88bcb0b8-8f76-4663-9b64-e49735600635-00001.parquet.crc new file mode 100644 index 0000000..fa3394f Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/.00001-40-88bcb0b8-8f76-4663-9b64-e49735600635-00001.parquet.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00000-1-0eed84dc-1041-4abd-b9d0-e962a205c9c7-00001.parquet b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00000-1-0eed84dc-1041-4abd-b9d0-e962a205c9c7-00001.parquet new file mode 100644 index 0000000..7437b4a Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00000-1-0eed84dc-1041-4abd-b9d0-e962a205c9c7-00001.parquet differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00000-10-ff6b10ea-872b-45cf-afe3-b72a51dc50b6-00001.parquet b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00000-10-ff6b10ea-872b-45cf-afe3-b72a51dc50b6-00001.parquet new file mode 100644 index 0000000..29553da Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00000-10-ff6b10ea-872b-45cf-afe3-b72a51dc50b6-00001.parquet differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00000-15-081e45d9-4d32-40fe-960e-d04b49236ff2-00001.parquet b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00000-15-081e45d9-4d32-40fe-960e-d04b49236ff2-00001.parquet new file mode 100644 index 0000000..92a72f9 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00000-15-081e45d9-4d32-40fe-960e-d04b49236ff2-00001.parquet differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00000-28-41f607a2-a570-438f-8c8e-d03cefe4d340-00001.parquet b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00000-28-41f607a2-a570-438f-8c8e-d03cefe4d340-00001.parquet new file mode 100644 index 0000000..8a061ec Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00000-28-41f607a2-a570-438f-8c8e-d03cefe4d340-00001.parquet differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00000-39-88bcb0b8-8f76-4663-9b64-e49735600635-00001.parquet b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00000-39-88bcb0b8-8f76-4663-9b64-e49735600635-00001.parquet new file mode 100644 index 0000000..a0d6310 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00000-39-88bcb0b8-8f76-4663-9b64-e49735600635-00001.parquet differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00000-6-0d140de0-7c9a-460b-8e08-1ee4284451ef-00001.parquet b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00000-6-0d140de0-7c9a-460b-8e08-1ee4284451ef-00001.parquet new file mode 100644 index 0000000..b4fff2f Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00000-6-0d140de0-7c9a-460b-8e08-1ee4284451ef-00001.parquet differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00001-16-081e45d9-4d32-40fe-960e-d04b49236ff2-00001.parquet b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00001-16-081e45d9-4d32-40fe-960e-d04b49236ff2-00001.parquet new file mode 100644 index 0000000..c7c436c Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00001-16-081e45d9-4d32-40fe-960e-d04b49236ff2-00001.parquet differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00001-29-41f607a2-a570-438f-8c8e-d03cefe4d340-00001.parquet b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00001-29-41f607a2-a570-438f-8c8e-d03cefe4d340-00001.parquet new file mode 100644 index 0000000..2820775 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00001-29-41f607a2-a570-438f-8c8e-d03cefe4d340-00001.parquet differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00001-40-88bcb0b8-8f76-4663-9b64-e49735600635-00001.parquet b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00001-40-88bcb0b8-8f76-4663-9b64-e49735600635-00001.parquet new file mode 100644 index 0000000..678077b Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/data/00001-40-88bcb0b8-8f76-4663-9b64-e49735600635-00001.parquet differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.4460dd70-b98d-49a9-b690-05b8cbc84ea9-m0.avro.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.4460dd70-b98d-49a9-b690-05b8cbc84ea9-m0.avro.crc new file mode 100644 index 0000000..3dd380c Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.4460dd70-b98d-49a9-b690-05b8cbc84ea9-m0.avro.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.4460dd70-b98d-49a9-b690-05b8cbc84ea9-m1.avro.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.4460dd70-b98d-49a9-b690-05b8cbc84ea9-m1.avro.crc new file mode 100644 index 0000000..4a7edca Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.4460dd70-b98d-49a9-b690-05b8cbc84ea9-m1.avro.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.5cf4c8f1-c28a-448c-a9ad-71b2920cf145-m0.avro.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.5cf4c8f1-c28a-448c-a9ad-71b2920cf145-m0.avro.crc new file mode 100644 index 0000000..cf5b373 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.5cf4c8f1-c28a-448c-a9ad-71b2920cf145-m0.avro.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.5cf4c8f1-c28a-448c-a9ad-71b2920cf145-m1.avro.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.5cf4c8f1-c28a-448c-a9ad-71b2920cf145-m1.avro.crc new file mode 100644 index 0000000..4fc65fc Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.5cf4c8f1-c28a-448c-a9ad-71b2920cf145-m1.avro.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.6b161bb4-5fb9-4f06-8ccf-e1b9e0236893-m0.avro.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.6b161bb4-5fb9-4f06-8ccf-e1b9e0236893-m0.avro.crc new file mode 100644 index 0000000..76ddb2d Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.6b161bb4-5fb9-4f06-8ccf-e1b9e0236893-m0.avro.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.6b161bb4-5fb9-4f06-8ccf-e1b9e0236893-m1.avro.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.6b161bb4-5fb9-4f06-8ccf-e1b9e0236893-m1.avro.crc new file mode 100644 index 0000000..352da42 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.6b161bb4-5fb9-4f06-8ccf-e1b9e0236893-m1.avro.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.a4381dd0-8507-4690-9710-4e412acce18c-m0.avro.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.a4381dd0-8507-4690-9710-4e412acce18c-m0.avro.crc new file mode 100644 index 0000000..dfd5168 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.a4381dd0-8507-4690-9710-4e412acce18c-m0.avro.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.a4381dd0-8507-4690-9710-4e412acce18c-m1.avro.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.a4381dd0-8507-4690-9710-4e412acce18c-m1.avro.crc new file mode 100644 index 0000000..ac3ef3a Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.a4381dd0-8507-4690-9710-4e412acce18c-m1.avro.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.a4381dd0-8507-4690-9710-4e412acce18c-m2.avro.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.a4381dd0-8507-4690-9710-4e412acce18c-m2.avro.crc new file mode 100644 index 0000000..023b960 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.a4381dd0-8507-4690-9710-4e412acce18c-m2.avro.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.bbab5b8f-dd0a-40e7-8ae6-af228cd96267-m0.avro.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.bbab5b8f-dd0a-40e7-8ae6-af228cd96267-m0.avro.crc new file mode 100644 index 0000000..ac3417e Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.bbab5b8f-dd0a-40e7-8ae6-af228cd96267-m0.avro.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.cf96b501-e4d6-43df-9bab-3f4ef5769510-m0.avro.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.cf96b501-e4d6-43df-9bab-3f4ef5769510-m0.avro.crc new file mode 100644 index 0000000..db12aaa Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.cf96b501-e4d6-43df-9bab-3f4ef5769510-m0.avro.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-1427538264954246454-1-bbab5b8f-dd0a-40e7-8ae6-af228cd96267.avro.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-1427538264954246454-1-bbab5b8f-dd0a-40e7-8ae6-af228cd96267.avro.crc new file mode 100644 index 0000000..5bd5f2d Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-1427538264954246454-1-bbab5b8f-dd0a-40e7-8ae6-af228cd96267.avro.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-182121606153922595-1-4460dd70-b98d-49a9-b690-05b8cbc84ea9.avro.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-182121606153922595-1-4460dd70-b98d-49a9-b690-05b8cbc84ea9.avro.crc new file mode 100644 index 0000000..0a584a5 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-182121606153922595-1-4460dd70-b98d-49a9-b690-05b8cbc84ea9.avro.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-4614971797487332201-1-5cf4c8f1-c28a-448c-a9ad-71b2920cf145.avro.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-4614971797487332201-1-5cf4c8f1-c28a-448c-a9ad-71b2920cf145.avro.crc new file mode 100644 index 0000000..2fe3c78 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-4614971797487332201-1-5cf4c8f1-c28a-448c-a9ad-71b2920cf145.avro.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-6939941997387914529-1-a4381dd0-8507-4690-9710-4e412acce18c.avro.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-6939941997387914529-1-a4381dd0-8507-4690-9710-4e412acce18c.avro.crc new file mode 100644 index 0000000..969aa64 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-6939941997387914529-1-a4381dd0-8507-4690-9710-4e412acce18c.avro.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-7334263894966731819-1-a7f34407-3101-4480-b866-3721aca735e7.avro.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-7334263894966731819-1-a7f34407-3101-4480-b866-3721aca735e7.avro.crc new file mode 100644 index 0000000..85c8d5b Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-7334263894966731819-1-a7f34407-3101-4480-b866-3721aca735e7.avro.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-7383008947754115722-1-cf96b501-e4d6-43df-9bab-3f4ef5769510.avro.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-7383008947754115722-1-cf96b501-e4d6-43df-9bab-3f4ef5769510.avro.crc new file mode 100644 index 0000000..039a7de Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-7383008947754115722-1-cf96b501-e4d6-43df-9bab-3f4ef5769510.avro.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-9177163199645670044-1-6b161bb4-5fb9-4f06-8ccf-e1b9e0236893.avro.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-9177163199645670044-1-6b161bb4-5fb9-4f06-8ccf-e1b9e0236893.avro.crc new file mode 100644 index 0000000..baf6952 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.snap-9177163199645670044-1-6b161bb4-5fb9-4f06-8ccf-e1b9e0236893.avro.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v1.metadata.json.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v1.metadata.json.crc new file mode 100644 index 0000000..1b92448 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v1.metadata.json.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v2.metadata.json.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v2.metadata.json.crc new file mode 100644 index 0000000..d8f2f0d Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v2.metadata.json.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v3.metadata.json.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v3.metadata.json.crc new file mode 100644 index 0000000..c17f9fd Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v3.metadata.json.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v4.metadata.json.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v4.metadata.json.crc new file mode 100644 index 0000000..bb2eda1 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v4.metadata.json.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v5.metadata.json.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v5.metadata.json.crc new file mode 100644 index 0000000..c5f2bf5 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v5.metadata.json.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v6.metadata.json.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v6.metadata.json.crc new file mode 100644 index 0000000..4af87d5 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v6.metadata.json.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v7.metadata.json.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v7.metadata.json.crc new file mode 100644 index 0000000..02a89c0 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v7.metadata.json.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v8.metadata.json.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v8.metadata.json.crc new file mode 100644 index 0000000..465a363 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v8.metadata.json.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v9.metadata.json.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v9.metadata.json.crc new file mode 100644 index 0000000..f1ea545 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.v9.metadata.json.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.version-hint.text.crc b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.version-hint.text.crc new file mode 100644 index 0000000..1f8d85e Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/.version-hint.text.crc differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/4460dd70-b98d-49a9-b690-05b8cbc84ea9-m0.avro b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/4460dd70-b98d-49a9-b690-05b8cbc84ea9-m0.avro new file mode 100644 index 0000000..dffdada Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/4460dd70-b98d-49a9-b690-05b8cbc84ea9-m0.avro differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/4460dd70-b98d-49a9-b690-05b8cbc84ea9-m1.avro b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/4460dd70-b98d-49a9-b690-05b8cbc84ea9-m1.avro new file mode 100644 index 0000000..be13fd7 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/4460dd70-b98d-49a9-b690-05b8cbc84ea9-m1.avro differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/5cf4c8f1-c28a-448c-a9ad-71b2920cf145-m0.avro b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/5cf4c8f1-c28a-448c-a9ad-71b2920cf145-m0.avro new file mode 100644 index 0000000..9945552 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/5cf4c8f1-c28a-448c-a9ad-71b2920cf145-m0.avro differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/5cf4c8f1-c28a-448c-a9ad-71b2920cf145-m1.avro b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/5cf4c8f1-c28a-448c-a9ad-71b2920cf145-m1.avro new file mode 100644 index 0000000..3599dac Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/5cf4c8f1-c28a-448c-a9ad-71b2920cf145-m1.avro differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/6b161bb4-5fb9-4f06-8ccf-e1b9e0236893-m0.avro b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/6b161bb4-5fb9-4f06-8ccf-e1b9e0236893-m0.avro new file mode 100644 index 0000000..e702fe2 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/6b161bb4-5fb9-4f06-8ccf-e1b9e0236893-m0.avro differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/6b161bb4-5fb9-4f06-8ccf-e1b9e0236893-m1.avro b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/6b161bb4-5fb9-4f06-8ccf-e1b9e0236893-m1.avro new file mode 100644 index 0000000..255ca0a Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/6b161bb4-5fb9-4f06-8ccf-e1b9e0236893-m1.avro differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/a4381dd0-8507-4690-9710-4e412acce18c-m0.avro b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/a4381dd0-8507-4690-9710-4e412acce18c-m0.avro new file mode 100644 index 0000000..4dc663e Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/a4381dd0-8507-4690-9710-4e412acce18c-m0.avro differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/a4381dd0-8507-4690-9710-4e412acce18c-m1.avro b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/a4381dd0-8507-4690-9710-4e412acce18c-m1.avro new file mode 100644 index 0000000..f1d31f0 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/a4381dd0-8507-4690-9710-4e412acce18c-m1.avro differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/a4381dd0-8507-4690-9710-4e412acce18c-m2.avro b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/a4381dd0-8507-4690-9710-4e412acce18c-m2.avro new file mode 100644 index 0000000..afb62e4 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/a4381dd0-8507-4690-9710-4e412acce18c-m2.avro differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/bbab5b8f-dd0a-40e7-8ae6-af228cd96267-m0.avro b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/bbab5b8f-dd0a-40e7-8ae6-af228cd96267-m0.avro new file mode 100644 index 0000000..001d0a9 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/bbab5b8f-dd0a-40e7-8ae6-af228cd96267-m0.avro differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/cf96b501-e4d6-43df-9bab-3f4ef5769510-m0.avro b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/cf96b501-e4d6-43df-9bab-3f4ef5769510-m0.avro new file mode 100644 index 0000000..a859aee Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/cf96b501-e4d6-43df-9bab-3f4ef5769510-m0.avro differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-1427538264954246454-1-bbab5b8f-dd0a-40e7-8ae6-af228cd96267.avro b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-1427538264954246454-1-bbab5b8f-dd0a-40e7-8ae6-af228cd96267.avro new file mode 100644 index 0000000..59d6e6d Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-1427538264954246454-1-bbab5b8f-dd0a-40e7-8ae6-af228cd96267.avro differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-182121606153922595-1-4460dd70-b98d-49a9-b690-05b8cbc84ea9.avro b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-182121606153922595-1-4460dd70-b98d-49a9-b690-05b8cbc84ea9.avro new file mode 100644 index 0000000..f00d84a Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-182121606153922595-1-4460dd70-b98d-49a9-b690-05b8cbc84ea9.avro differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-4614971797487332201-1-5cf4c8f1-c28a-448c-a9ad-71b2920cf145.avro b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-4614971797487332201-1-5cf4c8f1-c28a-448c-a9ad-71b2920cf145.avro new file mode 100644 index 0000000..215c43a Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-4614971797487332201-1-5cf4c8f1-c28a-448c-a9ad-71b2920cf145.avro differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-6939941997387914529-1-a4381dd0-8507-4690-9710-4e412acce18c.avro b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-6939941997387914529-1-a4381dd0-8507-4690-9710-4e412acce18c.avro new file mode 100644 index 0000000..d6a3155 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-6939941997387914529-1-a4381dd0-8507-4690-9710-4e412acce18c.avro differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7334263894966731819-1-a7f34407-3101-4480-b866-3721aca735e7.avro b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7334263894966731819-1-a7f34407-3101-4480-b866-3721aca735e7.avro new file mode 100644 index 0000000..3315e7c Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7334263894966731819-1-a7f34407-3101-4480-b866-3721aca735e7.avro differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7383008947754115722-1-cf96b501-e4d6-43df-9bab-3f4ef5769510.avro b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7383008947754115722-1-cf96b501-e4d6-43df-9bab-3f4ef5769510.avro new file mode 100644 index 0000000..8574da0 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7383008947754115722-1-cf96b501-e4d6-43df-9bab-3f4ef5769510.avro differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-9177163199645670044-1-6b161bb4-5fb9-4f06-8ccf-e1b9e0236893.avro b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-9177163199645670044-1-6b161bb4-5fb9-4f06-8ccf-e1b9e0236893.avro new file mode 100644 index 0000000..8e14a81 Binary files /dev/null and b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-9177163199645670044-1-6b161bb4-5fb9-4f06-8ccf-e1b9e0236893.avro differ diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json new file mode 100644 index 0000000..bd18f45 --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json @@ -0,0 +1,217 @@ +{ + "format-version" : 1, + "table-uuid" : "f424ef42-5477-4841-b127-2f18e7dad530", + "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table", + "last-updated-ms" : 1710862058098, + "last-column-id" : 15, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 1427538264954246454, + "refs" : { + "main" : { + "snapshot-id" : 1427538264954246454, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 1427538264954246454, + "timestamp-ms" : 1710862058098, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "added-records" : "60175", + "added-files-size" : "4115457", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "4115457", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-1427538264954246454-1-bbab5b8f-dd0a-40e7-8ae6-af228cd96267.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1710862058098, + "snapshot-id" : 1427538264954246454 + } ], + "metadata-log" : [ ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json new file mode 100644 index 0000000..ddbbd5b --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json @@ -0,0 +1,246 @@ +{ + "format-version" : 1, + "table-uuid" : "f424ef42-5477-4841-b127-2f18e7dad530", + "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table", + "last-updated-ms" : 1710862059247, + "last-column-id" : 15, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 9177163199645670044, + "refs" : { + "main" : { + "snapshot-id" : 9177163199645670044, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 1427538264954246454, + "timestamp-ms" : 1710862058098, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "added-records" : "60175", + "added-files-size" : "4115457", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "4115457", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-1427538264954246454-1-bbab5b8f-dd0a-40e7-8ae6-af228cd96267.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 9177163199645670044, + "parent-snapshot-id" : 1427538264954246454, + "timestamp-ms" : 1710862059247, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "60175", + "deleted-records" : "60175", + "added-files-size" : "3207951", + "removed-files-size" : "4115457", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "3207951", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-9177163199645670044-1-6b161bb4-5fb9-4f06-8ccf-e1b9e0236893.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1710862058098, + "snapshot-id" : 1427538264954246454 + }, { + "timestamp-ms" : 1710862059247, + "snapshot-id" : 9177163199645670044 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1710862058098, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json new file mode 100644 index 0000000..250657e --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json @@ -0,0 +1,272 @@ +{ + "format-version" : 1, + "table-uuid" : "f424ef42-5477-4841-b127-2f18e7dad530", + "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table", + "last-updated-ms" : 1710862060042, + "last-column-id" : 15, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 7383008947754115722, + "refs" : { + "main" : { + "snapshot-id" : 7383008947754115722, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 1427538264954246454, + "timestamp-ms" : 1710862058098, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "added-records" : "60175", + "added-files-size" : "4115457", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "4115457", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-1427538264954246454-1-bbab5b8f-dd0a-40e7-8ae6-af228cd96267.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 9177163199645670044, + "parent-snapshot-id" : 1427538264954246454, + "timestamp-ms" : 1710862059247, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "60175", + "deleted-records" : "60175", + "added-files-size" : "3207951", + "removed-files-size" : "4115457", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "3207951", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-9177163199645670044-1-6b161bb4-5fb9-4f06-8ccf-e1b9e0236893.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 7383008947754115722, + "parent-snapshot-id" : 9177163199645670044, + "timestamp-ms" : 1710862060042, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "added-records" : "13126", + "added-files-size" : "933466", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4141417", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7383008947754115722-1-cf96b501-e4d6-43df-9bab-3f4ef5769510.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1710862058098, + "snapshot-id" : 1427538264954246454 + }, { + "timestamp-ms" : 1710862059247, + "snapshot-id" : 9177163199645670044 + }, { + "timestamp-ms" : 1710862060042, + "snapshot-id" : 7383008947754115722 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1710862058098, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1710862059247, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v4.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v4.metadata.json new file mode 100644 index 0000000..e4c3123 --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v4.metadata.json @@ -0,0 +1,301 @@ +{ + "format-version" : 1, + "table-uuid" : "f424ef42-5477-4841-b127-2f18e7dad530", + "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table", + "last-updated-ms" : 1710862060844, + "last-column-id" : 15, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 6939941997387914529, + "refs" : { + "main" : { + "snapshot-id" : 6939941997387914529, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 1427538264954246454, + "timestamp-ms" : 1710862058098, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "added-records" : "60175", + "added-files-size" : "4115457", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "4115457", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-1427538264954246454-1-bbab5b8f-dd0a-40e7-8ae6-af228cd96267.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 9177163199645670044, + "parent-snapshot-id" : 1427538264954246454, + "timestamp-ms" : 1710862059247, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "60175", + "deleted-records" : "60175", + "added-files-size" : "3207951", + "removed-files-size" : "4115457", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "3207951", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-9177163199645670044-1-6b161bb4-5fb9-4f06-8ccf-e1b9e0236893.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 7383008947754115722, + "parent-snapshot-id" : 9177163199645670044, + "timestamp-ms" : 1710862060042, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "added-records" : "13126", + "added-files-size" : "933466", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4141417", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7383008947754115722-1-cf96b501-e4d6-43df-9bab-3f4ef5769510.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 6939941997387914529, + "parent-snapshot-id" : 7383008947754115722, + "timestamp-ms" : 1710862060844, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "2", + "deleted-data-files" : "2", + "added-records" : "73301", + "deleted-records" : "73301", + "added-files-size" : "4141417", + "removed-files-size" : "4141417", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4141417", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-6939941997387914529-1-a4381dd0-8507-4690-9710-4e412acce18c.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1710862058098, + "snapshot-id" : 1427538264954246454 + }, { + "timestamp-ms" : 1710862059247, + "snapshot-id" : 9177163199645670044 + }, { + "timestamp-ms" : 1710862060042, + "snapshot-id" : 7383008947754115722 + }, { + "timestamp-ms" : 1710862060844, + "snapshot-id" : 6939941997387914529 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1710862058098, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1710862059247, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1710862060042, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v5.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v5.metadata.json new file mode 100644 index 0000000..5b3b5e6 --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v5.metadata.json @@ -0,0 +1,324 @@ +{ + "format-version" : 1, + "table-uuid" : "f424ef42-5477-4841-b127-2f18e7dad530", + "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table", + "last-updated-ms" : 1710862061286, + "last-column-id" : 15, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 7334263894966731819, + "refs" : { + "main" : { + "snapshot-id" : 7334263894966731819, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 1427538264954246454, + "timestamp-ms" : 1710862058098, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "added-records" : "60175", + "added-files-size" : "4115457", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "4115457", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-1427538264954246454-1-bbab5b8f-dd0a-40e7-8ae6-af228cd96267.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 9177163199645670044, + "parent-snapshot-id" : 1427538264954246454, + "timestamp-ms" : 1710862059247, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "60175", + "deleted-records" : "60175", + "added-files-size" : "3207951", + "removed-files-size" : "4115457", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "3207951", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-9177163199645670044-1-6b161bb4-5fb9-4f06-8ccf-e1b9e0236893.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 7383008947754115722, + "parent-snapshot-id" : 9177163199645670044, + "timestamp-ms" : 1710862060042, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "added-records" : "13126", + "added-files-size" : "933466", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4141417", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7383008947754115722-1-cf96b501-e4d6-43df-9bab-3f4ef5769510.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 6939941997387914529, + "parent-snapshot-id" : 7383008947754115722, + "timestamp-ms" : 1710862060844, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "2", + "deleted-data-files" : "2", + "added-records" : "73301", + "deleted-records" : "73301", + "added-files-size" : "4141417", + "removed-files-size" : "4141417", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4141417", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-6939941997387914529-1-a4381dd0-8507-4690-9710-4e412acce18c.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 7334263894966731819, + "parent-snapshot-id" : 6939941997387914529, + "timestamp-ms" : 1710862061286, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "changed-partition-count" : "0", + "total-records" : "73301", + "total-files-size" : "4141417", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7334263894966731819-1-a7f34407-3101-4480-b866-3721aca735e7.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1710862058098, + "snapshot-id" : 1427538264954246454 + }, { + "timestamp-ms" : 1710862059247, + "snapshot-id" : 9177163199645670044 + }, { + "timestamp-ms" : 1710862060042, + "snapshot-id" : 7383008947754115722 + }, { + "timestamp-ms" : 1710862060844, + "snapshot-id" : 6939941997387914529 + }, { + "timestamp-ms" : 1710862061286, + "snapshot-id" : 7334263894966731819 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1710862058098, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1710862059247, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1710862060042, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1710862060844, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v6.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v6.metadata.json new file mode 100644 index 0000000..ef663a5 --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v6.metadata.json @@ -0,0 +1,353 @@ +{ + "format-version" : 1, + "table-uuid" : "f424ef42-5477-4841-b127-2f18e7dad530", + "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table", + "last-updated-ms" : 1710862062007, + "last-column-id" : 15, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 182121606153922595, + "refs" : { + "main" : { + "snapshot-id" : 182121606153922595, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 1427538264954246454, + "timestamp-ms" : 1710862058098, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "added-records" : "60175", + "added-files-size" : "4115457", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "4115457", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-1427538264954246454-1-bbab5b8f-dd0a-40e7-8ae6-af228cd96267.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 9177163199645670044, + "parent-snapshot-id" : 1427538264954246454, + "timestamp-ms" : 1710862059247, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "60175", + "deleted-records" : "60175", + "added-files-size" : "3207951", + "removed-files-size" : "4115457", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "3207951", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-9177163199645670044-1-6b161bb4-5fb9-4f06-8ccf-e1b9e0236893.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 7383008947754115722, + "parent-snapshot-id" : 9177163199645670044, + "timestamp-ms" : 1710862060042, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "added-records" : "13126", + "added-files-size" : "933466", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4141417", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7383008947754115722-1-cf96b501-e4d6-43df-9bab-3f4ef5769510.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 6939941997387914529, + "parent-snapshot-id" : 7383008947754115722, + "timestamp-ms" : 1710862060844, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "2", + "deleted-data-files" : "2", + "added-records" : "73301", + "deleted-records" : "73301", + "added-files-size" : "4141417", + "removed-files-size" : "4141417", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4141417", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-6939941997387914529-1-a4381dd0-8507-4690-9710-4e412acce18c.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 7334263894966731819, + "parent-snapshot-id" : 6939941997387914529, + "timestamp-ms" : 1710862061286, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "changed-partition-count" : "0", + "total-records" : "73301", + "total-files-size" : "4141417", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7334263894966731819-1-a7f34407-3101-4480-b866-3721aca735e7.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 182121606153922595, + "parent-snapshot-id" : 7334263894966731819, + "timestamp-ms" : 1710862062007, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "2", + "deleted-data-files" : "2", + "added-records" : "73301", + "deleted-records" : "73301", + "added-files-size" : "4141415", + "removed-files-size" : "4141417", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4141415", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-182121606153922595-1-4460dd70-b98d-49a9-b690-05b8cbc84ea9.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1710862058098, + "snapshot-id" : 1427538264954246454 + }, { + "timestamp-ms" : 1710862059247, + "snapshot-id" : 9177163199645670044 + }, { + "timestamp-ms" : 1710862060042, + "snapshot-id" : 7383008947754115722 + }, { + "timestamp-ms" : 1710862060844, + "snapshot-id" : 6939941997387914529 + }, { + "timestamp-ms" : 1710862061286, + "snapshot-id" : 7334263894966731819 + }, { + "timestamp-ms" : 1710862062007, + "snapshot-id" : 182121606153922595 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1710862058098, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1710862059247, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1710862060042, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1710862060844, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1710862061286, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v5.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v7.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v7.metadata.json new file mode 100644 index 0000000..c4df961 --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v7.metadata.json @@ -0,0 +1,445 @@ +{ + "format-version" : 1, + "table-uuid" : "f424ef42-5477-4841-b127-2f18e7dad530", + "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table", + "last-updated-ms" : 1710862062306, + "last-column-id" : 16, + "schema" : { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + }, + "current-schema-id" : 1, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 182121606153922595, + "refs" : { + "main" : { + "snapshot-id" : 182121606153922595, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 1427538264954246454, + "timestamp-ms" : 1710862058098, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "added-records" : "60175", + "added-files-size" : "4115457", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "4115457", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-1427538264954246454-1-bbab5b8f-dd0a-40e7-8ae6-af228cd96267.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 9177163199645670044, + "parent-snapshot-id" : 1427538264954246454, + "timestamp-ms" : 1710862059247, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "60175", + "deleted-records" : "60175", + "added-files-size" : "3207951", + "removed-files-size" : "4115457", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "3207951", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-9177163199645670044-1-6b161bb4-5fb9-4f06-8ccf-e1b9e0236893.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 7383008947754115722, + "parent-snapshot-id" : 9177163199645670044, + "timestamp-ms" : 1710862060042, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "added-records" : "13126", + "added-files-size" : "933466", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4141417", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7383008947754115722-1-cf96b501-e4d6-43df-9bab-3f4ef5769510.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 6939941997387914529, + "parent-snapshot-id" : 7383008947754115722, + "timestamp-ms" : 1710862060844, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "2", + "deleted-data-files" : "2", + "added-records" : "73301", + "deleted-records" : "73301", + "added-files-size" : "4141417", + "removed-files-size" : "4141417", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4141417", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-6939941997387914529-1-a4381dd0-8507-4690-9710-4e412acce18c.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 7334263894966731819, + "parent-snapshot-id" : 6939941997387914529, + "timestamp-ms" : 1710862061286, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "changed-partition-count" : "0", + "total-records" : "73301", + "total-files-size" : "4141417", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7334263894966731819-1-a7f34407-3101-4480-b866-3721aca735e7.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 182121606153922595, + "parent-snapshot-id" : 7334263894966731819, + "timestamp-ms" : 1710862062007, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "2", + "deleted-data-files" : "2", + "added-records" : "73301", + "deleted-records" : "73301", + "added-files-size" : "4141415", + "removed-files-size" : "4141417", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4141415", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-182121606153922595-1-4460dd70-b98d-49a9-b690-05b8cbc84ea9.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1710862058098, + "snapshot-id" : 1427538264954246454 + }, { + "timestamp-ms" : 1710862059247, + "snapshot-id" : 9177163199645670044 + }, { + "timestamp-ms" : 1710862060042, + "snapshot-id" : 7383008947754115722 + }, { + "timestamp-ms" : 1710862060844, + "snapshot-id" : 6939941997387914529 + }, { + "timestamp-ms" : 1710862061286, + "snapshot-id" : 7334263894966731819 + }, { + "timestamp-ms" : 1710862062007, + "snapshot-id" : 182121606153922595 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1710862058098, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1710862059247, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1710862060042, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1710862060844, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1710862061286, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1710862062007, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v6.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v8.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v8.metadata.json new file mode 100644 index 0000000..0506d46 --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v8.metadata.json @@ -0,0 +1,474 @@ +{ + "format-version" : 1, + "table-uuid" : "f424ef42-5477-4841-b127-2f18e7dad530", + "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table", + "last-updated-ms" : 1710862063027, + "last-column-id" : 16, + "schema" : { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + }, + "current-schema-id" : 1, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 4614971797487332201, + "refs" : { + "main" : { + "snapshot-id" : 4614971797487332201, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 1427538264954246454, + "timestamp-ms" : 1710862058098, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "added-records" : "60175", + "added-files-size" : "4115457", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "4115457", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-1427538264954246454-1-bbab5b8f-dd0a-40e7-8ae6-af228cd96267.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 9177163199645670044, + "parent-snapshot-id" : 1427538264954246454, + "timestamp-ms" : 1710862059247, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "60175", + "deleted-records" : "60175", + "added-files-size" : "3207951", + "removed-files-size" : "4115457", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "3207951", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-9177163199645670044-1-6b161bb4-5fb9-4f06-8ccf-e1b9e0236893.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 7383008947754115722, + "parent-snapshot-id" : 9177163199645670044, + "timestamp-ms" : 1710862060042, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "added-records" : "13126", + "added-files-size" : "933466", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4141417", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7383008947754115722-1-cf96b501-e4d6-43df-9bab-3f4ef5769510.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 6939941997387914529, + "parent-snapshot-id" : 7383008947754115722, + "timestamp-ms" : 1710862060844, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "2", + "deleted-data-files" : "2", + "added-records" : "73301", + "deleted-records" : "73301", + "added-files-size" : "4141417", + "removed-files-size" : "4141417", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4141417", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-6939941997387914529-1-a4381dd0-8507-4690-9710-4e412acce18c.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 7334263894966731819, + "parent-snapshot-id" : 6939941997387914529, + "timestamp-ms" : 1710862061286, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "changed-partition-count" : "0", + "total-records" : "73301", + "total-files-size" : "4141417", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7334263894966731819-1-a7f34407-3101-4480-b866-3721aca735e7.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 182121606153922595, + "parent-snapshot-id" : 7334263894966731819, + "timestamp-ms" : 1710862062007, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "2", + "deleted-data-files" : "2", + "added-records" : "73301", + "deleted-records" : "73301", + "added-files-size" : "4141415", + "removed-files-size" : "4141417", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4141415", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-182121606153922595-1-4460dd70-b98d-49a9-b690-05b8cbc84ea9.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4614971797487332201, + "parent-snapshot-id" : 182121606153922595, + "timestamp-ms" : 1710862063027, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "2", + "deleted-data-files" : "2", + "added-records" : "73301", + "deleted-records" : "73301", + "added-files-size" : "4159998", + "removed-files-size" : "4141415", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4159998", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-4614971797487332201-1-5cf4c8f1-c28a-448c-a9ad-71b2920cf145.avro", + "schema-id" : 1 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1710862058098, + "snapshot-id" : 1427538264954246454 + }, { + "timestamp-ms" : 1710862059247, + "snapshot-id" : 9177163199645670044 + }, { + "timestamp-ms" : 1710862060042, + "snapshot-id" : 7383008947754115722 + }, { + "timestamp-ms" : 1710862060844, + "snapshot-id" : 6939941997387914529 + }, { + "timestamp-ms" : 1710862061286, + "snapshot-id" : 7334263894966731819 + }, { + "timestamp-ms" : 1710862062007, + "snapshot-id" : 182121606153922595 + }, { + "timestamp-ms" : 1710862063027, + "snapshot-id" : 4614971797487332201 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1710862058098, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1710862059247, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1710862060042, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1710862060844, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1710862061286, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1710862062007, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v6.metadata.json" + }, { + "timestamp-ms" : 1710862062306, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v7.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v9.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v9.metadata.json new file mode 100644 index 0000000..fba7f83 --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v9.metadata.json @@ -0,0 +1,561 @@ +{ + "format-version" : 1, + "table-uuid" : "f424ef42-5477-4841-b127-2f18e7dad530", + "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table", + "last-updated-ms" : 1710862063307, + "last-column-id" : 16, + "schema" : { + "type" : "struct", + "schema-id" : 2, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "long" + } ] + }, + "current-schema-id" : 2, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + }, { + "type" : "struct", + "schema-id" : 2, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "long" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 4614971797487332201, + "refs" : { + "main" : { + "snapshot-id" : 4614971797487332201, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 1427538264954246454, + "timestamp-ms" : 1710862058098, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "added-records" : "60175", + "added-files-size" : "4115457", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "4115457", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-1427538264954246454-1-bbab5b8f-dd0a-40e7-8ae6-af228cd96267.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 9177163199645670044, + "parent-snapshot-id" : 1427538264954246454, + "timestamp-ms" : 1710862059247, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "60175", + "deleted-records" : "60175", + "added-files-size" : "3207951", + "removed-files-size" : "4115457", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "3207951", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-9177163199645670044-1-6b161bb4-5fb9-4f06-8ccf-e1b9e0236893.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 7383008947754115722, + "parent-snapshot-id" : 9177163199645670044, + "timestamp-ms" : 1710862060042, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "1", + "added-records" : "13126", + "added-files-size" : "933466", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4141417", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7383008947754115722-1-cf96b501-e4d6-43df-9bab-3f4ef5769510.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 6939941997387914529, + "parent-snapshot-id" : 7383008947754115722, + "timestamp-ms" : 1710862060844, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "2", + "deleted-data-files" : "2", + "added-records" : "73301", + "deleted-records" : "73301", + "added-files-size" : "4141417", + "removed-files-size" : "4141417", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4141417", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-6939941997387914529-1-a4381dd0-8507-4690-9710-4e412acce18c.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 7334263894966731819, + "parent-snapshot-id" : 6939941997387914529, + "timestamp-ms" : 1710862061286, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "changed-partition-count" : "0", + "total-records" : "73301", + "total-files-size" : "4141417", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7334263894966731819-1-a7f34407-3101-4480-b866-3721aca735e7.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 182121606153922595, + "parent-snapshot-id" : 7334263894966731819, + "timestamp-ms" : 1710862062007, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "2", + "deleted-data-files" : "2", + "added-records" : "73301", + "deleted-records" : "73301", + "added-files-size" : "4141415", + "removed-files-size" : "4141417", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4141415", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-182121606153922595-1-4460dd70-b98d-49a9-b690-05b8cbc84ea9.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4614971797487332201, + "parent-snapshot-id" : 182121606153922595, + "timestamp-ms" : 1710862063027, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862055070", + "added-data-files" : "2", + "deleted-data-files" : "2", + "added-records" : "73301", + "deleted-records" : "73301", + "added-files-size" : "4159998", + "removed-files-size" : "4141415", + "changed-partition-count" : "1", + "total-records" : "73301", + "total-files-size" : "4159998", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-4614971797487332201-1-5cf4c8f1-c28a-448c-a9ad-71b2920cf145.avro", + "schema-id" : 1 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1710862058098, + "snapshot-id" : 1427538264954246454 + }, { + "timestamp-ms" : 1710862059247, + "snapshot-id" : 9177163199645670044 + }, { + "timestamp-ms" : 1710862060042, + "snapshot-id" : 7383008947754115722 + }, { + "timestamp-ms" : 1710862060844, + "snapshot-id" : 6939941997387914529 + }, { + "timestamp-ms" : 1710862061286, + "snapshot-id" : 7334263894966731819 + }, { + "timestamp-ms" : 1710862062007, + "snapshot-id" : 182121606153922595 + }, { + "timestamp-ms" : 1710862063027, + "snapshot-id" : 4614971797487332201 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1710862058098, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1710862059247, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1710862060042, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1710862060844, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1710862061286, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1710862062007, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v6.metadata.json" + }, { + "timestamp-ms" : 1710862062306, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v7.metadata.json" + }, { + "timestamp-ms" : 1710862063027, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v8.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/version-hint.text b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/version-hint.text new file mode 100644 index 0000000..f11c82a --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/version-hint.text @@ -0,0 +1 @@ +9 \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-1-d7e8863e-aefb-440f-bf68-060432c1c6a4-00001.parquet.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-1-d7e8863e-aefb-440f-bf68-060432c1c6a4-00001.parquet.crc new file mode 100644 index 0000000..784136b Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-1-d7e8863e-aefb-440f-bf68-060432c1c6a4-00001.parquet.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-13-90c0a65f-8fde-49ff-8072-e8b8f2ace270-00001-deletes.parquet.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-13-90c0a65f-8fde-49ff-8072-e8b8f2ace270-00001-deletes.parquet.crc new file mode 100644 index 0000000..ec3222c Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-13-90c0a65f-8fde-49ff-8072-e8b8f2ace270-00001-deletes.parquet.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-13-90c0a65f-8fde-49ff-8072-e8b8f2ace270-00001.parquet.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-13-90c0a65f-8fde-49ff-8072-e8b8f2ace270-00001.parquet.crc new file mode 100644 index 0000000..5b941b8 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-13-90c0a65f-8fde-49ff-8072-e8b8f2ace270-00001.parquet.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-25-2a78457e-4117-4d1b-8f05-d2bdd60a9090-00001.parquet.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-25-2a78457e-4117-4d1b-8f05-d2bdd60a9090-00001.parquet.crc new file mode 100644 index 0000000..2e9dc8e Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-25-2a78457e-4117-4d1b-8f05-d2bdd60a9090-00001.parquet.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-35-4c376145-4729-4780-bd27-f8997b5cf85e-00001.parquet.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-35-4c376145-4729-4780-bd27-f8997b5cf85e-00001.parquet.crc new file mode 100644 index 0000000..110d506 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-35-4c376145-4729-4780-bd27-f8997b5cf85e-00001.parquet.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-4-9f6a215f-95c7-47b2-9eeb-567eecb88228-00001-deletes.parquet.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-4-9f6a215f-95c7-47b2-9eeb-567eecb88228-00001-deletes.parquet.crc new file mode 100644 index 0000000..06c1bc6 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-4-9f6a215f-95c7-47b2-9eeb-567eecb88228-00001-deletes.parquet.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-4-9f6a215f-95c7-47b2-9eeb-567eecb88228-00001.parquet.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-4-9f6a215f-95c7-47b2-9eeb-567eecb88228-00001.parquet.crc new file mode 100644 index 0000000..799996d Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-4-9f6a215f-95c7-47b2-9eeb-567eecb88228-00001.parquet.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-50-64d8698d-d87a-4fd2-b265-1cca524199ca-00001-deletes.parquet.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-50-64d8698d-d87a-4fd2-b265-1cca524199ca-00001-deletes.parquet.crc new file mode 100644 index 0000000..e2b1691 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-50-64d8698d-d87a-4fd2-b265-1cca524199ca-00001-deletes.parquet.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-50-64d8698d-d87a-4fd2-b265-1cca524199ca-00001.parquet.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-50-64d8698d-d87a-4fd2-b265-1cca524199ca-00001.parquet.crc new file mode 100644 index 0000000..b0982ce Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-50-64d8698d-d87a-4fd2-b265-1cca524199ca-00001.parquet.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-8-e8c7a535-bd8a-4dc3-b171-eaf95635de86-00001.parquet.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-8-e8c7a535-bd8a-4dc3-b171-eaf95635de86-00001.parquet.crc new file mode 100644 index 0000000..b9576eb Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/.00000-8-e8c7a535-bd8a-4dc3-b171-eaf95635de86-00001.parquet.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-1-d7e8863e-aefb-440f-bf68-060432c1c6a4-00001.parquet b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-1-d7e8863e-aefb-440f-bf68-060432c1c6a4-00001.parquet new file mode 100644 index 0000000..2127d0a Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-1-d7e8863e-aefb-440f-bf68-060432c1c6a4-00001.parquet differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-13-90c0a65f-8fde-49ff-8072-e8b8f2ace270-00001-deletes.parquet b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-13-90c0a65f-8fde-49ff-8072-e8b8f2ace270-00001-deletes.parquet new file mode 100644 index 0000000..78bea8c Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-13-90c0a65f-8fde-49ff-8072-e8b8f2ace270-00001-deletes.parquet differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-13-90c0a65f-8fde-49ff-8072-e8b8f2ace270-00001.parquet b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-13-90c0a65f-8fde-49ff-8072-e8b8f2ace270-00001.parquet new file mode 100644 index 0000000..cf75d40 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-13-90c0a65f-8fde-49ff-8072-e8b8f2ace270-00001.parquet differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-25-2a78457e-4117-4d1b-8f05-d2bdd60a9090-00001.parquet b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-25-2a78457e-4117-4d1b-8f05-d2bdd60a9090-00001.parquet new file mode 100644 index 0000000..5c23e41 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-25-2a78457e-4117-4d1b-8f05-d2bdd60a9090-00001.parquet differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-35-4c376145-4729-4780-bd27-f8997b5cf85e-00001.parquet b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-35-4c376145-4729-4780-bd27-f8997b5cf85e-00001.parquet new file mode 100644 index 0000000..287742c Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-35-4c376145-4729-4780-bd27-f8997b5cf85e-00001.parquet differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-4-9f6a215f-95c7-47b2-9eeb-567eecb88228-00001-deletes.parquet b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-4-9f6a215f-95c7-47b2-9eeb-567eecb88228-00001-deletes.parquet new file mode 100644 index 0000000..1e3eb4b Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-4-9f6a215f-95c7-47b2-9eeb-567eecb88228-00001-deletes.parquet differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-4-9f6a215f-95c7-47b2-9eeb-567eecb88228-00001.parquet b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-4-9f6a215f-95c7-47b2-9eeb-567eecb88228-00001.parquet new file mode 100644 index 0000000..772edd6 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-4-9f6a215f-95c7-47b2-9eeb-567eecb88228-00001.parquet differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-50-64d8698d-d87a-4fd2-b265-1cca524199ca-00001-deletes.parquet b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-50-64d8698d-d87a-4fd2-b265-1cca524199ca-00001-deletes.parquet new file mode 100644 index 0000000..b08422a Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-50-64d8698d-d87a-4fd2-b265-1cca524199ca-00001-deletes.parquet differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-50-64d8698d-d87a-4fd2-b265-1cca524199ca-00001.parquet b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-50-64d8698d-d87a-4fd2-b265-1cca524199ca-00001.parquet new file mode 100644 index 0000000..b0c7c7a Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-50-64d8698d-d87a-4fd2-b265-1cca524199ca-00001.parquet differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-8-e8c7a535-bd8a-4dc3-b171-eaf95635de86-00001.parquet b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-8-e8c7a535-bd8a-4dc3-b171-eaf95635de86-00001.parquet new file mode 100644 index 0000000..cc70ed5 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/data/00000-8-e8c7a535-bd8a-4dc3-b171-eaf95635de86-00001.parquet differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.08716a3b-1da4-4224-bafa-9fa91a03749f-m0.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.08716a3b-1da4-4224-bafa-9fa91a03749f-m0.avro.crc new file mode 100644 index 0000000..90cdebe Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.08716a3b-1da4-4224-bafa-9fa91a03749f-m0.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.08716a3b-1da4-4224-bafa-9fa91a03749f-m1.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.08716a3b-1da4-4224-bafa-9fa91a03749f-m1.avro.crc new file mode 100644 index 0000000..fa34191 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.08716a3b-1da4-4224-bafa-9fa91a03749f-m1.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.212e282c-ba0d-42b6-a962-bf0bceb80311-m0.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.212e282c-ba0d-42b6-a962-bf0bceb80311-m0.avro.crc new file mode 100644 index 0000000..d64b35b Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.212e282c-ba0d-42b6-a962-bf0bceb80311-m0.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.212e282c-ba0d-42b6-a962-bf0bceb80311-m1.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.212e282c-ba0d-42b6-a962-bf0bceb80311-m1.avro.crc new file mode 100644 index 0000000..c15bafc Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.212e282c-ba0d-42b6-a962-bf0bceb80311-m1.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.9b90d2c7-b977-401c-a764-f2bedf73122b-m0.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.9b90d2c7-b977-401c-a764-f2bedf73122b-m0.avro.crc new file mode 100644 index 0000000..4eced9c Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.9b90d2c7-b977-401c-a764-f2bedf73122b-m0.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.9b90d2c7-b977-401c-a764-f2bedf73122b-m1.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.9b90d2c7-b977-401c-a764-f2bedf73122b-m1.avro.crc new file mode 100644 index 0000000..cf8bab3 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.9b90d2c7-b977-401c-a764-f2bedf73122b-m1.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.a5fef285-1377-4bcd-ac03-faf32d0e8074-m0.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.a5fef285-1377-4bcd-ac03-faf32d0e8074-m0.avro.crc new file mode 100644 index 0000000..34e08aa Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.a5fef285-1377-4bcd-ac03-faf32d0e8074-m0.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.a5fef285-1377-4bcd-ac03-faf32d0e8074-m1.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.a5fef285-1377-4bcd-ac03-faf32d0e8074-m1.avro.crc new file mode 100644 index 0000000..a514f74 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.a5fef285-1377-4bcd-ac03-faf32d0e8074-m1.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.b7051a4c-4bab-4399-bed0-ea6e540506d6-m0.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.b7051a4c-4bab-4399-bed0-ea6e540506d6-m0.avro.crc new file mode 100644 index 0000000..602858a Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.b7051a4c-4bab-4399-bed0-ea6e540506d6-m0.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.b96e9651-c445-4ed9-b59e-6e2338c6df8e-m0.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.b96e9651-c445-4ed9-b59e-6e2338c6df8e-m0.avro.crc new file mode 100644 index 0000000..7346f88 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.b96e9651-c445-4ed9-b59e-6e2338c6df8e-m0.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94-m0.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94-m0.avro.crc new file mode 100644 index 0000000..61c8d63 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94-m0.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94-m1.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94-m1.avro.crc new file mode 100644 index 0000000..fdb87f7 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94-m1.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-1896028637661931044-1-08716a3b-1da4-4224-bafa-9fa91a03749f.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-1896028637661931044-1-08716a3b-1da4-4224-bafa-9fa91a03749f.avro.crc new file mode 100644 index 0000000..0c30498 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-1896028637661931044-1-08716a3b-1da4-4224-bafa-9fa91a03749f.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-4018563758056380634-1-a5fef285-1377-4bcd-ac03-faf32d0e8074.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-4018563758056380634-1-a5fef285-1377-4bcd-ac03-faf32d0e8074.avro.crc new file mode 100644 index 0000000..4a2d82f Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-4018563758056380634-1-a5fef285-1377-4bcd-ac03-faf32d0e8074.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-4181695983163187512-1-9b90d2c7-b977-401c-a764-f2bedf73122b.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-4181695983163187512-1-9b90d2c7-b977-401c-a764-f2bedf73122b.avro.crc new file mode 100644 index 0000000..ea097e0 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-4181695983163187512-1-9b90d2c7-b977-401c-a764-f2bedf73122b.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-5305247691388084493-1-b7051a4c-4bab-4399-bed0-ea6e540506d6.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-5305247691388084493-1-b7051a4c-4bab-4399-bed0-ea6e540506d6.avro.crc new file mode 100644 index 0000000..df63b81 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-5305247691388084493-1-b7051a4c-4bab-4399-bed0-ea6e540506d6.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-5593441557906709930-1-b96e9651-c445-4ed9-b59e-6e2338c6df8e.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-5593441557906709930-1-b96e9651-c445-4ed9-b59e-6e2338c6df8e.avro.crc new file mode 100644 index 0000000..93d124a Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-5593441557906709930-1-b96e9651-c445-4ed9-b59e-6e2338c6df8e.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-7765867412814781196-1-212e282c-ba0d-42b6-a962-bf0bceb80311.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-7765867412814781196-1-212e282c-ba0d-42b6-a962-bf0bceb80311.avro.crc new file mode 100644 index 0000000..e2b9388 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-7765867412814781196-1-212e282c-ba0d-42b6-a962-bf0bceb80311.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-9149274436460014835-1-bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94.avro.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-9149274436460014835-1-bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94.avro.crc new file mode 100644 index 0000000..5c7c9e9 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.snap-9149274436460014835-1-bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94.avro.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v1.metadata.json.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v1.metadata.json.crc new file mode 100644 index 0000000..f508a77 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v1.metadata.json.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v2.metadata.json.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v2.metadata.json.crc new file mode 100644 index 0000000..ca451f6 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v2.metadata.json.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v3.metadata.json.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v3.metadata.json.crc new file mode 100644 index 0000000..0be280f Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v3.metadata.json.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v4.metadata.json.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v4.metadata.json.crc new file mode 100644 index 0000000..d51d6a5 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v4.metadata.json.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v5.metadata.json.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v5.metadata.json.crc new file mode 100644 index 0000000..5aa868a Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v5.metadata.json.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v6.metadata.json.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v6.metadata.json.crc new file mode 100644 index 0000000..9b5024b Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v6.metadata.json.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v7.metadata.json.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v7.metadata.json.crc new file mode 100644 index 0000000..c88babf Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v7.metadata.json.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v8.metadata.json.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v8.metadata.json.crc new file mode 100644 index 0000000..c57ad12 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v8.metadata.json.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v9.metadata.json.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v9.metadata.json.crc new file mode 100644 index 0000000..b8e837e Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.v9.metadata.json.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.version-hint.text.crc b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.version-hint.text.crc new file mode 100644 index 0000000..1f8d85e Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/.version-hint.text.crc differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/08716a3b-1da4-4224-bafa-9fa91a03749f-m0.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/08716a3b-1da4-4224-bafa-9fa91a03749f-m0.avro new file mode 100644 index 0000000..22ba1ce Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/08716a3b-1da4-4224-bafa-9fa91a03749f-m0.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/08716a3b-1da4-4224-bafa-9fa91a03749f-m1.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/08716a3b-1da4-4224-bafa-9fa91a03749f-m1.avro new file mode 100644 index 0000000..83db1dd Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/08716a3b-1da4-4224-bafa-9fa91a03749f-m1.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/212e282c-ba0d-42b6-a962-bf0bceb80311-m0.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/212e282c-ba0d-42b6-a962-bf0bceb80311-m0.avro new file mode 100644 index 0000000..c2e0438 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/212e282c-ba0d-42b6-a962-bf0bceb80311-m0.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/212e282c-ba0d-42b6-a962-bf0bceb80311-m1.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/212e282c-ba0d-42b6-a962-bf0bceb80311-m1.avro new file mode 100644 index 0000000..2b6c33e Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/212e282c-ba0d-42b6-a962-bf0bceb80311-m1.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/9b90d2c7-b977-401c-a764-f2bedf73122b-m0.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/9b90d2c7-b977-401c-a764-f2bedf73122b-m0.avro new file mode 100644 index 0000000..9a61373 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/9b90d2c7-b977-401c-a764-f2bedf73122b-m0.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/9b90d2c7-b977-401c-a764-f2bedf73122b-m1.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/9b90d2c7-b977-401c-a764-f2bedf73122b-m1.avro new file mode 100644 index 0000000..0991520 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/9b90d2c7-b977-401c-a764-f2bedf73122b-m1.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/a5fef285-1377-4bcd-ac03-faf32d0e8074-m0.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/a5fef285-1377-4bcd-ac03-faf32d0e8074-m0.avro new file mode 100644 index 0000000..193b67f Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/a5fef285-1377-4bcd-ac03-faf32d0e8074-m0.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/a5fef285-1377-4bcd-ac03-faf32d0e8074-m1.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/a5fef285-1377-4bcd-ac03-faf32d0e8074-m1.avro new file mode 100644 index 0000000..53e99f6 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/a5fef285-1377-4bcd-ac03-faf32d0e8074-m1.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/b7051a4c-4bab-4399-bed0-ea6e540506d6-m0.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/b7051a4c-4bab-4399-bed0-ea6e540506d6-m0.avro new file mode 100644 index 0000000..719fa20 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/b7051a4c-4bab-4399-bed0-ea6e540506d6-m0.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/b96e9651-c445-4ed9-b59e-6e2338c6df8e-m0.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/b96e9651-c445-4ed9-b59e-6e2338c6df8e-m0.avro new file mode 100644 index 0000000..25f86e5 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/b96e9651-c445-4ed9-b59e-6e2338c6df8e-m0.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94-m0.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94-m0.avro new file mode 100644 index 0000000..bc00389 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94-m0.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94-m1.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94-m1.avro new file mode 100644 index 0000000..2c9199c Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94-m1.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1896028637661931044-1-08716a3b-1da4-4224-bafa-9fa91a03749f.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1896028637661931044-1-08716a3b-1da4-4224-bafa-9fa91a03749f.avro new file mode 100644 index 0000000..5e4a581 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1896028637661931044-1-08716a3b-1da4-4224-bafa-9fa91a03749f.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-4018563758056380634-1-a5fef285-1377-4bcd-ac03-faf32d0e8074.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-4018563758056380634-1-a5fef285-1377-4bcd-ac03-faf32d0e8074.avro new file mode 100644 index 0000000..3ca93a3 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-4018563758056380634-1-a5fef285-1377-4bcd-ac03-faf32d0e8074.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-4181695983163187512-1-9b90d2c7-b977-401c-a764-f2bedf73122b.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-4181695983163187512-1-9b90d2c7-b977-401c-a764-f2bedf73122b.avro new file mode 100644 index 0000000..b9403a8 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-4181695983163187512-1-9b90d2c7-b977-401c-a764-f2bedf73122b.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5305247691388084493-1-b7051a4c-4bab-4399-bed0-ea6e540506d6.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5305247691388084493-1-b7051a4c-4bab-4399-bed0-ea6e540506d6.avro new file mode 100644 index 0000000..245afd8 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5305247691388084493-1-b7051a4c-4bab-4399-bed0-ea6e540506d6.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5593441557906709930-1-b96e9651-c445-4ed9-b59e-6e2338c6df8e.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5593441557906709930-1-b96e9651-c445-4ed9-b59e-6e2338c6df8e.avro new file mode 100644 index 0000000..75dadb6 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5593441557906709930-1-b96e9651-c445-4ed9-b59e-6e2338c6df8e.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-7765867412814781196-1-212e282c-ba0d-42b6-a962-bf0bceb80311.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-7765867412814781196-1-212e282c-ba0d-42b6-a962-bf0bceb80311.avro new file mode 100644 index 0000000..ff03c41 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-7765867412814781196-1-212e282c-ba0d-42b6-a962-bf0bceb80311.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-9149274436460014835-1-bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94.avro b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-9149274436460014835-1-bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94.avro new file mode 100644 index 0000000..f751e37 Binary files /dev/null and b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-9149274436460014835-1-bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94.avro differ diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json new file mode 100644 index 0000000..62bbe0f --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json @@ -0,0 +1,139 @@ +{ + "format-version" : 2, + "table-uuid" : "6f901565-5100-4773-bba7-5ae4a88124cb", + "location" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table", + "last-sequence-number" : 1, + "last-updated-ms" : 1710862068149, + "last-column-id" : 15, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 5593441557906709930, + "refs" : { + "main" : { + "snapshot-id" : 5593441557906709930, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 5593441557906709930, + "timestamp-ms" : 1710862068149, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-records" : "60175", + "added-files-size" : "4114898", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "4114898", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5593441557906709930-1-b96e9651-c445-4ed9-b59e-6e2338c6df8e.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1710862068149, + "snapshot-id" : 5593441557906709930 + } ], + "metadata-log" : [ ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json new file mode 100644 index 0000000..d6039fe --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json @@ -0,0 +1,169 @@ +{ + "format-version" : 2, + "table-uuid" : "6f901565-5100-4773-bba7-5ae4a88124cb", + "location" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table", + "last-sequence-number" : 2, + "last-updated-ms" : 1710862069031, + "last-column-id" : 15, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 4018563758056380634, + "refs" : { + "main" : { + "snapshot-id" : 4018563758056380634, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 5593441557906709930, + "timestamp-ms" : 1710862068149, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-records" : "60175", + "added-files-size" : "4114898", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "4114898", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5593441557906709930-1-b96e9651-c445-4ed9-b59e-6e2338c6df8e.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4018563758056380634, + "parent-snapshot-id" : 5593441557906709930, + "timestamp-ms" : 1710862069031, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "30037", + "added-files-size" : "1078093", + "added-position-deletes" : "30037", + "changed-partition-count" : "1", + "total-records" : "90212", + "total-files-size" : "5192991", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "30037", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-4018563758056380634-1-a5fef285-1377-4bcd-ac03-faf32d0e8074.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1710862068149, + "snapshot-id" : 5593441557906709930 + }, { + "timestamp-ms" : 1710862069031, + "snapshot-id" : 4018563758056380634 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1710862068149, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json new file mode 100644 index 0000000..c6eb845 --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json @@ -0,0 +1,196 @@ +{ + "format-version" : 2, + "table-uuid" : "6f901565-5100-4773-bba7-5ae4a88124cb", + "location" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table", + "last-sequence-number" : 3, + "last-updated-ms" : 1710862070028, + "last-column-id" : 15, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 5305247691388084493, + "refs" : { + "main" : { + "snapshot-id" : 5305247691388084493, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 5593441557906709930, + "timestamp-ms" : 1710862068149, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-records" : "60175", + "added-files-size" : "4114898", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "4114898", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5593441557906709930-1-b96e9651-c445-4ed9-b59e-6e2338c6df8e.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4018563758056380634, + "parent-snapshot-id" : 5593441557906709930, + "timestamp-ms" : 1710862069031, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "30037", + "added-files-size" : "1078093", + "added-position-deletes" : "30037", + "changed-partition-count" : "1", + "total-records" : "90212", + "total-files-size" : "5192991", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "30037", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-4018563758056380634-1-a5fef285-1377-4bcd-ac03-faf32d0e8074.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 5305247691388084493, + "parent-snapshot-id" : 4018563758056380634, + "timestamp-ms" : 1710862070028, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-records" : "13126", + "added-files-size" : "932934", + "changed-partition-count" : "1", + "total-records" : "103338", + "total-files-size" : "6125925", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "30037", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5305247691388084493-1-b7051a4c-4bab-4399-bed0-ea6e540506d6.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1710862068149, + "snapshot-id" : 5593441557906709930 + }, { + "timestamp-ms" : 1710862069031, + "snapshot-id" : 4018563758056380634 + }, { + "timestamp-ms" : 1710862070028, + "snapshot-id" : 5305247691388084493 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1710862068149, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1710862069031, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v4.metadata.json b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v4.metadata.json new file mode 100644 index 0000000..7fe32ca --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v4.metadata.json @@ -0,0 +1,226 @@ +{ + "format-version" : 2, + "table-uuid" : "6f901565-5100-4773-bba7-5ae4a88124cb", + "location" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table", + "last-sequence-number" : 4, + "last-updated-ms" : 1710862071156, + "last-column-id" : 15, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 1896028637661931044, + "refs" : { + "main" : { + "snapshot-id" : 1896028637661931044, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 5593441557906709930, + "timestamp-ms" : 1710862068149, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-records" : "60175", + "added-files-size" : "4114898", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "4114898", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5593441557906709930-1-b96e9651-c445-4ed9-b59e-6e2338c6df8e.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4018563758056380634, + "parent-snapshot-id" : 5593441557906709930, + "timestamp-ms" : 1710862069031, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "30037", + "added-files-size" : "1078093", + "added-position-deletes" : "30037", + "changed-partition-count" : "1", + "total-records" : "90212", + "total-files-size" : "5192991", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "30037", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-4018563758056380634-1-a5fef285-1377-4bcd-ac03-faf32d0e8074.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 5305247691388084493, + "parent-snapshot-id" : 4018563758056380634, + "timestamp-ms" : 1710862070028, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-records" : "13126", + "added-files-size" : "932934", + "changed-partition-count" : "1", + "total-records" : "103338", + "total-files-size" : "6125925", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "30037", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5305247691388084493-1-b7051a4c-4bab-4399-bed0-ea6e540506d6.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 1896028637661931044, + "parent-snapshot-id" : 5305247691388084493, + "timestamp-ms" : 1710862071156, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "73301", + "added-files-size" : "4048500", + "added-position-deletes" : "73301", + "changed-partition-count" : "1", + "total-records" : "176639", + "total-files-size" : "10174425", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "103338", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1896028637661931044-1-08716a3b-1da4-4224-bafa-9fa91a03749f.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1710862068149, + "snapshot-id" : 5593441557906709930 + }, { + "timestamp-ms" : 1710862069031, + "snapshot-id" : 4018563758056380634 + }, { + "timestamp-ms" : 1710862070028, + "snapshot-id" : 5305247691388084493 + }, { + "timestamp-ms" : 1710862071156, + "snapshot-id" : 1896028637661931044 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1710862068149, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1710862069031, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1710862070028, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v5.metadata.json b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v5.metadata.json new file mode 100644 index 0000000..38888f6 --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v5.metadata.json @@ -0,0 +1,256 @@ +{ + "format-version" : 2, + "table-uuid" : "6f901565-5100-4773-bba7-5ae4a88124cb", + "location" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table", + "last-sequence-number" : 5, + "last-updated-ms" : 1710862072235, + "last-column-id" : 15, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 7765867412814781196, + "refs" : { + "main" : { + "snapshot-id" : 7765867412814781196, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 5593441557906709930, + "timestamp-ms" : 1710862068149, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-records" : "60175", + "added-files-size" : "4114898", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "4114898", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5593441557906709930-1-b96e9651-c445-4ed9-b59e-6e2338c6df8e.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4018563758056380634, + "parent-snapshot-id" : 5593441557906709930, + "timestamp-ms" : 1710862069031, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "30037", + "added-files-size" : "1078093", + "added-position-deletes" : "30037", + "changed-partition-count" : "1", + "total-records" : "90212", + "total-files-size" : "5192991", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "30037", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-4018563758056380634-1-a5fef285-1377-4bcd-ac03-faf32d0e8074.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 5305247691388084493, + "parent-snapshot-id" : 4018563758056380634, + "timestamp-ms" : 1710862070028, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-records" : "13126", + "added-files-size" : "932934", + "changed-partition-count" : "1", + "total-records" : "103338", + "total-files-size" : "6125925", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "30037", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5305247691388084493-1-b7051a4c-4bab-4399-bed0-ea6e540506d6.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 1896028637661931044, + "parent-snapshot-id" : 5305247691388084493, + "timestamp-ms" : 1710862071156, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "73301", + "added-files-size" : "4048500", + "added-position-deletes" : "73301", + "changed-partition-count" : "1", + "total-records" : "176639", + "total-files-size" : "10174425", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "103338", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1896028637661931044-1-08716a3b-1da4-4224-bafa-9fa91a03749f.avro", + "schema-id" : 0 + }, { + "sequence-number" : 5, + "snapshot-id" : 7765867412814781196, + "parent-snapshot-id" : 1896028637661931044, + "timestamp-ms" : 1710862072235, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "65057", + "deleted-records" : "73301", + "added-files-size" : "3405378", + "removed-files-size" : "3938531", + "changed-partition-count" : "1", + "total-records" : "168395", + "total-files-size" : "9641272", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "103338", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-7765867412814781196-1-212e282c-ba0d-42b6-a962-bf0bceb80311.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1710862068149, + "snapshot-id" : 5593441557906709930 + }, { + "timestamp-ms" : 1710862069031, + "snapshot-id" : 4018563758056380634 + }, { + "timestamp-ms" : 1710862070028, + "snapshot-id" : 5305247691388084493 + }, { + "timestamp-ms" : 1710862071156, + "snapshot-id" : 1896028637661931044 + }, { + "timestamp-ms" : 1710862072235, + "snapshot-id" : 7765867412814781196 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1710862068149, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1710862069031, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1710862070028, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1710862071156, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v6.metadata.json b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v6.metadata.json new file mode 100644 index 0000000..b665be6 --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v6.metadata.json @@ -0,0 +1,286 @@ +{ + "format-version" : 2, + "table-uuid" : "6f901565-5100-4773-bba7-5ae4a88124cb", + "location" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table", + "last-sequence-number" : 6, + "last-updated-ms" : 1710862073176, + "last-column-id" : 15, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 4181695983163187512, + "refs" : { + "main" : { + "snapshot-id" : 4181695983163187512, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 5593441557906709930, + "timestamp-ms" : 1710862068149, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-records" : "60175", + "added-files-size" : "4114898", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "4114898", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5593441557906709930-1-b96e9651-c445-4ed9-b59e-6e2338c6df8e.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4018563758056380634, + "parent-snapshot-id" : 5593441557906709930, + "timestamp-ms" : 1710862069031, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "30037", + "added-files-size" : "1078093", + "added-position-deletes" : "30037", + "changed-partition-count" : "1", + "total-records" : "90212", + "total-files-size" : "5192991", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "30037", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-4018563758056380634-1-a5fef285-1377-4bcd-ac03-faf32d0e8074.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 5305247691388084493, + "parent-snapshot-id" : 4018563758056380634, + "timestamp-ms" : 1710862070028, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-records" : "13126", + "added-files-size" : "932934", + "changed-partition-count" : "1", + "total-records" : "103338", + "total-files-size" : "6125925", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "30037", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5305247691388084493-1-b7051a4c-4bab-4399-bed0-ea6e540506d6.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 1896028637661931044, + "parent-snapshot-id" : 5305247691388084493, + "timestamp-ms" : 1710862071156, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "73301", + "added-files-size" : "4048500", + "added-position-deletes" : "73301", + "changed-partition-count" : "1", + "total-records" : "176639", + "total-files-size" : "10174425", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "103338", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1896028637661931044-1-08716a3b-1da4-4224-bafa-9fa91a03749f.avro", + "schema-id" : 0 + }, { + "sequence-number" : 5, + "snapshot-id" : 7765867412814781196, + "parent-snapshot-id" : 1896028637661931044, + "timestamp-ms" : 1710862072235, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "65057", + "deleted-records" : "73301", + "added-files-size" : "3405378", + "removed-files-size" : "3938531", + "changed-partition-count" : "1", + "total-records" : "168395", + "total-files-size" : "9641272", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "103338", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-7765867412814781196-1-212e282c-ba0d-42b6-a962-bf0bceb80311.avro", + "schema-id" : 0 + }, { + "sequence-number" : 6, + "snapshot-id" : 4181695983163187512, + "parent-snapshot-id" : 7765867412814781196, + "timestamp-ms" : 1710862073176, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "62729", + "deleted-records" : "65057", + "added-files-size" : "3242827", + "removed-files-size" : "3405378", + "changed-partition-count" : "1", + "total-records" : "166067", + "total-files-size" : "9478721", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "103338", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-4181695983163187512-1-9b90d2c7-b977-401c-a764-f2bedf73122b.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1710862068149, + "snapshot-id" : 5593441557906709930 + }, { + "timestamp-ms" : 1710862069031, + "snapshot-id" : 4018563758056380634 + }, { + "timestamp-ms" : 1710862070028, + "snapshot-id" : 5305247691388084493 + }, { + "timestamp-ms" : 1710862071156, + "snapshot-id" : 1896028637661931044 + }, { + "timestamp-ms" : 1710862072235, + "snapshot-id" : 7765867412814781196 + }, { + "timestamp-ms" : 1710862073176, + "snapshot-id" : 4181695983163187512 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1710862068149, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1710862069031, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1710862070028, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1710862071156, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1710862072235, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v5.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v7.metadata.json b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v7.metadata.json new file mode 100644 index 0000000..330ee4f --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v7.metadata.json @@ -0,0 +1,373 @@ +{ + "format-version" : 2, + "table-uuid" : "6f901565-5100-4773-bba7-5ae4a88124cb", + "location" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table", + "last-sequence-number" : 6, + "last-updated-ms" : 1710862073575, + "last-column-id" : 16, + "current-schema-id" : 1, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 4181695983163187512, + "refs" : { + "main" : { + "snapshot-id" : 4181695983163187512, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 5593441557906709930, + "timestamp-ms" : 1710862068149, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-records" : "60175", + "added-files-size" : "4114898", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "4114898", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5593441557906709930-1-b96e9651-c445-4ed9-b59e-6e2338c6df8e.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4018563758056380634, + "parent-snapshot-id" : 5593441557906709930, + "timestamp-ms" : 1710862069031, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "30037", + "added-files-size" : "1078093", + "added-position-deletes" : "30037", + "changed-partition-count" : "1", + "total-records" : "90212", + "total-files-size" : "5192991", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "30037", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-4018563758056380634-1-a5fef285-1377-4bcd-ac03-faf32d0e8074.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 5305247691388084493, + "parent-snapshot-id" : 4018563758056380634, + "timestamp-ms" : 1710862070028, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-records" : "13126", + "added-files-size" : "932934", + "changed-partition-count" : "1", + "total-records" : "103338", + "total-files-size" : "6125925", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "30037", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5305247691388084493-1-b7051a4c-4bab-4399-bed0-ea6e540506d6.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 1896028637661931044, + "parent-snapshot-id" : 5305247691388084493, + "timestamp-ms" : 1710862071156, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "73301", + "added-files-size" : "4048500", + "added-position-deletes" : "73301", + "changed-partition-count" : "1", + "total-records" : "176639", + "total-files-size" : "10174425", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "103338", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1896028637661931044-1-08716a3b-1da4-4224-bafa-9fa91a03749f.avro", + "schema-id" : 0 + }, { + "sequence-number" : 5, + "snapshot-id" : 7765867412814781196, + "parent-snapshot-id" : 1896028637661931044, + "timestamp-ms" : 1710862072235, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "65057", + "deleted-records" : "73301", + "added-files-size" : "3405378", + "removed-files-size" : "3938531", + "changed-partition-count" : "1", + "total-records" : "168395", + "total-files-size" : "9641272", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "103338", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-7765867412814781196-1-212e282c-ba0d-42b6-a962-bf0bceb80311.avro", + "schema-id" : 0 + }, { + "sequence-number" : 6, + "snapshot-id" : 4181695983163187512, + "parent-snapshot-id" : 7765867412814781196, + "timestamp-ms" : 1710862073176, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "62729", + "deleted-records" : "65057", + "added-files-size" : "3242827", + "removed-files-size" : "3405378", + "changed-partition-count" : "1", + "total-records" : "166067", + "total-files-size" : "9478721", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "103338", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-4181695983163187512-1-9b90d2c7-b977-401c-a764-f2bedf73122b.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1710862068149, + "snapshot-id" : 5593441557906709930 + }, { + "timestamp-ms" : 1710862069031, + "snapshot-id" : 4018563758056380634 + }, { + "timestamp-ms" : 1710862070028, + "snapshot-id" : 5305247691388084493 + }, { + "timestamp-ms" : 1710862071156, + "snapshot-id" : 1896028637661931044 + }, { + "timestamp-ms" : 1710862072235, + "snapshot-id" : 7765867412814781196 + }, { + "timestamp-ms" : 1710862073176, + "snapshot-id" : 4181695983163187512 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1710862068149, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1710862069031, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1710862070028, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1710862071156, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1710862072235, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1710862073176, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v6.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v8.metadata.json b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v8.metadata.json new file mode 100644 index 0000000..09e1efa --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v8.metadata.json @@ -0,0 +1,403 @@ +{ + "format-version" : 2, + "table-uuid" : "6f901565-5100-4773-bba7-5ae4a88124cb", + "location" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table", + "last-sequence-number" : 7, + "last-updated-ms" : 1710862074244, + "last-column-id" : 16, + "current-schema-id" : 1, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 9149274436460014835, + "refs" : { + "main" : { + "snapshot-id" : 9149274436460014835, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 5593441557906709930, + "timestamp-ms" : 1710862068149, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-records" : "60175", + "added-files-size" : "4114898", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "4114898", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5593441557906709930-1-b96e9651-c445-4ed9-b59e-6e2338c6df8e.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4018563758056380634, + "parent-snapshot-id" : 5593441557906709930, + "timestamp-ms" : 1710862069031, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "30037", + "added-files-size" : "1078093", + "added-position-deletes" : "30037", + "changed-partition-count" : "1", + "total-records" : "90212", + "total-files-size" : "5192991", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "30037", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-4018563758056380634-1-a5fef285-1377-4bcd-ac03-faf32d0e8074.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 5305247691388084493, + "parent-snapshot-id" : 4018563758056380634, + "timestamp-ms" : 1710862070028, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-records" : "13126", + "added-files-size" : "932934", + "changed-partition-count" : "1", + "total-records" : "103338", + "total-files-size" : "6125925", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "30037", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5305247691388084493-1-b7051a4c-4bab-4399-bed0-ea6e540506d6.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 1896028637661931044, + "parent-snapshot-id" : 5305247691388084493, + "timestamp-ms" : 1710862071156, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "73301", + "added-files-size" : "4048500", + "added-position-deletes" : "73301", + "changed-partition-count" : "1", + "total-records" : "176639", + "total-files-size" : "10174425", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "103338", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1896028637661931044-1-08716a3b-1da4-4224-bafa-9fa91a03749f.avro", + "schema-id" : 0 + }, { + "sequence-number" : 5, + "snapshot-id" : 7765867412814781196, + "parent-snapshot-id" : 1896028637661931044, + "timestamp-ms" : 1710862072235, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "65057", + "deleted-records" : "73301", + "added-files-size" : "3405378", + "removed-files-size" : "3938531", + "changed-partition-count" : "1", + "total-records" : "168395", + "total-files-size" : "9641272", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "103338", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-7765867412814781196-1-212e282c-ba0d-42b6-a962-bf0bceb80311.avro", + "schema-id" : 0 + }, { + "sequence-number" : 6, + "snapshot-id" : 4181695983163187512, + "parent-snapshot-id" : 7765867412814781196, + "timestamp-ms" : 1710862073176, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "62729", + "deleted-records" : "65057", + "added-files-size" : "3242827", + "removed-files-size" : "3405378", + "changed-partition-count" : "1", + "total-records" : "166067", + "total-files-size" : "9478721", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "103338", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-4181695983163187512-1-9b90d2c7-b977-401c-a764-f2bedf73122b.avro", + "schema-id" : 0 + }, { + "sequence-number" : 7, + "snapshot-id" : 9149274436460014835, + "parent-snapshot-id" : 4181695983163187512, + "timestamp-ms" : 1710862074244, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "6684", + "added-files-size" : "445198", + "added-position-deletes" : "6684", + "changed-partition-count" : "1", + "total-records" : "172751", + "total-files-size" : "9923919", + "total-data-files" : "5", + "total-delete-files" : "3", + "total-position-deletes" : "110022", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-9149274436460014835-1-bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94.avro", + "schema-id" : 1 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1710862068149, + "snapshot-id" : 5593441557906709930 + }, { + "timestamp-ms" : 1710862069031, + "snapshot-id" : 4018563758056380634 + }, { + "timestamp-ms" : 1710862070028, + "snapshot-id" : 5305247691388084493 + }, { + "timestamp-ms" : 1710862071156, + "snapshot-id" : 1896028637661931044 + }, { + "timestamp-ms" : 1710862072235, + "snapshot-id" : 7765867412814781196 + }, { + "timestamp-ms" : 1710862073176, + "snapshot-id" : 4181695983163187512 + }, { + "timestamp-ms" : 1710862074244, + "snapshot-id" : 9149274436460014835 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1710862068149, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1710862069031, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1710862070028, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1710862071156, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1710862072235, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1710862073176, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v6.metadata.json" + }, { + "timestamp-ms" : 1710862073575, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v7.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v9.metadata.json b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v9.metadata.json new file mode 100644 index 0000000..28de108 --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v9.metadata.json @@ -0,0 +1,490 @@ +{ + "format-version" : 2, + "table-uuid" : "6f901565-5100-4773-bba7-5ae4a88124cb", + "location" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table", + "last-sequence-number" : 7, + "last-updated-ms" : 1710862074681, + "last-column-id" : 16, + "current-schema-id" : 2, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + }, { + "type" : "struct", + "schema-id" : 2, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "long" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 9149274436460014835, + "refs" : { + "main" : { + "snapshot-id" : 9149274436460014835, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 5593441557906709930, + "timestamp-ms" : 1710862068149, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-records" : "60175", + "added-files-size" : "4114898", + "changed-partition-count" : "1", + "total-records" : "60175", + "total-files-size" : "4114898", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5593441557906709930-1-b96e9651-c445-4ed9-b59e-6e2338c6df8e.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4018563758056380634, + "parent-snapshot-id" : 5593441557906709930, + "timestamp-ms" : 1710862069031, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "30037", + "added-files-size" : "1078093", + "added-position-deletes" : "30037", + "changed-partition-count" : "1", + "total-records" : "90212", + "total-files-size" : "5192991", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "30037", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-4018563758056380634-1-a5fef285-1377-4bcd-ac03-faf32d0e8074.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 5305247691388084493, + "parent-snapshot-id" : 4018563758056380634, + "timestamp-ms" : 1710862070028, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-records" : "13126", + "added-files-size" : "932934", + "changed-partition-count" : "1", + "total-records" : "103338", + "total-files-size" : "6125925", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "30037", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5305247691388084493-1-b7051a4c-4bab-4399-bed0-ea6e540506d6.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 1896028637661931044, + "parent-snapshot-id" : 5305247691388084493, + "timestamp-ms" : 1710862071156, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "73301", + "added-files-size" : "4048500", + "added-position-deletes" : "73301", + "changed-partition-count" : "1", + "total-records" : "176639", + "total-files-size" : "10174425", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "103338", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1896028637661931044-1-08716a3b-1da4-4224-bafa-9fa91a03749f.avro", + "schema-id" : 0 + }, { + "sequence-number" : 5, + "snapshot-id" : 7765867412814781196, + "parent-snapshot-id" : 1896028637661931044, + "timestamp-ms" : 1710862072235, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "65057", + "deleted-records" : "73301", + "added-files-size" : "3405378", + "removed-files-size" : "3938531", + "changed-partition-count" : "1", + "total-records" : "168395", + "total-files-size" : "9641272", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "103338", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-7765867412814781196-1-212e282c-ba0d-42b6-a962-bf0bceb80311.avro", + "schema-id" : 0 + }, { + "sequence-number" : 6, + "snapshot-id" : 4181695983163187512, + "parent-snapshot-id" : 7765867412814781196, + "timestamp-ms" : 1710862073176, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "62729", + "deleted-records" : "65057", + "added-files-size" : "3242827", + "removed-files-size" : "3405378", + "changed-partition-count" : "1", + "total-records" : "166067", + "total-files-size" : "9478721", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "103338", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-4181695983163187512-1-9b90d2c7-b977-401c-a764-f2bedf73122b.avro", + "schema-id" : 0 + }, { + "sequence-number" : 7, + "snapshot-id" : 9149274436460014835, + "parent-snapshot-id" : 4181695983163187512, + "timestamp-ms" : 1710862074244, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1710862065256", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "6684", + "added-files-size" : "445198", + "added-position-deletes" : "6684", + "changed-partition-count" : "1", + "total-records" : "172751", + "total-files-size" : "9923919", + "total-data-files" : "5", + "total-delete-files" : "3", + "total-position-deletes" : "110022", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-9149274436460014835-1-bf9efbb6-c0f2-4b2d-bd28-a15f2cce9f94.avro", + "schema-id" : 1 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1710862068149, + "snapshot-id" : 5593441557906709930 + }, { + "timestamp-ms" : 1710862069031, + "snapshot-id" : 4018563758056380634 + }, { + "timestamp-ms" : 1710862070028, + "snapshot-id" : 5305247691388084493 + }, { + "timestamp-ms" : 1710862071156, + "snapshot-id" : 1896028637661931044 + }, { + "timestamp-ms" : 1710862072235, + "snapshot-id" : 7765867412814781196 + }, { + "timestamp-ms" : 1710862073176, + "snapshot-id" : 4181695983163187512 + }, { + "timestamp-ms" : 1710862074244, + "snapshot-id" : 9149274436460014835 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1710862068149, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1710862069031, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1710862070028, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1710862071156, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1710862072235, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1710862073176, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v6.metadata.json" + }, { + "timestamp-ms" : 1710862073575, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v7.metadata.json" + }, { + "timestamp-ms" : 1710862074244, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v8.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/version-hint.text b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/version-hint.text new file mode 100644 index 0000000..f11c82a --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/version-hint.text @@ -0,0 +1 @@ +9 \ No newline at end of file diff --git a/test/sql/iceberg_metadata.test b/test/sql/iceberg_metadata.test index 3238c1c..67407d1 100644 --- a/test/sql/iceberg_metadata.test +++ b/test/sql/iceberg_metadata.test @@ -2,11 +2,12 @@ # description: test iceberg metadata function # group: [iceberg] -# Before we load the extension, this will fail -statement error -SELECT * FROM ICEBERG_METADATA('data/iceberg/lineitem_iceberg'); ----- -Catalog Error +# Commented out because we preload iceberg extension in MD +# # Before we load the extension, this will fail +# statement error +# SELECT * FROM ICEBERG_METADATA('data/iceberg/lineitem_iceberg'); +# ---- +# Catalog Error require iceberg diff --git a/test/sql/iceberg_scan.test b/test/sql/iceberg_scan.test index e505d1c..550aeee 100644 --- a/test/sql/iceberg_scan.test +++ b/test/sql/iceberg_scan.test @@ -2,19 +2,21 @@ # description: test iceberg extension # group: [iceberg] -# Before we load the extension, this will fail -statement error -SELECT * FROM ICEBERG_SCAN('data/iceberg/lineitem_iceberg'); ----- -Catalog Error +# Commented out because we preload iceberg extension in MD +# # Before we load the extension, this will fail +# statement error +# SELECT * FROM ICEBERG_SCAN('data/iceberg/lineitem_iceberg'); +# ---- +# Catalog Error require iceberg # parquet is required for scanning this parquet-backed iceberg table -statement error -SELECT count(*) FROM ICEBERG_SCAN('data/iceberg/lineitem_iceberg', ALLOW_MOVED_PATHS=TRUE); ----- -Catalog Error +# commented out in MD because (hybrid) the bind-replace that generates parquet_scans is executed on duckling, so this succeeds +# statement error +# SELECT count(*) FROM ICEBERG_SCAN('data/iceberg/lineitem_iceberg', ALLOW_MOVED_PATHS=TRUE); +# ---- +# Catalog Error require parquet diff --git a/test/sql/iceberg_snapshots.test b/test/sql/iceberg_snapshots.test index 39d29d8..24c3ad0 100644 --- a/test/sql/iceberg_snapshots.test +++ b/test/sql/iceberg_snapshots.test @@ -2,11 +2,12 @@ # description: test iceberg snapshots function # group: [iceberg] -# Before we load the extension, this will fail -statement error -SELECT * FROM ICEBERG_SNAPSHOTS('data/iceberg/lineitem_iceberg'); ----- -Catalog Error +# Commented out because we preload iceberg extension in MD +# # Before we load the extension, this will fail +# statement error +# SELECT * FROM ICEBERG_SNAPSHOTS('data/iceberg/lineitem_iceberg'); +# ---- +# Catalog Error require notwindows diff --git a/test/sql/md_hive_partitioning_pushdown.test b/test/sql/md_hive_partitioning_pushdown.test new file mode 100644 index 0000000..4c14e30 --- /dev/null +++ b/test/sql/md_hive_partitioning_pushdown.test @@ -0,0 +1,28 @@ +# name: hive_partioning_pushdown.test +# description: confirm that hive-specific filter pushdown now works in MD + +require parquet + +# Confirm +statement ok +COPY (SELECT i::VARCHAR as a, (i*10)::VARCHAR as b, (i*100)::VARCHAR as c from range(0,10) tbl(i)) TO '__TEST_DIR__/hive_pushdown_bug' (FORMAT PARQUET, PARTITION_BY c); + +# Check that hive specific filters are pushed down into the explain plan regardless of format type +query II +explain SELECT * FROM parquet_scan('__TEST_DIR__/hive_pushdown_bug/*/*.parquet', HIVE_PARTITIONING=1, HIVE_TYPES_AUTOCAST=1) where c=500; +---- +physical_plan :.*PARQUET_SCAN.*Filters:.*c[ ]*=[ ]*500.* + +query II +explain SELECT * FROM parquet_scan('__TEST_DIR__/hive_pushdown_bug/*/*.parquet', HIVE_PARTITIONING=1, HIVE_TYPES_AUTOCAST=1) where c=500 and b=20; +---- +physical_plan :.*PARQUET_SCAN.*Filters:.*c[ ]*=[ ]*500.* + +# File Filters show up in read csv auto for hive partitioned csv files. +statement ok +COPY (SELECT i::VARCHAR as a, (i*10)::VARCHAR as b, (i*100)::VARCHAR as c from range(0,10) tbl(i)) TO '__TEST_DIR__/hive_pushdown_bug_csv' (FORMAT CSV, PARTITION_BY c); + +query II +explain SELECT * FROM read_csv_auto('__TEST_DIR__/hive_pushdown_bug_csv/*/*.csv', HIVE_PARTITIONING=1, HIVE_TYPES_AUTOCAST=1, names=['a','b','c']) where c=500; +---- +physical_plan :.*READ_CSV_AUTO.*Filters:.*c[ ]*=[ ]*500.* diff --git a/test/sql/md_iceberg_scan_generated_data_0_01.test b/test/sql/md_iceberg_scan_generated_data_0_01.test new file mode 100644 index 0000000..62ab4d9 --- /dev/null +++ b/test/sql/md_iceberg_scan_generated_data_0_01.test @@ -0,0 +1,64 @@ +# name: test/sql/iceberg_scan_generated_data_0_01.test_slow +# description: test iceberg extension with the sf0.01 generated test set +# group: [iceberg] + +require parquet + +require iceberg + +### Iceberg spec v1 + +foreach iceberg_spec 1 + +# Check count matches +query II +EXPLAIN SELECT count(*) FROM ICEBERG_SCAN('data/iceberg/generated_spec1_0_01/pyspark_iceberg_table') WHERE l_partkey_int < 30; +---- +physical_plan :.*PARQUET_SCAN.*Filters:.*l_partkey_int<30.* + +# Check count matches +query I +SELECT count(*) FROM ICEBERG_SCAN('data/iceberg/generated_spec1_0_01/pyspark_iceberg_table') WHERE l_partkey_int < 30; +---- +716 + +# We should also be able to scan the metadata file directly +query I +SELECT count(l_orderkey_bool) FROM ICEBERG_SCAN('data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v6.metadata.json') WHERE l_partkey_int < 30; +---- +716 + +# Check data is identical, sorting by uuid to guarantee unique order +query I nosort q1 +SELECT * FROM ICEBERG_SCAN('data/iceberg/generated_spec1_0_01/pyspark_iceberg_table') WHERE l_partkey_int < 30 ORDER BY uuid; +---- + +query I nosort q1 +SELECT * FROM ICEBERG_SCAN('data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v9.metadata.json') WHERE l_partkey_int < 30 ORDER BY uuid; +---- + +### Iceberg spec v2 + +# Check count matches +query I +SELECT count(*) FROM ICEBERG_SCAN('data/iceberg/generated_spec2_0_01/pyspark_iceberg_table') WHERE l_partkey_int < 30; +---- +550 + +# We should also be able to scan the metadata file directly +query I +SELECT count(l_partkey_int) FROM ICEBERG_SCAN('data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v6.metadata.json') WHERE l_partkey_int < 30; +---- +550 + +# Check data is identical, sorting by uuid to guarantee unique order +query I nosort q2 +SELECT * FROM ICEBERG_SCAN('data/iceberg/generated_spec2_0_01/pyspark_iceberg_table') WHERE l_partkey_int < 30 ORDER BY uuid; +---- + +# Check data is identical, sorting by uuid to guarantee unique order +query I nosort q2 +SELECT * FROM ICEBERG_SCAN('data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v9.metadata.json') WHERE l_partkey_int < 30 ORDER BY uuid; +---- + +endloop