Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace dropped PartitonField with AnyType #11542

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion api/src/main/java/org/apache/iceberg/PartitionSpec.java
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,14 @@ public StructType partitionType() {

for (PartitionField field : fields) {
Type sourceType = schema.findType(field.sourceId());
Type resultType = field.transform().getResultType(sourceType);

final Type resultType;
if (sourceType == null) {
resultType = Types.AnyType.get();
} else {
resultType = field.transform().getResultType(sourceType);
}

structFields.add(Types.NestedField.optional(field.fieldId(), field.name(), resultType));
}

Expand Down
3 changes: 2 additions & 1 deletion api/src/main/java/org/apache/iceberg/types/Type.java
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@ enum TypeID {
DECIMAL(BigDecimal.class),
STRUCT(StructLike.class),
LIST(List.class),
MAP(Map.class);
MAP(Map.class),
ANY(Object.class);

private final Class<?> javaClass;

Expand Down
19 changes: 19 additions & 0 deletions api/src/main/java/org/apache/iceberg/types/Types.java
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ private Types() {}
.put(StringType.get().toString(), StringType.get())
.put(UUIDType.get().toString(), UUIDType.get())
.put(BinaryType.get().toString(), BinaryType.get())
.put(AnyType.get().toString(), AnyType.get())
.buildOrThrow();

private static final Pattern FIXED = Pattern.compile("fixed\\[\\s*(\\d+)\\s*\\]");
Expand Down Expand Up @@ -412,6 +413,24 @@ public String toString() {
}
}

public static class AnyType extends PrimitiveType {
private static final AnyType INSTANCE = new AnyType();

public static AnyType get() {
return INSTANCE;
}

@Override
public TypeID typeId() {
return TypeID.ANY;
}

@Override
public String toString() {
return "any";
}
}

public static class DecimalType extends PrimitiveType {
public static DecimalType of(int precision, int scale) {
return new DecimalType(precision, scale);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,8 @@ static Schema toOption(Schema schema) {
Preconditions.checkArgument(
isOptionSchema(schema), "Union schemas are not supported: %s", schema);
return schema;
} else if (schema.getType() == Schema.Type.NULL) {
return schema;
} else {
return Schema.createUnion(NULL, schema);
}
Expand Down
4 changes: 4 additions & 0 deletions core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ abstract class TypeToSchema extends TypeUtil.SchemaVisitor<Schema> {
private static final Schema UUID_SCHEMA =
LogicalTypes.uuid().addToSchema(Schema.createFixed("uuid_fixed", null, null, 16));
private static final Schema BINARY_SCHEMA = Schema.create(Schema.Type.BYTES);
private static final Schema NULL_SCHEMA = Schema.create(Schema.Type.NULL);

static {
TIMESTAMP_SCHEMA.addProp(AvroSchemaUtil.ADJUST_TO_UTC_PROP, false);
Expand Down Expand Up @@ -243,6 +244,9 @@ public Schema primitive(Type.PrimitiveType primitive) {
null,
TypeUtil.decimalRequiredBytes(decimal.precision())));
break;
case ANY:
primitiveSchema = NULL_SCHEMA;
break;
default:
throw new UnsupportedOperationException("Unsupported type ID: " + primitive.typeId());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,20 @@ public void testReplacePartitionAndRename() {
.isEqualTo(expected);
}

@TestTemplate
public void testDropPartitionAndUnderlyingField() {
sql(
"CREATE TABLE %s (col0 BIGINT, col1 BIGINT, col2 BIGINT) USING ICEBERG TBLPROPERTIES ('format-version' = %d, 'write.delete.mode' = 'merge-on-read')",
tableName, formatVersion);
sql("INSERT INTO %s VALUES (1, 11, 21)", tableName);
sql("ALTER TABLE %s ADD PARTITION FIELD col2", tableName);
sql("INSERT INTO %s VALUES (2, 12, 22)", tableName);
sql("ALTER TABLE %s DROP PARTITION FIELD col2", tableName);
sql("INSERT INTO %s VALUES (3, 13, 23)", tableName);
sql("ALTER TABLE %s DROP COLUMN col2", tableName);
sql("SELECT * FROM %s", tableName);
}

@TestTemplate
public void testReplaceNamedPartition() {
createTable("id bigint NOT NULL, category string, ts timestamp, data string");
Expand Down
Loading