-
Notifications
You must be signed in to change notification settings - Fork 1.7k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[Kernel] [CC Refactor #1] Add ConfigurationProvider, TableIdentifier, TableBuilder APIs #3781
Changes from all commits
1485372
5c16b28
778705d
87e199c
8cc78f9
21735a1
9e4e53a
ef353e1
5d1f382
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,76 @@ | ||
/* | ||
* Copyright (2024) The Delta Lake Project Authors. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package io.delta.kernel; | ||
|
||
import io.delta.kernel.annotation.Evolving; | ||
import io.delta.kernel.config.ConfigurationProvider; | ||
import io.delta.kernel.engine.Engine; | ||
import io.delta.kernel.exceptions.TableNotFoundException; | ||
|
||
/** | ||
* Builder for creating a {@link Table} instance. | ||
* | ||
* <p>Required parameters for building a {@link TableBuilder} instance include: | ||
* | ||
* <ul> | ||
* <li>{@code Engine}: The {@link Engine} instance required to interact with Delta Kernel. | ||
* <li>{@code Path}: The location of the table, resolved by the provided {@code Engine}. | ||
* </ul> | ||
* | ||
* @since 3.3.0 | ||
*/ | ||
@Evolving | ||
public interface TableBuilder { | ||
|
||
/** | ||
* Set the {@link ConfigurationProvider} to use for the table. | ||
* | ||
* @param configProvider {@link ConfigurationProvider} to use for the table | ||
* @return this updated {@link TableBuilder} instance | ||
*/ | ||
TableBuilder withConfigurationProvider(ConfigurationProvider configProvider); | ||
|
||
/** | ||
* Set the {@link TableIdentifier} for the table. | ||
* | ||
* @param tableId {@link TableIdentifier} for the table | ||
* @return this updated {@link TableBuilder} instance | ||
*/ | ||
TableBuilder withTableId(TableIdentifier tableId); | ||
|
||
/** | ||
* Builds the table object for the Delta Lake table at the given path. | ||
* | ||
* <ul> | ||
* <li>Behavior when the table location doesn't exist: | ||
* <ul> | ||
* <li>Reads will fail with a {@link TableNotFoundException} | ||
* <li>Writes will create the location | ||
* </ul> | ||
* <li>Behavior when the table location exists (with contents or not) but is not a Delta table: | ||
* <ul> | ||
* <li>Reads will fail with a {@link TableNotFoundException} | ||
* <li>Writes will create a Delta table at the given location. If there are any existing | ||
* files in the location that are not already part of the Delta table, they will | ||
* remain excluded from the Delta table. | ||
* </ul> | ||
* </ul> | ||
* | ||
* @return the new {@link Table} instance | ||
*/ | ||
Table build(); | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
/* | ||
* Copyright (2024) The Delta Lake Project Authors. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package io.delta.kernel; | ||
|
||
/** Identifier for a table. e.g. <catalog> / <schema> / <tableName> */ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
public class TableIdentifier { | ||
/** The name of the table. */ | ||
private final String name; | ||
|
||
/** The namespace of the table. e.g. <catalog> / <schema> */ | ||
private final String[] namespace; | ||
Comment on lines
+21
to
+25
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: The order of constructor arguments & getters is There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks! Will do! I'm still working on some more PRs to fully flesh out this design. Will get to this later. |
||
|
||
public TableIdentifier(String[] namespace, String name) { | ||
this.namespace = namespace; | ||
this.name = name; | ||
Comment on lines
+28
to
+29
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
} | ||
|
||
/** Returns the namespace of the table. */ | ||
public String[] getNamespace() { | ||
return namespace; | ||
} | ||
|
||
/** Returns the name of the table. */ | ||
public String getName() { | ||
return name; | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. toString (for printing into logs). equals, hashCode (may add if needed later, but I do see this is useful in near future). |
||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
/* | ||
* Copyright (2024) The Delta Lake Project Authors. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package io.delta.kernel.config; | ||
|
||
import io.delta.kernel.annotation.Evolving; | ||
import java.util.NoSuchElementException; | ||
import java.util.Optional; | ||
|
||
/** | ||
* Provides per-session configuration values for Delta Kernel, specific to the current execution. | ||
* | ||
* <p>These values are not Delta table properties. Delta table properties are stored in the Delta | ||
* log metadata and apply globally to all readers and writers interacting with the table. | ||
Comment on lines
+26
to
+27
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. can you give an example of where this gets used or what sort of configuration can it contain? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. First example usageThe implementation of CommitCoordinatorClient Engine::getCommitCoordinatorClient(
String commitCoordinatorName,
ConfigurationProvider sessionConfig,
Map<String, String> commitCoordinatorConf); will use it to lookup which builder to use for the given It will lookup a value like Second example usageThen, the actual builder referenced above will lookup various keys / configurations needed to instantiate the client. |
||
* | ||
* <p>Instead, the configuration values provided by this interface are scoped to a single instance | ||
* of Delta Kernel during its execution. | ||
* | ||
* @since 3.3.0 | ||
*/ | ||
@Evolving | ||
public interface ConfigurationProvider { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. what is the lifecycle of this object? Can Kernel maintain a reference to it? expected to release it after some event? can Engine update it while Kernel is using it (i.e return different values at different times for the same key)? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. UC Commit Coordinator Client needs to be able to iterate through all configs |
||
/** | ||
* Retrieves the configuration value for the given key. | ||
* | ||
* @param key the configuration key | ||
* @return the configuration value associated with the key | ||
* @throws NoSuchElementException if the key is not found | ||
*/ | ||
String get(String key); | ||
|
||
/** | ||
* Retrieves the configuration value for the given key. If it doesn't exist, returns {@link | ||
* Optional#empty}. | ||
* | ||
* @param key the configuration key | ||
* @return the configuration value associated with the key, if it exists | ||
*/ | ||
Optional<String> getOptional(String key); | ||
|
||
/** | ||
* Checks if the configuration provider contains the given key. | ||
* | ||
* @param key the configuration key | ||
* @return {@code true} if the key exists, else {@code false} | ||
*/ | ||
boolean contains(String key); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Is it better to just have There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. That makes sense. Thanks! |
||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
/* | ||
* Copyright (2024) The Delta Lake Project Authors. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package io.delta.kernel.internal; | ||
|
||
import io.delta.kernel.config.ConfigurationProvider; | ||
import java.util.NoSuchElementException; | ||
import java.util.Optional; | ||
|
||
/** | ||
* An implementation of {@link ConfigurationProvider} that always returns empty results or throws | ||
* exceptions for configuration values. This class is intended to be used in cases where no | ||
* configuration is provided or expected. | ||
*/ | ||
public class EmptyConfigurationProvider implements ConfigurationProvider { | ||
@Override | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. can define just one static instance and use it everywhere rather than creating a new instance everytime. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. that could be even in |
||
public String get(String key) { | ||
throw new NoSuchElementException(); | ||
} | ||
|
||
@Override | ||
public Optional<String> getOptional(String key) { | ||
return Optional.empty(); | ||
} | ||
|
||
@Override | ||
public boolean contains(String key) { | ||
return false; | ||
} | ||
} |
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
@@ -0,0 +1,77 @@ | ||||||
/* | ||||||
* Copyright (2024) The Delta Lake Project Authors. | ||||||
* | ||||||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||||||
* you may not use this file except in compliance with the License. | ||||||
* You may obtain a copy of the License at | ||||||
* | ||||||
* http://www.apache.org/licenses/LICENSE-2.0 | ||||||
* | ||||||
* Unless required by applicable law or agreed to in writing, software | ||||||
* distributed under the License is distributed on an "AS IS" BASIS, | ||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||||||
* See the License for the specific language governing permissions and | ||||||
* limitations under the License. | ||||||
*/ | ||||||
|
||||||
package io.delta.kernel.internal; | ||||||
|
||||||
import static java.util.Objects.requireNonNull; | ||||||
|
||||||
import io.delta.kernel.Table; | ||||||
import io.delta.kernel.TableBuilder; | ||||||
import io.delta.kernel.TableIdentifier; | ||||||
import io.delta.kernel.config.ConfigurationProvider; | ||||||
import io.delta.kernel.engine.Engine; | ||||||
import io.delta.kernel.internal.util.Clock; | ||||||
import java.util.Optional; | ||||||
|
||||||
public class TableBuilderImpl implements TableBuilder { | ||||||
private final Engine engine; | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. we avoid this kind of pattern where There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. That makes sense. We will have to think about how to work around this, then // Table.java
static TableBuilder builder() { return new TableBuilderImpl(); }
// TableBuilder.java
public Table build(Engine engine, String path) |
||||||
private final String path; | ||||||
|
||||||
private Optional<ConfigurationProvider> configProviderOpt = Optional.empty(); | ||||||
private Optional<TableIdentifier> tableIdOpt = Optional.empty(); | ||||||
|
||||||
/** Note: This is not for a public API. It's for internal testing use only. */ | ||||||
private Optional<Clock> clockOpt = Optional.empty(); | ||||||
Comment on lines
+33
to
+37
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There's no need to add There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I hear that. This is certainly one of my personal preferences. I like variables being fully self-describing. The value isn't a |
||||||
|
||||||
public TableBuilderImpl(Engine engine, String path) { | ||||||
requireNonNull(engine, "Expected non-null value for 'engine'"); | ||||||
requireNonNull(path, "Expected non-null value for 'path'"); | ||||||
Comment on lines
+40
to
+41
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The existing message pattern of |
||||||
|
||||||
this.engine = engine; | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
this.path = path; | ||||||
} | ||||||
|
||||||
@Override | ||||||
public TableBuilder withConfigurationProvider(ConfigurationProvider configProvider) { | ||||||
requireNonNull(configProvider, "Expected non-null value for 'configProvider'"); | ||||||
this.configProviderOpt = Optional.of(configProvider); | ||||||
return this; | ||||||
} | ||||||
|
||||||
@Override | ||||||
public TableBuilder withTableId(TableIdentifier tableId) { | ||||||
requireNonNull(tableId, "Expected non-null value for 'tableId'"); | ||||||
this.tableIdOpt = Optional.of(tableId); | ||||||
return null; | ||||||
} | ||||||
|
||||||
/** Note: This is not a public API. It's for internal testing use only. */ | ||||||
public TableBuilder withClock(Clock clock) { | ||||||
requireNonNull(clock, "Expected non-null value for 'clock'"); | ||||||
this.clockOpt = Optional.of(clock); | ||||||
return this; | ||||||
} | ||||||
|
||||||
@Override | ||||||
public Table build() { | ||||||
return TableImpl.create( | ||||||
engine, | ||||||
path, | ||||||
clockOpt.orElse(System::currentTimeMillis), | ||||||
configProviderOpt.orElse(new EmptyConfigurationProvider()), | ||||||
tableIdOpt); | ||||||
} | ||||||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@since 3.3.0
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@Evolving