Skip to content

Commit

Permalink
PARQUET-2465: Fall back to HadoopConfig (#1339)
Browse files Browse the repository at this point in the history
Add fallback logic

We see that this causes the 1.14 to be incompatible
with the previous releases. The config will be created
and right after that the `getWriteSupport(conf)` is called.

But since this method is freshly introduced:

```java
    protected WriteSupport<T> getWriteSupport(ParquetConfiguration conf) {
      throw new UnsupportedOperationException(
          "Override ParquetWriter$Builder#getWriteSupport(ParquetConfiguration)");
    }
```
  • Loading branch information
Fokko authored May 3, 2024
1 parent 6860919 commit 408c18b
Show file tree
Hide file tree
Showing 4 changed files with 16 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -495,16 +495,17 @@ protected Builder(OutputFile path) {
/**
* @param conf a configuration
* @return an appropriate WriteSupport for the object model.
* @deprecated Use {@link #getWriteSupport(ParquetConfiguration)} instead
*/
@Deprecated
protected abstract WriteSupport<T> getWriteSupport(Configuration conf);

/**
* @param conf a configuration
* @return an appropriate WriteSupport for the object model.
*/
protected WriteSupport<T> getWriteSupport(ParquetConfiguration conf) {
throw new UnsupportedOperationException(
"Override ParquetWriter$Builder#getWriteSupport(ParquetConfiguration)");
return getWriteSupport(ConfigurationUtil.createHadoopConfiguration(conf));
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,9 @@ public Map<String, String> getMergedKeyValueMetaData() {

/**
* @return the configuration for this job
* @deprecated Use {@link #getParquetConfiguration()} instead
*/
@Deprecated
public Configuration getConfiguration() {
return ConfigurationUtil.createHadoopConfiguration(configuration);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.parquet.conf.ParquetConfiguration;
import org.apache.parquet.hadoop.util.ConfigurationUtil;
import org.apache.parquet.io.api.RecordMaterializer;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;
Expand Down Expand Up @@ -76,12 +77,12 @@ public ReadContext init(Configuration configuration, Map<String, String> keyValu
* @param keyValueMetaData the app specific metadata from the file
* @param fileSchema the schema of the file
* @return the readContext that defines how to read the file
* @deprecated override {@link ReadSupport#init(InitContext)} instead
* @deprecated override {@link #init(InitContext)} instead
*/
@Deprecated
public ReadContext init(
ParquetConfiguration configuration, Map<String, String> keyValueMetaData, MessageType fileSchema) {
throw new UnsupportedOperationException("Override ReadSupport.init(InitContext)");
return init(ConfigurationUtil.createHadoopConfiguration(configuration), keyValueMetaData, fileSchema);
}

/**
Expand All @@ -103,7 +104,9 @@ public ReadContext init(InitContext context) {
* @param fileSchema the schema of the file
* @param readContext returned by the init method
* @return the recordMaterializer that will materialize the records
* @deprecated override {@link #prepareForRead(ParquetConfiguration,Map,MessageType,ReadContext)} instead
*/
@Deprecated
public abstract RecordMaterializer<T> prepareForRead(
Configuration configuration,
Map<String, String> keyValueMetaData,
Expand All @@ -125,8 +128,8 @@ public RecordMaterializer<T> prepareForRead(
Map<String, String> keyValueMetaData,
MessageType fileSchema,
ReadContext readContext) {
throw new UnsupportedOperationException(
"Override ReadSupport.prepareForRead(ParquetConfiguration, Map<String, String>, MessageType, ReadContext)");
return prepareForRead(
ConfigurationUtil.createHadoopConfiguration(configuration), keyValueMetaData, fileSchema, readContext);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import java.util.Objects;
import org.apache.hadoop.conf.Configuration;
import org.apache.parquet.conf.ParquetConfiguration;
import org.apache.parquet.hadoop.util.ConfigurationUtil;
import org.apache.parquet.io.api.RecordConsumer;
import org.apache.parquet.schema.MessageType;

Expand Down Expand Up @@ -99,7 +100,9 @@ public Map<String, String> getExtraMetaData() {
*
* @param configuration the job's configuration
* @return the information needed to write the file
* @deprecated override {@link #init(ParquetConfiguration)} instead
*/
@Deprecated
public abstract WriteContext init(Configuration configuration);

/**
Expand All @@ -109,7 +112,7 @@ public Map<String, String> getExtraMetaData() {
* @return the information needed to write the file
*/
public WriteContext init(ParquetConfiguration configuration) {
throw new UnsupportedOperationException("Override WriteSupport#init(ParquetConfiguration)");
return init(ConfigurationUtil.createHadoopConfiguration(configuration));
}

/**
Expand Down

0 comments on commit 408c18b

Please sign in to comment.