Skip to content

Commit

Permalink
Merge branch 'master' of https://github.com/honeyaya/hudi into HUDI-4809
Browse files Browse the repository at this point in the history
  • Loading branch information
honeyaya committed Oct 25, 2022
2 parents a13855e + 4965526 commit 57a1cf5
Show file tree
Hide file tree
Showing 79 changed files with 1,072 additions and 743 deletions.
4 changes: 4 additions & 0 deletions .asf.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ github:
squash: true
merge: false
rebase: false
protected_branches:
master:
required_pull_request_reviews:
required_approving_review_count: 1
notifications:
commits: [email protected]
issues: [email protected]
Expand Down
9 changes: 9 additions & 0 deletions .github/workflows/bot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,15 @@ jobs:
FLINK_PROFILE: ${{ matrix.flinkProfile }}
run:
mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -DfailIfNoTests=false -pl hudi-examples/hudi-examples-flink,hudi-examples/hudi-examples-java,hudi-examples/hudi-examples-spark $MVN_ARGS
- name: Bundle Validation
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
SPARK_PROFILE: ${{ matrix.sparkProfile }}
FLINK_PROFILE: ${{ matrix.flinkProfile }}
if: ${{ !endsWith(env.SPARK_PROFILE, '2.4') }} # skip test spark 2.4 as it's covered by Azure CI
run: |
HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
./packaging/bundle-validation/spark-write-hive-sync/ci_run.sh $HUDI_VERSION
- name: Spark SQL Test
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
Expand Down
4 changes: 0 additions & 4 deletions dependencies/hudi-flink-bundle_2.11.txt
Original file line number Diff line number Diff line change
Expand Up @@ -162,12 +162,8 @@ httpclient/org.apache.httpcomponents/4.4.1//httpclient-4.4.1.jar
httpcore/org.apache.httpcomponents/4.4.1//httpcore-4.4.1.jar
ivy/org.apache.ivy/2.4.0//ivy-2.4.0.jar
jackson-annotations/com.fasterxml.jackson.core/2.6.7//jackson-annotations-2.6.7.jar
jackson-core-asl/org.codehaus.jackson/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/com.fasterxml.jackson.core/2.6.7//jackson-core-2.6.7.jar
jackson-databind/com.fasterxml.jackson.core/2.6.7.3//jackson-databind-2.6.7.3.jar
jackson-jaxrs/org.codehaus.jackson/1.9.13//jackson-jaxrs-1.9.13.jar
jackson-mapper-asl/org.codehaus.jackson/1.9.13//jackson-mapper-asl-1.9.13.jar
jackson-xc/org.codehaus.jackson/1.9.13//jackson-xc-1.9.13.jar
jamon-runtime/org.jamon/2.3.1//jamon-runtime-2.3.1.jar
janino/org.codehaus.janino/2.7.6//janino-2.7.6.jar
jasper-compiler/tomcat/5.5.23//jasper-compiler-5.5.23.jar
Expand Down
4 changes: 0 additions & 4 deletions dependencies/hudi-flink-bundle_2.12.txt
Original file line number Diff line number Diff line change
Expand Up @@ -163,12 +163,8 @@ httpclient/org.apache.httpcomponents/4.4.1//httpclient-4.4.1.jar
httpcore/org.apache.httpcomponents/4.4.1//httpcore-4.4.1.jar
ivy/org.apache.ivy/2.4.0//ivy-2.4.0.jar
jackson-annotations/com.fasterxml.jackson.core/2.10.0//jackson-annotations-2.10.0.jar
jackson-core-asl/org.codehaus.jackson/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/com.fasterxml.jackson.core/2.10.0//jackson-core-2.10.0.jar
jackson-databind/com.fasterxml.jackson.core/2.10.0//jackson-databind-2.10.0.jar
jackson-jaxrs/org.codehaus.jackson/1.9.13//jackson-jaxrs-1.9.13.jar
jackson-mapper-asl/org.codehaus.jackson/1.9.13//jackson-mapper-asl-1.9.13.jar
jackson-xc/org.codehaus.jackson/1.9.13//jackson-xc-1.9.13.jar
jamon-runtime/org.jamon/2.3.1//jamon-runtime-2.3.1.jar
janino/org.codehaus.janino/2.7.6//janino-2.7.6.jar
jasper-compiler/tomcat/5.5.23//jasper-compiler-5.5.23.jar
Expand Down
4 changes: 0 additions & 4 deletions dependencies/hudi-hadoop-mr-bundle.txt
Original file line number Diff line number Diff line change
Expand Up @@ -81,12 +81,8 @@ htrace-core/org.apache.htrace/3.1.0-incubating//htrace-core-3.1.0-incubating.jar
httpclient/org.apache.httpcomponents/4.4.1//httpclient-4.4.1.jar
httpcore/org.apache.httpcomponents/4.4.1//httpcore-4.4.1.jar
jackson-annotations/com.fasterxml.jackson.core/2.6.7//jackson-annotations-2.6.7.jar
jackson-core-asl/org.codehaus.jackson/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/com.fasterxml.jackson.core/2.6.7//jackson-core-2.6.7.jar
jackson-databind/com.fasterxml.jackson.core/2.6.7.3//jackson-databind-2.6.7.3.jar
jackson-jaxrs/org.codehaus.jackson/1.9.13//jackson-jaxrs-1.9.13.jar
jackson-mapper-asl/org.codehaus.jackson/1.9.13//jackson-mapper-asl-1.9.13.jar
jackson-xc/org.codehaus.jackson/1.9.13//jackson-xc-1.9.13.jar
jamon-runtime/org.jamon/2.4.1//jamon-runtime-2.4.1.jar
java-xmlbuilder/com.jamesmurty.utils/0.4//java-xmlbuilder-0.4.jar
javax.inject/javax.inject/1//javax.inject-1.jar
Expand Down
2 changes: 0 additions & 2 deletions dependencies/hudi-hive-sync-bundle.txt
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,8 @@ htrace-core/org.apache.htrace/3.1.0-incubating//htrace-core-3.1.0-incubating.jar
httpclient/org.apache.httpcomponents/4.4.1//httpclient-4.4.1.jar
httpcore/org.apache.httpcomponents/4.4.1//httpcore-4.4.1.jar
jackson-annotations/com.fasterxml.jackson.core/2.6.7//jackson-annotations-2.6.7.jar
jackson-core-asl/org.codehaus.jackson/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/com.fasterxml.jackson.core/2.6.7//jackson-core-2.6.7.jar
jackson-databind/com.fasterxml.jackson.core/2.6.7.3//jackson-databind-2.6.7.3.jar
jackson-mapper-asl/org.codehaus.jackson/1.9.13//jackson-mapper-asl-1.9.13.jar
jamon-runtime/org.jamon/2.4.1//jamon-runtime-2.4.1.jar
java-xmlbuilder/com.jamesmurty.utils/0.4//java-xmlbuilder-0.4.jar
javax.inject/javax.inject/1//javax.inject-1.jar
Expand Down
4 changes: 0 additions & 4 deletions dependencies/hudi-integ-test-bundle.txt
Original file line number Diff line number Diff line change
Expand Up @@ -159,17 +159,13 @@ httpcore/org.apache.httpcomponents/4.4.1//httpcore-4.4.1.jar
ion-java/software.amazon.ion/1.0.2//ion-java-1.0.2.jar
ivy/org.apache.ivy/2.4.0//ivy-2.4.0.jar
jackson-annotations/com.fasterxml.jackson.core/2.6.7//jackson-annotations-2.6.7.jar
jackson-core-asl/org.codehaus.jackson/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/com.fasterxml.jackson.core/2.6.7//jackson-core-2.6.7.jar
jackson-databind/com.fasterxml.jackson.core/2.6.7.3//jackson-databind-2.6.7.3.jar
jackson-dataformat-cbor/com.fasterxml.jackson.dataformat/2.12.3//jackson-dataformat-cbor-2.12.3.jar
jackson-dataformat-csv/com.fasterxml.jackson.dataformat/2.6.7//jackson-dataformat-csv-2.6.7.jar
jackson-dataformat-yaml/com.fasterxml.jackson.dataformat/2.7.4//jackson-dataformat-yaml-2.7.4.jar
jackson-jaxrs/org.codehaus.jackson/1.9.13//jackson-jaxrs-1.9.13.jar
jackson-mapper-asl/org.codehaus.jackson/1.9.13//jackson-mapper-asl-1.9.13.jar
jackson-module-paranamer/com.fasterxml.jackson.module/2.7.9//jackson-module-paranamer-2.7.9.jar
jackson-module-scala_2.11/com.fasterxml.jackson.module/2.6.7.1//jackson-module-scala_2.11-2.6.7.1.jar
jackson-xc/org.codehaus.jackson/1.9.13//jackson-xc-1.9.13.jar
jamon-runtime/org.jamon/2.4.1//jamon-runtime-2.4.1.jar
janino/org.codehaus.janino/2.7.6//janino-2.7.6.jar
jasper-compiler/tomcat/5.5.23//jasper-compiler-5.5.23.jar
Expand Down
4 changes: 0 additions & 4 deletions dependencies/hudi-kafka-connect-bundle.txt
Original file line number Diff line number Diff line change
Expand Up @@ -130,16 +130,12 @@ httpcore/org.apache.httpcomponents/4.4.1//httpcore-4.4.1.jar
ion-java/software.amazon.ion/1.0.2//ion-java-1.0.2.jar
ivy/org.apache.ivy/2.4.0//ivy-2.4.0.jar
jackson-annotations/com.fasterxml.jackson.core/2.6.7//jackson-annotations-2.6.7.jar
jackson-core-asl/org.codehaus.jackson/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/com.fasterxml.jackson.core/2.6.7//jackson-core-2.6.7.jar
jackson-databind/com.fasterxml.jackson.core/2.6.7.3//jackson-databind-2.6.7.3.jar
jackson-dataformat-cbor/com.fasterxml.jackson.dataformat/2.12.3//jackson-dataformat-cbor-2.12.3.jar
jackson-dataformat-csv/com.fasterxml.jackson.dataformat/2.6.7//jackson-dataformat-csv-2.6.7.jar
jackson-jaxrs/org.codehaus.jackson/1.9.13//jackson-jaxrs-1.9.13.jar
jackson-mapper-asl/org.codehaus.jackson/1.9.13//jackson-mapper-asl-1.9.13.jar
jackson-module-paranamer/com.fasterxml.jackson.module/2.7.9//jackson-module-paranamer-2.7.9.jar
jackson-module-scala_2.11/com.fasterxml.jackson.module/2.6.7.1//jackson-module-scala_2.11-2.6.7.1.jar
jackson-xc/org.codehaus.jackson/1.9.13//jackson-xc-1.9.13.jar
jamon-runtime/org.jamon/2.4.1//jamon-runtime-2.4.1.jar
java-xmlbuilder/com.jamesmurty.utils/0.4//java-xmlbuilder-0.4.jar
javalin/io.javalin/2.8.0//javalin-2.8.0.jar
Expand Down
2 changes: 0 additions & 2 deletions dependencies/hudi-presto-bundle.txt
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,8 @@ htrace-core/org.apache.htrace/3.1.0-incubating//htrace-core-3.1.0-incubating.jar
httpclient/org.apache.httpcomponents/4.4.1//httpclient-4.4.1.jar
httpcore/org.apache.httpcomponents/4.4.1//httpcore-4.4.1.jar
jackson-annotations/com.fasterxml.jackson.core/2.6.7//jackson-annotations-2.6.7.jar
jackson-core-asl/org.codehaus.jackson/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/com.fasterxml.jackson.core/2.6.7//jackson-core-2.6.7.jar
jackson-databind/com.fasterxml.jackson.core/2.6.7.3//jackson-databind-2.6.7.3.jar
jackson-mapper-asl/org.codehaus.jackson/1.9.13//jackson-mapper-asl-1.9.13.jar
jamon-runtime/org.jamon/2.4.1//jamon-runtime-2.4.1.jar
java-xmlbuilder/com.jamesmurty.utils/0.4//java-xmlbuilder-0.4.jar
javax.inject/javax.inject/1//javax.inject-1.jar
Expand Down
4 changes: 0 additions & 4 deletions dependencies/hudi-spark-bundle_2.11.txt
Original file line number Diff line number Diff line change
Expand Up @@ -128,14 +128,10 @@ httpclient/org.apache.httpcomponents/4.4.1//httpclient-4.4.1.jar
httpcore/org.apache.httpcomponents/4.4.1//httpcore-4.4.1.jar
ivy/org.apache.ivy/2.4.0//ivy-2.4.0.jar
jackson-annotations/com.fasterxml.jackson.core/2.6.7//jackson-annotations-2.6.7.jar
jackson-core-asl/org.codehaus.jackson/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/com.fasterxml.jackson.core/2.6.7//jackson-core-2.6.7.jar
jackson-databind/com.fasterxml.jackson.core/2.6.7.3//jackson-databind-2.6.7.3.jar
jackson-jaxrs/org.codehaus.jackson/1.9.13//jackson-jaxrs-1.9.13.jar
jackson-mapper-asl/org.codehaus.jackson/1.9.13//jackson-mapper-asl-1.9.13.jar
jackson-module-paranamer/com.fasterxml.jackson.module/2.7.9//jackson-module-paranamer-2.7.9.jar
jackson-module-scala_2.11/com.fasterxml.jackson.module/2.6.7.1//jackson-module-scala_2.11-2.6.7.1.jar
jackson-xc/org.codehaus.jackson/1.9.13//jackson-xc-1.9.13.jar
jamon-runtime/org.jamon/2.3.1//jamon-runtime-2.3.1.jar
janino/org.codehaus.janino/2.7.6//janino-2.7.6.jar
jasper-compiler/tomcat/5.5.23//jasper-compiler-5.5.23.jar
Expand Down
4 changes: 0 additions & 4 deletions dependencies/hudi-spark-bundle_2.12.txt
Original file line number Diff line number Diff line change
Expand Up @@ -128,14 +128,10 @@ httpclient/org.apache.httpcomponents/4.4.1//httpclient-4.4.1.jar
httpcore/org.apache.httpcomponents/4.4.1//httpcore-4.4.1.jar
ivy/org.apache.ivy/2.4.0//ivy-2.4.0.jar
jackson-annotations/com.fasterxml.jackson.core/2.6.7//jackson-annotations-2.6.7.jar
jackson-core-asl/org.codehaus.jackson/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/com.fasterxml.jackson.core/2.6.7//jackson-core-2.6.7.jar
jackson-databind/com.fasterxml.jackson.core/2.6.7.3//jackson-databind-2.6.7.3.jar
jackson-jaxrs/org.codehaus.jackson/1.9.13//jackson-jaxrs-1.9.13.jar
jackson-mapper-asl/org.codehaus.jackson/1.9.13//jackson-mapper-asl-1.9.13.jar
jackson-module-paranamer/com.fasterxml.jackson.module/2.7.9//jackson-module-paranamer-2.7.9.jar
jackson-module-scala_2.11/com.fasterxml.jackson.module/2.6.7.1//jackson-module-scala_2.11-2.6.7.1.jar
jackson-xc/org.codehaus.jackson/1.9.13//jackson-xc-1.9.13.jar
jamon-runtime/org.jamon/2.3.1//jamon-runtime-2.3.1.jar
janino/org.codehaus.janino/2.7.6//janino-2.7.6.jar
jasper-compiler/tomcat/5.5.23//jasper-compiler-5.5.23.jar
Expand Down
4 changes: 0 additions & 4 deletions dependencies/hudi-spark3-bundle_2.12.txt
Original file line number Diff line number Diff line change
Expand Up @@ -128,14 +128,10 @@ httpclient/org.apache.httpcomponents/4.4.1//httpclient-4.4.1.jar
httpcore/org.apache.httpcomponents/4.4.1//httpcore-4.4.1.jar
ivy/org.apache.ivy/2.4.0//ivy-2.4.0.jar
jackson-annotations/com.fasterxml.jackson.core/2.10.0//jackson-annotations-2.10.0.jar
jackson-core-asl/org.codehaus.jackson/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/com.fasterxml.jackson.core/2.10.0//jackson-core-2.10.0.jar
jackson-databind/com.fasterxml.jackson.core/2.10.0//jackson-databind-2.10.0.jar
jackson-jaxrs/org.codehaus.jackson/1.9.13//jackson-jaxrs-1.9.13.jar
jackson-mapper-asl/org.codehaus.jackson/1.9.13//jackson-mapper-asl-1.9.13.jar
jackson-module-paranamer/com.fasterxml.jackson.module/2.7.9//jackson-module-paranamer-2.7.9.jar
jackson-module-scala_2.11/com.fasterxml.jackson.module/2.6.7.1//jackson-module-scala_2.11-2.6.7.1.jar
jackson-xc/org.codehaus.jackson/1.9.13//jackson-xc-1.9.13.jar
jamon-runtime/org.jamon/2.3.1//jamon-runtime-2.3.1.jar
janino/org.codehaus.janino/2.7.6//janino-2.7.6.jar
jasper-compiler/tomcat/5.5.23//jasper-compiler-5.5.23.jar
Expand Down
4 changes: 0 additions & 4 deletions dependencies/hudi-timeline-server-bundle.txt
Original file line number Diff line number Diff line change
Expand Up @@ -77,12 +77,8 @@ htrace-core/org.apache.htrace/3.1.0-incubating//htrace-core-3.1.0-incubating.jar
httpclient/org.apache.httpcomponents/4.4.1//httpclient-4.4.1.jar
httpcore/org.apache.httpcomponents/4.4.1//httpcore-4.4.1.jar
jackson-annotations/com.fasterxml.jackson.core/2.6.7//jackson-annotations-2.6.7.jar
jackson-core-asl/org.codehaus.jackson/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/com.fasterxml.jackson.core/2.6.7//jackson-core-2.6.7.jar
jackson-databind/com.fasterxml.jackson.core/2.6.7.3//jackson-databind-2.6.7.3.jar
jackson-jaxrs/org.codehaus.jackson/1.9.13//jackson-jaxrs-1.9.13.jar
jackson-mapper-asl/org.codehaus.jackson/1.9.13//jackson-mapper-asl-1.9.13.jar
jackson-xc/org.codehaus.jackson/1.9.13//jackson-xc-1.9.13.jar
jamon-runtime/org.jamon/2.4.1//jamon-runtime-2.4.1.jar
java-xmlbuilder/com.jamesmurty.utils/0.4//java-xmlbuilder-0.4.jar
javalin/io.javalin/2.8.0//javalin-2.8.0.jar
Expand Down
4 changes: 0 additions & 4 deletions dependencies/hudi-utilities-bundle_2.11.txt
Original file line number Diff line number Diff line change
Expand Up @@ -147,16 +147,12 @@ httpcore/org.apache.httpcomponents/4.4.1//httpcore-4.4.1.jar
ion-java/software.amazon.ion/1.0.2//ion-java-1.0.2.jar
ivy/org.apache.ivy/2.4.0//ivy-2.4.0.jar
jackson-annotations/com.fasterxml.jackson.core/2.6.7//jackson-annotations-2.6.7.jar
jackson-core-asl/org.codehaus.jackson/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/com.fasterxml.jackson.core/2.6.7//jackson-core-2.6.7.jar
jackson-databind/com.fasterxml.jackson.core/2.6.7.3//jackson-databind-2.6.7.3.jar
jackson-dataformat-cbor/com.fasterxml.jackson.dataformat/2.12.3//jackson-dataformat-cbor-2.12.3.jar
jackson-dataformat-csv/com.fasterxml.jackson.dataformat/2.6.7//jackson-dataformat-csv-2.6.7.jar
jackson-jaxrs/org.codehaus.jackson/1.9.13//jackson-jaxrs-1.9.13.jar
jackson-mapper-asl/org.codehaus.jackson/1.9.13//jackson-mapper-asl-1.9.13.jar
jackson-module-paranamer/com.fasterxml.jackson.module/2.7.9//jackson-module-paranamer-2.7.9.jar
jackson-module-scala_2.11/com.fasterxml.jackson.module/2.6.7.1//jackson-module-scala_2.11-2.6.7.1.jar
jackson-xc/org.codehaus.jackson/1.9.13//jackson-xc-1.9.13.jar
jamon-runtime/org.jamon/2.3.1//jamon-runtime-2.3.1.jar
janino/org.codehaus.janino/2.7.6//janino-2.7.6.jar
jasper-compiler/tomcat/5.5.23//jasper-compiler-5.5.23.jar
Expand Down
4 changes: 0 additions & 4 deletions dependencies/hudi-utilities-bundle_2.12.txt
Original file line number Diff line number Diff line change
Expand Up @@ -147,16 +147,12 @@ httpcore/org.apache.httpcomponents/4.4.1//httpcore-4.4.1.jar
ion-java/software.amazon.ion/1.0.2//ion-java-1.0.2.jar
ivy/org.apache.ivy/2.4.0//ivy-2.4.0.jar
jackson-annotations/com.fasterxml.jackson.core/2.10.0//jackson-annotations-2.10.0.jar
jackson-core-asl/org.codehaus.jackson/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/com.fasterxml.jackson.core/2.10.0//jackson-core-2.10.0.jar
jackson-databind/com.fasterxml.jackson.core/2.10.0//jackson-databind-2.10.0.jar
jackson-dataformat-cbor/com.fasterxml.jackson.dataformat/2.12.3//jackson-dataformat-cbor-2.12.3.jar
jackson-dataformat-csv/com.fasterxml.jackson.dataformat/2.6.7//jackson-dataformat-csv-2.6.7.jar
jackson-jaxrs/org.codehaus.jackson/1.9.13//jackson-jaxrs-1.9.13.jar
jackson-mapper-asl/org.codehaus.jackson/1.9.13//jackson-mapper-asl-1.9.13.jar
jackson-module-paranamer/com.fasterxml.jackson.module/2.7.9//jackson-module-paranamer-2.7.9.jar
jackson-module-scala_2.11/com.fasterxml.jackson.module/2.6.7.1//jackson-module-scala_2.11-2.6.7.1.jar
jackson-xc/org.codehaus.jackson/1.9.13//jackson-xc-1.9.13.jar
jamon-runtime/org.jamon/2.3.1//jamon-runtime-2.3.1.jar
janino/org.codehaus.janino/2.7.6//janino-2.7.6.jar
jasper-compiler/tomcat/5.5.23//jasper-compiler-5.5.23.jar
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

import org.apache.hudi.exception.HoodieCommitCallbackException;

import org.codehaus.jackson.map.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectMapper;

import java.io.IOException;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -492,7 +492,8 @@ public abstract O bulkInsertPreppedRecords(I preppedRecords, final String instan
public void preWrite(String instantTime, WriteOperationType writeOperationType,
HoodieTableMetaClient metaClient) {
setOperationType(writeOperationType);
this.lastCompletedTxnAndMetadata = TransactionUtils.getLastCompletedTxnInstantAndMetadata(metaClient);
this.lastCompletedTxnAndMetadata = txnManager.isOptimisticConcurrencyControlEnabled()
? TransactionUtils.getLastCompletedTxnInstantAndMetadata(metaClient) : Option.empty();
this.pendingInflightAndRequestedInstants = TransactionUtils.getInflightAndRequestedInstants(metaClient);
this.pendingInflightAndRequestedInstants.remove(instantTime);
if (null == this.asyncCleanerService) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,4 +97,7 @@ public Option<HoodieInstant> getCurrentTransactionOwner() {
return currentTxnOwnerInstant;
}

public boolean isOptimisticConcurrencyControlEnabled() {
return isOptimisticConcurrencyControlEnabled;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,15 @@

package org.apache.hudi.client.transaction.lock.metrics;

import org.apache.hudi.common.util.HoodieTimer;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.metrics.Metrics;

import com.codahale.metrics.Counter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SlidingWindowReservoir;
import com.codahale.metrics.Timer;

import org.apache.hudi.common.util.HoodieTimer;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.metrics.Metrics;

import java.util.concurrent.TimeUnit;

public class HoodieLockMetrics {
Expand All @@ -46,6 +46,7 @@ public class HoodieLockMetrics {
private transient Counter failedLockAttempts;
private transient Timer lockDuration;
private transient Timer lockApiRequestDuration;
private static final Object REGISTRY_LOCK = new Object();

public HoodieLockMetrics(HoodieWriteConfig writeConfig) {
this.isMetricsEnabled = writeConfig.isLockingMetricsEnabled();
Expand All @@ -69,10 +70,12 @@ private String getMetricsName(String metric) {

private Timer createTimerForMetrics(MetricRegistry registry, String metric) {
String metricName = getMetricsName(metric);
if (registry.getMetrics().get(metricName) == null) {
lockDuration = new Timer(new SlidingWindowReservoir(keepLastNtimes));
registry.register(metricName, lockDuration);
return lockDuration;
synchronized (REGISTRY_LOCK) {
if (registry.getMetrics().get(metricName) == null) {
lockDuration = new Timer(new SlidingWindowReservoir(keepLastNtimes));
registry.register(metricName, lockDuration);
return lockDuration;
}
}
return (Timer) registry.getMetrics().get(metricName);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,8 @@ public BaseCommitActionExecutor(HoodieEngineContext context, HoodieWriteConfig c
this.taskContextSupplier = context.getTaskContextSupplier();
// TODO : Remove this once we refactor and move out autoCommit method from here, since the TxnManager is held in {@link BaseHoodieWriteClient}.
this.txnManager = new TransactionManager(config, table.getMetaClient().getFs());
this.lastCompletedTxn = TransactionUtils.getLastCompletedTxnInstantAndMetadata(table.getMetaClient());
this.lastCompletedTxn = txnManager.isOptimisticConcurrencyControlEnabled()
? TransactionUtils.getLastCompletedTxnInstantAndMetadata(table.getMetaClient()) : Option.empty();
this.pendingInflightAndRequestedInstants = TransactionUtils.getInflightAndRequestedInstants(table.getMetaClient());
this.pendingInflightAndRequestedInstants.remove(instantTime);
if (!table.getStorageLayout().writeOperationSupported(operationType)) {
Expand Down
Loading

0 comments on commit 57a1cf5

Please sign in to comment.