Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Bump iceberg to 0.14 #4738

Merged
merged 7 commits into from
Jul 20, 2022
Merged

Bump iceberg to 0.14 #4738

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
4de01a9
(Most) changes to bump to Iceberg 0.14.0
snazy Mar 28, 2022
bf51fbd
Fix NessieViewCatalog, because `NessieUtil.CONFIG_CLIENT_BUILDER_IMPL…
snazy Jun 15, 2022
f7d03c1
Fix NessieExtCatalog after 'Nessie: Extract Catalog client code to Ne…
snazy Jun 15, 2022
027af93
Adopt iceberg-views
snazy Jul 14, 2022
ed147bd
Actual version bumps
snazy Jul 19, 2022
8953441
Unify Scala + Spark version
snazy Jul 19, 2022
868c883
Clarify
snazy Jul 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 13 additions & 12 deletions build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ val versionCel = "0.3.8"
val versionCheckstyle = "10.3.1"
// to fix circular dependencies with NessieClient, certain projects need to use the same Nessie
// version as Iceberg/Delta has
var versionClientNessie = "0.18.0"
var versionClientNessie = "0.30.0"
val versionDeltalake = "1.1.0-nessie"
val versionDockerjava = "3.2.13"
val versionErrorProneAnnotations = "2.14.0"
Expand All @@ -59,7 +59,7 @@ val versionGoogleJavaFormat = "1.15.0"
val versionGraalSvm = "22.1.0.1"
val versionH2 = "2.1.214"
val versionHadoop = "3.3.3"
var versionIceberg = "0.13.2"
var versionIceberg = "0.14.0"
val versionImmutables = "2.9.0"
val versionJackson = "2.13.3"
val versionJacksonSpark3 = "2.13.3"
Expand Down Expand Up @@ -91,11 +91,11 @@ val versionProtobuf = "3.21.2"
val versionReactor = "2020.0.21"
val versionRestAssured = "5.1.1"
val versionRocksDb = "7.3.1"
val versionScala2_12 = "2.12.13"
val versionScala2_13 = "2.13.8"
val versionScala_2_12 = "2.12.13"
val versionScala_2_13 = "2.13.8"
val versionSlf4j = "1.7.36"
val versionSpark31 = "3.1.2"
val versionSpark32 = "3.2.1"
val versionSpark_3_1 = "3.1.2"
val versionSpark_3_2 = "3.2.1"
val versionTestcontainers = "1.17.3"
val versionWeld = "3.1.8.Final"

Expand Down Expand Up @@ -129,13 +129,13 @@ extra["versionProtobuf"] = versionProtobuf

extra["versionRocksDb"] = versionRocksDb

extra["versionScala2_12"] = versionScala2_12
extra["versionScala_2.12"] = versionScala_2_12

extra["versionScala2_13"] = versionScala2_13
extra["versionScala_2.13"] = versionScala_2_13

extra["versionSpark31"] = versionSpark31
extra["versionSpark_3.1"] = versionSpark_3_1

extra["versionSpark32"] = versionSpark32
extra["versionSpark_3.2"] = versionSpark_3_2

extra["quarkus.builder-image"] = "quay.io/quarkus/ubi-quarkus-native-image:22.1-java17"

Expand Down Expand Up @@ -190,8 +190,9 @@ dependencies {
api("org.apache.iceberg:iceberg-nessie:$versionIceberg")
api("org.apache.iceberg:iceberg-parquet:$versionIceberg")
api("org.apache.iceberg:iceberg-spark-3.2_2.12:$versionIceberg")
api("org.apache.iceberg:iceberg-spark3:$versionIceberg")
api("org.apache.iceberg:iceberg-spark3-extensions:$versionIceberg")
api("org.apache.iceberg:iceberg-spark-3.1_2.12:$versionIceberg")
api("org.apache.iceberg:iceberg-spark-extensions-3.1_2.12:$versionIceberg")
api("org.apache.iceberg:iceberg-spark-extensions-3.2_2.12:$versionIceberg")
api("org.apache.maven:maven-resolver-provider:$versionMaven")
api("org.apache.maven.resolver:maven-resolver-connector-basic:$versionMavenResolver")
api("org.apache.maven.resolver:maven-resolver-transport-file:$versionMavenResolver")
Expand Down
18 changes: 11 additions & 7 deletions clients/deltalake/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,13 @@ plugins {
`nessie-conventions`
}

val scalaVersion = dependencyVersion("versionScala2_12")
val scalaMajorVersion = "2.12"

val sparkVersion = dependencyVersion("versionSpark32")
val scalaVersion = dependencyVersion("versionScala_$scalaMajorVersion")

val sparkMajorVersion = "3.2"

val sparkVersion = dependencyVersion("versionSpark_$sparkMajorVersion")

dependencies {
// picks the right dependencies for scala compilation
Expand All @@ -36,14 +40,14 @@ dependencies {
implementation(platform(nessieRootProject()))
implementation(nessieProject("nessie-model"))
implementation(nessieProject("nessie-client"))
implementation("org.apache.spark:spark-core_2.12") { forSpark(sparkVersion) }
implementation("org.apache.spark:spark-sql_2.12") { forSpark(sparkVersion) }
implementation("io.delta:delta-core_2.12")
implementation("org.apache.spark:spark-core_$scalaMajorVersion") { forSpark(sparkVersion) }
implementation("org.apache.spark:spark-sql_$scalaMajorVersion") { forSpark(sparkVersion) }
implementation("io.delta:delta-core_$scalaMajorVersion")
compileOnly("org.eclipse.microprofile.openapi:microprofile-openapi-api")

testImplementation(platform(nessieRootProject()))
testImplementation(nessieProject("nessie-spark-3.2-extensions"))
testImplementation("com.fasterxml.jackson.module:jackson-module-scala_2.12")
testImplementation(nessieProject("nessie-spark-${sparkMajorVersion}-extensions"))
testImplementation("com.fasterxml.jackson.module:jackson-module-scala_$scalaMajorVersion")
testImplementation("com.fasterxml.jackson.core:jackson-databind")
testImplementation("org.eclipse.microprofile.openapi:microprofile-openapi-api")
testImplementation("ch.qos.logback:logback-classic")
Expand Down
9 changes: 7 additions & 2 deletions clients/iceberg-views/src/main/java/org/apache/iceberg/nessie/NessieExtCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package org.apache.iceberg.nessie;

import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
Expand Down Expand Up @@ -104,11 +105,15 @@ public void setConf(Configuration conf) {
}

public void refresh() throws NessieNotFoundException {
tableCatalog.refresh();
// TODO This NessieExtCatalog is a *hack* to get some Iceberg view support into Nessie.
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the whole method can be just removed

// The refactoring of `NessieCatalog` (https://github.com/apache/iceberg/pull/4491)
// removed the explicit ability to refresh the catalog. Client code should actually NEVER
// call this function explicitly.
// tableCatalog.refresh();
viewCatalog.refresh();
}

public void close() {
public void close() throws IOException {
tableCatalog.close();
viewCatalog.close();
}
Expand Down
3 changes: 2 additions & 1 deletion clients/iceberg-views/src/main/java/org/apache/iceberg/nessie/NessieViewCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,8 @@ public void initialize(String inputName, Map<String, String> options) {
x -> x.replace(NessieUtil.NESSIE_CONFIG_PREFIX, "");

this.api =
createNessieClientBuilder(options.get(NessieUtil.CONFIG_CLIENT_BUILDER_IMPL))
createNessieClientBuilder(
options.get(NessieUtil.NESSIE_CONFIG_PREFIX + "client-builder-impl"))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: Should we use the existing NessieConfigConstants.CONF_NESSIE_CLIENT_BUILDER_IMPL?

.fromConfig(x -> options.get(removePrefix.apply(x)))
.build(NessieApiV1.class);

Expand Down
22 changes: 13 additions & 9 deletions clients/spark-3.2-extensions/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,13 @@ plugins {
`nessie-conventions`
}

val scalaVersion = dependencyVersion("versionScala2_12")
val scalaMajorVersion = "2.12"

val sparkVersion = dependencyVersion("versionSpark32")
val scalaVersion = dependencyVersion("versionScala_$scalaMajorVersion")

val sparkMajorVersion = "3.2"

val sparkVersion = dependencyVersion("versionSpark_$sparkMajorVersion")

dependencies {
// picks the right dependencies for scala compilation
Expand All @@ -36,21 +40,21 @@ dependencies {
implementation(platform(nessieRootProject()))
implementation(project(":nessie-spark-extensions-grammar"))
implementation(project(":nessie-spark-extensions-base"))
compileOnly("org.apache.spark:spark-sql_2.12") { forSpark(sparkVersion) }
compileOnly("org.apache.spark:spark-core_2.12") { forSpark(sparkVersion) }
compileOnly("org.apache.spark:spark-hive_2.12") { forSpark(sparkVersion) }
compileOnly("org.apache.spark:spark-sql_$scalaMajorVersion") { forSpark(sparkVersion) }
compileOnly("org.apache.spark:spark-core_$scalaMajorVersion") { forSpark(sparkVersion) }
compileOnly("org.apache.spark:spark-hive_$scalaMajorVersion") { forSpark(sparkVersion) }
implementation(nessieClientForIceberg())

testImplementation(platform(nessieRootProject()))
testImplementation(project(":nessie-spark-extensions-base")) { testJarCapability() }
testImplementation("org.apache.iceberg:iceberg-nessie")
testImplementation("org.apache.iceberg:iceberg-spark-3.2_2.12")
testImplementation("org.apache.iceberg:iceberg-spark-${sparkMajorVersion}_$scalaMajorVersion")
testImplementation("org.apache.iceberg:iceberg-hive-metastore")
testImplementation("ch.qos.logback:logback-classic")
testImplementation("org.slf4j:log4j-over-slf4j")
testImplementation("org.apache.spark:spark-sql_2.12") { forSpark(sparkVersion) }
testImplementation("org.apache.spark:spark-core_2.12") { forSpark(sparkVersion) }
testImplementation("org.apache.spark:spark-hive_2.12") { forSpark(sparkVersion) }
testImplementation("org.apache.spark:spark-sql_$scalaMajorVersion") { forSpark(sparkVersion) }
testImplementation("org.apache.spark:spark-core_$scalaMajorVersion") { forSpark(sparkVersion) }
testImplementation("org.apache.spark:spark-hive_$scalaMajorVersion") { forSpark(sparkVersion) }

testImplementation("org.assertj:assertj-core")
testImplementation(platform("org.junit:junit-bom"))
Expand Down
12 changes: 8 additions & 4 deletions clients/spark-extensions-base/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,23 +22,27 @@ plugins {
id("org.projectnessie.buildsupport.attach-test-jar")
}

val scalaVersion = dependencyVersion("versionScala2_12")
val scalaMajorVersion = "2.12"

val sparkVersion = dependencyVersion("versionSpark31")
val scalaVersion = dependencyVersion("versionScala_$scalaMajorVersion")

val sparkMajorVersion = "3.1"

val sparkVersion = dependencyVersion("versionSpark_$sparkMajorVersion")

dependencies {
// picks the right dependencies for scala compilation
forScala(scalaVersion)

implementation(platform(nessieRootProject()))
implementation(nessieProject("nessie-spark-extensions-grammar"))
compileOnly("org.apache.spark:spark-hive_2.12") { forSpark(sparkVersion) }
compileOnly("org.apache.spark:spark-hive_$scalaMajorVersion") { forSpark(sparkVersion) }
compileOnly("org.eclipse.microprofile.openapi:microprofile-openapi-api")
implementation(nessieClientForIceberg())

testImplementation(platform(nessieRootProject()))
testAnnotationProcessor(platform(nessieRootProject()))
testImplementation("org.apache.spark:spark-sql_2.12") { forSpark(sparkVersion) }
testImplementation("org.apache.spark:spark-sql_$scalaMajorVersion") { forSpark(sparkVersion) }
testImplementation("org.eclipse.microprofile.openapi:microprofile-openapi-api")

testCompileOnly("org.immutables:value-annotations")
Expand Down
5 changes: 1 addition & 4 deletions ...s-base/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BaseShowLogExec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,7 @@ abstract class BaseShowLogExec(
)
val stream = StreamingUtil.getCommitLogStream(
api,
refName,
null,
null,
null,
builder => builder.refName(refName),
OptionalInt.empty
)

Expand Down
24 changes: 13 additions & 11 deletions ...sions-base/src/main/scala/org/apache/spark/sql/execution/datasources/v2/NessieUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,10 +89,10 @@ object NessieUtils {
StreamingUtil
.getCommitLogStream(
api,
branch,
Validation.validateHash(requestedHash),
null,
null,
builder =>
builder
.refName(branch)
.hashOnRef(Validation.validateHash(requestedHash)),
OptionalInt.empty
)
.findFirst()
Expand Down Expand Up @@ -124,13 +124,15 @@ object NessieUtils {
StreamingUtil
.getCommitLogStream(
api,
branch,
null,
null,
String.format(
"timestamp(commit.commitTime) <= timestamp('%s')",
timestamp
),
builder =>
builder
.refName(branch)
.filter(
String.format(
"timestamp(commit.commitTime) <= timestamp('%s')",
timestamp
)
),
OptionalInt.empty
)
.findFirst()
Expand Down
5 changes: 2 additions & 3 deletions ...xtensions-base/src/test/java/org/projectnessie/spark/extensions/AbstractSparkSqlTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -505,7 +505,7 @@ void useShowReferencesAtWithFailureConditions()
String randomHash = "dd8d46a3dd5478ce69749a5455dba29d74f6d1171188f4c21d0e15ff4a0a9a9c";
String invalidTimestamp = "01-01-01";
String invalidBranch = "invalidBranch";
String invalidHash = "abcdef123";
String invalidHash = "abcdef1234";
assertThatThrownBy(() -> sql("USE REFERENCE %s AT %s IN nessie ", invalidBranch, defaultHash()))
.isInstanceOf(NessieNotFoundException.class)
.hasMessage(String.format("Named reference '%s' not found", invalidBranch));
Expand All @@ -524,8 +524,7 @@ void useShowReferencesAtWithFailureConditions()
assertThatThrownBy(() -> sql("USE REFERENCE %s AT %s IN nessie ", refName, invalidHash))
.isInstanceOf(NessieNotFoundException.class)
.hasMessageStartingWith(
String.format(
"Invalid timestamp provided: Text '%s' could not be parsed", invalidHash));
String.format("Could not find commit '%s' in reference '%s'", invalidHash, refName));
}

@Test
Expand Down
22 changes: 13 additions & 9 deletions clients/spark-extensions/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,13 @@ plugins {
`nessie-conventions`
}

val scalaVersion = dependencyVersion("versionScala2_12")
val scalaMajorVersion = "2.12"

val sparkVersion = dependencyVersion("versionSpark31")
val scalaVersion = dependencyVersion("versionScala_$scalaMajorVersion")

val sparkMajorVersion = "3.1"

val sparkVersion = dependencyVersion("versionSpark_$sparkMajorVersion")

dependencies {
// picks the right dependencies for scala compilation
Expand All @@ -36,21 +40,21 @@ dependencies {
implementation(platform(nessieRootProject()))
implementation(project(":nessie-spark-extensions-grammar"))
implementation(project(":nessie-spark-extensions-base"))
compileOnly("org.apache.spark:spark-sql_2.12") { forSpark(sparkVersion) }
compileOnly("org.apache.spark:spark-core_2.12") { forSpark(sparkVersion) }
compileOnly("org.apache.spark:spark-hive_2.12") { forSpark(sparkVersion) }
compileOnly("org.apache.spark:spark-sql_$scalaMajorVersion") { forSpark(sparkVersion) }
compileOnly("org.apache.spark:spark-core_$scalaMajorVersion") { forSpark(sparkVersion) }
compileOnly("org.apache.spark:spark-hive_$scalaMajorVersion") { forSpark(sparkVersion) }
implementation(nessieClientForIceberg())

testImplementation(platform(nessieRootProject()))
testImplementation(project(":nessie-spark-extensions-base")) { testJarCapability() }
testImplementation("org.apache.iceberg:iceberg-nessie")
testImplementation("org.apache.iceberg:iceberg-spark3")
testImplementation("org.apache.iceberg:iceberg-spark-${sparkMajorVersion}_$scalaMajorVersion")
testImplementation("org.apache.iceberg:iceberg-hive-metastore")
testImplementation("ch.qos.logback:logback-classic")
testImplementation("org.slf4j:log4j-over-slf4j")
testImplementation("org.apache.spark:spark-sql_2.12") { forSpark(sparkVersion) }
testImplementation("org.apache.spark:spark-core_2.12") { forSpark(sparkVersion) }
testImplementation("org.apache.spark:spark-hive_2.12") { forSpark(sparkVersion) }
testImplementation("org.apache.spark:spark-sql_$scalaMajorVersion") { forSpark(sparkVersion) }
testImplementation("org.apache.spark:spark-core_$scalaMajorVersion") { forSpark(sparkVersion) }
testImplementation("org.apache.spark:spark-hive_$scalaMajorVersion") { forSpark(sparkVersion) }

testImplementation("org.assertj:assertj-core")
testImplementation(platform("org.junit:junit-bom"))
Expand Down
20 changes: 13 additions & 7 deletions gc/gc-base/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,13 @@ plugins {

extra["maven.name"] = "Nessie - GC - Base Implementation"

val scalaVersion = dependencyVersion("versionScala2_12")
val scalaMajorVersion = "2.12"

val sparkVersion = dependencyVersion("versionSpark31")
val scalaVersion = dependencyVersion("versionScala_$scalaMajorVersion")

val sparkMajorVersion = "3.1"

val sparkVersion = dependencyVersion("versionSpark_$sparkMajorVersion")

dependencies {
implementation(platform(nessieRootProject()))
Expand All @@ -43,11 +47,11 @@ dependencies {
implementation("com.fasterxml.jackson.core:jackson-annotations")
implementation("com.google.code.findbugs:jsr305")

compileOnly("org.apache.spark:spark-sql_2.12") { forSpark(sparkVersion) }
compileOnly("org.apache.spark:spark-sql_$scalaMajorVersion") { forSpark(sparkVersion) }
compileOnly("org.apache.iceberg:iceberg-api")
compileOnly("org.apache.iceberg:iceberg-core")
compileOnly("org.apache.iceberg:iceberg-nessie") { exclude("org.projectnessie") }
compileOnly("org.apache.iceberg:iceberg-spark3")
compileOnly("org.apache.iceberg:iceberg-spark-${sparkMajorVersion}_$scalaMajorVersion")
compileOnly("org.apache.iceberg:iceberg-parquet")
compileOnly("org.apache.parquet:parquet-column")

Expand All @@ -56,7 +60,7 @@ dependencies {
testCompileOnly("org.eclipse.microprofile.openapi:microprofile-openapi-api")
testImplementation(nessieProject("nessie-jaxrs-testextension"))
testImplementation(nessieProject("nessie-jaxrs-tests"))
testImplementation("org.apache.spark:spark-sql_2.12") {
testImplementation("org.apache.spark:spark-sql_$scalaMajorVersion") {
forSpark(sparkVersion)
exclude("com.sun.jersey", "jersey-servlet")
}
Expand All @@ -69,8 +73,10 @@ dependencies {
testImplementation(project(":nessie-spark-extensions-base")) { testJarCapability() }
testImplementation(project(":nessie-spark-extensions"))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: Is it good idea to rename nessie-spark-extensions to nessie-spark-3.1-extensions (just to have a similar syntax as spark-3.2 module)? Now reader has to figure out which version of spark is nessie-spark-extensions

testImplementation("org.apache.iceberg:iceberg-nessie")
testImplementation("org.apache.iceberg:iceberg-spark3")
testImplementation("org.apache.iceberg:iceberg-spark3-extensions")
testImplementation("org.apache.iceberg:iceberg-spark-${sparkMajorVersion}_$scalaMajorVersion")
testImplementation(
"org.apache.iceberg:iceberg-spark-extensions-${sparkMajorVersion}_$scalaMajorVersion"
)
testImplementation("org.apache.iceberg:iceberg-hive-metastore")

testImplementation("org.assertj:assertj-core")
Expand Down
4 changes: 3 additions & 1 deletion perftest/gatling/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@ plugins {

extra["maven.name"] = "Nessie - Perf Test - Gatling"

val scalaVersion = dependencyVersion("versionScala2_13")
val scalaMajorVersion = "2.13"

val scalaVersion = dependencyVersion("versionScala_$scalaMajorVersion")

dependencies {
// picks the right dependencies for scala compilation
Expand Down
2 changes: 0 additions & 2 deletions perftest/simulations/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@ plugins {

extra["maven.name"] = "Nessie - Perf Test - Simulations"

val scalaVersion = dependencyVersion("versionScala2_13")

dependencies {
gatling(platform(rootProject))
gatling(project(":nessie-model"))
Expand Down