diff --git a/.github/deploy.sh b/.github/deploy.sh new file mode 100755 index 000000000..421fb8e4b --- /dev/null +++ b/.github/deploy.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +project=$1 +tag=$2 + +export RDB_LOADER_VERSION=$tag + +release-manager \ + --config "./.github/release_${project}.yml" \ + --check-version \ + --make-version \ + --make-artifact \ + --upload-artifact diff --git a/.travis/is_release_tag.sh b/.github/is_release_tag.sh similarity index 100% rename from .travis/is_release_tag.sh rename to .github/is_release_tag.sh diff --git a/.travis/release_rdb_loader.yml b/.github/release_rdb_loader.yml similarity index 90% rename from .travis/release_rdb_loader.yml rename to .github/release_rdb_loader.yml index 68ff48d3f..662e02bfe 100644 --- a/.travis/release_rdb_loader.yml +++ b/.github/release_rdb_loader.yml @@ -1,11 +1,4 @@ --- -# --- Variables --- # - -local: - root_dir : <%= ENV['TRAVIS_BUILD_DIR'] %> - -# --- Release Manager Config --- # - # Required: deployment targets targets: - type : "awss3" @@ -64,7 +57,7 @@ packages: version : <%= FUNC['sbt_version(.)'] %> # Required IF '--check-version' is passed: will assert that both versions are the same - build_version : <%= ENV['TRAVIS_BUILD_RELEASE_TAG'] %> + build_version : <%= ENV['RDB_LOADER_VERSION'] %> # Optional: Build commands build_commands: @@ -76,4 +69,4 @@ packages: prefix : "snowplow-rdb-loader-" suffix : ".jar" binary_paths: - - "target/scala-2.11/snowplow-rdb-loader-{{ packages.0.build_version }}.jar" + - "modules/loader/target/scala-2.11/snowplow-rdb-loader-{{ packages.0.build_version }}.jar" diff --git a/.travis/release_rdb_shredder.yml b/.github/release_rdb_shredder.yml similarity index 85% rename from .travis/release_rdb_shredder.yml rename to .github/release_rdb_shredder.yml index 65e5bc048..d535af2f1 100644 --- a/.travis/release_rdb_shredder.yml +++ b/.github/release_rdb_shredder.yml @@ -1,11 +1,4 @@ --- -# --- Variables --- # - -local: - root_dir : <%= ENV['TRAVIS_BUILD_DIR'] %> - -# --- Release Manager Config --- # - # Required: deployment targets targets: - type : "awss3" @@ -61,10 +54,10 @@ packages: publish : true override : false continue_on_conflict : false - version : <%= CMD['sbt "project shredder" version -Dsbt.log.noformat=true | awk "END{print}" | sed -r "s/\[info\]\s(.*)/\1/" | xargs echo -n'] %> + version : <%= FUNC['sbt_version(.)'] %> # Required IF '--check-version' is passed: will assert that both versions are the same - build_version : <%= ENV['TRAVIS_BUILD_RELEASE_TAG'] %> + build_version : <%= ENV['RDB_LOADER_VERSION'] %> # Optional: Build commands build_commands: @@ -76,4 +69,4 @@ packages: prefix : "snowplow-rdb-shredder-" suffix : ".jar" binary_paths: - - "shredder/target/scala-2.11/snowplow-rdb-shredder-{{ packages.0.build_version }}.jar" + - "modules/shredder/target/scala-2.11/snowplow-rdb-shredder-{{ packages.0.build_version }}.jar" diff --git a/.travis/server.conf b/.github/server.conf similarity index 100% rename from .travis/server.conf rename to .github/server.conf diff --git a/.travis/start_iglu.sh b/.github/start_iglu.sh similarity index 85% rename from .travis/start_iglu.sh rename to .github/start_iglu.sh index ff21b5512..18c6d3255 100755 --- a/.travis/start_iglu.sh +++ b/.github/start_iglu.sh @@ -4,13 +4,13 @@ set -e IGLUCTL_ZIP="igluctl_0.6.0.zip" IGLUCTL_URI="http://dl.bintray.com/snowplow/snowplow-generic/$IGLUCTL_ZIP" -SCHEMAS_PATH="$TRAVIS_BUILD_DIR/iglu-central/schemas/" +SCHEMAS_PATH="$pwd/iglu-central/schemas/" git clone https://github.com/snowplow/iglu-central.git docker run \ -p 8080:8080 \ - -v $TRAVIS_BUILD_DIR/.travis:/iglu \ + -v $pwd/.github:/iglu \ --rm -d \ snowplow-docker-registry.bintray.io/snowplow/iglu-server:0.6.0-rc16 \ --config /iglu/server.conf diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..bb7016c51 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,37 @@ +name: Test + +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Run tests + run: sbt test + + deploy: + needs: test + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Install release manager + run: pip install --user release-manager==0.4.1 + - name: Compare SBT version with git tag + run: .github/check_tag.sh ${GITHUB_REF##*/} + - name: Build and publish Shredder + run: ./github/deploy.sh rdb_shredder ${GITHUB_REF##*/} + - name: Build and publish Loader + run: ./github/deploy.sh rdb_loader ${GITHUB_REF##*/} + + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f7bc45e5e..000000000 --- a/.travis.yml +++ /dev/null @@ -1,28 +0,0 @@ -language: scala -scala: -- 2.11.12 -jdk: -- openjdk8 -before_deploy: - - sudo python -m easy_install --upgrade pyOpenSSL # https://stackoverflow.com/questions/43267157/python-attributeerror-module-object-has-no-attribute-ssl-st-init - - pip install --user release-manager==0.4.1 -before_script: ./.travis/start_iglu.sh -env: - global: - - secure: my2pgROi/Xut3sVM47URmR+A/e29Z+pW3dtcHfqJHn3c6CP1NaSSKM+6e3PdM7BYNl9HgT0zUYfnw4lstIi9nCP0vumcA4zbx+zOd9brV10/jhpO/RtMsSQB1ycpGffJP5z+sc6AzK11ECkOoXlGHZNx48Lf9SaDTL79tMn5MQOiJphy+4OKYBiyZEdiPX6C45uvItG9zMN6RhlVpryHzRwdeiqykdFUCd/SeKAiXnAnNFPUyEYAlKz9P3setkZKVzp/yKBNVK41W1myePOWB0Is0MSisOapf1MD/k+tXovH9Ilo+K+HZId5hMpGQMo5tCvb2N2xr7j+VsNwcW3kjgROH/0QfJD9l5pB7P67iKv0JPrOjGpI2DgO7o4/YspAYUferWEeg3JboF+eytSJgXkDF6y4Ll5xQEhEXF5VDZfKamg2MOQBnRa13egn9oom5KcQ/7UrxyirRUFQI04fp3M8hGuYjxIy98t3/7Zw448ayj+oJd0miz449/3iz/YZfuaqlTFHxc22DL9BG9hX7ygOitDDoacGEeK0Z5iL7samU8GI+V5aJP23SaV+t1gvutCEU4NwRGa+ZTPmEt4RInEko0kF69gtGEHq026vjvBGW8uKFcyZcrq1ZJTcRHo3oj2L3Ub1KKuyTxs78Jd3dAI59DSG5ZSkACcuVt5gQdI= - - secure: Q/H5Mcb1PxmpyJ8wgCWgnof098r3OA9iVqQiySb+TQSnEjv1FKfvzMU1X3HK2TePwG5SPI4QoXygsXnDuo2lV4cPtofwULDvtNoqd8ii2ojFHbp6nvSrD7fxSVq0Vr20FfavrWzqN9/dB8m7/vyuLl2DGTsSFNMrB40Gt1OfoYFvnah3RvyhFYw0WNluPp3n4+l1zd4I64WJKU60zP2mvzRaNzXPfvXf9igGbC9vtIxaaVb9jRr7DtQrk3wqxdjY3t5Zr6ZDYtqFeUkOCskIb0ghL4Idegm7D8MMKemz8N+ZMZPQgJkGMZQj4gxBwFH0YtROnx1y4O8c21CwKA8LVg1H2n7nrwEHyjaZIwCbZbT8eoYHyyvTeXhhMFTsk/GCksMWvV4K9SuvSsDz33yDPsopdEdwfGJ6F4wqhb5MezjSlktaGeAEGsp04H0AXjprTumCboodYAgLz3CZuv0OtMyxHyH+V2q2hLnhJiWC5ue0WsWLx1WzfzIhHIxUvVqtY4fHyZW3OBycvbvDm/Auxj8xkWAXzBP315vJSyFLHkde3VThPnQC17zdN4JBcyeGXVNd7ZbIvoEQfjKOUmaW1plFIZtQPyqfx6vXWak/6ICI6hX+lAgSBjMHkvv/4lGrbgQOrM+VzkfRBkVb3Ybi+IDBkE5wodAntB5KQG9GsSA= - - secure: MJp/yUl9TViL4mCMu5v2gNuxvrfG5/atcOMjNchtvs3fzY9RNEMp0d0Qj2HVzSdTY0HizWlprc9vOWtiMeoDnGaMNbPQSxNoujnTq+xK4U6HycFuD1iGGTr/Zf5NfGAmQP9ty9CgHZJkXKrKHIlw2fJV18vq82fykiOkb+lFXhrSd373gO6tmG9AQsfd/+DTTpGVE9yblfFyqa8Zed59Qrr9ZC2/qlJwouFKDjIOc/iU0rwFR+MaxiaezLIOyPNT/K/zXyoiNQwr3uMv3EZqvO1yv/ymW3y6sk1IqyWzgP9fjFllHdhGOPLoUMSx898NihaCeXUsMZeFEFQ95sgevPiRpOJ47qd1Qimx8+o8Q0kTqU9evY9wo+Rsd8Odqx1+MIPabk1xzDB4BvDMqsHQfRx0CJT82sfW+fXxoHNTFU7I7jJBBL5T7GdntXSMxFZuYL34TFTiEkUBz5e/k6oY5c987iU4EX5Jbgq9sWbvJ6UTLqemMBMIR4TMaYlWNpXp5bHnsaBQqlS1VzkF/PWnx8edVVvfk7CW27b9asnEvBYeM62XibSihGbRqydv8QU3OFjH3mq4bZAOc/ePlnky3OOCiXKhh2PJusQpZa6PrRsDE+/4q7jkebLiC16S9Dah1FZeqzGJDXMabH4WGG4BdDflgrYueMKA8HnH2rtLvMA= - - secure: sP7R6sZqekVEeAt/NfgTO9kLZ+tpWrVCRWJJkulh2cW6flPkGqnSH+5HHa8dj00ZNojl7NZqyIwnmBOCfkYS+ncIpXNvHah2DLEuwZqxXtZhLU2mZg6DKGBaCfArhtXRf6Y8FQ+e9unZi9A5yqT7HScaABeWglYCJIIZ6o0tFL9MB0Zx6b1bHDAgrsvFCqUeyWvt7lEQ8fHi/pgHaVuXAmyO2bYKgY2Vwb6bILUTtFMSvknfIXPzdUNQeOalGXUjHGtsSvyuKZbR8kOhnAn9fjgNtxIyUT+u1BUY0HfuYH8653o8XzQMdifja9C0LPAoJ+eaCBQOXV/ZxoXr1/deNPZfLsaIXtmmzXpHbniWDYuM69zvrZM0y+VsuhIOCKVtllvyvd3EGNwjIP76HYTYxCeGh0p3R7Vde2BgsBBf45VkGTrNKA7sOyZ8JilaGnlEfB7ujyZ6L1QXd1ihRryuoZ8iXzngxobKmqC9m+uCyeC6+cvU2jeNVTkYfoVaRW3zw6RgSDLG5mJfZPREwOBCRQXms6ifzgj7HFw1W5FdlHrM+EyvTznnXWk1OGFeIVXezqtCIq8d9TpRQbG1ezlK0y/TYasidUrJUo2tri4LFTkkGyqUSQxNU+Tkjm18nUUjlphPLMANO6uuD1E6mouG7nQOo/ipR/46jUCPCgcrl20= -deploy: - - provider: script - script: ./.travis/deploy.sh rdb_loader $TRAVIS_TAG - skip_cleanup: true - on: - tags: true - condition: '"$(.travis/is_release_tag.sh rdb_loader $TRAVIS_TAG)" == "" && $? == 0' - - provider: script - script: ./.travis/deploy.sh rdb_shredder $TRAVIS_TAG - skip_cleanup: true - on: - tags: true - condition: '"$(.travis/is_release_tag.sh rdb_shredder $TRAVIS_TAG)" == "" && $? == 0' diff --git a/.travis/deploy.sh b/.travis/deploy.sh deleted file mode 100755 index 887b8b453..000000000 --- a/.travis/deploy.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -set -e - -project=$1 -tag=$2 - -slashed="${project}/" -slashed_len=${#slashed} - -cicd=${tag:0:${slashed_len}} -release=${tag:${slashed_len}} - -if [ "${cicd}" == "${slashed}" ]; then - if [ "${release}" == "" ]; then - echo "WARNING! No release specified! Ignoring." - exit 2 - fi -else - echo "This can't be deployed - there's no ${project} tag! (Is the travis condition set?)" - exit 1 -fi - -cd "${TRAVIS_BUILD_DIR}" - -export TRAVIS_BUILD_RELEASE_TAG="${release}" -release-manager \ - --config "./.travis/release_${project}.yml" \ - --check-version \ - --make-version \ - --make-artifact \ - --upload-artifact diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala index 86e3636f0..8eb85c6f2 100644 --- a/project/BuildSettings.scala +++ b/project/BuildSettings.scala @@ -67,6 +67,7 @@ object BuildSettings { lazy val assemblySettings = Seq( jarName, + test in assembly := {}, assembly / assemblyShadeRules := Seq( ShadeRule.rename( diff --git a/shredder/src/main/scala/com.snowplowanalytics.snowplow.storage/spark/singleton.scala b/shredder/src/main/scala/com.snowplowanalytics.snowplow.storage/spark/singleton.scala index 6135352a2..6b6681a4e 100644 --- a/shredder/src/main/scala/com.snowplowanalytics.snowplow.storage/spark/singleton.scala +++ b/shredder/src/main/scala/com.snowplowanalytics.snowplow.storage/spark/singleton.scala @@ -22,8 +22,10 @@ import cats.syntax.option._ import io.circe.Json +import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials} import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder + import com.snowplowanalytics.iglu.client.Client import com.snowplowanalytics.snowplow.eventsmanifest.{DynamoDbManifest, EventsManifest, EventsManifestConfig} @@ -81,6 +83,7 @@ object singleton { val client = AmazonDynamoDBClientBuilder .standard() .withEndpointConfiguration(new EndpointConfiguration("http://localhost:8000", region)) + .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("local", "local"))) .build() Some(new DynamoDbManifest(client, table)) case Some(config) => EventsManifest.initStorage(config).fold(e => throw ShredJob.FatalEtlError(e.toString), _.some) diff --git a/shredder/src/test/scala/com.snowplowanalytics.snowplow.storage.spark/ShredJobSpec.scala b/shredder/src/test/scala/com.snowplowanalytics.snowplow.storage.spark/ShredJobSpec.scala index c4c2c592f..ebbb6b576 100644 --- a/shredder/src/test/scala/com.snowplowanalytics.snowplow.storage.spark/ShredJobSpec.scala +++ b/shredder/src/test/scala/com.snowplowanalytics.snowplow.storage.spark/ShredJobSpec.scala @@ -255,6 +255,16 @@ object ShredJobSpec { |"uri": "http://iglucentral.com" |} |} + |}, + |{ + |"name": "Iglu Central Mirror", + |"priority": 0, + |"vendorPrefixes": [ "com.snowplowanalytics" ], + |"connection": { + |"http": { + |"uri": "https://com-iglucentral-eu1-prod.iglu.snplow.net/api" + |} + |} |} |] |} diff --git a/shredder/src/test/scala/com.snowplowanalytics.snowplow.storage.spark/good/CrossBatchDeduplicationSpec.scala b/shredder/src/test/scala/com.snowplowanalytics.snowplow.storage.spark/good/CrossBatchDeduplicationSpec.scala index 8f7dbd295..25744811d 100644 --- a/shredder/src/test/scala/com.snowplowanalytics.snowplow.storage.spark/good/CrossBatchDeduplicationSpec.scala +++ b/shredder/src/test/scala/com.snowplowanalytics.snowplow.storage.spark/good/CrossBatchDeduplicationSpec.scala @@ -16,13 +16,15 @@ package good import java.time.Instant import java.util.UUID +import scala.collection.JavaConverters._ + import cats.syntax.either._ -import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration -import com.snowplowanalytics.snowplow.eventsmanifest.DynamoDbManifest -import scala.collection.JavaConverters._ +import com.snowplowanalytics.snowplow.eventsmanifest.{ EventsManifestConfig, EventsManifest, DynamoDbManifest } // AWS SDK +import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials} +import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder import com.amazonaws.services.dynamodbv2.document.Table import com.amazonaws.services.dynamodbv2.model.{ResourceNotFoundException, ScanRequest} @@ -30,10 +32,6 @@ import com.amazonaws.services.dynamodbv2.model.{ResourceNotFoundException, ScanR // Specs2 import org.specs2.mutable.Specification -import com.snowplowanalytics.snowplow.eventsmanifest.{ EventsManifestConfig, EventsManifest, DynamoDbManifest => DynamoDbEventsManifest } - -import com.snowplowanalytics.snowplow.eventsmanifest.EventsManifestConfig.DynamoDb.Credentials - object CrossBatchDeduplicationSpec { import ShredJobSpec._ @@ -143,6 +141,7 @@ object CrossBatchDeduplicationSpec { private val client = AmazonDynamoDBClientBuilder .standard() .withEndpointConfiguration(new EndpointConfiguration("http://localhost:8000", dynamodbDuplicateStorageRegion)) + .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("local", "local"))) .build() /** Helper container class to hold components stored in DuplicationStorage */ @@ -185,15 +184,13 @@ object CrossBatchDeduplicationSpec { println("The table didn't exist, skipping") } - val config = EventsManifestConfig.DynamoDb( + EventsManifestConfig.DynamoDb( + None, + name = "local", None, - name = "Duplicate Storage Integration Test", - auth = Some(Credentials("fake", "fake")), awsRegion = dynamodbDuplicateStorageRegion, dynamodbTable = dynamodbDuplicateStorageTable ) - - config } val config = build() @@ -201,7 +198,7 @@ object CrossBatchDeduplicationSpec { EventsManifest.initStorage(config) match { case Right(t) => t.asRight[Throwable] case Left(_) => - DynamoDbEventsManifest.createTable(client, config.dynamodbTable, None, None) + DynamoDbManifest.createTable(client, config.dynamodbTable, None, None) println("Table has been created") val table = DynamoDbManifest.checkTable(client, dynamodbDuplicateStorageTable) new DynamoDbManifest(client, table).asRight