Common: switch to Github Actions (close #189)

.github/deploy.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+project=$1
+tag=$2
+
+export RDB_LOADER_VERSION=$tag
+
+release-manager \
+    --config "./.github/release_${project}.yml" \
+    --check-version \
+    --make-version \
+    --make-artifact \
+    --upload-artifact

.travis/release_rdb_loader.yml → .github/release_rdb_loader.yml

@@ -1,11 +1,4 @@
 ---
-# --- Variables --- #
-
-local:
-  root_dir : <%= ENV['TRAVIS_BUILD_DIR'] %>
-
-# --- Release Manager Config --- #
-
 # Required: deployment targets
 targets:
   - type              : "awss3"
@@ -64,7 +57,7 @@ packages:
     version  : <%= FUNC['sbt_version(.)'] %>
 
     # Required IF '--check-version' is passed: will assert that both versions are the same
-    build_version : <%= ENV['TRAVIS_BUILD_RELEASE_TAG'] %>
+    build_version : <%= ENV['RDB_LOADER_VERSION'] %>
 
     # Optional: Build commands
     build_commands:
@@ -76,4 +69,4 @@ packages:
         prefix : "snowplow-rdb-loader-"
         suffix : ".jar"
         binary_paths:
-          - "target/scala-2.11/snowplow-rdb-loader-{{ packages.0.build_version }}.jar"
+          - "modules/loader/target/scala-2.11/snowplow-rdb-loader-{{ packages.0.build_version }}.jar"

.travis/release_rdb_shredder.yml → .github/release_rdb_shredder.yml

@@ -1,11 +1,4 @@
 ---
-# --- Variables --- #
-
-local:
-  root_dir : <%= ENV['TRAVIS_BUILD_DIR'] %>
-
-# --- Release Manager Config --- #
-
 # Required: deployment targets
 targets:
   - type              : "awss3"
@@ -61,10 +54,10 @@ packages:
     publish  : true
     override : false
     continue_on_conflict : false
-    version  : <%= CMD['sbt "project shredder" version -Dsbt.log.noformat=true | awk "END{print}" | sed -r "s/\[info\]\s(.*)/\1/" | xargs echo -n'] %>
+    version  : <%= FUNC['sbt_version(.)'] %>
 
     # Required IF '--check-version' is passed: will assert that both versions are the same
-    build_version : <%= ENV['TRAVIS_BUILD_RELEASE_TAG'] %>
+    build_version : <%= ENV['RDB_LOADER_VERSION'] %>
 
     # Optional: Build commands
     build_commands:
@@ -76,4 +69,4 @@ packages:
         prefix : "snowplow-rdb-shredder-"
         suffix : ".jar"
         binary_paths:
-          - "shredder/target/scala-2.11/snowplow-rdb-shredder-{{ packages.0.build_version }}.jar"
+          - "modules/shredder/target/scala-2.11/snowplow-rdb-shredder-{{ packages.0.build_version }}.jar"

.travis/start_iglu.sh → .github/start_iglu.sh

@@ -4,13 +4,13 @@ set -e
 
 IGLUCTL_ZIP="igluctl_0.6.0.zip"
 IGLUCTL_URI="http://dl.bintray.com/snowplow/snowplow-generic/$IGLUCTL_ZIP"
-SCHEMAS_PATH="$TRAVIS_BUILD_DIR/iglu-central/schemas/"
+SCHEMAS_PATH="$pwd/iglu-central/schemas/"
 
 git clone https://github.com/snowplow/iglu-central.git
 
 docker run \
     -p 8080:8080 \
-    -v $TRAVIS_BUILD_DIR/.travis:/iglu \
+    -v $pwd/.github:/iglu \
     --rm -d \
     snowplow-docker-registry.bintray.io/snowplow/iglu-server:0.6.0-rc16 \
     --config /iglu/server.conf

.github/workflows/test.yml

@@ -0,0 +1,37 @@
+name: Test
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up JDK 1.8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.8
+    - name: Run tests
+      run: sbt test
+
+  deploy:
+    needs: test
+    if: startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up JDK 1.8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.8
+    - name: Install release manager
+      run: pip install --user release-manager==0.4.1
+    - name: Compare SBT version with git tag
+      run: .github/check_tag.sh ${GITHUB_REF##*/}
+    - name: Build and publish Shredder
+      run: ./github/deploy.sh rdb_shredder ${GITHUB_REF##*/}
+    - name: Build and publish Loader
+      run: ./github/deploy.sh rdb_loader ${GITHUB_REF##*/}
+
+

project/BuildSettings.scala

@@ -67,6 +67,7 @@ object BuildSettings {
 
   lazy val assemblySettings = Seq(
     jarName,
+    test in assembly := {},
 
     assembly / assemblyShadeRules := Seq(
       ShadeRule.rename(

shredder/src/main/scala/com.snowplowanalytics.snowplow.storage/spark/singleton.scala

@@ -22,8 +22,10 @@ import cats.syntax.option._
 
 import io.circe.Json
 
+import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials}
 import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder
+
 import com.snowplowanalytics.iglu.client.Client
 import com.snowplowanalytics.snowplow.eventsmanifest.{DynamoDbManifest, EventsManifest, EventsManifestConfig}
 
@@ -81,6 +83,7 @@ object singleton {
                 val client = AmazonDynamoDBClientBuilder
                   .standard()
                   .withEndpointConfiguration(new EndpointConfiguration("http://localhost:8000", region))
+                  .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("local", "local")))
                   .build()
                 Some(new DynamoDbManifest(client, table))
               case Some(config) => EventsManifest.initStorage(config).fold(e => throw ShredJob.FatalEtlError(e.toString), _.some)

shredder/src/test/scala/com.snowplowanalytics.snowplow.storage.spark/ShredJobSpec.scala

@@ -255,6 +255,16 @@ object ShredJobSpec {
          |"uri": "http://iglucentral.com"
          |}
          |}
+         |},
+         |{
+         |"name": "Iglu Central Mirror",
+         |"priority": 0,
+         |"vendorPrefixes": [ "com.snowplowanalytics" ],
+         |"connection": {
+         |"http": {
+         |"uri": "https://com-iglucentral-eu1-prod.iglu.snplow.net/api"
+         |}
+         |}
          |}
          |]
          |}

shredder/src/test/scala/com.snowplowanalytics.snowplow.storage.spark/good/CrossBatchDeduplicationSpec.scala

@@ -16,24 +16,22 @@ package good
 import java.time.Instant
 import java.util.UUID
 
+import scala.collection.JavaConverters._
+
 import cats.syntax.either._
-import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
-import com.snowplowanalytics.snowplow.eventsmanifest.DynamoDbManifest
 
-import scala.collection.JavaConverters._
+import com.snowplowanalytics.snowplow.eventsmanifest.{ EventsManifestConfig, EventsManifest, DynamoDbManifest }
 
 // AWS SDK
+import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials}
+import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder
 import com.amazonaws.services.dynamodbv2.document.Table
 import com.amazonaws.services.dynamodbv2.model.{ResourceNotFoundException, ScanRequest}
 
 // Specs2
 import org.specs2.mutable.Specification
 
-import com.snowplowanalytics.snowplow.eventsmanifest.{ EventsManifestConfig, EventsManifest, DynamoDbManifest => DynamoDbEventsManifest }
-
-import com.snowplowanalytics.snowplow.eventsmanifest.EventsManifestConfig.DynamoDb.Credentials
-
 object CrossBatchDeduplicationSpec {
   import ShredJobSpec._
 
@@ -143,6 +141,7 @@ object CrossBatchDeduplicationSpec {
     private val client = AmazonDynamoDBClientBuilder
       .standard()
       .withEndpointConfiguration(new EndpointConfiguration("http://localhost:8000", dynamodbDuplicateStorageRegion))
+      .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("local", "local")))
       .build()
 
     /** Helper container class to hold components stored in DuplicationStorage */
@@ -185,23 +184,21 @@ object CrossBatchDeduplicationSpec {
             println("The table didn't exist, skipping")
         }
 
-        val config = EventsManifestConfig.DynamoDb(
+        EventsManifestConfig.DynamoDb(
+          None,
+          name = "local",
           None,
-          name = "Duplicate Storage Integration Test",
-          auth = Some(Credentials("fake", "fake")),
           awsRegion = dynamodbDuplicateStorageRegion,
           dynamodbTable = dynamodbDuplicateStorageTable
         )
-
-        config
       }
 
       val config = build()
 
       EventsManifest.initStorage(config) match {
         case Right(t) => t.asRight[Throwable]
         case Left(_) =>
-          DynamoDbEventsManifest.createTable(client, config.dynamodbTable, None, None)
+          DynamoDbManifest.createTable(client, config.dynamodbTable, None, None)
           println("Table has been created")
           val table = DynamoDbManifest.checkTable(client, dynamodbDuplicateStorageTable)
           new DynamoDbManifest(client, table).asRight