Merge pull request #48 from tiki/feat/sar

Feat: Publish to SAR

.github/workflows/codeql.yml

@@ -8,7 +8,9 @@ on:
     branches:
       - main
 
-# Cancel any active builds when new commits are pushed
+env:
+  UTILS_VERSION: 0.2.1
+
 concurrency:
   group: ${{ github.ref }}-${{ github.workflow }}
   cancel-in-progress: true
@@ -34,6 +36,17 @@ jobs:
           java-version: '20'
           distribution: 'zulu'
 
+      - name: Check out utils
+        uses: actions/checkout@v4
+        with:
+          repository: tiki/core-iceberg-utils
+          token: ${{ github.token }}
+          path: utils
+          ref: ${{ env.UTILS_VERSION }}
+
+      - name: Maven Install Utils
+        run: cd utils && mvn clean package install
+
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2
         with:

.github/workflows/release.yml

@@ -10,6 +10,7 @@ on:
 env:
   SESSION_NAME : "gh_action"
   AWS_REGION : "us-east-2"
+  UTILS_VERSION: 0.2.1
 
 concurrency:
   group: ${{ github.ref }}-${{ github.workflow }}
@@ -34,28 +35,39 @@ jobs:
           java-version: '20'
           distribution: 'zulu'
 
+      - name: Check out utils
+        uses: actions/checkout@v4
+        with:
+          repository: tiki/core-iceberg-utils
+          token: ${{ github.token }}
+          path: utils
+          ref: ${{ env.UTILS_VERSION }}
+
+      - name: Maven Install Utils
+        run: cd utils && mvn clean package install
+
       - name: Install xmlstarlet
         run: sudo apt-get update && sudo apt-get install -y xmlstarlet
 
       - name: AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v3
+        uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: arn:aws:iam::254962200554:role/GitHubDeployBase
           role-session-name: ${{ env.SESSION_NAME }}
           aws-region: ${{ env.AWS_REGION }}
 
       - name: SAM Deploy GitHub
-        run: cd infra/github && sam build && sam package --force-upload && sam deploy --force-upload
+        run: cd infra/github && sam build && sam package && sam deploy
 
       - name: AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v3
+        uses: aws-actions/configure-aws-credentials@v4
         with:
-          role-to-assume: arn:aws:iam::254962200554:role/GitHubDeployOceanMetadata
+          role-to-assume: arn:aws:iam::254962200554:role/GitHubDeployIcebergMetadata
           role-session-name: ${{ env.SESSION_NAME }}
           aws-region: ${{ env.AWS_REGION }}
 
-      - name: SAM Deploy AWS
-        run: cd infra/aws && sam build && sam package --force-upload && sam deploy --force-upload
+      - name: SAM Publish
+        run: cd infra/aws && sam build && sam package --output-template-file packaged.yaml && sam publish --template packaged.yaml
 
       - name: Get Version
         id: version

.github/workflows/tests.yml

@@ -6,6 +6,9 @@ on:
     branches:
       - main
 
+env:
+  UTILS_VERSION: 0.2.1
+
 # Cancel any active builds when new commits are pushed
 concurrency:
   group: ${{ github.ref }}-${{ github.workflow }}
@@ -28,6 +31,17 @@ jobs:
         with:
           use-installer: true
 
+      - name: Check out utils
+        uses: actions/checkout@v4
+        with:
+          repository: tiki/core-iceberg-utils
+          token: ${{ github.token }}
+          path: utils
+          ref: ${{ env.UTILS_VERSION }}
+
+      - name: Maven Install Utils
+        run: cd utils && mvn clean package install
+
       - name: Maven Package
         run: mvn clean package
 

.github/workflows/version.yml

@@ -5,7 +5,6 @@ on:
     branches:
       - main
 
-# Cancel any active builds when new commits are pushed
 concurrency:
   group: ${{ github.ref }}-${{ github.workflow }}
   cancel-in-progress: true
@@ -39,9 +38,13 @@ jobs:
           token: ${{ github.token }}
           branch: ${{ steps.getpr.outputs.branch }}
 
-      - name: Update Version
+      - name: Update POM Version
         run: xmlstarlet ed -L -u "_:project/_:version" -v "${{ steps.semver.outputs.nextStrict }}" pom.xml
 
+      - name: Update SAR Version
+        run: |
+          sed -i 's/SemanticVersion: .*/SemanticVersion: ${{ steps.semver.outputs.nextStrict }}/' infra/aws/template.yaml
+
       - name: Commit Changes
         continue-on-error: true
         env:

.gitignore

@@ -28,6 +28,8 @@ replay_pid*
 
 # AWS
 .aws-sam
+packaged.yaml
+packaged.yml
 
 # Generated Outputs
 target

README.md

@@ -0,0 +1 @@
+🐡

infra/aws/samconfig.toml

@@ -1,9 +1,9 @@
 version = 0.1
 
 [default.global.parameters]
-stack_name = "ocean-metadata-aws"
+stack_name = "core-iceberg-metadata-aws"
 s3_bucket = "mytiki-sam-deploy"
-s3_prefix = "ocean/metadata/aws"
+s3_prefix = "core/iceberg/metadata/aws"
 region = "us-east-2"
 
 [default.deploy.parameters]

infra/aws/template.yml

@@ -1,20 +1,86 @@
 AWSTemplateFormatVersion: '2010-09-09'
 Transform: AWS::Serverless-2016-10-31
-Description: Ocean - Metadata Infrastructure
+Description: Iceberg Metadata
+
+Parameters:
+  QueueName:
+    Type: String
+    Description: The name of the FIFO Queue.
+  BucketName:
+    Description: The S3 bucket name for the lake
+    Type: String
+  DatabaseName:
+    Description: The Glue database name
+    Type: String
+  UtilsName:
+    Type: String
+    Default: core-iceberg-utils
+    Description: Enter the name of the utils layer.
+  UtilsVersion:
+    Type: Number
+    Description: Enter the version number of the utils layer.
+
+Metadata:
+  AWS::ServerlessRepo::Application:
+    Name: core-iceberg-metadata
+    Description: Iceberg Metadata
+    Author: tiki
+    SpdxLicenseId: MIT
+    LicenseUrl: ../../LICENSE
+    ReadmeUrl: ../../README.md
+    Labels: ['iceberg']
+    HomePageUrl: https://github.com/tiki/core-iceberg-metadata
+    SemanticVersion: 0.1.2
+    SourceCodeUrl: https://github.com/tiki/core-iceberg-metadata
 
 Resources:
   Queue:
-    Type: AWS::Serverless::Application
+    Type: AWS::SQS::Queue
     Properties:
-      Location: queue.yml
-      Parameters:
-        QueueName: cleanroom-sample-catalog
+      QueueName: !Sub "${QueueName}.fifo"
+      ContentBasedDeduplication: true
+      DeduplicationScope: messageGroup
+      FifoQueue: true
 
   Function:
-    Type: AWS::Serverless::Application
+    Type: AWS::Serverless::Function
     Properties:
-      Location: function.yml
-      Parameters:
-        QueueArn: !GetAtt [Queue, Outputs.Arn]
-        BucketName: mytiki-cleanroom-sample
-        DatabaseName: cleanroom_sample
+      CodeUri: ../..
+      Handler: com.mytiki.ocean.metadata.App::handleRequest
+      Runtime: java17
+      Architectures:
+        - arm64
+      MemorySize: 512
+      Timeout: 20
+      Events:
+        SQSEvent:
+          Type: SQS
+          Properties:
+            Enabled: true
+            FunctionResponseTypes:
+              - ReportBatchItemFailures
+            BatchSize: 10
+            Queue: !GetAtt Queue.Arn
+      Policies:
+        - S3CrudPolicy:
+            BucketName: !Ref BucketName
+        - Version: "2012-10-17"
+          Statement:
+            - Effect: Allow
+              Action:
+                - glue:GetTable
+                - glue:GetTables
+                - glue:UpdateTable
+              Resource:
+                - !Sub "arn:aws:glue:${AWS::Region}:${AWS::AccountId}:catalog"
+                - !Sub "arn:aws:glue:${AWS::Region}:${AWS::AccountId}:database/${DatabaseName}"
+                - !Sub "arn:aws:glue:${AWS::Region}:${AWS::AccountId}:table/${DatabaseName}/*"
+      Layers:
+        - !Sub "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:layer:${UtilsName}:${UtilsVersion}"
+Outputs:
+  Function:
+    Description: Lambda Arn
+    Value: !GetAtt Function.Arn
+  Queue:
+    Description: SQS Arn
+    Value: !GetAtt Queue.Arn

infra/github/policy.yml

@@ -18,33 +18,5 @@ Resources:
         Statement:
           - Effect: Allow
             Action:
-              - iam:CreateRole
-              - iam:TagRole
-              - iam:GetRolePolicy
-              - iam:PutRolePolicy
-              - iam:AttachRolePolicy
-              - iam:GetRole
-            Resource: "*"
-          - Effect: Allow
-            Action:
-              - iam:PassRole
-            Resource: "*"
-            Condition:
-              StringEquals:
-                iam:PassedToService: lambda.amazonaws.com
-          - Effect: Allow
-            Action:
-              - lambda:CreateFunction
-              - lambda:TagResource
-              - lambda:GetFunction
-              - lambda:UpdateFunctionCode
-              - lambda:ListTags
-              - lambda:CreateEventSourceMapping
-              - lambda:GetEventSourceMapping
-            Resource: "*"
-          - Effect: Allow
-            Action:
-              - sqs:CreateQueue
-              - sqs:GetQueueAttributes
-              - sqs:SetQueueAttributes
+              - serverlessrepo:*
             Resource: "*"

infra/github/samconfig.toml

@@ -1,9 +1,9 @@
 version = 0.1
 
 [default.global.parameters]
-stack_name = "ocean-metadata-github"
+stack_name = "core-iceberg-metadata-github"
 s3_bucket = "mytiki-sam-deploy"
-s3_prefix = "ocean/metadata/github"
+s3_prefix = "core/iceberg/metadata/github"
 region = "us-east-2"
 
 [default.deploy.parameters]