Merge pull request #75 from tiki/release/lake-formation

Release/lake formation
tiki-archive · Dec 13, 2023 · 384ac70 · 384ac70
2 parents 3fabf73 + 75cd13f
commit 384ac70
Show file tree

Hide file tree

Showing 10 changed files with 105 additions and 56 deletions.
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -28,15 +28,15 @@ jobs:
           fetch-depth: 1
 
       - name: Set up JDK
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '20'
           distribution: 'zulu'
 
       - name: Check out utils
         uses: actions/checkout@v4
         with:
-          repository: tiki/core-iceberg-utils
+          repository: tiki/core-ocean-iceberg-utils
           token: ${{ github.token }}
           path: utils
           ref: ${{ vars.UTILS_VERSION }}

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -27,21 +27,23 @@ jobs:
           sed -i 's/${AWS::AccountId}/${{ secrets.AWS_ACCOUNT_ID }}/' infra/role/template.yml
 
       - name: Set up JDK
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '20'
           distribution: 'zulu'
 
       - name: Check out utils
         uses: actions/checkout@v4
         with:
-          repository: tiki/core-iceberg-utils
+          repository: tiki/core-ocean-iceberg-utils
           token: ${{ github.token }}
           path: utils
           ref: ${{ vars.UTILS_VERSION }}
 
       - name: Maven Install Utils
-        run: cd utils && mvn clean package install
+        run: | 
+          cd utils
+          mvn clean package install
 
       - name: Install xmlstarlet
         run: sudo apt-get update && sudo apt-get install -y xmlstarlet
@@ -54,17 +56,25 @@ jobs:
           aws-region: ${{ vars.AWS_REGION }}
 
       - name: SAM Deploy GitHub
-        run: cd infra/role && sam build && sam package && sam deploy
+        run: |
+          cd infra/role
+          sam build
+          sam package
+          sam deploy
 
       - name: AWS Credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
-          role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/GitHubDeployCoreIcebergMetadata
+          role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/GitHubDeployCoreOceanMetadata
           role-session-name: ${{ vars.SESSION_NAME }}
           aws-region: ${{ vars.AWS_REGION }}
 
-      - name: SAM Publish
-        run: cd infra/sar && sam build && sam package --output-template-file packaged.yaml && sam publish --template packaged.yaml
+      - name: SAM Deploy Function
+        run: |
+          cd infra/function
+          sam build
+          sam package
+          sam deploy
 
       - name: Get Version
         id: version

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -18,7 +18,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up JDK
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '20'
           distribution: 'zulu'
@@ -30,7 +30,7 @@ jobs:
       - name: Check out utils
         uses: actions/checkout@v4
         with:
-          repository: tiki/core-iceberg-utils
+          repository: tiki/core-ocean-iceberg-utils
           token: ${{ github.token }}
           path: utils
           ref: ${{ vars.UTILS_VERSION }}
@@ -49,7 +49,11 @@ jobs:
           path: target/surefire-reports/**/*.xml
 
       - name: Validate AWS Template
-        run: cd infra/sar && sam validate
+        run: |
+          cd infra/function
+          sam validate
 
       - name: Build AWS Template
-        run: cd infra/sar && sam build
+        run: |
+          cd infra/function
+          sam build
diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml
@@ -41,10 +41,6 @@ jobs:
       - name: Update POM Version
         run: xmlstarlet ed -L -u "_:project/_:version" -v "${{ steps.semver.outputs.nextStrict }}" pom.xml
 
-      - name: Update SAR Version
-        run: |
-          sed -i 's/SemanticVersion: .*/SemanticVersion: ${{ steps.semver.outputs.nextStrict }}/' infra/sar/template.yml
-
       - name: Commit Changes
         continue-on-error: true
         env:
@@ -53,7 +49,6 @@ jobs:
           git config --global user.email "[email protected]"
           git config --global user.name "GH Action"
           git add pom.xml
-          git add infra/sar/template.yml
           git commit -m 'version bump'
           git push
 

diff --git a/README.md b/README.md
diff --git a/infra/sar/samconfig.toml → infra/function/samconfig.toml b/infra/sar/samconfig.toml → infra/function/samconfig.toml
@@ -1,9 +1,9 @@
 version = 0.1
 
 [default.global.parameters]
-stack_name = "core-iceberg-metadata-sar"
+stack_name = "core-ocean-metadata-function"
 s3_bucket = "mytiki-sam-deploy"
-s3_prefix = "core/iceberg/metadata/sar"
+s3_prefix = "core/ocean/metadata/function"
 region = "us-east-2"
 
 [default.deploy.parameters]

diff --git a/infra/sar/template.yml → infra/function/template.yml b/infra/sar/template.yml → infra/function/template.yml
@@ -1,37 +1,28 @@
 AWSTemplateFormatVersion: '2010-09-09'
 Transform: AWS::Serverless-2016-10-31
-Description: Core Iceberg Metadata SAR
+Description: Core Ocean Iceberg Metadata Function
 
 Parameters:
   QueueName:
     Type: String
     Description: The name of the FIFO Queue.
+    Default: ocean-metadata
   BucketName:
     Description: The S3 bucket name for the lake
     Type: String
+    Default: mytiki-ocean
   DatabaseName:
     Description: The Glue database name
     Type: String
+    Default: ocean
   UtilsName:
     Type: String
-    Default: core-iceberg-utils
+    Default: core-ocean-iceberg-utils
     Description: Enter the name of the utils layer.
   UtilsVersion:
     Type: Number
     Description: Enter the version number of the utils layer.
-
-Metadata:
-  AWS::ServerlessRepo::Application:
-    Name: core-iceberg-metadata
-    Description: Iceberg Metadata
-    Author: tiki
-    SpdxLicenseId: MIT
-    LicenseUrl: ../../LICENSE
-    ReadmeUrl: ../../README.md
-    Labels: [ 'iceberg' ]
-    HomePageUrl: https://github.com/tiki/core-iceberg-metadata
-    SemanticVersion: 0.1.9
-    SourceCodeUrl: https://github.com/tiki/core-iceberg-metadata
+    Default: 1
 
 Resources:
   Queue:
@@ -88,6 +79,10 @@ Resources:
                 - xray:PutTraceSegments
                 - xray:PutTelemetryRecords
               Resource: "*"
+            - Effect: Allow
+              Action:
+                - lakeformation:GetDataAccess
+              Resource: "*"
       Layers:
         - !Sub "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:layer:${UtilsName}:${UtilsVersion}"
         - !Sub "arn:aws:lambda:${AWS::Region}:901920570463:layer:aws-otel-java-wrapper-amd64-ver-1-30-0:1"

diff --git a/infra/role/samconfig.toml b/infra/role/samconfig.toml
@@ -1,9 +1,9 @@
 version = 0.1
 
 [default.global.parameters]
-stack_name = "core-iceberg-metadata-role"
+stack_name = "core-ocean-metadata-role"
 s3_bucket = "mytiki-sam-deploy"
-s3_prefix = "core/iceberg/metadata/role"
+s3_prefix = "core/ocean/metadata/role"
 region = "us-east-2"
 
 [default.deploy.parameters]

diff --git a/infra/role/template.yml b/infra/role/template.yml
@@ -6,7 +6,7 @@ Parameters:
   RoleName:
     Description: Name of the IAM Role for the deployment
     Type: String
-    Default: "GitHubDeployCoreIcebergMetadata"
+    Default: "GitHubDeployCoreOceanMetadata"
 
 Resources:
   Role:
@@ -17,7 +17,7 @@ Resources:
         SemanticVersion: 0.0.3
       Parameters:
         GitHubOrg: tiki
-        RepositoryName: core-iceberg-metadata
+        RepositoryName: core-ocean-metadata
         RoleName: !Ref RoleName
 
   Policy:
@@ -31,16 +31,62 @@ Resources:
         Statement:
           - Effect: Allow
             Action:
-              - serverlessrepo:ListApplications
-              - serverlessrepo:CreateApplication
-              - serverlessrepo:SearchApplications
+              - s3:PutObject
+              - s3:GetObject
+            Resource: !Sub arn:aws:s3:::mytiki-sam-deploy/*
+          - Effect: Allow
+            Action: cloudformation:*
+            Resource:
+              - !Sub arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/core-ocean-metadata-function/*
+              - !Sub arn:aws:cloudformation:${AWS::Region}:aws:transform/Serverless-2016-10-31
+          - Effect: Allow
+            Action: cloudformation:ListStacks
             Resource: "*"
+          - Effect: Allow
+            Action: iam:PassRole
+            Resource:
+              - !Sub arn:aws:iam::${AWS::AccountId}:role/${RoleName}
+              - !Sub arn:aws:iam::${AWS::AccountId}:role/core-ocean-metadata-function-*
+            Condition:
+              StringEquals:
+                iam:PassedToService: lambda.amazonaws.com
           - Effect: Allow
             Action:
-              - serverlessrepo:*
-            Resource: !Sub arn:aws:serverlessrepo:${AWS::Region}:${AWS::AccountId}:applications/core-iceberg-metadata
+              - lambda:CreateFunction
+              - lambda:TagResource
+              - lambda:GetFunction
+              - lambda:UpdateFunctionCode
+              - lambda:ListTags
+              - lambda:UpdateFunctionConfiguration
+            Resource: !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:*
           - Effect: Allow
             Action:
-              - s3:PutObject
-              - s3:GetObject
-            Resource: !Sub arn:aws:s3:::mytiki-sam-deploy/*
+              - lambda:CreateEventSourceMapping
+              - lambda:GetEventSourceMapping
+              - lambda:UpdateEventSourceMapping
+            Resource: "*"
+          - Effect: Allow
+            Action:
+              - sqs:CreateQueue
+              - sqs:GetQueueAttributes
+              - sqs:SetQueueAttributes
+              - sqs:TagQueue
+            Resource: !Sub arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:*
+          - Effect: Allow
+            Action: lambda:GetLayerVersion
+            Resource:
+              - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:layer:core-ocean-iceberg-utils:*
+              - !Sub arn:aws:lambda:${AWS::Region}:901920570463:layer:aws-otel-java-wrapper-*
+          - Effect: Allow
+            Action:
+              - iam:CreateRole
+              - iam:GetRole
+              - iam:UpdateRole
+              - iam:DeleteRole
+              - iam:TagRole
+              - iam:GetRolePolicy
+              - iam:PutRolePolicy
+              - iam:DeleteRolePolicy
+              - iam:AttachRolePolicy
+              - iam:DetachRolePolicy
+            Resource: !Sub arn:aws:iam::${AWS::AccountId}:role/core-ocean-metadata-function-*
diff --git a/pom.xml b/pom.xml
@@ -7,7 +7,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>com.mytiki</groupId>
   <artifactId>core-iceberg-metadata</artifactId>
-  <version>0.1.9</version>
+  <version>0.1.11</version>
   <packaging>jar</packaging>
   <name>Iceberg Metadata</name>
   <properties>
@@ -19,23 +19,23 @@
     <dependency>
       <groupId>com.mytiki</groupId>
       <artifactId>core-iceberg-utils</artifactId>
-      <version>0.2.6</version>
+      <version>0.2.7</version>
       <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-api</artifactId>
-      <version>2.21.0</version>
+      <version>2.22.0</version>
     </dependency>
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-core</artifactId>
-      <version>2.21.0</version>
+      <version>2.22.0</version>
     </dependency>
     <dependency>
       <groupId>com.amazonaws</groupId>
       <artifactId>aws-lambda-java-log4j2</artifactId>
-      <version>1.5.1</version>
+      <version>1.6.0</version>
     </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
@@ -50,13 +50,13 @@
     <dependency>
       <groupId>org.junit.jupiter</groupId>
       <artifactId>junit-jupiter-engine</artifactId>
-      <version>5.10.0</version>
+      <version>5.10.1</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
-      <version>5.6.0</version>
+      <version>5.8.0</version>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -85,7 +85,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
-        <version>3.1.2</version>
+        <version>3.2.2</version>
       </plugin>
     </plugins>
   </build>