Document permissions for AWS (DynamoDB & Redshift) (#1753)

* Document permissions for AWS (DynamoDB & Redshift) Signed-off-by: Tsotne Tabidze <[email protected]> * Add dynamodb & redshift pages in parent pages Signed-off-by: Tsotne Tabidze <[email protected]>
feast-dev · Aug 1, 2021 · 8f6e8df · 8f6e8df
1 parent 082bbff
commit 8f6e8df
Show file tree

Hide file tree

Showing 8 changed files with 236 additions and 97 deletions.
diff --git a/docs/getting-started/create-a-feature-repository.md b/docs/getting-started/create-a-feature-repository.md
@@ -24,13 +24,13 @@ Creating a new Feast repository in /<...>/tiny_pika.
 {% tab title="AWS template" %}
 ```text
 feast init -t aws
-[?] AWS Region (e.g. us-west-2): ...
-[?] Redshift Cluster ID: ...
-[?] Redshift Database Name: ...
-[?] Redshift User Name: ...
-[?] Redshift S3 Staging Location (s3://*): ...
-[?] Redshift IAM Role for S3 (arn:aws:iam::*:role/*): ...
-[?] Should I upload example data to Redshift (overwriting 'feast_driver_hourly_stats' table)? (Y/n): 
+AWS Region (e.g. us-west-2): ...
+Redshift Cluster ID: ...
+Redshift Database Name: ...
+Redshift User Name: ...
+Redshift S3 Staging Location (s3://*): ...
+Redshift IAM Role for S3 (arn:aws:iam::*:role/*): ...
+Should I upload example data to Redshift (overwriting 'feast_driver_hourly_stats' table)? (Y/n): 
 
 Creating a new Feast repository in /<...>/tiny_pika.
 ```

diff --git a/docs/reference/data-sources/README.md b/docs/reference/data-sources/README.md
@@ -6,3 +6,4 @@ Please see [Data Source](../../concepts/feature-view.md#data-source) for an expl
 
 {% page-ref page="bigquery.md" %}
 
+{% page-ref page="redshift.md" %}
diff --git a/docs/reference/offline-stores/README.md b/docs/reference/offline-stores/README.md
@@ -6,3 +6,4 @@ Please see [Offline Store](../../concepts/offline-store.md) for an explanation o
 
 {% page-ref page="bigquery.md" %}
 
+{% page-ref page="redshift.md" %}
diff --git a/docs/reference/offline-stores/redshift.md b/docs/reference/offline-stores/redshift.md
@@ -29,3 +29,166 @@ offline_store:
 
 Configuration options are available [here](https://github.com/feast-dev/feast/blob/bf557bcb72c7878a16dccb48443bbbe9dc3efa49/sdk/python/feast/infra/offline_stores/redshift.py#L22).
 
+### Permissions
+
+Feast requires the following permissions in order to execute commands for Redshift offline store:
+
+<table>
+  <thead>
+    <tr>
+      <th style="text-align:left"><b>Command</b></th>
+      <th style="text-align:left">Permissions</th>
+      <th style="text-align:left">Resources</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td style="text-align:left"><b>Apply</b></td>
+      <td style="text-align:left">
+        <p>redshift-data:DescribeTable</p>
+        <p>redshift:GetClusterCredentials</p>
+      </td>
+      <td style="text-align:left">
+          <p>arn:aws:redshift:&lt;region&gt;:&lt;account_id&gt;:dbuser:&lt;redshift_cluster_id&gt;/&lt;redshift_username&gt;</p>
+          <p>arn:aws:redshift:&lt;region&gt;:&lt;account_id&gt;:dbname:&lt;redshift_cluster_id&gt;/&lt;redshift_database_name&gt;</p>
+          <p>arn:aws:redshift:&lt;region&gt;:&lt;account_id&gt;:cluster:&lt;redshift_cluster_id&gt;</p>
+      </td>
+    </tr>
+    <tr>
+      <td style="text-align:left" rowspan="3"><b>Materialize</b></td>
+      <td style="text-align:left">
+        <p>redshift-data:ExecuteStatement</p>
+      </td>
+      <td style="text-align:left">
+        <p>arn:aws:redshift:&lt;region&gt;:&lt;account_id&gt;:cluster:&lt;redshift_cluster_id&gt;</p>
+      </td>
+    </tr>
+    <tr>
+      <td style="text-align:left">
+        <p>redshift-data:DescribeStatement</p>
+      </td>
+      <td style="text-align:left">*</td>
+    </tr>
+    <tr>
+      <td style="text-align:left">
+        <p>s3:ListBucket</p>
+        <p>s3:GetObject</p>
+        <p>s3:DeleteObject</p>
+      </td>
+      <td style="text-align:left">
+        <p>arn:aws:s3:::&lt;bucket_name&gt;</p>
+        <p>arn:aws:s3:::&lt;bucket_name&gt;/*</p>
+      </td>
+    </tr>
+    <tr>
+      <td style="text-align:left" rowspan="3"><b>Get Historical Features</b></td>
+      <td style="text-align:left">
+        <p>redshift-data:ExecuteStatement</p>
+        <p>redshift:GetClusterCredentials</p>
+      </td>
+      <td style="text-align:left">
+          <p>arn:aws:redshift:&lt;region&gt;:&lt;account_id&gt;:dbuser:&lt;redshift_cluster_id&gt;/&lt;redshift_username&gt;</p>
+          <p>arn:aws:redshift:&lt;region&gt;:&lt;account_id&gt;:dbname:&lt;redshift_cluster_id&gt;/&lt;redshift_database_name&gt;</p>
+          <p>arn:aws:redshift:&lt;region&gt;:&lt;account_id&gt;:cluster:&lt;redshift_cluster_id&gt;</p>
+      </td>
+    </tr>
+    <tr>
+      <td style="text-align:left">
+        <p>redshift-data:DescribeStatement</p>
+      </td>
+      <td style="text-align:left">*</td>
+    </tr>
+    <tr>
+      <td style="text-align:left">
+        <p>s3:ListBucket</p>
+        <p>s3:GetObject</p>
+        <p>s3:PutObject</p>
+        <p>s3:DeleteObject</p>
+      </td>
+      <td style="text-align:left">
+        <p>arn:aws:s3:::&lt;bucket_name&gt;</p>
+        <p>arn:aws:s3:::&lt;bucket_name&gt;/*</p>
+      </td>
+    </tr>
+  </tbody>
+</table>
+
+The following inline policy can be used to grant Feast the necessary permissions:
+
+```json
+{
+    "Statement": [
+        {
+            "Action": [
+                "s3:ListBucket",
+                "s3:PutObject",
+                "s3:GetObject",
+                "s3:DeleteObject"
+            ],
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:s3:::<bucket_name>/*",
+                "arn:aws:s3:::<bucket_name>"
+            ]
+        },
+        {
+            "Action": [
+                "redshift-data:DescribeTable",
+                "redshift:GetClusterCredentials",
+                "redshift-data:ExecuteStatement"
+            ],
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:redshift:<region>:<account_id>:dbuser:<redshift_cluster_id>/<redshift_username>",
+                "arn:aws:redshift:<region>:<account_id>:dbname:<redshift_cluster_id>/<redshift_database_name>",
+                "arn:aws:redshift:<region>:<account_id>:cluster:<redshift_cluster_id>"
+            ]
+        },
+        {
+            "Action": [
+                "redshift-data:DescribeStatement"
+            ],
+            "Effect": "Allow",
+            "Resource": "*"
+        }
+    ],
+    "Version": "2012-10-17"
+}
+```
+
+In addition to this, Redshift offline store requires an IAM role that will be used by Redshift itself to interact with S3. More concretely, Redshift has to use this IAM role to run <a href="https://docs.aws.amazon.com/redshift/latest/dg/r_UNLOAD.html">UNLOAD</a> and <a href="https://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html">COPY</a> commands. Once created, this IAM role needs to be configured in `feature_store.yaml` file as `offline_store: iam_role`.
+
+The following inline policy can be used to grant Redshift necessary permissions to access S3:
+
+```json
+{
+    "Statement": [
+        {
+            "Action": "s3:*",
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:s3:::feast-integration-tests",
+                "arn:aws:s3:::feast-integration-tests/*"
+            ]
+        }
+    ],
+    "Version": "2012-10-17"
+}
+```
+
+While the following trust relationship is necessary to make sure that Redshift, and only Redshift can assume this role:
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "redshift.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
+```
diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md
@@ -8,3 +8,4 @@ Please see [Online Store](../../concepts/online-store.md) for an explanation of
 
 {% page-ref page="datastore.md" %}
 
+{% page-ref page="dynamodb.md" %}
diff --git a/docs/reference/online-stores/dynamodb.md b/docs/reference/online-stores/dynamodb.md
@@ -25,3 +25,65 @@ online_store:
 {% endcode %}
 
 Configuration options are available [here](https://github.com/feast-dev/feast/blob/17bfa6118d6658d2bff53d7de8e2ccef5681714d/sdk/python/feast/infra/online_stores/dynamodb.py#L36).
+
+### Permissions
+
+Feast requires the following permissions in order to execute commands for DynamoDB online store:
+
+<table>
+  <thead>
+    <tr>
+      <th style="text-align:left"><b>Command</b></th>
+      <th style="text-align:left">Permissions</th>
+      <th style="text-align:left">Resources</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td style="text-align:left"><b>Apply</b></td>
+      <td style="text-align:left">
+        <p>dynamodb:CreateTable</p>
+        <p>dynamodb:DescribeTable</p>
+        <p>dynamodb:DeleteTable</p>
+      </td>
+      <td style="text-align:left">arn:aws:dynamodb:&lt;region&gt;:&lt;account_id&gt;:table/*</td>
+    </tr>
+    <tr>
+      <td style="text-align:left"><b>Materialize</b></td>
+      <td style="text-align:left">
+        <p>dynamodb.BatchWriteItem</p>
+      </td>
+      <td style="text-align:left">arn:aws:dynamodb:&lt;region&gt;:&lt;account_id&gt;:table/*</td>
+    </tr>
+    <tr>
+      <td style="text-align:left"><b>Get Online Features</b></td>
+      <td style="text-align:left">dynamodb.GetItem</td>
+      <td style="text-align:left">arn:aws:dynamodb:&lt;region&gt;:&lt;account_id&gt;:table/*</td>
+    </tr>
+  </tbody>
+</table>
+
+The following inline policy can be used to grant Feast the necessary permissions:
+
+```json
+{
+    "Statement": [
+        {
+            "Action": [
+                "dynamodb:CreateTable",
+                "dynamodb:DescribeTable",
+                "dynamodb:DeleteTable",
+                "dynamodb:BatchWriteItem",
+                "dynamodb:GetItem"
+            ],
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:dynamodb:<region>:<account_id>:table/*"
+            ]
+        }
+    ],
+    "Version": "2012-10-17"
+}
+```
+
+Lastly, this IAM role needs to be associated with the desired Redshift cluster. Please follow the official AWS guide for the necessary steps [here](https://docs.aws.amazon.com/redshift/latest/dg/c-getting-started-using-spectrum-add-role.html).
diff --git a/docs/reference/providers/README.md b/docs/reference/providers/README.md
@@ -6,3 +6,4 @@ Please see [Provider](../../concepts/provider.md) for an explanation of provider
 
 {% page-ref page="google-cloud-platform.md" %}
 
+{% page-ref page="amazon-web-services.md" %}
diff --git a/docs/reference/providers/amazon-web-services.md b/docs/reference/providers/amazon-web-services.md
@@ -25,93 +25,3 @@ offline_store:
   iam_role: arn:aws:iam::123456789012:role/redshift_s3_access_role
 ```
 {% endcode %}
-
-<!--
-TODO: figure out the permissions
-
-### **Permissions**
-
-<table>
-  <thead>
-    <tr>
-      <th style="text-align:left"><b>Command</b>
-      </th>
-      <th style="text-align:left">Component</th>
-      <th style="text-align:left">Permissions</th>
-      <th style="text-align:left">Recommended Role</th>
-    </tr>
-  </thead>
-  <tbody>
-    <tr>
-      <td style="text-align:left"><b>Apply</b>
-      </td>
-      <td style="text-align:left">BigQuery (source)</td>
-      <td style="text-align:left">
-        <p>bigquery.jobs.create</p>
-        <p>bigquery.readsessions.create</p>
-        <p>bigquery.readsessions.getData</p>
-      </td>
-      <td style="text-align:left">roles/bigquery.user</td>
-    </tr>
-    <tr>
-      <td style="text-align:left"><b>Apply</b>
-      </td>
-      <td style="text-align:left">Datastore (destination)</td>
-      <td style="text-align:left">
-        <p>datastore.entities.allocateIds</p>
-        <p>datastore.entities.create</p>
-        <p>datastore.entities.delete</p>
-        <p>datastore.entities.get</p>
-        <p>datastore.entities.list</p>
-        <p>datastore.entities.update</p>
-      </td>
-      <td style="text-align:left">roles/datastore.owner</td>
-    </tr>
-    <tr>
-      <td style="text-align:left"><b>Materialize</b>
-      </td>
-      <td style="text-align:left">BigQuery (source)</td>
-      <td style="text-align:left">bigquery.jobs.create</td>
-      <td style="text-align:left">roles/bigquery.user</td>
-    </tr>
-    <tr>
-      <td style="text-align:left"><b>Materialize</b>
-      </td>
-      <td style="text-align:left">Datastore (destination)</td>
-      <td style="text-align:left">
-        <p>datastore.entities.allocateIds</p>
-        <p>datastore.entities.create</p>
-        <p>datastore.entities.delete</p>
-        <p>datastore.entities.get</p>
-        <p>datastore.entities.list</p>
-        <p>datastore.entities.update</p>
-        <p>datastore.databases.get</p>
-      </td>
-      <td style="text-align:left">roles/datastore.owner</td>
-    </tr>
-    <tr>
-      <td style="text-align:left"><b>Get Online Features</b>
-      </td>
-      <td style="text-align:left">Datastore</td>
-      <td style="text-align:left">datastore.entities.get</td>
-      <td style="text-align:left">roles/datastore.user</td>
-    </tr>
-    <tr>
-      <td style="text-align:left"><b>Get Historical Features</b>
-      </td>
-      <td style="text-align:left">BigQuery (source)</td>
-      <td style="text-align:left">
-        <p>bigquery.datasets.get</p>
-        <p>bigquery.tables.get</p>
-        <p>bigquery.tables.create</p>
-        <p>bigquery.tables.updateData</p>
-        <p>bigquery.tables.update</p>
-        <p>bigquery.tables.delete</p>
-        <p>bigquery.tables.getData</p>
-      </td>
-      <td style="text-align:left">roles/bigquery.dataEditor</td>
-    </tr>
-  </tbody>
-</table>
-
--->