Setup synapse

admin/roles.sql

@@ -1,15 +1,12 @@
 USE WAREHOUSE COMPUTE_ORG;
-
 use role securityadmin;
 
 // Grant system roles to users
 GRANT ROLE SYSADMIN
 TO USER "[email protected]";
-
 // GENIE
 USE ROLE USERADMIN;
 CREATE ROLE IF NOT EXISTS genie_admin;
-
 USE ROLE SECURITYADMIN;
 grant role genie_admin
 to role useradmin;
@@ -46,55 +43,26 @@ TO ROLE useradmin;
 GRANT ROLE AD
 TO USER "[email protected]";
 
-
 // Public role
 // Synapse data warehouse
 // GRANT SELECT ON ALL TABLES IN SCHEMA synapse_data_warehouse.synapse TO ROLE PUBLIC;
 -- TODO: Add these back in after governance
 -- GRANT SELECT ON FUTURE TABLES IN SCHEMA synapse_data_warehouse.synapse
 -- TO ROLE PUBLIC;
-
-
 USE ROLE USERADMIN;
 CREATE ROLE IF NOT EXISTS masking_admin;
-
-use role securityadmin;
-GRANT CREATE MASKING POLICY ON SCHEMA SYNAPSE_DATA_WAREHOUSE.synapse
-TO ROLE masking_admin;
-
 GRANT ROLE masking_admin
 TO USER "[email protected]";
 USE ROLE ACCOUNTADMIN;
-
 GRANT APPLY MASKING POLICY on ACCOUNT
 to ROLE masking_admin;
 
 USE ROLE USERADMIN;
-
 CREATE ROLE IF NOT EXISTS data_engineer;
 USE ROLE SECURITYADMIN;
 grant role data_engineer
 to role useradmin;
-GRANT CREATE SCHEMA, USAGE ON DATABASE SYNAPSE_DATA_WAREHOUSE
-TO ROLE data_engineer;
--- GRANT ALL PRIVILEGES ON ALL SCHEMAS IN DATABASE SYNAPSE_DATA_WAREHOUSE
--- TO ROLE data_engineer;
--- GRANT ALL PRIVILEGES ON ALL TABLES IN DATABASE SYNAPSE_DATA_WAREHOUSE
--- TO ROLE data_engineer;
-GRANT ALL PRIVILEGES ON FUTURE SCHEMAS IN DATABASE SYNAPSE_DATA_WAREHOUSE
-TO ROLE data_engineer;
-GRANT ALL PRIVILEGES ON FUTURE TABLES IN DATABASE SYNAPSE_DATA_WAREHOUSE
-TO ROLE data_engineer;
-GRANT CREATE SCHEMA, USAGE ON DATABASE SAGE
-TO ROLE data_engineer;
--- GRANT ALL PRIVILEGES ON ALL SCHEMAS IN DATABASE SAGE
--- TO ROLE data_engineer;
--- GRANT ALL PRIVILEGES ON ALL TABLES IN DATABASE SAGE
--- TO ROLE data_engineer;
-GRANT ALL PRIVILEGES ON FUTURE SCHEMAS IN DATABASE SAGE
-TO ROLE data_engineer;
-GRANT ALL PRIVILEGES ON FUTURE TABLES IN DATABASE SAGE
-TO ROLE data_engineer;
+
 GRANT ROLE data_engineer
 TO USER "[email protected]";
 GRANT ROLE data_engineer

admin/sage_setup.sql

@@ -12,6 +12,17 @@ GRANT SELECT ON FUTURE TABLES IN DATABASE sage
 TO ROLE PUBLIC;
 GRANT USAGE ON DATABASE sage
 TO ROLE PUBLIC;
+GRANT ALL PRIVILEGES ON FUTURE SCHEMAS IN DATABASE SAGE
+TO ROLE data_engineer;
+GRANT ALL PRIVILEGES ON FUTURE TABLES IN DATABASE SAGE
+TO ROLE data_engineer;
+GRANT CREATE SCHEMA, USAGE ON DATABASE SAGE
+TO ROLE data_engineer;
+
+-- GRANT ALL PRIVILEGES ON ALL SCHEMAS IN DATABASE SAGE
+-- TO ROLE data_engineer;
+-- GRANT ALL PRIVILEGES ON ALL TABLES IN DATABASE SAGE
+-- TO ROLE data_engineer;
 
 CREATE SCHEMA IF NOT EXISTS AD
     WITH MANAGED ACCESS;

admin/synapse_setup.sql

@@ -0,0 +1,65 @@
+// Follow this blog https://www.snowflake.com/blog/how-to-load-terabytes-into-snowflake-speeds-feeds-and-techniques/#:~:text=Best%20Practices%20for%20Parquet%20and%20ORC
+USE DATABASE synapse_data_warehouse;
+USE SCHEMA synapse_raw;
+USE WAREHOUSE COMPUTE_ORG;
+USE ROLE ACCOUNTADMIN;
+-- * Test Integration
+CREATE STORAGE INTEGRATION IF NOT EXISTS synapse_dev_warehouse_s3
+  TYPE = EXTERNAL_STAGE
+  STORAGE_PROVIDER = 'S3'
+  ENABLED = TRUE
+  STORAGE_AWS_ROLE_ARN = 'arn:aws:iam::449435941126:role/test-snowflake-access-SnowflakeServiceRole-1LXZYAMMKTHJY'
+  STORAGE_ALLOWED_LOCATIONS = ('s3://dev.datawarehouse.sagebase.org');
+-- * Integration to prod (SNOW-14)
+CREATE STORAGE INTEGRATION IF NOT EXISTS synapse_prod_warehouse_s3
+  TYPE = EXTERNAL_STAGE
+  STORAGE_PROVIDER = 'S3'
+  ENABLED = TRUE
+  STORAGE_AWS_ROLE_ARN = 'arn:aws:iam::325565585839:role/snowflake-accesss-SnowflakeServiceRole-HL66JOP7K4BT'
+  STORAGE_ALLOWED_LOCATIONS = ('s3://prod.datawarehouse.sagebase.org');
+DESC INTEGRATION synapse_dev_warehouse_s3;
+DESC INTEGRATION synapse_prod_warehouse_s3;
+
+USE SCHEMA synapse_data_warehouse.synapse_raw;
+USE ROLE SECURITYADMIN;
+GRANT USAGE ON INTEGRATION synapse_dev_warehouse_s3
+TO ROLE SYSADMIN;
+GRANT USAGE ON INTEGRATION synapse_prod_warehouse_s3
+TO ROLE SYSADMIN;
+
+-- * Create external stage
+USE ROLE sysadmin;
+USE DATABASE synapse_data_warehouse;
+USE SCHEMA synapse_raw;
+CREATE STAGE IF NOT EXISTS synapse_dev_warehouse_s3_stage
+  STORAGE_INTEGRATION = synapse_dev_warehouse_s3
+  URL = 's3://dev.datawarehouse.sagebase.org/datawarehouse/'
+  FILE_FORMAT = (TYPE = PARQUET COMPRESSION = AUTO)
+  DIRECTORY = (ENABLE = TRUE);
+
+ALTER STAGE IF EXISTS synapse_dev_warehouse_s3_stage REFRESH;
+LIST @synapse_dev_warehouse_s3_stage;
+
+-- * SNOW-14
+CREATE STAGE IF NOT EXISTS synapse_prod_warehouse_s3_stage
+  STORAGE_INTEGRATION = synapse_prod_warehouse_s3
+  URL = 's3://prod.datawarehouse.sagebase.org/warehouse/'
+  FILE_FORMAT = (TYPE = PARQUET COMPRESSION = AUTO)
+  DIRECTORY = (ENABLE = TRUE);
+
+ALTER STAGE IF EXISTS synapse_prod_warehouse_s3_stage REFRESH;
+
+USE ROLE SECURITYADMIN;
+
+GRANT CREATE MASKING POLICY ON SCHEMA SYNAPSE_DATA_WAREHOUSE.synapse
+TO ROLE masking_admin;
+GRANT CREATE SCHEMA, USAGE ON DATABASE SYNAPSE_DATA_WAREHOUSE
+TO ROLE data_engineer;
+-- GRANT ALL PRIVILEGES ON ALL SCHEMAS IN DATABASE SYNAPSE_DATA_WAREHOUSE
+-- TO ROLE data_engineer;
+-- GRANT ALL PRIVILEGES ON ALL TABLES IN DATABASE SYNAPSE_DATA_WAREHOUSE
+-- TO ROLE data_engineer;
+GRANT ALL PRIVILEGES ON FUTURE SCHEMAS IN DATABASE SYNAPSE_DATA_WAREHOUSE
+TO ROLE data_engineer;
+GRANT ALL PRIVILEGES ON FUTURE TABLES IN DATABASE SYNAPSE_DATA_WAREHOUSE
+TO ROLE data_engineer;