diff --git a/docs/data-sources/schema.md b/docs/data-sources/schema.md index eb2a78b..21beb36 100644 --- a/docs/data-sources/schema.md +++ b/docs/data-sources/schema.md @@ -27,6 +27,7 @@ data "redshift_schema" "schema" { ### Optional +- **external_schema** (Block List, Max: 1) Configures the schema as an external schema. See https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_SCHEMA.html (see [below for nested schema](#nestedblock--external_schema)) - **id** (String) The ID of this resource. ### Read-Only @@ -34,4 +35,139 @@ data "redshift_schema" "schema" { - **owner** (String) Name of the schema owner. - **quota** (Number) The maximum amount of disk space that the specified schema can use. GB is the default unit of measurement. + +### Nested Schema for `external_schema` + +Optional: + +- **data_catalog_source** (Block List, Max: 1) Configures the external schema from the AWS Glue Data Catalog (see [below for nested schema](#nestedblock--external_schema--data_catalog_source)) +- **hive_metastore_source** (Block List, Max: 1) Configures the external schema from a Hive Metastore. (see [below for nested schema](#nestedblock--external_schema--hive_metastore_source)) +- **rds_mysql_source** (Block List, Max: 1) Configures the external schema to reference data using a federated query to RDS MYSQL or Aurora MySQL. (see [below for nested schema](#nestedblock--external_schema--rds_mysql_source)) +- **rds_postgres_source** (Block List, Max: 1) Configures the external schema to reference data using a federated query to RDS POSTGRES or Aurora PostgreSQL. (see [below for nested schema](#nestedblock--external_schema--rds_postgres_source)) +- **redshift_source** (Block List, Max: 1) Configures the external schema to reference datashare database. (see [below for nested schema](#nestedblock--external_schema--redshift_source)) + +Read-Only: + +- **database_name** (String) The database where the external schema can be found + + +### Nested Schema for `external_schema.data_catalog_source` + +Optional: + +- **catalog_role_arns** (List of String) The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization for the data catalog. + If this is not specified, Amazon Redshift uses the specified iam_role_arns. The catalog role must have permission to access the Data Catalog in AWS Glue or Athena. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles +- **region** (String) If the external database is defined in an Athena data catalog or the AWS Glue Data Catalog, the AWS Region in which the database is located. This parameter is required if the database is defined in an external Data Catalog. + +Read-Only: + +- **iam_role_arns** (List of String) The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization. + As a minimum, the IAM roles must have permission to perform a LIST operation on the Amazon S3 bucket to be accessed and a GET operation on the Amazon S3 objects the bucket contains. + If the external database is defined in an Amazon Athena data catalog or the AWS Glue Data Catalog, the IAM role must have permission to access Athena unless catalog_role is specified. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + When you attach a role to your cluster, your cluster can assume that role to access Amazon S3, Athena, and AWS Glue on your behalf. + If a role attached to your cluster doesn't have access to the necessary resources, you can chain another role, possibly belonging to another account. + Your cluster then temporarily assumes the chained role to access the data. You can also grant cross-account access by chaining roles. + You can chain a maximum of 10 roles. Each role in the chain assumes the next role in the chain, until the cluster assumes the role at the end of chain. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles + + + +### Nested Schema for `external_schema.hive_metastore_source` + +Optional: + +- **port** (Number) The port number of the hive metastore. The default port number is 9083. + +Read-Only: + +- **hostname** (String) The hostname of the hive metastore database. +- **iam_role_arns** (List of String) The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization. + As a minimum, the IAM roles must have permission to perform a LIST operation on the Amazon S3 bucket to be accessed and a GET operation on the Amazon S3 objects the bucket contains. + If the external database is defined in an Amazon Athena data catalog or the AWS Glue Data Catalog, the IAM role must have permission to access Athena unless catalog_role is specified. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + When you attach a role to your cluster, your cluster can assume that role to access Amazon S3, Athena, and AWS Glue on your behalf. + If a role attached to your cluster doesn't have access to the necessary resources, you can chain another role, possibly belonging to another account. + Your cluster then temporarily assumes the chained role to access the data. You can also grant cross-account access by chaining roles. + You can chain a maximum of 10 roles. Each role in the chain assumes the next role in the chain, until the cluster assumes the role at the end of chain. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles + + + +### Nested Schema for `external_schema.rds_mysql_source` + +Optional: + +- **port** (Number) The port number of the MySQL database. The default port number is 3306. + +Read-Only: + +- **hostname** (String) The hostname of the head node of the MySQL database replica set. +- **iam_role_arns** (List of String) The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization. + As a minimum, the IAM roles must have permission to perform a LIST operation on the Amazon S3 bucket to be accessed and a GET operation on the Amazon S3 objects the bucket contains. + If the external database is defined in an Amazon Athena data catalog or the AWS Glue Data Catalog, the IAM role must have permission to access Athena unless catalog_role is specified. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + When you attach a role to your cluster, your cluster can assume that role to access Amazon S3, Athena, and AWS Glue on your behalf. + If a role attached to your cluster doesn't have access to the necessary resources, you can chain another role, possibly belonging to another account. + Your cluster then temporarily assumes the chained role to access the data. You can also grant cross-account access by chaining roles. + You can chain a maximum of 10 roles. Each role in the chain assumes the next role in the chain, until the cluster assumes the role at the end of chain. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles +- **secret_arn** (String) The Amazon Resource Name (ARN) of a supported MySQL database engine secret created using AWS Secrets Manager. + For information about how to create and retrieve an ARN for a secret, see https://docs.aws.amazon.com/secretsmanager/latest/userguide/manage_create-basic-secret.html + and https://docs.aws.amazon.com/secretsmanager/latest/userguide/manage_retrieve-secret.html in the AWS Secrets Manager User Guide. + + + +### Nested Schema for `external_schema.rds_postgres_source` + +Optional: + +- **port** (Number) The port number of the PostgreSQL database. The default port number is 5432. +- **schema** (String) The name of the PostgreSQL schema. The default schema is 'public' + +Read-Only: + +- **hostname** (String) The hostname of the head node of the PostgreSQL database replica set. +- **iam_role_arns** (List of String) The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization. + As a minimum, the IAM roles must have permission to perform a LIST operation on the Amazon S3 bucket to be accessed and a GET operation on the Amazon S3 objects the bucket contains. + If the external database is defined in an Amazon Athena data catalog or the AWS Glue Data Catalog, the IAM role must have permission to access Athena unless catalog_role is specified. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + When you attach a role to your cluster, your cluster can assume that role to access Amazon S3, Athena, and AWS Glue on your behalf. + If a role attached to your cluster doesn't have access to the necessary resources, you can chain another role, possibly belonging to another account. + Your cluster then temporarily assumes the chained role to access the data. You can also grant cross-account access by chaining roles. + You can chain a maximum of 10 roles. Each role in the chain assumes the next role in the chain, until the cluster assumes the role at the end of chain. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles +- **secret_arn** (String) The Amazon Resource Name (ARN) of a supported PostgreSQL database engine secret created using AWS Secrets Manager. + For information about how to create and retrieve an ARN for a secret, see https://docs.aws.amazon.com/secretsmanager/latest/userguide/manage_create-basic-secret.html + and https://docs.aws.amazon.com/secretsmanager/latest/userguide/manage_retrieve-secret.html in the AWS Secrets Manager User Guide. + + + +### Nested Schema for `external_schema.redshift_source` + +Optional: + +- **schema** (String) The name of the datashare schema. The default schema is 'public'. + diff --git a/docs/resources/schema.md b/docs/resources/schema.md index 64c1848..a71eff1 100644 --- a/docs/resources/schema.md +++ b/docs/resources/schema.md @@ -17,11 +17,104 @@ resource "redshift_user" "owner" { name = "owner" } +# Internal schema resource "redshift_schema" "schema" { name = "my_schema" owner = redshift_user.owner.name quota = 150 } + +# External schema using AWS Glue Data Catalog +resource "redshift_schema" "external_from_glue_data_catalog" { + name = "spectrum_schema" + owner = redshift_user.owner.name + external_schema { + database_name = "spectrum_db" # Required. Name of the db in glue catalog + data_catalog_source { + region = "us-west-2" # Optional. If not specified, Redshift will use the same region as the cluster. + iam_role_arns = [ + # Required. Must be at least 1 ARN and not more than 10. + "arn:aws:iam::123456789012:role/myRedshiftRole", + "arn:aws:iam::123456789012:role/myS3Role", + ] + catalog_role_arns = [ + # Optional. If specified, must be at least 1 ARN and not more than 10. + # If not specified, Redshift will use iam_role_arns for accessing the glue data catalog. + "arn:aws:iam::123456789012:role/myAthenaRole", + # ... + ] + create_external_database_if_not_exists = true # Optional. Defaults to false. + } + } +} + +# External schema using Hive Metastore +resource "redshift_schema" "external_from_hive_metastore" { + name = "hive_schema" + owner = redshift_user.owner.name + external_schema { + database_name = "hive_db" # Required. Name of the db in hive metastore + hive_metastore_source { + hostname = "172.10.10.10" # Required + port = 99 # Optional. Default is 9083 + iam_role_arns = [ + # Required. Must be at least 1 ARN and not more than 10. + "arn:aws:iam::123456789012:role/MySpectrumRole", + ] + } + } +} + +# External schema using federated query from RDS/Aurora Postgres +resource "redshift_schema" "external_from_postgres" { + name = "myRedshiftPostgresSchema" + owner = redshift_user.owner.name + external_schema { + database_name = "my_aurora_db" # Required. Name of the db in postgres + rds_postgres_source { + hostname = "endpoint to aurora hostname" # Required + port = 5432 # Optional. Default is 5432 + schema = "my_aurora_schema" # Optional, default is "public" + iam_role_arns = [ + # Required. Must be at least 1 ARN and not more than 10. + "arn:aws:iam::123456789012:role/MyAuroraRole", + # ... + ] + secret_arn = "arn:aws:secretsmanager:us-east-2:123456789012:secret:development/MyTestDatabase-AbCdEf" # Required + } + } +} + +# External schema using federated query from RDS/Aurora MySQL +resource "redshift_schema" "external_from_mysql" { + name = "myRedshiftMysqlSchema" + owner = redshift_user.owner.name + external_schema { + database_name = "my_aurora_db" # Required. Name of the db in mysql + rds_mysql_source { + hostname = "endpoint to aurora hostname" # Required + port = 3306 # Optional. Default is 3306 + iam_role_arns = [ + # Required. Must be at least 1 ARN and not more than 10. + "arn:aws:iam::123456789012:role/MyAuroraRole", + # ... + ] + secret_arn = "arn:aws:secretsmanager:us-east-2:123456789012:secret:development/MyTestDatabase-AbCdEf" # Required + } + } +} + +# External schema using federated query from Redshift data share database +resource "redshift" "external_from_redshift" { + name = "Sales_schema" + owner = redshift_user.owner.name + external_schema { + database_name = "Sales_db" # Required. Name of the datashare db + redshift_source { + schema = "public" # Optional. Name of the schema in the datashare db. Default is "public" + } + } +} ``` @@ -34,8 +127,149 @@ resource "redshift_schema" "schema" { ### Optional - **cascade_on_delete** (Boolean) Indicates to automatically drop all objects in the schema. The default action is TO NOT drop a schema if it contains any objects. +- **external_schema** (Block List, Max: 1) Configures the schema as an external schema. See https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_SCHEMA.html (see [below for nested schema](#nestedblock--external_schema)) - **id** (String) The ID of this resource. - **owner** (String) Name of the schema owner. - **quota** (Number) The maximum amount of disk space that the specified schema can use. GB is the default unit of measurement. + +### Nested Schema for `external_schema` + +Required: + +- **database_name** (String) The database where the external schema can be found + +Optional: + +- **data_catalog_source** (Block List, Max: 1) Configures the external schema from the AWS Glue Data Catalog (see [below for nested schema](#nestedblock--external_schema--data_catalog_source)) +- **hive_metastore_source** (Block List, Max: 1) Configures the external schema from a Hive Metastore. (see [below for nested schema](#nestedblock--external_schema--hive_metastore_source)) +- **rds_mysql_source** (Block List, Max: 1) Configures the external schema to reference data using a federated query to RDS MYSQL or Aurora MySQL. (see [below for nested schema](#nestedblock--external_schema--rds_mysql_source)) +- **rds_postgres_source** (Block List, Max: 1) Configures the external schema to reference data using a federated query to RDS POSTGRES or Aurora PostgreSQL. (see [below for nested schema](#nestedblock--external_schema--rds_postgres_source)) +- **redshift_source** (Block List, Max: 1) Configures the external schema to reference datashare database. (see [below for nested schema](#nestedblock--external_schema--redshift_source)) + + +### Nested Schema for `external_schema.data_catalog_source` + +Required: + +- **iam_role_arns** (List of String) The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization. + As a minimum, the IAM roles must have permission to perform a LIST operation on the Amazon S3 bucket to be accessed and a GET operation on the Amazon S3 objects the bucket contains. + If the external database is defined in an Amazon Athena data catalog or the AWS Glue Data Catalog, the IAM role must have permission to access Athena unless catalog_role is specified. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + When you attach a role to your cluster, your cluster can assume that role to access Amazon S3, Athena, and AWS Glue on your behalf. + If a role attached to your cluster doesn't have access to the necessary resources, you can chain another role, possibly belonging to another account. + Your cluster then temporarily assumes the chained role to access the data. You can also grant cross-account access by chaining roles. + You can chain a maximum of 10 roles. Each role in the chain assumes the next role in the chain, until the cluster assumes the role at the end of chain. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles + +Optional: + +- **catalog_role_arns** (List of String) The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization for the data catalog. + If this is not specified, Amazon Redshift uses the specified iam_role_arns. The catalog role must have permission to access the Data Catalog in AWS Glue or Athena. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles +- **create_external_database_if_not_exists** (Boolean) When enabled, creates an external database with the name specified by the database argument, + if the specified external database doesn't exist. If the specified external database exists, the command makes no changes. + In this case, the command returns a message that the external database exists, rather than terminating with an error. + + To use create_external_database_if_not_exists with a Data Catalog enabled for AWS Lake Formation, you need CREATE_DATABASE permission on the Data Catalog. +- **region** (String) If the external database is defined in an Athena data catalog or the AWS Glue Data Catalog, the AWS Region in which the database is located. This parameter is required if the database is defined in an external Data Catalog. + + + +### Nested Schema for `external_schema.hive_metastore_source` + +Required: + +- **hostname** (String) The hostname of the hive metastore database. +- **iam_role_arns** (List of String) The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization. + As a minimum, the IAM roles must have permission to perform a LIST operation on the Amazon S3 bucket to be accessed and a GET operation on the Amazon S3 objects the bucket contains. + If the external database is defined in an Amazon Athena data catalog or the AWS Glue Data Catalog, the IAM role must have permission to access Athena unless catalog_role is specified. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + When you attach a role to your cluster, your cluster can assume that role to access Amazon S3, Athena, and AWS Glue on your behalf. + If a role attached to your cluster doesn't have access to the necessary resources, you can chain another role, possibly belonging to another account. + Your cluster then temporarily assumes the chained role to access the data. You can also grant cross-account access by chaining roles. + You can chain a maximum of 10 roles. Each role in the chain assumes the next role in the chain, until the cluster assumes the role at the end of chain. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles + +Optional: + +- **port** (Number) The port number of the hive metastore. The default port number is 9083. + + + +### Nested Schema for `external_schema.rds_mysql_source` + +Required: + +- **hostname** (String) The hostname of the head node of the MySQL database replica set. +- **iam_role_arns** (List of String) The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization. + As a minimum, the IAM roles must have permission to perform a LIST operation on the Amazon S3 bucket to be accessed and a GET operation on the Amazon S3 objects the bucket contains. + If the external database is defined in an Amazon Athena data catalog or the AWS Glue Data Catalog, the IAM role must have permission to access Athena unless catalog_role is specified. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + When you attach a role to your cluster, your cluster can assume that role to access Amazon S3, Athena, and AWS Glue on your behalf. + If a role attached to your cluster doesn't have access to the necessary resources, you can chain another role, possibly belonging to another account. + Your cluster then temporarily assumes the chained role to access the data. You can also grant cross-account access by chaining roles. + You can chain a maximum of 10 roles. Each role in the chain assumes the next role in the chain, until the cluster assumes the role at the end of chain. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles +- **secret_arn** (String) The Amazon Resource Name (ARN) of a supported MySQL database engine secret created using AWS Secrets Manager. + For information about how to create and retrieve an ARN for a secret, see https://docs.aws.amazon.com/secretsmanager/latest/userguide/manage_create-basic-secret.html + and https://docs.aws.amazon.com/secretsmanager/latest/userguide/manage_retrieve-secret.html in the AWS Secrets Manager User Guide. + +Optional: + +- **port** (Number) The port number of the MySQL database. The default port number is 3306. + + + +### Nested Schema for `external_schema.rds_postgres_source` + +Required: + +- **hostname** (String) The hostname of the head node of the PostgreSQL database replica set. +- **iam_role_arns** (List of String) The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization. + As a minimum, the IAM roles must have permission to perform a LIST operation on the Amazon S3 bucket to be accessed and a GET operation on the Amazon S3 objects the bucket contains. + If the external database is defined in an Amazon Athena data catalog or the AWS Glue Data Catalog, the IAM role must have permission to access Athena unless catalog_role is specified. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + When you attach a role to your cluster, your cluster can assume that role to access Amazon S3, Athena, and AWS Glue on your behalf. + If a role attached to your cluster doesn't have access to the necessary resources, you can chain another role, possibly belonging to another account. + Your cluster then temporarily assumes the chained role to access the data. You can also grant cross-account access by chaining roles. + You can chain a maximum of 10 roles. Each role in the chain assumes the next role in the chain, until the cluster assumes the role at the end of chain. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles +- **secret_arn** (String) The Amazon Resource Name (ARN) of a supported PostgreSQL database engine secret created using AWS Secrets Manager. + For information about how to create and retrieve an ARN for a secret, see https://docs.aws.amazon.com/secretsmanager/latest/userguide/manage_create-basic-secret.html + and https://docs.aws.amazon.com/secretsmanager/latest/userguide/manage_retrieve-secret.html in the AWS Secrets Manager User Guide. + +Optional: + +- **port** (Number) The port number of the PostgreSQL database. The default port number is 5432. +- **schema** (String) The name of the PostgreSQL schema. The default schema is 'public' + + + +### Nested Schema for `external_schema.redshift_source` + +Optional: + +- **schema** (String) The name of the datashare schema. The default schema is 'public'. + diff --git a/examples/resources/redshift_schema/resource.tf b/examples/resources/redshift_schema/resource.tf index a491fbf..1fdc8e5 100644 --- a/examples/resources/redshift_schema/resource.tf +++ b/examples/resources/redshift_schema/resource.tf @@ -2,8 +2,101 @@ resource "redshift_user" "owner" { name = "owner" } +# Internal schema resource "redshift_schema" "schema" { name = "my_schema" owner = redshift_user.owner.name quota = 150 } + +# External schema using AWS Glue Data Catalog +resource "redshift_schema" "external_from_glue_data_catalog" { + name = "spectrum_schema" + owner = redshift_user.owner.name + external_schema { + database_name = "spectrum_db" # Required. Name of the db in glue catalog + data_catalog_source { + region = "us-west-2" # Optional. If not specified, Redshift will use the same region as the cluster. + iam_role_arns = [ + # Required. Must be at least 1 ARN and not more than 10. + "arn:aws:iam::123456789012:role/myRedshiftRole", + "arn:aws:iam::123456789012:role/myS3Role", + ] + catalog_role_arns = [ + # Optional. If specified, must be at least 1 ARN and not more than 10. + # If not specified, Redshift will use iam_role_arns for accessing the glue data catalog. + "arn:aws:iam::123456789012:role/myAthenaRole", + # ... + ] + create_external_database_if_not_exists = true # Optional. Defaults to false. + } + } +} + +# External schema using Hive Metastore +resource "redshift_schema" "external_from_hive_metastore" { + name = "hive_schema" + owner = redshift_user.owner.name + external_schema { + database_name = "hive_db" # Required. Name of the db in hive metastore + hive_metastore_source { + hostname = "172.10.10.10" # Required + port = 99 # Optional. Default is 9083 + iam_role_arns = [ + # Required. Must be at least 1 ARN and not more than 10. + "arn:aws:iam::123456789012:role/MySpectrumRole", + ] + } + } +} + +# External schema using federated query from RDS/Aurora Postgres +resource "redshift_schema" "external_from_postgres" { + name = "myRedshiftPostgresSchema" + owner = redshift_user.owner.name + external_schema { + database_name = "my_aurora_db" # Required. Name of the db in postgres + rds_postgres_source { + hostname = "endpoint to aurora hostname" # Required + port = 5432 # Optional. Default is 5432 + schema = "my_aurora_schema" # Optional, default is "public" + iam_role_arns = [ + # Required. Must be at least 1 ARN and not more than 10. + "arn:aws:iam::123456789012:role/MyAuroraRole", + # ... + ] + secret_arn = "arn:aws:secretsmanager:us-east-2:123456789012:secret:development/MyTestDatabase-AbCdEf" # Required + } + } +} + +# External schema using federated query from RDS/Aurora MySQL +resource "redshift_schema" "external_from_mysql" { + name = "myRedshiftMysqlSchema" + owner = redshift_user.owner.name + external_schema { + database_name = "my_aurora_db" # Required. Name of the db in mysql + rds_mysql_source { + hostname = "endpoint to aurora hostname" # Required + port = 3306 # Optional. Default is 3306 + iam_role_arns = [ + # Required. Must be at least 1 ARN and not more than 10. + "arn:aws:iam::123456789012:role/MyAuroraRole", + # ... + ] + secret_arn = "arn:aws:secretsmanager:us-east-2:123456789012:secret:development/MyTestDatabase-AbCdEf" # Required + } + } +} + +# External schema using federated query from Redshift data share database +resource "redshift" "external_from_redshift" { + name = "Sales_schema" + owner = redshift_user.owner.name + external_schema { + database_name = "Sales_db" # Required. Name of the datashare db + redshift_source { + schema = "public" # Optional. Name of the schema in the datashare db. Default is "public" + } + } +} diff --git a/redshift/acc_test.go b/redshift/acc_test.go new file mode 100644 index 0000000..85b5dde --- /dev/null +++ b/redshift/acc_test.go @@ -0,0 +1,27 @@ +// This file shouldn't contain actual test cases, +// but rather common utility methods for acceptance tests. +package redshift + +import ( + "fmt" + "os" + "strings" + "testing" +) + +// Get the value of an environment variable, or skip the +// current test if the variable is not set. +func getEnvOrSkip(key string, t *testing.T) string { + v := os.Getenv(key) + if v == "" { + t.Skipf(fmt.Sprintf("Environment variable %s was not set. Skipping...", key)) + } + return v +} + +// Renders a string slice as a terraform array +func tfArray(s []string) string { + semiformat := fmt.Sprintf("%q\n", s) + tokens := strings.Split(semiformat, " ") + return fmt.Sprintf(strings.Join(tokens, ",")) +} diff --git a/redshift/custom_diff.go b/redshift/custom_diff.go new file mode 100644 index 0000000..58564c9 --- /dev/null +++ b/redshift/custom_diff.go @@ -0,0 +1,16 @@ +package redshift + +import ( + "context" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/customdiff" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" +) + +func forceNewIfListSizeChanged(key string) schema.CustomizeDiffFunc { + return customdiff.ForceNewIfChange(key, listSizeChanged) +} + +func listSizeChanged(ctx context.Context, old, new, meta interface{}) bool { + return len(old.([]interface{})) != len(new.([]interface{})) +} diff --git a/redshift/data_source_redshift_schema.go b/redshift/data_source_redshift_schema.go index 66a0da1..1ddd03b 100644 --- a/redshift/data_source_redshift_schema.go +++ b/redshift/data_source_redshift_schema.go @@ -1,6 +1,7 @@ package redshift import ( + "fmt" "strings" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" @@ -31,33 +32,263 @@ A database contains one or more named schemas. Each schema in a database contain Computed: true, Description: "The maximum amount of disk space that the specified schema can use. GB is the default unit of measurement.", }, + schemaExternalSchemaAttr: { + Type: schema.TypeList, + Optional: true, + Computed: true, + Description: "Configures the schema as an external schema. See https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_SCHEMA.html", + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "database_name": { + Type: schema.TypeString, + Computed: true, + Description: "The database where the external schema can be found", + }, + "data_catalog_source": { + Type: schema.TypeList, + Description: "Configures the external schema from the AWS Glue Data Catalog", + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "region": { + Type: schema.TypeString, + Optional: true, + Computed: true, + Description: "If the external database is defined in an Athena data catalog or the AWS Glue Data Catalog, the AWS Region in which the database is located. This parameter is required if the database is defined in an external Data Catalog.", + }, + "iam_role_arns": { + Type: schema.TypeList, + Computed: true, + Description: `The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization. + As a minimum, the IAM roles must have permission to perform a LIST operation on the Amazon S3 bucket to be accessed and a GET operation on the Amazon S3 objects the bucket contains. + If the external database is defined in an Amazon Athena data catalog or the AWS Glue Data Catalog, the IAM role must have permission to access Athena unless catalog_role is specified. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + When you attach a role to your cluster, your cluster can assume that role to access Amazon S3, Athena, and AWS Glue on your behalf. + If a role attached to your cluster doesn't have access to the necessary resources, you can chain another role, possibly belonging to another account. + Your cluster then temporarily assumes the chained role to access the data. You can also grant cross-account access by chaining roles. + You can chain a maximum of 10 roles. Each role in the chain assumes the next role in the chain, until the cluster assumes the role at the end of chain. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles`, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + "catalog_role_arns": { + Type: schema.TypeList, + Optional: true, + Computed: true, + Description: `The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization for the data catalog. + If this is not specified, Amazon Redshift uses the specified iam_role_arns. The catalog role must have permission to access the Data Catalog in AWS Glue or Athena. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles`, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + }, + }, + }, + "hive_metastore_source": { + Type: schema.TypeList, + Description: "Configures the external schema from a Hive Metastore.", + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "hostname": { + Type: schema.TypeString, + Description: "The hostname of the hive metastore database.", + Computed: true, + }, + "port": { + Type: schema.TypeInt, + Description: "The port number of the hive metastore. The default port number is 9083.", + Optional: true, + Computed: true, + }, + "iam_role_arns": { + Type: schema.TypeList, + Computed: true, + Description: `The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization. + As a minimum, the IAM roles must have permission to perform a LIST operation on the Amazon S3 bucket to be accessed and a GET operation on the Amazon S3 objects the bucket contains. + If the external database is defined in an Amazon Athena data catalog or the AWS Glue Data Catalog, the IAM role must have permission to access Athena unless catalog_role is specified. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + When you attach a role to your cluster, your cluster can assume that role to access Amazon S3, Athena, and AWS Glue on your behalf. + If a role attached to your cluster doesn't have access to the necessary resources, you can chain another role, possibly belonging to another account. + Your cluster then temporarily assumes the chained role to access the data. You can also grant cross-account access by chaining roles. + You can chain a maximum of 10 roles. Each role in the chain assumes the next role in the chain, until the cluster assumes the role at the end of chain. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles`, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + }, + }, + }, + "rds_postgres_source": { + Type: schema.TypeList, + Description: "Configures the external schema to reference data using a federated query to RDS POSTGRES or Aurora PostgreSQL.", + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "hostname": { + Type: schema.TypeString, + Description: "The hostname of the head node of the PostgreSQL database replica set.", + Computed: true, + }, + "port": { + Type: schema.TypeInt, + Description: "The port number of the PostgreSQL database. The default port number is 5432.", + Optional: true, + Computed: true, + }, + "schema": { + Type: schema.TypeString, + Description: "The name of the PostgreSQL schema. The default schema is 'public'", + Optional: true, + Computed: true, + }, + "iam_role_arns": { + Type: schema.TypeList, + Computed: true, + Description: `The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization. + As a minimum, the IAM roles must have permission to perform a LIST operation on the Amazon S3 bucket to be accessed and a GET operation on the Amazon S3 objects the bucket contains. + If the external database is defined in an Amazon Athena data catalog or the AWS Glue Data Catalog, the IAM role must have permission to access Athena unless catalog_role is specified. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + When you attach a role to your cluster, your cluster can assume that role to access Amazon S3, Athena, and AWS Glue on your behalf. + If a role attached to your cluster doesn't have access to the necessary resources, you can chain another role, possibly belonging to another account. + Your cluster then temporarily assumes the chained role to access the data. You can also grant cross-account access by chaining roles. + You can chain a maximum of 10 roles. Each role in the chain assumes the next role in the chain, until the cluster assumes the role at the end of chain. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles`, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + "secret_arn": { + Type: schema.TypeString, + Description: `The Amazon Resource Name (ARN) of a supported PostgreSQL database engine secret created using AWS Secrets Manager. + For information about how to create and retrieve an ARN for a secret, see https://docs.aws.amazon.com/secretsmanager/latest/userguide/manage_create-basic-secret.html + and https://docs.aws.amazon.com/secretsmanager/latest/userguide/manage_retrieve-secret.html in the AWS Secrets Manager User Guide.`, + Computed: true, + }, + }, + }, + }, + "rds_mysql_source": { + Type: schema.TypeList, + Description: "Configures the external schema to reference data using a federated query to RDS MYSQL or Aurora MySQL.", + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "hostname": { + Type: schema.TypeString, + Description: "The hostname of the head node of the MySQL database replica set.", + Computed: true, + }, + "port": { + Type: schema.TypeInt, + Description: "The port number of the MySQL database. The default port number is 3306.", + Optional: true, + Computed: true, + }, + "iam_role_arns": { + Type: schema.TypeList, + Computed: true, + Description: `The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization. + As a minimum, the IAM roles must have permission to perform a LIST operation on the Amazon S3 bucket to be accessed and a GET operation on the Amazon S3 objects the bucket contains. + If the external database is defined in an Amazon Athena data catalog or the AWS Glue Data Catalog, the IAM role must have permission to access Athena unless catalog_role is specified. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + When you attach a role to your cluster, your cluster can assume that role to access Amazon S3, Athena, and AWS Glue on your behalf. + If a role attached to your cluster doesn't have access to the necessary resources, you can chain another role, possibly belonging to another account. + Your cluster then temporarily assumes the chained role to access the data. You can also grant cross-account access by chaining roles. + You can chain a maximum of 10 roles. Each role in the chain assumes the next role in the chain, until the cluster assumes the role at the end of chain. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles`, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + "secret_arn": { + Type: schema.TypeString, + Description: `The Amazon Resource Name (ARN) of a supported MySQL database engine secret created using AWS Secrets Manager. + For information about how to create and retrieve an ARN for a secret, see https://docs.aws.amazon.com/secretsmanager/latest/userguide/manage_create-basic-secret.html + and https://docs.aws.amazon.com/secretsmanager/latest/userguide/manage_retrieve-secret.html in the AWS Secrets Manager User Guide.`, + Computed: true, + }, + }, + }, + }, + "redshift_source": { + Type: schema.TypeList, + Description: "Configures the external schema to reference datashare database.", + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "schema": { + Type: schema.TypeString, + Description: "The name of the datashare schema. The default schema is 'public'.", + Optional: true, + Computed: true, + }, + }, + }, + }, + }, + }, + }, }, } } func dataSourceRedshiftSchemaRead(db *DBConnection, d *schema.ResourceData) error { - var schemaOwner, schemaId string - var schemaQuota int + var schemaOwner, schemaId, schemaType string + // Step 1: get basic schema info err := db.QueryRow(` SELECT - pg_namespace.oid, - trim(usename), - COALESCE(quota, 0) - FROM pg_namespace - LEFT JOIN svv_schema_quota_state - ON svv_schema_quota_state.schema_id = pg_namespace.oid - LEFT JOIN pg_user_info - ON pg_user_info.usesysid = pg_namespace.nspowner - WHERE pg_namespace.nspname = $1`, d.Get(schemaNameAttr).(string)).Scan(&schemaId, &schemaOwner, &schemaQuota) - + pg_namespace.oid, + trim(pg_user_info.usename), + trim(svv_all_schemas.schema_type) + FROM svv_all_schemas + INNER JOIN pg_namespace ON (svv_all_schemas.database_name = $1 and svv_all_schemas.schema_name = pg_namespace.nspname) + LEFT JOIN pg_user_info + ON (svv_all_schemas.database_name = $1 and pg_user_info.usesysid = svv_all_schemas.schema_owner) + where svv_all_schemas.database_name = $1 + AND svv_all_schemas.schema_name = $2`, db.client.databaseName, d.Get(schemaNameAttr).(string)).Scan(&schemaId, &schemaOwner, &schemaType) if err != nil { return err } - d.SetId(schemaId) d.Set(schemaOwnerAttr, schemaOwner) - d.Set(schemaQuotaAttr, schemaQuota) - return nil + switch { + case schemaType == "local": + return resourceRedshiftSchemaReadLocal(db, d) + case schemaType == "external": + return resourceRedshiftSchemaReadExternal(db, d) + default: + return fmt.Errorf(`Unsupported schema type "%s". Supported types are "local" and "external".`, schemaType) + } } diff --git a/redshift/data_source_redshift_schema_test.go b/redshift/data_source_redshift_schema_test.go index 78b809a..8fbaa87 100644 --- a/redshift/data_source_redshift_schema_test.go +++ b/redshift/data_source_redshift_schema_test.go @@ -2,6 +2,7 @@ package redshift import ( "fmt" + "os" "strings" "testing" @@ -39,3 +40,310 @@ data "redshift_schema" "schema" { } `, schemaNameAttr, schemaName) } + +// Acceptance test for external redshift schema using AWS Glue Data Catalog +// The following environment variables must be set, otherwise the test will be skipped: +// REDSHIFT_EXTERNAL_SCHEMA_DATA_CATALOG_DATABASE - source database name +// REDSHIFT_EXTERNAL_SCHEMA_RDS_DATA_CATALOG_IAM_ROLE_ARNS - comma-separated list of ARNs to use +func TestAccDataSourceRedshiftSchema_ExternalDataCatalog(t *testing.T) { + dbName := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_DATA_CATALOG_DATABASE", t) + iamRoleArnsRaw := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_DATA_CATALOG_IAM_ROLE_ARNS", t) + iamRoleArns := strings.Split(iamRoleArnsRaw, ",") + schemaName := strings.ReplaceAll(acctest.RandomWithPrefix("tf_acc_external_data_schema_data_catalog"), "-", "_") + configCreate := fmt.Sprintf(` +resource "redshift_schema" "spectrum" { + %[1]s = %[2]q + %[3]s { + database_name = %[4]q + data_catalog_source { + iam_role_arns = %[5]s + } + } +} + +data "redshift_schema" "spectrum" { + %[1]s = redshift_schema.spectrum.%[1]s +} +`, + schemaNameAttr, schemaName, schemaExternalSchemaAttr, dbName, tfArray(iamRoleArns)) + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckRedshiftSchemaDestroy, + Steps: []resource.TestStep{ + { + Config: configCreate, + Check: resource.ComposeTestCheckFunc( + testAccCheckRedshiftSchemaExists(schemaName), + resource.TestCheckResourceAttr("data.redshift_schema.spectrum", "name", schemaName), + resource.TestCheckResourceAttr("data.redshift_schema.spectrum", fmt.Sprintf("%s.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("data.redshift_schema.spectrum", fmt.Sprintf("%s.0.database_name", schemaExternalSchemaAttr), dbName), + resource.TestCheckResourceAttr("data.redshift_schema.spectrum", fmt.Sprintf("%s.0.data_catalog_source.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("data.redshift_schema.spectrum", fmt.Sprintf("%s.0.data_catalog_source.0.iam_role_arns.#", schemaExternalSchemaAttr), fmt.Sprintf("%d", len(iamRoleArns))), + resource.ComposeTestCheckFunc(func() []resource.TestCheckFunc { + results := []resource.TestCheckFunc{} + for i, arn := range iamRoleArns { + results = append(results, resource.TestCheckResourceAttr("data.redshift_schema.spectrum", fmt.Sprintf("%s.0.data_catalog_source.0.iam_role_arns.%d", schemaExternalSchemaAttr, i), arn)) + } + return results + }()...), + ), + }, + }, + }) +} + +// Acceptance test for external redshift schema using Hive metastore +// The following environment variables must be set, otherwise the test will be skipped: +// REDSHIFT_EXTERNAL_SCHEMA_HIVE_DATABASE - source database name +// REDSHIFT_EXTERNAL_SCHEMA_HIVE_HOSTNAME - hive metastore database endpoint FQDN or IP address +// REDSHIFT_EXTERNAL_SCHEMA_HIVE_IAM_ROLE_ARNS - comma-separated list of ARNs to use +// Additionally, the following environment variables may be optionally set: +// REDSHIFT_EXTERNAL_SCHEMA_HIVE_PORT - hive metastore port. Default is 9083 +func TestAccDataSourceRedshiftSchema_ExternalHive(t *testing.T) { + dbName := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_HIVE_DATABASE", t) + dbHostname := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_HIVE_HOSTNAME", t) + iamRoleArnsRaw := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_HIVE_IAM_ROLE_ARNS", t) + iamRoleArns := strings.Split(iamRoleArnsRaw, ",") + dbPort := os.Getenv("REDSHIFT_EXTERNAL_SCHEMA_HIVE_PORT") + if dbPort == "" { + dbPort = "9083" + } + schemaName := strings.ReplaceAll(acctest.RandomWithPrefix("tf_acc_external_data_schema_hive"), "-", "_") + configCreate := fmt.Sprintf(` +resource "redshift_schema" "hive" { + %[1]s = %[2]q + %[3]s { + database_name = %[4]q + hive_metastore_source { + hostname = %[5]q + port = %[6]s + iam_role_arns = %[7]s + } + } +} + +data "redshift_schema" "hive" { + %[1]s = redshift_schema.hive.%[1]s +} +`, + schemaNameAttr, schemaName, schemaExternalSchemaAttr, dbName, dbHostname, dbPort, tfArray(iamRoleArns)) + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckRedshiftSchemaDestroy, + Steps: []resource.TestStep{ + { + Config: configCreate, + Check: resource.ComposeTestCheckFunc( + testAccCheckRedshiftSchemaExists(schemaName), + resource.TestCheckResourceAttr("data.redshift_schema.hive", "name", schemaName), + resource.TestCheckResourceAttr("data.redshift_schema.hive", fmt.Sprintf("%s.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("data.redshift_schema.hive", fmt.Sprintf("%s.0.database_name", schemaExternalSchemaAttr), dbName), + resource.TestCheckResourceAttr("data.redshift_schema.hive", fmt.Sprintf("%s.0.hive_metastore_source.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("data.redshift_schema.hive", fmt.Sprintf("%s.0.hive_metastore_source.0.hostname", schemaExternalSchemaAttr), dbHostname), + resource.TestCheckResourceAttr("data.redshift_schema.hive", fmt.Sprintf("%s.0.hive_metastore_source.0.port", schemaExternalSchemaAttr), dbPort), + resource.TestCheckResourceAttr("data.redshift_schema.hive", fmt.Sprintf("%s.0.hive_metastore_source.0.iam_role_arns.#", schemaExternalSchemaAttr), fmt.Sprintf("%d", len(iamRoleArns))), + resource.ComposeTestCheckFunc(func() []resource.TestCheckFunc { + results := []resource.TestCheckFunc{} + for i, arn := range iamRoleArns { + results = append(results, resource.TestCheckResourceAttr("data.redshift_schema.hive", fmt.Sprintf("%s.0.hive_metastore_source.0.iam_role_arns.%d", schemaExternalSchemaAttr, i), arn)) + } + return results + }()...), + ), + }, + }, + }) +} + +// Acceptance test for external redshift schema using RDS Postgres +// The following environment variables must be set, otherwise the test will be skipped: +// REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_DATABASE - source database name +// REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_HOSTNAME - RDS endpoint FQDN or IP address +// REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_IAM_ROLE_ARNS - comma-separated list of ARNs to use +// REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_SECRET_ARN - ARN of the secret in Secrets Manager containing credentials for authenticating to RDS +// Additionally, the following environment variables may be optionally set: +// REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_PORT - RDS port. Default is 5432 +// REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_SCHEMA - source database schema. Default is "public" +func TestAccDataSourceRedshiftSchema_ExternalRdsPostgres(t *testing.T) { + dbName := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_DATABASE", t) + dbHostname := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_HOSTNAME", t) + iamRoleArnsRaw := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_IAM_ROLE_ARNS", t) + iamRoleArns := strings.Split(iamRoleArnsRaw, ",") + dbSecretArn := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_SECRET_ARN", t) + dbPort := os.Getenv("REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_PORT") + if dbPort == "" { + dbPort = "5432" + } + dbSchema := os.Getenv("REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_SCHEMA") + if dbSchema == "" { + dbSchema = "public" + } + schemaName := strings.ReplaceAll(acctest.RandomWithPrefix("tf_acc_external_data_schema_rds_pg"), "-", "_") + configCreate := fmt.Sprintf(` +resource "redshift_schema" "postgres" { + %[1]s = %[2]q + %[3]s { + database_name = %[4]q + rds_postgres_source { + hostname = %[5]q + port = %[6]s + schema = %[7]q + iam_role_arns = %[8]s + secret_arn = %[9]q + } + } +} + +data "redshift_schema" "postgres" { + %[1]s = redshift_schema.postgres.%[1]s +} +`, + schemaNameAttr, schemaName, schemaExternalSchemaAttr, dbName, dbHostname, dbPort, dbSchema, tfArray(iamRoleArns), dbSecretArn) + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckRedshiftSchemaDestroy, + Steps: []resource.TestStep{ + { + Config: configCreate, + Check: resource.ComposeTestCheckFunc( + testAccCheckRedshiftSchemaExists(schemaName), + resource.TestCheckResourceAttr("data.redshift_schema.postgres", "name", schemaName), + resource.TestCheckResourceAttr("data.redshift_schema.postgres", fmt.Sprintf("%s.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("data.redshift_schema.postgres", fmt.Sprintf("%s.0.database_name", schemaExternalSchemaAttr), dbName), + resource.TestCheckResourceAttr("data.redshift_schema.postgres", fmt.Sprintf("%s.0.rds_postgres_source.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("data.redshift_schema.postgres", fmt.Sprintf("%s.0.rds_postgres_source.0.hostname", schemaExternalSchemaAttr), dbHostname), + resource.TestCheckResourceAttr("data.redshift_schema.postgres", fmt.Sprintf("%s.0.rds_postgres_source.0.port", schemaExternalSchemaAttr), dbPort), + resource.TestCheckResourceAttr("data.redshift_schema.postgres", fmt.Sprintf("%s.0.rds_postgres_source.0.schema", schemaExternalSchemaAttr), dbSchema), + resource.TestCheckResourceAttr("data.redshift_schema.postgres", fmt.Sprintf("%s.0.rds_postgres_source.0.secret_arn", schemaExternalSchemaAttr), dbSecretArn), + resource.TestCheckResourceAttr("data.redshift_schema.postgres", fmt.Sprintf("%s.0.rds_postgres_source.0.iam_role_arns.#", schemaExternalSchemaAttr), fmt.Sprintf("%d", len(iamRoleArns))), + resource.ComposeTestCheckFunc(func() []resource.TestCheckFunc { + results := []resource.TestCheckFunc{} + for i, arn := range iamRoleArns { + results = append(results, resource.TestCheckResourceAttr("data.redshift_schema.postgres", fmt.Sprintf("%s.0.rds_postgres_source.0.iam_role_arns.%d", schemaExternalSchemaAttr, i), arn)) + } + return results + }()...), + ), + }, + }, + }) +} + +// Acceptance test for external redshift schema using RDS Mysql +// The following environment variables must be set, otherwise the test will be skipped: +// REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_DATABASE - source database name +// REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_HOSTNAME - RDS endpoint FQDN or IP address +// REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_IAM_ROLE_ARNS - comma-separated list of ARNs to use +// REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_SECRET_ARN - ARN of the secret in Secrets Manager containing credentials for authenticating to RDS +// Additionally, the following environment variables may be optionally set: +// REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_PORT - RDS port. Default is 3306 +func TestAccDataSourceRedshiftSchema_ExternalRdsMysql(t *testing.T) { + dbName := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_DATABASE", t) + dbHostname := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_HOSTNAME", t) + iamRoleArnsRaw := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_IAM_ROLE_ARNS", t) + iamRoleArns := strings.Split(iamRoleArnsRaw, ",") + dbSecretArn := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_SECRET_ARN", t) + dbPort := os.Getenv("REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_PORT") + if dbPort == "" { + dbPort = "3306" + } + schemaName := strings.ReplaceAll(acctest.RandomWithPrefix("tf_acc_external_data_schema_rds_mysql"), "-", "_") + configCreate := fmt.Sprintf(` +resource "redshift_schema" "mysql" { + %[1]s = %[2]q + %[3]s { + database_name = %[4]q + rds_mysql_source { + hostname = %[5]q + port = %[6]s + iam_role_arns = %[7]s + secret_arn = %[8]q + } + } +} + +data "redshift_schema" "mysql" { + %[1]s = redshift_schema.mysql.%[1]s +} +`, + schemaNameAttr, schemaName, schemaExternalSchemaAttr, dbName, dbHostname, dbPort, tfArray(iamRoleArns), dbSecretArn) + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckRedshiftSchemaDestroy, + Steps: []resource.TestStep{ + { + Config: configCreate, + Check: resource.ComposeTestCheckFunc( + testAccCheckRedshiftSchemaExists(schemaName), + resource.TestCheckResourceAttr("data.redshift_schema.mysql", "name", schemaName), + resource.TestCheckResourceAttr("data.redshift_schema.mysql", fmt.Sprintf("%s.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("data.redshift_schema.mysql", fmt.Sprintf("%s.0.database_name", schemaExternalSchemaAttr), dbName), + resource.TestCheckResourceAttr("data.redshift_schema.mysql", fmt.Sprintf("%s.0.rds_mysql_source.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("data.redshift_schema.mysql", fmt.Sprintf("%s.0.rds_mysql_source.0.hostname", schemaExternalSchemaAttr), dbHostname), + resource.TestCheckResourceAttr("data.redshift_schema.mysql", fmt.Sprintf("%s.0.rds_mysql_source.0.port", schemaExternalSchemaAttr), dbPort), + resource.TestCheckResourceAttr("data.redshift_schema.mysql", fmt.Sprintf("%s.0.rds_mysql_source.0.secret_arn", schemaExternalSchemaAttr), dbSecretArn), + resource.TestCheckResourceAttr("data.redshift_schema.mysql", fmt.Sprintf("%s.0.rds_mysql_source.0.iam_role_arns.#", schemaExternalSchemaAttr), fmt.Sprintf("%d", len(iamRoleArns))), + resource.ComposeTestCheckFunc(func() []resource.TestCheckFunc { + results := []resource.TestCheckFunc{} + for i, arn := range iamRoleArns { + results = append(results, resource.TestCheckResourceAttr("data.redshift_schema.mysql", fmt.Sprintf("%s.0.rds_mysql_source.0.iam_role_arns.%d", schemaExternalSchemaAttr, i), arn)) + } + return results + }()...), + ), + }, + }, + }) +} + +// Acceptance test for external redshift schema using datashare database +// The following environment variables must be set, otherwise the test will be skipped: +// REDSHIFT_EXTERNAL_SCHEMA_REDSHIFT_DATABASE - source database name +// Additionally, the following environment variables may be optionally set: +// REDSHIFT_EXTERNAL_SCHEMA_REDSHIFT_SCHEMA - datashare schema name. Default is "public" +func TestAccDataSourceRedshiftSchema_ExternalRedshift(t *testing.T) { + dbName := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_REDSHIFT_DATABASE", t) + dbSchema := os.Getenv("REDSHIFT_EXTERNAL_SCHEMA_REDSHIFT_SCHEMA") + if dbSchema == "" { + dbSchema = "public" + } + schemaName := strings.ReplaceAll(acctest.RandomWithPrefix("tf_acc_external_data_schema_redshift"), "-", "_") + configCreate := fmt.Sprintf(` +resource "redshift_schema" "redshift" { + %[1]s = %[2]q + %[3]s { + database_name = %[4]q + redshift_source { + schema = %[5]q + } + } +} + +data "redshift_schema" "redshift" { + %[1]s = redshift_schema.redshift.%[1]s +} +`, + schemaNameAttr, schemaName, schemaExternalSchemaAttr, dbName, dbSchema) + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckRedshiftSchemaDestroy, + Steps: []resource.TestStep{ + { + Config: configCreate, + Check: resource.ComposeTestCheckFunc( + testAccCheckRedshiftSchemaExists(schemaName), + resource.TestCheckResourceAttr("data.redshift_schema.redshift", "name", schemaName), + resource.TestCheckResourceAttr("data.redshift_schema.redshift", fmt.Sprintf("%s.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("data.redshift_schema.redshift", fmt.Sprintf("%s.0.database_name", schemaExternalSchemaAttr), dbName), + resource.TestCheckResourceAttr("data.redshift_schema.redshift", fmt.Sprintf("%s.0.redshift_source.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("data.redshift_schema.redshift", fmt.Sprintf("%s.0.redshift_source.0.schema", schemaExternalSchemaAttr), dbSchema), + ), + }, + }, + }) +} diff --git a/redshift/helpers.go b/redshift/helpers.go index 127b427..4d9ae94 100644 --- a/redshift/helpers.go +++ b/redshift/helpers.go @@ -2,6 +2,7 @@ package redshift import ( "database/sql" + "encoding/csv" "fmt" "log" "strings" @@ -126,3 +127,22 @@ func isRetryablePQError(code string) bool { _, ok := retryable[code] return ok } + +func splitCsvAndTrim(raw string) ([]string, error) { + if raw == "" { + return []string{}, nil + } + reader := csv.NewReader(strings.NewReader(raw)) + rawSlice, err := reader.Read() + if err != nil { + return nil, err + } + result := []string{} + for _, s := range rawSlice { + trimmed := strings.TrimSpace(s) + if trimmed != "" { + result = append(result, trimmed) + } + } + return result, nil +} diff --git a/redshift/provider_test.go b/redshift/provider_test.go index 28f797e..8a20f44 100644 --- a/redshift/provider_test.go +++ b/redshift/provider_test.go @@ -46,10 +46,7 @@ func testAccPreCheck(t *testing.T) { } func initTemporaryCredentialsProvider(t *testing.T, provider *schema.Provider) { - var clusterIdentifier string - if clusterIdentifier = os.Getenv("REDSHIFT_CLUSTER_IDENTIFIER"); clusterIdentifier == "" { - t.Skip("REDSHIFT_CLUSTER_IDENTIFIER must be set for acceptance tests") - } + clusterIdentifier := getEnvOrSkip("REDSHIFT_CLUSTER_IDENTIFIER", t) sdkClient, err := stsClient(t) if err != nil { diff --git a/redshift/resource_redshift_schema.go b/redshift/resource_redshift_schema.go index 8c2fd0f..f9b5740 100644 --- a/redshift/resource_redshift_schema.go +++ b/redshift/resource_redshift_schema.go @@ -3,8 +3,11 @@ package redshift import ( "database/sql" "fmt" + "log" + "strconv" "strings" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/customdiff" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" "github.com/lib/pq" @@ -15,6 +18,12 @@ const ( schemaOwnerAttr = "owner" schemaQuotaAttr = "quota" schemaCascadeOnDeleteAttr = "cascade_on_delete" + schemaExternalSchemaAttr = "external_schema" + dataCatalogAttr = "external_schema.0.data_catalog_source.0" + hiveMetastoreAttr = "external_schema.0.hive_metastore_source.0" + rdsPostgresAttr = "external_schema.0.rds_postgres_source.0" + rdsMysqlAttr = "external_schema.0.rds_mysql_source.0" + redshiftAttr = "external_schema.0.redshift_source.0" ) func redshiftSchema() *schema.Resource { @@ -32,7 +41,7 @@ A database contains one or more named schemas. Each schema in a database contain Importer: &schema.ResourceImporter{ State: schema.ImportStatePassthrough, }, - + CustomizeDiff: forceNewIfListSizeChanged(schemaExternalSchemaAttr), Schema: map[string]*schema.Schema{ schemaNameAttr: { Type: schema.TypeString, @@ -63,12 +72,324 @@ A database contains one or more named schemas. Each schema in a database contain StateFunc: func(val interface{}) string { return fmt.Sprintf("%d", val.(int)*1024) }, + ConflictsWith: []string{ + schemaExternalSchemaAttr, + }, }, schemaCascadeOnDeleteAttr: { Type: schema.TypeBool, Optional: true, - Default: false, Description: "Indicates to automatically drop all objects in the schema. The default action is TO NOT drop a schema if it contains any objects.", + ConflictsWith: []string{ + schemaExternalSchemaAttr, + }, + }, + schemaExternalSchemaAttr: { + Type: schema.TypeList, + Optional: true, + Description: "Configures the schema as an external schema. See https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_SCHEMA.html", + MaxItems: 1, + ConflictsWith: []string{ + schemaQuotaAttr, + schemaCascadeOnDeleteAttr, + }, + Elem: &schema.Resource{ + CustomizeDiff: customdiff.All( + forceNewIfListSizeChanged("data_catalog_source"), + forceNewIfListSizeChanged("hive_metastore_source"), + forceNewIfListSizeChanged("rds_postgres_source"), + forceNewIfListSizeChanged("rds_mysql_source"), + forceNewIfListSizeChanged("redshift_source"), + ), + Schema: map[string]*schema.Schema{ + "database_name": { + Type: schema.TypeString, + Required: true, + Description: "The database where the external schema can be found", + ForceNew: true, + }, + "data_catalog_source": { + Type: schema.TypeList, + Description: "Configures the external schema from the AWS Glue Data Catalog", + Optional: true, + MaxItems: 1, + ConflictsWith: []string{ + fmt.Sprintf("%s.0.hive_metastore_source", schemaExternalSchemaAttr), + fmt.Sprintf("%s.0.rds_postgres_source", schemaExternalSchemaAttr), + fmt.Sprintf("%s.0.rds_mysql_source", schemaExternalSchemaAttr), + fmt.Sprintf("%s.0.redshift_source", schemaExternalSchemaAttr), + }, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "region": { + Type: schema.TypeString, + Optional: true, + Description: "If the external database is defined in an Athena data catalog or the AWS Glue Data Catalog, the AWS Region in which the database is located. This parameter is required if the database is defined in an external Data Catalog.", + ForceNew: true, + }, + "iam_role_arns": { + Type: schema.TypeList, + Required: true, + MinItems: 1, + MaxItems: 10, + Description: `The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization. + As a minimum, the IAM roles must have permission to perform a LIST operation on the Amazon S3 bucket to be accessed and a GET operation on the Amazon S3 objects the bucket contains. + If the external database is defined in an Amazon Athena data catalog or the AWS Glue Data Catalog, the IAM role must have permission to access Athena unless catalog_role is specified. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + When you attach a role to your cluster, your cluster can assume that role to access Amazon S3, Athena, and AWS Glue on your behalf. + If a role attached to your cluster doesn't have access to the necessary resources, you can chain another role, possibly belonging to another account. + Your cluster then temporarily assumes the chained role to access the data. You can also grant cross-account access by chaining roles. + You can chain a maximum of 10 roles. Each role in the chain assumes the next role in the chain, until the cluster assumes the role at the end of chain. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles`, + ForceNew: true, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + "catalog_role_arns": { + Type: schema.TypeList, + Optional: true, + MinItems: 1, + MaxItems: 10, + Description: `The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization for the data catalog. + If this is not specified, Amazon Redshift uses the specified iam_role_arns. The catalog role must have permission to access the Data Catalog in AWS Glue or Athena. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles`, + ForceNew: true, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + "create_external_database_if_not_exists": { + Type: schema.TypeBool, + Optional: true, + Default: false, + DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { + return true // this attribute only applies at create time so we should never flag it as a change + }, + Description: `When enabled, creates an external database with the name specified by the database argument, + if the specified external database doesn't exist. If the specified external database exists, the command makes no changes. + In this case, the command returns a message that the external database exists, rather than terminating with an error. + + To use create_external_database_if_not_exists with a Data Catalog enabled for AWS Lake Formation, you need CREATE_DATABASE permission on the Data Catalog.`, + }, + }, + }, + }, + "hive_metastore_source": { + Type: schema.TypeList, + Description: "Configures the external schema from a Hive Metastore.", + Optional: true, + MaxItems: 1, + ConflictsWith: []string{ + fmt.Sprintf("%s.0.data_catalog_source", schemaExternalSchemaAttr), + fmt.Sprintf("%s.0.rds_postgres_source", schemaExternalSchemaAttr), + fmt.Sprintf("%s.0.rds_mysql_source", schemaExternalSchemaAttr), + fmt.Sprintf("%s.0.redshift_source", schemaExternalSchemaAttr), + }, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "hostname": { + Type: schema.TypeString, + Description: "The hostname of the hive metastore database.", + Required: true, + ForceNew: true, + }, + "port": { + Type: schema.TypeInt, + Description: "The port number of the hive metastore. The default port number is 9083.", + Optional: true, + Default: 9083, + ValidateFunc: validation.IntBetween(1, 65535), + ForceNew: true, + }, + "iam_role_arns": { + Type: schema.TypeList, + Required: true, + MinItems: 1, + MaxItems: 10, + Description: `The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization. + As a minimum, the IAM roles must have permission to perform a LIST operation on the Amazon S3 bucket to be accessed and a GET operation on the Amazon S3 objects the bucket contains. + If the external database is defined in an Amazon Athena data catalog or the AWS Glue Data Catalog, the IAM role must have permission to access Athena unless catalog_role is specified. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + When you attach a role to your cluster, your cluster can assume that role to access Amazon S3, Athena, and AWS Glue on your behalf. + If a role attached to your cluster doesn't have access to the necessary resources, you can chain another role, possibly belonging to another account. + Your cluster then temporarily assumes the chained role to access the data. You can also grant cross-account access by chaining roles. + You can chain a maximum of 10 roles. Each role in the chain assumes the next role in the chain, until the cluster assumes the role at the end of chain. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles`, + ForceNew: true, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + }, + }, + }, + "rds_postgres_source": { + Type: schema.TypeList, + Description: "Configures the external schema to reference data using a federated query to RDS POSTGRES or Aurora PostgreSQL.", + Optional: true, + MaxItems: 1, + ConflictsWith: []string{ + fmt.Sprintf("%s.0.data_catalog_source", schemaExternalSchemaAttr), + fmt.Sprintf("%s.0.hive_metastore_source", schemaExternalSchemaAttr), + fmt.Sprintf("%s.0.rds_mysql_source", schemaExternalSchemaAttr), + fmt.Sprintf("%s.0.redshift_source", schemaExternalSchemaAttr), + }, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "hostname": { + Type: schema.TypeString, + Description: "The hostname of the head node of the PostgreSQL database replica set.", + Required: true, + ForceNew: true, + }, + "port": { + Type: schema.TypeInt, + Description: "The port number of the PostgreSQL database. The default port number is 5432.", + Optional: true, + Default: 5432, + ValidateFunc: validation.IntBetween(1, 65535), + ForceNew: true, + }, + "schema": { + Type: schema.TypeString, + Description: "The name of the PostgreSQL schema. The default schema is 'public'", + Optional: true, + Default: "public", + ForceNew: true, + }, + "iam_role_arns": { + Type: schema.TypeList, + Required: true, + MinItems: 1, + MaxItems: 10, + Description: `The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization. + As a minimum, the IAM roles must have permission to perform a LIST operation on the Amazon S3 bucket to be accessed and a GET operation on the Amazon S3 objects the bucket contains. + If the external database is defined in an Amazon Athena data catalog or the AWS Glue Data Catalog, the IAM role must have permission to access Athena unless catalog_role is specified. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + When you attach a role to your cluster, your cluster can assume that role to access Amazon S3, Athena, and AWS Glue on your behalf. + If a role attached to your cluster doesn't have access to the necessary resources, you can chain another role, possibly belonging to another account. + Your cluster then temporarily assumes the chained role to access the data. You can also grant cross-account access by chaining roles. + You can chain a maximum of 10 roles. Each role in the chain assumes the next role in the chain, until the cluster assumes the role at the end of chain. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles`, + ForceNew: true, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + "secret_arn": { + Type: schema.TypeString, + Description: `The Amazon Resource Name (ARN) of a supported PostgreSQL database engine secret created using AWS Secrets Manager. + For information about how to create and retrieve an ARN for a secret, see https://docs.aws.amazon.com/secretsmanager/latest/userguide/manage_create-basic-secret.html + and https://docs.aws.amazon.com/secretsmanager/latest/userguide/manage_retrieve-secret.html in the AWS Secrets Manager User Guide.`, + Required: true, + ForceNew: true, + }, + }, + }, + }, + "rds_mysql_source": { + Type: schema.TypeList, + Description: "Configures the external schema to reference data using a federated query to RDS MYSQL or Aurora MySQL.", + Optional: true, + MaxItems: 1, + ConflictsWith: []string{ + fmt.Sprintf("%s.0.data_catalog_source", schemaExternalSchemaAttr), + fmt.Sprintf("%s.0.hive_metastore_source", schemaExternalSchemaAttr), + fmt.Sprintf("%s.0.rds_postgres_source", schemaExternalSchemaAttr), + fmt.Sprintf("%s.0.redshift_source", schemaExternalSchemaAttr), + }, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "hostname": { + Type: schema.TypeString, + Description: "The hostname of the head node of the MySQL database replica set.", + Required: true, + ForceNew: true, + }, + "port": { + Type: schema.TypeInt, + Description: "The port number of the MySQL database. The default port number is 3306.", + Optional: true, + Default: 3306, + ValidateFunc: validation.IntBetween(1, 65535), + ForceNew: true, + }, + "iam_role_arns": { + Type: schema.TypeList, + Required: true, + MinItems: 1, + MaxItems: 10, + Description: `The Amazon Resource Name (ARN) for the IAM roles that your cluster uses for authentication and authorization. + As a minimum, the IAM roles must have permission to perform a LIST operation on the Amazon S3 bucket to be accessed and a GET operation on the Amazon S3 objects the bucket contains. + If the external database is defined in an Amazon Athena data catalog or the AWS Glue Data Catalog, the IAM role must have permission to access Athena unless catalog_role is specified. + For more information, see https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-iam-policies.html. + + When you attach a role to your cluster, your cluster can assume that role to access Amazon S3, Athena, and AWS Glue on your behalf. + If a role attached to your cluster doesn't have access to the necessary resources, you can chain another role, possibly belonging to another account. + Your cluster then temporarily assumes the chained role to access the data. You can also grant cross-account access by chaining roles. + You can chain a maximum of 10 roles. Each role in the chain assumes the next role in the chain, until the cluster assumes the role at the end of chain. + + To chain roles, you establish a trust relationship between the roles. A role that assumes another role must have a permissions policy that allows it to assume the specified role. + In turn, the role that passes permissions must have a trust policy that allows it to pass its permissions to another role. + For more information, see https://docs.aws.amazon.com/redshift/latest/mgmt/authorizing-redshift-service.html#authorizing-redshift-service-chaining-roles`, + ForceNew: true, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + "secret_arn": { + Type: schema.TypeString, + Description: `The Amazon Resource Name (ARN) of a supported MySQL database engine secret created using AWS Secrets Manager. + For information about how to create and retrieve an ARN for a secret, see https://docs.aws.amazon.com/secretsmanager/latest/userguide/manage_create-basic-secret.html + and https://docs.aws.amazon.com/secretsmanager/latest/userguide/manage_retrieve-secret.html in the AWS Secrets Manager User Guide.`, + Required: true, + ForceNew: true, + }, + }, + }, + }, + "redshift_source": { + Type: schema.TypeList, + Description: "Configures the external schema to reference datashare database.", + Optional: true, + MaxItems: 1, + ConflictsWith: []string{ + fmt.Sprintf("%s.0.data_catalog_source", schemaExternalSchemaAttr), + fmt.Sprintf("%s.0.hive_metastore_source", schemaExternalSchemaAttr), + fmt.Sprintf("%s.0.rds_postgres_source", schemaExternalSchemaAttr), + fmt.Sprintf("%s.0.rds_mysql_source", schemaExternalSchemaAttr), + }, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "schema": { + Type: schema.TypeString, + Description: "The name of the datashare schema. The default schema is 'public'.", + Optional: true, + Default: "public", + ForceNew: true, + }, + }, + }, + }, + }, + }, }, }, } @@ -93,28 +414,154 @@ func resourceRedshiftSchemaRead(db *DBConnection, d *schema.ResourceData) error } func resourceRedshiftSchemaReadImpl(db *DBConnection, d *schema.ResourceData) error { - var schemaOwner, schemaName string - var schemaQuota int + var schemaOwner, schemaName, schemaType string + // Step 1: get basic schema info err := db.QueryRow(` - SELECT - trim(nspname), - trim(usename), - COALESCE(quota, 0) - FROM pg_namespace - LEFT JOIN svv_schema_quota_state - ON svv_schema_quota_state.schema_id = pg_namespace.oid - LEFT JOIN pg_user_info - ON pg_user_info.usesysid = pg_namespace.nspowner - WHERE pg_namespace.oid = $1`, d.Id()).Scan(&schemaName, &schemaOwner, &schemaQuota) - + SELECT + trim(svv_all_schemas.schema_name), + trim(pg_user_info.usename), + trim(svv_all_schemas.schema_type) + FROM svv_all_schemas + INNER JOIN pg_namespace ON (svv_all_schemas.database_name = $1 and svv_all_schemas.schema_name = pg_namespace.nspname) + LEFT JOIN pg_user_info + ON (svv_all_schemas.database_name = $1 and pg_user_info.usesysid = svv_all_schemas.schema_owner) + where svv_all_schemas.database_name = $1 + AND pg_namespace.oid = $2`, db.client.databaseName, d.Id()).Scan(&schemaName, &schemaOwner, &schemaType) if err != nil { return err } - d.Set(schemaNameAttr, schemaName) d.Set(schemaOwnerAttr, schemaOwner) + switch { + case schemaType == "local": + return resourceRedshiftSchemaReadLocal(db, d) + case schemaType == "external": + return resourceRedshiftSchemaReadExternal(db, d) + default: + return fmt.Errorf(`Unsupported schema type "%s". Supported types are "local" and "external".`, schemaType) + } +} + +func resourceRedshiftSchemaReadLocal(db *DBConnection, d *schema.ResourceData) error { + var schemaQuota int + + err := db.QueryRow(` + SELECT + COALESCE(quota, 0) + FROM svv_schema_quota_state + WHERE schema_id = $1 + `, d.Id()).Scan(&schemaQuota) + switch { + case err == sql.ErrNoRows: + schemaQuota = 0 + case err != nil: + return err + } + d.Set(schemaQuotaAttr, schemaQuota) + d.Set(schemaExternalSchemaAttr, nil) + + return nil +} + +func resourceRedshiftSchemaReadExternal(db *DBConnection, d *schema.ResourceData) error { + var sourceType, sourceDbName, iamRole, catalogRole, region, sourceSchema, hostName, port, secretArn string + err := db.QueryRow(` + SELECT + CASE + WHEN eskind = 1 THEN 'data_catalog_source' + WHEN eskind = 2 THEN 'hive_metastore_source' + WHEN eskind = 3 THEN 'rds_postgres_source' + WHEN eskind = 4 THEN 'redshift_source' + WHEN eskind = 7 THEN 'rds_mysql_source' + ELSE 'unknown' + END, + trim(databasename), + COALESCE(CASE WHEN is_valid_json(esoptions) THEN json_extract_path_text(esoptions, 'IAM_ROLE') END, ''), + COALESCE(CASE WHEN is_valid_json(esoptions) THEN json_extract_path_text(esoptions, 'CATALOG_ROLE') END, ''), + COALESCE(CASE WHEN is_valid_json(esoptions) THEN json_extract_path_text(esoptions, 'REGION') END, ''), + COALESCE(CASE WHEN is_valid_json(esoptions) THEN json_extract_path_text(esoptions, 'SCHEMA') END, ''), + COALESCE(CASE WHEN is_valid_json(esoptions) THEN json_extract_path_text(esoptions, 'URI') END, ''), + COALESCE(CASE WHEN is_valid_json(esoptions) THEN json_extract_path_text(esoptions, 'PORT') END, ''), + COALESCE(CASE WHEN is_valid_json(esoptions) THEN json_extract_path_text(esoptions, 'SECRET_ARN') END, '') + FROM + svv_external_schemas + WHERE + esoid = $1`, d.Id()).Scan(&sourceType, &sourceDbName, &iamRole, &catalogRole, ®ion, &sourceSchema, &hostName, &port, &secretArn) + + if err != nil { + return err + } + externalSchemaConfiguration := make(map[string]interface{}) + sourceConfiguration := make(map[string]interface{}) + externalSchemaConfiguration["database_name"] = &sourceDbName + switch { + case sourceType == "data_catalog_source": + sourceConfiguration["region"] = ®ion + sourceConfiguration["iam_role_arns"], err = splitCsvAndTrim(iamRole) + if err != nil { + return fmt.Errorf("Error parsing iam_role_arns: %v", err) + } + sourceConfiguration["catalog_role_arns"], err = splitCsvAndTrim(catalogRole) + if err != nil { + return fmt.Errorf("Error parsing catalog_role_arns: %v", err) + } + case sourceType == "hive_metastore_source": + sourceConfiguration["hostname"] = &hostName + if port != "" { + portNum, err := strconv.Atoi(port) + if err != nil { + return fmt.Errorf("hive_metastore_source port was not an integer") + } + sourceConfiguration["port"] = &portNum + } + sourceConfiguration["iam_role_arns"], err = splitCsvAndTrim(iamRole) + if err != nil { + return fmt.Errorf("Error parsing iam_role_arns: %v", err) + } + case sourceType == "rds_postgres_source": + sourceConfiguration["hostname"] = &hostName + if port != "" { + portNum, err := strconv.Atoi(port) + if err != nil { + return fmt.Errorf("rds_postgres_source port was not an integer") + } + sourceConfiguration["port"] = &portNum + } + if sourceSchema != "" { + sourceConfiguration["schema"] = &sourceSchema + } + sourceConfiguration["iam_role_arns"], err = splitCsvAndTrim(iamRole) + if err != nil { + return fmt.Errorf("Error parsing iam_role_arns: %v", err) + } + sourceConfiguration["secret_arn"] = &secretArn + case sourceType == "rds_mysql_source": + sourceConfiguration["hostname"] = &hostName + if port != "" { + portNum, err := strconv.Atoi(port) + if err != nil { + return fmt.Errorf("rds_mysql_source port was not an integer") + } + sourceConfiguration["port"] = &portNum + } + sourceConfiguration["iam_role_arns"], err = splitCsvAndTrim(iamRole) + if err != nil { + return fmt.Errorf("Error parsing iam_role_arns: %v", err) + } + sourceConfiguration["secret_arn"] = &secretArn + case sourceType == "redshift_source": + if sourceSchema != "" { + sourceConfiguration["schema"] = &sourceSchema + } + default: + return fmt.Errorf(`Unsupported source database type %s`, sourceType) + } + externalSchemaConfiguration[sourceType] = []map[string]interface{}{sourceConfiguration} + + d.Set(schemaQuotaAttr, 0) + d.Set(schemaExternalSchemaAttr, []map[string]interface{}{externalSchemaConfiguration}) return nil } @@ -132,8 +579,8 @@ func resourceRedshiftSchemaDelete(db *DBConnection, d *schema.ResourceData) erro cascade_or_restrict = "CASCADE" } - sql := fmt.Sprintf("DROP SCHEMA %s %s", pq.QuoteIdentifier(schemaName), cascade_or_restrict) - if _, err := tx.Exec(sql); err != nil { + query := fmt.Sprintf("DROP SCHEMA %s %s", pq.QuoteIdentifier(schemaName), cascade_or_restrict) + if _, err := tx.Exec(query); err != nil { return err } @@ -147,6 +594,23 @@ func resourceRedshiftSchemaCreate(db *DBConnection, d *schema.ResourceData) erro } defer deferredRollback(tx) + if _, isExternal := d.GetOk(fmt.Sprintf("%s.0.%s", schemaExternalSchemaAttr, "database_name")); isExternal { + err = resourceRedshiftSchemaCreateExternal(tx, d) + } else { + err = resourceRedshiftSchemaCreateInternal(tx, d) + } + if err != nil { + return err + } + + if err = tx.Commit(); err != nil { + return fmt.Errorf("could not commit transaction: %w", err) + } + + return resourceRedshiftSchemaReadImpl(db, d) +} + +func resourceRedshiftSchemaCreateInternal(tx *sql.Tx, d *schema.ResourceData) error { schemaName := d.Get(schemaNameAttr).(string) schemaQuota := d.Get(schemaQuotaAttr).(int) createOpts := []string{} @@ -161,9 +625,9 @@ func resourceRedshiftSchemaCreate(db *DBConnection, d *schema.ResourceData) erro } createOpts = append(createOpts, quotaValue) - sql := fmt.Sprintf("CREATE SCHEMA %s %s", pq.QuoteIdentifier(schemaName), strings.Join(createOpts, " ")) + query := fmt.Sprintf("CREATE SCHEMA %s %s", pq.QuoteIdentifier(schemaName), strings.Join(createOpts, " ")) - if _, err := tx.Exec(sql); err != nil { + if _, err := tx.Exec(query); err != nil { return err } @@ -174,11 +638,145 @@ func resourceRedshiftSchemaCreate(db *DBConnection, d *schema.ResourceData) erro d.SetId(schemaOID) - if err = tx.Commit(); err != nil { - return fmt.Errorf("could not commit transaction: %w", err) + return nil +} + +func resourceRedshiftSchemaCreateExternal(tx *sql.Tx, d *schema.ResourceData) error { + schemaName := d.Get(schemaNameAttr).(string) + query := fmt.Sprintf("CREATE EXTERNAL SCHEMA %s", pq.QuoteIdentifier(schemaName)) + sourceDbName := d.Get(fmt.Sprintf("%s.0.%s", schemaExternalSchemaAttr, "database_name")).(string) + var configQuery string + if _, isDataCatalog := d.GetOk(dataCatalogAttr); isDataCatalog { + // data catalog source + configQuery = getDataCatalogConfigQueryPart(d, sourceDbName) + } else if _, isHiveMetastore := d.GetOk(hiveMetastoreAttr); isHiveMetastore { + // hive metastore source + configQuery = getHiveMetastoreConfigQueryPart(d, sourceDbName) + } else if _, isRdsPostgres := d.GetOk(rdsPostgresAttr); isRdsPostgres { + // rds postgres source + configQuery = getRdsPostgresConfigQueryPart(d, sourceDbName) + } else if _, isRdsMysql := d.GetOk(rdsMysqlAttr); isRdsMysql { + // rds mysql source + configQuery = getRdsMysqlConfigQueryPart(d, sourceDbName) + } else if _, isRedshift := d.GetOk(redshiftAttr); isRedshift { + // redshift source + configQuery = getRedshiftConfigQueryPart(d, sourceDbName) + } else { + return fmt.Errorf("Can't create external schema. No source configuration found.") } - return resourceRedshiftSchemaReadImpl(db, d) + query = fmt.Sprintf("%s %s", query, configQuery) + + log.Printf("[DEBUG] creating external schema: %s\n", query) + if _, err := tx.Exec(query); err != nil { + return err + } + + if v, ok := d.GetOk(schemaOwnerAttr); ok { + query = fmt.Sprintf("ALTER SCHEMA %s OWNER TO %s", pq.QuoteIdentifier(schemaName), pq.QuoteIdentifier(v.(string))) + log.Printf("[DEBUG] setting schema owner: %s\n", query) + if _, err := tx.Exec(query); err != nil { + return err + } + } + + var schemaOID string + if err := tx.QueryRow("SELECT oid FROM pg_namespace WHERE nspname = $1", strings.ToLower(schemaName)).Scan(&schemaOID); err != nil { + return err + } + + d.SetId(schemaOID) + + return nil +} + +func getDataCatalogConfigQueryPart(d *schema.ResourceData, sourceDbName string) string { + query := fmt.Sprintf("FROM DATA CATALOG DATABASE '%s'", pqQuoteLiteral(sourceDbName)) + if region, hasRegion := d.GetOk(fmt.Sprintf("%s.%s", dataCatalogAttr, "region")); hasRegion { + query = fmt.Sprintf("%s REGION '%s'", query, pqQuoteLiteral(region.(string))) + } + iamRoleArnsRaw := d.Get(fmt.Sprintf("%s.%s", dataCatalogAttr, "iam_role_arns")).([]interface{}) + iamRoleArns := []string{} + for _, arn := range iamRoleArnsRaw { + iamRoleArns = append(iamRoleArns, arn.(string)) + } + query = fmt.Sprintf("%s IAM_ROLE '%s'", query, pqQuoteLiteral(strings.Join(iamRoleArns, ","))) + if catalogRoleArnsRaw, hasCatalogRoleArns := d.GetOk(fmt.Sprintf("%s.%s", dataCatalogAttr, "catalog_role_arns")); hasCatalogRoleArns { + catalogRoleArns := []string{} + for _, arn := range catalogRoleArnsRaw.([]interface{}) { + catalogRoleArns = append(catalogRoleArns, arn.(string)) + } + if len(catalogRoleArns) > 0 { + query = fmt.Sprintf("%s CATALOG_ROLE '%s'", query, pqQuoteLiteral(strings.Join(catalogRoleArns, ","))) + } + } + if d.Get(fmt.Sprintf("%s.%s", dataCatalogAttr, "create_external_database_if_not_exists")).(bool) { + query = fmt.Sprintf("%s CREATE EXTERNAL DATABASE IF NOT EXISTS", query) + } + return query +} + +func getHiveMetastoreConfigQueryPart(d *schema.ResourceData, sourceDbName string) string { + query := fmt.Sprintf("FROM HIVE METASTORE DATABASE '%s'", pqQuoteLiteral(sourceDbName)) + hostName := d.Get(fmt.Sprintf("%s.%s", hiveMetastoreAttr, "hostname")).(string) + query = fmt.Sprintf("%s URI '%s'", query, pqQuoteLiteral(hostName)) + if port, portIsSet := d.GetOk(fmt.Sprintf("%s.%s", hiveMetastoreAttr, "port")); portIsSet { + query = fmt.Sprintf("%s PORT %d", query, port.(int)) + } + iamRoleArnsRaw := d.Get(fmt.Sprintf("%s.%s", hiveMetastoreAttr, "iam_role_arns")).([]interface{}) + iamRoleArns := []string{} + for _, arn := range iamRoleArnsRaw { + iamRoleArns = append(iamRoleArns, arn.(string)) + } + query = fmt.Sprintf("%s IAM_ROLE '%s'", query, pqQuoteLiteral(strings.Join(iamRoleArns, ","))) + return query +} + +func getRdsPostgresConfigQueryPart(d *schema.ResourceData, sourceDbName string) string { + query := fmt.Sprintf("FROM POSTGRES DATABASE '%s'", pqQuoteLiteral(sourceDbName)) + if sourceSchema, sourceSchemaIsSet := d.GetOk(fmt.Sprintf("%s.%s", rdsPostgresAttr, "schema")); sourceSchemaIsSet { + query = fmt.Sprintf("%s SCHEMA '%s'", query, pqQuoteLiteral(sourceSchema.(string))) + } + hostName := d.Get(fmt.Sprintf("%s.%s", rdsPostgresAttr, "hostname")).(string) + query = fmt.Sprintf("%s URI '%s'", query, pqQuoteLiteral(hostName)) + if port, portIsSet := d.GetOk(fmt.Sprintf("%s.%s", rdsPostgresAttr, "port")); portIsSet { + query = fmt.Sprintf("%s PORT %d", query, port.(int)) + } + iamRoleArnsRaw := d.Get(fmt.Sprintf("%s.%s", rdsPostgresAttr, "iam_role_arns")).([]interface{}) + iamRoleArns := []string{} + for _, arn := range iamRoleArnsRaw { + iamRoleArns = append(iamRoleArns, arn.(string)) + } + query = fmt.Sprintf("%s IAM_ROLE '%s'", query, pqQuoteLiteral(strings.Join(iamRoleArns, ","))) + secretArn := d.Get(fmt.Sprintf("%s.%s", rdsPostgresAttr, "secret_arn")).(string) + query = fmt.Sprintf("%s SECRET_ARN '%s'", query, pqQuoteLiteral(secretArn)) + return query +} + +func getRdsMysqlConfigQueryPart(d *schema.ResourceData, sourceDbName string) string { + query := fmt.Sprintf("FROM MYSQL DATABASE '%s'", pqQuoteLiteral(sourceDbName)) + hostName := d.Get(fmt.Sprintf("%s.%s", rdsMysqlAttr, "hostname")).(string) + query = fmt.Sprintf("%s URI '%s'", query, pqQuoteLiteral(hostName)) + if port, portIsSet := d.GetOk(fmt.Sprintf("%s.%s", rdsMysqlAttr, "port")); portIsSet { + query = fmt.Sprintf("%s PORT %d", query, port.(int)) + } + iamRoleArnsRaw := d.Get(fmt.Sprintf("%s.%s", rdsMysqlAttr, "iam_role_arns")).([]interface{}) + iamRoleArns := []string{} + for _, arn := range iamRoleArnsRaw { + iamRoleArns = append(iamRoleArns, arn.(string)) + } + query = fmt.Sprintf("%s IAM_ROLE '%s'", query, pqQuoteLiteral(strings.Join(iamRoleArns, ","))) + secretArn := d.Get(fmt.Sprintf("%s.%s", rdsMysqlAttr, "secret_arn")).(string) + query = fmt.Sprintf("%s SECRET_ARN '%s'", query, pqQuoteLiteral(secretArn)) + return query +} + +func getRedshiftConfigQueryPart(d *schema.ResourceData, sourceDbName string) string { + query := fmt.Sprintf("FROM REDSHIFT DATABASE '%s'", pqQuoteLiteral(sourceDbName)) + if sourceSchema, sourceSchemaIsSet := d.GetOk(fmt.Sprintf("%s.%s", redshiftAttr, "schema")); sourceSchemaIsSet { + query = fmt.Sprintf("%s SCHEMA '%s'", query, pqQuoteLiteral(sourceSchema.(string))) + } + return query } func resourceRedshiftSchemaUpdate(db *DBConnection, d *schema.ResourceData) error { @@ -220,8 +818,8 @@ func setSchemaName(tx *sql.Tx, d *schema.ResourceData) error { return fmt.Errorf("Error setting schema name to an empty string") } - sql := fmt.Sprintf("ALTER SCHEMA %s RENAME TO %s", pq.QuoteIdentifier(oldValue), pq.QuoteIdentifier(newValue)) - if _, err := tx.Exec(sql); err != nil { + query := fmt.Sprintf("ALTER SCHEMA %s RENAME TO %s", pq.QuoteIdentifier(oldValue), pq.QuoteIdentifier(newValue)) + if _, err := tx.Exec(query); err != nil { return fmt.Errorf("Error updating schema NAME: %w", err) } diff --git a/redshift/resource_redshift_schema_test.go b/redshift/resource_redshift_schema_test.go index 728d120..17357de 100644 --- a/redshift/resource_redshift_schema_test.go +++ b/redshift/resource_redshift_schema_test.go @@ -3,9 +3,11 @@ package redshift import ( "database/sql" "fmt" + "os" "strings" "testing" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" ) @@ -147,6 +149,330 @@ resource "redshift_user" "schema_dl_user1" { }) } +// Acceptance test for external redshift schema using AWS Glue Data Catalog +// The following environment variables must be set, otherwise the test will be skipped: +// REDSHIFT_EXTERNAL_SCHEMA_DATA_CATALOG_DATABASE - source database name +// REDSHIFT_EXTERNAL_SCHEMA_DATA_CATALOG_IAM_ROLE_ARNS - comma-separated list of ARNs to use +func TestAccRedshiftSchema_ExternalDataCatalog(t *testing.T) { + dbName := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_DATA_CATALOG_DATABASE", t) + iamRoleArnsRaw := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_DATA_CATALOG_IAM_ROLE_ARNS", t) + iamRoleArns, err := splitCsvAndTrim(iamRoleArnsRaw) + if err != nil { + t.Errorf("REDSHIFT_EXTERNAL_SCHEMA_DATA_CATALOG_IAM_ROLE_ARNS could not be parsed: %v", err) + } + schemaName := strings.ReplaceAll(acctest.RandomWithPrefix("tf_acc_external_schema_data_catalog"), "-", "_") + configCreate := fmt.Sprintf(` +resource "redshift_schema" "spectrum" { + %[1]s = %[2]q + %[3]s { + database_name = %[4]q + data_catalog_source { + iam_role_arns = %[5]s + } + } +} +`, + schemaNameAttr, schemaName, schemaExternalSchemaAttr, dbName, tfArray(iamRoleArns)) + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckRedshiftSchemaDestroy, + Steps: []resource.TestStep{ + { + Config: configCreate, + Check: resource.ComposeTestCheckFunc( + testAccCheckRedshiftSchemaExists(schemaName), + resource.TestCheckResourceAttr("redshift_schema.spectrum", "name", schemaName), + resource.TestCheckResourceAttr("redshift_schema.spectrum", fmt.Sprintf("%s.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("redshift_schema.spectrum", fmt.Sprintf("%s.0.database_name", schemaExternalSchemaAttr), dbName), + resource.TestCheckResourceAttr("redshift_schema.spectrum", fmt.Sprintf("%s.0.data_catalog_source.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("redshift_schema.spectrum", fmt.Sprintf("%s.0.data_catalog_source.0.iam_role_arns.#", schemaExternalSchemaAttr), fmt.Sprintf("%d", len(iamRoleArns))), + resource.ComposeTestCheckFunc(func() []resource.TestCheckFunc { + results := []resource.TestCheckFunc{} + for i, arn := range iamRoleArns { + results = append(results, resource.TestCheckResourceAttr("redshift_schema.spectrum", fmt.Sprintf("%s.0.data_catalog_source.0.iam_role_arns.%d", schemaExternalSchemaAttr, i), arn)) + } + return results + }()...), + ), + }, + { + ResourceName: "redshift_schema.spectrum", + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +// Acceptance test for external redshift schema using Hive metastore +// The following environment variables must be set, otherwise the test will be skipped: +// REDSHIFT_EXTERNAL_SCHEMA_HIVE_DATABASE - source database name +// REDSHIFT_EXTERNAL_SCHEMA_HIVE_HOSTNAME - hive metastore database endpoint FQDN or IP address +// REDSHIFT_EXTERNAL_SCHEMA_HIVE_IAM_ROLE_ARNS - comma-separated list of ARNs to use +// Additionally, the following environment variables may be optionally set: +// REDSHIFT_EXTERNAL_SCHEMA_HIVE_PORT - hive metastore port. Default is 9083 +func TestAccRedshiftSchema_ExternalHive(t *testing.T) { + dbName := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_HIVE_DATABASE", t) + dbHostname := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_HIVE_HOSTNAME", t) + iamRoleArnsRaw := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_HIVE_IAM_ROLE_ARNS", t) + iamRoleArns, err := splitCsvAndTrim(iamRoleArnsRaw) + if err != nil { + t.Errorf("REDSHIFT_EXTERNAL_SCHEMA_DATA_CATALOG_IAM_ROLE_ARNS could not be parsed: %v", err) + } + dbPort := os.Getenv("REDSHIFT_EXTERNAL_SCHEMA_HIVE_PORT") + if dbPort == "" { + dbPort = "9083" + } + schemaName := strings.ReplaceAll(acctest.RandomWithPrefix("tf_acc_external_schema_hive"), "-", "_") + configCreate := fmt.Sprintf(` +resource "redshift_schema" "hive" { + %[1]s = %[2]q + %[3]s { + database_name = %[4]q + hive_metastore_source { + hostname = %[5]q + port = %[6]s + iam_role_arns = %[7]s + } + } +} +`, + schemaNameAttr, schemaName, schemaExternalSchemaAttr, dbName, dbHostname, dbPort, tfArray(iamRoleArns)) + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckRedshiftSchemaDestroy, + Steps: []resource.TestStep{ + { + Config: configCreate, + Check: resource.ComposeTestCheckFunc( + testAccCheckRedshiftSchemaExists(schemaName), + resource.TestCheckResourceAttr("redshift_schema.hive", "name", schemaName), + resource.TestCheckResourceAttr("redshift_schema.hive", fmt.Sprintf("%s.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("redshift_schema.hive", fmt.Sprintf("%s.0.database_name", schemaExternalSchemaAttr), dbName), + resource.TestCheckResourceAttr("redshift_schema.hive", fmt.Sprintf("%s.0.hive_metastore_source.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("redshift_schema.hive", fmt.Sprintf("%s.0.hive_metastore_source.0.hostname", schemaExternalSchemaAttr), dbHostname), + resource.TestCheckResourceAttr("redshift_schema.hive", fmt.Sprintf("%s.0.hive_metastore_source.0.port", schemaExternalSchemaAttr), dbPort), + resource.TestCheckResourceAttr("redshift_schema.hive", fmt.Sprintf("%s.0.hive_metastore_source.0.iam_role_arns.#", schemaExternalSchemaAttr), fmt.Sprintf("%d", len(iamRoleArns))), + resource.ComposeTestCheckFunc(func() []resource.TestCheckFunc { + results := []resource.TestCheckFunc{} + for i, arn := range iamRoleArns { + results = append(results, resource.TestCheckResourceAttr("redshift_schema.hive", fmt.Sprintf("%s.0.hive_metastore_source.0.iam_role_arns.%d", schemaExternalSchemaAttr, i), arn)) + } + return results + }()...), + ), + }, + { + ResourceName: "redshift_schema.hive", + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +// Acceptance test for external redshift schema using RDS Postgres +// The following environment variables must be set, otherwise the test will be skipped: +// REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_DATABASE - source database name +// REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_HOSTNAME - RDS endpoint FQDN or IP address +// REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_IAM_ROLE_ARNS - comma-separated list of ARNs to use +// REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_SECRET_ARN - ARN of the secret in Secrets Manager containing credentials for authenticating to RDS +// Additionally, the following environment variables may be optionally set: +// REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_PORT - RDS port. Default is 5432 +// REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_SCHEMA - source database schema. Default is "public" +func TestAccRedshiftSchema_ExternalRdsPostgres(t *testing.T) { + dbName := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_DATABASE", t) + dbHostname := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_HOSTNAME", t) + iamRoleArnsRaw := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_IAM_ROLE_ARNS", t) + iamRoleArns, err := splitCsvAndTrim(iamRoleArnsRaw) + if err != nil { + t.Errorf("REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_IAM_ROLE_ARNS could not be parsed: %v", err) + } + dbSecretArn := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_SECRET_ARN", t) + dbPort := os.Getenv("REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_PORT") + if dbPort == "" { + dbPort = "5432" + } + dbSchema := os.Getenv("REDSHIFT_EXTERNAL_SCHEMA_RDS_POSTGRES_SCHEMA") + if dbSchema == "" { + dbSchema = "public" + } + schemaName := strings.ReplaceAll(acctest.RandomWithPrefix("tf_acc_external_schema_rds_pg"), "-", "_") + configCreate := fmt.Sprintf(` +resource "redshift_schema" "postgres" { + %[1]s = %[2]q + %[3]s { + database_name = %[4]q + rds_postgres_source { + hostname = %[5]q + port = %[6]s + schema = %[7]q + iam_role_arns = %[8]s + secret_arn = %[9]q + } + } +} +`, + schemaNameAttr, schemaName, schemaExternalSchemaAttr, dbName, dbHostname, dbPort, dbSchema, tfArray(iamRoleArns), dbSecretArn) + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckRedshiftSchemaDestroy, + Steps: []resource.TestStep{ + { + Config: configCreate, + Check: resource.ComposeTestCheckFunc( + testAccCheckRedshiftSchemaExists(schemaName), + resource.TestCheckResourceAttr("redshift_schema.postgres", "name", schemaName), + resource.TestCheckResourceAttr("redshift_schema.postgres", fmt.Sprintf("%s.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("redshift_schema.postgres", fmt.Sprintf("%s.0.database_name", schemaExternalSchemaAttr), dbName), + resource.TestCheckResourceAttr("redshift_schema.postgres", fmt.Sprintf("%s.0.rds_postgres_source.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("redshift_schema.postgres", fmt.Sprintf("%s.0.rds_postgres_source.0.hostname", schemaExternalSchemaAttr), dbHostname), + resource.TestCheckResourceAttr("redshift_schema.postgres", fmt.Sprintf("%s.0.rds_postgres_source.0.port", schemaExternalSchemaAttr), dbPort), + resource.TestCheckResourceAttr("redshift_schema.postgres", fmt.Sprintf("%s.0.rds_postgres_source.0.schema", schemaExternalSchemaAttr), dbSchema), + resource.TestCheckResourceAttr("redshift_schema.postgres", fmt.Sprintf("%s.0.rds_postgres_source.0.secret_arn", schemaExternalSchemaAttr), dbSecretArn), + resource.TestCheckResourceAttr("redshift_schema.postgres", fmt.Sprintf("%s.0.rds_postgres_source.0.iam_role_arns.#", schemaExternalSchemaAttr), fmt.Sprintf("%d", len(iamRoleArns))), + resource.ComposeTestCheckFunc(func() []resource.TestCheckFunc { + results := []resource.TestCheckFunc{} + for i, arn := range iamRoleArns { + results = append(results, resource.TestCheckResourceAttr("redshift_schema.postgres", fmt.Sprintf("%s.0.rds_postgres_source.0.iam_role_arns.%d", schemaExternalSchemaAttr, i), arn)) + } + return results + }()...), + ), + }, + { + ResourceName: "redshift_schema.postgres", + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +// Acceptance test for external redshift schema using RDS Mysql +// The following environment variables must be set, otherwise the test will be skipped: +// REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_DATABASE - source database name +// REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_HOSTNAME - RDS endpoint FQDN or IP address +// REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_IAM_ROLE_ARNS - comma-separated list of ARNs to use +// REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_SECRET_ARN - ARN of the secret in Secrets Manager containing credentials for authenticating to RDS +// Additionally, the following environment variables may be optionally set: +// REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_PORT - RDS port. Default is 3306 +func TestAccRedshiftSchema_ExternalRdsMysql(t *testing.T) { + dbName := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_DATABASE", t) + dbHostname := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_HOSTNAME", t) + iamRoleArnsRaw := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_IAM_ROLE_ARNS", t) + iamRoleArns, err := splitCsvAndTrim(iamRoleArnsRaw) + if err != nil { + t.Errorf("REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_IAM_ROLE_ARNS could not be parsed: %v", err) + } + dbSecretArn := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_SECRET_ARN", t) + dbPort := os.Getenv("REDSHIFT_EXTERNAL_SCHEMA_RDS_MYSQL_PORT") + if dbPort == "" { + dbPort = "3306" + } + schemaName := strings.ReplaceAll(acctest.RandomWithPrefix("tf_acc_external_schema_rds_mysql"), "-", "_") + configCreate := fmt.Sprintf(` +resource "redshift_schema" "mysql" { + %[1]s = %[2]q + %[3]s { + database_name = %[4]q + rds_mysql_source { + hostname = %[5]q + port = %[6]s + iam_role_arns = %[7]s + secret_arn = %[8]q + } + } +} +`, + schemaNameAttr, schemaName, schemaExternalSchemaAttr, dbName, dbHostname, dbPort, tfArray(iamRoleArns), dbSecretArn) + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckRedshiftSchemaDestroy, + Steps: []resource.TestStep{ + { + Config: configCreate, + Check: resource.ComposeTestCheckFunc( + testAccCheckRedshiftSchemaExists(schemaName), + resource.TestCheckResourceAttr("redshift_schema.mysql", "name", schemaName), + resource.TestCheckResourceAttr("redshift_schema.mysql", fmt.Sprintf("%s.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("redshift_schema.mysql", fmt.Sprintf("%s.0.database_name", schemaExternalSchemaAttr), dbName), + resource.TestCheckResourceAttr("redshift_schema.mysql", fmt.Sprintf("%s.0.rds_mysql_source.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("redshift_schema.mysql", fmt.Sprintf("%s.0.rds_mysql_source.0.hostname", schemaExternalSchemaAttr), dbHostname), + resource.TestCheckResourceAttr("redshift_schema.mysql", fmt.Sprintf("%s.0.rds_mysql_source.0.port", schemaExternalSchemaAttr), dbPort), + resource.TestCheckResourceAttr("redshift_schema.mysql", fmt.Sprintf("%s.0.rds_mysql_source.0.secret_arn", schemaExternalSchemaAttr), dbSecretArn), + resource.TestCheckResourceAttr("redshift_schema.mysql", fmt.Sprintf("%s.0.rds_mysql_source.0.iam_role_arns.#", schemaExternalSchemaAttr), fmt.Sprintf("%d", len(iamRoleArns))), + resource.ComposeTestCheckFunc(func() []resource.TestCheckFunc { + results := []resource.TestCheckFunc{} + for i, arn := range iamRoleArns { + results = append(results, resource.TestCheckResourceAttr("redshift_schema.mysql", fmt.Sprintf("%s.0.rds_mysql_source.0.iam_role_arns.%d", schemaExternalSchemaAttr, i), arn)) + } + return results + }()...), + ), + }, + { + ResourceName: "redshift_schema.mysql", + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +// Acceptance test for external redshift schema using datashare database +// The following environment variables must be set, otherwise the test will be skipped: +// REDSHIFT_EXTERNAL_SCHEMA_REDSHIFT_DATABASE - source database name +// Additionally, the following environment variables may be optionally set: +// REDSHIFT_EXTERNAL_SCHEMA_REDSHIFT_SCHEMA - datashare schema name. Default is "public" +func TestAccRedshiftSchema_ExternalRedshift(t *testing.T) { + dbName := getEnvOrSkip("REDSHIFT_EXTERNAL_SCHEMA_REDSHIFT_DATABASE", t) + dbSchema := os.Getenv("REDSHIFT_EXTERNAL_SCHEMA_REDSHIFT_SCHEMA") + if dbSchema == "" { + dbSchema = "public" + } + schemaName := strings.ReplaceAll(acctest.RandomWithPrefix("tf_acc_external_schema_redshift"), "-", "_") + configCreate := fmt.Sprintf(` +resource "redshift_schema" "redshift" { + %[1]s = %[2]q + %[3]s { + database_name = %[4]q + redshift_source { + schema = %[5]q + } + } +} +`, + schemaNameAttr, schemaName, schemaExternalSchemaAttr, dbName, dbSchema) + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckRedshiftSchemaDestroy, + Steps: []resource.TestStep{ + { + Config: configCreate, + Check: resource.ComposeTestCheckFunc( + testAccCheckRedshiftSchemaExists(schemaName), + resource.TestCheckResourceAttr("redshift_schema.redshift", "name", schemaName), + resource.TestCheckResourceAttr("redshift_schema.redshift", fmt.Sprintf("%s.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("redshift_schema.redshift", fmt.Sprintf("%s.0.database_name", schemaExternalSchemaAttr), dbName), + resource.TestCheckResourceAttr("redshift_schema.redshift", fmt.Sprintf("%s.0.redshift_source.#", schemaExternalSchemaAttr), "1"), + resource.TestCheckResourceAttr("redshift_schema.redshift", fmt.Sprintf("%s.0.redshift_source.0.schema", schemaExternalSchemaAttr), dbSchema), + ), + }, + { + ResourceName: "redshift_schema.redshift", + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + func testAccCheckRedshiftSchemaDestroy(s *terraform.State) error { client := testAccProvider.Meta().(*Client)