diff --git a/client/client.go b/client/client.go index 4e940cdc6..b7cafbefb 100644 --- a/client/client.go +++ b/client/client.go @@ -48,6 +48,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/elasticsearchservice" "github.com/aws/aws-sdk-go-v2/service/emr" "github.com/aws/aws-sdk-go-v2/service/eventbridge" + "github.com/aws/aws-sdk-go-v2/service/firehose" "github.com/aws/aws-sdk-go-v2/service/fsx" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/guardduty" @@ -157,6 +158,7 @@ type Services struct { ELBv2 ElbV2Client EMR EmrClient EventBridge EventBridgeClient + Firehose FirehoseClient FSX FsxClient Glue GlueClient GuardDuty GuardDutyClient @@ -659,6 +661,7 @@ func initServices(region string, c aws.Config) Services { ELBv2: elbv2.NewFromConfig(awsCfg), EMR: emr.NewFromConfig(awsCfg), EventBridge: eventbridge.NewFromConfig(awsCfg), + Firehose: firehose.NewFromConfig(awsCfg), FSX: fsx.NewFromConfig(awsCfg), Glue: glue.NewFromConfig(awsCfg), GuardDuty: guardduty.NewFromConfig(awsCfg), diff --git a/client/mocks/firehose.go b/client/mocks/firehose.go new file mode 100644 index 000000000..163aee993 --- /dev/null +++ b/client/mocks/firehose.go @@ -0,0 +1,96 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudquery/cq-provider-aws/client (interfaces: FirehoseClient) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + firehose "github.com/aws/aws-sdk-go-v2/service/firehose" + gomock "github.com/golang/mock/gomock" +) + +// MockFirehoseClient is a mock of FirehoseClient interface. +type MockFirehoseClient struct { + ctrl *gomock.Controller + recorder *MockFirehoseClientMockRecorder +} + +// MockFirehoseClientMockRecorder is the mock recorder for MockFirehoseClient. +type MockFirehoseClientMockRecorder struct { + mock *MockFirehoseClient +} + +// NewMockFirehoseClient creates a new mock instance. +func NewMockFirehoseClient(ctrl *gomock.Controller) *MockFirehoseClient { + mock := &MockFirehoseClient{ctrl: ctrl} + mock.recorder = &MockFirehoseClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockFirehoseClient) EXPECT() *MockFirehoseClientMockRecorder { + return m.recorder +} + +// DescribeDeliveryStream mocks base method. +func (m *MockFirehoseClient) DescribeDeliveryStream(arg0 context.Context, arg1 *firehose.DescribeDeliveryStreamInput, arg2 ...func(*firehose.Options)) (*firehose.DescribeDeliveryStreamOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeDeliveryStream", varargs...) + ret0, _ := ret[0].(*firehose.DescribeDeliveryStreamOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeDeliveryStream indicates an expected call of DescribeDeliveryStream. +func (mr *MockFirehoseClientMockRecorder) DescribeDeliveryStream(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeDeliveryStream", reflect.TypeOf((*MockFirehoseClient)(nil).DescribeDeliveryStream), varargs...) +} + +// ListDeliveryStreams mocks base method. +func (m *MockFirehoseClient) ListDeliveryStreams(arg0 context.Context, arg1 *firehose.ListDeliveryStreamsInput, arg2 ...func(*firehose.Options)) (*firehose.ListDeliveryStreamsOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListDeliveryStreams", varargs...) + ret0, _ := ret[0].(*firehose.ListDeliveryStreamsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListDeliveryStreams indicates an expected call of ListDeliveryStreams. +func (mr *MockFirehoseClientMockRecorder) ListDeliveryStreams(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDeliveryStreams", reflect.TypeOf((*MockFirehoseClient)(nil).ListDeliveryStreams), varargs...) +} + +// ListTagsForDeliveryStream mocks base method. +func (m *MockFirehoseClient) ListTagsForDeliveryStream(arg0 context.Context, arg1 *firehose.ListTagsForDeliveryStreamInput, arg2 ...func(*firehose.Options)) (*firehose.ListTagsForDeliveryStreamOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListTagsForDeliveryStream", varargs...) + ret0, _ := ret[0].(*firehose.ListTagsForDeliveryStreamOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListTagsForDeliveryStream indicates an expected call of ListTagsForDeliveryStream. +func (mr *MockFirehoseClientMockRecorder) ListTagsForDeliveryStream(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListTagsForDeliveryStream", reflect.TypeOf((*MockFirehoseClient)(nil).ListTagsForDeliveryStream), varargs...) +} diff --git a/client/services.go b/client/services.go index 9ad2cb72d..7f2d03f38 100644 --- a/client/services.go +++ b/client/services.go @@ -38,6 +38,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/elasticsearchservice" "github.com/aws/aws-sdk-go-v2/service/emr" "github.com/aws/aws-sdk-go-v2/service/eventbridge" + "github.com/aws/aws-sdk-go-v2/service/firehose" "github.com/aws/aws-sdk-go-v2/service/fsx" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/guardduty" @@ -414,6 +415,13 @@ type EventBridgeClient interface { ListTagsForResource(ctx context.Context, params *eventbridge.ListTagsForResourceInput, optFns ...func(*eventbridge.Options)) (*eventbridge.ListTagsForResourceOutput, error) } +//go:generate mockgen -package=mocks -destination=./mocks/firehose.go . FirehoseClient +type FirehoseClient interface { + DescribeDeliveryStream(ctx context.Context, params *firehose.DescribeDeliveryStreamInput, optFns ...func(*firehose.Options)) (*firehose.DescribeDeliveryStreamOutput, error) + ListDeliveryStreams(ctx context.Context, params *firehose.ListDeliveryStreamsInput, optFns ...func(*firehose.Options)) (*firehose.ListDeliveryStreamsOutput, error) + ListTagsForDeliveryStream(ctx context.Context, params *firehose.ListTagsForDeliveryStreamInput, optFns ...func(*firehose.Options)) (*firehose.ListTagsForDeliveryStreamOutput, error) +} + //go:generate mockgen -package=mocks -destination=./mocks/mock_fsx.go . FsxClient type FsxClient interface { DescribeBackups(ctx context.Context, params *fsx.DescribeBackupsInput, optFns ...func(*fsx.Options)) (*fsx.DescribeBackupsOutput, error) diff --git a/docs/tables/aws_firehose_delivery_stream_elasticsearch_destination.md b/docs/tables/aws_firehose_delivery_stream_elasticsearch_destination.md new file mode 100644 index 000000000..ed84fa380 --- /dev/null +++ b/docs/tables/aws_firehose_delivery_stream_elasticsearch_destination.md @@ -0,0 +1,38 @@ + +# Table: aws_firehose_delivery_stream_elasticsearch_destination +The destination description in Amazon ES +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|delivery_stream_cq_id|uuid|Unique CloudQuery ID of aws_firehose_delivery_streams table (FK)| +|processing_configuration_processors|jsonb|Describes a data processing configuration| +|buffering_hints_interval_in_seconds|bigint|Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination| +|buffering_hints_size_in_mb_s|bigint|Buffer incoming data to the specified size, in MBs, before delivering it to the destination| +|cloud_watch_logging_options_enabled|boolean|Enables or disables CloudWatch logging| +|cloud_watch_logging_options_log_group_name|text|The CloudWatch group name for logging| +|cloud_watch_logging_options_log_stream_name|text|The CloudWatch log stream name for logging| +|cluster_endpoint|text|The endpoint to use when communicating with the cluster| +|domain_arn|text|The ARN of the Amazon ES domain| +|index_name|text|The Elasticsearch index name| +|index_rotation_period|text|The Elasticsearch index rotation period| +|processing_configuration_enabled|boolean|Enables or disables data processing| +|retry_options_duration_in_seconds|bigint|After an initial failure to deliver to Amazon ES, the total amount of time during which Kinesis Data Firehose retries delivery (including the first attempt)| +|role_arn|text|The Amazon Resource Name (ARN) of the AWS credentials| +|s3_backup_mode|text|The Amazon S3 backup mode| +|s3_destination_bucket_arn|text|The ARN of the S3 bucket| +|s3_destination_buffering_hints_interval_in_seconds|bigint|Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination| +|s3_destination_buffering_hints_size_in_mb_s|bigint|Buffer incoming data to the specified size, in MiBs, before delivering it to the destination| +|s3_destination_compression_format|text|The compression format| +|s3_destination_kms_encryption_config_aws_kms_key_arn|text|The Amazon Resource Name (ARN) of the encryption key| +|s3_destination_no_encryption_config|text|Specifically override existing encryption information to ensure that no encryption is used| +|s3_destination_role_arn|text|The Amazon Resource Name (ARN) of the AWS credentials| +|s3_destination_cloud_watch_logging_options_enabled|boolean|Enables or disables CloudWatch logging| +|s3_destination_cloud_watch_logging_options_log_group_name|text|The CloudWatch group name for logging| +|s3_destination_cloud_watch_logging_options_log_stream_name|text|The CloudWatch log stream name for logging| +|s3_destination_error_output_prefix|text|A prefix that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3| +|s3_destination_prefix|text|The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered Amazon S3 files| +|type_name|text|The Elasticsearch type name| +|vpc_configuration_description_role_arn|text|The ARN of the IAM role that the delivery stream uses to create endpoints in the destination VPC| +|vpc_configuration_description_security_group_ids|text[]|The IDs of the security groups that Kinesis Data Firehose uses when it creates ENIs in the VPC of the Amazon ES destination| +|vpc_configuration_description_subnet_ids|text[]|The IDs of the subnets that Kinesis Data Firehose uses to create ENIs in the VPC of the Amazon ES destination| +|vpc_configuration_description_vpc_id|text|The ID of the Amazon ES destination's VPC| diff --git a/docs/tables/aws_firehose_delivery_stream_extended_s3_destination.md b/docs/tables/aws_firehose_delivery_stream_extended_s3_destination.md new file mode 100644 index 000000000..e10dd9500 --- /dev/null +++ b/docs/tables/aws_firehose_delivery_stream_extended_s3_destination.md @@ -0,0 +1,63 @@ + +# Table: aws_firehose_delivery_stream_extended_s3_destination +Describes a destination in Amazon S3 +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|delivery_stream_cq_id|uuid|Unique CloudQuery ID of aws_firehose_delivery_streams table (FK)| +|processing_configuration_processors|jsonb|Describes a data processing configuration| +|bucket_arn|text|The ARN of the S3 bucket| +|buffering_hints_interval_in_seconds|bigint|Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination| +|buffering_hints_size_in_mb_s|bigint|Buffer incoming data to the specified size, in MiBs, before delivering it to the destination| +|compression_format|text|The compression format| +|encryption_configuration_kms_encryption_config_aws_kms_key_arn|text|The Amazon Resource Name (ARN) of the encryption key| +|encryption_configuration_no_encryption_config|text|Specifically override existing encryption information to ensure that no encryption is used| +|role_arn|text|The Amazon Resource Name (ARN) of the AWS credentials| +|cloud_watch_logging_options_enabled|boolean|Enables or disables CloudWatch logging| +|cloud_watch_logging_options_log_group_name|text|The CloudWatch group name for logging| +|cloud_watch_logging_options_log_stream_name|text|The CloudWatch log stream name for logging| +|enabled|boolean|Defaults to true| +|deserializer_hive_json_ser_de_timestamp_formats|text[]|Indicates how you want Kinesis Data Firehose to parse the date and timestamps that may be present in your input data JSON| +|deserializer_open_x_json_ser_de_case_insensitive|boolean|When set to true, which is the default, Kinesis Data Firehose converts JSON keys to lowercase before deserializing them| +|deserializer_open_x_json_ser_de_column_to_json_key_mappings|jsonb|Maps column names to JSON keys that aren't identical to the column names| +|deserializer_open_x_json_ser_de_convert_dots_to_underscores|boolean|When set to true, specifies that the names of the keys include dots and that you want Kinesis Data Firehose to replace them with underscores| +|serializer_orc_ser_de_block_size_bytes|bigint|The Hadoop Distributed File System (HDFS) block size| +|serializer_orc_ser_de_bloom_filter_columns|text[]|The column names for which you want Kinesis Data Firehose to create bloom filters| +|serializer_orc_ser_de_bloom_filter_false_positive_probability|float|The Bloom filter false positive probability (FPP)| +|serializer_orc_ser_de_compression|text|The compression code to use over data blocks| +|serializer_orc_ser_de_dictionary_key_threshold|float|Represents the fraction of the total number of non-null rows| +|serializer_orc_ser_de_enable_padding|boolean|Set this to true to indicate that you want stripes to be padded to the HDFS block boundaries| +|serializer_orc_ser_de_format_version|text|The version of the file to write| +|serializer_orc_ser_de_padding_tolerance|float|A number between 0 and 1 that defines the tolerance for block padding as a decimal fraction of stripe size| +|serializer_orc_ser_de_row_index_stride|bigint|The number of rows between index entries| +|serializer_orc_ser_de_stripe_size_bytes|bigint|The number of bytes in each stripe| +|serializer_parquet_ser_de_block_size_bytes|bigint|The Hadoop Distributed File System (HDFS) block size| +|serializer_parquet_ser_de_compression|text|The compression code to use over data blocks| +|serializer_parquet_ser_de_enable_dictionary_compression|boolean|Indicates whether to enable dictionary compression| +|serializer_parquet_ser_de_max_padding_bytes|bigint|The maximum amount of padding to apply| +|serializer_parquet_ser_de_page_size_bytes|bigint|The Parquet page size| +|serializer_parquet_ser_de_writer_version|text|Indicates the version of row format to output| +|schema_configuration_catalog_id|text|The ID of the AWS Glue Data Catalog| +|schema_configuration_database_name|text|Specifies the name of the AWS Glue database that contains the schema for the output data| +|schema_configuration_region|text|If you don't specify an AWS Region, the default is the current Region| +|schema_configuration_role_arn|text|The role that Kinesis Data Firehose can use to access AWS Glue| +|schema_configuration_table_name|text|Specifies the AWS Glue table that contains the column information that constitutes your data schema| +|schema_configuration_version_id|text|Specifies the table version for the output data schema| +|dynamic_partitioning_enabled|boolean|Specifies that the dynamic partitioning is enabled for this Kinesis Data Firehose delivery stream| +|dynamic_partitioning_retry_options_duration_in_seconds|bigint|The period of time during which Kinesis Data Firehose retries to deliver data to the specified Amazon S3 prefix| +|error_output_prefix|text|A prefix that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3| +|prefix|text|The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered Amazon S3 files| +|processing_configuration_enabled|boolean|Enables or disables data processing| +|s3_backup_bucket_arn|text|The ARN of the S3 bucket| +|s3_backup_buffering_hints_interval_in_seconds|bigint|Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination| +|s3_backup_buffering_hints_size_in_mb_s|bigint|Buffer incoming data to the specified size, in MiBs, before delivering it to the destination| +|s3_backup_compression_format|text|The compression format| +|s3_backup_kms_encryption_config_aws_kms_key_arn|text|The Amazon Resource Name (ARN) of the encryption key| +|s3_backup_no_encryption_config|text|Specifically override existing encryption information to ensure that no encryption is used| +|s3_backup_role_arn|text|The Amazon Resource Name (ARN) of the AWS credentials| +|s3_backup_cloud_watch_logging_options_enabled|boolean|Enables or disables CloudWatch logging| +|s3_backup_cloud_watch_logging_options_log_group_name|text|The CloudWatch group name for logging| +|s3_backup_cloud_watch_logging_options_log_stream_name|text|The CloudWatch log stream name for logging| +|s3_backup_error_output_prefix|text|A prefix that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3| +|s3_backup_prefix|text|The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered Amazon S3 files| +|s3_backup_mode|text|The Amazon S3 backup mode| diff --git a/docs/tables/aws_firehose_delivery_stream_http_destination.md b/docs/tables/aws_firehose_delivery_stream_http_destination.md new file mode 100644 index 000000000..c0500b568 --- /dev/null +++ b/docs/tables/aws_firehose_delivery_stream_http_destination.md @@ -0,0 +1,33 @@ + +# Table: aws_firehose_delivery_stream_http_destination +Describes the HTTP endpoint destination +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|delivery_stream_cq_id|uuid|Unique CloudQuery ID of aws_firehose_delivery_streams table (FK)| +|processing_configuration_processors|jsonb|Describes a data processing configuration| +|buffering_hints_interval_in_seconds|bigint|Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination| +|buffering_hints_size_in_mb_s|bigint|Buffer incoming data to the specified size, in MBs, before delivering it to the destination| +|cloud_watch_logging_options_enabled|boolean|Enables or disables CloudWatch logging| +|cloud_watch_logging_options_log_group_name|text|The CloudWatch group name for logging| +|cloud_watch_logging_options_log_stream_name|text|The CloudWatch log stream name for logging| +|endpoint_configuration_name|text|The name of the HTTP endpoint selected as the destination| +|endpoint_configuration_url|text|The URL of the HTTP endpoint selected as the destination| +|processing_configuration_enabled|boolean|Enables or disables data processing| +|request_configuration_common_attributes|jsonb|Describes the metadata sent to the HTTP endpoint destination| +|request_configuration_content_encoding|text|Kinesis Data Firehose uses the content encoding to compress the body of a request before sending the request to the destination| +|retry_options_duration_in_seconds|bigint|The total amount of time that Kinesis Data Firehose spends on retries| +|role_arn|text|Kinesis Data Firehose uses this IAM role for all the permissions that the delivery stream needs| +|s3_backup_mode|text|Describes the S3 bucket backup options for the data that Kinesis Firehose delivers to the HTTP endpoint destination| +|s3_destination_bucket_arn|text|The ARN of the S3 bucket| +|s3_destination_buffering_hints_interval_in_seconds|bigint|Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination| +|s3_destination_buffering_hints_size_in_mb_s|bigint|Buffer incoming data to the specified size, in MiBs, before delivering it to the destination| +|s3_destination_compression_format|text|The compression format| +|s3_destination_kms_encryption_config_aws_kms_key_arn|text|The Amazon Resource Name (ARN) of the encryption key| +|s3_destination_no_encryption_config|text|Specifically override existing encryption information to ensure that no encryption is used| +|s3_destination_role_arn|text|The Amazon Resource Name (ARN) of the AWS credentials| +|s3_destination_cloud_watch_logging_options_enabled|boolean|Enables or disables CloudWatch logging| +|s3_destination_cloud_watch_logging_options_log_group_name|text|The CloudWatch group name for logging| +|s3_destination_cloud_watch_logging_options_log_stream_name|text|The CloudWatch log stream name for logging| +|s3_destination_error_output_prefix|text|A prefix that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3| +|s3_destination_prefix|text|The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered Amazon S3 files| diff --git a/docs/tables/aws_firehose_delivery_stream_open_search_destination.md b/docs/tables/aws_firehose_delivery_stream_open_search_destination.md new file mode 100644 index 000000000..c851083b0 --- /dev/null +++ b/docs/tables/aws_firehose_delivery_stream_open_search_destination.md @@ -0,0 +1,38 @@ + +# Table: aws_firehose_delivery_stream_open_search_destination + +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|delivery_stream_cq_id|uuid|Unique CloudQuery ID of aws_firehose_delivery_streams table (FK)| +|processing_configuration_processors|jsonb|Describes a data processing configuration| +|buffering_hints_interval_in_seconds|bigint|| +|buffering_hints_size_in_mb_s|bigint|| +|cloud_watch_logging_options_enabled|boolean|Enables or disables CloudWatch logging| +|cloud_watch_logging_options_log_group_name|text|The CloudWatch group name for logging| +|cloud_watch_logging_options_log_stream_name|text|The CloudWatch log stream name for logging| +|cluster_endpoint|text|| +|domain_arn|text|| +|index_name|text|| +|index_rotation_period|text|| +|processing_configuration_enabled|boolean|Enables or disables data processing| +|retry_options_duration_in_seconds|bigint|| +|role_arn|text|| +|s3_backup_mode|text|| +|s3_destination_bucket_arn|text|The ARN of the S3 bucket| +|s3_destination_buffering_hints_interval_in_seconds|bigint|Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination| +|s3_destination_buffering_hints_size_in_mb_s|bigint|Buffer incoming data to the specified size, in MiBs, before delivering it to the destination| +|s3_destination_compression_format|text|The compression format| +|s3_destination_kms_encryption_config_aws_kms_key_arn|text|The Amazon Resource Name (ARN) of the encryption key| +|s3_destination_no_encryption_config|text|Specifically override existing encryption information to ensure that no encryption is used| +|s3_destination_role_arn|text|The Amazon Resource Name (ARN) of the AWS credentials| +|s3_destination_cloud_watch_logging_options_enabled|boolean|Enables or disables CloudWatch logging| +|s3_destination_cloud_watch_logging_options_log_group_name|text|The CloudWatch group name for logging| +|s3_destination_cloud_watch_logging_options_log_stream_name|text|The CloudWatch log stream name for logging| +|s3_destination_error_output_prefix|text|A prefix that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3| +|s3_destination_prefix|text|The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered Amazon S3 files| +|type_name|text|| +|vpc_configuration_description_role_arn|text|The ARN of the IAM role that the delivery stream uses to create endpoints in the destination VPC| +|vpc_configuration_description_security_group_ids|text[]|The IDs of the security groups that Kinesis Data Firehose uses when it creates ENIs in the VPC of the Amazon ES destination| +|vpc_configuration_description_subnet_ids|text[]|The IDs of the subnets that Kinesis Data Firehose uses to create ENIs in the VPC of the Amazon ES destination| +|vpc_configuration_description_vpc_id|text|The ID of the Amazon ES destination's VPC| diff --git a/docs/tables/aws_firehose_delivery_stream_redshift_destination.md b/docs/tables/aws_firehose_delivery_stream_redshift_destination.md new file mode 100644 index 000000000..d07cc2199 --- /dev/null +++ b/docs/tables/aws_firehose_delivery_stream_redshift_destination.md @@ -0,0 +1,44 @@ + +# Table: aws_firehose_delivery_stream_redshift_destination +Describes a destination in Amazon Redshift +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|delivery_stream_cq_id|uuid|Unique CloudQuery ID of aws_firehose_delivery_streams table (FK)| +|processing_configuration_processors|jsonb|Describes a data processing configuration| +|cluster_j_db_c_url|text|The database connection string| +|copy_command_data_table_name|text|The name of the target table| +|copy_command_copy_options|text|Optional parameters to use with the Amazon Redshift COPY command| +|copy_command_data_table_columns|text|A comma-separated list of column names| +|role_arn|text|The Amazon Resource Name (ARN) of the AWS credentials| +|s3_destination_bucket_arn|text|The ARN of the S3 bucket| +|s3_destination_buffering_hints_interval_in_seconds|bigint|Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination| +|s3_destination_buffering_hints_size_in_mb_s|bigint|Buffer incoming data to the specified size, in MiBs, before delivering it to the destination| +|s3_destination_compression_format|text|The compression format| +|s3_destination_kms_encryption_config_aws_kms_key_arn|text|The Amazon Resource Name (ARN) of the encryption key| +|s3_destination_no_encryption_config|text|Specifically override existing encryption information to ensure that no encryption is used| +|s3_destination_role_arn|text|The Amazon Resource Name (ARN) of the AWS credentials| +|s3_destination_cloud_watch_logging_options_enabled|boolean|Enables or disables CloudWatch logging| +|s3_destination_cloud_watch_logging_options_log_group_name|text|The CloudWatch group name for logging| +|s3_destination_cloud_watch_logging_options_log_stream_name|text|The CloudWatch log stream name for logging| +|s3_destination_error_output_prefix|text|A prefix that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3| +|s3_destination_prefix|text|The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered Amazon S3 files| +|username|text|The name of the user| +|cloud_watch_logging_options_enabled|boolean|Enables or disables CloudWatch logging| +|cloud_watch_logging_options_log_group_name|text|The CloudWatch group name for logging| +|cloud_watch_logging_options_log_stream_name|text|The CloudWatch log stream name for logging| +|processing_configuration_enabled|boolean|Enables or disables data processing| +|retry_options_duration_in_seconds|bigint|The length of time during which Kinesis Data Firehose retries delivery after a failure, starting from the initial request and including the first attempt| +|s3_backup_bucket_arn|text|The ARN of the S3 bucket| +|s3_backup_buffering_hints_interval_in_seconds|bigint|Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination| +|s3_backup_buffering_hints_size_in_mb_s|bigint|Buffer incoming data to the specified size, in MiBs, before delivering it to the destination| +|s3_backup_compression_format|text|The compression format| +|s3_backup_kms_encryption_config_aws_kms_key_arn|text|The Amazon Resource Name (ARN) of the encryption key| +|s3_backup_no_encryption_config|text|Specifically override existing encryption information to ensure that no encryption is used| +|s3_backup_role_arn|text|The Amazon Resource Name (ARN) of the AWS credentials| +|s3_backup_cloud_watch_logging_options_enabled|boolean|Enables or disables CloudWatch logging| +|s3_backup_cloud_watch_logging_options_log_group_name|text|The CloudWatch group name for logging| +|s3_backup_cloud_watch_logging_options_log_stream_name|text|The CloudWatch log stream name for logging| +|s3_backup_error_output_prefix|text|A prefix that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3| +|s3_backup_prefix|text|The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered Amazon S3 files| +|s3_backup_mode|text|The Amazon S3 backup mode| diff --git a/docs/tables/aws_firehose_delivery_stream_splunk_destination.md b/docs/tables/aws_firehose_delivery_stream_splunk_destination.md new file mode 100644 index 000000000..ec10f5b75 --- /dev/null +++ b/docs/tables/aws_firehose_delivery_stream_splunk_destination.md @@ -0,0 +1,30 @@ + +# Table: aws_firehose_delivery_stream_splunk_destination +Describes a destination in Splunk +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|delivery_stream_cq_id|uuid|Unique CloudQuery ID of aws_firehose_delivery_streams table (FK)| +|processing_configuration_processors|jsonb|Describes a data processing configuration| +|cloud_watch_logging_options_enabled|boolean|Enables or disables CloudWatch logging| +|cloud_watch_logging_options_log_group_name|text|The CloudWatch group name for logging| +|cloud_watch_logging_options_log_stream_name|text|The CloudWatch log stream name for logging| +|h_e_ca_cknowledgment_timeout_in_seconds|bigint|The amount of time that Kinesis Data Firehose waits to receive an acknowledgment from Splunk after it sends it data| +|h_e_c_endpoint|text|The HTTP Event Collector (HEC) endpoint to which Kinesis Data Firehose sends your data| +|h_e_c_endpoint_type|text|This type can be either "Raw" or "Event"| +|h_e_c_token|text|A GUID you obtain from your Splunk cluster when you create a new HEC endpoint| +|processing_configuration_enabled|boolean|Enables or disables data processing| +|retry_options_duration_in_seconds|bigint|The total amount of time that Kinesis Data Firehose spends on retries| +|s3_backup_mode|text|Defines how documents should be delivered to Amazon S3| +|s3_destination_bucket_arn|text|The ARN of the S3 bucket| +|s3_destination_buffering_hints_interval_in_seconds|bigint|Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination| +|s3_destination_buffering_hints_size_in_mb_s|bigint|Buffer incoming data to the specified size, in MiBs, before delivering it to the destination| +|s3_destination_compression_format|text|The compression format| +|s3_destination_kms_encryption_config_aws_kms_key_arn|text|The Amazon Resource Name (ARN) of the encryption key| +|s3_destination_no_encryption_config|text|Specifically override existing encryption information to ensure that no encryption is used| +|s3_destination_role_arn|text|The Amazon Resource Name (ARN) of the AWS credentials| +|s3_destination_cloud_watch_logging_options_enabled|boolean|Enables or disables CloudWatch logging| +|s3_destination_cloud_watch_logging_options_log_group_name|text|The CloudWatch group name for logging| +|s3_destination_cloud_watch_logging_options_log_stream_name|text|The CloudWatch log stream name for logging| +|s3_destination_error_output_prefix|text|A prefix that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3| +|s3_destination_prefix|text|The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered Amazon S3 files| diff --git a/docs/tables/aws_firehose_delivery_streams.md b/docs/tables/aws_firehose_delivery_streams.md new file mode 100644 index 000000000..557628e5d --- /dev/null +++ b/docs/tables/aws_firehose_delivery_streams.md @@ -0,0 +1,27 @@ + +# Table: aws_firehose_delivery_streams +Contains information about a delivery stream +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|account_id|text|The AWS Account ID of the resource.| +|region|text|The AWS Region of the resource.| +|tags|jsonb|| +|arn|text|The Amazon Resource Name (ARN) of the delivery stream| +|delivery_stream_arn|text|The Amazon Resource Name (ARN) of the delivery stream| +|delivery_stream_name|text|The name of the delivery stream| +|delivery_stream_status|text|The status of the delivery stream| +|delivery_stream_type|text|The delivery stream type| +|version_id|text|Each time the destination is updated for a delivery stream, the version ID is changed, and the current version ID is required when updating the destination This is so that the service knows it is applying the changes to the correct version of the delivery stream| +|create_timestamp|timestamp without time zone|The date and time that the delivery stream was created| +|encryption_config_failure_description_details|text|A message providing details about the error that caused the failure| +|encryption_config_failure_description_type|text|The type of error that caused the failure| +|encryption_config_key_arn|text|If KeyType is CUSTOMER_MANAGED_CMK, this field contains the ARN of the customer managed CMK| +|encryption_config_key_type|text|Indicates the type of customer master key (CMK) that is used for encryption| +|encryption_config_status|text|This is the server-side encryption (SSE) status for the delivery stream| +|failure_description_details|text|A message providing details about the error that caused the failure| +|failure_description_type|text|The type of error that caused the failure| +|last_update_timestamp|timestamp without time zone|The date and time that the delivery stream was last updated| +|source_kinesis_stream_delivery_start_timestamp|timestamp without time zone|Kinesis Data Firehose starts retrieving records from the Kinesis data stream starting with this timestamp| +|source_kinesis_stream_kinesis_stream_arn|text|The Amazon Resource Name (ARN) of the source Kinesis data stream| +|source_kinesis_stream_role_arn|text|The ARN of the role used by the source Kinesis data stream| diff --git a/go.mod b/go.mod index e3ecf993b..37d8a700a 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,8 @@ require ( github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.18.7 github.com/aws/aws-sdk-go-v2/service/elasticsearchservice v1.15.7 github.com/aws/aws-sdk-go-v2/service/emr v1.20.0 + github.com/aws/aws-sdk-go-v2/service/eventbridge v1.16.8 + github.com/aws/aws-sdk-go-v2/service/firehose v1.14.10 github.com/aws/aws-sdk-go-v2/service/fsx v1.24.2 github.com/aws/aws-sdk-go-v2/service/guardduty v1.14.1 github.com/aws/aws-sdk-go-v2/service/iam v1.18.8 @@ -88,7 +90,6 @@ require ( require ( github.com/BurntSushi/toml v1.2.0 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.8 // indirect - github.com/aws/aws-sdk-go-v2/service/eventbridge v1.16.8 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.9 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/elliotchance/orderedmap v1.4.0 // indirect diff --git a/go.sum b/go.sum index e64048aa7..8cb1eeb2b 100644 --- a/go.sum +++ b/go.sum @@ -154,6 +154,8 @@ github.com/aws/aws-sdk-go-v2/service/emr v1.20.0 h1:2xjz2hES5SnQLgmW1bBVdVz6j0mj github.com/aws/aws-sdk-go-v2/service/emr v1.20.0/go.mod h1:OVVv6VrQG33CuCR+V0GiaBuWbwYGogGxWGi9TDrM2yk= github.com/aws/aws-sdk-go-v2/service/eventbridge v1.16.8 h1:RE7eIYoWMJRqMNM8cdQfEOV0ruexieh/J3yM3PYh+HU= github.com/aws/aws-sdk-go-v2/service/eventbridge v1.16.8/go.mod h1:ShtRcolaihIMdVmjL7qqWXkOlMCz64L3XfjaeEBXnTg= +github.com/aws/aws-sdk-go-v2/service/firehose v1.14.10 h1:QBmdueOBazMIf7IIEOXFpeFgqZwtZfYVNvdq77J4210= +github.com/aws/aws-sdk-go-v2/service/firehose v1.14.10/go.mod h1:SLUVnKAVS6cKErShWaD30AkG97hsaFzmYSxJg1nn594= github.com/aws/aws-sdk-go-v2/service/fsx v1.24.2 h1:8ko+AFpvJUbpjtCIEgtaXcXtndkZBi0N7e2ePGocqf8= github.com/aws/aws-sdk-go-v2/service/fsx v1.24.2/go.mod h1:K3Ym90NBYdXV+BCHvpuiDXCeMAtayFdiGdJ0I1uop5Q= github.com/aws/aws-sdk-go-v2/service/glue v1.28.1 h1:rG+jzafWyw73tdv+48e4jZYyehihEORcEcqzyBbZUGA= diff --git a/resources/provider/provider.go b/resources/provider/provider.go index 591e62c2f..0d267fa14 100644 --- a/resources/provider/provider.go +++ b/resources/provider/provider.go @@ -38,6 +38,7 @@ import ( "github.com/cloudquery/cq-provider-aws/resources/services/elbv2" "github.com/cloudquery/cq-provider-aws/resources/services/emr" "github.com/cloudquery/cq-provider-aws/resources/services/eventbridge" + "github.com/cloudquery/cq-provider-aws/resources/services/firehose" "github.com/cloudquery/cq-provider-aws/resources/services/fsx" "github.com/cloudquery/cq-provider-aws/resources/services/glue" "github.com/cloudquery/cq-provider-aws/resources/services/guardduty" @@ -209,6 +210,7 @@ func Provider() *provider.Provider { "iot.things": iot.IotThings(), "iot.topic_rules": iot.IotTopicRules(), "kinesis.data_streams": kinesis.Streams(), + "firehose.delivery_streams": firehose.DeliveryStreams(), "kms.keys": kms.Keys(), "lambda.functions": lambda.Functions(), "lambda.layers": lambda.LambdaLayers(), diff --git a/resources/services/firehose/delivery_streams.go b/resources/services/firehose/delivery_streams.go new file mode 100644 index 000000000..d4d148108 --- /dev/null +++ b/resources/services/firehose/delivery_streams.go @@ -0,0 +1,1487 @@ +package firehose + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/firehose" + "github.com/aws/aws-sdk-go-v2/service/firehose/types" + "github.com/cloudquery/cq-provider-aws/client" + "github.com/cloudquery/cq-provider-sdk/provider/diag" + "github.com/cloudquery/cq-provider-sdk/provider/schema" +) + +//go:generate cq-gen --resource delivery_streams --config gen.hcl --output . +func DeliveryStreams() *schema.Table { + return &schema.Table{ + Name: "aws_firehose_delivery_streams", + Description: "Contains information about a delivery stream", + Resolver: fetchFirehoseDeliveryStreams, + Multiplex: client.ServiceAccountRegionMultiplexer("firehose"), + IgnoreError: client.IgnoreCommonErrors, + DeleteFilter: client.DeleteAccountRegionFilter, + Options: schema.TableCreationOptions{PrimaryKeys: []string{"arn"}}, + Columns: []schema.Column{ + { + Name: "account_id", + Description: "The AWS Account ID of the resource.", + Type: schema.TypeString, + Resolver: client.ResolveAWSAccount, + }, + { + Name: "region", + Description: "The AWS Region of the resource.", + Type: schema.TypeString, + Resolver: client.ResolveAWSRegion, + }, + { + Name: "tags", + Type: schema.TypeJSON, + Resolver: resolveFirehoseDeliveryStreamTags, + }, + { + Name: "arn", + Description: "The Amazon Resource Name (ARN) of the delivery stream", + Type: schema.TypeString, + Resolver: schema.PathResolver("DeliveryStreamARN"), + }, + { + Name: "delivery_stream_arn", + Description: "The Amazon Resource Name (ARN) of the delivery stream", + Type: schema.TypeString, + Resolver: schema.PathResolver("DeliveryStreamARN"), + }, + { + Name: "delivery_stream_name", + Description: "The name of the delivery stream", + Type: schema.TypeString, + }, + { + Name: "delivery_stream_status", + Description: "The status of the delivery stream", + Type: schema.TypeString, + }, + { + Name: "delivery_stream_type", + Description: "The delivery stream type", + Type: schema.TypeString, + }, + { + Name: "version_id", + Description: "Each time the destination is updated for a delivery stream, the version ID is changed, and the current version ID is required when updating the destination This is so that the service knows it is applying the changes to the correct version of the delivery stream", + Type: schema.TypeString, + }, + { + Name: "create_timestamp", + Description: "The date and time that the delivery stream was created", + Type: schema.TypeTimestamp, + }, + { + Name: "encryption_config_failure_description_details", + Description: "A message providing details about the error that caused the failure", + Type: schema.TypeString, + Resolver: schema.PathResolver("DeliveryStreamEncryptionConfiguration.FailureDescription.Details"), + }, + { + Name: "encryption_config_failure_description_type", + Description: "The type of error that caused the failure", + Type: schema.TypeString, + Resolver: schema.PathResolver("DeliveryStreamEncryptionConfiguration.FailureDescription.Type"), + }, + { + Name: "encryption_config_key_arn", + Description: "If KeyType is CUSTOMER_MANAGED_CMK, this field contains the ARN of the customer managed CMK", + Type: schema.TypeString, + Resolver: schema.PathResolver("DeliveryStreamEncryptionConfiguration.KeyARN"), + }, + { + Name: "encryption_config_key_type", + Description: "Indicates the type of customer master key (CMK) that is used for encryption", + Type: schema.TypeString, + Resolver: schema.PathResolver("DeliveryStreamEncryptionConfiguration.KeyType"), + }, + { + Name: "encryption_config_status", + Description: "This is the server-side encryption (SSE) status for the delivery stream", + Type: schema.TypeString, + Resolver: schema.PathResolver("DeliveryStreamEncryptionConfiguration.Status"), + }, + { + Name: "failure_description_details", + Description: "A message providing details about the error that caused the failure", + Type: schema.TypeString, + Resolver: schema.PathResolver("FailureDescription.Details"), + }, + { + Name: "failure_description_type", + Description: "The type of error that caused the failure", + Type: schema.TypeString, + Resolver: schema.PathResolver("FailureDescription.Type"), + }, + { + Name: "last_update_timestamp", + Description: "The date and time that the delivery stream was last updated", + Type: schema.TypeTimestamp, + }, + { + Name: "source_kinesis_stream_delivery_start_timestamp", + Description: "Kinesis Data Firehose starts retrieving records from the Kinesis data stream starting with this timestamp", + Type: schema.TypeTimestamp, + Resolver: schema.PathResolver("Source.KinesisStreamSourceDescription.DeliveryStartTimestamp"), + }, + { + Name: "source_kinesis_stream_kinesis_stream_arn", + Description: "The Amazon Resource Name (ARN) of the source Kinesis data stream", + Type: schema.TypeString, + Resolver: schema.PathResolver("Source.KinesisStreamSourceDescription.KinesisStreamARN"), + }, + { + Name: "source_kinesis_stream_role_arn", + Description: "The ARN of the role used by the source Kinesis data stream", + Type: schema.TypeString, + Resolver: schema.PathResolver("Source.KinesisStreamSourceDescription.RoleARN"), + }, + }, + Relations: []*schema.Table{ + { + Name: "aws_firehose_delivery_stream_open_search_destination", + Resolver: schema.PathTableResolver("Destinations.AmazonopensearchserviceDestinationDescription"), + Columns: []schema.Column{ + { + Name: "delivery_stream_cq_id", + Description: "Unique CloudQuery ID of aws_firehose_delivery_streams table (FK)", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "processing_configuration_processors", + Description: "Describes a data processing configuration", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("ProcessingConfiguration.Processors"), + }, + { + Name: "buffering_hints_interval_in_seconds", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("BufferingHints.IntervalInSeconds"), + }, + { + Name: "buffering_hints_size_in_mb_s", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("BufferingHints.SizeInMBs"), + }, + { + Name: "cloud_watch_logging_options_enabled", + Description: "Enables or disables CloudWatch logging", + Type: schema.TypeBool, + Resolver: schema.PathResolver("CloudWatchLoggingOptions.Enabled"), + }, + { + Name: "cloud_watch_logging_options_log_group_name", + Description: "The CloudWatch group name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("CloudWatchLoggingOptions.LogGroupName"), + }, + { + Name: "cloud_watch_logging_options_log_stream_name", + Description: "The CloudWatch log stream name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("CloudWatchLoggingOptions.LogStreamName"), + }, + { + Name: "cluster_endpoint", + Type: schema.TypeString, + }, + { + Name: "domain_arn", + Type: schema.TypeString, + Resolver: schema.PathResolver("DomainARN"), + }, + { + Name: "index_name", + Type: schema.TypeString, + }, + { + Name: "index_rotation_period", + Type: schema.TypeString, + }, + { + Name: "processing_configuration_enabled", + Description: "Enables or disables data processing", + Type: schema.TypeBool, + Resolver: schema.PathResolver("ProcessingConfiguration.Enabled"), + }, + { + Name: "retry_options_duration_in_seconds", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("RetryOptions.DurationInSeconds"), + }, + { + Name: "role_arn", + Type: schema.TypeString, + Resolver: schema.PathResolver("RoleARN"), + }, + { + Name: "s3_backup_mode", + Type: schema.TypeString, + }, + { + Name: "s3_destination_bucket_arn", + Description: "The ARN of the S3 bucket", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.BucketARN"), + }, + { + Name: "s3_destination_buffering_hints_interval_in_seconds", + Description: "Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("S3DestinationDescription.BufferingHints.IntervalInSeconds"), + }, + { + Name: "s3_destination_buffering_hints_size_in_mb_s", + Description: "Buffer incoming data to the specified size, in MiBs, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("S3DestinationDescription.BufferingHints.SizeInMBs"), + }, + { + Name: "s3_destination_compression_format", + Description: "The compression format", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.CompressionFormat"), + }, + { + Name: "s3_destination_kms_encryption_config_aws_kms_key_arn", + Description: "The Amazon Resource Name (ARN) of the encryption key", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.EncryptionConfiguration.KMSEncryptionConfig.AWSKMSKeyARN"), + }, + { + Name: "s3_destination_no_encryption_config", + Description: "Specifically override existing encryption information to ensure that no encryption is used", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.EncryptionConfiguration.NoEncryptionConfig"), + }, + { + Name: "s3_destination_role_arn", + Description: "The Amazon Resource Name (ARN) of the AWS credentials", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.RoleARN"), + }, + { + Name: "s3_destination_cloud_watch_logging_options_enabled", + Description: "Enables or disables CloudWatch logging", + Type: schema.TypeBool, + Resolver: schema.PathResolver("S3DestinationDescription.CloudWatchLoggingOptions.Enabled"), + }, + { + Name: "s3_destination_cloud_watch_logging_options_log_group_name", + Description: "The CloudWatch group name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.CloudWatchLoggingOptions.LogGroupName"), + }, + { + Name: "s3_destination_cloud_watch_logging_options_log_stream_name", + Description: "The CloudWatch log stream name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.CloudWatchLoggingOptions.LogStreamName"), + }, + { + Name: "s3_destination_error_output_prefix", + Description: "A prefix that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.ErrorOutputPrefix"), + }, + { + Name: "s3_destination_prefix", + Description: "The \"YYYY/MM/DD/HH\" time format prefix is automatically used for delivered Amazon S3 files", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.Prefix"), + }, + { + Name: "type_name", + Type: schema.TypeString, + }, + { + Name: "vpc_configuration_description_role_arn", + Description: "The ARN of the IAM role that the delivery stream uses to create endpoints in the destination VPC", + Type: schema.TypeString, + Resolver: schema.PathResolver("VpcConfigurationDescription.RoleARN"), + }, + { + Name: "vpc_configuration_description_security_group_ids", + Description: "The IDs of the security groups that Kinesis Data Firehose uses when it creates ENIs in the VPC of the Amazon ES destination", + Type: schema.TypeStringArray, + Resolver: schema.PathResolver("VpcConfigurationDescription.SecurityGroupIds"), + }, + { + Name: "vpc_configuration_description_subnet_ids", + Description: "The IDs of the subnets that Kinesis Data Firehose uses to create ENIs in the VPC of the Amazon ES destination", + Type: schema.TypeStringArray, + Resolver: schema.PathResolver("VpcConfigurationDescription.SubnetIds"), + }, + { + Name: "vpc_configuration_description_vpc_id", + Description: "The ID of the Amazon ES destination's VPC", + Type: schema.TypeString, + Resolver: schema.PathResolver("VpcConfigurationDescription.VpcId"), + }, + }, + }, + { + Name: "aws_firehose_delivery_stream_elasticsearch_destination", + Description: "The destination description in Amazon ES", + Resolver: schema.PathTableResolver("Destinations.ElasticsearchDestinationDescription"), + Columns: []schema.Column{ + { + Name: "delivery_stream_cq_id", + Description: "Unique CloudQuery ID of aws_firehose_delivery_streams table (FK)", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "processing_configuration_processors", + Description: "Describes a data processing configuration", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("ProcessingConfiguration.Processors"), + }, + { + Name: "buffering_hints_interval_in_seconds", + Description: "Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("BufferingHints.IntervalInSeconds"), + }, + { + Name: "buffering_hints_size_in_mb_s", + Description: "Buffer incoming data to the specified size, in MBs, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("BufferingHints.SizeInMBs"), + }, + { + Name: "cloud_watch_logging_options_enabled", + Description: "Enables or disables CloudWatch logging", + Type: schema.TypeBool, + Resolver: schema.PathResolver("CloudWatchLoggingOptions.Enabled"), + }, + { + Name: "cloud_watch_logging_options_log_group_name", + Description: "The CloudWatch group name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("CloudWatchLoggingOptions.LogGroupName"), + }, + { + Name: "cloud_watch_logging_options_log_stream_name", + Description: "The CloudWatch log stream name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("CloudWatchLoggingOptions.LogStreamName"), + }, + { + Name: "cluster_endpoint", + Description: "The endpoint to use when communicating with the cluster", + Type: schema.TypeString, + }, + { + Name: "domain_arn", + Description: "The ARN of the Amazon ES domain", + Type: schema.TypeString, + Resolver: schema.PathResolver("DomainARN"), + }, + { + Name: "index_name", + Description: "The Elasticsearch index name", + Type: schema.TypeString, + }, + { + Name: "index_rotation_period", + Description: "The Elasticsearch index rotation period", + Type: schema.TypeString, + }, + { + Name: "processing_configuration_enabled", + Description: "Enables or disables data processing", + Type: schema.TypeBool, + Resolver: schema.PathResolver("ProcessingConfiguration.Enabled"), + }, + { + Name: "retry_options_duration_in_seconds", + Description: "After an initial failure to deliver to Amazon ES, the total amount of time during which Kinesis Data Firehose retries delivery (including the first attempt)", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("RetryOptions.DurationInSeconds"), + }, + { + Name: "role_arn", + Description: "The Amazon Resource Name (ARN) of the AWS credentials", + Type: schema.TypeString, + Resolver: schema.PathResolver("RoleARN"), + }, + { + Name: "s3_backup_mode", + Description: "The Amazon S3 backup mode", + Type: schema.TypeString, + }, + { + Name: "s3_destination_bucket_arn", + Description: "The ARN of the S3 bucket", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.BucketARN"), + }, + { + Name: "s3_destination_buffering_hints_interval_in_seconds", + Description: "Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("S3DestinationDescription.BufferingHints.IntervalInSeconds"), + }, + { + Name: "s3_destination_buffering_hints_size_in_mb_s", + Description: "Buffer incoming data to the specified size, in MiBs, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("S3DestinationDescription.BufferingHints.SizeInMBs"), + }, + { + Name: "s3_destination_compression_format", + Description: "The compression format", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.CompressionFormat"), + }, + { + Name: "s3_destination_kms_encryption_config_aws_kms_key_arn", + Description: "The Amazon Resource Name (ARN) of the encryption key", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.EncryptionConfiguration.KMSEncryptionConfig.AWSKMSKeyARN"), + }, + { + Name: "s3_destination_no_encryption_config", + Description: "Specifically override existing encryption information to ensure that no encryption is used", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.EncryptionConfiguration.NoEncryptionConfig"), + }, + { + Name: "s3_destination_role_arn", + Description: "The Amazon Resource Name (ARN) of the AWS credentials", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.RoleARN"), + }, + { + Name: "s3_destination_cloud_watch_logging_options_enabled", + Description: "Enables or disables CloudWatch logging", + Type: schema.TypeBool, + Resolver: schema.PathResolver("S3DestinationDescription.CloudWatchLoggingOptions.Enabled"), + }, + { + Name: "s3_destination_cloud_watch_logging_options_log_group_name", + Description: "The CloudWatch group name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.CloudWatchLoggingOptions.LogGroupName"), + }, + { + Name: "s3_destination_cloud_watch_logging_options_log_stream_name", + Description: "The CloudWatch log stream name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.CloudWatchLoggingOptions.LogStreamName"), + }, + { + Name: "s3_destination_error_output_prefix", + Description: "A prefix that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.ErrorOutputPrefix"), + }, + { + Name: "s3_destination_prefix", + Description: "The \"YYYY/MM/DD/HH\" time format prefix is automatically used for delivered Amazon S3 files", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.Prefix"), + }, + { + Name: "type_name", + Description: "The Elasticsearch type name", + Type: schema.TypeString, + }, + { + Name: "vpc_configuration_description_role_arn", + Description: "The ARN of the IAM role that the delivery stream uses to create endpoints in the destination VPC", + Type: schema.TypeString, + Resolver: schema.PathResolver("VpcConfigurationDescription.RoleARN"), + }, + { + Name: "vpc_configuration_description_security_group_ids", + Description: "The IDs of the security groups that Kinesis Data Firehose uses when it creates ENIs in the VPC of the Amazon ES destination", + Type: schema.TypeStringArray, + Resolver: schema.PathResolver("VpcConfigurationDescription.SecurityGroupIds"), + }, + { + Name: "vpc_configuration_description_subnet_ids", + Description: "The IDs of the subnets that Kinesis Data Firehose uses to create ENIs in the VPC of the Amazon ES destination", + Type: schema.TypeStringArray, + Resolver: schema.PathResolver("VpcConfigurationDescription.SubnetIds"), + }, + { + Name: "vpc_configuration_description_vpc_id", + Description: "The ID of the Amazon ES destination's VPC", + Type: schema.TypeString, + Resolver: schema.PathResolver("VpcConfigurationDescription.VpcId"), + }, + }, + }, + { + Name: "aws_firehose_delivery_stream_extended_s3_destination", + Description: "Describes a destination in Amazon S3", + Resolver: schema.PathTableResolver("Destinations.ExtendedS3DestinationDescription"), + Columns: []schema.Column{ + { + Name: "delivery_stream_cq_id", + Description: "Unique CloudQuery ID of aws_firehose_delivery_streams table (FK)", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "processing_configuration_processors", + Description: "Describes a data processing configuration", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("ProcessingConfiguration.Processors"), + }, + { + Name: "bucket_arn", + Description: "The ARN of the S3 bucket", + Type: schema.TypeString, + Resolver: schema.PathResolver("BucketARN"), + }, + { + Name: "buffering_hints_interval_in_seconds", + Description: "Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("BufferingHints.IntervalInSeconds"), + }, + { + Name: "buffering_hints_size_in_mb_s", + Description: "Buffer incoming data to the specified size, in MiBs, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("BufferingHints.SizeInMBs"), + }, + { + Name: "compression_format", + Description: "The compression format", + Type: schema.TypeString, + }, + { + Name: "encryption_configuration_kms_encryption_config_aws_kms_key_arn", + Description: "The Amazon Resource Name (ARN) of the encryption key", + Type: schema.TypeString, + Resolver: schema.PathResolver("EncryptionConfiguration.KMSEncryptionConfig.AWSKMSKeyARN"), + }, + { + Name: "encryption_configuration_no_encryption_config", + Description: "Specifically override existing encryption information to ensure that no encryption is used", + Type: schema.TypeString, + Resolver: schema.PathResolver("EncryptionConfiguration.NoEncryptionConfig"), + }, + { + Name: "role_arn", + Description: "The Amazon Resource Name (ARN) of the AWS credentials", + Type: schema.TypeString, + Resolver: schema.PathResolver("RoleARN"), + }, + { + Name: "cloud_watch_logging_options_enabled", + Description: "Enables or disables CloudWatch logging", + Type: schema.TypeBool, + Resolver: schema.PathResolver("CloudWatchLoggingOptions.Enabled"), + }, + { + Name: "cloud_watch_logging_options_log_group_name", + Description: "The CloudWatch group name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("CloudWatchLoggingOptions.LogGroupName"), + }, + { + Name: "cloud_watch_logging_options_log_stream_name", + Description: "The CloudWatch log stream name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("CloudWatchLoggingOptions.LogStreamName"), + }, + { + Name: "enabled", + Description: "Defaults to true", + Type: schema.TypeBool, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.Enabled"), + }, + { + Name: "deserializer_hive_json_ser_de_timestamp_formats", + Description: "Indicates how you want Kinesis Data Firehose to parse the date and timestamps that may be present in your input data JSON", + Type: schema.TypeStringArray, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.InputFormatConfiguration.Deserializer.HiveJsonSerDe.TimestampFormats"), + }, + { + Name: "deserializer_open_x_json_ser_de_case_insensitive", + Description: "When set to true, which is the default, Kinesis Data Firehose converts JSON keys to lowercase before deserializing them", + Type: schema.TypeBool, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.InputFormatConfiguration.Deserializer.OpenXJsonSerDe.CaseInsensitive"), + }, + { + Name: "deserializer_open_x_json_ser_de_column_to_json_key_mappings", + Description: "Maps column names to JSON keys that aren't identical to the column names", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.InputFormatConfiguration.Deserializer.OpenXJsonSerDe.ColumnToJsonKeyMappings"), + }, + { + Name: "deserializer_open_x_json_ser_de_convert_dots_to_underscores", + Description: "When set to true, specifies that the names of the keys include dots and that you want Kinesis Data Firehose to replace them with underscores", + Type: schema.TypeBool, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.InputFormatConfiguration.Deserializer.OpenXJsonSerDe.ConvertDotsInJsonKeysToUnderscores"), + }, + { + Name: "serializer_orc_ser_de_block_size_bytes", + Description: "The Hadoop Distributed File System (HDFS) block size", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.OutputFormatConfiguration.Serializer.OrcSerDe.BlockSizeBytes"), + }, + { + Name: "serializer_orc_ser_de_bloom_filter_columns", + Description: "The column names for which you want Kinesis Data Firehose to create bloom filters", + Type: schema.TypeStringArray, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.OutputFormatConfiguration.Serializer.OrcSerDe.BloomFilterColumns"), + }, + { + Name: "serializer_orc_ser_de_bloom_filter_false_positive_probability", + Description: "The Bloom filter false positive probability (FPP)", + Type: schema.TypeFloat, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.OutputFormatConfiguration.Serializer.OrcSerDe.BloomFilterFalsePositiveProbability"), + }, + { + Name: "serializer_orc_ser_de_compression", + Description: "The compression code to use over data blocks", + Type: schema.TypeString, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.OutputFormatConfiguration.Serializer.OrcSerDe.Compression"), + }, + { + Name: "serializer_orc_ser_de_dictionary_key_threshold", + Description: "Represents the fraction of the total number of non-null rows", + Type: schema.TypeFloat, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.OutputFormatConfiguration.Serializer.OrcSerDe.DictionaryKeyThreshold"), + }, + { + Name: "serializer_orc_ser_de_enable_padding", + Description: "Set this to true to indicate that you want stripes to be padded to the HDFS block boundaries", + Type: schema.TypeBool, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.OutputFormatConfiguration.Serializer.OrcSerDe.EnablePadding"), + }, + { + Name: "serializer_orc_ser_de_format_version", + Description: "The version of the file to write", + Type: schema.TypeString, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.OutputFormatConfiguration.Serializer.OrcSerDe.FormatVersion"), + }, + { + Name: "serializer_orc_ser_de_padding_tolerance", + Description: "A number between 0 and 1 that defines the tolerance for block padding as a decimal fraction of stripe size", + Type: schema.TypeFloat, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.OutputFormatConfiguration.Serializer.OrcSerDe.PaddingTolerance"), + }, + { + Name: "serializer_orc_ser_de_row_index_stride", + Description: "The number of rows between index entries", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.OutputFormatConfiguration.Serializer.OrcSerDe.RowIndexStride"), + }, + { + Name: "serializer_orc_ser_de_stripe_size_bytes", + Description: "The number of bytes in each stripe", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.OutputFormatConfiguration.Serializer.OrcSerDe.StripeSizeBytes"), + }, + { + Name: "serializer_parquet_ser_de_block_size_bytes", + Description: "The Hadoop Distributed File System (HDFS) block size", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.OutputFormatConfiguration.Serializer.ParquetSerDe.BlockSizeBytes"), + }, + { + Name: "serializer_parquet_ser_de_compression", + Description: "The compression code to use over data blocks", + Type: schema.TypeString, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.OutputFormatConfiguration.Serializer.ParquetSerDe.Compression"), + }, + { + Name: "serializer_parquet_ser_de_enable_dictionary_compression", + Description: "Indicates whether to enable dictionary compression", + Type: schema.TypeBool, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.OutputFormatConfiguration.Serializer.ParquetSerDe.EnableDictionaryCompression"), + }, + { + Name: "serializer_parquet_ser_de_max_padding_bytes", + Description: "The maximum amount of padding to apply", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.OutputFormatConfiguration.Serializer.ParquetSerDe.MaxPaddingBytes"), + }, + { + Name: "serializer_parquet_ser_de_page_size_bytes", + Description: "The Parquet page size", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.OutputFormatConfiguration.Serializer.ParquetSerDe.PageSizeBytes"), + }, + { + Name: "serializer_parquet_ser_de_writer_version", + Description: "Indicates the version of row format to output", + Type: schema.TypeString, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.OutputFormatConfiguration.Serializer.ParquetSerDe.WriterVersion"), + }, + { + Name: "schema_configuration_catalog_id", + Description: "The ID of the AWS Glue Data Catalog", + Type: schema.TypeString, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.SchemaConfiguration.CatalogId"), + }, + { + Name: "schema_configuration_database_name", + Description: "Specifies the name of the AWS Glue database that contains the schema for the output data", + Type: schema.TypeString, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.SchemaConfiguration.DatabaseName"), + }, + { + Name: "schema_configuration_region", + Description: "If you don't specify an AWS Region, the default is the current Region", + Type: schema.TypeString, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.SchemaConfiguration.Region"), + }, + { + Name: "schema_configuration_role_arn", + Description: "The role that Kinesis Data Firehose can use to access AWS Glue", + Type: schema.TypeString, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.SchemaConfiguration.RoleARN"), + }, + { + Name: "schema_configuration_table_name", + Description: "Specifies the AWS Glue table that contains the column information that constitutes your data schema", + Type: schema.TypeString, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.SchemaConfiguration.TableName"), + }, + { + Name: "schema_configuration_version_id", + Description: "Specifies the table version for the output data schema", + Type: schema.TypeString, + Resolver: schema.PathResolver("DataFormatConversionConfiguration.SchemaConfiguration.VersionId"), + }, + { + Name: "dynamic_partitioning_enabled", + Description: "Specifies that the dynamic partitioning is enabled for this Kinesis Data Firehose delivery stream", + Type: schema.TypeBool, + Resolver: schema.PathResolver("DynamicPartitioningConfiguration.Enabled"), + }, + { + Name: "dynamic_partitioning_retry_options_duration_in_seconds", + Description: "The period of time during which Kinesis Data Firehose retries to deliver data to the specified Amazon S3 prefix", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("DynamicPartitioningConfiguration.RetryOptions.DurationInSeconds"), + }, + { + Name: "error_output_prefix", + Description: "A prefix that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3", + Type: schema.TypeString, + }, + { + Name: "prefix", + Description: "The \"YYYY/MM/DD/HH\" time format prefix is automatically used for delivered Amazon S3 files", + Type: schema.TypeString, + }, + { + Name: "processing_configuration_enabled", + Description: "Enables or disables data processing", + Type: schema.TypeBool, + Resolver: schema.PathResolver("ProcessingConfiguration.Enabled"), + }, + { + Name: "s3_backup_bucket_arn", + Description: "The ARN of the S3 bucket", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3BackupDescription.BucketARN"), + }, + { + Name: "s3_backup_buffering_hints_interval_in_seconds", + Description: "Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("S3BackupDescription.BufferingHints.IntervalInSeconds"), + }, + { + Name: "s3_backup_buffering_hints_size_in_mb_s", + Description: "Buffer incoming data to the specified size, in MiBs, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("S3BackupDescription.BufferingHints.SizeInMBs"), + }, + { + Name: "s3_backup_compression_format", + Description: "The compression format", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3BackupDescription.CompressionFormat"), + }, + { + Name: "s3_backup_kms_encryption_config_aws_kms_key_arn", + Description: "The Amazon Resource Name (ARN) of the encryption key", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3BackupDescription.EncryptionConfiguration.KMSEncryptionConfig.AWSKMSKeyARN"), + }, + { + Name: "s3_backup_no_encryption_config", + Description: "Specifically override existing encryption information to ensure that no encryption is used", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3BackupDescription.EncryptionConfiguration.NoEncryptionConfig"), + }, + { + Name: "s3_backup_role_arn", + Description: "The Amazon Resource Name (ARN) of the AWS credentials", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3BackupDescription.RoleARN"), + }, + { + Name: "s3_backup_cloud_watch_logging_options_enabled", + Description: "Enables or disables CloudWatch logging", + Type: schema.TypeBool, + Resolver: schema.PathResolver("S3BackupDescription.CloudWatchLoggingOptions.Enabled"), + }, + { + Name: "s3_backup_cloud_watch_logging_options_log_group_name", + Description: "The CloudWatch group name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3BackupDescription.CloudWatchLoggingOptions.LogGroupName"), + }, + { + Name: "s3_backup_cloud_watch_logging_options_log_stream_name", + Description: "The CloudWatch log stream name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3BackupDescription.CloudWatchLoggingOptions.LogStreamName"), + }, + { + Name: "s3_backup_error_output_prefix", + Description: "A prefix that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3BackupDescription.ErrorOutputPrefix"), + }, + { + Name: "s3_backup_prefix", + Description: "The \"YYYY/MM/DD/HH\" time format prefix is automatically used for delivered Amazon S3 files", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3BackupDescription.Prefix"), + }, + { + Name: "s3_backup_mode", + Description: "The Amazon S3 backup mode", + Type: schema.TypeString, + }, + }, + }, + { + Name: "aws_firehose_delivery_stream_http_destination", + Description: "Describes the HTTP endpoint destination", + Resolver: schema.PathTableResolver("Destinations.HttpEndpointDestinationDescription"), + Columns: []schema.Column{ + { + Name: "delivery_stream_cq_id", + Description: "Unique CloudQuery ID of aws_firehose_delivery_streams table (FK)", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "processing_configuration_processors", + Description: "Describes a data processing configuration", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("ProcessingConfiguration.Processors"), + }, + { + Name: "buffering_hints_interval_in_seconds", + Description: "Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("BufferingHints.IntervalInSeconds"), + }, + { + Name: "buffering_hints_size_in_mb_s", + Description: "Buffer incoming data to the specified size, in MBs, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("BufferingHints.SizeInMBs"), + }, + { + Name: "cloud_watch_logging_options_enabled", + Description: "Enables or disables CloudWatch logging", + Type: schema.TypeBool, + Resolver: schema.PathResolver("CloudWatchLoggingOptions.Enabled"), + }, + { + Name: "cloud_watch_logging_options_log_group_name", + Description: "The CloudWatch group name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("CloudWatchLoggingOptions.LogGroupName"), + }, + { + Name: "cloud_watch_logging_options_log_stream_name", + Description: "The CloudWatch log stream name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("CloudWatchLoggingOptions.LogStreamName"), + }, + { + Name: "endpoint_configuration_name", + Description: "The name of the HTTP endpoint selected as the destination", + Type: schema.TypeString, + Resolver: schema.PathResolver("EndpointConfiguration.Name"), + }, + { + Name: "endpoint_configuration_url", + Description: "The URL of the HTTP endpoint selected as the destination", + Type: schema.TypeString, + Resolver: schema.PathResolver("EndpointConfiguration.Url"), + }, + { + Name: "processing_configuration_enabled", + Description: "Enables or disables data processing", + Type: schema.TypeBool, + Resolver: schema.PathResolver("ProcessingConfiguration.Enabled"), + }, + { + Name: "request_configuration_common_attributes", + Description: "Describes the metadata sent to the HTTP endpoint destination", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("RequestConfiguration.CommonAttributes"), + }, + { + Name: "request_configuration_content_encoding", + Description: "Kinesis Data Firehose uses the content encoding to compress the body of a request before sending the request to the destination", + Type: schema.TypeString, + Resolver: schema.PathResolver("RequestConfiguration.ContentEncoding"), + }, + { + Name: "retry_options_duration_in_seconds", + Description: "The total amount of time that Kinesis Data Firehose spends on retries", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("RetryOptions.DurationInSeconds"), + }, + { + Name: "role_arn", + Description: "Kinesis Data Firehose uses this IAM role for all the permissions that the delivery stream needs", + Type: schema.TypeString, + Resolver: schema.PathResolver("RoleARN"), + }, + { + Name: "s3_backup_mode", + Description: "Describes the S3 bucket backup options for the data that Kinesis Firehose delivers to the HTTP endpoint destination", + Type: schema.TypeString, + }, + { + Name: "s3_destination_bucket_arn", + Description: "The ARN of the S3 bucket", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.BucketARN"), + }, + { + Name: "s3_destination_buffering_hints_interval_in_seconds", + Description: "Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("S3DestinationDescription.BufferingHints.IntervalInSeconds"), + }, + { + Name: "s3_destination_buffering_hints_size_in_mb_s", + Description: "Buffer incoming data to the specified size, in MiBs, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("S3DestinationDescription.BufferingHints.SizeInMBs"), + }, + { + Name: "s3_destination_compression_format", + Description: "The compression format", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.CompressionFormat"), + }, + { + Name: "s3_destination_kms_encryption_config_aws_kms_key_arn", + Description: "The Amazon Resource Name (ARN) of the encryption key", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.EncryptionConfiguration.KMSEncryptionConfig.AWSKMSKeyARN"), + }, + { + Name: "s3_destination_no_encryption_config", + Description: "Specifically override existing encryption information to ensure that no encryption is used", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.EncryptionConfiguration.NoEncryptionConfig"), + }, + { + Name: "s3_destination_role_arn", + Description: "The Amazon Resource Name (ARN) of the AWS credentials", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.RoleARN"), + }, + { + Name: "s3_destination_cloud_watch_logging_options_enabled", + Description: "Enables or disables CloudWatch logging", + Type: schema.TypeBool, + Resolver: schema.PathResolver("S3DestinationDescription.CloudWatchLoggingOptions.Enabled"), + }, + { + Name: "s3_destination_cloud_watch_logging_options_log_group_name", + Description: "The CloudWatch group name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.CloudWatchLoggingOptions.LogGroupName"), + }, + { + Name: "s3_destination_cloud_watch_logging_options_log_stream_name", + Description: "The CloudWatch log stream name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.CloudWatchLoggingOptions.LogStreamName"), + }, + { + Name: "s3_destination_error_output_prefix", + Description: "A prefix that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.ErrorOutputPrefix"), + }, + { + Name: "s3_destination_prefix", + Description: "The \"YYYY/MM/DD/HH\" time format prefix is automatically used for delivered Amazon S3 files", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.Prefix"), + }, + }, + }, + { + Name: "aws_firehose_delivery_stream_redshift_destination", + Description: "Describes a destination in Amazon Redshift", + Resolver: schema.PathTableResolver("Destinations.RedshiftDestinationDescription"), + Columns: []schema.Column{ + { + Name: "delivery_stream_cq_id", + Description: "Unique CloudQuery ID of aws_firehose_delivery_streams table (FK)", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "processing_configuration_processors", + Description: "Describes a data processing configuration", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("ProcessingConfiguration.Processors"), + }, + { + Name: "cluster_j_db_c_url", + Description: "The database connection string", + Type: schema.TypeString, + Resolver: schema.PathResolver("ClusterJDBCURL"), + }, + { + Name: "copy_command_data_table_name", + Description: "The name of the target table", + Type: schema.TypeString, + Resolver: schema.PathResolver("CopyCommand.DataTableName"), + }, + { + Name: "copy_command_copy_options", + Description: "Optional parameters to use with the Amazon Redshift COPY command", + Type: schema.TypeString, + Resolver: schema.PathResolver("CopyCommand.CopyOptions"), + }, + { + Name: "copy_command_data_table_columns", + Description: "A comma-separated list of column names", + Type: schema.TypeString, + Resolver: schema.PathResolver("CopyCommand.DataTableColumns"), + }, + { + Name: "role_arn", + Description: "The Amazon Resource Name (ARN) of the AWS credentials", + Type: schema.TypeString, + Resolver: schema.PathResolver("RoleARN"), + }, + { + Name: "s3_destination_bucket_arn", + Description: "The ARN of the S3 bucket", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.BucketARN"), + }, + { + Name: "s3_destination_buffering_hints_interval_in_seconds", + Description: "Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("S3DestinationDescription.BufferingHints.IntervalInSeconds"), + }, + { + Name: "s3_destination_buffering_hints_size_in_mb_s", + Description: "Buffer incoming data to the specified size, in MiBs, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("S3DestinationDescription.BufferingHints.SizeInMBs"), + }, + { + Name: "s3_destination_compression_format", + Description: "The compression format", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.CompressionFormat"), + }, + { + Name: "s3_destination_kms_encryption_config_aws_kms_key_arn", + Description: "The Amazon Resource Name (ARN) of the encryption key", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.EncryptionConfiguration.KMSEncryptionConfig.AWSKMSKeyARN"), + }, + { + Name: "s3_destination_no_encryption_config", + Description: "Specifically override existing encryption information to ensure that no encryption is used", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.EncryptionConfiguration.NoEncryptionConfig"), + }, + { + Name: "s3_destination_role_arn", + Description: "The Amazon Resource Name (ARN) of the AWS credentials", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.RoleARN"), + }, + { + Name: "s3_destination_cloud_watch_logging_options_enabled", + Description: "Enables or disables CloudWatch logging", + Type: schema.TypeBool, + Resolver: schema.PathResolver("S3DestinationDescription.CloudWatchLoggingOptions.Enabled"), + }, + { + Name: "s3_destination_cloud_watch_logging_options_log_group_name", + Description: "The CloudWatch group name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.CloudWatchLoggingOptions.LogGroupName"), + }, + { + Name: "s3_destination_cloud_watch_logging_options_log_stream_name", + Description: "The CloudWatch log stream name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.CloudWatchLoggingOptions.LogStreamName"), + }, + { + Name: "s3_destination_error_output_prefix", + Description: "A prefix that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.ErrorOutputPrefix"), + }, + { + Name: "s3_destination_prefix", + Description: "The \"YYYY/MM/DD/HH\" time format prefix is automatically used for delivered Amazon S3 files", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.Prefix"), + }, + { + Name: "username", + Description: "The name of the user", + Type: schema.TypeString, + }, + { + Name: "cloud_watch_logging_options_enabled", + Description: "Enables or disables CloudWatch logging", + Type: schema.TypeBool, + Resolver: schema.PathResolver("CloudWatchLoggingOptions.Enabled"), + }, + { + Name: "cloud_watch_logging_options_log_group_name", + Description: "The CloudWatch group name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("CloudWatchLoggingOptions.LogGroupName"), + }, + { + Name: "cloud_watch_logging_options_log_stream_name", + Description: "The CloudWatch log stream name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("CloudWatchLoggingOptions.LogStreamName"), + }, + { + Name: "processing_configuration_enabled", + Description: "Enables or disables data processing", + Type: schema.TypeBool, + Resolver: schema.PathResolver("ProcessingConfiguration.Enabled"), + }, + { + Name: "retry_options_duration_in_seconds", + Description: "The length of time during which Kinesis Data Firehose retries delivery after a failure, starting from the initial request and including the first attempt", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("RetryOptions.DurationInSeconds"), + }, + { + Name: "s3_backup_bucket_arn", + Description: "The ARN of the S3 bucket", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3BackupDescription.BucketARN"), + }, + { + Name: "s3_backup_buffering_hints_interval_in_seconds", + Description: "Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("S3BackupDescription.BufferingHints.IntervalInSeconds"), + }, + { + Name: "s3_backup_buffering_hints_size_in_mb_s", + Description: "Buffer incoming data to the specified size, in MiBs, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("S3BackupDescription.BufferingHints.SizeInMBs"), + }, + { + Name: "s3_backup_compression_format", + Description: "The compression format", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3BackupDescription.CompressionFormat"), + }, + { + Name: "s3_backup_kms_encryption_config_aws_kms_key_arn", + Description: "The Amazon Resource Name (ARN) of the encryption key", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3BackupDescription.EncryptionConfiguration.KMSEncryptionConfig.AWSKMSKeyARN"), + }, + { + Name: "s3_backup_no_encryption_config", + Description: "Specifically override existing encryption information to ensure that no encryption is used", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3BackupDescription.EncryptionConfiguration.NoEncryptionConfig"), + }, + { + Name: "s3_backup_role_arn", + Description: "The Amazon Resource Name (ARN) of the AWS credentials", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3BackupDescription.RoleARN"), + }, + { + Name: "s3_backup_cloud_watch_logging_options_enabled", + Description: "Enables or disables CloudWatch logging", + Type: schema.TypeBool, + Resolver: schema.PathResolver("S3BackupDescription.CloudWatchLoggingOptions.Enabled"), + }, + { + Name: "s3_backup_cloud_watch_logging_options_log_group_name", + Description: "The CloudWatch group name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3BackupDescription.CloudWatchLoggingOptions.LogGroupName"), + }, + { + Name: "s3_backup_cloud_watch_logging_options_log_stream_name", + Description: "The CloudWatch log stream name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3BackupDescription.CloudWatchLoggingOptions.LogStreamName"), + }, + { + Name: "s3_backup_error_output_prefix", + Description: "A prefix that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3BackupDescription.ErrorOutputPrefix"), + }, + { + Name: "s3_backup_prefix", + Description: "The \"YYYY/MM/DD/HH\" time format prefix is automatically used for delivered Amazon S3 files", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3BackupDescription.Prefix"), + }, + { + Name: "s3_backup_mode", + Description: "The Amazon S3 backup mode", + Type: schema.TypeString, + }, + }, + }, + { + Name: "aws_firehose_delivery_stream_splunk_destination", + Description: "Describes a destination in Splunk", + Resolver: schema.PathTableResolver("Destinations.SplunkDestinationDescription"), + Columns: []schema.Column{ + { + Name: "delivery_stream_cq_id", + Description: "Unique CloudQuery ID of aws_firehose_delivery_streams table (FK)", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "processing_configuration_processors", + Description: "Describes a data processing configuration", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("ProcessingConfiguration.Processors"), + }, + { + Name: "cloud_watch_logging_options_enabled", + Description: "Enables or disables CloudWatch logging", + Type: schema.TypeBool, + Resolver: schema.PathResolver("CloudWatchLoggingOptions.Enabled"), + }, + { + Name: "cloud_watch_logging_options_log_group_name", + Description: "The CloudWatch group name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("CloudWatchLoggingOptions.LogGroupName"), + }, + { + Name: "cloud_watch_logging_options_log_stream_name", + Description: "The CloudWatch log stream name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("CloudWatchLoggingOptions.LogStreamName"), + }, + { + Name: "h_e_ca_cknowledgment_timeout_in_seconds", + Description: "The amount of time that Kinesis Data Firehose waits to receive an acknowledgment from Splunk after it sends it data", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("HECAcknowledgmentTimeoutInSeconds"), + }, + { + Name: "h_e_c_endpoint", + Description: "The HTTP Event Collector (HEC) endpoint to which Kinesis Data Firehose sends your data", + Type: schema.TypeString, + }, + { + Name: "h_e_c_endpoint_type", + Description: "This type can be either \"Raw\" or \"Event\"", + Type: schema.TypeString, + }, + { + Name: "h_e_c_token", + Description: "A GUID you obtain from your Splunk cluster when you create a new HEC endpoint", + Type: schema.TypeString, + }, + { + Name: "processing_configuration_enabled", + Description: "Enables or disables data processing", + Type: schema.TypeBool, + Resolver: schema.PathResolver("ProcessingConfiguration.Enabled"), + }, + { + Name: "retry_options_duration_in_seconds", + Description: "The total amount of time that Kinesis Data Firehose spends on retries", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("RetryOptions.DurationInSeconds"), + }, + { + Name: "s3_backup_mode", + Description: "Defines how documents should be delivered to Amazon S3", + Type: schema.TypeString, + }, + { + Name: "s3_destination_bucket_arn", + Description: "The ARN of the S3 bucket", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.BucketARN"), + }, + { + Name: "s3_destination_buffering_hints_interval_in_seconds", + Description: "Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("S3DestinationDescription.BufferingHints.IntervalInSeconds"), + }, + { + Name: "s3_destination_buffering_hints_size_in_mb_s", + Description: "Buffer incoming data to the specified size, in MiBs, before delivering it to the destination", + Type: schema.TypeBigInt, + Resolver: schema.PathResolver("S3DestinationDescription.BufferingHints.SizeInMBs"), + }, + { + Name: "s3_destination_compression_format", + Description: "The compression format", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.CompressionFormat"), + }, + { + Name: "s3_destination_kms_encryption_config_aws_kms_key_arn", + Description: "The Amazon Resource Name (ARN) of the encryption key", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.EncryptionConfiguration.KMSEncryptionConfig.AWSKMSKeyARN"), + }, + { + Name: "s3_destination_no_encryption_config", + Description: "Specifically override existing encryption information to ensure that no encryption is used", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.EncryptionConfiguration.NoEncryptionConfig"), + }, + { + Name: "s3_destination_role_arn", + Description: "The Amazon Resource Name (ARN) of the AWS credentials", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.RoleARN"), + }, + { + Name: "s3_destination_cloud_watch_logging_options_enabled", + Description: "Enables or disables CloudWatch logging", + Type: schema.TypeBool, + Resolver: schema.PathResolver("S3DestinationDescription.CloudWatchLoggingOptions.Enabled"), + }, + { + Name: "s3_destination_cloud_watch_logging_options_log_group_name", + Description: "The CloudWatch group name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.CloudWatchLoggingOptions.LogGroupName"), + }, + { + Name: "s3_destination_cloud_watch_logging_options_log_stream_name", + Description: "The CloudWatch log stream name for logging", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.CloudWatchLoggingOptions.LogStreamName"), + }, + { + Name: "s3_destination_error_output_prefix", + Description: "A prefix that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.ErrorOutputPrefix"), + }, + { + Name: "s3_destination_prefix", + Description: "The \"YYYY/MM/DD/HH\" time format prefix is automatically used for delivered Amazon S3 files", + Type: schema.TypeString, + Resolver: schema.PathResolver("S3DestinationDescription.Prefix"), + }, + }, + }, + }, + } +} + +// ==================================================================================================================== +// Table Resolver Functions +// ==================================================================================================================== + +func fetchFirehoseDeliveryStreams(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { + return diag.WrapError(client.ListAndDetailResolver(ctx, meta, res, listDeliveryStreams, deliveryStreamDetail)) +} +func resolveFirehoseDeliveryStreamTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { + cl := meta.(*client.Client) + svc := cl.Services().Firehose + summary := resource.Item.(*types.DeliveryStreamDescription) + input := firehose.ListTagsForDeliveryStreamInput{ + DeliveryStreamName: summary.DeliveryStreamName, + } + var tags []types.Tag + for { + output, err := svc.ListTagsForDeliveryStream(ctx, &input) + if err != nil { + return diag.WrapError(err) + } + tags = append(tags, output.Tags...) + if !aws.ToBool(output.HasMoreTags) { + break + } + input.ExclusiveStartTagKey = aws.String(*output.Tags[len(output.Tags)-1].Key) + } + return diag.WrapError(resource.Set(c.Name, client.TagsToMap(tags))) +} + +// ==================================================================================================================== +// User Defined Helpers +// ==================================================================================================================== + +func listDeliveryStreams(ctx context.Context, meta schema.ClientMeta, detailChan chan<- interface{}) error { + c := meta.(*client.Client) + svc := c.Services().Firehose + input := firehose.ListDeliveryStreamsInput{} + for { + response, err := svc.ListDeliveryStreams(ctx, &input) + if err != nil { + return diag.WrapError(err) + } + for _, item := range response.DeliveryStreamNames { + detailChan <- item + } + if !aws.ToBool(response.HasMoreDeliveryStreams) { + break + } + input.ExclusiveStartDeliveryStreamName = aws.String(response.DeliveryStreamNames[len(response.DeliveryStreamNames)-1]) + } + return nil +} +func deliveryStreamDetail(ctx context.Context, meta schema.ClientMeta, resultsChan chan<- interface{}, errorChan chan<- error, listInfo interface{}) { + c := meta.(*client.Client) + streamName := listInfo.(string) + svc := c.Services().Firehose + streamSummary, err := svc.DescribeDeliveryStream(ctx, &firehose.DescribeDeliveryStreamInput{ + DeliveryStreamName: aws.String(streamName), + }) + if err != nil { + if c.IsNotFoundError(err) { + return + } + errorChan <- diag.WrapError(err) + return + } + resultsChan <- streamSummary.DeliveryStreamDescription +} diff --git a/resources/services/firehose/delivery_streams_mock_test.go b/resources/services/firehose/delivery_streams_mock_test.go new file mode 100644 index 000000000..836d47b72 --- /dev/null +++ b/resources/services/firehose/delivery_streams_mock_test.go @@ -0,0 +1,52 @@ +package firehose + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/firehose" + "github.com/aws/aws-sdk-go-v2/service/firehose/types" + "github.com/bxcodec/faker" + "github.com/cloudquery/cq-provider-aws/client" + "github.com/cloudquery/cq-provider-aws/client/mocks" + "github.com/golang/mock/gomock" +) + +func buildKinesisFirehoses(t *testing.T, ctrl *gomock.Controller) client.Services { + f := mocks.NewMockFirehoseClient(ctrl) + + streams := firehose.ListDeliveryStreamsOutput{} + err := faker.FakeData(&streams) + if err != nil { + t.Fatal(err) + } + streams.HasMoreDeliveryStreams = aws.Bool(false) + streams.DeliveryStreamNames = []string{"test-stream"} + f.EXPECT().ListDeliveryStreams(gomock.Any(), gomock.Any(), gomock.Any()).Return(&streams, nil) + + stream := firehose.DescribeDeliveryStreamOutput{} + + err = faker.FakeData(&stream) + if err != nil { + t.Fatal(err) + } + stream.DeliveryStreamDescription.Destinations = []types.DestinationDescription{stream.DeliveryStreamDescription.Destinations[0]} + + f.EXPECT().DescribeDeliveryStream(gomock.Any(), gomock.Any(), gomock.Any()).MinTimes(1).Return(&stream, nil) + + tags := firehose.ListTagsForDeliveryStreamOutput{} + err = faker.FakeData(&tags) + if err != nil { + t.Fatal(err) + } + tags.HasMoreTags = aws.Bool(false) + f.EXPECT().ListTagsForDeliveryStream(gomock.Any(), gomock.Any(), gomock.Any()).MinTimes(1).Return(&tags, nil) + + return client.Services{ + Firehose: f, + } +} + +func TestFirehoses(t *testing.T) { + client.AwsMockTestHelper(t, DeliveryStreams(), buildKinesisFirehoses, client.TestOptions{}) +} diff --git a/resources/services/firehose/gen.hcl b/resources/services/firehose/gen.hcl new file mode 100644 index 000000000..f81ec0c03 --- /dev/null +++ b/resources/services/firehose/gen.hcl @@ -0,0 +1,256 @@ +//check-for-changes +service = "aws" +output_directory = "." +add_generate = true + +description_modifier "remove_read_only" { + words = [" This member is required."] +} + + +resource "aws" "firehose" "delivery_streams" { + path = "github.com/aws/aws-sdk-go-v2/service/firehose/types.DeliveryStreamDescription" + ignoreError "IgnoreCommonErrors" { + path = "github.com/cloudquery/cq-provider-aws/client.IgnoreCommonErrors" + } + deleteFilter "AccountRegionFilter" { + path = "github.com/cloudquery/cq-provider-aws/client.DeleteAccountRegionFilter" + } + multiplex "AwsAccountRegion" { + path = "github.com/cloudquery/cq-provider-aws/client.ServiceAccountRegionMultiplexer" + params = ["firehose"] + } + userDefinedColumn "account_id" { + description = "The AWS Account ID of the resource." + type = "string" + resolver "resolveAWSAccount" { + path = "github.com/cloudquery/cq-provider-aws/client.ResolveAWSAccount" + } + } + userDefinedColumn "region" { + type = "string" + description = "The AWS Region of the resource." + resolver "resolveAWSRegion" { + path = "github.com/cloudquery/cq-provider-aws/client.ResolveAWSRegion" + } + } + options { + primary_keys = ["arn"] + } + userDefinedColumn "tags" { + type = "json" + generate_resolver = true + } + column "destinations" { + skip = true + } + column "has_more_destinations" { + skip = true + } + userDefinedColumn "arn" { + type = "string" + description = "The Amazon Resource Name (ARN) of the delivery stream" + resolver "resolveStreamArn" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["DeliveryStreamARN"] + } + } + column "delivery_stream_encryption_configuration" { + rename = "encryption_config" + } + column "source" { + rename = "source" + } + column "source_kinesis_stream_source_description" { + rename = "_kinesis_stream" + // skip_prefix = true + } + user_relation "aws" "kinesis" "open_search_destination" { + path = "github.com/aws/aws-sdk-go-v2/service/firehose/types.AmazonopensearchserviceDestinationDescription" + resolver "resolveTable" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathTableResolver" + path_resolver = true + params = ["Destinations.AmazonopensearchserviceDestinationDescription"] + } + column "s3_destination_description" { + rename = "s3_destination" + } + column "s3_destination_encryption_configuration" { + skip_prefix = true + } + column "s3_destination_cloud_watch_logging_options_" { + skip_prefix = true + } + column "processing_configuration_processors" { + skip = true + } + userDefinedColumn "processing_configuration_processors" { + type = "json" + description = "Describes a data processing configuration" + resolver "resolveProcessors" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["ProcessingConfiguration.Processors"] + } + } + } + user_relation "aws" "kinesis" "elasticsearch_destination" { + path = "github.com/aws/aws-sdk-go-v2/service/firehose/types.ElasticsearchDestinationDescription" + resolver "resolveTable" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathTableResolver" + path_resolver = true + params = ["Destinations.ElasticsearchDestinationDescription"] + } + column "s3_destination_description" { + rename = "s3_destination" + } + column "s3_destination_encryption_configuration" { + skip_prefix = true + } + column "processing_configuration_processors" { + skip = true + } + userDefinedColumn "processing_configuration_processors" { + type = "json" + description = "Describes a data processing configuration" + resolver "resolveProcessors" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["ProcessingConfiguration.Processors"] + } + } + } + user_relation "aws" "kinesis" "extended_s3_destination" { + path = "github.com/aws/aws-sdk-go-v2/service/firehose/types.ExtendedS3DestinationDescription" + resolver "resolveTable" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathTableResolver" + path_resolver = true + params = ["Destinations.ExtendedS3DestinationDescription"] + } + column "data_format_conversion_configuration" { + skip_prefix = true + } + column "input_format_configuration" { + skip_prefix = true + } + column "deserializer_open_x_json_ser_de_convert_dots_in_json_keys_to_underscores" { + rename = "deserializer_open_x_json_ser_de_convert_dots_to_underscores" + } + column "dynamic_partitioning_configuration" { + rename = "dynamic_partitioning" + } + column "output_format_configuration" { + skip_prefix = true + } + column "s3_backup_description" { + rename = "s3_backup" + } + column "s3_backup_encryption_configuration" { + skip_prefix = true + } + column "processing_configuration_processors" { + skip = true + } + userDefinedColumn "processing_configuration_processors" { + type = "json" + description = "Describes a data processing configuration" + resolver "resolveProcessors" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["ProcessingConfiguration.Processors"] + } + } + } + user_relation "aws" "kinesis" "http_destination" { + path = "github.com/aws/aws-sdk-go-v2/service/firehose/types.HttpEndpointDestinationDescription" + resolver "resolveTable" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathTableResolver" + path_resolver = true + params = ["Destinations.HttpEndpointDestinationDescription"] + } + column "s3_destination_description" { + rename = "s3_destination" + } + column "s3_destination_encryption_configuration" { + skip_prefix = true + } + column "processing_configuration_processors" { + skip = true + } + userDefinedColumn "processing_configuration_processors" { + type = "json" + description = "Describes a data processing configuration" + resolver "resolveProcessors" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["ProcessingConfiguration.Processors"] + } + } + column "request_configuration_common_attributes" { + // skip = true + type = "json" + resolver "pathResolver" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["RequestConfiguration.CommonAttributes"] + } + } + } + user_relation "aws" "kinesis" "redshift_destination" { + path = "github.com/aws/aws-sdk-go-v2/service/firehose/types.RedshiftDestinationDescription" + resolver "resolveTable" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathTableResolver" + path_resolver = true + params = ["Destinations.RedshiftDestinationDescription"] + } + column "s3_destination_encryption_configuration" { + skip_prefix = true + } + //s3_backup_encryption_configuration_kms_encryption_config_aws_kms_key_arn + // column "s3_destination_encryption_configuration_kms_encryption" { + // skip_prefix = true + // } + column "s3_destination_description" { + rename = "s3_destination" + } + column "s3_backup_description" { + rename = "s3_backup" + } + + column "s3_backup_encryption_configuration" { + skip_prefix = true + } + column "processing_configuration_processors" { + skip = true + } + userDefinedColumn "processing_configuration_processors" { + type = "json" + description = "Describes a data processing configuration" + resolver "resolveProcessors" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["ProcessingConfiguration.Processors"] + } + } + } + user_relation "aws" "kinesis" "splunk_destination" { + path = "github.com/aws/aws-sdk-go-v2/service/firehose/types.SplunkDestinationDescription" + resolver "resolveTable" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathTableResolver" + path_resolver = true + params = ["Destinations.SplunkDestinationDescription"] + } + column "s3_destination_description" { + rename = "s3_destination" + } + column "s3_destination_encryption_configuration" { + skip_prefix = true + } + + column "processing_configuration_processors" { + skip = true + } + userDefinedColumn "processing_configuration_processors" { + type = "json" + description = "Describes a data processing configuration" + resolver "resolveProcessors" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["ProcessingConfiguration.Processors"] + } + } + } +} diff --git a/resources/services/kinesis/gen.hcl b/resources/services/kinesis/gen.hcl index d73afcb1d..207d13834 100644 --- a/resources/services/kinesis/gen.hcl +++ b/resources/services/kinesis/gen.hcl @@ -1,11 +1,15 @@ +//check-for-changes +service = "aws" +output_directory = "." +add_generate = true + description_modifier "remove_read_only" { words = [" This member is required."] } - resource "aws" "kinesis" "streams" { path = "github.com/aws/aws-sdk-go-v2/service/kinesis/types.StreamDescriptionSummary" - ignoreError "IgnoreAccessDenied" { - path = "github.com/cloudquery/cq-provider-aws/client.IgnoreAccessDeniedServiceDisabled" + ignoreError "IgnoreCommonErrors" { + path = "github.com/cloudquery/cq-provider-aws/client.IgnoreCommonErrors" } deleteFilter "AccountRegionFilter" { path = "github.com/cloudquery/cq-provider-aws/client.DeleteAccountRegionFilter" @@ -14,7 +18,6 @@ resource "aws" "kinesis" "streams" { path = "github.com/cloudquery/cq-provider-aws/client.ServiceAccountRegionMultiplexer" params = ["kinesis"] } - userDefinedColumn "account_id" { description = "The AWS Account ID of the resource." type = "string" @@ -22,7 +25,6 @@ resource "aws" "kinesis" "streams" { path = "github.com/cloudquery/cq-provider-aws/client.ResolveAWSAccount" } } - userDefinedColumn "region" { type = "string" description = "The AWS Region of the resource." @@ -31,11 +33,9 @@ resource "aws" "kinesis" "streams" { } } ignore_columns_in_tests = ["kms_key_id", "retention_in_days"] - options { primary_keys = ["arn"] } - userDefinedColumn "arn" { type = "string" resolver "resolveStreamArn" { @@ -45,9 +45,255 @@ resource "aws" "kinesis" "streams" { params = ["stream_arn"] } } + userDefinedColumn "tags" { + type = "json" + generate_resolver = true + } +} +resource "aws" "kinesis" "firehoses" { + path = "github.com/aws/aws-sdk-go-v2/service/firehose/types.DeliveryStreamDescription" + ignoreError "IgnoreCommonErrors" { + path = "github.com/cloudquery/cq-provider-aws/client.IgnoreCommonErrors" + } + deleteFilter "AccountRegionFilter" { + path = "github.com/cloudquery/cq-provider-aws/client.DeleteAccountRegionFilter" + } + multiplex "AwsAccountRegion" { + path = "github.com/cloudquery/cq-provider-aws/client.ServiceAccountRegionMultiplexer" + params = ["firehose"] + } + userDefinedColumn "account_id" { + description = "The AWS Account ID of the resource." + type = "string" + resolver "resolveAWSAccount" { + path = "github.com/cloudquery/cq-provider-aws/client.ResolveAWSAccount" + } + } + userDefinedColumn "region" { + type = "string" + description = "The AWS Region of the resource." + resolver "resolveAWSRegion" { + path = "github.com/cloudquery/cq-provider-aws/client.ResolveAWSRegion" + } + } + options { + primary_keys = ["arn"] + } userDefinedColumn "tags" { type = "json" generate_resolver = true } + column "destinations" { + skip = true + } + column "has_more_destinations" { + skip = true + } + userDefinedColumn "arn" { + type = "string" + description = "The Amazon Resource Name (ARN) of the delivery stream" + resolver "resolveStreamArn" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["DeliveryStreamARN"] + } + } + column "delivery_stream_encryption_configuration" { + rename = "encryption_config" + } + column "source" { + rename = "source" + } + column "source_kinesis_stream_source_description" { + rename = "_kinesis_stream" + // skip_prefix = true + } + user_relation "aws" "kinesis" "open_search_destination" { + path = "github.com/aws/aws-sdk-go-v2/service/firehose/types.AmazonopensearchserviceDestinationDescription" + resolver "resolveTable" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathTableResolver" + path_resolver = true + params = ["Destinations.AmazonopensearchserviceDestinationDescription"] + } + column "s3_destination_description" { + rename = "s3_destination" + } + column "s3_destination_encryption_configuration" { + skip_prefix = true + } + column "s3_destination_cloud_watch_logging_options_" { + skip_prefix = true + } + column "processing_configuration_processors" { + skip = true + } + userDefinedColumn "processing_configuration_processors" { + type = "json" + description = "Describes a data processing configuration" + resolver "resolveProcessors" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["ProcessingConfiguration.Processors"] + } + } + } + user_relation "aws" "kinesis" "elasticsearch_destination" { + path = "github.com/aws/aws-sdk-go-v2/service/firehose/types.ElasticsearchDestinationDescription" + resolver "resolveTable" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathTableResolver" + path_resolver = true + params = ["Destinations.ElasticsearchDestinationDescription"] + } + column "s3_destination_description" { + rename = "s3_destination" + } + column "s3_destination_encryption_configuration" { + skip_prefix = true + } + column "processing_configuration_processors" { + skip = true + } + userDefinedColumn "processing_configuration_processors" { + type = "json" + description = "Describes a data processing configuration" + resolver "resolveProcessors" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["ProcessingConfiguration.Processors"] + } + } + } + user_relation "aws" "kinesis" "extended_s3_destination" { + path = "github.com/aws/aws-sdk-go-v2/service/firehose/types.ExtendedS3DestinationDescription" + resolver "resolveTable" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathTableResolver" + path_resolver = true + params = ["Destinations.ExtendedS3DestinationDescription"] + } + column "data_format_conversion_configuration" { + skip_prefix = true + } + column "input_format_configuration" { + skip_prefix = true + } + column "deserializer_open_x_json_ser_de_convert_dots_in_json_keys_to_underscores" { + rename = "deserializer_open_x_json_ser_de_convert_dots_to_underscores" + } + column "dynamic_partitioning_configuration" { + rename = "dynamic_partitioning" + } + column "output_format_configuration" { + skip_prefix = true + } + column "s3_backup_description" { + rename = "s3_backup" + } + column "s3_backup_encryption_configuration" { + skip_prefix = true + } + column "processing_configuration_processors" { + skip = true + } + userDefinedColumn "processing_configuration_processors" { + type = "json" + description = "Describes a data processing configuration" + resolver "resolveProcessors" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["ProcessingConfiguration.Processors"] + } + } + } + user_relation "aws" "kinesis" "http_destination" { + path = "github.com/aws/aws-sdk-go-v2/service/firehose/types.HttpEndpointDestinationDescription" + resolver "resolveTable" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathTableResolver" + path_resolver = true + params = ["Destinations.HttpEndpointDestinationDescription"] + } + column "s3_destination_description" { + rename = "s3_destination" + } + column "s3_destination_encryption_configuration" { + skip_prefix = true + } + column "processing_configuration_processors" { + skip = true + } + userDefinedColumn "processing_configuration_processors" { + type = "json" + description = "Describes a data processing configuration" + resolver "resolveProcessors" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["ProcessingConfiguration.Processors"] + } + } + column "request_configuration_common_attributes" { + // skip = true + type = "json" + resolver "pathResolver" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["RequestConfiguration.CommonAttributes"] + } + } + } + user_relation "aws" "kinesis" "redshift_destination" { + path = "github.com/aws/aws-sdk-go-v2/service/firehose/types.RedshiftDestinationDescription" + resolver "resolveTable" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathTableResolver" + path_resolver = true + params = ["Destinations.RedshiftDestinationDescription"] + } + column "s3_destination_encryption_configuration" { + skip_prefix = true + } + //s3_backup_encryption_configuration_kms_encryption_config_aws_kms_key_arn + // column "s3_destination_encryption_configuration_kms_encryption" { + // skip_prefix = true + // } + column "s3_destination_description" { + rename = "s3_destination" + } + column "s3_backup_description" { + rename = "s3_backup" + } + + column "s3_backup_encryption_configuration" { + skip_prefix = true + } + column "processing_configuration_processors" { + skip = true + } + userDefinedColumn "processing_configuration_processors" { + type = "json" + description = "Describes a data processing configuration" + resolver "resolveProcessors" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["ProcessingConfiguration.Processors"] + } + } + } + user_relation "aws" "kinesis" "splunk_destination" { + path = "github.com/aws/aws-sdk-go-v2/service/firehose/types.SplunkDestinationDescription" + resolver "resolveTable" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathTableResolver" + path_resolver = true + params = ["Destinations.SplunkDestinationDescription"] + } + column "s3_destination_description" { + rename = "s3_destination" + } + column "s3_destination_encryption_configuration" { + skip_prefix = true + } + + column "processing_configuration_processors" { + skip = true + } + userDefinedColumn "processing_configuration_processors" { + type = "json" + description = "Describes a data processing configuration" + resolver "resolveProcessors" { + path = "github.com/cloudquery/cq-provider-sdk/provider/schema.PathResolver" + params = ["ProcessingConfiguration.Processors"] + } + } + } }