fix: Update timestamps fields (#891)

* fix: Update timestamps fields * progress * lint * add docs * fixed tests * remove migrations
cloudquery · May 27, 2022 · 48b9e6f · 48b9e6f
1 parent 5702860
commit 48b9e6f
Show file tree

Hide file tree

Showing 18 changed files with 73 additions and 21 deletions.
diff --git a/client/resolvers.go b/client/resolvers.go
@@ -3,9 +3,11 @@ package client
 import (
 	"context"
 	"reflect"
+	"time"
 
 	"github.com/cloudquery/cq-provider-sdk/provider/diag"
 	"github.com/cloudquery/cq-provider-sdk/provider/schema"
+	"github.com/thoas/go-funk"
 )
 
 func ResolveAWSAccount(_ context.Context, meta schema.ClientMeta, r *schema.Resource, _ schema.Column) error {
@@ -54,3 +56,29 @@ func ResolveTagField(fieldName string) func(context.Context, schema.ClientMeta,
 		return diag.WrapError(r.Set(c.Name, data))
 	}
 }
+
+func ResolveTimestampField(path string, rfcs ...string) func(_ context.Context, _ schema.ClientMeta, r *schema.Resource, c schema.Column) error {
+	return func(ctx context.Context, cl schema.ClientMeta, r *schema.Resource, c schema.Column) error {
+		var val reflect.Value
+
+		value := funk.Get(r.Item, path, funk.WithAllowZero())
+		if value == nil {
+			return diag.WrapError(r.Set(c.Name, nil))
+		}
+
+		if reflect.TypeOf(value).Kind() == reflect.Ptr {
+			val = reflect.ValueOf(value).Elem()
+		} else {
+			val = reflect.ValueOf(value)
+		}
+
+		switch val.Kind() {
+		case reflect.Int32, reflect.Int64:
+			return diag.WrapError(r.Set(c.Name, time.Unix(val.Int(), 0)))
+		case reflect.String:
+			return schema.DateResolver(path, rfcs...)(ctx, cl, r, c)
+		default:
+			return diag.WrapError(r.Set(c.Name, nil))
+		}
+	}
+}
diff --git a/docs/tables/aws_ec2_images.md b/docs/tables/aws_ec2_images.md
@@ -9,7 +9,7 @@ Describes an image.
 |arn|text|The Amazon Resource Name (ARN) for the resource.|
 |id|text|The ID of the AMI.|
 |architecture|text|The architecture of the image.|
-|creation_date|text|The date and time the image was created.|
+|creation_date|timestamp without time zone|The date and time the image was created.|
 |description|text|The description of the AMI that was provided during image creation.|
 |ena_support|boolean|Specifies whether enhanced networking with ENA is enabled.|
 |hypervisor|text|The hypervisor type of the image.|

diff --git a/docs/tables/aws_ec2_instance_elastic_gpu_associations.md b/docs/tables/aws_ec2_instance_elastic_gpu_associations.md
@@ -7,5 +7,5 @@ Describes the association between an instance and an Elastic Graphics accelerato
 |instance_cq_id|uuid|Unique CloudQuery ID of aws_ec2_instances table (FK)|
 |elastic_gpu_association_id|text|The ID of the association.|
 |elastic_gpu_association_state|text|The state of the association between the instance and the Elastic Graphics accelerator.|
-|elastic_gpu_association_time|text|The time the Elastic Graphics accelerator was associated with the instance.|
+|elastic_gpu_association_time|timestamp without time zone|The time the Elastic Graphics accelerator was associated with the instance.|
 |elastic_gpu_id|text|The ID of the Elastic Graphics accelerator.|
diff --git a/docs/tables/aws_lambda_function_concurrency_configs.md b/docs/tables/aws_lambda_function_concurrency_configs.md
@@ -8,7 +8,7 @@ Details about the provisioned concurrency configuration for a function alias or
 |allocated_provisioned_concurrent_executions|integer|The amount of provisioned concurrency allocated.|
 |available_provisioned_concurrent_executions|integer|The amount of provisioned concurrency available.|
 |function_arn|text|The Amazon Resource Name (ARN) of the alias or version.|
-|last_modified|text|The date and time that a user last updated the configuration, in ISO 8601 format (https://www.iso.org/iso-8601-date-and-time-format.html).|
+|last_modified|timestamp without time zone|The date and time that a user last updated the configuration, in ISO 8601 format (https://www.iso.org/iso-8601-date-and-time-format.html).|
 |requested_provisioned_concurrent_executions|integer|The amount of provisioned concurrency requested.|
 |status|text|The status of the allocation process.|
 |status_reason|text|For failed allocations, the reason that provisioned concurrency could not be allocated.|
diff --git a/docs/tables/aws_lambda_function_versions.md b/docs/tables/aws_lambda_function_versions.md
@@ -23,7 +23,7 @@ Details about a function's configuration.
 |image_config_entry_point|text[]|Specifies the entry point to their application, which is typically the location of the runtime executable.|
 |image_config_working_directory|text|Specifies the working directory.|
 |kms_key_arn|text|The KMS key that's used to encrypt the function's environment variables|
-|last_modified|text|The date and time that the function was last updated, in ISO-8601 format (https://www.w3.org/TR/NOTE-datetime) (YYYY-MM-DDThh:mm:ss.sTZD).|
+|last_modified|timestamp without time zone|The date and time that the function was last updated, in ISO-8601 format (https://www.w3.org/TR/NOTE-datetime) (YYYY-MM-DDThh:mm:ss.sTZD).|
 |last_update_status|text|The status of the last update that was performed on the function|
 |last_update_status_reason|text|The reason for the last update that was performed on the function.|
 |last_update_status_reason_code|text|The reason code for the last update that was performed on the function.|

diff --git a/docs/tables/aws_lambda_functions.md b/docs/tables/aws_lambda_functions.md
@@ -37,7 +37,7 @@ AWS Lambda is a serverless compute service that lets you run code without provis
 |image_config_entry_point|text[]|Specifies the entry point to their application, which is typically the location of the runtime executable.|
 |image_config_working_directory|text|Specifies the working directory.|
 |kms_key_arn|text|The KMS key that's used to encrypt the function's environment variables|
-|last_modified|text|The date and time that the function was last updated, in ISO-8601 format (https://www.w3.org/TR/NOTE-datetime) (YYYY-MM-DDThh:mm:ss.sTZD).|
+|last_modified|timestamp without time zone|The date and time that the function was last updated, in ISO-8601 format (https://www.w3.org/TR/NOTE-datetime) (YYYY-MM-DDThh:mm:ss.sTZD).|
 |last_update_status|text|The status of the last update that was performed on the function|
 |last_update_status_reason|text|The reason for the last update that was performed on the function.|
 |last_update_status_reason_code|text|The reason code for the last update that was performed on the function.|

diff --git a/docs/tables/aws_lambda_layer_versions.md b/docs/tables/aws_lambda_layer_versions.md
@@ -6,7 +6,7 @@ Details about a version of an AWS Lambda layer (https://docs.aws.amazon.com/lamb
 | ------------- | ------------- | -----  |
 |layer_cq_id|uuid|Unique CloudQuery ID of aws_lambda_layers table (FK)|
 |compatible_runtimes|text[]|The layer's compatible runtimes.|
-|created_date|text|The date that the version was created, in ISO 8601 format|
+|created_date|timestamp without time zone|The date that the version was created, in ISO 8601 format|
 |description|text|The description of the version.|
 |layer_version_arn|text|The ARN of the layer version.|
 |license_info|text|The layer's open-source license.|

diff --git a/docs/tables/aws_lambda_layers.md b/docs/tables/aws_lambda_layers.md
@@ -7,7 +7,7 @@ Details about an AWS Lambda layer (https://docs.aws.amazon.com/lambda/latest/dg/
 |account_id|text|The AWS Account ID of the resource.|
 |region|text|The AWS Region of the resource.|
 |latest_matching_version_compatible_runtimes|text[]|The layer's compatible runtimes.|
-|latest_matching_version_created_date|text|The date that the version was created, in ISO 8601 format|
+|latest_matching_version_created_date|timestamp without time zone|The date that the version was created, in ISO 8601 format|
 |latest_matching_version_description|text|The description of the version.|
 |latest_matching_version_layer_version_arn|text|The ARN of the layer version.|
 |latest_matching_version_license_info|text|The layer's open-source license.|

diff --git a/docs/tables/aws_sqs_queues.md b/docs/tables/aws_sqs_queues.md
@@ -13,8 +13,8 @@ Simple Queue Service
 |message_retention_period|integer|The length of time, in seconds, for which Amazon SQS retains a message.|
 |approximate_number_of_messages|integer|The approximate number of messages available for retrieval from the queue.|
 |approximate_number_of_messages_not_visible|integer|The approximate number of messages that are in flight.|
-|created_timestamp|integer|UNIX time when the queue was created.|
-|last_modified_timestamp|integer|UNIX time when the queue was last changed.|
+|created_timestamp|timestamp without time zone|UNIX time when the queue was created.|
+|last_modified_timestamp|timestamp without time zone|UNIX time when the queue was last changed.|
 |arn|text|Amazon resource name (ARN) of the queue.|
 |approximate_number_of_messages_delayed|integer|The approximate number of messages in the queue that are delayed and not available for reading immediately.|
 |delay_seconds|integer|The default delay on the queue in seconds.|

diff --git a/resources/services/ec2/images.go b/resources/services/ec2/images.go
@@ -57,7 +57,8 @@ func Ec2Images() *schema.Table {
 			{
 				Name:        "creation_date",
 				Description: "The date and time the image was created.",
-				Type:        schema.TypeString,
+				Type:        schema.TypeTimestamp,
+				Resolver:    client.ResolveTimestampField("CreationDate"),
 			},
 			{
 				Name:        "description",

diff --git a/resources/services/ec2/images_mock_test.go b/resources/services/ec2/images_mock_test.go
@@ -23,7 +23,9 @@ func buildEc2ImagesMock(t *testing.T, ctrl *gomock.Controller) client.Services {
 		t.Fatal(err)
 	}
 
+	creationDate := "1994-11-05T08:15:30-05:00"
 	g.OwnerId = aws.String("testAccount")
+	g.CreationDate = &creationDate
 
 	m.EXPECT().DescribeImages(gomock.Any(), gomock.Any(), gomock.Any()).Return(
 		&ec2.DescribeImagesOutput{

diff --git a/resources/services/ec2/instances.go b/resources/services/ec2/instances.go
@@ -467,7 +467,8 @@ func Ec2Instances() *schema.Table {
 					{
 						Name:        "elastic_gpu_association_time",
 						Description: "The time the Elastic Graphics accelerator was associated with the instance.",
-						Type:        schema.TypeString,
+						Type:        schema.TypeTimestamp,
+						Resolver:    client.ResolveTimestampField("ElasticGpuAssociationTime"),
 					},
 					{
 						Name:        "elastic_gpu_id",

diff --git a/resources/services/ec2/instances_mock_test.go b/resources/services/ec2/instances_mock_test.go
@@ -20,6 +20,8 @@ func buildEc2Instances(t *testing.T, ctrl *gomock.Controller) client.Services {
 		t.Fatal(err)
 	}
 	l.Instances[0].StateTransitionReason = aws.String("User initiated (2021-11-26 11:33:00 GMT)")
+	creationDate := "1994-11-05T08:15:30-05:00"
+	l.Instances[0].ElasticGpuAssociations[0].ElasticGpuAssociationTime = &creationDate
 	m.EXPECT().DescribeInstances(gomock.Any(), gomock.Any(), gomock.Any()).Return(
 		&ec2.DescribeInstancesOutput{
 			Reservations: []ec2Types.Reservation{l},

diff --git a/resources/services/lambda/functions.go b/resources/services/lambda/functions.go
@@ -240,8 +240,8 @@ func Functions() *schema.Table {
 			{
 				Name:        "last_modified",
 				Description: "The date and time that the function was last updated, in ISO-8601 format (https://www.w3.org/TR/NOTE-datetime) (YYYY-MM-DDThh:mm:ss.sTZD).",
-				Type:        schema.TypeString,
-				Resolver:    schema.PathResolver("Configuration.LastModified"),
+				Type:        schema.TypeTimestamp,
+				Resolver:    client.ResolveTimestampField("Configuration.LastModified", TimestampLayoutNano),
 			},
 			{
 				Name:        "last_update_status",
@@ -712,7 +712,8 @@ func Functions() *schema.Table {
 					{
 						Name:        "last_modified",
 						Description: "The date and time that the function was last updated, in ISO-8601 format (https://www.w3.org/TR/NOTE-datetime) (YYYY-MM-DDThh:mm:ss.sTZD).",
-						Type:        schema.TypeString,
+						Type:        schema.TypeTimestamp,
+						Resolver:    client.ResolveTimestampField("LastModified", TimestampLayout),
 					},
 					{
 						Name:        "last_update_status",
@@ -919,7 +920,8 @@ func Functions() *schema.Table {
 					{
 						Name:        "last_modified",
 						Description: "The date and time that a user last updated the configuration, in ISO 8601 format (https://www.iso.org/iso-8601-date-and-time-format.html).",
-						Type:        schema.TypeString,
+						Type:        schema.TypeTimestamp,
+						Resolver:    client.ResolveTimestampField("LastModified", TimestampLayout),
 					},
 					{
 						Name:        "requested_provisioned_concurrent_executions",

diff --git a/resources/services/lambda/functions_mock_test.go b/resources/services/lambda/functions_mock_test.go
@@ -15,11 +15,14 @@ import (
 func buildLambdaFunctionsMock(t *testing.T, ctrl *gomock.Controller) client.Services {
 	m := mocks.NewMockLambdaClient(ctrl)
 
+	lastModified := "1994-11-05T08:15:30.000+0500"
+
 	f := lambda.GetFunctionOutput{}
 	err := faker.FakeData(&f)
 	if err != nil {
 		t.Fatal(err)
 	}
+	f.Configuration.LastModified = &lastModified
 	m.EXPECT().GetFunction(gomock.Any(), gomock.Any(), gomock.Any()).Return(
 		&f, nil)
 
@@ -58,6 +61,7 @@ func buildLambdaFunctionsMock(t *testing.T, ctrl *gomock.Controller) client.Serv
 	if err != nil {
 		t.Fatal(err)
 	}
+	cc.LastModified = &lastModified
 	m.EXPECT().ListProvisionedConcurrencyConfigs(gomock.Any(), gomock.Any(), gomock.Any()).Return(
 		&lambda.ListProvisionedConcurrencyConfigsOutput{
 			ProvisionedConcurrencyConfigs: []types.ProvisionedConcurrencyConfigListItem{cc},
@@ -102,7 +106,7 @@ func buildLambdaFunctionsMock(t *testing.T, ctrl *gomock.Controller) client.Serv
 		&lambda.GetCodeSigningConfigOutput{
 			CodeSigningConfig: &csc,
 		}, nil)
-
+	fc.LastModified = &lastModified
 	m.EXPECT().ListVersionsByFunction(gomock.Any(), gomock.Any(), gomock.Any()).Return(
 		&lambda.ListVersionsByFunctionOutput{
 			Versions: []types.FunctionConfiguration{fc},

diff --git a/resources/services/lambda/layers.go b/resources/services/lambda/layers.go
@@ -11,6 +11,11 @@ import (
 	"github.com/cloudquery/cq-provider-sdk/provider/schema"
 )
 
+const (
+	TimestampLayoutNano = "2006-01-02T15:04:05.000-0700"
+	TimestampLayout     = "2006-01-02T15:04:05-0700"
+)
+
 func LambdaLayers() *schema.Table {
 	return &schema.Table{
 		Name:         "aws_lambda_layers",
@@ -42,8 +47,8 @@ func LambdaLayers() *schema.Table {
 			{
 				Name:        "latest_matching_version_created_date",
 				Description: "The date that the version was created, in ISO 8601 format",
-				Type:        schema.TypeString,
-				Resolver:    schema.PathResolver("LatestMatchingVersion.CreatedDate"),
+				Type:        schema.TypeTimestamp,
+				Resolver:    client.ResolveTimestampField("LatestMatchingVersion.CreatedDate", TimestampLayoutNano),
 			},
 			{
 				Name:        "latest_matching_version_description",
@@ -103,7 +108,8 @@ func LambdaLayers() *schema.Table {
 					{
 						Name:        "created_date",
 						Description: "The date that the version was created, in ISO 8601 format",
-						Type:        schema.TypeString,
+						Type:        schema.TypeTimestamp,
+						Resolver:    client.ResolveTimestampField("CreatedDate", TimestampLayoutNano),
 					},
 					{
 						Name:        "description",

diff --git a/resources/services/lambda/layers_mock_test.go b/resources/services/lambda/layers_mock_test.go
@@ -14,11 +14,14 @@ import (
 func buildLambdaLayersMock(t *testing.T, ctrl *gomock.Controller) client.Services {
 	m := mocks.NewMockLambdaClient(ctrl)
 
+	creationDate := "1994-11-05T08:15:30.000+0500"
+
 	l := types.LayersListItem{}
 	err := faker.FakeData(&l)
 	if err != nil {
 		t.Fatal(err)
 	}
+	l.LatestMatchingVersion.CreatedDate = &creationDate
 	m.EXPECT().ListLayers(gomock.Any(), gomock.Any(), gomock.Any()).Return(
 		&lambda.ListLayersOutput{
 			Layers: []types.LayersListItem{l},
@@ -31,6 +34,7 @@ func buildLambdaLayersMock(t *testing.T, ctrl *gomock.Controller) client.Service
 	}
 	arn := "arn:aws:s3:::my_corporate_bucket/test:exampleobject.png:1"
 	lv.LayerVersionArn = &arn
+	lv.CreatedDate = &creationDate
 	m.EXPECT().ListLayerVersions(gomock.Any(), gomock.Any(), gomock.Any()).Return(
 		&lambda.ListLayerVersionsOutput{
 			LayerVersions: []types.LayerVersionsListItem{lv},

diff --git a/resources/services/sqs/queues.go b/resources/services/sqs/queues.go
@@ -102,12 +102,14 @@ func SQSQueues() *schema.Table {
 			{
 				Name:        "created_timestamp",
 				Description: "UNIX time when the queue was created.",
-				Type:        schema.TypeInt,
+				Type:        schema.TypeTimestamp,
+				Resolver:    client.ResolveTimestampField("CreatedTimestamp"),
 			},
 			{
 				Name:        "last_modified_timestamp",
 				Description: "UNIX time when the queue was last changed.",
-				Type:        schema.TypeInt,
+				Type:        schema.TypeTimestamp,
+				Resolver:    client.ResolveTimestampField("LastModifiedTimestamp"),
 			},
 			{
 				Name:        "arn",