diff --git a/helper/helper.go b/helper/helper.go index 0c5f5f1..d3a07dc 100644 --- a/helper/helper.go +++ b/helper/helper.go @@ -33,6 +33,20 @@ import ( "google.golang.org/api/iterator" ) +// Key Service account json key +type Key struct { + Type string `json:"type"` + ProjectID string `json:"project_id"` + PrivateKeyID string `json:"private_key_id"` + PrivateKey string `json:"private_key"` + ClientEmail string `json:"client_email"` + ClientID string `json:"client_id"` + AuthURI string `json:"auth_uri"` + TokenURI string `json:"token_uri"` + AuthProviderX509CertURL string `json:"auth_provider_x509_cert_url"` + ClientX509CertURL string `json:"client_x509_cert_url"` +} + // PublishRequest Pub/sub type PublishRequest struct { Topic string `json:"topic"` diff --git a/listgroups/listgroups.go b/listgroups/listgroups.go index c956ea4..68104ac 100644 --- a/listgroups/listgroups.go +++ b/listgroups/listgroups.go @@ -32,7 +32,6 @@ import ( "google.golang.org/api/iam/v1" "google.golang.org/api/option" - "cloud.google.com/go/functions/metadata" "cloud.google.com/go/pubsub" admin "google.golang.org/api/admin/directory/v1" ) @@ -64,20 +63,6 @@ type Global struct { retryTimeOutSeconds int64 } -// Key Service account json key -type Key struct { - Type string `json:"type"` - ProjectID string `json:"project_id"` - PrivateKeyID string `json:"private_key_id"` - PrivateKey string `json:"private_key"` - ClientEmail string `json:"client_email"` - ClientID string `json:"client_id"` - AuthURI string `json:"auth_uri"` - TokenURI string `json:"token_uri"` - AuthProviderX509CertURL string `json:"auth_provider_x509_cert_url"` - ClientX509CertURL string `json:"client_x509_cert_url"` -} - // FeedMessage Cloud Asset Inventory feed message type FeedMessage struct { Asset Asset `json:"asset"` @@ -163,7 +148,7 @@ func Initialize(ctx context.Context, global *Global) { global.initFailed = true return } - var key Key + var key helper.Key err = json.Unmarshal(keyJSONdata, &key) if err != nil { log.Printf("ERROR - json.Unmarshal(keyJSONdata, &key): %v", err) @@ -230,22 +215,9 @@ func Initialize(ctx context.Context, global *Global) { // EntryPoint is the function to be executed for each cloud function occurence func EntryPoint(ctxEvent context.Context, PubSubMessage helper.PubSubMessage, global *Global) error { // log.Println(string(PubSubMessage.Data)) - if global.initFailed { - log.Println("ERROR - init function failed") - return nil // NO RETRY - } - - metadata, err := metadata.FromContext(ctxEvent) - if err != nil { - // Assume an error on the function invoker and try again. - return fmt.Errorf("metadata.FromContext: %v", err) // RETRY - } - - // Ignore events that are too old. - expiration := metadata.Timestamp.Add(time.Duration(global.retryTimeOutSeconds) * time.Second) - if time.Now().After(expiration) { - log.Printf("ERROR - too many retries for expired event '%q'", metadata.EventID) - return nil // NO MORE RETRY + ok, metadata, err := helper.IntialRetryCheck(ctxEvent, global.initFailed, global.retryTimeOutSeconds) + if !ok { + return err } // log.Printf("EventType %s EventID %s Resource %s Timestamp %v", metadata.EventType, metadata.EventID, metadata.Resource.Type, metadata.Timestamp) diff --git a/stream2bq/stream2bq.go b/stream2bq/stream2bq.go index 4c97aba..71ed3fb 100644 --- a/stream2bq/stream2bq.go +++ b/stream2bq/stream2bq.go @@ -29,7 +29,6 @@ import ( "cloud.google.com/go/bigquery" "cloud.google.com/go/firestore" - "cloud.google.com/go/functions/metadata" cloudresourcemanagerv2 "google.golang.org/api/cloudresourcemanager/v2" ) @@ -299,25 +298,12 @@ func Initialize(ctx context.Context, global *Global) { // EntryPoint is the function to be executed for each cloud function occurence func EntryPoint(ctxEvent context.Context, PubSubMessage helper.PubSubMessage, global *Global) error { // log.Println(string(PubSubMessage.Data)) - if global.initFailed { - log.Println("ERROR - init function failed") - return nil // NO RETRY - } - - metadata, err := metadata.FromContext(ctxEvent) - if err != nil { - // Assume an error on the function invoker and try again. - return fmt.Errorf("metadata.FromContext: %v", err) // RETRY - } - - // Ignore events that are too old. - expiration := metadata.Timestamp.Add(time.Duration(global.retryTimeOutSeconds) * time.Second) - if time.Now().After(expiration) { - log.Printf("ERROR - too many retries for expired event '%q'", metadata.EventID) - return nil // NO MORE RETRY + if ok, _, err := helper.IntialRetryCheck(ctxEvent, global.initFailed, global.retryTimeOutSeconds); !ok { + return err } // log.Printf("EventType %s EventID %s Resource %s Timestamp %v", metadata.EventType, metadata.EventID, metadata.Resource.Type, metadata.Timestamp) + var err error switch global.tableName { case "complianceStatus": err = persistComplianceStatus(PubSubMessage.Data, global) diff --git a/upload2gcs/upload2gcs.go b/upload2gcs/upload2gcs.go index 46564bd..f12392f 100644 --- a/upload2gcs/upload2gcs.go +++ b/upload2gcs/upload2gcs.go @@ -22,13 +22,11 @@ import ( "os" "strconv" "strings" - "time" "github.com/BrunoReboul/ram/helper" "google.golang.org/api/cloudresourcemanager/v1" "cloud.google.com/go/firestore" - "cloud.google.com/go/functions/metadata" "cloud.google.com/go/storage" cloudresourcemanagerv2 "google.golang.org/api/cloudresourcemanager/v2" ) @@ -127,27 +125,13 @@ func Initialize(ctx context.Context, global *Global) { // EntryPoint is the function to be executed for each cloud function occurence func EntryPoint(ctxEvent context.Context, PubSubMessage helper.PubSubMessage, global *Global) error { // log.Println(string(PubSubMessage.Data)) - if global.initFailed { - log.Println("ERROR - init function failed") - return nil // NO RETRY - } - - metadata, err := metadata.FromContext(ctxEvent) - if err != nil { - // Assume an error on the function invoker and try again. - return fmt.Errorf("metadata.FromContext: %v", err) // RETRY - } - - // Ignore events that are too old. - expiration := metadata.Timestamp.Add(time.Duration(global.retryTimeOutSeconds) * time.Second) - if time.Now().After(expiration) { - log.Printf("ERROR - too many retries for expired event '%q'", metadata.EventID) - return nil // NO MORE RETRY + if ok, _, err := helper.IntialRetryCheck(ctxEvent, global.initFailed, global.retryTimeOutSeconds); !ok { + return err } // log.Printf("EventType %s EventID %s Resource %s Timestamp %v", metadata.EventType, metadata.EventID, metadata.Resource.Type, metadata.Timestamp) var feedMessage FeedMessage - err = json.Unmarshal(PubSubMessage.Data, &feedMessage) + err := json.Unmarshal(PubSubMessage.Data, &feedMessage) if err != nil { log.Printf("ERROR - json.Unmarshal: %v", err) return nil // NO RETRY