From bc1730d811386086617e6f5958935b63b86a5a8b Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Tue, 20 Aug 2024 21:37:03 +0200 Subject: [PATCH 01/19] v1.0 preparations --- cmd/sqs-sqs/main.go | 2 +- go.mod | 10 +-- go.sum | 31 +-------- sns/pub_test.go | 2 +- sqs/config.go | 155 ++++++++++++++++++++++++++++++++++++++++++++ sqs/marshaler.go | 2 +- sqs/publisher.go | 54 +++++++-------- sqs/pubsub_test.go | 4 +- sqs/sqs.go | 88 ++++++++++++------------- sqs/subscriber.go | 124 ++++++++++++++++++----------------- 10 files changed, 298 insertions(+), 174 deletions(-) create mode 100644 sqs/config.go diff --git a/cmd/sqs-sqs/main.go b/cmd/sqs-sqs/main.go index 3578729..cae5285 100644 --- a/cmd/sqs-sqs/main.go +++ b/cmd/sqs-sqs/main.go @@ -36,7 +36,7 @@ func main() { sub, err := sqs.NewSubscriber(sqs.SubscriberConfig{ AWSConfig: cfg, - CreateQueueInitializerConfig: sqs.QueueConfigAtrributes{}, + CreateQueueInitializerConfig: sqs.QueueConfigAttributes{}, Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, }, logger) if err != nil { diff --git a/go.mod b/go.mod index 0149ab4..0acb39f 100644 --- a/go.mod +++ b/go.mod @@ -7,18 +7,10 @@ require ( github.com/aws/aws-sdk-go-v2 v1.24.1 github.com/aws/aws-sdk-go-v2/config v1.26.6 github.com/aws/aws-sdk-go-v2/credentials v1.16.16 - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect github.com/aws/aws-sdk-go-v2/service/sns v1.26.7 github.com/aws/aws-sdk-go-v2/service/sqs v1.29.7 - github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect github.com/aws/smithy-go v1.19.0 github.com/hashicorp/go-multierror v1.1.1 + github.com/mitchellh/mapstructure v1.5.0 github.com/stretchr/testify v1.8.1 ) diff --git a/go.sum b/go.sum index 53ead85..bfdf143 100644 --- a/go.sum +++ b/go.sum @@ -41,62 +41,34 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY= -github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU= github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= -github.com/aws/aws-sdk-go-v2/config v1.18.25 h1:JuYyZcnMPBiFqn87L2cRppo+rNwgah6YwD3VuyvaW6Q= -github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= github.com/aws/aws-sdk-go-v2/config v1.26.6 h1:Z/7w9bUqlRI0FFQpetVuFYEsjzE3h7fpU6HuGmfPL/o= github.com/aws/aws-sdk-go-v2/config v1.26.6/go.mod h1:uKU6cnDmYCvJ+pxO9S4cWDb2yWWIH5hra+32hVh1MI4= -github.com/aws/aws-sdk-go-v2/credentials v1.13.24 h1:PjiYyls3QdCrzqUN35jMWtUK1vqVZ+zLfdOa/UPFDp0= -github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= github.com/aws/aws-sdk-go-v2/credentials v1.16.16 h1:8q6Rliyv0aUFAVtzaldUEcS+T5gbadPbWdV1WcAddK8= github.com/aws/aws-sdk-go-v2/credentials v1.16.16/go.mod h1:UHVZrdUsv63hPXFo1H7c5fEneoVo9UXiz36QG1GEPi0= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 h1:jJPgroehGvjrde3XufFIJUZVK5A2L9a3KwSFgKy9n8w= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 h1:kG5eQilShqmJbv11XL1VpyDbaEJzWxd4zRiCG30GSn4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 h1:vFQlirhuM8lLlpI7imKOMsjdQLuN9CPi+k44F/OFVsk= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 h1:gGLG7yKaXG02/jBlg210R7VgQIotiQntNhsCFejawx8= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 h1:n3GDfwqF2tzEkXlv5cuy4iy7LpKDtqDMcNLfZDu9rls= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 h1:0iKliEXAcCa2qVtRs7Ot5hItA2MsufrphbRFlz1Owxo= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino= -github.com/aws/aws-sdk-go-v2/service/sns v1.20.11 h1:kUKAkuOhCCq/Av372Dtzg0oaAD5VEUYdDtU4lGIYKkw= -github.com/aws/aws-sdk-go-v2/service/sns v1.20.11/go.mod h1:WjBcrd28zNbbuAcIRO/n89sSeOxTuOZPiuxNXU/2WrI= github.com/aws/aws-sdk-go-v2/service/sns v1.26.7 h1:DylmW2c1Z7qGxN3Y02k+voPbtM1mh7Rp+gV+7maG5io= github.com/aws/aws-sdk-go-v2/service/sns v1.26.7/go.mod h1:mLFiISZfiZAqZEfPWUsZBK8gD4dYCKuKAfapV+KrIVQ= -github.com/aws/aws-sdk-go-v2/service/sqs v1.22.0 h1:ikSvot5NdywduxtkOwOa2GJFzFuJq1ZjXsGjoIA82Ao= -github.com/aws/aws-sdk-go-v2/service/sqs v1.22.0/go.mod h1:ujUjm+PrcKUeIiKu2PT7MWjcyY0D6YZRZF3fSswiO+0= github.com/aws/aws-sdk-go-v2/service/sqs v1.29.7 h1:tRNrFDGRm81e6nTX5Q4CFblea99eAfm0dxXazGpLceU= github.com/aws/aws-sdk-go-v2/service/sqs v1.29.7/go.mod h1:8GWUDux5Z2h6z2efAtr54RdHXtLm8sq7Rg85ZNY/CZM= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 h1:UBQjaMTCKwyUYwiVnUt6toEJwGXsLBI6al083tpjJzY= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI= github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 h1:eajuO3nykDPdYicLlP3AGgOyVN3MOlFmZv7WGTuJPow= github.com/aws/aws-sdk-go-v2/service/sso v1.18.7/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 h1:PkHIIJs8qvq0e5QybnZoG1K/9QTrLr9OsqCIo59jOBA= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 h1:QPMJf+Jw8E1l7zqhZmMlFw6w1NmfkfiSK8mS4zOx3BA= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 h1:2DQLAKDteoEDI8zpCzqBMaZlJuoE9iTYD0gFmXVax9E= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0= github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U= -github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -201,7 +173,6 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -230,6 +201,8 @@ github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJV github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= diff --git a/sns/pub_test.go b/sns/pub_test.go index 21398e2..55b44b9 100644 --- a/sns/pub_test.go +++ b/sns/pub_test.go @@ -93,7 +93,7 @@ func createPubSub(t *testing.T) (message.Publisher, message.Subscriber) { sub, err := sqs.NewSubscriber(sqs.SubscriberConfig{ AWSConfig: cfg, - CreateQueueInitializerConfig: sqs.QueueConfigAtrributes{ + CreateQueueInitializerConfig: sqs.QueueConfigAttributes{ // Defalt value is 30 seconds - need to be lower for tests VisibilityTimeout: "15", }, diff --git a/sqs/config.go b/sqs/config.go new file mode 100644 index 0000000..d1191d1 --- /dev/null +++ b/sqs/config.go @@ -0,0 +1,155 @@ +package sqs + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sqs/types" +) + +type SubscriberConfig struct { + AWSConfig aws.Config + + // How long about unsuccessful reconnecting next reconnect will occur. + ReconnectRetrySleep time.Duration + // Delete message attempts + CreateQueueInitializerConfig QueueConfigAttributes + + GenerateCreateQueueInput GenerateCreateQueueInputFunc + + GenerateGetQueueUrlInput GenerateGetQueueUrlInputFunc + + GenerateReceiveMessageInput GenerateReceiveMessageInputFunc + + GenerateDeleteMessageInput GenerateDeleteMessageInputFunc + + Unmarshaler Unmarshaler +} + +func (c *SubscriberConfig) SetDefaults() { + if c.Unmarshaler == nil { + c.Unmarshaler = DefaultMarshalerUnmarshaler{} + } + + if c.ReconnectRetrySleep == 0 { + c.ReconnectRetrySleep = time.Second + } + + if c.GenerateCreateQueueInput == nil { + c.GenerateCreateQueueInput = GenerateCreateQueueInputDefault + } + + if c.GenerateGetQueueUrlInput == nil { + c.GenerateGetQueueUrlInput = GenerateGetQueueUrlInputDefault + } + + if c.GenerateReceiveMessageInput == nil { + c.GenerateReceiveMessageInput = GenerateReceiveMessageInputDefault + } + + if c.GenerateDeleteMessageInput == nil { + c.GenerateDeleteMessageInput = GenerateDeleteMessageInputDefault + } +} + +func (c SubscriberConfig) Validate() error { + var err error + + if c.AWSConfig.Credentials == nil { + err = errors.Join(err, errors.New("missing Config.Credentials")) + + } + if c.Unmarshaler == nil { + err = errors.Join(err, errors.New("missing Config.Marshaler")) + } + + return err +} + +type PublisherConfig struct { + AWSConfig aws.Config + + CreateQueueConfig QueueConfigAttributes + CreateQueueIfNotExists bool + + GenerateGetQueueUrlInput GenerateGetQueueUrlInputFunc + + GenerateSendMessageInput GenerateSendMessageInputFunc + + GenerateCreateQueueInput GenerateCreateQueueInputFunc + + Marshaler Marshaler +} + +func (c *PublisherConfig) setDefaults() { + if c.Marshaler == nil { + c.Marshaler = DefaultMarshalerUnmarshaler{} + } + + if c.GenerateGetQueueUrlInput == nil { + c.GenerateGetQueueUrlInput = GenerateGetQueueUrlInputDefault + } + + if c.GenerateSendMessageInput == nil { + c.GenerateSendMessageInput = GenerateSendMessageInputDefault + } + + if c.GenerateCreateQueueInput == nil { + c.GenerateCreateQueueInput = GenerateCreateQueueInputDefault + } +} + +type GenerateCreateQueueInputFunc func(ctx context.Context, topic string, attrs QueueConfigAttributes) (*sqs.CreateQueueInput, error) + +func GenerateCreateQueueInputDefault(ctx context.Context, topic string, attrs QueueConfigAttributes) (*sqs.CreateQueueInput, error) { + attrsMap, err := attrs.Attributes() + if err != nil { + return nil, fmt.Errorf("cannot generate attributes for queue %s: %w", topic, err) + } + + return &sqs.CreateQueueInput{ + QueueName: aws.String(topic), + Attributes: attrsMap, + }, nil +} + +type GenerateGetQueueUrlInputFunc func(ctx context.Context, topic string) (*sqs.GetQueueUrlInput, error) + +func GenerateGetQueueUrlInputDefault(ctx context.Context, topic string) (*sqs.GetQueueUrlInput, error) { + return &sqs.GetQueueUrlInput{ + QueueName: aws.String(topic), + }, nil +} + +type GenerateReceiveMessageInputFunc func(ctx context.Context, queueURL string) (*sqs.ReceiveMessageInput, error) + +func GenerateReceiveMessageInputDefault(ctx context.Context, queueURL string) (*sqs.ReceiveMessageInput, error) { + return &sqs.ReceiveMessageInput{ + QueueUrl: aws.String(queueURL), + MessageAttributeNames: []string{"All"}, + WaitTimeSeconds: 30, + }, nil +} + +type GenerateDeleteMessageInputFunc func(ctx context.Context, queueURL string, receiptHandle *string) (*sqs.DeleteMessageInput, error) + +func GenerateDeleteMessageInputDefault(ctx context.Context, queueURL string, receiptHandle *string) (*sqs.DeleteMessageInput, error) { + return &sqs.DeleteMessageInput{ + QueueUrl: aws.String(queueURL), + ReceiptHandle: receiptHandle, + }, nil +} + +type GenerateSendMessageInputFunc func(ctx context.Context, queueURL string, msg *types.Message) (*sqs.SendMessageInput, error) + +func GenerateSendMessageInputDefault(ctx context.Context, queueURL string, msg *types.Message) (*sqs.SendMessageInput, error) { + return &sqs.SendMessageInput{ + QueueUrl: &queueURL, + MessageAttributes: msg.MessageAttributes, + MessageBody: msg.Body, + }, nil +} diff --git a/sqs/marshaler.go b/sqs/marshaler.go index 002e6e3..6090a0d 100644 --- a/sqs/marshaler.go +++ b/sqs/marshaler.go @@ -13,7 +13,7 @@ type Marshaler interface { Marshal(msg *message.Message) (*types.Message, error) } -type UnMarshaler interface { +type Unmarshaler interface { Unmarshal(msg *types.Message) (*message.Message, error) } diff --git a/sqs/publisher.go b/sqs/publisher.go index 978f132..bc75174 100644 --- a/sqs/publisher.go +++ b/sqs/publisher.go @@ -2,20 +2,13 @@ package sqs import ( "context" + "fmt" "github.com/ThreeDotsLabs/watermill" "github.com/ThreeDotsLabs/watermill/message" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/sqs" ) -type PublisherConfig struct { - AWSConfig aws.Config - CreateQueueConfig QueueConfigAtrributes - CreateQueueIfNotExists bool - Marshaler Marshaler -} - type Publisher struct { config PublisherConfig logger watermill.LoggerAdapter @@ -24,6 +17,7 @@ type Publisher struct { func NewPublisher(config PublisherConfig, logger watermill.LoggerAdapter) (*Publisher, error) { config.setDefaults() + return &Publisher{ sqs: sqs.NewFromConfig(config.AWSConfig), config: config, @@ -33,24 +27,32 @@ func NewPublisher(config PublisherConfig, logger watermill.LoggerAdapter) (*Publ func (p Publisher) Publish(topic string, messages ...*message.Message) error { ctx := context.Background() + + // todo: cache it queueUrl, err := p.GetQueueUrl(ctx, topic) if err != nil { - return err + return fmt.Errorf("cannot get queue url: %w", err) } + if queueUrl == nil { + return fmt.Errorf("returned queueUrl is nil") + } + for _, msg := range messages { sqsMsg, err := p.config.Marshaler.Marshal(msg) if err != nil { - return err + return fmt.Errorf("cannot marshal message: %w", err) } p.logger.Debug("Sending message", watermill.LogFields{"msg": msg}) - _, err = p.sqs.SendMessage(ctx, &sqs.SendMessageInput{ - QueueUrl: queueUrl, - MessageAttributes: sqsMsg.MessageAttributes, - MessageBody: sqsMsg.Body, - }) + + input, err := p.config.GenerateSendMessageInput(ctx, *queueUrl, sqsMsg) if err != nil { - return err + return fmt.Errorf("cannot generate send message input: %w", err) + } + + _, err = p.sqs.SendMessage(ctx, input) + if err != nil { + return fmt.Errorf("cannot send message: %w", err) } } @@ -58,12 +60,16 @@ func (p Publisher) Publish(topic string, messages ...*message.Message) error { } func (p Publisher) GetQueueUrl(ctx context.Context, topic string) (*string, error) { - queueUrl, err := GetQueueUrl(ctx, p.sqs, topic) + queueUrl, err := getQueueUrl(ctx, p.sqs, topic, p.config.GenerateGetQueueUrlInput) if err != nil { + // todo: check exact error here if p.config.CreateQueueIfNotExists { - queueUrl, err = CreateQueue(ctx, p.sqs, topic, sqs.CreateQueueInput{ - Attributes: p.config.CreateQueueConfig.Attributes(), - }) + input, err := p.config.GenerateCreateQueueInput(ctx, topic, p.config.CreateQueueConfig) + if err != nil { + return nil, fmt.Errorf("cannot generate create queue input: %w", err) + } + + queueUrl, err = greateQueue(ctx, p.sqs, input) if err == nil { return queueUrl, nil } @@ -74,15 +80,11 @@ func (p Publisher) GetQueueUrl(ctx context.Context, topic string) (*string, erro } func (p Publisher) GetQueueArn(ctx context.Context, url *string) (*string, error) { - return GetARNUrl(ctx, p.sqs, url) + return getARNUrl(ctx, p.sqs, url) } func (p Publisher) Close() error { return nil } -func (c *PublisherConfig) setDefaults() { - if c.Marshaler == nil { - c.Marshaler = DefaultMarshalerUnmarshaler{} - } -} +// todo: missing validate? diff --git a/sqs/pubsub_test.go b/sqs/pubsub_test.go index 73e06c0..0ab2b8b 100644 --- a/sqs/pubsub_test.go +++ b/sqs/pubsub_test.go @@ -65,7 +65,7 @@ func createPubSub(t *testing.T) (message.Publisher, message.Subscriber) { pub, err := sqs.NewPublisher(sqs.PublisherConfig{ AWSConfig: cfg, - CreateQueueConfig: sqs.QueueConfigAtrributes{ + CreateQueueConfig: sqs.QueueConfigAttributes{ // Defalt value is 30 seconds - need to be lower for tests VisibilityTimeout: "1", }, @@ -76,7 +76,7 @@ func createPubSub(t *testing.T) (message.Publisher, message.Subscriber) { sub, err := sqs.NewSubscriber(sqs.SubscriberConfig{ AWSConfig: cfg, - CreateQueueInitializerConfig: sqs.QueueConfigAtrributes{ + CreateQueueInitializerConfig: sqs.QueueConfigAttributes{ // Defalt value is 30 seconds - need to be lower for tests VisibilityTimeout: "1", }, diff --git a/sqs/sqs.go b/sqs/sqs.go index 6c12ce3..bb76da1 100644 --- a/sqs/sqs.go +++ b/sqs/sqs.go @@ -2,45 +2,51 @@ package sqs import ( "context" - "encoding/json" - "errors" "fmt" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/types" + "github.com/mitchellh/mapstructure" ) -type QueueConfigAtrributes struct { - DelaySeconds string `json:"DelaySeconds,omitempty"` - MaximumMessageSize string `json:"MaximumMessageSize,omitempty"` - MessageRetentionPeriod string `json:"MessageRetentionPeriod,omitempty"` - Policy string `json:"Policy,omitempty"` - ReceiveMessageWaitTimeSeconds string `json:"ReceiveMessageWaitTimeSeconds,omitempty"` - RedrivePolicy string `json:"RedrivePolicy,omitempty"` - DeadLetterTargetArn string `json:"deadLetterTargetArn,omitempty"` - MaxReceiveCount string `json:"maxReceiveCount,omitempty"` - VisibilityTimeout string `json:"VisibilityTimeout,omitempty"` - KmsMasterKeyId string `json:"KmsMasterKeyId,omitempty"` - KmsDataKeyReusePeriodSeconds string `json:"KmsDataKeyReusePeriodSeconds,omitempty"` - SqsManagedSseEnabled string `json:"SqsManagedSseEnabled,omitempty"` - FifoQueue bool `json:"FifoQueue,omitempty"` - ContentBasedDeduplication bool `json:"ContentBasedDeduplication,omitempty"` - DeduplicationScope string `json:"DeduplicationScope,omitempty"` - FifoThroughputLimit string `json:"FifoThroughputLimit,omitempty"` +type QueueConfigAttributes struct { + DelaySeconds string `mapstructure:"DelaySeconds,omitempty"` + MaximumMessageSize string `mapstructure:"MaximumMessageSize,omitempty"` + MessageRetentionPeriod string `mapstructure:"MessageRetentionPeriod,omitempty"` + Policy string `mapstructure:"Policy,omitempty"` + ReceiveMessageWaitTimeSeconds string `mapstructure:"ReceiveMessageWaitTimeSeconds,omitempty"` + RedrivePolicy string `mapstructure:"RedrivePolicy,omitempty"` + DeadLetterTargetArn string `mapstructure:"deadLetterTargetArn,omitempty"` + MaxReceiveCount string `mapstructure:"maxReceiveCount,omitempty"` + VisibilityTimeout string `mapstructure:"VisibilityTimeout,omitempty"` + KmsMasterKeyId string `mapstructure:"KmsMasterKeyId,omitempty"` + KmsDataKeyReusePeriodSeconds string `mapstructure:"KmsDataKeyReusePeriodSeconds,omitempty"` + SqsManagedSseEnabled string `mapstructure:"SqsManagedSseEnabled,omitempty"` + FifoQueue bool `mapstructure:"FifoQueue,omitempty"` + ContentBasedDeduplication bool `mapstructure:"ContentBasedDeduplication,omitempty"` + DeduplicationScope string `mapstructure:"DeduplicationScope,omitempty"` + FifoThroughputLimit string `mapstructure:"FifoThroughputLimit,omitempty"` } -func (q QueueConfigAtrributes) Attributes() map[string]string { - b, _ := json.Marshal(q) +func (q QueueConfigAttributes) Attributes() (map[string]string, error) { var m map[string]string - _ = json.Unmarshal(b, &m) - return m + + // todo: test + err := mapstructure.Decode(q, &m) + if err != nil { + return nil, fmt.Errorf("cannot decode queue attributes: %w", err) + } + + return m, nil } -func GetQueueUrl(ctx context.Context, sqsClient *sqs.Client, topic string) (*string, error) { - getQueueOutput, err := sqsClient.GetQueueUrl(ctx, &sqs.GetQueueUrlInput{ - QueueName: aws.String(topic), - }) +func getQueueUrl(ctx context.Context, sqsClient *sqs.Client, topic string, generateGetQueueUrlInput GenerateGetQueueUrlInputFunc) (*string, error) { + input, err := generateGetQueueUrlInput(ctx, topic) + if err != nil { + return nil, fmt.Errorf("cannot generate input for queue %s: %w", topic, err) + } + + getQueueOutput, err := sqsClient.GetQueueUrl(ctx, input) if err != nil || getQueueOutput.QueueUrl == nil { return nil, fmt.Errorf("cannot get queue %s: %w", topic, err) @@ -48,27 +54,19 @@ func GetQueueUrl(ctx context.Context, sqsClient *sqs.Client, topic string) (*str return getQueueOutput.QueueUrl, nil } -func CreateQueue(ctx context.Context, sqsClient *sqs.Client, topic string, createQueueParams sqs.CreateQueueInput) (*string, error) { - createQueueParams.QueueName = aws.String(topic) - createQueueOutput, err := sqsClient.CreateQueue(ctx, &createQueueParams) - if err != nil || createQueueOutput.QueueUrl == nil { - return nil, fmt.Errorf("cannot create queue %s: %w", topic, err) +func greateQueue(ctx context.Context, sqsClient *sqs.Client, createQueueParams *sqs.CreateQueueInput) (*string, error) { + createQueueOutput, err := sqsClient.CreateQueue(ctx, createQueueParams) + if err != nil { + return nil, fmt.Errorf("cannot create queue %w", err) } - return createQueueOutput.QueueUrl, nil -} - -func GetOrCreateQueue(ctx context.Context, sqsClient *sqs.Client, topic string, createQueueParams sqs.CreateQueueInput) (*string, error) { - queueUrl, err := GetQueueUrl(ctx, sqsClient, topic) - if err != nil || queueUrl == nil { - var qne *types.QueueDoesNotExist - if errors.As(err, &qne) { - return CreateQueue(ctx, sqsClient, topic, createQueueParams) - } + if createQueueOutput.QueueUrl == nil { + return nil, fmt.Errorf("cannot create queue, queueUrl is nil") } - return queueUrl, nil + + return createQueueOutput.QueueUrl, nil } -func GetARNUrl(ctx context.Context, sqsClient *sqs.Client, url *string) (*string, error) { +func getARNUrl(ctx context.Context, sqsClient *sqs.Client, url *string) (*string, error) { attrResult, err := sqsClient.GetQueueAttributes(ctx, &sqs.GetQueueAttributesInput{ QueueUrl: url, AttributeNames: []types.QueueAttributeName{ diff --git a/sqs/subscriber.go b/sqs/subscriber.go index 3cb4237..f53aaab 100644 --- a/sqs/subscriber.go +++ b/sqs/subscriber.go @@ -3,29 +3,17 @@ package sqs import ( "context" "errors" + "fmt" "sync" "time" - "github.com/aws/aws-sdk-go-v2/aws" + "github.com/ThreeDotsLabs/watermill" + "github.com/ThreeDotsLabs/watermill/message" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/types" "github.com/aws/smithy-go" - "github.com/hashicorp/go-multierror" - - "github.com/ThreeDotsLabs/watermill" - "github.com/ThreeDotsLabs/watermill/message" ) -type SubscriberConfig struct { - AWSConfig aws.Config - // How long about unsuccessful reconnecting next reconnect will occur. - ReconnectRetrySleep time.Duration - // Delete message attemps - CreateQueueInitializerConfig QueueConfigAtrributes - - Unmarshaler UnMarshaler -} - type Subscriber struct { config SubscriberConfig logger watermill.LoggerAdapter @@ -64,20 +52,44 @@ func (s *Subscriber) Subscribe(ctx context.Context, topic string) (<-chan *messa return nil, errors.New("subscriber closed") } + s.logger.With(watermill.LogFields{"topic": topic}).Trace("Getting queue", nil) + ctx, cancel := context.WithCancel(ctx) output := make(chan *message.Message) - queueURL, err := GetQueueUrl(ctx, s.sqs, topic) + queueURL, err := getQueueUrl(ctx, s.sqs, topic, s.config.GenerateGetQueueUrlInput) if err != nil { + // todo: should be logged later + // todo: better err handling + s.logger.With(watermill.LogFields{ + "queue": queueURL, + "topic": topic, + }).Error("Failed to get queue", err, nil) + close(output) cancel() return nil, err } + if queueURL == nil { + s.logger.With(watermill.LogFields{"topic": topic}).Trace("Queue not found", nil) + close(output) + cancel() + return nil, fmt.Errorf("queue %s not found", topic) + } + + receiveInput, err := s.config.GenerateReceiveMessageInput(ctx, *queueURL) + if err != nil { + close(output) + cancel() + return nil, fmt.Errorf("cannot generate input for queue %s: %w", topic, err) + } + + s.logger.With(watermill.LogFields{"queue": queueURL}).Trace("Subscribing to queue", nil) s.subscribersWg.Add(1) go func() { - s.receive(ctx, *queueURL, output) + s.receive(ctx, *queueURL, output, receiveInput) close(output) cancel() }() @@ -85,7 +97,7 @@ func (s *Subscriber) Subscribe(ctx context.Context, topic string) (<-chan *messa return output, nil } -func (s *Subscriber) receive(ctx context.Context, queueURL string, output chan *message.Message) { +func (s *Subscriber) receive(ctx context.Context, queueURL string, output chan *message.Message, input *sqs.ReceiveMessageInput) { defer s.subscribersWg.Done() ctx, cancelCtx := context.WithCancel(ctx) defer cancelCtx() @@ -98,29 +110,28 @@ func (s *Subscriber) receive(ctx context.Context, queueURL string, output chan * for { select { case <-s.closing: - s.logger.Info("Discarding queued message, subscriber closing", logFields) + s.logger.Trace("Discarding queued message, subscriber closing", logFields) return case <-ctx.Done(): - s.logger.Info("Stopping consume, context canceled", logFields) + s.logger.Trace("Stopping consume, context canceled", logFields) return case <-time.After(sleepTime): // Wait if needed + s.logger.Trace("Timeout?", logFields) } - result, err := s.sqs.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{ - QueueUrl: aws.String(queueURL), - MessageAttributeNames: []string{"All"}, - }) + result, err := s.sqs.ReceiveMessage(ctx, input) if err != nil { - s.logger.Error("Cannot connect recieve messages", err, logFields) + s.logger.Error("Cannot connect receive messages", err, logFields) sleepTime = s.config.ReconnectRetrySleep continue } sleepTime = NoSleep if result == nil || len(result.Messages) == 0 { + s.logger.Trace("No messages", logFields) continue } s.ConsumeMessages(ctx, result.Messages, queueURL, output, logFields) @@ -134,10 +145,13 @@ func (s *Subscriber) ConsumeMessages( output chan *message.Message, logFields watermill.LogFields, ) { + s.logger.Trace("ConsumeMessages", logFields) for _, sqsMsg := range messages { + s.logger.Trace("ConsumeMessages", logFields) + ctx, cancelCtx := context.WithCancel(ctx) - defer cancelCtx() + defer cancelCtx() // todo: leak msg, err := s.config.Unmarshaler.Unmarshal(&sqsMsg) if err != nil { s.logger.Error("Cannot unmarshal message", err, logFields) @@ -166,16 +180,19 @@ func (s *Subscriber) ConsumeMessages( } func (s *Subscriber) deleteMessage(ctx context.Context, queueURL string, receiptHandle *string) error { - _, err := s.sqs.DeleteMessage(ctx, &sqs.DeleteMessageInput{ - QueueUrl: aws.String(queueURL), - ReceiptHandle: receiptHandle, - }) + input, err := s.config.GenerateDeleteMessageInput(ctx, queueURL, receiptHandle) + if err != nil { + return fmt.Errorf("cannot generate input for delete message: %w", err) + } + + _, err = s.sqs.DeleteMessage(ctx, input) if err != nil { var oe *smithy.GenericAPIError if errors.As(err, &oe) { if oe.Message == "The specified queue does not contain the message specified." { // Message was already deleted or is not in queue + // todo: log? return nil } } @@ -199,43 +216,30 @@ func (s *Subscriber) Close() error { } func (s *Subscriber) SubscribeInitialize(topic string) error { - _, err := CreateQueue(context.Background(), s.sqs, topic, sqs.CreateQueueInput{ - Attributes: s.config.CreateQueueInitializerConfig.Attributes(), - }) - return err + return s.SubscribeInitializeWithContext(context.Background(), topic) +} + +func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, topic string) error { + input, err := s.config.GenerateCreateQueueInput(ctx, topic, s.config.CreateQueueInitializerConfig) + if err != nil { + return fmt.Errorf("cannot generate input for queue %s: %w", topic, err) + } + + _, err = greateQueue(ctx, s.sqs, input) + if err != nil { + return fmt.Errorf("cannot create queue %s: %w", topic, err) + } + + return nil } func (s *Subscriber) GetQueueUrl(ctx context.Context, topic string) (*string, error) { - queueUrl, err := GetQueueUrl(ctx, s.sqs, topic) + queueUrl, err := getQueueUrl(ctx, s.sqs, topic, s.config.GenerateGetQueueUrlInput) return queueUrl, err } func (s *Subscriber) GetQueueArn(ctx context.Context, url *string) (*string, error) { - return GetARNUrl(ctx, s.sqs, url) + return getARNUrl(ctx, s.sqs, url) } const NoSleep time.Duration = -1 - -func (c *SubscriberConfig) SetDefaults() { - - if c.Unmarshaler == nil { - c.Unmarshaler = DefaultMarshalerUnmarshaler{} - } - - if c.ReconnectRetrySleep == 0 { - c.ReconnectRetrySleep = time.Second - } -} -func (c SubscriberConfig) Validate() error { - var err error - - if c.AWSConfig.Credentials == nil { - err = multierror.Append(err, errors.New("missing Config.Credentials")) - - } - if c.Unmarshaler == nil { - err = multierror.Append(err, errors.New("missing Config.Marshaler")) - } - - return err -} From fdde1a5c9eae790261f7f0325c87c7f4d74c5798 Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Tue, 20 Aug 2024 21:51:33 +0200 Subject: [PATCH 02/19] fix mapstructure --- sqs/sqs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqs/sqs.go b/sqs/sqs.go index bb76da1..04f53de 100644 --- a/sqs/sqs.go +++ b/sqs/sqs.go @@ -32,7 +32,7 @@ func (q QueueConfigAttributes) Attributes() (map[string]string, error) { var m map[string]string // todo: test - err := mapstructure.Decode(q, &m) + err := mapstructure.WeakDecode(q, &m) if err != nil { return nil, fmt.Errorf("cannot decode queue attributes: %w", err) } From 2fe21d570e90bc12341f5c39659342dd2f02aa85 Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Tue, 20 Aug 2024 21:59:57 +0200 Subject: [PATCH 03/19] go back to json --- sqs/sqs.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/sqs/sqs.go b/sqs/sqs.go index 04f53de..2eef343 100644 --- a/sqs/sqs.go +++ b/sqs/sqs.go @@ -2,11 +2,11 @@ package sqs import ( "context" + "encoding/json" "fmt" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/types" - "github.com/mitchellh/mapstructure" ) type QueueConfigAttributes struct { @@ -29,12 +29,15 @@ type QueueConfigAttributes struct { } func (q QueueConfigAttributes) Attributes() (map[string]string, error) { - var m map[string]string + b, err := json.Marshal(q) + if err != nil { + return nil, fmt.Errorf("cannot marshal queue attributes (json.Marshal): %w", err) + } - // todo: test - err := mapstructure.WeakDecode(q, &m) + var m map[string]string + err = json.Unmarshal(b, &m) if err != nil { - return nil, fmt.Errorf("cannot decode queue attributes: %w", err) + return nil, fmt.Errorf("cannot marshal queue attributes (json.Unmarshal): %w", err) } return m, nil From 18fdd8268c18801138247939f03f607f85c5ef70 Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Tue, 20 Aug 2024 22:17:27 +0200 Subject: [PATCH 04/19] really go back to json --- sqs/sqs.go | 49 ++++++++++++++++++++++++++++++++----------------- sqs/sqs_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 17 deletions(-) create mode 100644 sqs/sqs_test.go diff --git a/sqs/sqs.go b/sqs/sqs.go index 2eef343..9ad108f 100644 --- a/sqs/sqs.go +++ b/sqs/sqs.go @@ -10,22 +10,33 @@ import ( ) type QueueConfigAttributes struct { - DelaySeconds string `mapstructure:"DelaySeconds,omitempty"` - MaximumMessageSize string `mapstructure:"MaximumMessageSize,omitempty"` - MessageRetentionPeriod string `mapstructure:"MessageRetentionPeriod,omitempty"` - Policy string `mapstructure:"Policy,omitempty"` - ReceiveMessageWaitTimeSeconds string `mapstructure:"ReceiveMessageWaitTimeSeconds,omitempty"` - RedrivePolicy string `mapstructure:"RedrivePolicy,omitempty"` - DeadLetterTargetArn string `mapstructure:"deadLetterTargetArn,omitempty"` - MaxReceiveCount string `mapstructure:"maxReceiveCount,omitempty"` - VisibilityTimeout string `mapstructure:"VisibilityTimeout,omitempty"` - KmsMasterKeyId string `mapstructure:"KmsMasterKeyId,omitempty"` - KmsDataKeyReusePeriodSeconds string `mapstructure:"KmsDataKeyReusePeriodSeconds,omitempty"` - SqsManagedSseEnabled string `mapstructure:"SqsManagedSseEnabled,omitempty"` - FifoQueue bool `mapstructure:"FifoQueue,omitempty"` - ContentBasedDeduplication bool `mapstructure:"ContentBasedDeduplication,omitempty"` - DeduplicationScope string `mapstructure:"DeduplicationScope,omitempty"` - FifoThroughputLimit string `mapstructure:"FifoThroughputLimit,omitempty"` + DelaySeconds string `json:"DelaySeconds,omitempty"` + MaximumMessageSize string `json:"MaximumMessageSize,omitempty"` + MessageRetentionPeriod string `json:"MessageRetentionPeriod,omitempty"` + Policy string `json:"Policy,omitempty"` + ReceiveMessageWaitTimeSeconds string `json:"ReceiveMessageWaitTimeSeconds,omitempty"` + RedrivePolicy string `json:"RedrivePolicy,omitempty"` + DeadLetterTargetArn string `json:"deadLetterTargetArn,omitempty"` + MaxReceiveCount string `json:"maxReceiveCount,omitempty"` + VisibilityTimeout string `json:"VisibilityTimeout,omitempty"` + KmsMasterKeyId string `json:"KmsMasterKeyId,omitempty"` + KmsDataKeyReusePeriodSeconds string `json:"KmsDataKeyReusePeriodSeconds,omitempty"` + SqsManagedSseEnabled string `json:"SqsManagedSseEnabled,omitempty"` + FifoQueue QueueConfigAttributesBool `json:"FifoQueue,omitempty"` + ContentBasedDeduplication QueueConfigAttributesBool `json:"ContentBasedDeduplication,omitempty"` + DeduplicationScope string `json:"DeduplicationScope,omitempty"` + FifoThroughputLimit string `json:"FifoThroughputLimit,omitempty"` +} + +// QueueConfigAttributesBool is a custom type for bool values in QueueConfigAttributes +// that supports marshaling to string. +type QueueConfigAttributesBool bool + +func (q QueueConfigAttributesBool) MarshalText() ([]byte, error) { + if q { + return []byte("true"), nil + } + return []byte("false"), nil } func (q QueueConfigAttributes) Attributes() (map[string]string, error) { @@ -70,6 +81,10 @@ func greateQueue(ctx context.Context, sqsClient *sqs.Client, createQueueParams * } func getARNUrl(ctx context.Context, sqsClient *sqs.Client, url *string) (*string, error) { + if url == nil { + return nil, fmt.Errorf("queue URL is nil") + } + attrResult, err := sqsClient.GetQueueAttributes(ctx, &sqs.GetQueueAttributesInput{ QueueUrl: url, AttributeNames: []types.QueueAttributeName{ @@ -77,7 +92,7 @@ func getARNUrl(ctx context.Context, sqsClient *sqs.Client, url *string) (*string }, }) if err != nil { - return nil, fmt.Errorf("cannot get ARN queue %s: %w", url, err) + return nil, fmt.Errorf("cannot get ARN queue %s: %w", *url, err) } arn := attrResult.Attributes[string(types.QueueAttributeNameQueueArn)] diff --git a/sqs/sqs_test.go b/sqs/sqs_test.go new file mode 100644 index 0000000..ac40d4e --- /dev/null +++ b/sqs/sqs_test.go @@ -0,0 +1,40 @@ +package sqs_test + +import ( + "testing" + + "github.com/ThreeDotsLabs/watermill-amazonsqs/sqs" + "github.com/stretchr/testify/require" +) + +func TestQueueConfigAttributes_Attributes(t *testing.T) { + structAttrs := sqs.QueueConfigAttributes{ + DelaySeconds: "10", + MaximumMessageSize: "20", + MessageRetentionPeriod: "20", + Policy: "test", + ReceiveMessageWaitTimeSeconds: "30", + RedrivePolicy: "test", + DeadLetterTargetArn: "test", + FifoQueue: false, + ContentBasedDeduplication: true, + } + + attrs, err := structAttrs.Attributes() + require.NoError(t, err) + + require.Equal( + t, + map[string]string{ + "ContentBasedDeduplication": "true", + "DelaySeconds": "10", + "MaximumMessageSize": "20", + "MessageRetentionPeriod": "20", + "Policy": "test", + "ReceiveMessageWaitTimeSeconds": "30", + "RedrivePolicy": "test", + "deadLetterTargetArn": "test", + }, + attrs, + ) +} From 9f80d0ef8c63992a78b6a1573936480c36d3fb7a Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Fri, 23 Aug 2024 10:03:32 +0200 Subject: [PATCH 05/19] fix WaitTimeSeconds --- sqs/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqs/config.go b/sqs/config.go index d1191d1..eb1eeff 100644 --- a/sqs/config.go +++ b/sqs/config.go @@ -131,7 +131,7 @@ func GenerateReceiveMessageInputDefault(ctx context.Context, queueURL string) (* return &sqs.ReceiveMessageInput{ QueueUrl: aws.String(queueURL), MessageAttributeNames: []string{"All"}, - WaitTimeSeconds: 30, + WaitTimeSeconds: 20, // 20 is max at the moment }, nil } From 0f9426db95836acc5f2b5598ecd9811ebb7c3bd9 Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Fri, 23 Aug 2024 18:21:23 +0200 Subject: [PATCH 06/19] fix tests and update emulator --- README.md | 3 +- docker-compose.yml | 18 +- go.mod | 44 +++- go.sum | 581 +++------------------------------------------ sns/pub_test.go | 2 +- sqs/pubsub_test.go | 2 +- sqs/subscriber.go | 104 ++++---- 7 files changed, 144 insertions(+), 610 deletions(-) diff --git a/README.md b/README.md index a7e612a..a06f224 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,7 @@ You can run the unit tests by simply running the command: `go test -cover -race #### Development and Testing -To try the in test mode (using [goaws](https://github.com/p4tin/goaws)) +To try the in test mode (using [localstack](https://hub.docker.com/r/localstack/localstack)) - start docker using: `docker-compose up -d` -- export AWS_ENDPOINT=http://localhost:4100/ Now you can run the application: `go run cmd/main.go` diff --git a/docker-compose.yml b/docker-compose.yml index 60268df..a62343c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,16 @@ -version: '3' services: - goaws: - image: pafortin/goaws:latest + localstack: + container_name: localstack + image: localstack/localstack:latest + environment: + - SERVICES=sqs,sns + - AWS_DEFAULT_REGION=us-east-1 + - EDGE_PORT=4566 ports: - - 4100:4100 + - "4566-4597:4566-4597" + healthcheck: + test: awslocal sqs list-queues && awslocal sns list-topics + interval: 5s + timeout: 5s + retries: 5 + start_period: 30s \ No newline at end of file diff --git a/go.mod b/go.mod index 0acb39f..f00b2d2 100644 --- a/go.mod +++ b/go.mod @@ -1,16 +1,38 @@ module github.com/ThreeDotsLabs/watermill-amazonsqs -go 1.12 +go 1.21 require ( - github.com/ThreeDotsLabs/watermill v1.2.0 - github.com/aws/aws-sdk-go-v2 v1.24.1 - github.com/aws/aws-sdk-go-v2/config v1.26.6 - github.com/aws/aws-sdk-go-v2/credentials v1.16.16 - github.com/aws/aws-sdk-go-v2/service/sns v1.26.7 - github.com/aws/aws-sdk-go-v2/service/sqs v1.29.7 - github.com/aws/smithy-go v1.19.0 - github.com/hashicorp/go-multierror v1.1.1 - github.com/mitchellh/mapstructure v1.5.0 - github.com/stretchr/testify v1.8.1 + github.com/ThreeDotsLabs/watermill v1.3.5 + github.com/aws/aws-sdk-go-v2 v1.30.4 + github.com/aws/aws-sdk-go-v2/config v1.27.28 + github.com/aws/aws-sdk-go-v2/credentials v1.17.28 + github.com/aws/aws-sdk-go-v2/service/sns v1.31.4 + github.com/aws/aws-sdk-go-v2/service/sqs v1.34.4 + github.com/aws/smithy-go v1.20.4 + github.com/stretchr/testify v1.8.4 +) + +require ( + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.30.4 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/lithammer/shortuuid/v3 v3.0.7 // indirect + github.com/oklog/ulid v1.3.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index bfdf143..b3d1cb5 100644 --- a/go.sum +++ b/go.sum @@ -1,195 +1,46 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ThreeDotsLabs/watermill v1.2.0 h1:TU3TML1dnQ/ifK09F2+4JQk2EKhmhXe7Qv7eb5ZpTS8= -github.com/ThreeDotsLabs/watermill v1.2.0/go.mod h1:IuVxGk/kgCN0cex2S94BLglUiB0PwOm8hbUhm6g2Nx4= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU= -github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= -github.com/aws/aws-sdk-go-v2/config v1.26.6 h1:Z/7w9bUqlRI0FFQpetVuFYEsjzE3h7fpU6HuGmfPL/o= -github.com/aws/aws-sdk-go-v2/config v1.26.6/go.mod h1:uKU6cnDmYCvJ+pxO9S4cWDb2yWWIH5hra+32hVh1MI4= -github.com/aws/aws-sdk-go-v2/credentials v1.16.16 h1:8q6Rliyv0aUFAVtzaldUEcS+T5gbadPbWdV1WcAddK8= -github.com/aws/aws-sdk-go-v2/credentials v1.16.16/go.mod h1:UHVZrdUsv63hPXFo1H7c5fEneoVo9UXiz36QG1GEPi0= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 h1:n3GDfwqF2tzEkXlv5cuy4iy7LpKDtqDMcNLfZDu9rls= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino= -github.com/aws/aws-sdk-go-v2/service/sns v1.26.7 h1:DylmW2c1Z7qGxN3Y02k+voPbtM1mh7Rp+gV+7maG5io= -github.com/aws/aws-sdk-go-v2/service/sns v1.26.7/go.mod h1:mLFiISZfiZAqZEfPWUsZBK8gD4dYCKuKAfapV+KrIVQ= -github.com/aws/aws-sdk-go-v2/service/sqs v1.29.7 h1:tRNrFDGRm81e6nTX5Q4CFblea99eAfm0dxXazGpLceU= -github.com/aws/aws-sdk-go-v2/service/sqs v1.29.7/go.mod h1:8GWUDux5Z2h6z2efAtr54RdHXtLm8sq7Rg85ZNY/CZM= -github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 h1:eajuO3nykDPdYicLlP3AGgOyVN3MOlFmZv7WGTuJPow= -github.com/aws/aws-sdk-go-v2/service/sso v1.18.7/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 h1:QPMJf+Jw8E1l7zqhZmMlFw6w1NmfkfiSK8mS4zOx3BA= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8= -github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0= -github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U= -github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= -github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/ThreeDotsLabs/watermill v1.3.5 h1:50JEPEhMGZQMh08ct0tfO1PsgMOAOhV3zxK2WofkbXg= +github.com/ThreeDotsLabs/watermill v1.3.5/go.mod h1:O/u/Ptyrk5MPTxSeWM5vzTtZcZfxXfO9PK9eXTYiFZY= +github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8= +github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= +github.com/aws/aws-sdk-go-v2/config v1.27.28 h1:OTxWGW/91C61QlneCtnD62NLb4W616/NM1jA8LhJqbg= +github.com/aws/aws-sdk-go-v2/config v1.27.28/go.mod h1:uzVRVtJSU5EFv6Fu82AoVFKozJi2ZCY6WRCXj06rbvs= +github.com/aws/aws-sdk-go-v2/credentials v1.17.28 h1:m8+AHY/ND8CMHJnPoH7PJIRakWGa4gbfbxuY9TGTUXM= +github.com/aws/aws-sdk-go-v2/credentials v1.17.28/go.mod h1:6TF7dSc78ehD1SL6KpRIPKMA1GyyWflIkjqg+qmf4+c= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 h1:yjwoSyDZF8Jth+mUk5lSPJCkMC0lMy6FaCD51jm6ayE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12/go.mod h1:fuR57fAgMk7ot3WcNQfb6rSEn+SUffl7ri+aa8uKysI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 h1:tJ5RnkHCiSH0jyd6gROjlJtNwov0eGYNz8s8nFcR0jQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18/go.mod h1:++NHzT+nAF7ZPrHPsA+ENvsXkOO8wEu+C6RXltAG4/c= +github.com/aws/aws-sdk-go-v2/service/sns v1.31.4 h1:Bwb1nTBy6jrLJgSlI+jLt27rjyS1Kg030X5yWPnTecI= +github.com/aws/aws-sdk-go-v2/service/sns v1.31.4/go.mod h1:wDacBq+NshhM8KhdysbM4wRFxVyghyj7AAI+l8+o9f0= +github.com/aws/aws-sdk-go-v2/service/sqs v1.34.4 h1:FXPO72iKC5YmYNEANltl763bUj8A6qT20wx8Jwvxlsw= +github.com/aws/aws-sdk-go-v2/service/sqs v1.34.4/go.mod h1:7idt3XszF6sE9WPS1GqZRiDJOxw4oPtlRBXodWnCGjU= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 h1:zCsFCKvbj25i7p1u94imVoO447I/sFv8qq+lGJhRN0c= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.5/go.mod h1:ZeDX1SnKsVlejeuz41GiajjZpRSWR7/42q/EyA/QEiM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 h1:SKvPgvdvmiTWoi0GAJ7AsJfOz3ngVkD/ERbs5pUnHNI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5/go.mod h1:20sz31hv/WsPa3HhU3hfrIet2kxM4Pe0r20eBZ20Tac= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.4 h1:iAckBT2OeEK/kBDyN/jDtpEExhjeeA/Im2q4X0rJZT8= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.4/go.mod h1:vmSqFK+BVIwVpDAGZB3CoCXHzurt4qBE8lf+I/kRTh0= +github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4= +github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -199,378 +50,18 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8= github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/sns/pub_test.go b/sns/pub_test.go index 55b44b9..fd9131a 100644 --- a/sns/pub_test.go +++ b/sns/pub_test.go @@ -113,6 +113,6 @@ func createPubSubWithConsumerGroup(t *testing.T, consumerGroup string) (message. func GetAWSConfig() (aws.Config, error) { return awsconfig.LoadDefaultConfig( context.Background(), - connection.SetEndPoint("http://localhost:4100"), + connection.SetEndPoint("http://localhost:4566"), ) } diff --git a/sqs/pubsub_test.go b/sqs/pubsub_test.go index 0ab2b8b..dae5872 100644 --- a/sqs/pubsub_test.go +++ b/sqs/pubsub_test.go @@ -59,7 +59,7 @@ func createPubSub(t *testing.T) (message.Publisher, message.Subscriber) { SecretAccessKey: "test", }, }), - connection.SetEndPoint("http://localhost:4100"), + connection.SetEndPoint("http://localhost:4566"), ) require.NoError(t, err) diff --git a/sqs/subscriber.go b/sqs/subscriber.go index f53aaab..27f21ed 100644 --- a/sqs/subscriber.go +++ b/sqs/subscriber.go @@ -59,19 +59,11 @@ func (s *Subscriber) Subscribe(ctx context.Context, topic string) (<-chan *messa queueURL, err := getQueueUrl(ctx, s.sqs, topic, s.config.GenerateGetQueueUrlInput) if err != nil { - // todo: should be logged later - // todo: better err handling - s.logger.With(watermill.LogFields{ - "queue": queueURL, - "topic": topic, - }).Error("Failed to get queue", err, nil) - close(output) cancel() - return nil, err + return nil, fmt.Errorf("cannot get queue for topic %s: %w", topic, err) } if queueURL == nil { - s.logger.With(watermill.LogFields{"topic": topic}).Trace("Queue not found", nil) close(output) cancel() return nil, fmt.Errorf("queue %s not found", topic) @@ -84,7 +76,7 @@ func (s *Subscriber) Subscribe(ctx context.Context, topic string) (<-chan *messa return nil, fmt.Errorf("cannot generate input for queue %s: %w", topic, err) } - s.logger.With(watermill.LogFields{"queue": queueURL}).Trace("Subscribing to queue", nil) + s.logger.With(watermill.LogFields{"queue": queueURL}).Debug("Subscribing to queue", nil) s.subscribersWg.Add(1) @@ -106,6 +98,11 @@ func (s *Subscriber) receive(ctx context.Context, queueURL string, output chan * "queue": queueURL, } + go func() { + <-s.closing + cancelCtx() + }() + var sleepTime time.Duration = 0 for { select { @@ -117,16 +114,20 @@ func (s *Subscriber) receive(ctx context.Context, queueURL string, output chan * s.logger.Trace("Stopping consume, context canceled", logFields) return - case <-time.After(sleepTime): // Wait if needed - s.logger.Trace("Timeout?", logFields) + case <-time.After(sleepTime): + // Wait if needed } result, err := s.sqs.ReceiveMessage(ctx, input) - if err != nil { - s.logger.Error("Cannot connect receive messages", err, logFields) - sleepTime = s.config.ReconnectRetrySleep - continue + if errors.Is(err, context.Canceled) { + sleepTime = NoSleep + continue + } else { + s.logger.Error("Cannot connect receive messages", err, logFields) + sleepTime = s.config.ReconnectRetrySleep + continue + } } sleepTime = NoSleep @@ -145,58 +146,69 @@ func (s *Subscriber) ConsumeMessages( output chan *message.Message, logFields watermill.LogFields, ) { - s.logger.Trace("ConsumeMessages", logFields) - for _, sqsMsg := range messages { - s.logger.Trace("ConsumeMessages", logFields) + processed := s.processMessage(ctx, logFields, sqsMsg, output, queueURL) - ctx, cancelCtx := context.WithCancel(ctx) - defer cancelCtx() // todo: leak - msg, err := s.config.Unmarshaler.Unmarshal(&sqsMsg) - if err != nil { - s.logger.Error("Cannot unmarshal message", err, logFields) + if !processed { return } - msg.SetContext(ctx) - output <- msg + } +} - select { - case <-msg.Acked(): - err := s.deleteMessage(ctx, queueURL, sqsMsg.ReceiptHandle) - if err != nil { - s.logger.Error("Failed to delete message", err, logFields) - return - } - case <-msg.Nacked(): - // Do not delete message, it will be redelivered - case <-s.closing: - s.logger.Trace("Closing, message discarded before ack", logFields) - return - case <-ctx.Done(): - s.logger.Trace("Closing, ctx cancelled before ack", logFields) - return +func (s *Subscriber) processMessage(ctx context.Context, logFields watermill.LogFields, sqsMsg types.Message, output chan *message.Message, queueURL string) bool { + s.logger.Trace("processMessage", logFields) + + ctx, cancelCtx := context.WithCancel(ctx) + defer cancelCtx() + + msg, err := s.config.Unmarshaler.Unmarshal(&sqsMsg) + if err != nil { + s.logger.Error("Cannot unmarshal message", err, logFields) + return false + } + msg.SetContext(ctx) + output <- msg + + select { + case <-msg.Acked(): + err := s.deleteMessage(ctx, queueURL, sqsMsg.ReceiptHandle, logFields) + if errors.Is(err, context.Canceled) { + return false } + if err != nil { + s.logger.Error("Failed to delete message", err, logFields) + return false + } + case <-msg.Nacked(): + // Do not delete message, it will be redelivered + return false // we don't want to process next messages to preserve order + case <-s.closing: + s.logger.Trace("Closing, message discarded before ack", logFields) + return false + case <-ctx.Done(): + s.logger.Trace("Closing, ctx cancelled before ack", logFields) + return false } + + return true } -func (s *Subscriber) deleteMessage(ctx context.Context, queueURL string, receiptHandle *string) error { +func (s *Subscriber) deleteMessage(ctx context.Context, queueURL string, receiptHandle *string, logFields watermill.LogFields) error { input, err := s.config.GenerateDeleteMessageInput(ctx, queueURL, receiptHandle) if err != nil { return fmt.Errorf("cannot generate input for delete message: %w", err) } _, err = s.sqs.DeleteMessage(ctx, input) - if err != nil { var oe *smithy.GenericAPIError if errors.As(err, &oe) { if oe.Message == "The specified queue does not contain the message specified." { - // Message was already deleted or is not in queue - // todo: log? + s.logger.Trace("Message was already deleted or is not in queue", logFields) return nil } } - return err + return fmt.Errorf("cannot ack (delete) message: %w", err) } return nil From a7bea07ea1eccf6ebda4b8086e8a5bcd5873b18c Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Sat, 24 Aug 2024 10:23:04 +0200 Subject: [PATCH 07/19] cosmetics --- sqs/config.go | 2 + sqs/publisher.go | 106 +++++++++++++++++++++++++++------- sqs/pubsub_test.go | 140 ++++++++++++++++++++++++++++++++++++++++----- sqs/sqs.go | 22 ++++--- sqs/subscriber.go | 44 +++++++++----- 5 files changed, 258 insertions(+), 56 deletions(-) diff --git a/sqs/config.go b/sqs/config.go index eb1eeff..eaa41ad 100644 --- a/sqs/config.go +++ b/sqs/config.go @@ -74,6 +74,7 @@ type PublisherConfig struct { AWSConfig aws.Config CreateQueueConfig QueueConfigAttributes + DoNotCacheQueues bool CreateQueueIfNotExists bool GenerateGetQueueUrlInput GenerateGetQueueUrlInputFunc @@ -132,6 +133,7 @@ func GenerateReceiveMessageInputDefault(ctx context.Context, queueURL string) (* QueueUrl: aws.String(queueURL), MessageAttributeNames: []string{"All"}, WaitTimeSeconds: 20, // 20 is max at the moment + MaxNumberOfMessages: 1, // Currently default value. }, nil } diff --git a/sqs/publisher.go b/sqs/publisher.go index bc75174..b20722b 100644 --- a/sqs/publisher.go +++ b/sqs/publisher.go @@ -2,34 +2,41 @@ package sqs import ( "context" + "encoding/json" "fmt" + "sync" "github.com/ThreeDotsLabs/watermill" "github.com/ThreeDotsLabs/watermill/message" "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sqs/types" + "github.com/pkg/errors" ) type Publisher struct { config PublisherConfig logger watermill.LoggerAdapter sqs *sqs.Client + + queuesCache map[string]*string + queuesCacheLock sync.RWMutex } func NewPublisher(config PublisherConfig, logger watermill.LoggerAdapter) (*Publisher, error) { config.setDefaults() return &Publisher{ - sqs: sqs.NewFromConfig(config.AWSConfig), - config: config, - logger: logger, + sqs: sqs.NewFromConfig(config.AWSConfig), + config: config, + logger: logger, + queuesCache: make(map[string]*string), }, nil } -func (p Publisher) Publish(topic string, messages ...*message.Message) error { +func (p *Publisher) Publish(topic string, messages ...*message.Message) error { ctx := context.Background() - // todo: cache it - queueUrl, err := p.GetQueueUrl(ctx, topic) + queueUrl, err := p.GetOrCreateQueueUrl(ctx, topic) if err != nil { return fmt.Errorf("cannot get queue url: %w", err) } @@ -59,32 +66,87 @@ func (p Publisher) Publish(topic string, messages ...*message.Message) error { return nil } -func (p Publisher) GetQueueUrl(ctx context.Context, topic string) (*string, error) { - queueUrl, err := getQueueUrl(ctx, p.sqs, topic, p.config.GenerateGetQueueUrlInput) +func (p *Publisher) GetOrCreateQueueUrl(ctx context.Context, topic string) (queueUrl *string, err error) { + getQueueInput, err := p.config.GenerateGetQueueUrlInput(ctx, topic) if err != nil { - // todo: check exact error here - if p.config.CreateQueueIfNotExists { - input, err := p.config.GenerateCreateQueueInput(ctx, topic, p.config.CreateQueueConfig) - if err != nil { - return nil, fmt.Errorf("cannot generate create queue input: %w", err) - } + return nil, fmt.Errorf("cannot generate input for queue %s: %w", topic, err) + } + + var getQueueInputHash string + + if !p.config.DoNotCacheQueues { + getQueueInputHash = generateGetQueueUrlInputHash(getQueueInput) + + if getQueueInputHash != "" { + p.queuesCacheLock.RLock() + var ok bool + queueUrl, ok = p.queuesCache[getQueueInputHash] + p.queuesCacheLock.RUnlock() - queueUrl, err = greateQueue(ctx, p.sqs, input) - if err == nil { + if ok { + p.logger.Trace("Used cache", watermill.LogFields{ + "topic": topic, + "queue": *queueUrl, + "hash": getQueueInputHash, + }) return queueUrl, nil } } - return nil, err } - return queueUrl, nil + + defer func() { + if err == nil && getQueueInputHash != "" { + p.queuesCacheLock.Lock() + p.queuesCache[getQueueInputHash] = queueUrl + p.queuesCacheLock.Unlock() + + p.logger.Trace("Stored cache", watermill.LogFields{ + "topic": topic, + "queue": *queueUrl, + "hash": getQueueInputHash, + }) + } + }() + + queueUrl, err = getQueueUrl(ctx, p.sqs, topic, getQueueInput) + if err == nil { + return queueUrl, nil + } + + var queueDoesNotExistsErr *types.QueueDoesNotExist + if errors.As(err, &queueDoesNotExistsErr) && p.config.CreateQueueIfNotExists { + input, err := p.config.GenerateCreateQueueInput(ctx, topic, p.config.CreateQueueConfig) + if err != nil { + return nil, fmt.Errorf("cannot generate create queue input: %w", err) + } + + queueUrl, err = createQueue(ctx, p.sqs, input) + if err != nil { + return nil, fmt.Errorf("cannot create queue: %w", err) + } + // queue was created in the meantime + if queueUrl == nil { + return getQueueUrl(ctx, p.sqs, topic, getQueueInput) + } + + return queueUrl, nil + } else { + return nil, fmt.Errorf("cannot get queue url: %w", err) + } } -func (p Publisher) GetQueueArn(ctx context.Context, url *string) (*string, error) { +func generateGetQueueUrlInputHash(getQueueInput *sqs.GetQueueUrlInput) string { + // we are not using fmt.Sprintf because of pointers under the hood + // we are not hashing specific struct fields to keep forward compatibility + // also, json.Marshal is faster than fmt.Sprintf + b, _ := json.Marshal(getQueueInput) + return string(b) +} + +func (p *Publisher) GetQueueArn(ctx context.Context, url *string) (*string, error) { return getARNUrl(ctx, p.sqs, url) } -func (p Publisher) Close() error { +func (p *Publisher) Close() error { return nil } - -// todo: missing validate? diff --git a/sqs/pubsub_test.go b/sqs/pubsub_test.go index dae5872..41bcbd8 100644 --- a/sqs/pubsub_test.go +++ b/sqs/pubsub_test.go @@ -8,6 +8,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" + awssqs "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/stretchr/testify/require" "github.com/ThreeDotsLabs/watermill" @@ -17,7 +18,7 @@ import ( "github.com/ThreeDotsLabs/watermill/pubsub/tests" ) -func TestPublishSubscribe(t *testing.T) { +func TestPubSub(t *testing.T) { tests.TestPubSub( t, tests.Features{ @@ -31,7 +32,7 @@ func TestPublishSubscribe(t *testing.T) { ) } -func TestPublishSubscribe_stress(t *testing.T) { +func TestPubSub_stress(t *testing.T) { runtime.GOMAXPROCS(runtime.GOMAXPROCS(0) * 2) tests.TestPubSubStressTest( @@ -47,22 +48,117 @@ func TestPublishSubscribe_stress(t *testing.T) { ) } -func createPubSub(t *testing.T) (message.Publisher, message.Subscriber) { - logger := watermill.NewStdLogger(false, false) - - cfg, err := awsconfig.LoadDefaultConfig( - context.Background(), - awsconfig.WithRegion("us-west-2"), - awsconfig.WithCredentialsProvider(credentials.StaticCredentialsProvider{ - Value: aws.Credentials{ - AccessKeyID: "test", - SecretAccessKey: "test", +func TestPublishSubscribe_batching(t *testing.T) { + tests.TestPublishSubscribe( + t, + tests.TestContext{ + TestID: tests.NewTestID(), + Features: tests.Features{ + ConsumerGroups: true, + ExactlyOnceDelivery: false, + GuaranteedOrder: true, + GuaranteedOrderWithSingleSubscriber: true, + Persistent: true, }, - }), - connection.SetEndPoint("http://localhost:4566"), + }, + func(t *testing.T) (message.Publisher, message.Subscriber) { + logger := watermill.NewStdLogger(false, false) + + cfg := newAwsConfig(t) + + pub, err := sqs.NewPublisher(sqs.PublisherConfig{ + AWSConfig: cfg, + CreateQueueConfig: sqs.QueueConfigAttributes{ + // Defalt value is 30 seconds - need to be lower for tests + VisibilityTimeout: "1", + }, + CreateQueueIfNotExists: true, + Marshaler: sqs.DefaultMarshalerUnmarshaler{}, + }, logger) + require.NoError(t, err) + + sub, err := sqs.NewSubscriber(sqs.SubscriberConfig{ + AWSConfig: cfg, + CreateQueueInitializerConfig: sqs.QueueConfigAttributes{ + // Defalt value is 30 seconds - need to be lower for tests + VisibilityTimeout: "1", + }, + GenerateReceiveMessageInput: func(ctx context.Context, queueURL string) (*awssqs.ReceiveMessageInput, error) { + in, err := sqs.GenerateReceiveMessageInputDefault(ctx, queueURL) + if err != nil { + return nil, err + } + + // this will effectively enable batching + in.MaxNumberOfMessages = 10 + + return in, nil + }, + Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, + }, logger) + require.NoError(t, err) + + return pub, sub + }, ) +} + +func TestPublishSubscribe_creating_queue_with_different_settings_should_be_idempotent(t *testing.T) { + logger := watermill.NewStdLogger(false, false) + + sub1, err := sqs.NewSubscriber(sqs.SubscriberConfig{ + AWSConfig: newAwsConfig(t), + CreateQueueInitializerConfig: sqs.QueueConfigAttributes{ + // Defalt value is 30 seconds - need to be lower for tests + VisibilityTimeout: "1", + }, + Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, + }, logger) + require.NoError(t, err) + + sub2, err := sqs.NewSubscriber(sqs.SubscriberConfig{ + AWSConfig: newAwsConfig(t), + CreateQueueInitializerConfig: sqs.QueueConfigAttributes{ + // Defalt value is 30 seconds - need to be lower for tests + VisibilityTimeout: "20", + }, + Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, + }, logger) + require.NoError(t, err) + + topicName := watermill.NewUUID() + + require.NoError(t, sub1.SubscribeInitialize(topicName)) + require.NoError(t, sub2.SubscribeInitialize(topicName)) +} + +func TestPublisher_GetOrCreateQueueUrl_is_idempotent(t *testing.T) { + pub, _ := createPubSub(t) + + topicName := watermill.NewUUID() + + url1, err := pub.(*sqs.Publisher).GetOrCreateQueueUrl(context.Background(), topicName) + require.NoError(t, err) + + url2, err := pub.(*sqs.Publisher).GetOrCreateQueueUrl(context.Background(), topicName) require.NoError(t, err) + require.Equal(t, url1, url2) +} + +func TestSubscriber_doesnt_hang_when_queue_doesnt_exist(t *testing.T) { + _, sub := createPubSub(t) + msgs, err := sub.Subscribe(context.Background(), "non-existing-queue") + + require.ErrorContains(t, err, "cannot get queue for topic non-existing-queue: cannot get queue non-existing-queue") + require.Nil(t, msgs) +} + +func createPubSub(t *testing.T) (message.Publisher, message.Subscriber) { + logger := watermill.NewStdLogger(false, false) + + cfg := newAwsConfig(t) + pub, err := sqs.NewPublisher(sqs.PublisherConfig{ AWSConfig: cfg, CreateQueueConfig: sqs.QueueConfigAttributes{ @@ -87,6 +183,22 @@ func createPubSub(t *testing.T) (message.Publisher, message.Subscriber) { return pub, sub } +func newAwsConfig(t *testing.T) aws.Config { + cfg, err := awsconfig.LoadDefaultConfig( + context.Background(), + awsconfig.WithRegion("us-west-2"), + awsconfig.WithCredentialsProvider(credentials.StaticCredentialsProvider{ + Value: aws.Credentials{ + AccessKeyID: "test", + SecretAccessKey: "test", + }, + }), + connection.SetEndPoint("http://localhost:4566"), + ) + require.NoError(t, err) + return cfg +} + func createPubSubWithConsumerGroup(t *testing.T, consumerGroup string) (message.Publisher, message.Subscriber) { return createPubSub(t) } diff --git a/sqs/sqs.go b/sqs/sqs.go index 9ad108f..2dbe103 100644 --- a/sqs/sqs.go +++ b/sqs/sqs.go @@ -7,6 +7,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/types" + "github.com/pkg/errors" ) type QueueConfigAttributes struct { @@ -54,12 +55,7 @@ func (q QueueConfigAttributes) Attributes() (map[string]string, error) { return m, nil } -func getQueueUrl(ctx context.Context, sqsClient *sqs.Client, topic string, generateGetQueueUrlInput GenerateGetQueueUrlInputFunc) (*string, error) { - input, err := generateGetQueueUrlInput(ctx, topic) - if err != nil { - return nil, fmt.Errorf("cannot generate input for queue %s: %w", topic, err) - } - +func getQueueUrl(ctx context.Context, sqsClient *sqs.Client, topic string, input *sqs.GetQueueUrlInput) (*string, error) { getQueueOutput, err := sqsClient.GetQueueUrl(ctx, input) if err != nil || getQueueOutput.QueueUrl == nil { @@ -68,8 +64,20 @@ func getQueueUrl(ctx context.Context, sqsClient *sqs.Client, topic string, gener return getQueueOutput.QueueUrl, nil } -func greateQueue(ctx context.Context, sqsClient *sqs.Client, createQueueParams *sqs.CreateQueueInput) (*string, error) { +func createQueue( + ctx context.Context, + sqsClient *sqs.Client, + createQueueParams *sqs.CreateQueueInput, +) (*string, error) { createQueueOutput, err := sqsClient.CreateQueue(ctx, createQueueParams) + // possible scenarios: + // 1. queue already exists, but with different params + //(for example: "A queue already exists with the same name and a different value for attribute VisibilityTimeout") + // 2. queue was created in the meantime + var queueExistsErrr *types.QueueNameExists + if errors.As(err, &queueExistsErrr) { + return nil, nil + } if err != nil { return nil, fmt.Errorf("cannot create queue %w", err) } diff --git a/sqs/subscriber.go b/sqs/subscriber.go index 27f21ed..fa38339 100644 --- a/sqs/subscriber.go +++ b/sqs/subscriber.go @@ -52,12 +52,17 @@ func (s *Subscriber) Subscribe(ctx context.Context, topic string) (<-chan *messa return nil, errors.New("subscriber closed") } - s.logger.With(watermill.LogFields{"topic": topic}).Trace("Getting queue", nil) + s.logger.With(watermill.LogFields{"topic": topic}).Debug("Getting queue", nil) + + getQueueInput, err := s.config.GenerateGetQueueUrlInput(ctx, topic) + if err != nil { + return nil, fmt.Errorf("cannot generate input for queue %s: %w", topic, err) + } ctx, cancel := context.WithCancel(ctx) output := make(chan *message.Message) - queueURL, err := getQueueUrl(ctx, s.sqs, topic, s.config.GenerateGetQueueUrlInput) + queueURL, err := getQueueUrl(ctx, s.sqs, topic, getQueueInput) if err != nil { close(output) cancel() @@ -66,17 +71,17 @@ func (s *Subscriber) Subscribe(ctx context.Context, topic string) (<-chan *messa if queueURL == nil { close(output) cancel() - return nil, fmt.Errorf("queue %s not found", topic) + return nil, fmt.Errorf("queue for topic %s not found", topic) } receiveInput, err := s.config.GenerateReceiveMessageInput(ctx, *queueURL) if err != nil { close(output) cancel() - return nil, fmt.Errorf("cannot generate input for queue %s: %w", topic, err) + return nil, fmt.Errorf("cannot generate input for topic %s: %w", topic, err) } - s.logger.With(watermill.LogFields{"queue": queueURL}).Debug("Subscribing to queue", nil) + s.logger.With(watermill.LogFields{"queue": *queueURL}).Info("Subscribing to queue", nil) s.subscribersWg.Add(1) @@ -107,11 +112,11 @@ func (s *Subscriber) receive(ctx context.Context, queueURL string, output chan * for { select { case <-s.closing: - s.logger.Trace("Discarding queued message, subscriber closing", logFields) + s.logger.Debug("Discarding queued message, subscriber closing", logFields) return case <-ctx.Done(): - s.logger.Trace("Stopping consume, context canceled", logFields) + s.logger.Debug("Stopping consume, context canceled", logFields) return case <-time.After(sleepTime): @@ -183,10 +188,10 @@ func (s *Subscriber) processMessage(ctx context.Context, logFields watermill.Log // Do not delete message, it will be redelivered return false // we don't want to process next messages to preserve order case <-s.closing: - s.logger.Trace("Closing, message discarded before ack", logFields) + s.logger.Debug("Closing, message discarded before ack", logFields) return false case <-ctx.Done(): - s.logger.Trace("Closing, ctx cancelled before ack", logFields) + s.logger.Debug("Closing, ctx cancelled before ack", logFields) return false } @@ -199,12 +204,21 @@ func (s *Subscriber) deleteMessage(ctx context.Context, queueURL string, receipt return fmt.Errorf("cannot generate input for delete message: %w", err) } + // we are using ctx that may be canceled when subscriber is closed + // + // it may lead to re-delivery when message is processed and in the meantime + // subscriber is closed - but we don't know if context cancellation didn't cancel + // some SQL transactions or whatever - so someone may lose data + // + // in other words, we prefer re-delivery (as at least once delivery is a thing anyway) _, err = s.sqs.DeleteMessage(ctx, input) if err != nil { var oe *smithy.GenericAPIError if errors.As(err, &oe) { + // todo(roblaszczak): it would be nice to replace it with a specific error type + // but I wasn't able to reproduce it if oe.Message == "The specified queue does not contain the message specified." { - s.logger.Trace("Message was already deleted or is not in queue", logFields) + s.logger.Debug("Message was already deleted or is not in queue", logFields) return nil } } @@ -237,7 +251,7 @@ func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, topic s return fmt.Errorf("cannot generate input for queue %s: %w", topic, err) } - _, err = greateQueue(ctx, s.sqs, input) + _, err = createQueue(ctx, s.sqs, input) if err != nil { return fmt.Errorf("cannot create queue %s: %w", topic, err) } @@ -246,8 +260,12 @@ func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, topic s } func (s *Subscriber) GetQueueUrl(ctx context.Context, topic string) (*string, error) { - queueUrl, err := getQueueUrl(ctx, s.sqs, topic, s.config.GenerateGetQueueUrlInput) - return queueUrl, err + getQueueInput, err := s.config.GenerateGetQueueUrlInput(ctx, topic) + if err != nil { + return nil, fmt.Errorf("cannot generate input for queue %s: %w", topic, err) + } + + return getQueueUrl(ctx, s.sqs, topic, getQueueInput) } func (s *Subscriber) GetQueueArn(ctx context.Context, url *string) (*string, error) { From dc70a8a55b62575342960acf51ba3ac3eb484cb7 Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Sat, 24 Aug 2024 10:46:39 +0200 Subject: [PATCH 08/19] more docs --- sqs/config.go | 158 +++++++++++++++++++++++++++- sqs/{sqs_test.go => config_test.go} | 21 ++++ sqs/sqs.go | 46 -------- 3 files changed, 177 insertions(+), 48 deletions(-) rename sqs/{sqs_test.go => config_test.go} (73%) diff --git a/sqs/config.go b/sqs/config.go index eaa41ad..6de4b7b 100644 --- a/sqs/config.go +++ b/sqs/config.go @@ -2,6 +2,7 @@ package sqs import ( "context" + "encoding/json" "errors" "fmt" "time" @@ -14,17 +15,22 @@ import ( type SubscriberConfig struct { AWSConfig aws.Config - // How long about unsuccessful reconnecting next reconnect will occur. + // ReconnectRetrySleep is the time to sleep between reconnect attempts. ReconnectRetrySleep time.Duration - // Delete message attempts + + // CreateQueueInitializerConfig is a struct that holds the attributes of an SQS queue. CreateQueueInitializerConfig QueueConfigAttributes + // GenerateCreateQueueInput generates *sqs.CreateQueueInput for AWS SDK. GenerateCreateQueueInput GenerateCreateQueueInputFunc + // GenerateGetQueueUrlInput generates *sqs.GetQueueUrlInput for AWS SDK. GenerateGetQueueUrlInput GenerateGetQueueUrlInputFunc + // GenerateReceiveMessageInput generates *sqs.ReceiveMessageInput for AWS SDK. GenerateReceiveMessageInput GenerateReceiveMessageInputFunc + // GenerateDeleteMessageInput generates *sqs.DeleteMessageInput for AWS SDK. GenerateDeleteMessageInput GenerateDeleteMessageInputFunc Unmarshaler Unmarshaler @@ -77,10 +83,13 @@ type PublisherConfig struct { DoNotCacheQueues bool CreateQueueIfNotExists bool + // GenerateGetQueueUrlInput generates *sqs.GetQueueUrlInput for AWS SDK. GenerateGetQueueUrlInput GenerateGetQueueUrlInputFunc + // GenerateSendMessageInput generates *sqs.SendMessageInput for AWS SDK. GenerateSendMessageInput GenerateSendMessageInputFunc + // GenerateCreateQueueInput generates *sqs.CreateQueueInput for AWS SDK. GenerateCreateQueueInput GenerateCreateQueueInputFunc Marshaler Marshaler @@ -155,3 +164,148 @@ func GenerateSendMessageInputDefault(ctx context.Context, queueURL string, msg * MessageBody: msg.Body, }, nil } + +// QueueConfigAttributes is a struct that holds the attributes of an SQS queue. +// https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SetQueueAttributes.html +type QueueConfigAttributes struct { + // DelaySeconds – The length of time, in seconds, for which the delivery of all messages in the queue is delayed. + // Valid values: An integer from 0 to 900 (15 minutes). Default: 0. + DelaySeconds string `json:"DelaySeconds,omitempty"` + + // MaximumMessageSize – The limit of how many bytes a message can contain before Amazon SQS rejects it. + // Valid values: An integer from 1,024 bytes (1 KiB) up to 262,144 bytes (256 KiB). Default: 262,144 (256 KiB). + MaximumMessageSize string `json:"MaximumMessageSize,omitempty"` + + // MessageRetentionPeriod – The length of time, in seconds, for which Amazon SQS retains a message. + // Valid values: An integer representing seconds, from 60 (1 minute) to 1,209,600 (14 days). + // Default: 345,600 (4 days). + // + // When you change a queue's attributes, the change can take up to 60 seconds for most of the attributes to + // propagate throughout the Amazon SQS system. Changes made to the MessageRetentionPeriod attribute can take up to + // 15 minutes and will impact existing messages in the queue potentially causing them to be expired and deleted if + // the MessageRetentionPeriod is reduced below the age of existing messages. + MessageRetentionPeriod string `json:"MessageRetentionPeriod,omitempty"` + + // Policy – The queue's policy. A valid AWS policy. For more information about policy structure, see + // [Overview of AWS IAM Policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) in the + // AWS Identity and Access Management User Guide. + Policy string `json:"Policy,omitempty"` + + // ReceiveMessageWaitTimeSeconds – The length of time, in seconds, for which a ReceiveMessage action waits for a + // message to arrive. + // Valid values: An integer from 0 to 20 (seconds). Default: 0. + ReceiveMessageWaitTimeSeconds string `json:"ReceiveMessageWaitTimeSeconds,omitempty"` + + // RedrivePolicy – The string that includes the parameters for the dead-letter queue functionality of the source + // queue as a JSON object. The parameters are as follows: + RedrivePolicy string `json:"RedrivePolicy,omitempty"` + + // DeadLetterTargetArn – The Amazon Resource Name (ARN) of the dead-letter queue to which Amazon SQS moves + // messages after the value of maxReceiveCount is exceeded. + DeadLetterTargetArn string `json:"deadLetterTargetArn,omitempty"` + + // MaxReceiveCount – The number of times a message is delivered to the source queue before being moved to the + // dead-letter queue. Default: 10. When the ReceiveCount for a message exceeds the maxReceiveCount for a queue, + // Amazon SQS moves the message to the dead-letter-queue. + MaxReceiveCount string `json:"maxReceiveCount,omitempty"` + + // VisibilityTimeout – The visibility timeout for the queue, in seconds. + // Valid values: An integer from 0 to 43,200 (12 hours). Default: 30. + + // For more information about the visibility timeout, see + // [Visibility Timeout](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html) in the Amazon SQS Developer Guide. + VisibilityTimeout string `json:"VisibilityTimeout,omitempty"` + + // KmsMasterKeyId – The ID of an AWS managed customer master key (CMK) for Amazon SQS or a custom CMK. + // For more information, see [Key Terms](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-server-side-encryption.html#sqs-sse-key-terms). + // + // While the alias of the AWS-managed CMK for Amazon SQS is always + // alias/aws/sqs, the alias of a custom CMK can, for example, be alias/MyAlias. + // For more examples, see [KeyId](https://docs.aws.amazon.com/kms/latest/APIReference/API_DescribeKey.html#API_DescribeKey_RequestParameters) + // in the AWS Key Management Service API Reference. + KmsMasterKeyId string `json:"KmsMasterKeyId,omitempty"` + + // KmsDataKeyReusePeriodSeconds – The length of time, in seconds, for which Amazon SQS can reuse a [data key](https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#data-keys) + // to encrypt or decrypt messages before calling AWS KMS again. + // An integer representing seconds, between 60 seconds (1 minute) and 86,400 seconds (24 hours). + // Default: 300 (5 minutes). + // + // A shorter time period provides better security but results in more calls to KMS which might incur charges after Free Tier. + // For more information, see How Does the [Data Key Reuse Period Work?](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-server-side-encryption.html#sqs-how-does-the-data-key-reuse-period-work). + KmsDataKeyReusePeriodSeconds string `json:"KmsDataKeyReusePeriodSeconds,omitempty"` + + // SqsManagedSseEnabled – Enables server-side queue encryption using SQS owned encryption keys. + // Only one server-side encryption option is supported per queue (for example, [SSE-KMS](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-configure-sse-existing-queue.html) or [SSE-SQS](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-configure-sqs-sse-queue.html). + SqsManagedSseEnabled string `json:"SqsManagedSseEnabled,omitempty"` + + // FifoQueue - Designates a queue as FIFO. Valid values: true, false. Default: false. + FifoQueue QueueConfigAttributesBool `json:"FifoQueue,omitempty"` + + // ContentBasedDeduplication – Enables content-based deduplication. For more information, see [Exactly-once processing](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues-exactly-once-processing.html) + // in the Amazon SQS Developer Guide. Note the following: + // + // Every message must have a unique MessageDeduplicationId. + // You may provide a MessageDeduplicationId explicitly. + + // If you aren't able to provide a MessageDeduplicationId and you enable ContentBasedDeduplication for your queue, + // Amazon SQS uses a SHA-256 hash to generate the MessageDeduplicationId using the body of the message + // (but not the attributes of the message). + + // If you don't provide a MessageDeduplicationId and the queue doesn't have ContentBasedDeduplication set, the + // action fails with an error. + // If the queue has ContentBasedDeduplication set, your MessageDeduplicationId overrides the generated one. + // When ContentBasedDeduplication is in effect, messages with identical content sent within the deduplication + // interval are treated as duplicates and only one copy of the message is delivered. + // + // If you send one message with ContentBasedDeduplication enabled and then another message with a + // MessageDeduplicationId that is the same as the one generated for the first MessageDeduplicationId, the two + // messages are treated as duplicates and only one copy of the message is delivered. + ContentBasedDeduplication QueueConfigAttributesBool `json:"ContentBasedDeduplication,omitempty"` + + // DeduplicationScope – Specifies whether message deduplication occurs at the message group or queue level. + // Valid values are messageGroup and queue. + // + // Apply only to [high throughput for FIFO queues](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/high-throughput-fifo.html). + DeduplicationScope string `json:"DeduplicationScope,omitempty"` + + // FifoThroughputLimit – Specifies whether the FIFO queue throughput quota applies to the entire queue or per message group. + // Valid values are perQueue and perMessageGroupId. + // The perMessageGroupId value is allowed only when the value for DeduplicationScope is messageGroup. + // Apply only to [high throughput for FIFO queues](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/high-throughput-fifo.html). + FifoThroughputLimit string `json:"FifoThroughputLimit,omitempty"` + + // CustomAttributes is a map of custom attributes that are not mapped to the struct fields. + CustomAttributes map[string]string `json:"-"` +} + +// QueueConfigAttributesBool is a custom type for bool values in QueueConfigAttributes +// that supports marshaling to string. +type QueueConfigAttributesBool bool + +func (q QueueConfigAttributesBool) MarshalText() ([]byte, error) { + if q { + return []byte("true"), nil + } + return []byte("false"), nil +} + +func (q QueueConfigAttributes) Attributes() (map[string]string, error) { + b, err := json.Marshal(q) + if err != nil { + return nil, fmt.Errorf("cannot marshal queue attributes (json.Marshal): %w", err) + } + + var m map[string]string + err = json.Unmarshal(b, &m) + if err != nil { + return nil, fmt.Errorf("cannot marshal queue attributes (json.Unmarshal): %w", err) + } + + if q.CustomAttributes != nil { + for k, v := range q.CustomAttributes { + m[k] = v + } + } + + return m, nil +} diff --git a/sqs/sqs_test.go b/sqs/config_test.go similarity index 73% rename from sqs/sqs_test.go rename to sqs/config_test.go index ac40d4e..7d0ff0b 100644 --- a/sqs/sqs_test.go +++ b/sqs/config_test.go @@ -38,3 +38,24 @@ func TestQueueConfigAttributes_Attributes(t *testing.T) { attrs, ) } + +func TestQueueConfigAttributes_Attributes_custom_attributes(t *testing.T) { + structAttrs := sqs.QueueConfigAttributes{ + DelaySeconds: "10", + CustomAttributes: map[string]string{ + "test": "test", + }, + } + + attrs, err := structAttrs.Attributes() + require.NoError(t, err) + + require.Equal( + t, + map[string]string{ + "DelaySeconds": "10", + "test": "test", + }, + attrs, + ) +} diff --git a/sqs/sqs.go b/sqs/sqs.go index 2dbe103..55b1586 100644 --- a/sqs/sqs.go +++ b/sqs/sqs.go @@ -2,7 +2,6 @@ package sqs import ( "context" - "encoding/json" "fmt" "github.com/aws/aws-sdk-go-v2/service/sqs" @@ -10,51 +9,6 @@ import ( "github.com/pkg/errors" ) -type QueueConfigAttributes struct { - DelaySeconds string `json:"DelaySeconds,omitempty"` - MaximumMessageSize string `json:"MaximumMessageSize,omitempty"` - MessageRetentionPeriod string `json:"MessageRetentionPeriod,omitempty"` - Policy string `json:"Policy,omitempty"` - ReceiveMessageWaitTimeSeconds string `json:"ReceiveMessageWaitTimeSeconds,omitempty"` - RedrivePolicy string `json:"RedrivePolicy,omitempty"` - DeadLetterTargetArn string `json:"deadLetterTargetArn,omitempty"` - MaxReceiveCount string `json:"maxReceiveCount,omitempty"` - VisibilityTimeout string `json:"VisibilityTimeout,omitempty"` - KmsMasterKeyId string `json:"KmsMasterKeyId,omitempty"` - KmsDataKeyReusePeriodSeconds string `json:"KmsDataKeyReusePeriodSeconds,omitempty"` - SqsManagedSseEnabled string `json:"SqsManagedSseEnabled,omitempty"` - FifoQueue QueueConfigAttributesBool `json:"FifoQueue,omitempty"` - ContentBasedDeduplication QueueConfigAttributesBool `json:"ContentBasedDeduplication,omitempty"` - DeduplicationScope string `json:"DeduplicationScope,omitempty"` - FifoThroughputLimit string `json:"FifoThroughputLimit,omitempty"` -} - -// QueueConfigAttributesBool is a custom type for bool values in QueueConfigAttributes -// that supports marshaling to string. -type QueueConfigAttributesBool bool - -func (q QueueConfigAttributesBool) MarshalText() ([]byte, error) { - if q { - return []byte("true"), nil - } - return []byte("false"), nil -} - -func (q QueueConfigAttributes) Attributes() (map[string]string, error) { - b, err := json.Marshal(q) - if err != nil { - return nil, fmt.Errorf("cannot marshal queue attributes (json.Marshal): %w", err) - } - - var m map[string]string - err = json.Unmarshal(b, &m) - if err != nil { - return nil, fmt.Errorf("cannot marshal queue attributes (json.Unmarshal): %w", err) - } - - return m, nil -} - func getQueueUrl(ctx context.Context, sqsClient *sqs.Client, topic string, input *sqs.GetQueueUrlInput) (*string, error) { getQueueOutput, err := sqsClient.GetQueueUrl(ctx, input) From e457144a3c8de5f4bf75baf0ac647abc697519ed Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Sat, 24 Aug 2024 17:31:01 +0200 Subject: [PATCH 09/19] add support for sns subscribing + cleanups --- sns/config.go | 198 +++++++++++++++++++++++++++++++++++++++++++++ sns/marshaler.go | 7 +- sns/pub_test.go | 148 ++++++++++++++++++--------------- sns/publisher.go | 98 ++++++++++++---------- sns/sns.go | 48 +++++------ sns/subscriber.go | 120 +++++++++++++++++++++++++++ sqs/config.go | 6 +- sqs/pubsub_test.go | 24 +++--- sqs/subscriber.go | 33 +++++--- 9 files changed, 519 insertions(+), 163 deletions(-) create mode 100644 sns/config.go create mode 100644 sns/subscriber.go diff --git a/sns/config.go b/sns/config.go new file mode 100644 index 0000000..67e2cd3 --- /dev/null +++ b/sns/config.go @@ -0,0 +1,198 @@ +package sns + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/sns" +) + +type PublisherConfig struct { + // todo: add func for that? + AwsAccountID string + AWSConfig aws.Config + + CreateTopicConfig ConfigAttributes + DoNotCreateTopicIfNotExists bool + + GenerateCreateTopicInput GenerateCreateTopicInputFunc + + Marshaler Marshaler +} + +type GenerateCreateTopicInputFunc func(ctx context.Context, topic string, attrs ConfigAttributes) (sns.CreateTopicInput, error) + +func GenerateCreateTopicInputDefault(ctx context.Context, topic string, attrs ConfigAttributes) (sns.CreateTopicInput, error) { + attrsMap, err := attrs.Attributes() + if err != nil { + return sns.CreateTopicInput{}, fmt.Errorf("cannot generate attributes for topic %s: %w", topic, err) + } + + return sns.CreateTopicInput{ + Name: aws.String(topic), + Attributes: attrsMap, + }, nil +} + +func (c *PublisherConfig) setDefaults() { + if c.Marshaler == nil { + c.Marshaler = DefaultMarshalerUnmarshaler{} + } + if c.GenerateCreateTopicInput == nil { + c.GenerateCreateTopicInput = GenerateCreateTopicInputDefault + } +} + +func (c *PublisherConfig) Validate() error { + var err error + + if c.AwsAccountID == "" { + err = errors.Join(err, fmt.Errorf("sns.PublisherConfig.AwsAccountID is missing")) + } + if c.AWSConfig.Credentials == nil { + err = errors.Join(err, fmt.Errorf("sns.PublisherConfig.AWSConfig.Credentials is nil")) + } + + return err +} + +type SubscriberConfig struct { + AwsAccountID string + AWSConfig aws.Config + + GenerateSqsQueueName func(ctx context.Context, sqsTopic string) (string, error) + + GenerateSubscribeInput GenerateSubscribeInputFn + + DoNotSubscribeToSns bool +} + +func (c *SubscriberConfig) SetDefaults() { + if c.GenerateSqsQueueName == nil { + c.GenerateSqsQueueName = func(ctx context.Context, sqsTopic string) (string, error) { + return sqsTopic, nil + } + } + if c.GenerateSubscribeInput == nil { + c.GenerateSubscribeInput = GenerateSubscribeInputDefault + } +} + +func (c *SubscriberConfig) Validate() error { + var err error + + if c.AwsAccountID == "" { + err = errors.Join(err, fmt.Errorf("sns.SubscriberConfig.AwsAccountID is missing")) + } + if c.AWSConfig.Credentials == nil { + err = errors.Join(err, fmt.Errorf("sns.SubscriberConfig.AWSConfig.Credentials is nil")) + } + + return err +} + +type GenerateSubscribeInputFn func(ctx context.Context, params GenerateSubscribeInputParams) (*sns.SubscribeInput, error) + +type GenerateSubscribeInputParams struct { + SnsTopic string + SqsTopic string + + SnsTopicArn string + SqsQueueArn string +} + +func GenerateSubscribeInputDefault(ctx context.Context, params GenerateSubscribeInputParams) (*sns.SubscribeInput, error) { + return &sns.SubscribeInput{ + Protocol: aws.String("sqs"), + TopicArn: ¶ms.SnsTopicArn, + Endpoint: ¶ms.SqsQueueArn, + Attributes: map[string]string{ + "RawMessageDelivery": "true", + }, + }, nil +} + +// ConfigAttributes is a struct that holds the attributes of an SNS topic +type ConfigAttributes struct { + // DeliveryPolicy – The policy that defines how Amazon SNS retries failed + // deliveries to HTTP/S endpoints. + DeliveryPolicy string `json:"DeliveryPolicy,omitempty"` + + // DisplayName – The display name to use for a topic with SMS subscriptions. + DisplayName string `json:"DisplayName,omitempty"` + + // Policy – The policy that defines who can access your topic. By default, only + // the topic owner can publish or subscribe to the topic. + Policy string `json:"Policy,omitempty"` + + // SignatureVersion – The signature version corresponds to the hashing + // algorithm used while creating the signature of the notifications, subscription + // confirmations, or unsubscribe confirmation messages sent by Amazon SNS. By + // default, SignatureVersion is set to 1 . + SignatureVersion string `json:"SignatureVersion,omitempty"` + + // TracingConfig – Tracing mode of an Amazon SNS topic. + // By default TracingConfig is set to PassThrough , and the topic passes through the tracing + // header it receives from an Amazon SNS publisher to its subscriptions. If set to + // Active , Amazon SNS will vend X-Ray segment data to topic owner account if the + // sampled flag in the tracing header is true. This is only supported on standard + // topics. + TracingConfig string `json:"TracingConfig,omitempty"` + + // KmsMasterKeyId – The ID of an Amazon Web Services managed customer master + // key (CMK) for Amazon SNS or a custom CMK. For more information, see [Key Terms]. For + // more examples, see [KeyId]in the Key Management Service API Reference. + // + // Applies only to server-side encryption. + KmsMasterKeyId string `json:"KmsMasterKeyId,omitempty"` + + // FifoTopic – Set to true to create a FIFO topic. + FifoTopic string `json:"FifoTopic,omitempty"` + + // ArchivePolicy – Adds or updates an inline policy document to archive messages stored in the specified + // Amazon SNS topic. + ArchivePolicy string `json:"ArchivePolicy,omitempty"` + + // BeginningArchiveTime – The earliest starting point at which a message in the + // topic’s archive can be replayed from. This point in time is based on the + // configured message retention period set by the topic’s message archiving policy. + BeginningArchiveTime string `json:"BeginningArchiveTime,omitempty"` + + // ContentBasedDeduplication – Enables content-based deduplication for FIFO topics. + // + // By default, ContentBasedDeduplication is set to false . If you create a FIFO + // topic and this attribute is false , you must specify a value for the + // MessageDeduplicationId parameter for the [Publish]action. + // + // When you set ContentBasedDeduplication to true , Amazon SNS uses a SHA-256 + // hash to generate the MessageDeduplicationId using the body of the message (but + // not the attributes of the message). + ContentBasedDeduplication string `json:"ContentBasedDeduplication,omitempty"` + + // CustomAttributes is a map of custom attributes that are not mapped to the struct fields. + CustomAttributes map[string]string `json:"-"` +} + +func (s ConfigAttributes) Attributes() (map[string]string, error) { + b, err := json.Marshal(s) + if err != nil { + return nil, fmt.Errorf("cannot marshal attributes (json.Marshal): %w", err) + } + + var m map[string]string + err = json.Unmarshal(b, &m) + if err != nil { + return nil, fmt.Errorf("cannot marshal attributes (json.Unmarshal): %w", err) + } + + if s.CustomAttributes != nil { + for k, v := range s.CustomAttributes { + m[k] = v + } + } + + return m, nil +} diff --git a/sns/marshaler.go b/sns/marshaler.go index f7b8500..550cd2f 100644 --- a/sns/marshaler.go +++ b/sns/marshaler.go @@ -8,18 +8,20 @@ import ( "github.com/ThreeDotsLabs/watermill/message" ) +// todo: check if it can be renamed const UUIDAttribute = "UUID" type Marshaler interface { - Marshal(msg *message.Message) *sns.PublishInput + Marshal(topicArn string, msg *message.Message) *sns.PublishInput } type DefaultMarshalerUnmarshaler struct{} -func (d DefaultMarshalerUnmarshaler) Marshal(msg *message.Message) *sns.PublishInput { +func (d DefaultMarshalerUnmarshaler) Marshal(topicArn string, msg *message.Message) *sns.PublishInput { // client side uuid // there is a deduplication id that can be use for // fifo queues + // todo: check how it works attributes, deduplicationId, groupId := metadataToAttributes(msg.Metadata) attributes[UUIDAttribute] = types.MessageAttributeValue{ StringValue: aws.String(msg.UUID), @@ -31,6 +33,7 @@ func (d DefaultMarshalerUnmarshaler) Marshal(msg *message.Message) *sns.PublishI MessageAttributes: attributes, MessageDeduplicationId: deduplicationId, MessageGroupId: groupId, + TargetArn: &topicArn, } } diff --git a/sns/pub_test.go b/sns/pub_test.go index fd9131a..c274d79 100644 --- a/sns/pub_test.go +++ b/sns/pub_test.go @@ -4,69 +4,25 @@ import ( "context" "testing" + "github.com/ThreeDotsLabs/watermill-amazonsqs/sqs" awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/stretchr/testify/assert" "github.com/aws/aws-sdk-go-v2/aws" - snsaws "github.com/aws/aws-sdk-go-v2/service/sns" "github.com/stretchr/testify/require" "github.com/ThreeDotsLabs/watermill" "github.com/ThreeDotsLabs/watermill-amazonsqs/connection" "github.com/ThreeDotsLabs/watermill-amazonsqs/sns" - "github.com/ThreeDotsLabs/watermill-amazonsqs/sqs" "github.com/ThreeDotsLabs/watermill/message" "github.com/ThreeDotsLabs/watermill/pubsub/tests" ) -type TestSNS struct { - sns.Publisher - sub *sqs.Subscriber -} - -func (t *TestSNS) Publish(topic string, messages ...*message.Message) error { - ctx := context.Background() - pubArn, err := t.Publisher.GetArnTopic(ctx, topic) - if err != nil { - return err - } - sqsUrl, err := t.sub.GetQueueUrl(ctx, topic) - if err != nil { - return err - } - - err = t.Publisher.AddSubscription(context.Background(), &snsaws.SubscribeInput{ - Protocol: aws.String("sqs"), - TopicArn: pubArn, - Endpoint: sqsUrl, - Attributes: map[string]string{ - "RawMessageDelivery": "true", - }, - }) - if err != nil { - return err - } - err = t.Publisher.Publish(topic, messages...) - return err -} - -func TestCreatePub(t *testing.T) { - logger := watermill.NewStdLogger(true, true) - - cfg, err := GetAWSConfig() - require.NoError(t, err) - - _, err = sns.NewPublisher(sns.PublisherConfig{ - AWSConfig: cfg, - }, logger) - - require.NoError(t, err) -} - func TestPublishSubscribe(t *testing.T) { tests.TestPubSub( t, tests.Features{ - ConsumerGroups: false, + ConsumerGroups: true, ExactlyOnceDelivery: false, GuaranteedOrder: false, Persistent: true, @@ -76,43 +32,107 @@ func TestPublishSubscribe(t *testing.T) { ) } +func TestPublisher_CreateTopic_is_idempotent(t *testing.T) { + pub, _ := createPubSub(t) + + topicName := watermill.NewUUID() + + arn1, err := pub.(*sns.Publisher).CreateTopic(context.Background(), topicName) + require.NoError(t, err) + + arn2, err := pub.(*sns.Publisher).CreateTopic(context.Background(), topicName) + require.NoError(t, err) + + assert.Equal(t, arn1, arn2) +} + +func TestSubscriber_SubscribeInitialize_is_idempotent(t *testing.T) { + _, sub := createPubSub(t) + + topicName := watermill.NewUUID() + + err := sub.(*sns.Subscriber).SubscribeInitialize(topicName) + require.NoError(t, err) + + err = sub.(*sns.Subscriber).SubscribeInitialize(topicName) + require.NoError(t, err) +} + func createPubSub(t *testing.T) (message.Publisher, message.Subscriber) { - logger := watermill.NewStdLogger(false, false) + logger := watermill.NewStdLogger(true, true) cfg, err := GetAWSConfig() require.NoError(t, err) pub, err := sns.NewPublisher(sns.PublisherConfig{ + AwsAccountID: "000000000000", AWSConfig: cfg, - CreateTopicConfig: sns.SNSConfigAtrributes{ + CreateTopicConfig: sns.ConfigAttributes{ // FifoTopic: "true", }, - CreateTopicfNotExists: true, - Marshaler: sns.DefaultMarshalerUnmarshaler{}, + Marshaler: sns.DefaultMarshalerUnmarshaler{}, }, logger) require.NoError(t, err) - sub, err := sqs.NewSubscriber(sqs.SubscriberConfig{ - AWSConfig: cfg, - CreateQueueInitializerConfig: sqs.QueueConfigAttributes{ - // Defalt value is 30 seconds - need to be lower for tests - VisibilityTimeout: "15", + sub, err := sns.NewSubscriber( + sns.SubscriberConfig{ + AWSConfig: cfg, + AwsAccountID: "000000000000", }, - Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, - }, logger) + sqs.SubscriberConfig{ + AWSConfig: cfg, + QueueConfigAttributes: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests + VisibilityTimeout: "1", + }, + }, + logger, + ) require.NoError(t, err) - test := &TestSNS{ - Publisher: *pub, - sub: sub, - } - return test, sub + + return pub, sub } func createPubSubWithConsumerGroup(t *testing.T, consumerGroup string) (message.Publisher, message.Subscriber) { - return createPubSub(t) + logger := watermill.NewStdLogger(true, false) + cfg, err := GetAWSConfig() + require.NoError(t, err) + + pub, err := sns.NewPublisher(sns.PublisherConfig{ + AwsAccountID: "000000000000", + AWSConfig: cfg, + CreateTopicConfig: sns.ConfigAttributes{ + // FifoTopic: "true", + }, + Marshaler: sns.DefaultMarshalerUnmarshaler{}, + }, logger) + require.NoError(t, err) + + sub, err := sns.NewSubscriber( + sns.SubscriberConfig{ + AWSConfig: cfg, + AwsAccountID: "000000000000", + GenerateSqsQueueName: func(ctx context.Context, sqsTopic string) (string, error) { + return consumerGroup, nil + }, + }, + sqs.SubscriberConfig{ + AWSConfig: cfg, + QueueConfigAttributes: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests + VisibilityTimeout: "1", + }, + }, + logger, + ) + require.NoError(t, err) + + return pub, sub } + func GetAWSConfig() (aws.Config, error) { return awsconfig.LoadDefaultConfig( context.Background(), connection.SetEndPoint("http://localhost:4566"), + awsconfig.WithRegion("us-west-2"), ) } diff --git a/sns/publisher.go b/sns/publisher.go index b41e1de..55f5be5 100644 --- a/sns/publisher.go +++ b/sns/publisher.go @@ -4,20 +4,13 @@ import ( "context" "fmt" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/sns" - "github.com/ThreeDotsLabs/watermill" "github.com/ThreeDotsLabs/watermill/message" + "github.com/aws/aws-sdk-go-v2/service/sns" + "github.com/aws/aws-sdk-go-v2/service/sns/types" + "github.com/pkg/errors" ) -type PublisherConfig struct { - AWSConfig aws.Config - CreateTopicConfig SNSConfigAtrributes - CreateTopicfNotExists bool - Marshaler Marshaler -} - type Publisher struct { config PublisherConfig logger watermill.LoggerAdapter @@ -26,6 +19,11 @@ type Publisher struct { func NewPublisher(config PublisherConfig, logger watermill.LoggerAdapter) (*Publisher, error) { config.setDefaults() + + if err := config.Validate(); err != nil { + return nil, err + } + return &Publisher{ sns: sns.NewFromConfig(config.AWSConfig), config: config, @@ -33,59 +31,75 @@ func NewPublisher(config PublisherConfig, logger watermill.LoggerAdapter) (*Publ }, nil } -func (p Publisher) Publish(topic string, messages ...*message.Message) error { +func (p *Publisher) Publish(topic string, messages ...*message.Message) error { ctx := context.Background() - topicArn, err := p.GetArnTopic(ctx, topic) + + topicArn, err := GenerateTopicArn(p.config.AWSConfig.Region, p.config.AwsAccountID, topic) if err != nil { return err } for _, msg := range messages { - p.logger.Debug("Sending message", watermill.LogFields{"msg": msg}) + p.logger.Debug("Sending message", watermill.LogFields{ + "msg": msg, + "topic_arn": topicArn, + }) // Real messageId are generated on server side // so we can set our own here so we can use it in the tests // There is a deduplicationId but just for FIFO queues - input := p.config.Marshaler.Marshal(msg) - input.TopicArn = topicArn - _, err = p.sns.Publish(ctx, input) + input := p.config.Marshaler.Marshal(topicArn, msg) + _, err := p.sns.Publish(ctx, input) + + var topicNotFoundError *types.NotFoundException + if errors.As(err, &topicNotFoundError) && !p.config.DoNotCreateTopicIfNotExists { + // in most cases topic will already exist - as form of optimisation we + // assume that topic exists to avoid unnecessary API calls + // we create topic only if it doesn't exist + if _, err := p.CreateTopic(ctx, topic); err != nil { + return fmt.Errorf("failed to create topic: %w", err) + } + + // now topic should exist + _, err = p.sns.Publish(ctx, input) + } if err != nil { - return err + return fmt.Errorf("cannot publish message to topic %s [%s]: %w", topic, topicArn, err) } } return nil } -func (p Publisher) GetArnTopic(ctx context.Context, topic string) (*string, error) { - topicARN, err := CheckARNTopic(ctx, p.sns, topic) +func (p *Publisher) CreateTopic(ctx context.Context, topic string) (string, error) { + topicArn, err := GenerateTopicArn(p.config.AWSConfig.Region, p.config.AwsAccountID, topic) if err != nil { - if p.config.CreateTopicfNotExists { - topicARN, err = CreateSNS(ctx, p.sns, topic, sns.CreateTopicInput{ - Attributes: p.config.CreateTopicConfig.Attributes(), - }) - if err == nil { - return topicARN, nil - } - } - return nil, err + return "", err } - return topicARN, nil -} -func (p Publisher) Close() error { - return nil -} + input, err := p.config.GenerateCreateTopicInput(ctx, topic, p.config.CreateTopicConfig) + if err != nil { + return "", fmt.Errorf("cannot generate create topic input for %s: %w", topic, err) + } -func (p Publisher) AddSubscription(ctx context.Context, config *sns.SubscribeInput) error { - subcribeOutput, err := p.sns.Subscribe(ctx, config) - if err != nil || subcribeOutput == nil { - return fmt.Errorf("cannot subscribe to SNS[%s] from %s: %w", *config.TopicArn, *config.Endpoint, err) + createdTopicArn, err := createSnsTopic(ctx, p.sns, input) + if err != nil { + return "", fmt.Errorf("failed to create SNS topic: %w", err) + } + if createdTopicArn == nil { + return "", fmt.Errorf("created topic arn is nil") } - return nil -} -func (c *PublisherConfig) setDefaults() { - if c.Marshaler == nil { - c.Marshaler = DefaultMarshalerUnmarshaler{} + if *createdTopicArn != topicArn { + return "", fmt.Errorf( + "created topic arn (%s) is not equal to expected topic arn (%s), please check the configuration", + *createdTopicArn, + topicArn, + ) } + + return *createdTopicArn, nil +} + +func (p *Publisher) Close() error { + return nil } diff --git a/sns/sns.go b/sns/sns.go index dbabf76..07475aa 100644 --- a/sns/sns.go +++ b/sns/sns.go @@ -2,46 +2,34 @@ package sns import ( "context" - "encoding/json" + "errors" "fmt" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/sns" ) -type SNSConfigAtrributes struct { - DeliveryPolicy string `json:"DeliveryPolicy,omitempty"` - DisplayName string `json:"DisplayName,omitempty"` - Policy string `json:"Policy,omitempty"` - SignatureVersion string `json:"SignatureVersion,omitempty"` - TracingConfig string `json:"TracingConfig,omitempty"` - KmsMasterKeyId string `json:"KmsMasterKeyId,omitempty"` - FifoTopic string `json:"FifoTopic,omitempty"` - ContentBasedDeduplication string `json:"ContentBasedDeduplication,omitempty"` -} - -func (s SNSConfigAtrributes) Attributes() map[string]string { - b, _ := json.Marshal(s) - var m map[string]string - _ = json.Unmarshal(b, &m) - return m -} - -func CreateSNS(ctx context.Context, snsClient *sns.Client, topicARN string, createSNSParams sns.CreateTopicInput) (*string, error) { - createSNSParams.Name = aws.String(topicARN) +func createSnsTopic(ctx context.Context, snsClient *sns.Client, createSNSParams sns.CreateTopicInput) (*string, error) { createSNSOutput, err := snsClient.CreateTopic(ctx, &createSNSParams) if err != nil || createSNSOutput.TopicArn == nil { - return nil, fmt.Errorf("cannot create SNS %s: %w", topicARN, err) + return nil, fmt.Errorf("cannot create SNS topic %s: %w", *createSNSParams.Name, err) } return createSNSOutput.TopicArn, nil } -func CheckARNTopic(ctx context.Context, snsClient *sns.Client, topicARN string) (*string, error) { - createSNSOutput, err := snsClient.GetTopicAttributes(ctx, &sns.GetTopicAttributesInput{ - TopicArn: aws.String(topicARN), - }) - if err != nil || createSNSOutput == nil { - return nil, fmt.Errorf("cannot create SNS %s: %w", topicARN, err) +func GenerateTopicArn(region, accountID, topic string) (string, error) { + var err error + if region == "" { + err = errors.Join(err, fmt.Errorf("region is empty")) } - return &topicARN, nil + if accountID == "" { + err = errors.Join(err, fmt.Errorf("accountID is empty")) + } + if topic == "" { + err = errors.Join(err, fmt.Errorf("topic is empty")) + } + if err != nil { + return "", fmt.Errorf("can't generate topic arn: %w", err) + } + + return fmt.Sprintf("arn:aws:sns:%s:%s:%s", region, accountID, topic), nil } diff --git a/sns/subscriber.go b/sns/subscriber.go new file mode 100644 index 0000000..e71813e --- /dev/null +++ b/sns/subscriber.go @@ -0,0 +1,120 @@ +package sns + +import ( + "context" + "fmt" + + "github.com/ThreeDotsLabs/watermill" + "github.com/ThreeDotsLabs/watermill-amazonsqs/sqs" + "github.com/ThreeDotsLabs/watermill/message" + "github.com/aws/aws-sdk-go-v2/service/sns" +) + +type Subscriber struct { + config SubscriberConfig + logger watermill.LoggerAdapter + + sns *sns.Client + sqs *sqs.Subscriber +} + +func NewSubscriber( + config SubscriberConfig, + sqsConfig sqs.SubscriberConfig, + logger watermill.LoggerAdapter, +) (*Subscriber, error) { + config.SetDefaults() + if err := config.Validate(); err != nil { + return nil, err + } + + if logger == nil { + logger = watermill.NopLogger{} + } + + logger = logger.With(watermill.LogFields{ + "subscriber_uuid": watermill.NewShortUUID(), + }) + + sqs, err := sqs.NewSubscriber(sqsConfig, logger) + if err != nil { + return nil, fmt.Errorf("cannot create SQS subscriber: %w", err) + } + + return &Subscriber{ + config: config, + logger: logger, + sns: sns.NewFromConfig(config.AWSConfig), + sqs: sqs, + }, nil +} + +func (s *Subscriber) Subscribe(ctx context.Context, topic string) (<-chan *message.Message, error) { + if !s.config.DoNotSubscribeToSns { + if err := s.SubscribeInitializeWithContext(ctx, topic); err != nil { + return nil, err + } + } + + sqsTopic, err := s.config.GenerateSqsQueueName(ctx, topic) + if err != nil { + return nil, fmt.Errorf("failed to generate SQS queue name: %w", err) + } + + return s.sqs.Subscribe(ctx, sqsTopic) +} + +func (s *Subscriber) SubscribeInitialize(topic string) error { + return s.SubscribeInitializeWithContext(context.Background(), topic) +} + +func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, snsTopic string) error { + snsTopicArn, err := GenerateTopicArn(s.config.AWSConfig.Region, s.config.AwsAccountID, snsTopic) + if err != nil { + return err + } + + sqsTopic, err := s.config.GenerateSqsQueueName(ctx, snsTopic) + if err != nil { + return fmt.Errorf("failed to generate SQS queue name: %w", err) + } + + if err := s.sqs.SubscribeInitializeWithContext(ctx, sqsTopic); err != nil { + return fmt.Errorf("cannot initialize SQS subscription for topic %s: %w", snsTopic, err) + } + + sqsURL, err := s.sqs.GetQueueUrl(ctx, sqsTopic) + if err != nil { + return fmt.Errorf("cannot get queue url for topic %s: %w", snsTopic, err) + } + sqsQueueArn, err := s.sqs.GetQueueArn(ctx, sqsURL) + if err != nil { + return fmt.Errorf("cannot get queue ARN for topic %s: %w", snsTopic, err) + } + + s.logger.Info("Subscribing to SNS", watermill.LogFields{ + "sns_topic": snsTopic, + "sns_topic_arn": snsTopicArn, + "sqs_topic": sqsTopic, + "sqs_url": *sqsURL, + "sqs_arn": *sqsQueueArn, + }) + + input, err := s.config.GenerateSubscribeInput(ctx, GenerateSubscribeInputParams{ + SnsTopic: snsTopic, + SqsTopic: sqsTopic, + SnsTopicArn: snsTopicArn, + SqsQueueArn: *sqsQueueArn, + }) + + subscribeOutput, err := s.sns.Subscribe(ctx, input) + if err != nil || subscribeOutput == nil { + return fmt.Errorf("cannot subscribe to SNS[%s] from %s: %w", snsTopicArn, *sqsQueueArn, err) + } + + return nil +} + +func (s *Subscriber) Close() error { + return s.sqs.Close() +} diff --git a/sqs/config.go b/sqs/config.go index 6de4b7b..d61d452 100644 --- a/sqs/config.go +++ b/sqs/config.go @@ -18,8 +18,8 @@ type SubscriberConfig struct { // ReconnectRetrySleep is the time to sleep between reconnect attempts. ReconnectRetrySleep time.Duration - // CreateQueueInitializerConfig is a struct that holds the attributes of an SQS queue. - CreateQueueInitializerConfig QueueConfigAttributes + // QueueConfigAttributes is a struct that holds the attributes of an SQS queue. + QueueConfigAttributes QueueConfigAttributes // GenerateCreateQueueInput generates *sqs.CreateQueueInput for AWS SDK. GenerateCreateQueueInput GenerateCreateQueueInputFunc @@ -211,7 +211,7 @@ type QueueConfigAttributes struct { // VisibilityTimeout – The visibility timeout for the queue, in seconds. // Valid values: An integer from 0 to 43,200 (12 hours). Default: 30. - + // // For more information about the visibility timeout, see // [Visibility Timeout](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html) in the Amazon SQS Developer Guide. VisibilityTimeout string `json:"VisibilityTimeout,omitempty"` diff --git a/sqs/pubsub_test.go b/sqs/pubsub_test.go index 41bcbd8..e9f71be 100644 --- a/sqs/pubsub_test.go +++ b/sqs/pubsub_test.go @@ -24,7 +24,7 @@ func TestPubSub(t *testing.T) { tests.Features{ ConsumerGroups: false, ExactlyOnceDelivery: false, - GuaranteedOrder: false, + GuaranteedOrder: false, // todo? Persistent: true, }, createPubSub, @@ -40,7 +40,7 @@ func TestPubSub_stress(t *testing.T) { tests.Features{ ConsumerGroups: false, ExactlyOnceDelivery: false, - GuaranteedOrder: false, + GuaranteedOrder: false, // todo? Persistent: true, }, createPubSub, @@ -69,7 +69,7 @@ func TestPublishSubscribe_batching(t *testing.T) { pub, err := sqs.NewPublisher(sqs.PublisherConfig{ AWSConfig: cfg, CreateQueueConfig: sqs.QueueConfigAttributes{ - // Defalt value is 30 seconds - need to be lower for tests + // Default value is 30 seconds - need to be lower for tests VisibilityTimeout: "1", }, CreateQueueIfNotExists: true, @@ -79,8 +79,8 @@ func TestPublishSubscribe_batching(t *testing.T) { sub, err := sqs.NewSubscriber(sqs.SubscriberConfig{ AWSConfig: cfg, - CreateQueueInitializerConfig: sqs.QueueConfigAttributes{ - // Defalt value is 30 seconds - need to be lower for tests + QueueConfigAttributes: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests VisibilityTimeout: "1", }, GenerateReceiveMessageInput: func(ctx context.Context, queueURL string) (*awssqs.ReceiveMessageInput, error) { @@ -108,8 +108,8 @@ func TestPublishSubscribe_creating_queue_with_different_settings_should_be_idemp sub1, err := sqs.NewSubscriber(sqs.SubscriberConfig{ AWSConfig: newAwsConfig(t), - CreateQueueInitializerConfig: sqs.QueueConfigAttributes{ - // Defalt value is 30 seconds - need to be lower for tests + QueueConfigAttributes: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests VisibilityTimeout: "1", }, Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, @@ -118,8 +118,8 @@ func TestPublishSubscribe_creating_queue_with_different_settings_should_be_idemp sub2, err := sqs.NewSubscriber(sqs.SubscriberConfig{ AWSConfig: newAwsConfig(t), - CreateQueueInitializerConfig: sqs.QueueConfigAttributes{ - // Defalt value is 30 seconds - need to be lower for tests + QueueConfigAttributes: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests VisibilityTimeout: "20", }, Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, @@ -162,7 +162,7 @@ func createPubSub(t *testing.T) (message.Publisher, message.Subscriber) { pub, err := sqs.NewPublisher(sqs.PublisherConfig{ AWSConfig: cfg, CreateQueueConfig: sqs.QueueConfigAttributes{ - // Defalt value is 30 seconds - need to be lower for tests + // Default value is 30 seconds - need to be lower for tests VisibilityTimeout: "1", }, CreateQueueIfNotExists: true, @@ -172,8 +172,8 @@ func createPubSub(t *testing.T) (message.Publisher, message.Subscriber) { sub, err := sqs.NewSubscriber(sqs.SubscriberConfig{ AWSConfig: cfg, - CreateQueueInitializerConfig: sqs.QueueConfigAttributes{ - // Defalt value is 30 seconds - need to be lower for tests + QueueConfigAttributes: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests VisibilityTimeout: "1", }, Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, diff --git a/sqs/subscriber.go b/sqs/subscriber.go index fa38339..917687d 100644 --- a/sqs/subscriber.go +++ b/sqs/subscriber.go @@ -140,11 +140,11 @@ func (s *Subscriber) receive(ctx context.Context, queueURL string, output chan * s.logger.Trace("No messages", logFields) continue } - s.ConsumeMessages(ctx, result.Messages, queueURL, output, logFields) + s.consumeMessages(ctx, result.Messages, queueURL, output, logFields) } } -func (s *Subscriber) ConsumeMessages( +func (s *Subscriber) consumeMessages( ctx context.Context, messages []types.Message, queueURL string, @@ -160,18 +160,30 @@ func (s *Subscriber) ConsumeMessages( } } -func (s *Subscriber) processMessage(ctx context.Context, logFields watermill.LogFields, sqsMsg types.Message, output chan *message.Message, queueURL string) bool { - s.logger.Trace("processMessage", logFields) +func (s *Subscriber) processMessage( + ctx context.Context, + logFields watermill.LogFields, + sqsMsg types.Message, + output chan *message.Message, + queueURL string, +) bool { + logger := s.logger.With(logFields) + logger.Trace("processMessage", nil) ctx, cancelCtx := context.WithCancel(ctx) defer cancelCtx() msg, err := s.config.Unmarshaler.Unmarshal(&sqsMsg) if err != nil { - s.logger.Error("Cannot unmarshal message", err, logFields) + logger.Error("Cannot unmarshal message", err, logFields) return false } msg.SetContext(ctx) + + logger = s.logger.With(logFields).With(watermill.LogFields{ + "message_uuid": msg.UUID, + }) + output <- msg select { @@ -181,17 +193,18 @@ func (s *Subscriber) processMessage(ctx context.Context, logFields watermill.Log return false } if err != nil { - s.logger.Error("Failed to delete message", err, logFields) + logger.Error("Failed to delete message", err, logFields) return false } case <-msg.Nacked(): // Do not delete message, it will be redelivered - return false // we don't want to process next messages to preserve order + logger.Debug("Nacking message", logFields) + return false // we don't want to process next messages to preserve order for FIFO case <-s.closing: - s.logger.Debug("Closing, message discarded before ack", logFields) + logger.Debug("Closing, message discarded before ack", logFields) return false case <-ctx.Done(): - s.logger.Debug("Closing, ctx cancelled before ack", logFields) + logger.Debug("Closing, ctx cancelled before ack", logFields) return false } @@ -246,7 +259,7 @@ func (s *Subscriber) SubscribeInitialize(topic string) error { } func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, topic string) error { - input, err := s.config.GenerateCreateQueueInput(ctx, topic, s.config.CreateQueueInitializerConfig) + input, err := s.config.GenerateCreateQueueInput(ctx, topic, s.config.QueueConfigAttributes) if err != nil { return fmt.Errorf("cannot generate input for queue %s: %w", topic, err) } From e80774020757d0884a43deb9ddb820e95d398bc9 Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Sat, 24 Aug 2024 17:51:49 +0200 Subject: [PATCH 10/19] use topic ARN as topic name for sns --- sns/config.go | 33 +++++++++++++++------------------ sns/pub_test.go | 17 +++++++++-------- sns/publisher.go | 19 +++++++------------ sns/sns.go | 11 +++++++++++ sns/subscriber.go | 23 ++++++++--------------- 5 files changed, 50 insertions(+), 53 deletions(-) diff --git a/sns/config.go b/sns/config.go index 67e2cd3..d859008 100644 --- a/sns/config.go +++ b/sns/config.go @@ -11,9 +11,7 @@ import ( ) type PublisherConfig struct { - // todo: add func for that? - AwsAccountID string - AWSConfig aws.Config + AWSConfig aws.Config CreateTopicConfig ConfigAttributes DoNotCreateTopicIfNotExists bool @@ -49,9 +47,6 @@ func (c *PublisherConfig) setDefaults() { func (c *PublisherConfig) Validate() error { var err error - if c.AwsAccountID == "" { - err = errors.Join(err, fmt.Errorf("sns.PublisherConfig.AwsAccountID is missing")) - } if c.AWSConfig.Credentials == nil { err = errors.Join(err, fmt.Errorf("sns.PublisherConfig.AWSConfig.Credentials is nil")) } @@ -60,10 +55,9 @@ func (c *PublisherConfig) Validate() error { } type SubscriberConfig struct { - AwsAccountID string - AWSConfig aws.Config + AWSConfig aws.Config - GenerateSqsQueueName func(ctx context.Context, sqsTopic string) (string, error) + GenerateSqsQueueName func(ctx context.Context, sqsTopicArn string) (string, error) GenerateSubscribeInput GenerateSubscribeInputFn @@ -71,25 +65,29 @@ type SubscriberConfig struct { } func (c *SubscriberConfig) SetDefaults() { - if c.GenerateSqsQueueName == nil { - c.GenerateSqsQueueName = func(ctx context.Context, sqsTopic string) (string, error) { - return sqsTopic, nil - } - } if c.GenerateSubscribeInput == nil { c.GenerateSubscribeInput = GenerateSubscribeInputDefault } } +func GenerateSqsQueueNameEqualToTopicName(ctx context.Context, sqsTopicArn string) (string, error) { + topicName, err := TopicNameFromTopicArn(sqsTopicArn) + if err != nil { + return "", err + } + + return topicName, nil +} + func (c *SubscriberConfig) Validate() error { var err error - if c.AwsAccountID == "" { - err = errors.Join(err, fmt.Errorf("sns.SubscriberConfig.AwsAccountID is missing")) - } if c.AWSConfig.Credentials == nil { err = errors.Join(err, fmt.Errorf("sns.SubscriberConfig.AWSConfig.Credentials is nil")) } + if c.GenerateSqsQueueName == nil { + err = errors.Join(err, fmt.Errorf("sns.SubscriberConfig.GenerateSqsQueueName is nil")) + } return err } @@ -97,7 +95,6 @@ func (c *SubscriberConfig) Validate() error { type GenerateSubscribeInputFn func(ctx context.Context, params GenerateSubscribeInputParams) (*sns.SubscribeInput, error) type GenerateSubscribeInputParams struct { - SnsTopic string SqsTopic string SnsTopicArn string diff --git a/sns/pub_test.go b/sns/pub_test.go index c274d79..0c3d577 100644 --- a/sns/pub_test.go +++ b/sns/pub_test.go @@ -2,6 +2,7 @@ package sns_test import ( "context" + "fmt" "testing" "github.com/ThreeDotsLabs/watermill-amazonsqs/sqs" @@ -26,6 +27,9 @@ func TestPublishSubscribe(t *testing.T) { ExactlyOnceDelivery: false, GuaranteedOrder: false, Persistent: true, + GenerateTopicFunc: func(tctx tests.TestContext) string { + return fmt.Sprintf("arn:aws:sns:us-west-2:000000000000:%s", tctx.TestID) + }, }, createPubSub, createPubSubWithConsumerGroup, @@ -35,7 +39,7 @@ func TestPublishSubscribe(t *testing.T) { func TestPublisher_CreateTopic_is_idempotent(t *testing.T) { pub, _ := createPubSub(t) - topicName := watermill.NewUUID() + topicName := "arn:aws:sns:us-west-2:000000000000:" + watermill.NewUUID() arn1, err := pub.(*sns.Publisher).CreateTopic(context.Background(), topicName) require.NoError(t, err) @@ -49,7 +53,7 @@ func TestPublisher_CreateTopic_is_idempotent(t *testing.T) { func TestSubscriber_SubscribeInitialize_is_idempotent(t *testing.T) { _, sub := createPubSub(t) - topicName := watermill.NewUUID() + topicName := "arn:aws:sns:us-west-2:000000000000:" + watermill.NewUUID() err := sub.(*sns.Subscriber).SubscribeInitialize(topicName) require.NoError(t, err) @@ -64,7 +68,6 @@ func createPubSub(t *testing.T) (message.Publisher, message.Subscriber) { require.NoError(t, err) pub, err := sns.NewPublisher(sns.PublisherConfig{ - AwsAccountID: "000000000000", AWSConfig: cfg, CreateTopicConfig: sns.ConfigAttributes{ // FifoTopic: "true", @@ -75,8 +78,8 @@ func createPubSub(t *testing.T) (message.Publisher, message.Subscriber) { sub, err := sns.NewSubscriber( sns.SubscriberConfig{ - AWSConfig: cfg, - AwsAccountID: "000000000000", + AWSConfig: cfg, + GenerateSqsQueueName: sns.GenerateSqsQueueNameEqualToTopicName, }, sqs.SubscriberConfig{ AWSConfig: cfg, @@ -98,7 +101,6 @@ func createPubSubWithConsumerGroup(t *testing.T, consumerGroup string) (message. require.NoError(t, err) pub, err := sns.NewPublisher(sns.PublisherConfig{ - AwsAccountID: "000000000000", AWSConfig: cfg, CreateTopicConfig: sns.ConfigAttributes{ // FifoTopic: "true", @@ -109,8 +111,7 @@ func createPubSubWithConsumerGroup(t *testing.T, consumerGroup string) (message. sub, err := sns.NewSubscriber( sns.SubscriberConfig{ - AWSConfig: cfg, - AwsAccountID: "000000000000", + AWSConfig: cfg, GenerateSqsQueueName: func(ctx context.Context, sqsTopic string) (string, error) { return consumerGroup, nil }, diff --git a/sns/publisher.go b/sns/publisher.go index 55f5be5..295e06c 100644 --- a/sns/publisher.go +++ b/sns/publisher.go @@ -31,14 +31,9 @@ func NewPublisher(config PublisherConfig, logger watermill.LoggerAdapter) (*Publ }, nil } -func (p *Publisher) Publish(topic string, messages ...*message.Message) error { +func (p *Publisher) Publish(topicArn string, messages ...*message.Message) error { ctx := context.Background() - topicArn, err := GenerateTopicArn(p.config.AWSConfig.Region, p.config.AwsAccountID, topic) - if err != nil { - return err - } - for _, msg := range messages { p.logger.Debug("Sending message", watermill.LogFields{ "msg": msg, @@ -55,7 +50,7 @@ func (p *Publisher) Publish(topic string, messages ...*message.Message) error { // in most cases topic will already exist - as form of optimisation we // assume that topic exists to avoid unnecessary API calls // we create topic only if it doesn't exist - if _, err := p.CreateTopic(ctx, topic); err != nil { + if _, err := p.CreateTopic(ctx, topicArn); err != nil { return fmt.Errorf("failed to create topic: %w", err) } @@ -63,22 +58,22 @@ func (p *Publisher) Publish(topic string, messages ...*message.Message) error { _, err = p.sns.Publish(ctx, input) } if err != nil { - return fmt.Errorf("cannot publish message to topic %s [%s]: %w", topic, topicArn, err) + return fmt.Errorf("cannot publish message to topic %s: %w", topicArn, err) } } return nil } -func (p *Publisher) CreateTopic(ctx context.Context, topic string) (string, error) { - topicArn, err := GenerateTopicArn(p.config.AWSConfig.Region, p.config.AwsAccountID, topic) +func (p *Publisher) CreateTopic(ctx context.Context, topicArn string) (string, error) { + topicName, err := TopicNameFromTopicArn(topicArn) if err != nil { return "", err } - input, err := p.config.GenerateCreateTopicInput(ctx, topic, p.config.CreateTopicConfig) + input, err := p.config.GenerateCreateTopicInput(ctx, topicName, p.config.CreateTopicConfig) if err != nil { - return "", fmt.Errorf("cannot generate create topic input for %s: %w", topic, err) + return "", fmt.Errorf("cannot generate create topic input for %s: %w", topicName, err) } createdTopicArn, err := createSnsTopic(ctx, p.sns, input) diff --git a/sns/sns.go b/sns/sns.go index 07475aa..d8bc961 100644 --- a/sns/sns.go +++ b/sns/sns.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "github.com/aws/aws-sdk-go-v2/service/sns" ) @@ -33,3 +34,13 @@ func GenerateTopicArn(region, accountID, topic string) (string, error) { return fmt.Sprintf("arn:aws:sns:%s:%s:%s", region, accountID, topic), nil } + +func TopicNameFromTopicArn(topicArn string) (string, error) { + topicArnParts := strings.Split(topicArn, ":") + if len(topicArnParts) != 6 { + return "", fmt.Errorf("topic arn should have 6 segments, has %d (%s)", len(topicArnParts), topicArn) + } + + topicName := topicArnParts[5] + return topicName, nil +} diff --git a/sns/subscriber.go b/sns/subscriber.go index e71813e..0403613 100644 --- a/sns/subscriber.go +++ b/sns/subscriber.go @@ -49,14 +49,14 @@ func NewSubscriber( }, nil } -func (s *Subscriber) Subscribe(ctx context.Context, topic string) (<-chan *message.Message, error) { +func (s *Subscriber) Subscribe(ctx context.Context, snsTopicArn string) (<-chan *message.Message, error) { if !s.config.DoNotSubscribeToSns { - if err := s.SubscribeInitializeWithContext(ctx, topic); err != nil { + if err := s.SubscribeInitializeWithContext(ctx, snsTopicArn); err != nil { return nil, err } } - sqsTopic, err := s.config.GenerateSqsQueueName(ctx, topic) + sqsTopic, err := s.config.GenerateSqsQueueName(ctx, snsTopicArn) if err != nil { return nil, fmt.Errorf("failed to generate SQS queue name: %w", err) } @@ -68,32 +68,26 @@ func (s *Subscriber) SubscribeInitialize(topic string) error { return s.SubscribeInitializeWithContext(context.Background(), topic) } -func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, snsTopic string) error { - snsTopicArn, err := GenerateTopicArn(s.config.AWSConfig.Region, s.config.AwsAccountID, snsTopic) - if err != nil { - return err - } - - sqsTopic, err := s.config.GenerateSqsQueueName(ctx, snsTopic) +func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, snsTopicArn string) error { + sqsTopic, err := s.config.GenerateSqsQueueName(ctx, snsTopicArn) if err != nil { return fmt.Errorf("failed to generate SQS queue name: %w", err) } if err := s.sqs.SubscribeInitializeWithContext(ctx, sqsTopic); err != nil { - return fmt.Errorf("cannot initialize SQS subscription for topic %s: %w", snsTopic, err) + return fmt.Errorf("cannot initialize SQS subscription for topic %s: %w", snsTopicArn, err) } sqsURL, err := s.sqs.GetQueueUrl(ctx, sqsTopic) if err != nil { - return fmt.Errorf("cannot get queue url for topic %s: %w", snsTopic, err) + return fmt.Errorf("cannot get queue url for topic %s: %w", snsTopicArn, err) } sqsQueueArn, err := s.sqs.GetQueueArn(ctx, sqsURL) if err != nil { - return fmt.Errorf("cannot get queue ARN for topic %s: %w", snsTopic, err) + return fmt.Errorf("cannot get queue ARN for topic %s: %w", snsTopicArn, err) } s.logger.Info("Subscribing to SNS", watermill.LogFields{ - "sns_topic": snsTopic, "sns_topic_arn": snsTopicArn, "sqs_topic": sqsTopic, "sqs_url": *sqsURL, @@ -101,7 +95,6 @@ func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, snsTopi }) input, err := s.config.GenerateSubscribeInput(ctx, GenerateSubscribeInputParams{ - SnsTopic: snsTopic, SqsTopic: sqsTopic, SnsTopicArn: snsTopicArn, SqsQueueArn: *sqsQueueArn, From bd3a73feeeb2b2bdf8151af27a819b7b999e9243 Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Sat, 24 Aug 2024 18:02:48 +0200 Subject: [PATCH 11/19] rename --- sns/{pub_test.go => pubsub_test.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sns/{pub_test.go => pubsub_test.go} (100%) diff --git a/sns/pub_test.go b/sns/pubsub_test.go similarity index 100% rename from sns/pub_test.go rename to sns/pubsub_test.go From 6713aa604e2d8beee19e3b45bb958b72e25da5df Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Sun, 25 Aug 2024 10:52:04 +0200 Subject: [PATCH 12/19] added topic resolver --- sns/config.go | 57 ++++++++++++--------- sns/publisher.go | 16 ++++-- sns/pubsub_test.go | 123 ++++++++++++++++++++++++++++++++------------- sns/subscriber.go | 16 ++++-- sns/topic.go | 44 ++++++++++++++++ 5 files changed, 193 insertions(+), 63 deletions(-) create mode 100644 sns/topic.go diff --git a/sns/config.go b/sns/config.go index d859008..03ac784 100644 --- a/sns/config.go +++ b/sns/config.go @@ -16,29 +16,18 @@ type PublisherConfig struct { CreateTopicConfig ConfigAttributes DoNotCreateTopicIfNotExists bool + TopicResolver TopicResolver + GenerateCreateTopicInput GenerateCreateTopicInputFunc Marshaler Marshaler } -type GenerateCreateTopicInputFunc func(ctx context.Context, topic string, attrs ConfigAttributes) (sns.CreateTopicInput, error) - -func GenerateCreateTopicInputDefault(ctx context.Context, topic string, attrs ConfigAttributes) (sns.CreateTopicInput, error) { - attrsMap, err := attrs.Attributes() - if err != nil { - return sns.CreateTopicInput{}, fmt.Errorf("cannot generate attributes for topic %s: %w", topic, err) - } - - return sns.CreateTopicInput{ - Name: aws.String(topic), - Attributes: attrsMap, - }, nil -} - func (c *PublisherConfig) setDefaults() { if c.Marshaler == nil { c.Marshaler = DefaultMarshalerUnmarshaler{} } + if c.GenerateCreateTopicInput == nil { c.GenerateCreateTopicInput = GenerateCreateTopicInputDefault } @@ -50,15 +39,34 @@ func (c *PublisherConfig) Validate() error { if c.AWSConfig.Credentials == nil { err = errors.Join(err, fmt.Errorf("sns.PublisherConfig.AWSConfig.Credentials is nil")) } + if c.TopicResolver == nil { + err = errors.Join(err, fmt.Errorf("sns.PublisherConfig.TopicResolver is nil")) + } return err } +type GenerateCreateTopicInputFunc func(ctx context.Context, topic string, attrs ConfigAttributes) (sns.CreateTopicInput, error) + +func GenerateCreateTopicInputDefault(ctx context.Context, topic string, attrs ConfigAttributes) (sns.CreateTopicInput, error) { + attrsMap, err := attrs.Attributes() + if err != nil { + return sns.CreateTopicInput{}, fmt.Errorf("cannot generate attributes for topic %s: %w", topic, err) + } + + return sns.CreateTopicInput{ + Name: aws.String(topic), + Attributes: attrsMap, + }, nil +} + type SubscriberConfig struct { AWSConfig aws.Config GenerateSqsQueueName func(ctx context.Context, sqsTopicArn string) (string, error) + TopicResolver TopicResolver + GenerateSubscribeInput GenerateSubscribeInputFn DoNotSubscribeToSns bool @@ -70,15 +78,6 @@ func (c *SubscriberConfig) SetDefaults() { } } -func GenerateSqsQueueNameEqualToTopicName(ctx context.Context, sqsTopicArn string) (string, error) { - topicName, err := TopicNameFromTopicArn(sqsTopicArn) - if err != nil { - return "", err - } - - return topicName, nil -} - func (c *SubscriberConfig) Validate() error { var err error @@ -88,10 +87,22 @@ func (c *SubscriberConfig) Validate() error { if c.GenerateSqsQueueName == nil { err = errors.Join(err, fmt.Errorf("sns.SubscriberConfig.GenerateSqsQueueName is nil")) } + if c.TopicResolver == nil { + err = errors.Join(err, fmt.Errorf("sns.SubscriberConfig.TopicResolver is nil")) + } return err } +func GenerateSqsQueueNameEqualToTopicName(ctx context.Context, sqsTopicArn string) (string, error) { + topicName, err := TopicNameFromTopicArn(sqsTopicArn) + if err != nil { + return "", err + } + + return topicName, nil +} + type GenerateSubscribeInputFn func(ctx context.Context, params GenerateSubscribeInputParams) (*sns.SubscribeInput, error) type GenerateSubscribeInputParams struct { diff --git a/sns/publisher.go b/sns/publisher.go index 295e06c..9ae2da0 100644 --- a/sns/publisher.go +++ b/sns/publisher.go @@ -31,9 +31,14 @@ func NewPublisher(config PublisherConfig, logger watermill.LoggerAdapter) (*Publ }, nil } -func (p *Publisher) Publish(topicArn string, messages ...*message.Message) error { +func (p *Publisher) Publish(topic string, messages ...*message.Message) error { ctx := context.Background() + topicArn, err := p.config.TopicResolver.ResolveTopic(ctx, topic) + if err != nil { + return err + } + for _, msg := range messages { p.logger.Debug("Sending message", watermill.LogFields{ "msg": msg, @@ -50,7 +55,7 @@ func (p *Publisher) Publish(topicArn string, messages ...*message.Message) error // in most cases topic will already exist - as form of optimisation we // assume that topic exists to avoid unnecessary API calls // we create topic only if it doesn't exist - if _, err := p.CreateTopic(ctx, topicArn); err != nil { + if _, err := p.CreateTopic(ctx, topic); err != nil { return fmt.Errorf("failed to create topic: %w", err) } @@ -65,7 +70,12 @@ func (p *Publisher) Publish(topicArn string, messages ...*message.Message) error return nil } -func (p *Publisher) CreateTopic(ctx context.Context, topicArn string) (string, error) { +func (p *Publisher) CreateTopic(ctx context.Context, topic string) (string, error) { + topicArn, err := p.config.TopicResolver.ResolveTopic(ctx, topic) + if err != nil { + return "", err + } + topicName, err := TopicNameFromTopicArn(topicArn) if err != nil { return "", err diff --git a/sns/pubsub_test.go b/sns/pubsub_test.go index 0c3d577..26b2139 100644 --- a/sns/pubsub_test.go +++ b/sns/pubsub_test.go @@ -27,19 +27,60 @@ func TestPublishSubscribe(t *testing.T) { ExactlyOnceDelivery: false, GuaranteedOrder: false, Persistent: true, - GenerateTopicFunc: func(tctx tests.TestContext) string { - return fmt.Sprintf("arn:aws:sns:us-west-2:000000000000:%s", tctx.TestID) - }, }, createPubSub, createPubSubWithConsumerGroup, ) } +func TestPubSub_arn_topic_resolver(t *testing.T) { + tests.TestPublishSubscribe( + t, + tests.TestContext{ + TestID: tests.NewTestID(), + Features: tests.Features{ + ConsumerGroups: true, + ExactlyOnceDelivery: false, + GuaranteedOrder: true, + GuaranteedOrderWithSingleSubscriber: true, + Persistent: true, + GenerateTopicFunc: func(tctx tests.TestContext) string { + return fmt.Sprintf("arn:aws:sns:us-west-2:000000000000:%s", tctx.TestID) + }, + }, + }, + func(t *testing.T) (message.Publisher, message.Subscriber) { + cfg := GetAWSConfig(t) + + return createPubSubWithConfig( + t, + sns.PublisherConfig{ + AWSConfig: cfg, + CreateTopicConfig: sns.ConfigAttributes{}, + Marshaler: sns.DefaultMarshalerUnmarshaler{}, + TopicResolver: sns.TransparentTopicResolver{}, + }, + sns.SubscriberConfig{ + AWSConfig: cfg, + GenerateSqsQueueName: sns.GenerateSqsQueueNameEqualToTopicName, + TopicResolver: sns.TransparentTopicResolver{}, + }, + sqs.SubscriberConfig{ + AWSConfig: cfg, + QueueConfigAttributes: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests + VisibilityTimeout: "1", + }, + }, + ) + }, + ) +} + func TestPublisher_CreateTopic_is_idempotent(t *testing.T) { pub, _ := createPubSub(t) - topicName := "arn:aws:sns:us-west-2:000000000000:" + watermill.NewUUID() + topicName := watermill.NewUUID() arn1, err := pub.(*sns.Publisher).CreateTopic(context.Background(), topicName) require.NoError(t, err) @@ -53,7 +94,7 @@ func TestPublisher_CreateTopic_is_idempotent(t *testing.T) { func TestSubscriber_SubscribeInitialize_is_idempotent(t *testing.T) { _, sub := createPubSub(t) - topicName := "arn:aws:sns:us-west-2:000000000000:" + watermill.NewUUID() + topicName := watermill.NewUUID() err := sub.(*sns.Subscriber).SubscribeInitialize(topicName) require.NoError(t, err) @@ -63,22 +104,22 @@ func TestSubscriber_SubscribeInitialize_is_idempotent(t *testing.T) { } func createPubSub(t *testing.T) (message.Publisher, message.Subscriber) { - logger := watermill.NewStdLogger(true, true) - cfg, err := GetAWSConfig() - require.NoError(t, err) + cfg := GetAWSConfig(t) - pub, err := sns.NewPublisher(sns.PublisherConfig{ - AWSConfig: cfg, - CreateTopicConfig: sns.ConfigAttributes{ - // FifoTopic: "true", - }, - Marshaler: sns.DefaultMarshalerUnmarshaler{}, - }, logger) + topicResolver, err := sns.NewGenerateArnTopicResolver("000000000000", "us-west-2") require.NoError(t, err) - sub, err := sns.NewSubscriber( + return createPubSubWithConfig( + t, + sns.PublisherConfig{ + AWSConfig: cfg, + CreateTopicConfig: sns.ConfigAttributes{}, + TopicResolver: topicResolver, + Marshaler: sns.DefaultMarshalerUnmarshaler{}, + }, sns.SubscriberConfig{ AWSConfig: cfg, + TopicResolver: topicResolver, GenerateSqsQueueName: sns.GenerateSqsQueueNameEqualToTopicName, }, sqs.SubscriberConfig{ @@ -88,33 +129,29 @@ func createPubSub(t *testing.T) (message.Publisher, message.Subscriber) { VisibilityTimeout: "1", }, }, - logger, ) - require.NoError(t, err) - - return pub, sub } func createPubSubWithConsumerGroup(t *testing.T, consumerGroup string) (message.Publisher, message.Subscriber) { - logger := watermill.NewStdLogger(true, false) - cfg, err := GetAWSConfig() - require.NoError(t, err) + cfg := GetAWSConfig(t) - pub, err := sns.NewPublisher(sns.PublisherConfig{ - AWSConfig: cfg, - CreateTopicConfig: sns.ConfigAttributes{ - // FifoTopic: "true", - }, - Marshaler: sns.DefaultMarshalerUnmarshaler{}, - }, logger) + topicResolver, err := sns.NewGenerateArnTopicResolver("000000000000", "us-west-2") require.NoError(t, err) - sub, err := sns.NewSubscriber( + return createPubSubWithConfig( + t, + sns.PublisherConfig{ + AWSConfig: cfg, + CreateTopicConfig: sns.ConfigAttributes{}, + Marshaler: sns.DefaultMarshalerUnmarshaler{}, + TopicResolver: topicResolver, + }, sns.SubscriberConfig{ AWSConfig: cfg, GenerateSqsQueueName: func(ctx context.Context, sqsTopic string) (string, error) { return consumerGroup, nil }, + TopicResolver: topicResolver, }, sqs.SubscriberConfig{ AWSConfig: cfg, @@ -123,17 +160,35 @@ func createPubSubWithConsumerGroup(t *testing.T, consumerGroup string) (message. VisibilityTimeout: "1", }, }, - logger, ) +} + +func createPubSubWithConfig( + t *testing.T, + pubConfig sns.PublisherConfig, + subConfig sns.SubscriberConfig, + sqsSubConfig sqs.SubscriberConfig, +) (message.Publisher, message.Subscriber) { + logger := watermill.NewStdLogger(true, false) + + pub, err := sns.NewPublisher(pubConfig, logger) + require.NoError(t, err) + + sub, err := sns.NewSubscriber(subConfig, sqsSubConfig, logger) require.NoError(t, err) return pub, sub } -func GetAWSConfig() (aws.Config, error) { - return awsconfig.LoadDefaultConfig( +func GetAWSConfig(t *testing.T) aws.Config { + t.Helper() + + cfg, err := awsconfig.LoadDefaultConfig( context.Background(), connection.SetEndPoint("http://localhost:4566"), awsconfig.WithRegion("us-west-2"), ) + require.NoError(t, err) + + return cfg } diff --git a/sns/subscriber.go b/sns/subscriber.go index 0403613..5de7975 100644 --- a/sns/subscriber.go +++ b/sns/subscriber.go @@ -49,9 +49,14 @@ func NewSubscriber( }, nil } -func (s *Subscriber) Subscribe(ctx context.Context, snsTopicArn string) (<-chan *message.Message, error) { +func (s *Subscriber) Subscribe(ctx context.Context, topic string) (<-chan *message.Message, error) { + snsTopicArn, err := s.config.TopicResolver.ResolveTopic(ctx, topic) + if err != nil { + return nil, err + } + if !s.config.DoNotSubscribeToSns { - if err := s.SubscribeInitializeWithContext(ctx, snsTopicArn); err != nil { + if err := s.SubscribeInitializeWithContext(ctx, topic); err != nil { return nil, err } } @@ -68,7 +73,12 @@ func (s *Subscriber) SubscribeInitialize(topic string) error { return s.SubscribeInitializeWithContext(context.Background(), topic) } -func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, snsTopicArn string) error { +func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, topic string) error { + snsTopicArn, err := s.config.TopicResolver.ResolveTopic(ctx, topic) + if err != nil { + return err + } + sqsTopic, err := s.config.GenerateSqsQueueName(ctx, snsTopicArn) if err != nil { return fmt.Errorf("failed to generate SQS queue name: %w", err) diff --git a/sns/topic.go b/sns/topic.go new file mode 100644 index 0000000..1953a14 --- /dev/null +++ b/sns/topic.go @@ -0,0 +1,44 @@ +package sns + +import ( + "context" + "errors" + "fmt" +) + +type TopicResolver interface { + ResolveTopic(ctx context.Context, topic string) (snsTopic string, err error) +} + +type TransparentTopicResolver struct{} + +func (a TransparentTopicResolver) ResolveTopic(ctx context.Context, topic string) (snsTopic string, err error) { + // we are passing topic ARN as topic + return topic, nil +} + +type GenerateArnTopicResolver struct { + accountID string + region string +} + +func NewGenerateArnTopicResolver(accountID string, region string) (*GenerateArnTopicResolver, error) { + var err error + + if accountID == "" { + err = errors.Join(err, fmt.Errorf("accountID is empty")) + } + if region == "" { + err = errors.Join(err, fmt.Errorf("region is empty")) + } + + if err != nil { + return nil, fmt.Errorf("failed to create GenerateArnTopicResolver: %w", err) + } + + return &GenerateArnTopicResolver{accountID: accountID, region: region}, nil +} + +func (g GenerateArnTopicResolver) ResolveTopic(ctx context.Context, topic string) (snsTopic string, err error) { + return GenerateTopicArn(g.region, g.accountID, topic) +} From f6f0f3718a2af073f71f275f3da9cfeddcc10aad Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Sun, 25 Aug 2024 16:21:53 +0200 Subject: [PATCH 13/19] added queue url resolver --- sns/config.go | 6 - sqs/config.go | 62 +++++----- sqs/publisher.go | 128 ++++++++++----------- sqs/pubsub_test.go | 274 ++++++++++++++++++++++++++++++++++++--------- sqs/sqs.go | 3 +- sqs/subscriber.go | 71 ++++++++---- sqs/url.go | 141 +++++++++++++++++++++++ 7 files changed, 511 insertions(+), 174 deletions(-) create mode 100644 sqs/url.go diff --git a/sns/config.go b/sns/config.go index 03ac784..f603ba6 100644 --- a/sns/config.go +++ b/sns/config.go @@ -39,9 +39,6 @@ func (c *PublisherConfig) Validate() error { if c.AWSConfig.Credentials == nil { err = errors.Join(err, fmt.Errorf("sns.PublisherConfig.AWSConfig.Credentials is nil")) } - if c.TopicResolver == nil { - err = errors.Join(err, fmt.Errorf("sns.PublisherConfig.TopicResolver is nil")) - } return err } @@ -87,9 +84,6 @@ func (c *SubscriberConfig) Validate() error { if c.GenerateSqsQueueName == nil { err = errors.Join(err, fmt.Errorf("sns.SubscriberConfig.GenerateSqsQueueName is nil")) } - if c.TopicResolver == nil { - err = errors.Join(err, fmt.Errorf("sns.SubscriberConfig.TopicResolver is nil")) - } return err } diff --git a/sqs/config.go b/sqs/config.go index d61d452..a555dfc 100644 --- a/sqs/config.go +++ b/sqs/config.go @@ -15,6 +15,10 @@ import ( type SubscriberConfig struct { AWSConfig aws.Config + DoNotCreateQueueIfNotExists bool + + QueueUrlResolver QueueUrlResolver + // ReconnectRetrySleep is the time to sleep between reconnect attempts. ReconnectRetrySleep time.Duration @@ -24,9 +28,6 @@ type SubscriberConfig struct { // GenerateCreateQueueInput generates *sqs.CreateQueueInput for AWS SDK. GenerateCreateQueueInput GenerateCreateQueueInputFunc - // GenerateGetQueueUrlInput generates *sqs.GetQueueUrlInput for AWS SDK. - GenerateGetQueueUrlInput GenerateGetQueueUrlInputFunc - // GenerateReceiveMessageInput generates *sqs.ReceiveMessageInput for AWS SDK. GenerateReceiveMessageInput GenerateReceiveMessageInputFunc @@ -49,10 +50,6 @@ func (c *SubscriberConfig) SetDefaults() { c.GenerateCreateQueueInput = GenerateCreateQueueInputDefault } - if c.GenerateGetQueueUrlInput == nil { - c.GenerateGetQueueUrlInput = GenerateGetQueueUrlInputDefault - } - if c.GenerateReceiveMessageInput == nil { c.GenerateReceiveMessageInput = GenerateReceiveMessageInputDefault } @@ -60,6 +57,10 @@ func (c *SubscriberConfig) SetDefaults() { if c.GenerateDeleteMessageInput == nil { c.GenerateDeleteMessageInput = GenerateDeleteMessageInputDefault } + + if c.QueueUrlResolver == nil { + c.QueueUrlResolver = NewGetQueueUrlByNameUrlResolver(GetQueueUrlByNameUrlResolverConfig{}) + } } func (c SubscriberConfig) Validate() error { @@ -72,6 +73,9 @@ func (c SubscriberConfig) Validate() error { if c.Unmarshaler == nil { err = errors.Join(err, errors.New("missing Config.Marshaler")) } + if c.QueueUrlResolver == nil { + err = errors.Join(err, fmt.Errorf("sqs.SubscriberConfig.QueueUrlResolver is nil")) + } return err } @@ -79,12 +83,12 @@ func (c SubscriberConfig) Validate() error { type PublisherConfig struct { AWSConfig aws.Config - CreateQueueConfig QueueConfigAttributes - DoNotCacheQueues bool - CreateQueueIfNotExists bool + CreateQueueConfig QueueConfigAttributes + DoNotCacheQueues bool + + DoNotCreateQueueIfNotExists bool - // GenerateGetQueueUrlInput generates *sqs.GetQueueUrlInput for AWS SDK. - GenerateGetQueueUrlInput GenerateGetQueueUrlInputFunc + QueueUrlResolver QueueUrlResolver // GenerateSendMessageInput generates *sqs.SendMessageInput for AWS SDK. GenerateSendMessageInput GenerateSendMessageInputFunc @@ -100,10 +104,6 @@ func (c *PublisherConfig) setDefaults() { c.Marshaler = DefaultMarshalerUnmarshaler{} } - if c.GenerateGetQueueUrlInput == nil { - c.GenerateGetQueueUrlInput = GenerateGetQueueUrlInputDefault - } - if c.GenerateSendMessageInput == nil { c.GenerateSendMessageInput = GenerateSendMessageInputDefault } @@ -111,30 +111,36 @@ func (c *PublisherConfig) setDefaults() { if c.GenerateCreateQueueInput == nil { c.GenerateCreateQueueInput = GenerateCreateQueueInputDefault } + + if c.QueueUrlResolver == nil { + c.QueueUrlResolver = NewGetQueueUrlByNameUrlResolver(GetQueueUrlByNameUrlResolverConfig{}) + } +} + +func (c *PublisherConfig) Validate() error { + var err error + + if c.QueueUrlResolver == nil { + err = errors.Join(err, fmt.Errorf("sqs.SubscriberConfig.QueueUrlResolver is nil")) + } + + return err } -type GenerateCreateQueueInputFunc func(ctx context.Context, topic string, attrs QueueConfigAttributes) (*sqs.CreateQueueInput, error) +type GenerateCreateQueueInputFunc func(ctx context.Context, queueName string, attrs QueueConfigAttributes) (*sqs.CreateQueueInput, error) -func GenerateCreateQueueInputDefault(ctx context.Context, topic string, attrs QueueConfigAttributes) (*sqs.CreateQueueInput, error) { +func GenerateCreateQueueInputDefault(ctx context.Context, queueName string, attrs QueueConfigAttributes) (*sqs.CreateQueueInput, error) { attrsMap, err := attrs.Attributes() if err != nil { - return nil, fmt.Errorf("cannot generate attributes for queue %s: %w", topic, err) + return nil, fmt.Errorf("cannot generate attributes for queue %s: %w", queueName, err) } return &sqs.CreateQueueInput{ - QueueName: aws.String(topic), + QueueName: aws.String(queueName), Attributes: attrsMap, }, nil } -type GenerateGetQueueUrlInputFunc func(ctx context.Context, topic string) (*sqs.GetQueueUrlInput, error) - -func GenerateGetQueueUrlInputDefault(ctx context.Context, topic string) (*sqs.GetQueueUrlInput, error) { - return &sqs.GetQueueUrlInput{ - QueueName: aws.String(topic), - }, nil -} - type GenerateReceiveMessageInputFunc func(ctx context.Context, queueURL string) (*sqs.ReceiveMessageInput, error) func GenerateReceiveMessageInputDefault(ctx context.Context, queueURL string) (*sqs.ReceiveMessageInput, error) { diff --git a/sqs/publisher.go b/sqs/publisher.go index b20722b..5bd4040 100644 --- a/sqs/publisher.go +++ b/sqs/publisher.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "sync" "github.com/ThreeDotsLabs/watermill" "github.com/ThreeDotsLabs/watermill/message" @@ -17,32 +16,29 @@ type Publisher struct { config PublisherConfig logger watermill.LoggerAdapter sqs *sqs.Client - - queuesCache map[string]*string - queuesCacheLock sync.RWMutex } func NewPublisher(config PublisherConfig, logger watermill.LoggerAdapter) (*Publisher, error) { config.setDefaults() + if err := config.Validate(); err != nil { + return nil, err + } + return &Publisher{ - sqs: sqs.NewFromConfig(config.AWSConfig), - config: config, - logger: logger, - queuesCache: make(map[string]*string), + sqs: sqs.NewFromConfig(config.AWSConfig), + config: config, + logger: logger, }, nil } func (p *Publisher) Publish(topic string, messages ...*message.Message) error { ctx := context.Background() - queueUrl, err := p.GetOrCreateQueueUrl(ctx, topic) + queueName, queueUrl, err := p.GetOrCreateQueueUrl(ctx, topic) if err != nil { return fmt.Errorf("cannot get queue url: %w", err) } - if queueUrl == nil { - return fmt.Errorf("returned queueUrl is nil") - } for _, msg := range messages { sqsMsg, err := p.config.Marshaler.Marshal(msg) @@ -52,12 +48,20 @@ func (p *Publisher) Publish(topic string, messages ...*message.Message) error { p.logger.Debug("Sending message", watermill.LogFields{"msg": msg}) - input, err := p.config.GenerateSendMessageInput(ctx, *queueUrl, sqsMsg) + input, err := p.config.GenerateSendMessageInput(ctx, queueUrl, sqsMsg) if err != nil { return fmt.Errorf("cannot generate send message input: %w", err) } _, err = p.sqs.SendMessage(ctx, input) + var queueDoesNotExistErr *types.QueueDoesNotExist + if errors.As(err, &queueDoesNotExistErr) && !p.config.DoNotCreateQueueIfNotExists { + // GetOrCreateQueueUrl may not create queue if QueueUrlResolver doesn't check if queue exists + _, err := p.createQueue(ctx, topic, queueName, &queueUrl) + if err != nil { + return err + } + } if err != nil { return fmt.Errorf("cannot send message: %w", err) } @@ -66,75 +70,67 @@ func (p *Publisher) Publish(topic string, messages ...*message.Message) error { return nil } -func (p *Publisher) GetOrCreateQueueUrl(ctx context.Context, topic string) (queueUrl *string, err error) { - getQueueInput, err := p.config.GenerateGetQueueUrlInput(ctx, topic) +// todo: add types for queueName, etc. ? as it becomes messy what is what + +// todo: name is stupid as creation is conditional +func (p *Publisher) GetOrCreateQueueUrl(ctx context.Context, topic string) (queueName string, queueURL string, err error) { + queueName, queueUrl, exists, err := p.config.QueueUrlResolver.ResolveQueueUrl(ctx, ResolveQueueUrlParams{ + Topic: topic, + SqsClient: p.sqs, + Logger: p.logger, + }) if err != nil { - return nil, fmt.Errorf("cannot generate input for queue %s: %w", topic, err) + return "", "", err } - - var getQueueInputHash string - - if !p.config.DoNotCacheQueues { - getQueueInputHash = generateGetQueueUrlInputHash(getQueueInput) - - if getQueueInputHash != "" { - p.queuesCacheLock.RLock() - var ok bool - queueUrl, ok = p.queuesCache[getQueueInputHash] - p.queuesCacheLock.RUnlock() - - if ok { - p.logger.Trace("Used cache", watermill.LogFields{ - "topic": topic, - "queue": *queueUrl, - "hash": getQueueInputHash, - }) - return queueUrl, nil - } - } + if exists { + return queueName, *queueUrl, nil } - defer func() { - if err == nil && getQueueInputHash != "" { - p.queuesCacheLock.Lock() - p.queuesCache[getQueueInputHash] = queueUrl - p.queuesCacheLock.Unlock() - - p.logger.Trace("Stored cache", watermill.LogFields{ - "topic": topic, - "queue": *queueUrl, - "hash": getQueueInputHash, - }) + if !p.config.DoNotCreateQueueIfNotExists { + queueUrl, err := p.createQueue(ctx, topic, queueName, queueUrl) + if err != nil { + return "", "", err } - }() - queueUrl, err = getQueueUrl(ctx, p.sqs, topic, getQueueInput) - if err == nil { - return queueUrl, nil + return queueName, queueUrl, nil + + } else { + return "", "", fmt.Errorf("queue for topic %s doesn't exist", topic) } +} - var queueDoesNotExistsErr *types.QueueDoesNotExist - if errors.As(err, &queueDoesNotExistsErr) && p.config.CreateQueueIfNotExists { - input, err := p.config.GenerateCreateQueueInput(ctx, topic, p.config.CreateQueueConfig) - if err != nil { - return nil, fmt.Errorf("cannot generate create queue input: %w", err) - } +func (p *Publisher) createQueue(ctx context.Context, topic string, queueName string, queueUrl *string) (string, error) { + input, err := p.config.GenerateCreateQueueInput(ctx, queueName, p.config.CreateQueueConfig) + if err != nil { + return "", fmt.Errorf("cannot generate create queue input: %w", err) + } - queueUrl, err = createQueue(ctx, p.sqs, input) + queueUrl, err = createQueue(ctx, p.sqs, input) + if err != nil { + return "", fmt.Errorf("cannot create queue: %w", err) + } + // queue was created in the meantime + // todo: it's quite ugly + if queueUrl == nil { + _, queueUrl, exists, err := p.config.QueueUrlResolver.ResolveQueueUrl(ctx, ResolveQueueUrlParams{ + Topic: topic, + SqsClient: p.sqs, + Logger: p.logger, + }) if err != nil { - return nil, fmt.Errorf("cannot create queue: %w", err) + return "", err } - // queue was created in the meantime - if queueUrl == nil { - return getQueueUrl(ctx, p.sqs, topic, getQueueInput) + if !exists { + return "", fmt.Errorf("queue doesn't exist after creation") } - return queueUrl, nil - } else { - return nil, fmt.Errorf("cannot get queue url: %w", err) + return *queueUrl, nil } + + return *queueUrl, nil } +// todo: move (together with other funcs) to url.go func generateGetQueueUrlInputHash(getQueueInput *sqs.GetQueueUrlInput) string { // we are not using fmt.Sprintf because of pointers under the hood // we are not hashing specific struct fields to keep forward compatibility diff --git a/sqs/pubsub_test.go b/sqs/pubsub_test.go index e9f71be..1e32dbb 100644 --- a/sqs/pubsub_test.go +++ b/sqs/pubsub_test.go @@ -2,6 +2,7 @@ package sqs_test import ( "context" + "fmt" "runtime" "testing" @@ -48,7 +49,7 @@ func TestPubSub_stress(t *testing.T) { ) } -func TestPublishSubscribe_batching(t *testing.T) { +func TestPublishSubscribe_with_GenerateQueueUrlResolver(t *testing.T) { tests.TestPublishSubscribe( t, tests.TestContext{ @@ -62,43 +63,152 @@ func TestPublishSubscribe_batching(t *testing.T) { }, }, func(t *testing.T) (message.Publisher, message.Subscriber) { - logger := watermill.NewStdLogger(false, false) - cfg := newAwsConfig(t) - pub, err := sqs.NewPublisher(sqs.PublisherConfig{ - AWSConfig: cfg, - CreateQueueConfig: sqs.QueueConfigAttributes{ - // Default value is 30 seconds - need to be lower for tests - VisibilityTimeout: "1", + queueResolver := sqs.GenerateQueueUrlResolver{ + AwsRegion: "us-west-2", + AwsAccountID: "000000000000", + } + + return createPubSubWithConfig( + t, + sqs.PublisherConfig{ + AWSConfig: cfg, + CreateQueueConfig: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests + VisibilityTimeout: "1", + }, + Marshaler: sqs.DefaultMarshalerUnmarshaler{}, + QueueUrlResolver: queueResolver, }, - CreateQueueIfNotExists: true, - Marshaler: sqs.DefaultMarshalerUnmarshaler{}, - }, logger) - require.NoError(t, err) - - sub, err := sqs.NewSubscriber(sqs.SubscriberConfig{ - AWSConfig: cfg, - QueueConfigAttributes: sqs.QueueConfigAttributes{ - // Default value is 30 seconds - need to be lower for tests - VisibilityTimeout: "1", + sqs.SubscriberConfig{ + AWSConfig: cfg, + QueueConfigAttributes: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests + VisibilityTimeout: "1", + }, + GenerateReceiveMessageInput: func(ctx context.Context, queueURL string) (*awssqs.ReceiveMessageInput, error) { + in, err := sqs.GenerateReceiveMessageInputDefault(ctx, queueURL) + if err != nil { + return nil, err + } + + // this will effectively enable batching + in.MaxNumberOfMessages = 10 + + return in, nil + }, + Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, + QueueUrlResolver: queueResolver, }, - GenerateReceiveMessageInput: func(ctx context.Context, queueURL string) (*awssqs.ReceiveMessageInput, error) { - in, err := sqs.GenerateReceiveMessageInputDefault(ctx, queueURL) - if err != nil { - return nil, err - } + ) + }, + ) +} - // this will effectively enable batching - in.MaxNumberOfMessages = 10 +func TestPublishSubscribe_with_TransparentUrlResolver(t *testing.T) { + tests.TestPublishSubscribe( + t, + tests.TestContext{ + TestID: tests.NewTestID(), + Features: tests.Features{ + ConsumerGroups: true, + ExactlyOnceDelivery: false, + GuaranteedOrder: true, + GuaranteedOrderWithSingleSubscriber: true, + Persistent: true, + GenerateTopicFunc: func(tctx tests.TestContext) string { + return fmt.Sprintf("http://sqs.us-west-2.localhost.localstack.cloud:4566/000000000000/%s", tctx.TestID) + }, + }, + }, + func(t *testing.T) (message.Publisher, message.Subscriber) { + cfg := newAwsConfig(t) - return in, nil + queueResolver := sqs.TransparentUrlResolver{} + + return createPubSubWithConfig( + t, + sqs.PublisherConfig{ + AWSConfig: cfg, + CreateQueueConfig: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests + VisibilityTimeout: "1", + }, + Marshaler: sqs.DefaultMarshalerUnmarshaler{}, + QueueUrlResolver: queueResolver, }, - Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, - }, logger) - require.NoError(t, err) + sqs.SubscriberConfig{ + AWSConfig: cfg, + QueueConfigAttributes: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests + VisibilityTimeout: "1", + }, + GenerateReceiveMessageInput: func(ctx context.Context, queueURL string) (*awssqs.ReceiveMessageInput, error) { + in, err := sqs.GenerateReceiveMessageInputDefault(ctx, queueURL) + if err != nil { + return nil, err + } + + // this will effectively enable batching + in.MaxNumberOfMessages = 10 + + return in, nil + }, + Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, + QueueUrlResolver: queueResolver, + }, + ) + }, + ) +} + +func TestPublishSubscribe_batching(t *testing.T) { + tests.TestPublishSubscribe( + t, + tests.TestContext{ + TestID: tests.NewTestID(), + Features: tests.Features{ + ConsumerGroups: true, + ExactlyOnceDelivery: false, + GuaranteedOrder: true, + GuaranteedOrderWithSingleSubscriber: true, + Persistent: true, + }, + }, + func(t *testing.T) (message.Publisher, message.Subscriber) { + cfg := newAwsConfig(t) - return pub, sub + return createPubSubWithConfig( + t, + sqs.PublisherConfig{ + AWSConfig: cfg, + CreateQueueConfig: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests + VisibilityTimeout: "1", + }, + Marshaler: sqs.DefaultMarshalerUnmarshaler{}, + }, + sqs.SubscriberConfig{ + AWSConfig: cfg, + QueueConfigAttributes: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests + VisibilityTimeout: "1", + }, + GenerateReceiveMessageInput: func(ctx context.Context, queueURL string) (*awssqs.ReceiveMessageInput, error) { + in, err := sqs.GenerateReceiveMessageInputDefault(ctx, queueURL) + if err != nil { + return nil, err + } + + // this will effectively enable batching + in.MaxNumberOfMessages = 10 + + return in, nil + }, + Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, + }, + ) }, ) } @@ -109,7 +219,6 @@ func TestPublishSubscribe_creating_queue_with_different_settings_should_be_idemp sub1, err := sqs.NewSubscriber(sqs.SubscriberConfig{ AWSConfig: newAwsConfig(t), QueueConfigAttributes: sqs.QueueConfigAttributes{ - // Default value is 30 seconds - need to be lower for tests VisibilityTimeout: "1", }, Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, @@ -119,7 +228,6 @@ func TestPublishSubscribe_creating_queue_with_different_settings_should_be_idemp sub2, err := sqs.NewSubscriber(sqs.SubscriberConfig{ AWSConfig: newAwsConfig(t), QueueConfigAttributes: sqs.QueueConfigAttributes{ - // Default value is 30 seconds - need to be lower for tests VisibilityTimeout: "20", }, Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, @@ -137,47 +245,105 @@ func TestPublisher_GetOrCreateQueueUrl_is_idempotent(t *testing.T) { topicName := watermill.NewUUID() - url1, err := pub.(*sqs.Publisher).GetOrCreateQueueUrl(context.Background(), topicName) + name1, url1, err := pub.(*sqs.Publisher).GetOrCreateQueueUrl(context.Background(), topicName) require.NoError(t, err) - url2, err := pub.(*sqs.Publisher).GetOrCreateQueueUrl(context.Background(), topicName) + name2, url2, err := pub.(*sqs.Publisher).GetOrCreateQueueUrl(context.Background(), topicName) require.NoError(t, err) require.Equal(t, url1, url2) + require.Equal(t, name1, name2) + } func TestSubscriber_doesnt_hang_when_queue_doesnt_exist(t *testing.T) { - _, sub := createPubSub(t) + cfg := newAwsConfig(t) + + _, sub := createPubSubWithConfig( + t, + sqs.PublisherConfig{ + AWSConfig: cfg, + CreateQueueConfig: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests + VisibilityTimeout: "1", + }, + Marshaler: sqs.DefaultMarshalerUnmarshaler{}, + }, + sqs.SubscriberConfig{ + AWSConfig: cfg, + QueueConfigAttributes: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests + VisibilityTimeout: "1", + }, + Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, + DoNotCreateQueueIfNotExists: true, + }, + ) msgs, err := sub.Subscribe(context.Background(), "non-existing-queue") - require.ErrorContains(t, err, "cannot get queue for topic non-existing-queue: cannot get queue non-existing-queue") + require.ErrorContains(t, err, "queue for topic 'non-existing-queue' doesn't exists") require.Nil(t, msgs) } -func createPubSub(t *testing.T) (message.Publisher, message.Subscriber) { - logger := watermill.NewStdLogger(false, false) +func TestPublisher_do_not_create_queue(t *testing.T) { + cfg := newAwsConfig(t) + + pub, _ := createPubSubWithConfig( + t, + sqs.PublisherConfig{ + AWSConfig: cfg, + CreateQueueConfig: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests + VisibilityTimeout: "1", + }, + Marshaler: sqs.DefaultMarshalerUnmarshaler{}, + DoNotCreateQueueIfNotExists: true, + }, + sqs.SubscriberConfig{ + AWSConfig: cfg, + QueueConfigAttributes: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests + VisibilityTimeout: "1", + }, + Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, + }, + ) + err := pub.Publish("non-existing-queue-2", message.NewMessage("1", []byte("x"))) + + require.ErrorContains(t, err, "queue for topic non-existing-queue-2 doesn't exist") +} +func createPubSub(t *testing.T) (message.Publisher, message.Subscriber) { cfg := newAwsConfig(t) - pub, err := sqs.NewPublisher(sqs.PublisherConfig{ - AWSConfig: cfg, - CreateQueueConfig: sqs.QueueConfigAttributes{ - // Default value is 30 seconds - need to be lower for tests - VisibilityTimeout: "1", + return createPubSubWithConfig( + t, + sqs.PublisherConfig{ + AWSConfig: cfg, + CreateQueueConfig: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests + VisibilityTimeout: "1", + }, + Marshaler: sqs.DefaultMarshalerUnmarshaler{}, }, - CreateQueueIfNotExists: true, - Marshaler: sqs.DefaultMarshalerUnmarshaler{}, - }, logger) + sqs.SubscriberConfig{ + AWSConfig: cfg, + QueueConfigAttributes: sqs.QueueConfigAttributes{ + // Default value is 30 seconds - need to be lower for tests + VisibilityTimeout: "1", + }, + Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, + }, + ) +} + +func createPubSubWithConfig(t *testing.T, pubConfig sqs.PublisherConfig, subConfig sqs.SubscriberConfig) (message.Publisher, message.Subscriber) { + logger := watermill.NewStdLogger(false, false) + + pub, err := sqs.NewPublisher(pubConfig, logger) require.NoError(t, err) - sub, err := sqs.NewSubscriber(sqs.SubscriberConfig{ - AWSConfig: cfg, - QueueConfigAttributes: sqs.QueueConfigAttributes{ - // Default value is 30 seconds - need to be lower for tests - VisibilityTimeout: "1", - }, - Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, - }, logger) + sub, err := sqs.NewSubscriber(subConfig, logger) require.NoError(t, err) return pub, sub diff --git a/sqs/sqs.go b/sqs/sqs.go index 55b1586..a765d56 100644 --- a/sqs/sqs.go +++ b/sqs/sqs.go @@ -18,6 +18,7 @@ func getQueueUrl(ctx context.Context, sqsClient *sqs.Client, topic string, input return getQueueOutput.QueueUrl, nil } +// todo: wtf about that? func createQueue( ctx context.Context, sqsClient *sqs.Client, @@ -33,7 +34,7 @@ func createQueue( return nil, nil } if err != nil { - return nil, fmt.Errorf("cannot create queue %w", err) + return nil, fmt.Errorf("cannot create queue '%s': %w", *createQueueParams.QueueName, err) } if createQueueOutput.QueueUrl == nil { return nil, fmt.Errorf("cannot create queue, queueUrl is nil") diff --git a/sqs/subscriber.go b/sqs/subscriber.go index 917687d..52aaaad 100644 --- a/sqs/subscriber.go +++ b/sqs/subscriber.go @@ -54,36 +54,49 @@ func (s *Subscriber) Subscribe(ctx context.Context, topic string) (<-chan *messa s.logger.With(watermill.LogFields{"topic": topic}).Debug("Getting queue", nil) - getQueueInput, err := s.config.GenerateGetQueueUrlInput(ctx, topic) - if err != nil { - return nil, fmt.Errorf("cannot generate input for queue %s: %w", topic, err) + resolveQueueParams := ResolveQueueUrlParams{ + Topic: topic, + SqsClient: s.sqs, + Logger: s.logger, } - ctx, cancel := context.WithCancel(ctx) - output := make(chan *message.Message) - - queueURL, err := getQueueUrl(ctx, s.sqs, topic, getQueueInput) + // todo: what if doesn't exists? + queueName, queueURL, exists, err := s.config.QueueUrlResolver.ResolveQueueUrl(ctx, resolveQueueParams) if err != nil { - close(output) - cancel() - return nil, fmt.Errorf("cannot get queue for topic %s: %w", topic, err) + return nil, err } - if queueURL == nil { - close(output) - cancel() - return nil, fmt.Errorf("queue for topic %s not found", topic) + if !exists { + if s.config.DoNotCreateQueueIfNotExists { + return nil, fmt.Errorf("queue for topic '%s' doesn't exists", topic) + } + + input, err := s.config.GenerateCreateQueueInput(ctx, queueName, s.config.QueueConfigAttributes) + if err != nil { + return nil, fmt.Errorf("cannot generate input for queue %s: %w", topic, err) + } + + _, err = createQueue(ctx, s.sqs, input) + if err != nil { + return nil, fmt.Errorf("cannot create queue %s: %w", topic, err) + } + + // todo: it's quite ugly + _, queueURL, _, err = s.config.QueueUrlResolver.ResolveQueueUrl(ctx, resolveQueueParams) + if err != nil { + return nil, err + } } receiveInput, err := s.config.GenerateReceiveMessageInput(ctx, *queueURL) if err != nil { - close(output) - cancel() return nil, fmt.Errorf("cannot generate input for topic %s: %w", topic, err) } s.logger.With(watermill.LogFields{"queue": *queueURL}).Info("Subscribing to queue", nil) + ctx, cancel := context.WithCancel(ctx) s.subscribersWg.Add(1) + output := make(chan *message.Message) go func() { s.receive(ctx, *queueURL, output, receiveInput) @@ -259,7 +272,19 @@ func (s *Subscriber) SubscribeInitialize(topic string) error { } func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, topic string) error { - input, err := s.config.GenerateCreateQueueInput(ctx, topic, s.config.QueueConfigAttributes) + queueName, _, exists, err := s.config.QueueUrlResolver.ResolveQueueUrl(ctx, ResolveQueueUrlParams{ + Topic: topic, + SqsClient: s.sqs, + Logger: s.logger, + }) + if err != nil { + return err + } + if exists { + return nil + } + + input, err := s.config.GenerateCreateQueueInput(ctx, queueName, s.config.QueueConfigAttributes) if err != nil { return fmt.Errorf("cannot generate input for queue %s: %w", topic, err) } @@ -272,13 +297,21 @@ func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, topic s return nil } +// todo: duplicated in subscribe? func (s *Subscriber) GetQueueUrl(ctx context.Context, topic string) (*string, error) { - getQueueInput, err := s.config.GenerateGetQueueUrlInput(ctx, topic) + resolveQueueParams := ResolveQueueUrlParams{ + Topic: topic, + SqsClient: s.sqs, + Logger: s.logger, + } + + // todo: what if doesn't exists? + _, queueURL, _, err := s.config.QueueUrlResolver.ResolveQueueUrl(ctx, resolveQueueParams) if err != nil { return nil, fmt.Errorf("cannot generate input for queue %s: %w", topic, err) } - return getQueueUrl(ctx, s.sqs, topic, getQueueInput) + return queueURL, nil } func (s *Subscriber) GetQueueArn(ctx context.Context, url *string) (*string, error) { diff --git a/sqs/url.go b/sqs/url.go new file mode 100644 index 0000000..2268fdd --- /dev/null +++ b/sqs/url.go @@ -0,0 +1,141 @@ +package sqs + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + + "github.com/ThreeDotsLabs/watermill" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sqs/types" +) + +type QueueUrlResolver interface { + ResolveQueueUrl(ctx context.Context, params ResolveQueueUrlParams) (queueName string, sqsQueueUrl *string, exists bool, err error) +} + +type ResolveQueueUrlParams struct { + // todo: doc that it's topic in watermill's nomenclature + Topic string + SqsClient *sqs.Client + Logger watermill.LoggerAdapter +} + +// todo: add alternative one that statically generates queue name + +type GenerateQueueUrlResolver struct { + AwsRegion string + AwsAccountID string +} + +func (p GenerateQueueUrlResolver) ResolveQueueUrl(ctx context.Context, params ResolveQueueUrlParams) (queueName string, queueUrl *string, exists bool, err error) { + queueURL := fmt.Sprintf("https://sqs.%s.amazonaws.com/%s/%s", p.AwsRegion, p.AwsAccountID, params.Topic) + + // todo: how to deal with fact that exists is always true? what if it doesn't exist? + return params.Topic, &queueURL, true, nil +} + +type GetQueueUrlByNameUrlResolverConfig struct { + DoNotCacheQueues bool + + // GenerateGetQueueUrlInput generates *sqs.GetQueueUrlInput for AWS SDK. + GenerateGetQueueUrlInput GenerateGetQueueUrlInputFunc +} + +// GetQueueUrlByNameUrlResolver resolves queue url by calling AWS API. +// todo: more docs +type GetQueueUrlByNameUrlResolver struct { + config GetQueueUrlByNameUrlResolverConfig + + queuesCache map[string]*string + queuesCacheLock sync.RWMutex +} + +func NewGetQueueUrlByNameUrlResolver( + config GetQueueUrlByNameUrlResolverConfig, +) *GetQueueUrlByNameUrlResolver { + if config.GenerateGetQueueUrlInput == nil { + config.GenerateGetQueueUrlInput = GenerateGetQueueUrlInputDefault + } + + return &GetQueueUrlByNameUrlResolver{ + config: config, + queuesCache: map[string]*string{}, + } +} + +func (p *GetQueueUrlByNameUrlResolver) ResolveQueueUrl(ctx context.Context, params ResolveQueueUrlParams) (queueName string, queueUrl *string, exists bool, err error) { + getQueueInput, err := p.config.GenerateGetQueueUrlInput(ctx, params.Topic) + if err != nil { + return params.Topic, nil, false, fmt.Errorf("cannot generate input for queue %s: %w", params.Topic, err) + } + + var getQueueInputHash string + + if !p.config.DoNotCacheQueues { + getQueueInputHash = generateGetQueueUrlInputHash(getQueueInput) + + if getQueueInputHash != "" { + p.queuesCacheLock.RLock() + var ok bool + queueUrl, ok = p.queuesCache[getQueueInputHash] + p.queuesCacheLock.RUnlock() + + if ok { + params.Logger.Trace("Used cache", watermill.LogFields{ + "topic": params.Topic, + "queue": *queueUrl, + "hash": getQueueInputHash, + }) + return params.Topic, queueUrl, true, nil + } + } + } + + defer func() { + if err == nil && exists && getQueueInputHash != "" { + p.queuesCacheLock.Lock() + p.queuesCache[getQueueInputHash] = queueUrl + p.queuesCacheLock.Unlock() + + params.Logger.Trace("Stored cache", watermill.LogFields{ + "topic": params.Topic, + "queue": *queueUrl, + "hash": getQueueInputHash, + }) + } + }() + + queueUrl, err = getQueueUrl(ctx, params.SqsClient, params.Topic, getQueueInput) + if err == nil { + return params.Topic, queueUrl, true, nil + } + var queueDoesNotExistsErr *types.QueueDoesNotExist + if errors.As(err, &queueDoesNotExistsErr) { + // todo: this is not a good idea? + return params.Topic, queueUrl, false, nil + } + + return params.Topic, nil, false, err +} + +type GenerateGetQueueUrlInputFunc func(ctx context.Context, topic string) (*sqs.GetQueueUrlInput, error) + +func GenerateGetQueueUrlInputDefault(ctx context.Context, topic string) (*sqs.GetQueueUrlInput, error) { + return &sqs.GetQueueUrlInput{ + QueueName: aws.String(topic), + }, nil +} + +// todo: test +type TransparentUrlResolver struct{} + +func (p TransparentUrlResolver) ResolveQueueUrl(ctx context.Context, params ResolveQueueUrlParams) (queueName string, queueUrl *string, exists bool, err error) { + topicParts := strings.Split(params.Topic, "/") + queueName = topicParts[len(topicParts)-1] + + return queueName, ¶ms.Topic, true, nil +} From 09f2b7c7c8fb35922a07042b9f7b12ab3062dcf0 Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Sun, 25 Aug 2024 17:15:34 +0200 Subject: [PATCH 14/19] cleanups --- sns/config.go | 13 +++- sns/publisher.go | 2 +- sns/sns.go | 2 +- sqs/config.go | 26 ++++---- sqs/publisher.go | 36 ++++------- sqs/pubsub_test.go | 6 +- sqs/sqs.go | 23 +++++-- sqs/subscriber.go | 40 +++++++------ sqs/{url.go => url_resolver.go} | 102 ++++++++++++++++++++++++-------- 9 files changed, 158 insertions(+), 92 deletions(-) rename sqs/{url.go => url_resolver.go} (51%) diff --git a/sns/config.go b/sns/config.go index f603ba6..d258327 100644 --- a/sns/config.go +++ b/sns/config.go @@ -39,6 +39,9 @@ func (c *PublisherConfig) Validate() error { if c.AWSConfig.Credentials == nil { err = errors.Join(err, fmt.Errorf("sns.PublisherConfig.AWSConfig.Credentials is nil")) } + if c.TopicResolver == nil { + err = errors.Join(err, fmt.Errorf("sns.PublisherConfig.TopicResolver is nil")) + } return err } @@ -60,7 +63,8 @@ func GenerateCreateTopicInputDefault(ctx context.Context, topic string, attrs Co type SubscriberConfig struct { AWSConfig aws.Config - GenerateSqsQueueName func(ctx context.Context, sqsTopicArn string) (string, error) + // todo: better name? + GenerateSqsQueueName func(ctx context.Context, snsTopic string) (string, error) TopicResolver TopicResolver @@ -84,12 +88,15 @@ func (c *SubscriberConfig) Validate() error { if c.GenerateSqsQueueName == nil { err = errors.Join(err, fmt.Errorf("sns.SubscriberConfig.GenerateSqsQueueName is nil")) } + if c.TopicResolver == nil { + err = errors.Join(err, fmt.Errorf("sns.SubscriberConfig.TopicResolver is nil")) + } return err } -func GenerateSqsQueueNameEqualToTopicName(ctx context.Context, sqsTopicArn string) (string, error) { - topicName, err := TopicNameFromTopicArn(sqsTopicArn) +func GenerateSqsQueueNameEqualToTopicName(ctx context.Context, snsTopic string) (string, error) { + topicName, err := ExtractTopicNameFromTopicArn(snsTopic) if err != nil { return "", err } diff --git a/sns/publisher.go b/sns/publisher.go index 9ae2da0..e09a9cb 100644 --- a/sns/publisher.go +++ b/sns/publisher.go @@ -76,7 +76,7 @@ func (p *Publisher) CreateTopic(ctx context.Context, topic string) (string, erro return "", err } - topicName, err := TopicNameFromTopicArn(topicArn) + topicName, err := ExtractTopicNameFromTopicArn(topicArn) if err != nil { return "", err } diff --git a/sns/sns.go b/sns/sns.go index d8bc961..63c1517 100644 --- a/sns/sns.go +++ b/sns/sns.go @@ -35,7 +35,7 @@ func GenerateTopicArn(region, accountID, topic string) (string, error) { return fmt.Sprintf("arn:aws:sns:%s:%s:%s", region, accountID, topic), nil } -func TopicNameFromTopicArn(topicArn string) (string, error) { +func ExtractTopicNameFromTopicArn(topicArn string) (string, error) { topicArnParts := strings.Split(topicArn, ":") if len(topicArnParts) != 6 { return "", fmt.Errorf("topic arn should have 6 segments, has %d (%s)", len(topicArnParts), topicArn) diff --git a/sqs/config.go b/sqs/config.go index a555dfc..e2a0b02 100644 --- a/sqs/config.go +++ b/sqs/config.go @@ -127,45 +127,47 @@ func (c *PublisherConfig) Validate() error { return err } -type GenerateCreateQueueInputFunc func(ctx context.Context, queueName string, attrs QueueConfigAttributes) (*sqs.CreateQueueInput, error) +type GenerateCreateQueueInputFunc func(ctx context.Context, queueName QueueName, attrs QueueConfigAttributes) (*sqs.CreateQueueInput, error) -func GenerateCreateQueueInputDefault(ctx context.Context, queueName string, attrs QueueConfigAttributes) (*sqs.CreateQueueInput, error) { +func GenerateCreateQueueInputDefault(ctx context.Context, queueName QueueName, attrs QueueConfigAttributes) (*sqs.CreateQueueInput, error) { attrsMap, err := attrs.Attributes() if err != nil { return nil, fmt.Errorf("cannot generate attributes for queue %s: %w", queueName, err) } return &sqs.CreateQueueInput{ - QueueName: aws.String(queueName), + QueueName: aws.String(string(queueName)), Attributes: attrsMap, }, nil } -type GenerateReceiveMessageInputFunc func(ctx context.Context, queueURL string) (*sqs.ReceiveMessageInput, error) +type GenerateReceiveMessageInputFunc func(ctx context.Context, queueURL QueueURL) (*sqs.ReceiveMessageInput, error) -func GenerateReceiveMessageInputDefault(ctx context.Context, queueURL string) (*sqs.ReceiveMessageInput, error) { +func GenerateReceiveMessageInputDefault(ctx context.Context, queueURL QueueURL) (*sqs.ReceiveMessageInput, error) { return &sqs.ReceiveMessageInput{ - QueueUrl: aws.String(queueURL), + QueueUrl: aws.String(string(queueURL)), MessageAttributeNames: []string{"All"}, WaitTimeSeconds: 20, // 20 is max at the moment MaxNumberOfMessages: 1, // Currently default value. }, nil } -type GenerateDeleteMessageInputFunc func(ctx context.Context, queueURL string, receiptHandle *string) (*sqs.DeleteMessageInput, error) +type GenerateDeleteMessageInputFunc func(ctx context.Context, queueURL QueueURL, receiptHandle *string) (*sqs.DeleteMessageInput, error) -func GenerateDeleteMessageInputDefault(ctx context.Context, queueURL string, receiptHandle *string) (*sqs.DeleteMessageInput, error) { +func GenerateDeleteMessageInputDefault(ctx context.Context, queueURL QueueURL, receiptHandle *string) (*sqs.DeleteMessageInput, error) { return &sqs.DeleteMessageInput{ - QueueUrl: aws.String(queueURL), + QueueUrl: aws.String(string(queueURL)), ReceiptHandle: receiptHandle, }, nil } -type GenerateSendMessageInputFunc func(ctx context.Context, queueURL string, msg *types.Message) (*sqs.SendMessageInput, error) +type GenerateSendMessageInputFunc func(ctx context.Context, queueURL QueueURL, msg *types.Message) (*sqs.SendMessageInput, error) + +func GenerateSendMessageInputDefault(ctx context.Context, queueURL QueueURL, msg *types.Message) (*sqs.SendMessageInput, error) { + urlStr := string(queueURL) -func GenerateSendMessageInputDefault(ctx context.Context, queueURL string, msg *types.Message) (*sqs.SendMessageInput, error) { return &sqs.SendMessageInput{ - QueueUrl: &queueURL, + QueueUrl: &urlStr, MessageAttributes: msg.MessageAttributes, MessageBody: msg.Body, }, nil diff --git a/sqs/publisher.go b/sqs/publisher.go index 5bd4040..5b039b2 100644 --- a/sqs/publisher.go +++ b/sqs/publisher.go @@ -2,7 +2,6 @@ package sqs import ( "context" - "encoding/json" "fmt" "github.com/ThreeDotsLabs/watermill" @@ -57,7 +56,7 @@ func (p *Publisher) Publish(topic string, messages ...*message.Message) error { var queueDoesNotExistErr *types.QueueDoesNotExist if errors.As(err, &queueDoesNotExistErr) && !p.config.DoNotCreateQueueIfNotExists { // GetOrCreateQueueUrl may not create queue if QueueUrlResolver doesn't check if queue exists - _, err := p.createQueue(ctx, topic, queueName, &queueUrl) + _, err := p.createQueue(ctx, topic, queueName) if err != nil { return err } @@ -73,8 +72,8 @@ func (p *Publisher) Publish(topic string, messages ...*message.Message) error { // todo: add types for queueName, etc. ? as it becomes messy what is what // todo: name is stupid as creation is conditional -func (p *Publisher) GetOrCreateQueueUrl(ctx context.Context, topic string) (queueName string, queueURL string, err error) { - queueName, queueUrl, exists, err := p.config.QueueUrlResolver.ResolveQueueUrl(ctx, ResolveQueueUrlParams{ +func (p *Publisher) GetOrCreateQueueUrl(ctx context.Context, topic string) (QueueName, QueueURL, error) { + resolvedQueue, err := p.config.QueueUrlResolver.ResolveQueueUrl(ctx, ResolveQueueUrlParams{ Topic: topic, SqsClient: p.sqs, Logger: p.logger, @@ -82,37 +81,37 @@ func (p *Publisher) GetOrCreateQueueUrl(ctx context.Context, topic string) (queu if err != nil { return "", "", err } - if exists { - return queueName, *queueUrl, nil + if resolvedQueue.Exists != nil && *resolvedQueue.Exists { + return resolvedQueue.QueueName, *resolvedQueue.QueueURL, nil } if !p.config.DoNotCreateQueueIfNotExists { - queueUrl, err := p.createQueue(ctx, topic, queueName, queueUrl) + queueUrl, err := p.createQueue(ctx, topic, resolvedQueue.QueueName) if err != nil { return "", "", err } - return queueName, queueUrl, nil + return resolvedQueue.QueueName, queueUrl, nil } else { return "", "", fmt.Errorf("queue for topic %s doesn't exist", topic) } } -func (p *Publisher) createQueue(ctx context.Context, topic string, queueName string, queueUrl *string) (string, error) { +func (p *Publisher) createQueue(ctx context.Context, topic string, queueName QueueName) (QueueURL, error) { input, err := p.config.GenerateCreateQueueInput(ctx, queueName, p.config.CreateQueueConfig) if err != nil { return "", fmt.Errorf("cannot generate create queue input: %w", err) } - queueUrl, err = createQueue(ctx, p.sqs, input) + queueUrl, err := createQueue(ctx, p.sqs, input) if err != nil { return "", fmt.Errorf("cannot create queue: %w", err) } // queue was created in the meantime // todo: it's quite ugly if queueUrl == nil { - _, queueUrl, exists, err := p.config.QueueUrlResolver.ResolveQueueUrl(ctx, ResolveQueueUrlParams{ + resolvedQueue, err := p.config.QueueUrlResolver.ResolveQueueUrl(ctx, ResolveQueueUrlParams{ Topic: topic, SqsClient: p.sqs, Logger: p.logger, @@ -120,26 +119,17 @@ func (p *Publisher) createQueue(ctx context.Context, topic string, queueName str if err != nil { return "", err } - if !exists { + if resolvedQueue.Exists != nil && !*resolvedQueue.Exists { return "", fmt.Errorf("queue doesn't exist after creation") } - return *queueUrl, nil + return *resolvedQueue.QueueURL, nil } return *queueUrl, nil } -// todo: move (together with other funcs) to url.go -func generateGetQueueUrlInputHash(getQueueInput *sqs.GetQueueUrlInput) string { - // we are not using fmt.Sprintf because of pointers under the hood - // we are not hashing specific struct fields to keep forward compatibility - // also, json.Marshal is faster than fmt.Sprintf - b, _ := json.Marshal(getQueueInput) - return string(b) -} - -func (p *Publisher) GetQueueArn(ctx context.Context, url *string) (*string, error) { +func (p *Publisher) GetQueueArn(ctx context.Context, url *QueueURL) (*string, error) { return getARNUrl(ctx, p.sqs, url) } diff --git a/sqs/pubsub_test.go b/sqs/pubsub_test.go index 1e32dbb..d590ca0 100644 --- a/sqs/pubsub_test.go +++ b/sqs/pubsub_test.go @@ -87,7 +87,7 @@ func TestPublishSubscribe_with_GenerateQueueUrlResolver(t *testing.T) { // Default value is 30 seconds - need to be lower for tests VisibilityTimeout: "1", }, - GenerateReceiveMessageInput: func(ctx context.Context, queueURL string) (*awssqs.ReceiveMessageInput, error) { + GenerateReceiveMessageInput: func(ctx context.Context, queueURL sqs.QueueURL) (*awssqs.ReceiveMessageInput, error) { in, err := sqs.GenerateReceiveMessageInputDefault(ctx, queueURL) if err != nil { return nil, err @@ -144,7 +144,7 @@ func TestPublishSubscribe_with_TransparentUrlResolver(t *testing.T) { // Default value is 30 seconds - need to be lower for tests VisibilityTimeout: "1", }, - GenerateReceiveMessageInput: func(ctx context.Context, queueURL string) (*awssqs.ReceiveMessageInput, error) { + GenerateReceiveMessageInput: func(ctx context.Context, queueURL sqs.QueueURL) (*awssqs.ReceiveMessageInput, error) { in, err := sqs.GenerateReceiveMessageInputDefault(ctx, queueURL) if err != nil { return nil, err @@ -195,7 +195,7 @@ func TestPublishSubscribe_batching(t *testing.T) { // Default value is 30 seconds - need to be lower for tests VisibilityTimeout: "1", }, - GenerateReceiveMessageInput: func(ctx context.Context, queueURL string) (*awssqs.ReceiveMessageInput, error) { + GenerateReceiveMessageInput: func(ctx context.Context, queueURL sqs.QueueURL) (*awssqs.ReceiveMessageInput, error) { in, err := sqs.GenerateReceiveMessageInputDefault(ctx, queueURL) if err != nil { return nil, err diff --git a/sqs/sqs.go b/sqs/sqs.go index a765d56..62a6717 100644 --- a/sqs/sqs.go +++ b/sqs/sqs.go @@ -9,13 +9,20 @@ import ( "github.com/pkg/errors" ) -func getQueueUrl(ctx context.Context, sqsClient *sqs.Client, topic string, input *sqs.GetQueueUrlInput) (*string, error) { +type QueueURL string + +type QueueName string + +func getQueueUrl(ctx context.Context, sqsClient *sqs.Client, topic string, input *sqs.GetQueueUrlInput) (*QueueURL, error) { getQueueOutput, err := sqsClient.GetQueueUrl(ctx, input) if err != nil || getQueueOutput.QueueUrl == nil { return nil, fmt.Errorf("cannot get queue %s: %w", topic, err) } - return getQueueOutput.QueueUrl, nil + + queueURL := QueueURL(*getQueueOutput.QueueUrl) + + return &queueURL, nil } // todo: wtf about that? @@ -23,7 +30,7 @@ func createQueue( ctx context.Context, sqsClient *sqs.Client, createQueueParams *sqs.CreateQueueInput, -) (*string, error) { +) (*QueueURL, error) { createQueueOutput, err := sqsClient.CreateQueue(ctx, createQueueParams) // possible scenarios: // 1. queue already exists, but with different params @@ -40,16 +47,20 @@ func createQueue( return nil, fmt.Errorf("cannot create queue, queueUrl is nil") } - return createQueueOutput.QueueUrl, nil + queueURL := QueueURL(*createQueueOutput.QueueUrl) + + return &queueURL, nil } -func getARNUrl(ctx context.Context, sqsClient *sqs.Client, url *string) (*string, error) { +func getARNUrl(ctx context.Context, sqsClient *sqs.Client, url *QueueURL) (*string, error) { if url == nil { return nil, fmt.Errorf("queue URL is nil") } + urlStr := string(*url) + attrResult, err := sqsClient.GetQueueAttributes(ctx, &sqs.GetQueueAttributesInput{ - QueueUrl: url, + QueueUrl: &urlStr, AttributeNames: []types.QueueAttributeName{ types.QueueAttributeNameQueueArn, }, diff --git a/sqs/subscriber.go b/sqs/subscriber.go index 52aaaad..43d4053 100644 --- a/sqs/subscriber.go +++ b/sqs/subscriber.go @@ -61,16 +61,17 @@ func (s *Subscriber) Subscribe(ctx context.Context, topic string) (<-chan *messa } // todo: what if doesn't exists? - queueName, queueURL, exists, err := s.config.QueueUrlResolver.ResolveQueueUrl(ctx, resolveQueueParams) + resolvedQueue, err := s.config.QueueUrlResolver.ResolveQueueUrl(ctx, resolveQueueParams) if err != nil { return nil, err } - if !exists { + // if we already know we are creating the queue - if not we'll create it later + if resolvedQueue.Exists != nil && !*resolvedQueue.Exists { if s.config.DoNotCreateQueueIfNotExists { return nil, fmt.Errorf("queue for topic '%s' doesn't exists", topic) } - input, err := s.config.GenerateCreateQueueInput(ctx, queueName, s.config.QueueConfigAttributes) + input, err := s.config.GenerateCreateQueueInput(ctx, resolvedQueue.QueueName, s.config.QueueConfigAttributes) if err != nil { return nil, fmt.Errorf("cannot generate input for queue %s: %w", topic, err) } @@ -81,25 +82,25 @@ func (s *Subscriber) Subscribe(ctx context.Context, topic string) (<-chan *messa } // todo: it's quite ugly - _, queueURL, _, err = s.config.QueueUrlResolver.ResolveQueueUrl(ctx, resolveQueueParams) + resolvedQueue, err = s.config.QueueUrlResolver.ResolveQueueUrl(ctx, resolveQueueParams) if err != nil { return nil, err } } - receiveInput, err := s.config.GenerateReceiveMessageInput(ctx, *queueURL) + receiveInput, err := s.config.GenerateReceiveMessageInput(ctx, *resolvedQueue.QueueURL) if err != nil { return nil, fmt.Errorf("cannot generate input for topic %s: %w", topic, err) } - s.logger.With(watermill.LogFields{"queue": *queueURL}).Info("Subscribing to queue", nil) + s.logger.With(watermill.LogFields{"queue": *resolvedQueue.QueueURL}).Info("Subscribing to queue", nil) ctx, cancel := context.WithCancel(ctx) s.subscribersWg.Add(1) output := make(chan *message.Message) go func() { - s.receive(ctx, *queueURL, output, receiveInput) + s.receive(ctx, *resolvedQueue.QueueURL, output, receiveInput) close(output) cancel() }() @@ -107,7 +108,7 @@ func (s *Subscriber) Subscribe(ctx context.Context, topic string) (<-chan *messa return output, nil } -func (s *Subscriber) receive(ctx context.Context, queueURL string, output chan *message.Message, input *sqs.ReceiveMessageInput) { +func (s *Subscriber) receive(ctx context.Context, queueURL QueueURL, output chan *message.Message, input *sqs.ReceiveMessageInput) { defer s.subscribersWg.Done() ctx, cancelCtx := context.WithCancel(ctx) defer cancelCtx() @@ -160,7 +161,7 @@ func (s *Subscriber) receive(ctx context.Context, queueURL string, output chan * func (s *Subscriber) consumeMessages( ctx context.Context, messages []types.Message, - queueURL string, + queueURL QueueURL, output chan *message.Message, logFields watermill.LogFields, ) { @@ -178,7 +179,7 @@ func (s *Subscriber) processMessage( logFields watermill.LogFields, sqsMsg types.Message, output chan *message.Message, - queueURL string, + queueURL QueueURL, ) bool { logger := s.logger.With(logFields) logger.Trace("processMessage", nil) @@ -224,7 +225,7 @@ func (s *Subscriber) processMessage( return true } -func (s *Subscriber) deleteMessage(ctx context.Context, queueURL string, receiptHandle *string, logFields watermill.LogFields) error { +func (s *Subscriber) deleteMessage(ctx context.Context, queueURL QueueURL, receiptHandle *string, logFields watermill.LogFields) error { input, err := s.config.GenerateDeleteMessageInput(ctx, queueURL, receiptHandle) if err != nil { return fmt.Errorf("cannot generate input for delete message: %w", err) @@ -272,7 +273,7 @@ func (s *Subscriber) SubscribeInitialize(topic string) error { } func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, topic string) error { - queueName, _, exists, err := s.config.QueueUrlResolver.ResolveQueueUrl(ctx, ResolveQueueUrlParams{ + resolvedQueue, err := s.config.QueueUrlResolver.ResolveQueueUrl(ctx, ResolveQueueUrlParams{ Topic: topic, SqsClient: s.sqs, Logger: s.logger, @@ -280,11 +281,11 @@ func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, topic s if err != nil { return err } - if exists { + if resolvedQueue.Exists != nil && *resolvedQueue.Exists { return nil } - input, err := s.config.GenerateCreateQueueInput(ctx, queueName, s.config.QueueConfigAttributes) + input, err := s.config.GenerateCreateQueueInput(ctx, resolvedQueue.QueueName, s.config.QueueConfigAttributes) if err != nil { return fmt.Errorf("cannot generate input for queue %s: %w", topic, err) } @@ -298,7 +299,7 @@ func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, topic s } // todo: duplicated in subscribe? -func (s *Subscriber) GetQueueUrl(ctx context.Context, topic string) (*string, error) { +func (s *Subscriber) GetQueueUrl(ctx context.Context, topic string) (*QueueURL, error) { resolveQueueParams := ResolveQueueUrlParams{ Topic: topic, SqsClient: s.sqs, @@ -306,15 +307,18 @@ func (s *Subscriber) GetQueueUrl(ctx context.Context, topic string) (*string, er } // todo: what if doesn't exists? - _, queueURL, _, err := s.config.QueueUrlResolver.ResolveQueueUrl(ctx, resolveQueueParams) + resolvedQueue, err := s.config.QueueUrlResolver.ResolveQueueUrl(ctx, resolveQueueParams) if err != nil { return nil, fmt.Errorf("cannot generate input for queue %s: %w", topic, err) } + if resolvedQueue.Exists != nil && !*resolvedQueue.Exists { + return nil, fmt.Errorf("queue for topic '%s' doesn't exist", topic) + } - return queueURL, nil + return resolvedQueue.QueueURL, nil } -func (s *Subscriber) GetQueueArn(ctx context.Context, url *string) (*string, error) { +func (s *Subscriber) GetQueueArn(ctx context.Context, url *QueueURL) (*string, error) { return getARNUrl(ctx, s.sqs, url) } diff --git a/sqs/url.go b/sqs/url_resolver.go similarity index 51% rename from sqs/url.go rename to sqs/url_resolver.go index 2268fdd..b1870a0 100644 --- a/sqs/url.go +++ b/sqs/url_resolver.go @@ -2,6 +2,7 @@ package sqs import ( "context" + "encoding/json" "errors" "fmt" "strings" @@ -14,7 +15,7 @@ import ( ) type QueueUrlResolver interface { - ResolveQueueUrl(ctx context.Context, params ResolveQueueUrlParams) (queueName string, sqsQueueUrl *string, exists bool, err error) + ResolveQueueUrl(ctx context.Context, params ResolveQueueUrlParams) (QueueUrlResolverResult, error) } type ResolveQueueUrlParams struct { @@ -24,6 +25,17 @@ type ResolveQueueUrlParams struct { Logger watermill.LoggerAdapter } +type QueueUrlResolverResult struct { + QueueName QueueName + + // QueueURL is not present if queue doesn't exist. + QueueURL *QueueURL + + // Exists says if queue exists. + // May be nil, if resolver doesn't have information about queue existence. + Exists *bool +} + // todo: add alternative one that statically generates queue name type GenerateQueueUrlResolver struct { @@ -31,11 +43,17 @@ type GenerateQueueUrlResolver struct { AwsAccountID string } -func (p GenerateQueueUrlResolver) ResolveQueueUrl(ctx context.Context, params ResolveQueueUrlParams) (queueName string, queueUrl *string, exists bool, err error) { - queueURL := fmt.Sprintf("https://sqs.%s.amazonaws.com/%s/%s", p.AwsRegion, p.AwsAccountID, params.Topic) +func (p GenerateQueueUrlResolver) ResolveQueueUrl(ctx context.Context, params ResolveQueueUrlParams) (QueueUrlResolverResult, error) { + queueURL := QueueURL(fmt.Sprintf( + "https://sqs.%s.amazonaws.com/%s/%s", + p.AwsRegion, p.AwsAccountID, params.Topic, + )) - // todo: how to deal with fact that exists is always true? what if it doesn't exist? - return params.Topic, &queueURL, true, nil + return QueueUrlResolverResult{ + QueueName: QueueName(params.Topic), // in this case topic maps 1:1 to topic name + QueueURL: &queueURL, + Exists: nil, // we don't know + }, nil } type GetQueueUrlByNameUrlResolverConfig struct { @@ -50,7 +68,7 @@ type GetQueueUrlByNameUrlResolverConfig struct { type GetQueueUrlByNameUrlResolver struct { config GetQueueUrlByNameUrlResolverConfig - queuesCache map[string]*string + queuesCache map[string]QueueURL queuesCacheLock sync.RWMutex } @@ -63,14 +81,14 @@ func NewGetQueueUrlByNameUrlResolver( return &GetQueueUrlByNameUrlResolver{ config: config, - queuesCache: map[string]*string{}, + queuesCache: map[string]QueueURL{}, } } -func (p *GetQueueUrlByNameUrlResolver) ResolveQueueUrl(ctx context.Context, params ResolveQueueUrlParams) (queueName string, queueUrl *string, exists bool, err error) { +func (p *GetQueueUrlByNameUrlResolver) ResolveQueueUrl(ctx context.Context, params ResolveQueueUrlParams) (res QueueUrlResolverResult, err error) { getQueueInput, err := p.config.GenerateGetQueueUrlInput(ctx, params.Topic) if err != nil { - return params.Topic, nil, false, fmt.Errorf("cannot generate input for queue %s: %w", params.Topic, err) + return QueueUrlResolverResult{}, fmt.Errorf("cannot generate input for queue %s: %w", params.Topic, err) } var getQueueInputHash string @@ -81,45 +99,73 @@ func (p *GetQueueUrlByNameUrlResolver) ResolveQueueUrl(ctx context.Context, para if getQueueInputHash != "" { p.queuesCacheLock.RLock() var ok bool - queueUrl, ok = p.queuesCache[getQueueInputHash] + queueUrl, ok := p.queuesCache[getQueueInputHash] p.queuesCacheLock.RUnlock() if ok { params.Logger.Trace("Used cache", watermill.LogFields{ "topic": params.Topic, - "queue": *queueUrl, + "queue": queueUrl, "hash": getQueueInputHash, }) - return params.Topic, queueUrl, true, nil + + // it can be present in cache only if it was created + exists := true + + return QueueUrlResolverResult{ + QueueName: QueueName(params.Topic), // in this scenario topic maps to queue name + QueueURL: &queueUrl, + Exists: &exists, + }, nil } } } defer func() { - if err == nil && exists && getQueueInputHash != "" { + if err == nil && res.QueueURL != nil && *res.QueueURL != "" && getQueueInputHash != "" { p.queuesCacheLock.Lock() - p.queuesCache[getQueueInputHash] = queueUrl + p.queuesCache[getQueueInputHash] = *res.QueueURL p.queuesCacheLock.Unlock() params.Logger.Trace("Stored cache", watermill.LogFields{ - "topic": params.Topic, - "queue": *queueUrl, - "hash": getQueueInputHash, + "topic": params.Topic, + "queue_url": res.QueueURL, + "hash": getQueueInputHash, }) } }() - queueUrl, err = getQueueUrl(ctx, params.SqsClient, params.Topic, getQueueInput) + // it can be present in cache only if it was created + + queueUrl, err := getQueueUrl(ctx, params.SqsClient, params.Topic, getQueueInput) if err == nil { - return params.Topic, queueUrl, true, nil + exists := true + + return QueueUrlResolverResult{ + QueueName: QueueName(params.Topic), // in this scenario topic maps to queue name + QueueURL: queueUrl, + Exists: &exists, + }, nil } var queueDoesNotExistsErr *types.QueueDoesNotExist if errors.As(err, &queueDoesNotExistsErr) { - // todo: this is not a good idea? - return params.Topic, queueUrl, false, nil + exists := false + + return QueueUrlResolverResult{ + QueueName: QueueName(params.Topic), // in this scenario topic maps to queue name + Exists: &exists, + }, nil } - return params.Topic, nil, false, err + return QueueUrlResolverResult{}, err +} + +func generateGetQueueUrlInputHash(getQueueInput *sqs.GetQueueUrlInput) string { + // we are not using fmt.Sprintf because of pointers under the hood + // we are not hashing specific struct fields to keep forward compatibility + // also, json.Marshal is faster than fmt.Sprintf + b, _ := json.Marshal(getQueueInput) + return string(b) } type GenerateGetQueueUrlInputFunc func(ctx context.Context, topic string) (*sqs.GetQueueUrlInput, error) @@ -133,9 +179,15 @@ func GenerateGetQueueUrlInputDefault(ctx context.Context, topic string) (*sqs.Ge // todo: test type TransparentUrlResolver struct{} -func (p TransparentUrlResolver) ResolveQueueUrl(ctx context.Context, params ResolveQueueUrlParams) (queueName string, queueUrl *string, exists bool, err error) { +func (p TransparentUrlResolver) ResolveQueueUrl(ctx context.Context, params ResolveQueueUrlParams) (res QueueUrlResolverResult, err error) { topicParts := strings.Split(params.Topic, "/") - queueName = topicParts[len(topicParts)-1] + queueName := topicParts[len(topicParts)-1] + + queueURL := QueueURL(params.Topic) - return queueName, ¶ms.Topic, true, nil + return QueueUrlResolverResult{ + QueueName: QueueName(queueName), + QueueURL: &queueURL, // in this case topic maps to queue URL + Exists: nil, // we don't know + }, nil } From 5974b5f204224991d8d618d2ad2edb0e04aae6d5 Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Mon, 26 Aug 2024 16:26:28 +0200 Subject: [PATCH 15/19] downgrade watermill temporarly --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f00b2d2..546d75c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ThreeDotsLabs/watermill-amazonsqs go 1.21 require ( - github.com/ThreeDotsLabs/watermill v1.3.5 + github.com/ThreeDotsLabs/watermill v1.2.0 github.com/aws/aws-sdk-go-v2 v1.30.4 github.com/aws/aws-sdk-go-v2/config v1.27.28 github.com/aws/aws-sdk-go-v2/credentials v1.17.28 From 67fc34dda5b513af044e67997c357071c4569bc6 Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Thu, 5 Sep 2024 17:46:32 +0200 Subject: [PATCH 16/19] fix locks and add support for setting queue access policy --- sns/config.go | 42 +++++++++++++++++++++++++++++++++++++- sns/subscriber.go | 51 +++++++++++++++++++++++++++++++++++++++++------ sqs/subscriber.go | 6 +++++- 3 files changed, 91 insertions(+), 8 deletions(-) diff --git a/sns/config.go b/sns/config.go index d258327..82e1b7f 100644 --- a/sns/config.go +++ b/sns/config.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" + "github.com/ThreeDotsLabs/watermill-amazonsqs/sqs" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/sns" ) @@ -63,20 +64,30 @@ func GenerateCreateTopicInputDefault(ctx context.Context, topic string, attrs Co type SubscriberConfig struct { AWSConfig aws.Config - // todo: better name? + // todo: better name? define signature GenerateSqsQueueName func(ctx context.Context, snsTopic string) (string, error) TopicResolver TopicResolver GenerateSubscribeInput GenerateSubscribeInputFn + GenerateQueueAccessPolicy GenerateQueueAccessPolicyFn + + // todo: rename? DoNotSubscribeToSns bool + + // todo: info what perm it requires ("sqs:SetQueueAttributes"?) + // link to docs + DoNotSetQueueAccessPolicy bool } func (c *SubscriberConfig) SetDefaults() { if c.GenerateSubscribeInput == nil { c.GenerateSubscribeInput = GenerateSubscribeInputDefault } + if c.GenerateQueueAccessPolicy == nil { + c.GenerateQueueAccessPolicy = GenerateQueueAccessPolicyDefault + } } func (c *SubscriberConfig) Validate() error { @@ -124,6 +135,35 @@ func GenerateSubscribeInputDefault(ctx context.Context, params GenerateSubscribe }, nil } +type GenerateQueueAccessPolicyFn func(ctx context.Context, params GenerateQueueAccessPolicyParams) (map[string]any, error) + +type GenerateQueueAccessPolicyParams struct { + SqsQueueArn string + SnsTopicArn string + SqsURL sqs.QueueURL +} + +func GenerateQueueAccessPolicyDefault(ctx context.Context, params GenerateQueueAccessPolicyParams) (map[string]any, error) { + return map[string]any{ + "Version": "2012-10-17", + "Statement": []map[string]any{ + { + "Effect": "Allow", + "Principal": map[string]string{ + "Service": "sns.amazonaws.com", + }, + "Action": "sqs:SendMessage", + "Resource": params.SqsQueueArn, + "Condition": map[string]any{ + "ArnEquals": map[string]string{ + "aws:SourceArn": params.SnsTopicArn, + }, + }, + }, + }, + }, nil +} + // ConfigAttributes is a struct that holds the attributes of an SNS topic type ConfigAttributes struct { // DeliveryPolicy – The policy that defines how Amazon SNS retries failed diff --git a/sns/subscriber.go b/sns/subscriber.go index 5de7975..d8047ec 100644 --- a/sns/subscriber.go +++ b/sns/subscriber.go @@ -2,20 +2,24 @@ package sns import ( "context" + "encoding/json" "fmt" "github.com/ThreeDotsLabs/watermill" "github.com/ThreeDotsLabs/watermill-amazonsqs/sqs" "github.com/ThreeDotsLabs/watermill/message" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/sns" + awsSqs "github.com/aws/aws-sdk-go-v2/service/sqs" ) type Subscriber struct { config SubscriberConfig logger watermill.LoggerAdapter - sns *sns.Client - sqs *sqs.Subscriber + sns *sns.Client + sqs *sqs.Subscriber + sqsClient *awsSqs.Client } func NewSubscriber( @@ -42,10 +46,11 @@ func NewSubscriber( } return &Subscriber{ - config: config, - logger: logger, - sns: sns.NewFromConfig(config.AWSConfig), - sqs: sqs, + config: config, + logger: logger, + sns: sns.NewFromConfig(config.AWSConfig), + sqsClient: awsSqs.NewFromConfig(config.AWSConfig), + sqs: sqs, }, nil } @@ -97,6 +102,12 @@ func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, topic s return fmt.Errorf("cannot get queue ARN for topic %s: %w", snsTopicArn, err) } + if !s.config.DoNotSetQueueAccessPolicy { + if err := s.setSqsQuePolicy(ctx, *sqsQueueArn, snsTopicArn, *sqsURL); err != nil { + return fmt.Errorf("cannot set queue access policy for topic %s: %w", snsTopicArn, err) + } + } + s.logger.Info("Subscribing to SNS", watermill.LogFields{ "sns_topic_arn": snsTopicArn, "sqs_topic": sqsTopic, @@ -118,6 +129,34 @@ func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, topic s return nil } +func (s *Subscriber) setSqsQuePolicy(ctx context.Context, sqsQueueArn string, snsTopicArn string, sqsURL sqs.QueueURL) error { + policy, err := s.config.GenerateQueueAccessPolicy(ctx, GenerateQueueAccessPolicyParams{ + SqsQueueArn: sqsQueueArn, + SnsTopicArn: snsTopicArn, + SqsURL: sqsURL, + }) + + policyJSON, err := json.Marshal(policy) + if err != nil { + return fmt.Errorf("cannot marshal policy: %w", err) + } + + s.logger.Debug("Setting queue access policy", watermill.LogFields{ + "policy": string(policyJSON), + }) + + _, err = s.sqsClient.SetQueueAttributes(ctx, &awsSqs.SetQueueAttributesInput{ + QueueUrl: aws.String(string(sqsURL)), + Attributes: map[string]string{ + "Policy": string(policyJSON), + }, + }) + if err != nil { + return fmt.Errorf("cannot set queue policy: %w", err) + } + return nil +} + func (s *Subscriber) Close() error { return s.sqs.Close() } diff --git a/sqs/subscriber.go b/sqs/subscriber.go index 43d4053..3101438 100644 --- a/sqs/subscriber.go +++ b/sqs/subscriber.go @@ -22,7 +22,8 @@ type Subscriber struct { closing chan struct{} subscribersWg sync.WaitGroup - closed bool + closed bool + closedLock sync.Mutex } func NewSubscriber(config SubscriberConfig, logger watermill.LoggerAdapter) (*Subscriber, error) { @@ -256,6 +257,9 @@ func (s *Subscriber) deleteMessage(ctx context.Context, queueURL QueueURL, recei } func (s *Subscriber) Close() error { + s.closedLock.Lock() + defer s.closedLock.Unlock() + if s.closed { return nil } From af4e21e1865281cdfbd7a2a9759f7a21af9b941e Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Thu, 5 Sep 2024 21:47:43 +0200 Subject: [PATCH 17/19] cleanups --- cmd/sns-sqs/main.go | 98 ---------------------------- cmd/sqs-sqs/main.go | 71 -------------------- {connection => internal}/endpoint.go | 2 +- sns/config.go | 39 +++++------ sns/marshaler.go | 6 +- sns/publisher.go | 2 +- sns/pubsub_test.go | 6 +- sns/sns.go | 14 ++-- sns/subscriber.go | 4 +- sns/topic.go | 8 +-- sqs/publisher.go | 14 ++-- sqs/pubsub_test.go | 8 +-- sqs/sqs.go | 7 +- sqs/subscriber.go | 12 +--- sqs/url_resolver.go | 6 +- 15 files changed, 61 insertions(+), 236 deletions(-) delete mode 100644 cmd/sns-sqs/main.go delete mode 100644 cmd/sqs-sqs/main.go rename {connection => internal}/endpoint.go (96%) diff --git a/cmd/sns-sqs/main.go b/cmd/sns-sqs/main.go deleted file mode 100644 index 9e543cf..0000000 --- a/cmd/sns-sqs/main.go +++ /dev/null @@ -1,98 +0,0 @@ -package main - -import ( - "context" - "os" - "time" - - "github.com/ThreeDotsLabs/watermill-amazonsqs/connection" - "github.com/ThreeDotsLabs/watermill-amazonsqs/sqs" - "github.com/aws/aws-sdk-go-v2/aws" - awsconfig "github.com/aws/aws-sdk-go-v2/config" - awssns "github.com/aws/aws-sdk-go-v2/service/sns" - - "github.com/ThreeDotsLabs/watermill" - "github.com/ThreeDotsLabs/watermill/message" - - "github.com/ThreeDotsLabs/watermill-amazonsqs/sns" -) - -const SNS_TOPIC = "local-topic1" -const SQS_QUEUE = "local-queue4" - -func main() { - logger := watermill.NewStdLogger(true, true) - - cfg, err := awsconfig.LoadDefaultConfig( - context.Background(), - awsconfig.WithRegion("eu-north-1"), - connection.SetEndPoint(os.Getenv("AWS_SNS_ENDPOINT")), - ) - if err != nil { - panic(err) - } - - pub, err := sns.NewPublisher(sns.PublisherConfig{ - AWSConfig: cfg, - CreateTopicfNotExists: true, - }, logger) - if err != nil { - panic(err) - } - - sub, err := sqs.NewSubscriber(sqs.SubscriberConfig{ - AWSConfig: cfg, - }, logger) - if err != nil { - panic(err) - } - - ctx := context.Background() - - messages, err := sub.Subscribe(ctx, SQS_QUEUE) - if err != nil { - panic(err) - } - - pubArn, err := pub.GetArnTopic(ctx, SNS_TOPIC) - if err != nil { - panic(err) - } - sqsUrl, err := sub.GetQueueUrl(ctx, SQS_QUEUE) - if err != nil { - panic(err) - } - sqsArn, err := sub.GetQueueArn(ctx, sqsUrl) - if err != nil { - panic(err) - } - - err = pub.AddSubscription(ctx, &awssns.SubscribeInput{ - Protocol: aws.String("sqs"), - TopicArn: pubArn, - Endpoint: sqsArn, - Attributes: map[string]string{ - "RawMessageDelivery": "true", - }, - }) - if err != nil { - panic(err) - } - - // Start consuming messages from SQS - go func() { - for m := range messages { - logger.With(watermill.LogFields{"message": string(m.Payload)}).Info("Received message", nil) - m.Ack() - } - }() - // Start sending messages to SNS - for { - msg := message.NewMessage(watermill.NewULID(), []byte(`{"some_json": "body"}`)) - err := pub.Publish(SNS_TOPIC, msg) - if err != nil { - panic(err) - } - time.Sleep(time.Second) - } -} diff --git a/cmd/sqs-sqs/main.go b/cmd/sqs-sqs/main.go deleted file mode 100644 index cae5285..0000000 --- a/cmd/sqs-sqs/main.go +++ /dev/null @@ -1,71 +0,0 @@ -package main - -import ( - "context" - "os" - "time" - - "github.com/ThreeDotsLabs/watermill" - "github.com/ThreeDotsLabs/watermill/message" - awsconfig "github.com/aws/aws-sdk-go-v2/config" - - "github.com/ThreeDotsLabs/watermill-amazonsqs/connection" - "github.com/ThreeDotsLabs/watermill-amazonsqs/sqs" -) - -func main() { - ctx := context.Background() - logger := watermill.NewStdLogger(true, true) - cfg, err := awsconfig.LoadDefaultConfig( - context.Background(), - awsconfig.WithRegion("eu-north-1"), - connection.SetEndPoint(os.Getenv("AWS_SNS_ENDPOINT")), - ) - if err != nil { - panic(err) - } - pub, err := sqs.NewPublisher(sqs.PublisherConfig{ - AWSConfig: cfg, - CreateQueueIfNotExists: true, - Marshaler: sqs.DefaultMarshalerUnmarshaler{}, - }, logger) - if err != nil { - panic(err) - } - _ = pub - - sub, err := sqs.NewSubscriber(sqs.SubscriberConfig{ - AWSConfig: cfg, - CreateQueueInitializerConfig: sqs.QueueConfigAttributes{}, - Unmarshaler: sqs.DefaultMarshalerUnmarshaler{}, - }, logger) - if err != nil { - panic(err) - } - - err = sub.SubscribeInitialize("any-topic") - if err != nil { - panic(err) - } - - messages, err := sub.Subscribe(ctx, "any-topic") - if err != nil { - panic(err) - } - - go func() { - for m := range messages { - logger.With(watermill.LogFields{"message": m}).Info("Received message", nil) - m.Ack() - } - }() - - for { - msg := message.NewMessage(watermill.NewULID(), []byte(`{"some_json": "body"}`)) - err := pub.Publish("any-topic", msg) - if err != nil { - panic(err) - } - time.Sleep(time.Second) - } -} diff --git a/connection/endpoint.go b/internal/endpoint.go similarity index 96% rename from connection/endpoint.go rename to internal/endpoint.go index 9ca4336..57d8d5b 100644 --- a/connection/endpoint.go +++ b/internal/endpoint.go @@ -1,4 +1,4 @@ -package connection +package internal import ( "github.com/aws/aws-sdk-go-v2/aws" diff --git a/sns/config.go b/sns/config.go index 82e1b7f..c3a7ad3 100644 --- a/sns/config.go +++ b/sns/config.go @@ -47,16 +47,16 @@ func (c *PublisherConfig) Validate() error { return err } -type GenerateCreateTopicInputFunc func(ctx context.Context, topic string, attrs ConfigAttributes) (sns.CreateTopicInput, error) +type GenerateCreateTopicInputFunc func(ctx context.Context, topic TopicName, attrs ConfigAttributes) (sns.CreateTopicInput, error) -func GenerateCreateTopicInputDefault(ctx context.Context, topic string, attrs ConfigAttributes) (sns.CreateTopicInput, error) { +func GenerateCreateTopicInputDefault(ctx context.Context, topic TopicName, attrs ConfigAttributes) (sns.CreateTopicInput, error) { attrsMap, err := attrs.Attributes() if err != nil { return sns.CreateTopicInput{}, fmt.Errorf("cannot generate attributes for topic %s: %w", topic, err) } return sns.CreateTopicInput{ - Name: aws.String(topic), + Name: aws.String(string(topic)), Attributes: attrsMap, }, nil } @@ -64,20 +64,19 @@ func GenerateCreateTopicInputDefault(ctx context.Context, topic string, attrs Co type SubscriberConfig struct { AWSConfig aws.Config - // todo: better name? define signature - GenerateSqsQueueName func(ctx context.Context, snsTopic string) (string, error) - TopicResolver TopicResolver + GenerateSqsQueueName GenerateSqsQueueNameFn + GenerateSubscribeInput GenerateSubscribeInputFn GenerateQueueAccessPolicy GenerateQueueAccessPolicyFn - // todo: rename? - DoNotSubscribeToSns bool + DoNotCreateSqsSubscription bool - // todo: info what perm it requires ("sqs:SetQueueAttributes"?) - // link to docs + // DoNotSetQueueAccessPolicy disables setting the queue access policy. + // Described in AWS docs: https://docs.aws.amazon.com/sns/latest/dg/subscribe-sqs-queue-to-sns-topic.html#SendMessageToSQS.sqs.permissions + // It requires "sqs:SetQueueAttributes" permission. DoNotSetQueueAccessPolicy bool } @@ -106,13 +105,15 @@ func (c *SubscriberConfig) Validate() error { return err } -func GenerateSqsQueueNameEqualToTopicName(ctx context.Context, snsTopic string) (string, error) { +type GenerateSqsQueueNameFn func(ctx context.Context, snsTopic TopicArn) (string, error) + +func GenerateSqsQueueNameEqualToTopicName(ctx context.Context, snsTopic TopicArn) (string, error) { topicName, err := ExtractTopicNameFromTopicArn(snsTopic) if err != nil { return "", err } - return topicName, nil + return string(topicName), nil } type GenerateSubscribeInputFn func(ctx context.Context, params GenerateSubscribeInputParams) (*sns.SubscribeInput, error) @@ -120,15 +121,15 @@ type GenerateSubscribeInputFn func(ctx context.Context, params GenerateSubscribe type GenerateSubscribeInputParams struct { SqsTopic string - SnsTopicArn string - SqsQueueArn string + SnsTopicArn TopicArn + SqsQueueArn sqs.QueueArn } func GenerateSubscribeInputDefault(ctx context.Context, params GenerateSubscribeInputParams) (*sns.SubscribeInput, error) { return &sns.SubscribeInput{ Protocol: aws.String("sqs"), - TopicArn: ¶ms.SnsTopicArn, - Endpoint: ¶ms.SqsQueueArn, + TopicArn: aws.String(string(params.SnsTopicArn)), + Endpoint: aws.String(string(params.SqsQueueArn)), Attributes: map[string]string{ "RawMessageDelivery": "true", }, @@ -138,8 +139,8 @@ func GenerateSubscribeInputDefault(ctx context.Context, params GenerateSubscribe type GenerateQueueAccessPolicyFn func(ctx context.Context, params GenerateQueueAccessPolicyParams) (map[string]any, error) type GenerateQueueAccessPolicyParams struct { - SqsQueueArn string - SnsTopicArn string + SqsQueueArn sqs.QueueArn + SnsTopicArn TopicArn SqsURL sqs.QueueURL } @@ -156,7 +157,7 @@ func GenerateQueueAccessPolicyDefault(ctx context.Context, params GenerateQueueA "Resource": params.SqsQueueArn, "Condition": map[string]any{ "ArnEquals": map[string]string{ - "aws:SourceArn": params.SnsTopicArn, + "aws:SourceArn": string(params.SnsTopicArn), }, }, }, diff --git a/sns/marshaler.go b/sns/marshaler.go index 550cd2f..9777309 100644 --- a/sns/marshaler.go +++ b/sns/marshaler.go @@ -12,12 +12,12 @@ import ( const UUIDAttribute = "UUID" type Marshaler interface { - Marshal(topicArn string, msg *message.Message) *sns.PublishInput + Marshal(topicArn TopicArn, msg *message.Message) *sns.PublishInput } type DefaultMarshalerUnmarshaler struct{} -func (d DefaultMarshalerUnmarshaler) Marshal(topicArn string, msg *message.Message) *sns.PublishInput { +func (d DefaultMarshalerUnmarshaler) Marshal(topicArn TopicArn, msg *message.Message) *sns.PublishInput { // client side uuid // there is a deduplication id that can be use for // fifo queues @@ -33,7 +33,7 @@ func (d DefaultMarshalerUnmarshaler) Marshal(topicArn string, msg *message.Messa MessageAttributes: attributes, MessageDeduplicationId: deduplicationId, MessageGroupId: groupId, - TargetArn: &topicArn, + TargetArn: aws.String(string(topicArn)), } } diff --git a/sns/publisher.go b/sns/publisher.go index e09a9cb..642bbe7 100644 --- a/sns/publisher.go +++ b/sns/publisher.go @@ -94,7 +94,7 @@ func (p *Publisher) CreateTopic(ctx context.Context, topic string) (string, erro return "", fmt.Errorf("created topic arn is nil") } - if *createdTopicArn != topicArn { + if *createdTopicArn != string(topicArn) { return "", fmt.Errorf( "created topic arn (%s) is not equal to expected topic arn (%s), please check the configuration", *createdTopicArn, diff --git a/sns/pubsub_test.go b/sns/pubsub_test.go index 26b2139..d40e66b 100644 --- a/sns/pubsub_test.go +++ b/sns/pubsub_test.go @@ -5,6 +5,7 @@ import ( "fmt" "testing" + "github.com/ThreeDotsLabs/watermill-amazonsqs/internal" "github.com/ThreeDotsLabs/watermill-amazonsqs/sqs" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/stretchr/testify/assert" @@ -13,7 +14,6 @@ import ( "github.com/stretchr/testify/require" "github.com/ThreeDotsLabs/watermill" - "github.com/ThreeDotsLabs/watermill-amazonsqs/connection" "github.com/ThreeDotsLabs/watermill-amazonsqs/sns" "github.com/ThreeDotsLabs/watermill/message" "github.com/ThreeDotsLabs/watermill/pubsub/tests" @@ -148,7 +148,7 @@ func createPubSubWithConsumerGroup(t *testing.T, consumerGroup string) (message. }, sns.SubscriberConfig{ AWSConfig: cfg, - GenerateSqsQueueName: func(ctx context.Context, sqsTopic string) (string, error) { + GenerateSqsQueueName: func(ctx context.Context, sqsTopic sns.TopicArn) (string, error) { return consumerGroup, nil }, TopicResolver: topicResolver, @@ -185,7 +185,7 @@ func GetAWSConfig(t *testing.T) aws.Config { cfg, err := awsconfig.LoadDefaultConfig( context.Background(), - connection.SetEndPoint("http://localhost:4566"), + internal.SetEndPoint("http://localhost:4566"), awsconfig.WithRegion("us-west-2"), ) require.NoError(t, err) diff --git a/sns/sns.go b/sns/sns.go index 63c1517..d937638 100644 --- a/sns/sns.go +++ b/sns/sns.go @@ -9,6 +9,10 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sns" ) +type TopicName string + +type TopicArn string + func createSnsTopic(ctx context.Context, snsClient *sns.Client, createSNSParams sns.CreateTopicInput) (*string, error) { createSNSOutput, err := snsClient.CreateTopic(ctx, &createSNSParams) if err != nil || createSNSOutput.TopicArn == nil { @@ -17,7 +21,7 @@ func createSnsTopic(ctx context.Context, snsClient *sns.Client, createSNSParams return createSNSOutput.TopicArn, nil } -func GenerateTopicArn(region, accountID, topic string) (string, error) { +func GenerateTopicArn(region, accountID, topic string) (TopicArn, error) { var err error if region == "" { err = errors.Join(err, fmt.Errorf("region is empty")) @@ -32,15 +36,15 @@ func GenerateTopicArn(region, accountID, topic string) (string, error) { return "", fmt.Errorf("can't generate topic arn: %w", err) } - return fmt.Sprintf("arn:aws:sns:%s:%s:%s", region, accountID, topic), nil + return TopicArn(fmt.Sprintf("arn:aws:sns:%s:%s:%s", region, accountID, topic)), nil } -func ExtractTopicNameFromTopicArn(topicArn string) (string, error) { - topicArnParts := strings.Split(topicArn, ":") +func ExtractTopicNameFromTopicArn(topicArn TopicArn) (TopicName, error) { + topicArnParts := strings.Split(string(topicArn), ":") if len(topicArnParts) != 6 { return "", fmt.Errorf("topic arn should have 6 segments, has %d (%s)", len(topicArnParts), topicArn) } topicName := topicArnParts[5] - return topicName, nil + return TopicName(topicName), nil } diff --git a/sns/subscriber.go b/sns/subscriber.go index d8047ec..233c8f5 100644 --- a/sns/subscriber.go +++ b/sns/subscriber.go @@ -60,7 +60,7 @@ func (s *Subscriber) Subscribe(ctx context.Context, topic string) (<-chan *messa return nil, err } - if !s.config.DoNotSubscribeToSns { + if !s.config.DoNotCreateSqsSubscription { if err := s.SubscribeInitializeWithContext(ctx, topic); err != nil { return nil, err } @@ -129,7 +129,7 @@ func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, topic s return nil } -func (s *Subscriber) setSqsQuePolicy(ctx context.Context, sqsQueueArn string, snsTopicArn string, sqsURL sqs.QueueURL) error { +func (s *Subscriber) setSqsQuePolicy(ctx context.Context, sqsQueueArn sqs.QueueArn, snsTopicArn TopicArn, sqsURL sqs.QueueURL) error { policy, err := s.config.GenerateQueueAccessPolicy(ctx, GenerateQueueAccessPolicyParams{ SqsQueueArn: sqsQueueArn, SnsTopicArn: snsTopicArn, diff --git a/sns/topic.go b/sns/topic.go index 1953a14..fce6c32 100644 --- a/sns/topic.go +++ b/sns/topic.go @@ -7,14 +7,14 @@ import ( ) type TopicResolver interface { - ResolveTopic(ctx context.Context, topic string) (snsTopic string, err error) + ResolveTopic(ctx context.Context, topic string) (snsTopic TopicArn, err error) } type TransparentTopicResolver struct{} -func (a TransparentTopicResolver) ResolveTopic(ctx context.Context, topic string) (snsTopic string, err error) { +func (a TransparentTopicResolver) ResolveTopic(ctx context.Context, topic string) (snsTopic TopicArn, err error) { // we are passing topic ARN as topic - return topic, nil + return TopicArn(topic), nil } type GenerateArnTopicResolver struct { @@ -39,6 +39,6 @@ func NewGenerateArnTopicResolver(accountID string, region string) (*GenerateArnT return &GenerateArnTopicResolver{accountID: accountID, region: region}, nil } -func (g GenerateArnTopicResolver) ResolveTopic(ctx context.Context, topic string) (snsTopic string, err error) { +func (g GenerateArnTopicResolver) ResolveTopic(ctx context.Context, topic string) (snsTopic TopicArn, err error) { return GenerateTopicArn(g.region, g.accountID, topic) } diff --git a/sqs/publisher.go b/sqs/publisher.go index 5b039b2..36bdc30 100644 --- a/sqs/publisher.go +++ b/sqs/publisher.go @@ -34,7 +34,7 @@ func NewPublisher(config PublisherConfig, logger watermill.LoggerAdapter) (*Publ func (p *Publisher) Publish(topic string, messages ...*message.Message) error { ctx := context.Background() - queueName, queueUrl, err := p.GetOrCreateQueueUrl(ctx, topic) + queueName, queueUrl, err := p.GetQueueUrl(ctx, topic, !p.config.DoNotCreateQueueIfNotExists) if err != nil { return fmt.Errorf("cannot get queue url: %w", err) } @@ -55,7 +55,7 @@ func (p *Publisher) Publish(topic string, messages ...*message.Message) error { _, err = p.sqs.SendMessage(ctx, input) var queueDoesNotExistErr *types.QueueDoesNotExist if errors.As(err, &queueDoesNotExistErr) && !p.config.DoNotCreateQueueIfNotExists { - // GetOrCreateQueueUrl may not create queue if QueueUrlResolver doesn't check if queue exists + // GetQueueUrl may not create queue if QueueUrlResolver doesn't check if queue exists _, err := p.createQueue(ctx, topic, queueName) if err != nil { return err @@ -69,10 +69,7 @@ func (p *Publisher) Publish(topic string, messages ...*message.Message) error { return nil } -// todo: add types for queueName, etc. ? as it becomes messy what is what - -// todo: name is stupid as creation is conditional -func (p *Publisher) GetOrCreateQueueUrl(ctx context.Context, topic string) (QueueName, QueueURL, error) { +func (p *Publisher) GetQueueUrl(ctx context.Context, topic string, createIfNotExists bool) (QueueName, QueueURL, error) { resolvedQueue, err := p.config.QueueUrlResolver.ResolveQueueUrl(ctx, ResolveQueueUrlParams{ Topic: topic, SqsClient: p.sqs, @@ -85,7 +82,7 @@ func (p *Publisher) GetOrCreateQueueUrl(ctx context.Context, topic string) (Queu return resolvedQueue.QueueName, *resolvedQueue.QueueURL, nil } - if !p.config.DoNotCreateQueueIfNotExists { + if createIfNotExists { queueUrl, err := p.createQueue(ctx, topic, resolvedQueue.QueueName) if err != nil { return "", "", err @@ -109,7 +106,6 @@ func (p *Publisher) createQueue(ctx context.Context, topic string, queueName Que return "", fmt.Errorf("cannot create queue: %w", err) } // queue was created in the meantime - // todo: it's quite ugly if queueUrl == nil { resolvedQueue, err := p.config.QueueUrlResolver.ResolveQueueUrl(ctx, ResolveQueueUrlParams{ Topic: topic, @@ -129,7 +125,7 @@ func (p *Publisher) createQueue(ctx context.Context, topic string, queueName Que return *queueUrl, nil } -func (p *Publisher) GetQueueArn(ctx context.Context, url *QueueURL) (*string, error) { +func (p *Publisher) GetQueueArn(ctx context.Context, url *QueueURL) (*QueueArn, error) { return getARNUrl(ctx, p.sqs, url) } diff --git a/sqs/pubsub_test.go b/sqs/pubsub_test.go index d590ca0..6689c76 100644 --- a/sqs/pubsub_test.go +++ b/sqs/pubsub_test.go @@ -6,6 +6,7 @@ import ( "runtime" "testing" + "github.com/ThreeDotsLabs/watermill-amazonsqs/internal" "github.com/aws/aws-sdk-go-v2/aws" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" @@ -13,7 +14,6 @@ import ( "github.com/stretchr/testify/require" "github.com/ThreeDotsLabs/watermill" - "github.com/ThreeDotsLabs/watermill-amazonsqs/connection" "github.com/ThreeDotsLabs/watermill-amazonsqs/sqs" "github.com/ThreeDotsLabs/watermill/message" "github.com/ThreeDotsLabs/watermill/pubsub/tests" @@ -245,10 +245,10 @@ func TestPublisher_GetOrCreateQueueUrl_is_idempotent(t *testing.T) { topicName := watermill.NewUUID() - name1, url1, err := pub.(*sqs.Publisher).GetOrCreateQueueUrl(context.Background(), topicName) + name1, url1, err := pub.(*sqs.Publisher).GetQueueUrl(context.Background(), topicName, true) require.NoError(t, err) - name2, url2, err := pub.(*sqs.Publisher).GetOrCreateQueueUrl(context.Background(), topicName) + name2, url2, err := pub.(*sqs.Publisher).GetQueueUrl(context.Background(), topicName, true) require.NoError(t, err) require.Equal(t, url1, url2) @@ -359,7 +359,7 @@ func newAwsConfig(t *testing.T) aws.Config { SecretAccessKey: "test", }, }), - connection.SetEndPoint("http://localhost:4566"), + internal.SetEndPoint("http://localhost:4566"), ) require.NoError(t, err) return cfg diff --git a/sqs/sqs.go b/sqs/sqs.go index 62a6717..2701285 100644 --- a/sqs/sqs.go +++ b/sqs/sqs.go @@ -13,6 +13,8 @@ type QueueURL string type QueueName string +type QueueArn string + func getQueueUrl(ctx context.Context, sqsClient *sqs.Client, topic string, input *sqs.GetQueueUrlInput) (*QueueURL, error) { getQueueOutput, err := sqsClient.GetQueueUrl(ctx, input) @@ -25,7 +27,6 @@ func getQueueUrl(ctx context.Context, sqsClient *sqs.Client, topic string, input return &queueURL, nil } -// todo: wtf about that? func createQueue( ctx context.Context, sqsClient *sqs.Client, @@ -52,7 +53,7 @@ func createQueue( return &queueURL, nil } -func getARNUrl(ctx context.Context, sqsClient *sqs.Client, url *QueueURL) (*string, error) { +func getARNUrl(ctx context.Context, sqsClient *sqs.Client, url *QueueURL) (*QueueArn, error) { if url == nil { return nil, fmt.Errorf("queue URL is nil") } @@ -69,7 +70,7 @@ func getARNUrl(ctx context.Context, sqsClient *sqs.Client, url *QueueURL) (*stri return nil, fmt.Errorf("cannot get ARN queue %s: %w", *url, err) } - arn := attrResult.Attributes[string(types.QueueAttributeNameQueueArn)] + arn := QueueArn(attrResult.Attributes[string(types.QueueAttributeNameQueueArn)]) return &arn, nil } diff --git a/sqs/subscriber.go b/sqs/subscriber.go index 3101438..b4bda02 100644 --- a/sqs/subscriber.go +++ b/sqs/subscriber.go @@ -61,7 +61,6 @@ func (s *Subscriber) Subscribe(ctx context.Context, topic string) (<-chan *messa Logger: s.logger, } - // todo: what if doesn't exists? resolvedQueue, err := s.config.QueueUrlResolver.ResolveQueueUrl(ctx, resolveQueueParams) if err != nil { return nil, err @@ -82,7 +81,6 @@ func (s *Subscriber) Subscribe(ctx context.Context, topic string) (<-chan *messa return nil, fmt.Errorf("cannot create queue %s: %w", topic, err) } - // todo: it's quite ugly resolvedQueue, err = s.config.QueueUrlResolver.ResolveQueueUrl(ctx, resolveQueueParams) if err != nil { return nil, err @@ -302,16 +300,12 @@ func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, topic s return nil } -// todo: duplicated in subscribe? func (s *Subscriber) GetQueueUrl(ctx context.Context, topic string) (*QueueURL, error) { - resolveQueueParams := ResolveQueueUrlParams{ + resolvedQueue, err := s.config.QueueUrlResolver.ResolveQueueUrl(ctx, ResolveQueueUrlParams{ Topic: topic, SqsClient: s.sqs, Logger: s.logger, - } - - // todo: what if doesn't exists? - resolvedQueue, err := s.config.QueueUrlResolver.ResolveQueueUrl(ctx, resolveQueueParams) + }) if err != nil { return nil, fmt.Errorf("cannot generate input for queue %s: %w", topic, err) } @@ -322,7 +316,7 @@ func (s *Subscriber) GetQueueUrl(ctx context.Context, topic string) (*QueueURL, return resolvedQueue.QueueURL, nil } -func (s *Subscriber) GetQueueArn(ctx context.Context, url *QueueURL) (*string, error) { +func (s *Subscriber) GetQueueArn(ctx context.Context, url *QueueURL) (*QueueArn, error) { return getARNUrl(ctx, s.sqs, url) } diff --git a/sqs/url_resolver.go b/sqs/url_resolver.go index b1870a0..db8e137 100644 --- a/sqs/url_resolver.go +++ b/sqs/url_resolver.go @@ -19,7 +19,8 @@ type QueueUrlResolver interface { } type ResolveQueueUrlParams struct { - // todo: doc that it's topic in watermill's nomenclature + // Topic passed to Publisher.Publish, Subscriber.Subscribe, etc. + // It may be mapped to a different name by QueueUrlResolver. Topic string SqsClient *sqs.Client Logger watermill.LoggerAdapter @@ -36,8 +37,6 @@ type QueueUrlResolverResult struct { Exists *bool } -// todo: add alternative one that statically generates queue name - type GenerateQueueUrlResolver struct { AwsRegion string AwsAccountID string @@ -176,7 +175,6 @@ func GenerateGetQueueUrlInputDefault(ctx context.Context, topic string) (*sqs.Ge }, nil } -// todo: test type TransparentUrlResolver struct{} func (p TransparentUrlResolver) ResolveQueueUrl(ctx context.Context, params ResolveQueueUrlParams) (res QueueUrlResolverResult, err error) { From 1cb38448dfc3c6e26e1989c386c191e3dbcc6681 Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Thu, 5 Sep 2024 21:50:41 +0200 Subject: [PATCH 18/19] added DEVELOPMENT.md --- DEVELOPMENT.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 DEVELOPMENT.md diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 0000000..1faf483 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,12 @@ +# Development instructions + +## Unit Tests + +You can run the unit tests by simply running the command: `go test -cover -race ./...` + +## Development and Testing + +To try the in test mode (using [localstack](https://hub.docker.com/r/localstack/localstack)) +- start docker using: `docker-compose up -d` + +Now you can run the application: `go run cmd/main.go` \ No newline at end of file From bcd0160a23402fdb5aeae186137ec6670694ccfd Mon Sep 17 00:00:00 2001 From: Robert Laszczak Date: Thu, 5 Sep 2024 22:08:01 +0200 Subject: [PATCH 19/19] missing err check --- sns/subscriber.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sns/subscriber.go b/sns/subscriber.go index 233c8f5..60ab53e 100644 --- a/sns/subscriber.go +++ b/sns/subscriber.go @@ -120,6 +120,9 @@ func (s *Subscriber) SubscribeInitializeWithContext(ctx context.Context, topic s SnsTopicArn: snsTopicArn, SqsQueueArn: *sqsQueueArn, }) + if err != nil { + return fmt.Errorf("cannot generate subscribe input for SNS[%s] from %s: %w", snsTopicArn, *sqsQueueArn, err) + } subscribeOutput, err := s.sns.Subscribe(ctx, input) if err != nil || subscribeOutput == nil {