S3 Object Lambda Support (#1183)

.changes/next-release/feature.s3.manager-feature-1616093328620827000.json

@@ -0,0 +1,9 @@
+{
+ "ID": "feature.s3.manager-feature-1616093328620827000",
+ "SchemaVersion": 1,
+ "Module": "feature/s3/manager",
+ "Type": "feature",
+ "Description": "Updates for S3 Object Lambda feature",
+ "MinVersion": "",
+ "AffectedModules": null
+}

.changes/next-release/service.internal.s3shared-feature-1616093429190725000.json

@@ -0,0 +1,9 @@
+{
+ "ID": "service.internal.s3shared-feature-1616093429190725000",
+ "SchemaVersion": 1,
+ "Module": "service/internal/s3shared",
+ "Type": "feature",
+ "Description": "Support for S3 Object Lambda",
+ "MinVersion": "",
+ "AffectedModules": null
+}

.changes/next-release/service.s3-feature-1616093381239174000.json

@@ -0,0 +1,9 @@
+{
+ "ID": "service.s3-feature-1616093381239174000",
+ "SchemaVersion": 1,
+ "Module": "service/s3",
+ "Type": "feature",
+ "Description": "S3 Object Lambda is a new S3 feature that enables users to apply their own custom code to process the output of a standard S3 GET request by automatically invoking a Lambda function with a GET request",
+ "MinVersion": "",
+ "AffectedModules": null
+}

.changes/next-release/service.s3control-feature-1616093391667799000.json

@@ -0,0 +1,9 @@
+{
+ "ID": "service.s3control-feature-1616093391667799000",
+ "SchemaVersion": 1,
+ "Module": "service/s3control",
+ "Type": "feature",
+ "Description": "S3 Object Lambda is a new S3 feature that enables users to apply their own custom code to process the output of a standard S3 GET request by automatically invoking a Lambda function with a GET request",
+ "MinVersion": "",
+ "AffectedModules": null
+}

.changes/next-release/service.wildcard-bugfix-1616093590156551000.json

@@ -0,0 +1,16 @@
+{
+ "ID": "service.wildcard-bugfix-1616093590156551000",
+ "SchemaVersion": 1,
+ "Module": "service/...",
+ "Type": "bugfix",
+ "Description": "Updated presign URLs to no longer include the X-Amz-User-Agent header",
+ "MinVersion": "",
+ "AffectedModules": [
+  "service/docdb",
+  "service/ec2",
+  "service/neptune",
+  "service/rds",
+  "service/s3",
+  "service/sts"
+ ]
+}

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java

@@ -344,6 +344,7 @@ private void writeConvertToPresignMiddleware(
                     writer.write("stack.Finalize.Clear()");
                     writer.write("stack.Deserialize.Clear()");
                     writer.write("stack.Build.Remove(($P)(nil).ID())", requestInvocationID);
+                    writer.write("stack.Build.Remove($S)", "UserAgent");
 
                     Symbol middlewareOptionsSymbol = SymbolUtils.createValueSymbolBuilder(
                             "PresignHTTPRequestMiddlewareOptions", AwsGoDependency.AWS_SIGNER_V4).build();

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/S3ContentSHA256Header.java

@@ -2,6 +2,7 @@
 
 import java.util.List;
 import software.amazon.smithy.aws.go.codegen.AwsGoDependency;
+import software.amazon.smithy.aws.traits.auth.UnsignedPayloadTrait;
 import software.amazon.smithy.go.codegen.SymbolUtils;
 import software.amazon.smithy.go.codegen.integration.GoIntegration;
 import software.amazon.smithy.go.codegen.integration.MiddlewareRegistrar;
@@ -26,8 +27,15 @@ public byte getOrder() {
     public List<RuntimeClientPlugin> getClientPlugins() {
         return ListUtils.of(
                 RuntimeClientPlugin.builder()
-                        .servicePredicate((model, service) -> S3ModelUtils.isServiceS3(model, service)
-                                || S3ModelUtils.isServiceS3Control(model, service))
+                        // Only add the middleware if UnsignedPayloadTrait is not specified, as this middleware
+                        // will have already been added.
+                        .operationPredicate((model, service, operation) -> {
+                            if (!(S3ModelUtils.isServiceS3(model, service)
+                                    || S3ModelUtils.isServiceS3Control(model, service))) {
+                                return false;
+                            }
+                            return !operation.hasTrait(UnsignedPayloadTrait.class);
+                        })
                         .registerMiddleware(MiddlewareRegistrar.builder()
                                 .resolvedFunction(SymbolUtils.createValueSymbolBuilder(
                                         "AddContentSHA256HeaderMiddleware",

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/S3UpdateEndpoint.java

@@ -200,6 +200,8 @@ private static class S3 {
                 "ListBuckets", "CreateBucket", "DeleteBucket"
         );
 
+        private final Set<String> TARGET_OBJECT_LAMBDAS = SetUtils.of("WriteGetObjectResponse");
+
         private S3(ServiceShape service) {
             this.service = service;
         }
@@ -285,14 +287,17 @@ private void writeMiddlewareHelper(
                     addMiddlewareFuncName(symbolProvider.toSymbol(operationShape).getName(),
                             UPDATE_ENDPOINT_INTERNAL_ADDER), () -> {
                         writer.write("return $T(stack, $T{ \n"
-                                        + "Accessor : $T{\n "
-                                        + "GetBucketFromInput: $L,\n}, \n"
-                                        + "UsePathStyle: options.$L,\n "
-                                        + "UseAccelerate: options.$L,\n "
-                                        + "SupportsAccelerate: $L,\n "
-                                        + "EndpointResolver: options.EndpointResolver,\n "
+                                        + "Accessor : $T{\n"
+                                        + "GetBucketFromInput: $L,\n},\n"
+                                        + "UsePathStyle: options.$L,\n"
+                                        + "UseAccelerate: options.$L,\n"
+                                        + "SupportsAccelerate: $L,\n"
+                                        + "TargetS3ObjectLambda: $L,\n"
+                                        + "EndpointResolver: options.EndpointResolver,\n"
                                         + "EndpointResolverOptions: options.EndpointOptions,\n"
-                                        + "UseDualstack: options.$L, \n UseARNRegion: options.$L, \n })",
+                                        + "UseDualstack: options.$L,\n"
+                                        + "UseARNRegion: options.$L,\n"
+                                        + "})",
                                 SymbolUtils.createValueSymbolBuilder(UPDATE_ENDPOINT_INTERNAL_ADDER,
                                         AwsCustomGoDependency.S3_CUSTOMIZATION).build(),
                                 SymbolUtils.createValueSymbolBuilder(UPDATE_ENDPOINT_INTERNAL_OPTIONS,
@@ -304,6 +309,7 @@ private void writeMiddlewareHelper(
                                 USE_PATH_STYLE_OPTION,
                                 USE_ACCELERATE_OPTION,
                                 !NOT_SUPPORT_ACCELERATE.contains(operationName),
+                                TARGET_OBJECT_LAMBDAS.contains(operationName),
                                 USE_DUALSTACK_OPTION,
                                 USE_ARNREGION_OPTION
                         );

config/go.mod

@@ -6,7 +6,6 @@ require (
 	github.com/aws/aws-sdk-go-v2 v1.2.1
 	github.com/aws/aws-sdk-go-v2/credentials v1.1.2
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.3
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.3
 	github.com/aws/aws-sdk-go-v2/service/sso v1.1.2
 	github.com/aws/aws-sdk-go-v2/service/sts v1.1.2
 	github.com/aws/smithy-go v1.2.0

credentials/go.mod

@@ -5,7 +5,6 @@ go 1.15
 require (
 	github.com/aws/aws-sdk-go-v2 v1.2.1
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.3
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.1.2
 	github.com/aws/aws-sdk-go-v2/service/sts v1.1.2
 	github.com/aws/smithy-go v1.2.0

example/service/s3/listObjects/go.mod

@@ -3,16 +3,8 @@ module github.com/aws/aws-sdk-go-v2/example/service/s3/listObjects
 go 1.15
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.2.1
 	github.com/aws/aws-sdk-go-v2/config v1.1.2
-	github.com/aws/aws-sdk-go-v2/credentials v1.1.2
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.3
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.3
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.1.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.2.1
-	github.com/aws/aws-sdk-go-v2/service/sso v1.1.2
-	github.com/aws/aws-sdk-go-v2/service/sts v1.1.2
 )
 
 replace github.com/aws/aws-sdk-go-v2/config => ../../../../config/

example/service/s3/usingPrivateLink/go.mod

@@ -5,15 +5,8 @@ go 1.15
 require (
 	github.com/aws/aws-sdk-go-v2 v1.2.1
 	github.com/aws/aws-sdk-go-v2/config v1.1.2
-	github.com/aws/aws-sdk-go-v2/credentials v1.1.2
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.3
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.3
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.1.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.2.1
 	github.com/aws/aws-sdk-go-v2/service/s3control v1.2.1
-	github.com/aws/aws-sdk-go-v2/service/sso v1.1.2
-	github.com/aws/aws-sdk-go-v2/service/sts v1.1.2
 )
 
 replace github.com/aws/aws-sdk-go-v2/config => ../../../../config/

feature/dynamodb/attributevalue/go.mod

@@ -6,7 +6,6 @@ require (
 	github.com/aws/aws-sdk-go-v2 v1.2.1
 	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.1.2
 	github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.1.2
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.2
 	github.com/google/go-cmp v0.5.4
 )
 

feature/dynamodb/expression/go.mod

@@ -6,8 +6,6 @@ require (
 	github.com/aws/aws-sdk-go-v2 v1.2.1
 	github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.0.3
 	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.1.2
-	github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.1.2
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.2 // indirect
 )
 
 replace (

feature/dynamodbstreams/attributevalue/go.mod

@@ -6,7 +6,6 @@ require (
 	github.com/aws/aws-sdk-go-v2 v1.2.1
 	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.1.2
 	github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.1.2
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.2
 	github.com/google/go-cmp v0.5.4
 )
 

feature/s3/manager/arn.go

@@ -0,0 +1,23 @@
+package manager
+
+import (
+	"fmt"
+	"github.com/aws/aws-sdk-go-v2/aws/arn"
+)
+
+func validateSupportedARNType(bucket string) error {
+	if !arn.IsARN(bucket) {
+		return nil
+	}
+
+	parsedARN, err := arn.Parse(bucket)
+	if err != nil {
+		return err
+	}
+
+	if parsedARN.Service == "s3-object-lambda" {
+		return fmt.Errorf("manager does not support s3-object-lambda service ARNs")
+	}
+
+	return nil
+}

feature/s3/manager/download.go

@@ -172,6 +172,10 @@ func NewDownloader(c DownloadAPIClient, options ...func(*Downloader)) *Downloade
 // to perform a single GetObjectInput request for that object's range. This will
 // caused the part size, and concurrency configurations to be ignored.
 func (d Downloader) Download(ctx context.Context, w io.WriterAt, input *s3.GetObjectInput, options ...func(*Downloader)) (n int64, err error) {
+	if err := validateSupportedARNType(aws.ToString(input.Bucket)); err != nil {
+		return 0, err
+	}
+
 	impl := downloader{w: w, in: input, cfg: d, ctx: ctx}
 
 	// Copy ClientOptions

feature/s3/manager/download_test.go

@@ -720,6 +720,53 @@ func TestDownloadBufferStrategy_Errors(t *testing.T) {
 	}
 }
 
+func TestDownloaderValidARN(t *testing.T) {
+	cases := map[string]struct {
+		input   s3.GetObjectInput
+		wantErr bool
+	}{
+		"standard bucket": {
+			input: s3.GetObjectInput{
+				Bucket: aws.String("test-bucket"),
+				Key:    aws.String("test-key"),
+			},
+		},
+		"accesspoint": {
+			input: s3.GetObjectInput{
+				Bucket: aws.String("arn:aws:s3:us-west-2:123456789012:accesspoint/myap"),
+				Key:    aws.String("test-key"),
+			},
+		},
+		"outpost accesspoint": {
+			input: s3.GetObjectInput{
+				Bucket: aws.String("arn:aws:s3-outposts:us-west-2:012345678901:outpost/op-1234567890123456/accesspoint/myaccesspoint"),
+				Key:    aws.String("test-key"),
+			},
+		},
+		"s3-object-lambda accesspoint": {
+			input: s3.GetObjectInput{
+				Bucket: aws.String("arn:aws:s3-object-lambda:us-west-2:123456789012:accesspoint/myap"),
+			},
+			wantErr: true,
+		},
+	}
+
+	for name, tt := range cases {
+		t.Run(name, func(t *testing.T) {
+			client, _ := newDownloadNonRangeClient(buf2MB)
+
+			downloader := manager.NewDownloader(client, func(downloader *manager.Downloader) {
+				downloader.Concurrency = 1
+			})
+
+			_, err := downloader.Download(context.Background(), &awstesting.DiscardAt{}, &tt.input)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("err: %v, wantErr: %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
 type recordedWriterReadFromProvider struct {
 	callbacksVended   uint32
 	callbacksExecuted uint32

feature/s3/manager/go.mod

@@ -5,14 +5,7 @@ go 1.15
 require (
 	github.com/aws/aws-sdk-go-v2 v1.2.1
 	github.com/aws/aws-sdk-go-v2/config v1.1.2
-	github.com/aws/aws-sdk-go-v2/credentials v1.1.2
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.3
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.3
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.1.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.2.1
-	github.com/aws/aws-sdk-go-v2/service/sso v1.1.2
-	github.com/aws/aws-sdk-go-v2/service/sts v1.1.2
 	github.com/aws/smithy-go v1.2.0
 	github.com/google/go-cmp v0.5.4
 )

feature/s3/manager/upload.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"io"
 	"net/http"
 	"sort"
@@ -275,6 +276,10 @@ func (u *uploader) upload() (*UploadOutput, error) {
 
 // init will initialize all default options.
 func (u *uploader) init() error {
+	if err := validateSupportedARNType(aws.ToString(u.in.Bucket)); err != nil {
+		return err
+	}
+
 	if u.cfg.Concurrency == 0 {
 		u.cfg.Concurrency = DefaultUploadConcurrency
 	}

feature/s3/manager/upload_test.go

@@ -963,6 +963,57 @@ func TestUploadBufferStrategy(t *testing.T) {
 	}
 }
 
+func TestUploaderValidARN(t *testing.T) {
+	cases := map[string]struct {
+		input   s3.PutObjectInput
+		wantErr bool
+	}{
+		"standard bucket": {
+			input: s3.PutObjectInput{
+				Bucket: aws.String("test-bucket"),
+				Key:    aws.String("test-key"),
+				Body:   bytes.NewReader([]byte("test body content")),
+			},
+		},
+		"accesspoint": {
+			input: s3.PutObjectInput{
+				Bucket: aws.String("arn:aws:s3:us-west-2:123456789012:accesspoint/myap"),
+				Key:    aws.String("test-key"),
+				Body:   bytes.NewReader([]byte("test body content")),
+			},
+		},
+		"outpost accesspoint": {
+			input: s3.PutObjectInput{
+				Bucket: aws.String("arn:aws:s3-outposts:us-west-2:012345678901:outpost/op-1234567890123456/accesspoint/myaccesspoint"),
+				Key:    aws.String("test-key"),
+				Body:   bytes.NewReader([]byte("test body content")),
+			},
+		},
+		"s3-object-lambda accesspoint": {
+			input: s3.PutObjectInput{
+				Bucket: aws.String("arn:aws:s3-object-lambda:us-west-2:123456789012:accesspoint/myap"),
+				Key:    aws.String("test-key"),
+				Body:   bytes.NewReader([]byte("test body content")),
+			},
+			wantErr: true,
+		},
+	}
+
+	for name, tt := range cases {
+		t.Run(name, func(t *testing.T) {
+			client, _, _ := s3testing.NewUploadLoggingClient(nil)
+			client.ConsumeBody = true
+
+			uploader := manager.NewUploader(client)
+
+			_, err := uploader.Upload(context.Background(), &tt.input)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("err: %v, wantErr: %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
 type mockS3UploadServer struct {
 	*http.ServeMux
 

service/acm/go.sum

@@ -13,6 +13,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=