Skip to content

Commit

Permalink
Merge customizations for service s3
Browse files Browse the repository at this point in the history
  • Loading branch information
AWS SDK for Go v2 automation user committed Nov 17, 2023
1 parent 24bd514 commit 4bd06b9
Show file tree
Hide file tree
Showing 44 changed files with 453 additions and 418 deletions.
9 changes: 9 additions & 0 deletions .changelog/331929f2c5c941d49d6160d2d2f34ee7.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"id": "331929f2-c5c9-41d4-9d61-60d2d2f34ee7",
"type": "feature",
"description": "**BREAKING CHANGE** Correct nullability of a large number of S3 structure fields. See https://github.com/aws/aws-sdk-go-v2/issues/2162.",
"modules": [
"feature/s3/manager",
"service/s3"
]
}
4 changes: 2 additions & 2 deletions feature/s3/manager/download.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -436,8 +436,8 @@ func (d *downloader) setTotalBytes(resp *s3.GetObjectOutput) {
if resp.ContentRange == nil {
// ContentRange is nil when the full file contents is provided, and
// is not chunked. Use ContentLength instead.
if resp.ContentLength > 0 {
d.totalBytes = resp.ContentLength
if aws.ToInt64(resp.ContentLength) > 0 {
d.totalBytes = aws.ToInt64(resp.ContentLength)
return
}
} else {
Expand Down
8 changes: 4 additions & 4 deletions feature/s3/manager/download_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ func newDownloadRangeClient(data []byte) (*downloadCaptureClient, *int, *[]strin
return &s3.GetObjectOutput{
Body: ioutil.NopCloser(bytes.NewReader(bodyBytes)),
ContentRange: aws.String(fmt.Sprintf("bytes %d-%d/%d", start, fin-1, len(data))),
ContentLength: int64(len(bodyBytes)),
ContentLength: aws.Int64(int64(len(bodyBytes))),
}, nil
}

Expand All @@ -83,7 +83,7 @@ func newDownloadNonRangeClient(data []byte) (*downloadCaptureClient, *int) {
capture.GetObjectFn = func(_ context.Context, params *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
return &s3.GetObjectOutput{
Body: ioutil.NopCloser(bytes.NewReader(data[:])),
ContentLength: int64(len(data)),
ContentLength: aws.Int64(int64(len(data))),
}, nil
}

Expand Down Expand Up @@ -139,7 +139,7 @@ func newDownloadWithErrReaderClient(cases []testErrReader) (*downloadCaptureClie
out := &s3.GetObjectOutput{
Body: ioutil.NopCloser(&c),
ContentRange: aws.String(fmt.Sprintf("bytes %d-%d/%d", 0, c.Len-1, c.Len)),
ContentLength: c.Len,
ContentLength: aws.Int64(c.Len),
}
index++
return out, nil
Expand Down Expand Up @@ -542,7 +542,7 @@ func TestDownload_WithFailure(t *testing.T) {
body := bytes.NewReader(make([]byte, manager.DefaultDownloadPartSize))
out = &s3.GetObjectOutput{
Body: ioutil.NopCloser(body),
ContentLength: int64(body.Len()),
ContentLength: aws.Int64(int64(body.Len())),
ContentRange: aws.String(fmt.Sprintf("bytes %d-%d/%d", startingByte, body.Len()-1, body.Len()*10)),
}

Expand Down
56 changes: 28 additions & 28 deletions feature/s3/manager/integ_upload_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,15 +228,15 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
expectParts: []s3types.CompletedPart{
{
ETag: aws.String(singlePartETag),
PartNumber: 1,
PartNumber: aws.Int32(1),
},
{
ETag: aws.String(singlePartETag),
PartNumber: 2,
PartNumber: aws.Int32(2),
},
{
ETag: aws.String(multiPartTailETag),
PartNumber: 3,
PartNumber: aws.Int32(3),
},
},
expectETag: multiPartETag,
Expand All @@ -248,17 +248,17 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
{
ChecksumCRC32: aws.String(singlePartCRC32),
ETag: aws.String(singlePartETag),
PartNumber: 1,
PartNumber: aws.Int32(1),
},
{
ChecksumCRC32: aws.String(singlePartCRC32),
ETag: aws.String(singlePartETag),
PartNumber: 2,
PartNumber: aws.Int32(2),
},
{
ChecksumCRC32: aws.String(multiPartTailCRC32),
ETag: aws.String(multiPartTailETag),
PartNumber: 3,
PartNumber: aws.Int32(3),
},
},
expectChecksumCRC32: multiPartCRC32,
Expand All @@ -271,17 +271,17 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
{
ChecksumCRC32C: aws.String(singlePartCRC32C),
ETag: aws.String(singlePartETag),
PartNumber: 1,
PartNumber: aws.Int32(1),
},
{
ChecksumCRC32C: aws.String(singlePartCRC32C),
ETag: aws.String(singlePartETag),
PartNumber: 2,
PartNumber: aws.Int32(2),
},
{
ChecksumCRC32C: aws.String(multiPartTailCRC32C),
ETag: aws.String(multiPartTailETag),
PartNumber: 3,
PartNumber: aws.Int32(3),
},
},
expectChecksumCRC32C: multiPartCRC32C,
Expand All @@ -294,17 +294,17 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
{
ChecksumSHA1: aws.String(singlePartSHA1),
ETag: aws.String(singlePartETag),
PartNumber: 1,
PartNumber: aws.Int32(1),
},
{
ChecksumSHA1: aws.String(singlePartSHA1),
ETag: aws.String(singlePartETag),
PartNumber: 2,
PartNumber: aws.Int32(2),
},
{
ChecksumSHA1: aws.String(multiPartTailSHA1),
ETag: aws.String(multiPartTailETag),
PartNumber: 3,
PartNumber: aws.Int32(3),
},
},
expectChecksumSHA1: multiPartSHA1,
Expand All @@ -317,17 +317,17 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
{
ChecksumSHA256: aws.String(singlePartSHA256),
ETag: aws.String(singlePartETag),
PartNumber: 1,
PartNumber: aws.Int32(1),
},
{
ChecksumSHA256: aws.String(singlePartSHA256),
ETag: aws.String(singlePartETag),
PartNumber: 2,
PartNumber: aws.Int32(2),
},
{
ChecksumSHA256: aws.String(multiPartTailSHA256),
ETag: aws.String(multiPartTailETag),
PartNumber: 3,
PartNumber: aws.Int32(3),
},
},
expectChecksumSHA256: multiPartSHA256,
Expand All @@ -343,17 +343,17 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
{
ChecksumCRC32: aws.String(singlePartCRC32),
ETag: aws.String(singlePartETag),
PartNumber: 1,
PartNumber: aws.Int32(1),
},
{
ChecksumCRC32: aws.String(singlePartCRC32),
ETag: aws.String(singlePartETag),
PartNumber: 2,
PartNumber: aws.Int32(2),
},
{
ChecksumCRC32: aws.String(multiPartTailCRC32),
ETag: aws.String(multiPartTailETag),
PartNumber: 3,
PartNumber: aws.Int32(3),
},
},
expectChecksumCRC32: multiPartCRC32,
Expand All @@ -367,17 +367,17 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
{
ChecksumCRC32C: aws.String(singlePartCRC32C),
ETag: aws.String(singlePartETag),
PartNumber: 1,
PartNumber: aws.Int32(1),
},
{
ChecksumCRC32C: aws.String(singlePartCRC32C),
ETag: aws.String(singlePartETag),
PartNumber: 2,
PartNumber: aws.Int32(2),
},
{
ChecksumCRC32C: aws.String(multiPartTailCRC32C),
ETag: aws.String(multiPartTailETag),
PartNumber: 3,
PartNumber: aws.Int32(3),
},
},
expectChecksumCRC32C: multiPartCRC32C,
Expand All @@ -391,17 +391,17 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
{
ChecksumSHA1: aws.String(singlePartSHA1),
ETag: aws.String(singlePartETag),
PartNumber: 1,
PartNumber: aws.Int32(1),
},
{
ChecksumSHA1: aws.String(singlePartSHA1),
ETag: aws.String(singlePartETag),
PartNumber: 2,
PartNumber: aws.Int32(2),
},
{
ChecksumSHA1: aws.String(multiPartTailSHA1),
ETag: aws.String(multiPartTailETag),
PartNumber: 3,
PartNumber: aws.Int32(3),
},
},
expectChecksumSHA1: multiPartSHA1,
Expand All @@ -415,17 +415,17 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
{
ChecksumSHA256: aws.String(singlePartSHA256),
ETag: aws.String(singlePartETag),
PartNumber: 1,
PartNumber: aws.Int32(1),
},
{
ChecksumSHA256: aws.String(singlePartSHA256),
ETag: aws.String(singlePartETag),
PartNumber: 2,
PartNumber: aws.Int32(2),
},
{
ChecksumSHA256: aws.String(multiPartTailSHA256),
ETag: aws.String(multiPartTailETag),
PartNumber: 3,
PartNumber: aws.Int32(3),
},
},
expectChecksumSHA256: multiPartSHA256,
Expand Down Expand Up @@ -499,7 +499,7 @@ func (b *invalidateHash) RegisterMiddleware(stack *middleware.Stack) error {
func (b *invalidateHash) HandleSerialize(ctx context.Context, in middleware.SerializeInput, next middleware.SerializeHandler) (
out middleware.SerializeOutput, metadata middleware.Metadata, err error,
) {
if input, ok := in.Parameters.(*s3.UploadPartInput); ok && input.PartNumber == 2 {
if input, ok := in.Parameters.(*s3.UploadPartInput); ok && input.PartNumber == aws.Int32(2) {
ctx = v4.SetPayloadHash(ctx, "000")
}

Expand Down
4 changes: 2 additions & 2 deletions feature/s3/manager/internal/integration/integration.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ func CleanupBucket(client *s3.Client, bucketName string) error {
errs = append(errs, fmt.Errorf("failed to delete %s, %s", aws.ToString(deleteError.Key), aws.ToString(deleteError.Message)))
}

if listObjectsV2.IsTruncated {
if aws.ToBool(listObjectsV2.IsTruncated) {
input.ContinuationToken = listObjectsV2.NextContinuationToken
} else {
break
Expand All @@ -182,7 +182,7 @@ func CleanupBucket(client *s3.Client, bucketName string) error {
})
}

if uploads.IsTruncated {
if aws.ToBool(uploads.IsTruncated) {
input.KeyMarker = uploads.NextKeyMarker
input.UploadIdMarker = uploads.NextUploadIdMarker
} else {
Expand Down
16 changes: 9 additions & 7 deletions feature/s3/manager/upload.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -501,7 +501,7 @@ func (u *uploader) singlePart(r io.ReadSeeker, cleanup func()) (*UploadOutput, e
return &UploadOutput{
Location: locationRecorder.location,

BucketKeyEnabled: out.BucketKeyEnabled,
BucketKeyEnabled: aws.ToBool(out.BucketKeyEnabled),
ChecksumCRC32: out.ChecksumCRC32,
ChecksumCRC32C: out.ChecksumCRC32C,
ChecksumSHA1: out.ChecksumSHA1,
Expand Down Expand Up @@ -568,9 +568,11 @@ type chunk struct {
// since S3 required this list to be sent in sorted order.
type completedParts []types.CompletedPart

func (a completedParts) Len() int { return len(a) }
func (a completedParts) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
func (a completedParts) Len() int { return len(a) }
func (a completedParts) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a completedParts) Less(i, j int) bool {
return aws.ToInt32(a[i].PartNumber) < aws.ToInt32(a[j].PartNumber)
}

// upload will perform a multipart upload using the firstBuf buffer containing
// the first chunk of data.
Expand Down Expand Up @@ -639,7 +641,7 @@ func (u *multiuploader) upload(firstBuf io.ReadSeeker, cleanup func()) (*UploadO
UploadID: u.uploadID,
CompletedParts: u.parts,

BucketKeyEnabled: completeOut.BucketKeyEnabled,
BucketKeyEnabled: aws.ToBool(completeOut.BucketKeyEnabled),
ChecksumCRC32: completeOut.ChecksumCRC32,
ChecksumCRC32C: completeOut.ChecksumCRC32C,
ChecksumSHA1: completeOut.ChecksumSHA1,
Expand Down Expand Up @@ -722,7 +724,7 @@ func (u *multiuploader) send(c chunk) error {
// PutObject as they are never valid for individual parts of a
// multipart upload.

PartNumber: c.num,
PartNumber: aws.Int32(c.num),
UploadId: &u.uploadID,
}
// TODO should do copy then clear?
Expand All @@ -734,7 +736,7 @@ func (u *multiuploader) send(c chunk) error {

var completed types.CompletedPart
awsutil.Copy(&completed, resp)
completed.PartNumber = c.num
completed.PartNumber = aws.Int32(c.num)

u.m.Lock()
u.parts = append(u.parts, completed)
Expand Down
2 changes: 1 addition & 1 deletion feature/s3/manager/upload_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func TestUploadOrderMulti(t *testing.T) {
num := parts[i].PartNumber
etag := aws.ToString(parts[i].ETag)

if int32(i+1) != num {
if int32(i+1) != aws.ToInt32(num) {
t.Errorf("expect %d, got %d", i+1, num)
}

Expand Down
6 changes: 3 additions & 3 deletions service/internal/integrationtest/s3/checksum_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ func TestInteg_ObjectChecksums(t *testing.T) {
"content length preset": {
params: &s3.PutObjectInput{
Body: strings.NewReader("hello world"),
ContentLength: 11,
ContentLength: aws.Int64(11),
ChecksumAlgorithm: s3types.ChecksumAlgorithmCrc32c,
},
getObjectChecksumMode: s3types.ChecksumModeEnabled,
Expand Down Expand Up @@ -198,7 +198,7 @@ func TestInteg_ObjectChecksums(t *testing.T) {
"content length preset": {
params: &s3.PutObjectInput{
Body: ioutil.NopCloser(strings.NewReader("hello world")),
ContentLength: 11,
ContentLength: aws.Int64(11),
ChecksumAlgorithm: s3types.ChecksumAlgorithmCrc32c,
},
getObjectChecksumMode: s3types.ChecksumModeEnabled,
Expand Down Expand Up @@ -449,7 +449,7 @@ func TestInteg_RequireChecksum(t *testing.T) {
Objects: []s3types.ObjectIdentifier{
{Key: aws.String(t.Name())},
},
Quiet: true,
Quiet: aws.Bool(true),
},
ChecksumAlgorithm: c.checksumAlgorithm,
})
Expand Down
2 changes: 1 addition & 1 deletion service/internal/integrationtest/s3/presign_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ func TestInteg_MultipartPresignURL(t *testing.T) {
uploadPartInput := &s3.UploadPartInput{
Bucket: &setupMetadata.Buckets.Source.Name,
Key: &key,
PartNumber: 1,
PartNumber: aws.Int32(1),
UploadId: multipartUpload.UploadId,
Body: c.body,
}
Expand Down
2 changes: 1 addition & 1 deletion service/s3/api_op_CompleteMultipartUpload.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions service/s3/api_op_CopyObject.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 4bd06b9

Please sign in to comment.