-
Notifications
You must be signed in to change notification settings - Fork 553
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add S3ObjectLambdaEvent #536
Merged
+343
−0
Merged
Changes from all commits
Commits
Show all changes
8 commits
Select commit
Hold shift + click to select a range
e6212d1
Add S3ObjectLambdaEvent
kdnakt 90a9fb6
Merge branch 'main' into s3-object-lambda-event
bmoffatt 0de482b
Update S3 Object Lambda sample code
kdnakt 8e3bb1e
Avoid the generic name of Configuration
kdnakt 0327fe5
Follow stylecheck
kdnakt 4a87eb4
Add S3ObjectLambda prefix
kdnakt 8df4a4b
fix Url to URL
kdnakt 8005661
Merge branch 'main' into s3-object-lambda-event
bmoffatt File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,83 @@ | ||
# Sample Function | ||
|
||
The following is a sample class and Lambda function that receives Amazon S3 Object Lambda event record data as an input and returns an object metadata output. | ||
|
||
```go | ||
|
||
// main.go | ||
package main | ||
|
||
import ( | ||
"context" | ||
"crypto/md5" | ||
"encoding/hex" | ||
"encoding/json" | ||
"io/ioutil" | ||
"net/http" | ||
"github.com/aws/aws-lambda-go/lambda" | ||
"github.com/aws/aws-lambda-go/events" | ||
"github.com/aws/aws-sdk-go-v2/config" | ||
"github.com/aws/aws-sdk-go-v2/service/s3" | ||
) | ||
|
||
func handler(ctx context.Context, event events.S3ObjectLambdaEvent) error { | ||
url := event.GetObjectContext.InputS3Url | ||
resp, err := http.Get(url) | ||
if err != nil { | ||
return err | ||
} | ||
defer resp.Body.Close() | ||
bodyBytes, err := ioutil.ReadAll(resp.Body) | ||
if err != nil { | ||
return err | ||
} | ||
transformedObject := TransformedObject{ | ||
Metadata: Metadata{ | ||
Length: len(bodyBytes), | ||
Md5: toMd5(bodyBytes), | ||
}, | ||
} | ||
jsonData, err := json.Marshal(transformedObject) | ||
if err != nil { | ||
return err | ||
} | ||
cfg, err := config.LoadDefaultConfig(context.TODO()) | ||
if err != nil { | ||
return err | ||
} | ||
svc := s3.NewFromConfig(cfg) | ||
input := &s3.WriteGetObjectResponseInput{ | ||
RequestRoute: &event.GetObjectContext.OutputRoute, | ||
RequestToken: &event.GetObjectContext.OutputToken, | ||
Body: strings.NewReader(string(jsonData)), | ||
} | ||
res, err := svc.WriteGetObjectResponse(ctx, input) | ||
if err != nil { | ||
return err | ||
} | ||
fmt.Printf("%v", res) | ||
return nil | ||
} | ||
|
||
func toMd5(data []byte) string { | ||
hasher := md5.New() | ||
hasher.Write(data) | ||
hash := hasher.Sum(nil) | ||
|
||
return hex.EncodeToString(hash) | ||
} | ||
|
||
type TransformedObject struct { | ||
Metadata Metadata `json:"metadata"` | ||
} | ||
|
||
type Metadata struct { | ||
Length int `json:"length"` | ||
Md5 string `json:"md5"` | ||
} | ||
|
||
func main() { | ||
lambda.Start(handler) | ||
} | ||
|
||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
package events | ||
|
||
type S3ObjectLambdaEvent struct { | ||
XAmzRequestID string `json:"xAmzRequestId"` | ||
GetObjectContext *S3ObjectLambdaGetObjectContext `json:"getObjectContext,omitempty"` | ||
ListObjectsContext *S3ObjectLambdaListObjectsContext `json:"listObjectsContext,omitempty"` | ||
ListObjectsV2Context *S3ObjectLambdaListObjectsV2Context `json:"listObjectsV2Context,omitempty"` | ||
HeadObjectContext *S3ObjectLambdaHeadObjectContext `json:"headObjectContext,omitempty"` | ||
Configuration S3ObjectLambdaConfiguration `json:"configuration"` | ||
UserRequest S3ObjectLambdaUserRequest `json:"userRequest"` | ||
UserIdentity S3ObjectLambdaUserIdentity `json:"userIdentity"` | ||
ProtocolVersion string `json:"protocolVersion"` | ||
} | ||
|
||
type S3ObjectLambdaGetObjectContext struct { | ||
InputS3URL string `json:"inputS3Url"` | ||
OutputRoute string `json:"outputRoute"` | ||
OutputToken string `json:"outputToken"` | ||
} | ||
|
||
type S3ObjectLambdaListObjectsContext struct { | ||
InputS3URL string `json:"inputS3Url"` | ||
} | ||
|
||
type S3ObjectLambdaListObjectsV2Context struct { | ||
InputS3URL string `json:"inputS3Url"` | ||
} | ||
|
||
type S3ObjectLambdaHeadObjectContext struct { | ||
InputS3URL string `json:"inputS3Url"` | ||
} | ||
|
||
type S3ObjectLambdaConfiguration struct { | ||
AccessPointARN string `json:"accessPointArn"` | ||
SupportingAccessPointARN string `json:"supportingAccessPointArn"` | ||
Payload string `json:"payload"` | ||
} | ||
|
||
type S3ObjectLambdaUserRequest struct { | ||
URL string `json:"url"` | ||
Headers map[string]string `json:"headers"` | ||
} | ||
|
||
type S3ObjectLambdaUserIdentity struct { | ||
Type string `json:"type"` | ||
PrincipalID string `json:"principalId"` | ||
ARN string `json:"arn"` | ||
AccountID string `json:"accountId"` | ||
AccessKeyID string `json:"accessKeyId"` | ||
SessionContext *S3ObjectLambdaSessionContext `json:"sessionContext,omitempty"` | ||
} | ||
|
||
type S3ObjectLambdaSessionContext struct { | ||
Attributes map[string]string `json:"attributes"` | ||
SessionIssuer *S3ObjectLambdaSessionIssuer `json:"sessionIssuer,omitempty"` | ||
} | ||
|
||
type S3ObjectLambdaSessionIssuer struct { | ||
Type string `json:"type"` | ||
PrincipalID string `json:"principalId"` | ||
ARN string `json:"arn"` | ||
AccountID string `json:"accountId"` | ||
UserName string `json:"userName"` | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
package events | ||
|
||
import ( | ||
"encoding/json" | ||
"testing" | ||
|
||
"github.com/aws/aws-lambda-go/events/test" | ||
"github.com/stretchr/testify/assert" | ||
) | ||
|
||
func TestS3ObjectLambdaEventMarshaling(t *testing.T) { | ||
tests := []struct { | ||
file string | ||
}{ | ||
{"./testdata/s3-object-lambda-event-get-object-iam.json"}, | ||
{"./testdata/s3-object-lambda-event-get-object-assumed-role.json"}, | ||
{"./testdata/s3-object-lambda-event-head-object-iam.json"}, | ||
{"./testdata/s3-object-lambda-event-list-objects-iam.json"}, | ||
{"./testdata/s3-object-lambda-event-list-objects-v2-iam.json"}, | ||
} | ||
|
||
for _, tc := range tests { | ||
tc := tc | ||
t.Run(tc.file, func(t *testing.T) { | ||
inputJSON := test.ReadJSONFromFile(t, tc.file) | ||
|
||
var inputEvent S3ObjectLambdaEvent | ||
if err := json.Unmarshal(inputJSON, &inputEvent); err != nil { | ||
t.Errorf("could not unmarshal event. details: %v", err) | ||
} | ||
|
||
outputJSON, err := json.Marshal(inputEvent) | ||
if err != nil { | ||
t.Errorf("could not marshal event. details: %v", err) | ||
} | ||
|
||
assert.JSONEq(t, string(inputJSON), string(outputJSON)) | ||
}) | ||
} | ||
} | ||
|
||
func TestS3ObjectLambdaMarshalingMalformedJson(t *testing.T) { | ||
test.TestMalformedJson(t, S3ObjectLambdaEvent{}) | ||
} |
42 changes: 42 additions & 0 deletions
42
events/testdata/s3-object-lambda-event-get-object-assumed-role.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
{ | ||
"xAmzRequestId": "requestId", | ||
"getObjectContext": { | ||
"inputS3Url": "https://my-s3-ap-111122223333.s3-accesspoint.us-east-1.amazonaws.com/example?X-Amz-Security-Token=<snip>", | ||
"outputRoute": "io-use1-001", | ||
"outputToken": "OutputToken" | ||
}, | ||
"configuration": { | ||
"accessPointArn": "arn:aws:s3-object-lambda:us-east-1:111122223333:accesspoint/example-object-lambda-ap", | ||
"supportingAccessPointArn": "arn:aws:s3:us-east-1:111122223333:accesspoint/example-ap", | ||
"payload": "{}" | ||
}, | ||
"userRequest": { | ||
"url": "https://object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com/example", | ||
"headers": { | ||
"Host": "object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com", | ||
"Accept-Encoding": "identity", | ||
"X-Amz-Content-SHA256": "e3b0c44298fc1example" | ||
} | ||
}, | ||
"userIdentity": { | ||
"type": "AssumedRole", | ||
"principalId": "principalId", | ||
"arn": "arn:aws:sts::111122223333:assumed-role/Admin/example", | ||
"accountId": "111122223333", | ||
"accessKeyId": "accessKeyId", | ||
"sessionContext": { | ||
"attributes": { | ||
"mfaAuthenticated": "false", | ||
"creationDate": "Wed Mar 10 23:41:52 UTC 2021" | ||
}, | ||
"sessionIssuer": { | ||
"type": "Role", | ||
"principalId": "principalId", | ||
"arn": "arn:aws:iam::111122223333:role/Admin", | ||
"accountId": "111122223333", | ||
"userName": "Admin" | ||
} | ||
} | ||
}, | ||
"protocolVersion": "1.00" | ||
} |
29 changes: 29 additions & 0 deletions
29
events/testdata/s3-object-lambda-event-get-object-iam.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
{ | ||
"xAmzRequestId": "requestId", | ||
"getObjectContext": { | ||
"inputS3Url": "https://my-s3-ap-111122223333.s3-accesspoint.us-east-1.amazonaws.com/example?X-Amz-Security-Token=<snip>", | ||
"outputRoute": "io-use1-001", | ||
"outputToken": "OutputToken" | ||
}, | ||
"configuration": { | ||
"accessPointArn": "arn:aws:s3-object-lambda:us-east-1:111122223333:accesspoint/example-object-lambda-ap", | ||
"supportingAccessPointArn": "arn:aws:s3:us-east-1:111122223333:accesspoint/example-ap", | ||
"payload": "{}" | ||
}, | ||
"userRequest": { | ||
"url": "https://object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com/example", | ||
"headers": { | ||
"Host": "object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com", | ||
"Accept-Encoding": "identity", | ||
"X-Amz-Content-SHA256": "e3b0c44298fc1example" | ||
} | ||
}, | ||
"userIdentity": { | ||
"type": "IAMUser", | ||
"principalId": "principalId", | ||
"arn": "arn:aws:iam::111122223333:user/username", | ||
"accountId": "111122223333", | ||
"accessKeyId": "accessKeyId" | ||
}, | ||
"protocolVersion": "1.00" | ||
} |
27 changes: 27 additions & 0 deletions
27
events/testdata/s3-object-lambda-event-head-object-iam.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
{ | ||
"xAmzRequestId": "requestId", | ||
"headObjectContext": { | ||
"inputS3Url": "https://my-s3-ap-111122223333.s3-accesspoint.us-east-1.amazonaws.com/example?X-Amz-Security-Token=<snip>" | ||
}, | ||
"configuration": { | ||
"accessPointArn": "arn:aws:s3-object-lambda:us-east-1:111122223333:accesspoint/example-object-lambda-ap", | ||
"supportingAccessPointArn": "arn:aws:s3:us-east-1:111122223333:accesspoint/example-ap", | ||
"payload": "{}" | ||
}, | ||
"userRequest": { | ||
"url": "https://object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com/example", | ||
"headers": { | ||
"Host": "object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com", | ||
"Accept-Encoding": "identity", | ||
"X-Amz-Content-SHA256": "e3b0c44298fc1example" | ||
} | ||
}, | ||
"userIdentity": { | ||
"type": "IAMUser", | ||
"principalId": "principalId", | ||
"arn": "arn:aws:iam::111122223333:user/username", | ||
"accountId": "111122223333", | ||
"accessKeyId": "accessKeyId" | ||
}, | ||
"protocolVersion": "1.01" | ||
} |
27 changes: 27 additions & 0 deletions
27
events/testdata/s3-object-lambda-event-list-objects-iam.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
{ | ||
"xAmzRequestId": "requestId", | ||
"listObjectsContext": { | ||
"inputS3Url": "https://my-s3-ap-111122223333.s3-accesspoint.us-east-1.amazonaws.com/example?X-Amz-Security-Token=<snip>" | ||
}, | ||
"configuration": { | ||
"accessPointArn": "arn:aws:s3-object-lambda:us-east-1:111122223333:accesspoint/example-object-lambda-ap", | ||
"supportingAccessPointArn": "arn:aws:s3:us-east-1:111122223333:accesspoint/example-ap", | ||
"payload": "{}" | ||
}, | ||
"userRequest": { | ||
"url": "https://object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com/example", | ||
"headers": { | ||
"Host": "object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com", | ||
"Accept-Encoding": "identity", | ||
"X-Amz-Content-SHA256": "e3b0c44298fc1example" | ||
} | ||
}, | ||
"userIdentity": { | ||
"type": "IAMUser", | ||
"principalId": "principalId", | ||
"arn": "arn:aws:iam::111122223333:user/username", | ||
"accountId": "111122223333", | ||
"accessKeyId": "accessKeyId" | ||
}, | ||
"protocolVersion": "1.01" | ||
} |
27 changes: 27 additions & 0 deletions
27
events/testdata/s3-object-lambda-event-list-objects-v2-iam.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
{ | ||
"xAmzRequestId": "requestId", | ||
"listObjectsV2Context": { | ||
"inputS3Url": "https://my-s3-ap-111122223333.s3-accesspoint.us-east-1.amazonaws.com/example?X-Amz-Security-Token=<snip>" | ||
}, | ||
"configuration": { | ||
"accessPointArn": "arn:aws:s3-object-lambda:us-east-1:111122223333:accesspoint/example-object-lambda-ap", | ||
"supportingAccessPointArn": "arn:aws:s3:us-east-1:111122223333:accesspoint/example-ap", | ||
"payload": "{}" | ||
}, | ||
"userRequest": { | ||
"url": "https://object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com/example", | ||
"headers": { | ||
"Host": "object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com", | ||
"Accept-Encoding": "identity", | ||
"X-Amz-Content-SHA256": "e3b0c44298fc1example" | ||
} | ||
}, | ||
"userIdentity": { | ||
"type": "IAMUser", | ||
"principalId": "principalId", | ||
"arn": "arn:aws:iam::111122223333:user/username", | ||
"accountId": "111122223333", | ||
"accessKeyId": "accessKeyId" | ||
}, | ||
"protocolVersion": "1.01" | ||
} |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Looking for another reference for these fields. The documentation I found so far only shows
getObjectContext
https://docs.aws.amazon.com/AmazonS3/latest/userguide/olap-event-context.html
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
PR for the rust event calavera/aws-lambda-events#127 didn't contain additional docs for List/Head object.
Guess I gotta check this by hand, or ask the service team for a pointer.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
https://docs.aws.amazon.com/AmazonS3/latest/userguide/olap-writing-lambda.html#olap-getobject-response
I followed these examples in the documentation.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
My bad! I forgot that I referenced that exact documentation when reviewing one of your previous commits! #536 (comment)