From e074eba8a8238afa7e88fe54c00eaead136cff7b Mon Sep 17 00:00:00 2001 From: Lachlan Donald Date: Fri, 16 Dec 2016 18:37:18 +1100 Subject: [PATCH 1/8] Add a lambda function handler --- .gitignore | 1 + Makefile | 16 ++++++++++------ lambda.go | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 lambda.go diff --git a/.gitignore b/.gitignore index 567609b1..a8df8a5b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ build/ +handler.zip diff --git a/Makefile b/Makefile index 2dbe536d..6de50a77 100644 --- a/Makefile +++ b/Makefile @@ -2,17 +2,21 @@ VERSION=$(shell git fetch --tags && git describe --tags --candidates=1 --dirty --always) FLAGS=-X main.Version=$(VERSION) BIN=build/buildkite-metrics-$(shell uname -s)-$(shell uname -m)-$(VERSION) -LATEST=build/buildkite-metrics-$(shell uname -s)-$(shell uname -m) +LAMBDA_ZIP=$(BIN)-lambda.zip +SRC=$(shell find . -name '*.go') -build: $(BIN) +build: $(BIN) $(LAMBDA_ZIP) clean: - -rm build/* + -rm -f build/ -$(BIN): main.go +$(BIN): $(SRC) -mkdir -p build/ go build -o $(BIN) -ldflags="$(FLAGS)" . - cp -a $(BIN) $(LATEST) + +$(LAMBDA_ZIP): $(SRC) + docker run --rm -v $(GOPATH):/go -v $(PWD):/tmp eawsy/aws-lambda-go + mv handler.zip $(LAMBDA_ZIP) upload: - aws s3 sync --acl public-read build s3://buildkite-metrics/ \ No newline at end of file + aws s3 sync --acl public-read build s3://buildkite-metrics/ diff --git a/lambda.go b/lambda.go new file mode 100644 index 00000000..afd69542 --- /dev/null +++ b/lambda.go @@ -0,0 +1,50 @@ +package main + +import ( + "bytes" + "encoding/json" + "log" + "os" + "time" + + "github.com/eawsy/aws-lambda-go/service/lambda/runtime" + "gopkg.in/buildkite/go-buildkite.v2/buildkite" +) + +func handle(evt json.RawMessage, ctx *runtime.Context) (interface{}, error) { + output := &bytes.Buffer{} + log.SetOutput(output) + + org := os.Getenv("BUILDKITE_ORG") + token := os.Getenv("BUILDKITE_TOKEN") + + config, err := buildkite.NewTokenConfig(token, false) + if err != nil { + return nil, err + } + + client := buildkite.NewClient(config.Client()) + t := time.Now() + + res, err := collectResults(client, collectOpts{ + OrgSlug: org, + Historical: time.Hour * 24, + }) + if err != nil { + return nil, err + } + + dumpResults(res) + + err = cloudwatchSend(res) + if err != nil { + return nil, err + } + + log.Printf("Finished in %s", time.Now().Sub(t)) + return output.String(), nil +} + +func init() { + runtime.HandleFunc(handle) +} From addf88f6e9303f19a7c8b5b0953893fad0a57755 Mon Sep 17 00:00:00 2001 From: Lachlan Donald Date: Fri, 16 Dec 2016 18:37:43 +1100 Subject: [PATCH 2/8] Add vendors for lambda --- vendor/github.com/eawsy/aws-lambda-go/LICENSE | 202 ++++++++++++++++ vendor/github.com/eawsy/aws-lambda-go/NOTICE | 10 + .../service/lambda/runtime/context.go | 134 +++++++++++ .../service/lambda/runtime/doc.go | 22 ++ .../service/lambda/runtime/handler.go | 57 +++++ .../service/lambda/runtime/proxy.c | 226 ++++++++++++++++++ .../service/lambda/runtime/proxy.go | 98 ++++++++ vendor/vendor.json | 9 +- 8 files changed, 757 insertions(+), 1 deletion(-) create mode 100644 vendor/github.com/eawsy/aws-lambda-go/LICENSE create mode 100644 vendor/github.com/eawsy/aws-lambda-go/NOTICE create mode 100644 vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/context.go create mode 100644 vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/doc.go create mode 100644 vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/handler.go create mode 100644 vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/proxy.c create mode 100644 vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/proxy.go diff --git a/vendor/github.com/eawsy/aws-lambda-go/LICENSE b/vendor/github.com/eawsy/aws-lambda-go/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/vendor/github.com/eawsy/aws-lambda-go/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/eawsy/aws-lambda-go/NOTICE b/vendor/github.com/eawsy/aws-lambda-go/NOTICE new file mode 100644 index 00000000..59f2b765 --- /dev/null +++ b/vendor/github.com/eawsy/aws-lambda-go/NOTICE @@ -0,0 +1,10 @@ +Copyright 2016 Alsanium, SAS. or its affiliates. All rights reserved. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"); you may not use this product except in compliance with the +License. + +This product may include a number of subcomponents with separate copyright +notices and license terms. Your use of these subcomponents is subject to the +terms and conditions of the subcomponent's license, as noted in the LICENSE +file. diff --git a/vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/context.go b/vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/context.go new file mode 100644 index 00000000..fe478b29 --- /dev/null +++ b/vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/context.go @@ -0,0 +1,134 @@ +// +// Copyright 2016 Alsanium, SAS. or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package runtime + +// CognitoIdentity provides information about the Amazon Cognito identity +// provider when Lambda function invoked through the AWS Mobile SDK. +// +// For more information about the exact values for a specific mobile platform, +// see Identity Context in the AWS Mobile SDK for iOS Developer Guide, and +// Identity Context in the AWS Mobile SDK for Android Developer Guide. +// +// http://aws.amazon.com/cognito/ +// http://docs.aws.amazon.com/mobile/sdkforios/developerguide/lambda.html#identitycontext +// http://docs.aws.amazon.com/mobile/sdkforandroid/developerguide/lambda.html#identity-context +type CognitoIdentity struct { + // Amazon Cognito identity ID + IdentityID string `json:"cognito_identity_id"` + + // Amazon Cognito identity pool ID + IdentityPoolID string `json:"cognito_identity_pool_id"` +} + +// Client provides information about the client application when Lambda function +// invoked through the AWS Mobile SDK. +// +// For more information about the exact values for a specific mobile platform, +// see Client Context in the AWS Mobile SDK for iOS Developer Guide, and Client +// Context in the AWS Mobile SDK for Android Developer Guide. +// +// http://docs.aws.amazon.com/mobile/sdkforios/developerguide/lambda.html#clientcontext +// http://docs.aws.amazon.com/mobile/sdkforandroid/developerguide/lambda.html#client-context +type Client struct { + InstallationID string `json:"installation_id"` + AppTitle string `json:"app_title"` + AppVersionName string `json:"app_version_name"` + AppVersionCode string `json:"app_version_code"` + AppPackageName string `json:"app_package_name"` +} + +// ClientContext provides information about the client application and device +// when Lambda function invoked through the AWS Mobile SDK. +// +// For more information about the exact values for a specific mobile platform, +// see Client Context in the AWS Mobile SDK for iOS Developer Guide, and Client +// Context in the AWS Mobile SDK for Android Developer Guide. +// +// http://docs.aws.amazon.com/mobile/sdkforios/developerguide/lambda.html#clientcontext +// http://docs.aws.amazon.com/mobile/sdkforandroid/developerguide/lambda.html#client-context +type ClientContext struct { + // Client information provided by AWS Mobile SDK. + Client *Client `json:"client,omitempty"` + + // Custom values set by mobile client application. + Custom map[string]string `json:"custom"` + + // Environment information provided by AWS Mobile SDK. + Environment map[string]string `json:"env"` +} + +// Context provides information about Lambda execution environment. +// +// For example, you can use the Context to determine the +// CloudWatch log stream associated with the function or use the ClientContext +// property of the Context to learn more about the application calling the +// Lambda function (when invoked through the AWS Mobile SDK). +type Context struct { + // The name of the Lambda function that is executing. + FunctionName string `json:"function_name"` + + // The version of the Lambda function that is executing. + // If an alias is used to invoke the function, then FunctionVersion will be + // the version the alias points to. + FunctionVersion string `json:"function_version"` + + // The ARN used to invoke the Lambda function. + // It can be function ARN or alias ARN. An unqualified ARN executes the + // $LATEST version and aliases execute the function version they are pointing + // to. + InvokedFunctionARN string `json:"invoked_function_arn"` + + // The Memory limit, in MB, configured for the Lambda function. + MemoryLimitInMB int `json:"memory_limit_in_mb,string"` + + // The AWS request ID associated with the request. + // This is the ID returned to the client invoked the function. The request ID + // can be used for any follow up enquiry with AWS support. Note that if Lambda + // retries the function (for example, in a situation where the Lambda function + // processing Amazon Kinesis records throw an exception), the request ID + // remains the same. + AWSRequestID string `json:"aws_request_id"` + + // The CloudWatch log group name of the Lambda function that is executing. + // It can be empty if the IAM user provided does not have permission for + // CloudWatch actions. + LogGroupName string `json:"log_group_name"` + + // The CloudWatch log stream name of the Lambda function that is executing. + // It can be empty if the IAM user provided does not have permission for + // CloudWatch actions. + LogStreamName string `json:"log_stream_name"` + + // The information about the Amazon Cognito identity provider when the Lambda + // function invoked through the AWS Mobile SDK. + // It can be nil. + Identity *CognitoIdentity `json:"identity,omitempty"` + + // The information about the client application and device when the Lambda + // function invoked through the AWS Mobile SDK. + // It can be nil. + ClientContext *ClientContext `json:"client_context,omitempty"` + + // RemainingTimeInMillis returns remaining execution time till the function + // will be terminated, in milliseconds. + // + // The maximum time limit at which Lambda will terminate the function + // execution is set at the time the Lambda function is created. Information + // about the remaining time of function execution can be used to specify + // function behavior when nearing the timeout. + RemainingTimeInMillis func() int64 +} diff --git a/vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/doc.go b/vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/doc.go new file mode 100644 index 00000000..f90f6f62 --- /dev/null +++ b/vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/doc.go @@ -0,0 +1,22 @@ +// +// Copyright 2016 Alsanium, SAS. or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +/* +Package runtime provides a fast and clean way to execute Go on AWS Lambda. + +Take a tour at https://github.com/eawsy/aws-lambda-go +*/ +package runtime diff --git a/vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/handler.go b/vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/handler.go new file mode 100644 index 00000000..82821280 --- /dev/null +++ b/vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/handler.go @@ -0,0 +1,57 @@ +// +// Copyright 2016 Alsanium, SAS. or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package runtime + +import "encoding/json" + +var handler Handler + +// Handler responds to a Lambda function invocation. +// +// HandleLambda can optionally return a value or an error. What happens to the +// returned value depends on the invocation type used when invoking the Lambda +// function. If the Lambda function returns an error, AWS Lambda recognizes the +// failure and sealizes the error information into JSON and returns it. How to +// get the error information back depends to the invocation type that client +// specifies at the time of function invocation. +// +// If HandleLambda panics, the handler (the caller of HandleLambda) recovers the +// panic, logs a stack trace to the CloudWatch log stream, and terminate the +// Lambda function execution. +type Handler interface { + HandleLambda(json.RawMessage, *Context) (interface{}, error) +} + +// HandlerFunc type is an adapter to allow the use of ordinary functions as +// Lambda handlers. If f is a function with the appropriate signature, +// HandlerFunc(f) is a Handler that calls f. +type HandlerFunc func(json.RawMessage, *Context) (interface{}, error) + +// HandleLambda calls f(evt, ctx) +func (f HandlerFunc) HandleLambda(evt json.RawMessage, ctx *Context) (interface{}, error) { + return f(evt, ctx) +} + +// Handle registers the given handler. +func Handle(h Handler) { + handler = h +} + +// HandleFunc registers the given handler function. +func HandleFunc(h func(json.RawMessage, *Context) (interface{}, error)) { + Handle(HandlerFunc(h)) +} diff --git a/vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/proxy.c b/vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/proxy.c new file mode 100644 index 00000000..79530f0f --- /dev/null +++ b/vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/proxy.c @@ -0,0 +1,226 @@ +// +build proxy + +// +// Copyright 2016 Alsanium, SAS. or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct handle_return { char* r0; char* r1; }; +extern struct handle_return handle(char*, char*, char*); + +static PyObject *module_error = NULL; +static PyObject* proxy_log_fn = NULL; +static PyObject* proxy_get_remaining_time_in_millis_fn = NULL; + +void +proxy_log(char* msg) +{ + PyObject* tmp = PyObject_CallFunction(proxy_log_fn, "s", msg); + Py_DECREF(tmp); + free(msg); +} + +long long +proxy_get_remaining_time_in_millis() +{ + PyObject* pms = PyObject_CallFunctionObjArgs(proxy_get_remaining_time_in_millis_fn, NULL); + unsigned long ms = PyLong_AsLongLong(pms); + Py_DECREF(pms); + return ms; +} + +static PyObject* py_json_module; +static PyObject* py_os_module; + +static void +PyDict_Free(PyObject* p) +{ + PyObject* es = PyDict_Values(p); + for (Py_ssize_t i = 0; i < PyList_Size(es); i++) { + PyObject* e = PyList_GetItem(es, i); + if (PyDict_Check(e)) { + PyDict_Free(e); + } else { + Py_DECREF(e); + } + } + Py_DECREF(es); + Py_DECREF(p); +} + +static void +PyDict_SetSomeItemString(PyObject* p, char *key, PyObject* val) +{ + if ((val != Py_None) && (!PyDict_Check(val) || (PyDict_Check(val) && PyDict_Size(val)))) { + PyDict_SetItemString(p, key, val); + } else { + Py_DECREF(val); + } +} + +static void +PyDict_CopyItemString(PyObject* p1, PyObject* p2, char *key) +{ + PyDict_SetSomeItemString(p1, key, PyObject_GetAttrString(p2, key)); +} + +static PyObject* +proxy_marshal_ctx(PyObject* raw) +{ + PyObject* ctx = PyDict_New(); + + PyDict_CopyItemString(ctx, raw, "function_name"); + PyDict_CopyItemString(ctx, raw, "function_version"); + PyDict_CopyItemString(ctx, raw, "invoked_function_arn"); + PyDict_CopyItemString(ctx, raw, "memory_limit_in_mb"); + PyDict_CopyItemString(ctx, raw, "aws_request_id"); + PyDict_CopyItemString(ctx, raw, "log_group_name"); + PyDict_CopyItemString(ctx, raw, "log_stream_name"); + + PyObject* raw_identity = PyObject_GetAttrString(raw, "identity"); + PyObject* identity = PyDict_New(); + if (raw_identity != Py_None) { + PyDict_CopyItemString(identity, raw_identity, "cognito_identity_id"); + PyDict_CopyItemString(identity, raw_identity, "cognito_identity_pool_id"); + } + PyDict_SetSomeItemString(ctx, "identity", identity); + Py_DECREF(raw_identity); + + PyObject* raw_client_context = PyObject_GetAttrString(raw, "client_context"); + PyObject* client_context = PyDict_New(); + if (raw_client_context != Py_None) { + PyObject* raw_client = PyObject_GetAttrString(raw_client_context, "client"); + PyObject* client = PyDict_New(); + if (raw_client != Py_None) { + PyDict_CopyItemString(client, raw_client, "installation_id"); + PyDict_CopyItemString(client, raw_client, "app_title"); + PyDict_CopyItemString(client, raw_client, "app_version_name"); + PyDict_CopyItemString(client, raw_client, "app_version_code"); + PyDict_CopyItemString(client, raw_client, "app_package_name"); + } + PyDict_SetSomeItemString(client_context, "client", client); + Py_DECREF(raw_client); + + PyDict_CopyItemString(client_context, raw_client_context, "env"); + PyDict_CopyItemString(client_context, raw_client_context, "custom"); + } + PyDict_SetSomeItemString(ctx, "client_context", client_context); + Py_DECREF(raw_client_context); + + PyObject* str = PyObject_CallMethod(py_json_module, "dumps", "O", ctx); + + PyDict_Free(ctx); + + return str; +} + +static PyObject* +proxy_marshal_env() +{ + PyObject* environ = PyObject_GetAttrString(py_os_module, "environ"); + PyObject* dict = PyObject_GetAttrString(environ, "__dict__"); + PyObject* env = PyDict_GetItemString(dict, "data"); + + PyObject* str = PyObject_CallMethod(py_json_module, "dumps", "O", env); + + Py_DECREF(environ); + Py_DECREF(dict); + + return str; +} + +static PyObject* +proxy_handle(PyObject* self, PyObject* args) +{ + PyObject *west = NULL; + PyObject *evt = NULL; + PyObject *ctx = NULL; + + PyArg_ParseTuple(args, "OO", &evt, &ctx); + + proxy_log_fn = PyObject_GetAttrString(ctx, "log"); + proxy_get_remaining_time_in_millis_fn = PyObject_GetAttrString(ctx, "get_remaining_time_in_millis"); + + PyObject* jevt = PyObject_CallMethod(py_json_module, "dumps", "O", evt); + PyObject* jctx = proxy_marshal_ctx(ctx); + PyObject* jenv = proxy_marshal_env(); + + char* sevt = PyString_AsString(jevt); + char* sctx = PyString_AsString(jctx); + char* senv = PyString_AsString(jenv); + + struct handle_return east = handle(sevt, sctx, senv); + + if (east.r0 != NULL) { + PyObject* tmp = PyString_FromString(east.r0); + west = PyObject_CallMethod(py_json_module, "loads", "O", tmp); + Py_DECREF(tmp); + free(east.r0); + } else if (east.r1 != NULL) { + PyErr_SetString(module_error, east.r1); + free(east.r1); + west = NULL; + } else { + Py_INCREF(Py_None); + west = Py_None; + } + + Py_DECREF(proxy_log_fn); + Py_DECREF(proxy_get_remaining_time_in_millis_fn); + Py_DECREF(jevt); + Py_DECREF(jctx); + Py_DECREF(jenv); + + return west; +} + +#define STR(s) #s +#define XSTR(s) STR(s) +#define INIT_MODULE_SIG(name) PyMODINIT_FUNC init ## name (void) +#define INIT_MODULE(name) INIT_MODULE_SIG(name) + +#ifndef FUNCTION +#define FUNCTION handler +#endif + +#ifndef HANDLER +#define HANDLER handle +#endif + +static PyMethodDef +module_methods[] = { + {XSTR(HANDLER), (PyCFunction)proxy_handle, METH_VARARGS}, + {NULL, NULL} +}; + +INIT_MODULE(FUNCTION) +{ + py_json_module = PyImport_ImportModule("json"); + py_os_module = PyImport_ImportModule("os"); + PyObject *module = Py_InitModule(XSTR(FUNCTION), module_methods); + module_error = PyErr_NewException(XSTR(FUNCTION)".error", NULL, NULL); + Py_INCREF(module_error); + PyModule_AddObject(module, "error", module_error); +} + +#ifdef __cplusplus +} +#endif diff --git a/vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/proxy.go b/vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/proxy.go new file mode 100644 index 00000000..7183d377 --- /dev/null +++ b/vendor/github.com/eawsy/aws-lambda-go/service/lambda/runtime/proxy.go @@ -0,0 +1,98 @@ +// +build proxy + +// +// Copyright 2016 Alsanium, SAS. or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package runtime + +// #cgo pkg-config: python2 +// #cgo CFLAGS: --std=gnu11 +// extern long long proxy_get_remaining_time_in_millis(); +// extern void proxy_log(char*); +import "C" + +import ( + "encoding/json" + "fmt" + "log" + "os" + "runtime" + "time" +) + +var logger *ilogger + +type ilogger struct { + id string +} + +func (l *ilogger) Write(info []byte) (int, error) { + now := time.Now().UTC().Format("2006-01-02T15:04:05.999Z") + C.proxy_log(C.CString(fmt.Sprintf("%s\t%s\t%s", now, l.id, string(info)))) + return len(info), nil +} + +//export handle +func handle(revt, rctx, renv *C.char) (rres *C.char, rerr *C.char) { + defer func() { + if err := recover(); err != nil { + const size = 64 << 10 + buf := make([]byte, size) + buf = buf[:runtime.Stack(buf, false)] + log.Printf("%s\n%s", err, buf) + rres = nil + rerr = C.CString(fmt.Sprintf("%s", err)) + } + }() + + evt := json.RawMessage([]byte(C.GoString(revt))) + ctx := &Context{} + + if err := json.Unmarshal([]byte(C.GoString(rctx)), ctx); err != nil { + return nil, C.CString(err.Error()) + } + + ctx.RemainingTimeInMillis = func() int64 { + return int64(C.proxy_get_remaining_time_in_millis()) + } + + logger.id = ctx.AWSRequestID + + var env map[string]string + if err := json.Unmarshal([]byte(C.GoString(renv)), &env); err != nil { + return nil, C.CString(err.Error()) + } + for k, v := range env { + os.Setenv(k, v) + } + + if res, err := handler.HandleLambda(evt, ctx); err != nil { + return nil, C.CString(err.Error()) + } else if res != nil { + tmp, err := json.Marshal(res) + if err != nil { + return nil, C.CString(err.Error()) + } + return C.CString(string(tmp)), nil + } + return nil, nil +} + +func init() { + logger = &ilogger{"00000000-0000-0000-0000-000000000000"} + log.SetFlags(0) + log.SetOutput(logger) +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 9ba6a151..1e62c2bb 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -1,6 +1,13 @@ { "comment": "", "ignore": "test", - "package": [], + "package": [ + { + "checksumSHA1": "/j5eaUxSjyT802BWlCZ8eWdxJZQ=", + "path": "github.com/eawsy/aws-lambda-go/service/lambda/runtime", + "revision": "39dc993a238c10d77d901357b39831d04468aa40", + "revisionTime": "2016-11-16T16:19:46Z" + } + ], "rootPath": "github.com/buildkite/buildkite-metrics" } From bf582a0c18e9b05f2c9f15fe336a9ab26be3c3b7 Mon Sep 17 00:00:00 2001 From: Lachlan Donald Date: Fri, 16 Dec 2016 18:53:02 +1100 Subject: [PATCH 3/8] Update bk pipeline to build lambda in second step --- .buildkite/pipeline.yml | 10 ++++++---- Makefile | 4 +++- docker-compose.yml | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 289822e3..ac4ecd02 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,16 +1,18 @@ steps: - - - name: "📦" + - name: ":golang:" command: "make build" artifact_paths: "build/*" plugins: - docker-compose: run: "build" + - name: ":lambda:" + command: "make build-lambda" + artifact_paths: "build/*" + - wait - - - name: "Release to :s3:" + - name: "Release to :s3:" command: ".buildkite/upload.sh" branches: "master" agents: diff --git a/Makefile b/Makefile index 6de50a77..fee353e2 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,9 @@ BIN=build/buildkite-metrics-$(shell uname -s)-$(shell uname -m)-$(VERSION) LAMBDA_ZIP=$(BIN)-lambda.zip SRC=$(shell find . -name '*.go') -build: $(BIN) $(LAMBDA_ZIP) +build: $(BIN) + +build-lambda: $(LAMBDA_ZIP) clean: -rm -f build/ diff --git a/docker-compose.yml b/docker-compose.yml index 7926eb25..d1b4eac3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,4 +3,4 @@ build: command: "make clean build" volumes: - .:/go/src/github.com/buildkite/buildkite-metrics - working_dir: /go/src/github.com/buildkite/buildkite-metrics \ No newline at end of file + working_dir: /go/src/github.com/buildkite/buildkite-metrics From e85c1d679f8e15b73b9bf958450d8293755706f1 Mon Sep 17 00:00:00 2001 From: Lachlan Donald Date: Fri, 16 Dec 2016 19:02:07 +1100 Subject: [PATCH 4/8] make build-lambda works without gopath --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index fee353e2..f3e8cb83 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,10 @@ $(BIN): $(SRC) -mkdir -p build/ go build -o $(BIN) -ldflags="$(FLAGS)" . +GODIR=/go/src/github.com/buildkite/buildkite-metrics + $(LAMBDA_ZIP): $(SRC) - docker run --rm -v $(GOPATH):/go -v $(PWD):/tmp eawsy/aws-lambda-go + docker run --rm -v $(PWD):$(GODIR) -w $(GODIR) eawsy/aws-lambda-go mv handler.zip $(LAMBDA_ZIP) upload: From 650a38a4d6fd699b01260d85ae80722b78a415d1 Mon Sep 17 00:00:00 2001 From: Lachlan Donald Date: Fri, 16 Dec 2016 19:04:08 +1100 Subject: [PATCH 5/8] Create build dir for lambda-build --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index f3e8cb83..cb5af271 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ GODIR=/go/src/github.com/buildkite/buildkite-metrics $(LAMBDA_ZIP): $(SRC) docker run --rm -v $(PWD):$(GODIR) -w $(GODIR) eawsy/aws-lambda-go + -mkdir -p build/ mv handler.zip $(LAMBDA_ZIP) upload: From 95767c2c89cebc99c27966e06567997d4e0a32aa Mon Sep 17 00:00:00 2001 From: Lachlan Donald Date: Fri, 16 Dec 2016 19:06:44 +1100 Subject: [PATCH 6/8] Run upload on branches --- .buildkite/pipeline.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index ac4ecd02..7a966952 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -14,6 +14,5 @@ steps: - name: "Release to :s3:" command: ".buildkite/upload.sh" - branches: "master" agents: queue: "deploy" From d54c3df5d7bb9c5dcd35321d32fed43cc1f295ab Mon Sep 17 00:00:00 2001 From: Lachlan Donald Date: Fri, 16 Dec 2016 19:08:45 +1100 Subject: [PATCH 7/8] Drop platform and arch from lambda package name --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cb5af271..56174985 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ VERSION=$(shell git fetch --tags && git describe --tags --candidates=1 --dirty --always) FLAGS=-X main.Version=$(VERSION) BIN=build/buildkite-metrics-$(shell uname -s)-$(shell uname -m)-$(VERSION) -LAMBDA_ZIP=$(BIN)-lambda.zip +LAMBDA_ZIP=build/buildkite-metrics-$(VERSION)-lambda.zip SRC=$(shell find . -name '*.go') build: $(BIN) From 4df37473ef0d8be0f15cbf629981162752a72373 Mon Sep 17 00:00:00 2001 From: Lachlan Donald Date: Fri, 16 Dec 2016 19:13:16 +1100 Subject: [PATCH 8/8] Improve upload output --- .buildkite/upload.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.buildkite/upload.sh b/.buildkite/upload.sh index 47d5c09e..ada01158 100755 --- a/.buildkite/upload.sh +++ b/.buildkite/upload.sh @@ -5,7 +5,14 @@ export AWS_DEFAULT_REGION=us-east-1 export AWS_ACCESS_KEY_ID=$SANDBOX_AWS_ACCESS_KEY_ID export AWS_SECRET_ACCESS_KEY=$SANDBOX_AWS_SECRET_ACCESS_KEY +echo "~~~ :buildkite: Downloading artifacts" mkdir -p build/ buildkite-agent artifact download "build/*" build/ -make upload branch=${BUILDKITE_BRANCH} \ No newline at end of file +echo "~~~ :s3: Uploading files" +make upload branch=${BUILDKITE_BRANCH} + +echo "+++ :s3: Upload complete" +for f in $(ls build/) ; do + printf "https://buildkite-metrics.s3.amazonaws.com/%s\n" $f +done \ No newline at end of file