From 0448a77b58f0601af77779520dfa89ee6df341d4 Mon Sep 17 00:00:00 2001 From: Peter Baumgartner Date: Mon, 30 Oct 2023 15:49:05 +0000 Subject: [PATCH 1/7] Use CNB builder for heroku-22 stack refs #5 --- builder/build/appjson.go | 25 ++++++++++++++++--------- builder/build/appjson_test.go | 4 ++-- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/builder/build/appjson.go b/builder/build/appjson.go index 212a49b..177b957 100644 --- a/builder/build/appjson.go +++ b/builder/build/appjson.go @@ -33,18 +33,18 @@ const DefaultStack = "heroku-20" var IncludedBuildpacks = map[string][]string{ "heroku-20": { // $ pack builder inspect heroku/buildpacks:20 -o json | jq '.remote_info.buildpacks[].id' + "heroku/builder-eol-warning", "heroku/go", "heroku/gradle", "heroku/java", - "heroku/java-function", "heroku/jvm", - "heroku/jvm-function-invoker", "heroku/maven", "heroku/nodejs", + "heroku/nodejs-corepack", "heroku/nodejs-engine", - "heroku/nodejs-function", - "heroku/nodejs-function-invoker", - "heroku/nodejs-npm", + "heroku/nodejs-npm-engine", + "heroku/nodejs-npm-install", + "heroku/nodejs-pnpm-install", "heroku/nodejs-yarn", "heroku/php", "heroku/procfile", @@ -53,16 +53,24 @@ var IncludedBuildpacks = map[string][]string{ "heroku/scala", }, "heroku-22": { - // $ pack builder inspect heroku/builder-classic:22 -o json | jq '.remote_info.buildpacks[].id' - "heroku/clojure", + // $ pack builder inspect heroku/builder:22 -o json | jq '.remote_info.buildpacks[].id' "heroku/go", "heroku/gradle", "heroku/java", + "heroku/jvm", + "heroku/maven", "heroku/nodejs", + "heroku/nodejs-corepack", + "heroku/nodejs-engine", + "heroku/nodejs-npm-engine", + "heroku/nodejs-npm-install", + "heroku/nodejs-pnpm-install", + "heroku/nodejs-yarn", "heroku/php", "heroku/procfile", "heroku/python", "heroku/ruby", + "heroku/sbt", "heroku/scala", }, } @@ -131,9 +139,8 @@ func (a *AppJSON) GetBuilders() []string { if a.Stack == "heroku-20" { return []string{"heroku/buildpacks:20", "heroku/heroku:20-cnb"} } - // TODO: use heroku/builder:22 can be used when all buildpacks are CNB if a.Stack == "heroku-22" { - return []string{"heroku/builder-classic:22", "heroku/heroku:22-cnb"} + return []string{"heroku/builder:22", "heroku/heroku:22-cnb"} } return []string{a.Stack} } diff --git a/builder/build/appjson_test.go b/builder/build/appjson_test.go index 7c8de36..416c6f6 100644 --- a/builder/build/appjson_test.go +++ b/builder/build/appjson_test.go @@ -76,7 +76,7 @@ func TestAppJsonBuilders(t *testing.T) { Stack: "heroku-22", ctx: testContext, } - expected := []string{"heroku/builder-classic:22", "heroku/heroku:22-cnb"} + expected := []string{"heroku/builder:22", "heroku/heroku:22-cnb"} if !stringSliceEqual(a.GetBuilders(), expected) { t.Errorf("expected %s, got %s", expected, a.GetBuilders()) } @@ -159,7 +159,7 @@ func TestAppJsonToApppackToml(t *testing.T) { Build: AppPackTomlBuild{ System: "buildpack", Buildpacks: []string{"urn:cnb:builder:heroku/nodejs", "urn:cnb:builder:heroku/python"}, - Builder: "heroku/builder-classic:22", + Builder: "heroku/builder:22", }, Test: AppPackTomlTest{ Command: "echo test", From 5ef20467d51d83928de0f252a8952fea699f0d7e Mon Sep 17 00:00:00 2001 From: Peter Baumgartner Date: Wed, 27 Mar 2024 09:43:56 -0600 Subject: [PATCH 2/7] Switch default builder to heroku/builder:22 --- builder/build/appjson.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/build/appjson.go b/builder/build/appjson.go index 177b957..c8ca705 100644 --- a/builder/build/appjson.go +++ b/builder/build/appjson.go @@ -27,7 +27,7 @@ type AppJSON struct { ctx context.Context } -const DefaultStack = "heroku-20" +const DefaultStack = "heroku-22" // buildpacks included in builder var IncludedBuildpacks = map[string][]string{ From 83f75f749346dd160c6a0de0485c16788427374b Mon Sep 17 00:00:00 2001 From: Peter Baumgartner Date: Wed, 27 Mar 2024 09:44:13 -0600 Subject: [PATCH 3/7] Add new buildpack --- builder/build/appjson.go | 1 + 1 file changed, 1 insertion(+) diff --git a/builder/build/appjson.go b/builder/build/appjson.go index c8ca705..3c32a0b 100644 --- a/builder/build/appjson.go +++ b/builder/build/appjson.go @@ -64,6 +64,7 @@ var IncludedBuildpacks = map[string][]string{ "heroku/nodejs-engine", "heroku/nodejs-npm-engine", "heroku/nodejs-npm-install", + "heroku/nodejs-pnpm-engine", "heroku/nodejs-pnpm-install", "heroku/nodejs-yarn", "heroku/php", From 31d3f402dc265b732c07a6abaf865d8baf8593db Mon Sep 17 00:00:00 2001 From: Peter Baumgartner Date: Tue, 16 Apr 2024 17:50:50 -0600 Subject: [PATCH 4/7] Test app.json too --- .github/workflows/build.yml | 60 ++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2faa98b..46fb848 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -99,6 +99,64 @@ jobs: set -ex cat apppack.toml test "$(python -c 'import tomllib; print(tomllib.load(open("apppack.toml", "rb"))["services"]["web"]["command"])')" = 'gunicorn --access-logfile - --bind 0.0.0.0:$PORT --forwarded-allow-ips '"'"'*'"' app:app" + integration-appjson: + runs-on: ubuntu-latest + needs: [test, build-image] + permissions: + id-token: write + contents: read + packages: read + steps: + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Pull image + run: docker pull ${{ needs.build-image.outputs.image }} + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1-node16 + with: + role-to-assume: arn:aws:iam::891426818781:role/github-actions-integration-tests + aws-region: us-east-1 + - name: Checkout sample repo + run: git clone --branch app.json https://github.com/apppackio/apppack-demo-python.git + - name: Run integration tests + working-directory: ./apppack-demo-python + run: | + cat < .envfile + APPNAME=gh-integration + CODEBUILD_BUILD_ID=demo-python:${{ github.run_id }} + CODEBUILD_SOURCE_VERSION=${{ github.sha }} + DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_ACCESS_TOKEN=${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + DOCKER_REPO=891426818781.dkr.ecr.us-east-1.amazonaws.com/github-integration-test + ARTIFACT_BUCKET=integration-test-buildartifacts + AWS_REGION + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY + AWS_SESSION_TOKEN + EOF + + docker run \ + --rm \ + --privileged \ + --env-file .envfile \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --volume "$(pwd):/app" \ + --workdir /app \ + ${{ needs.build-image.outputs.image }} \ + /bin/sh -c "set -x; git config --global --add safe.directory /app && apppack-builder prebuild; apppack-builder build; apppack-builder postbuild" + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Verify apppack.toml + working-directory: ./apppack-demo-python + run: | + set -ex + cat apppack.toml + test "$(python -c 'import tomllib; print(tomllib.load(open("apppack.toml", "rb"))["services"]["web"]["command"])')" = 'gunicorn --access-logfile - --bind 0.0.0.0:$PORT --forwarded-allow-ips '"'"'*'"' app:app" integration-docker: runs-on: ubuntu-latest needs: [test, build-image] @@ -161,7 +219,7 @@ jobs: runs-on: ubuntu-latest # Only run this job if the current commit is tagged with a version if: startswith(github.ref, 'refs/tags/v') - needs: [test, build-image, integration, integration-docker] + needs: [test, build-image, integration, integration-docker, integration-appjson] permissions: id-token: write contents: read From 62d0a686a98761a37614e39ca63b8083dd56d8c7 Mon Sep 17 00:00:00 2001 From: Peter Baumgartner Date: Tue, 16 Apr 2024 18:34:07 -0600 Subject: [PATCH 5/7] Procfile buildpack now uses command and args --- builder/build/metadatatoml.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/builder/build/metadatatoml.go b/builder/build/metadatatoml.go index b2e6d4a..b01f4a8 100644 --- a/builder/build/metadatatoml.go +++ b/builder/build/metadatatoml.go @@ -11,6 +11,7 @@ import ( type BuildpackMetadataTomlProcess struct { Command []string `toml:"command"` + Args []string `toml:"args"` Type string `toml:"type"` BuildpackID string `toml:"buildpack_id"` } @@ -37,7 +38,7 @@ func (m *BuildpackMetadataToml) UpdateAppPackToml(a *AppPackToml) { if process.BuildpackID == "heroku/ruby" && (process.Type == "rake" || process.Type == "console") { continue } - a.Services[process.Type] = AppPackTomlService{Command: commandSliceToString(process.Command)} + a.Services[process.Type] = AppPackTomlService{Command: commandSliceToString(append(process.Command, process.Args...))} } } From 84f9c210a8131069846cd5c5606783e3429ca797 Mon Sep 17 00:00:00 2001 From: Peter Baumgartner Date: Tue, 16 Apr 2024 18:38:19 -0600 Subject: [PATCH 6/7] Test against legacy Heroku builder --- .github/workflows/build.yml | 63 ++++++++++++++++++++++++++++++++++++- builder/build/build.go | 5 +++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 46fb848..720a92a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -157,6 +157,67 @@ jobs: set -ex cat apppack.toml test "$(python -c 'import tomllib; print(tomllib.load(open("apppack.toml", "rb"))["services"]["web"]["command"])')" = 'gunicorn --access-logfile - --bind 0.0.0.0:$PORT --forwarded-allow-ips '"'"'*'"' app:app" + + integration-heroku20: + runs-on: ubuntu-latest + needs: [test, build-image] + permissions: + id-token: write + contents: read + packages: read + steps: + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Pull image + run: docker pull ${{ needs.build-image.outputs.image }} + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1-node16 + with: + role-to-assume: arn:aws:iam::891426818781:role/github-actions-integration-tests + aws-region: us-east-1 + - name: Checkout sample repo + run: git clone --branch buildpacks-20 https://github.com/apppackio/apppack-demo-python.git + - name: Run integration tests + working-directory: ./apppack-demo-python + run: | + cat < .envfile + APPNAME=gh-integration + CODEBUILD_BUILD_ID=demo-python:${{ github.run_id }} + CODEBUILD_SOURCE_VERSION=${{ github.sha }} + DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_ACCESS_TOKEN=${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + DOCKER_REPO=891426818781.dkr.ecr.us-east-1.amazonaws.com/github-integration-test + ARTIFACT_BUCKET=integration-test-buildartifacts + ALLOW_EOL_SHIMMED_BUILDER=1 + AWS_REGION + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY + AWS_SESSION_TOKEN + EOF + + docker run \ + --rm \ + --privileged \ + --env-file .envfile \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --volume "$(pwd):/app" \ + --workdir /app \ + ${{ needs.build-image.outputs.image }} \ + /bin/sh -c "set -x; git config --global --add safe.directory /app && apppack-builder prebuild; apppack-builder build; apppack-builder postbuild" + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Verify apppack.toml + working-directory: ./apppack-demo-python + run: | + set -ex + cat apppack.toml + test "$(python -c 'import tomllib; print(tomllib.load(open("apppack.toml", "rb"))["services"]["web"]["command"])')" = 'gunicorn --access-logfile - --bind 0.0.0.0:$PORT --forwarded-allow-ips '"'"'*'"' app:app" + integration-docker: runs-on: ubuntu-latest needs: [test, build-image] @@ -219,7 +280,7 @@ jobs: runs-on: ubuntu-latest # Only run this job if the current commit is tagged with a version if: startswith(github.ref, 'refs/tags/v') - needs: [test, build-image, integration, integration-docker, integration-appjson] + needs: [test, build-image, integration, integration-docker, integration-appjson, integration-heroku20] permissions: id-token: write contents: read diff --git a/builder/build/build.go b/builder/build/build.go index 61f7621..ccc16f9 100644 --- a/builder/build/build.go +++ b/builder/build/build.go @@ -32,6 +32,11 @@ func (b *Build) LoadBuildEnv() (map[string]string, error) { env := map[string]string{ "CI": "true", } + // pass ALLOW_EOL_SHIMMED_BUILDER to pack if it is in the environment + // this facilitates testing of the eol shimmed builder + if val, ok := os.LookupEnv("ALLOW_EOL_SHIMMED_BUILDER"); ok { + env["ALLOW_EOL_SHIMMED_BUILDER"] = val + } params, err := b.aws.GetParametersByPath(paths[0]) stripParamPrefix(params, paths[0], &env) if err != nil { From 2123e7cbfd8b8377930ac76459552069ec2e3d1f Mon Sep 17 00:00:00 2001 From: Peter Baumgartner Date: Tue, 16 Apr 2024 18:45:21 -0600 Subject: [PATCH 7/7] Adjust tests for bash wrapper New procfile buildpacks add it --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 720a92a..351ff38 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -98,7 +98,8 @@ jobs: run: | set -ex cat apppack.toml - test "$(python -c 'import tomllib; print(tomllib.load(open("apppack.toml", "rb"))["services"]["web"]["command"])')" = 'gunicorn --access-logfile - --bind 0.0.0.0:$PORT --forwarded-allow-ips '"'"'*'"' app:app" + test "$(python -c 'import tomllib; print(tomllib.load(open("apppack.toml", "rb"))["services"]["web"]["command"])')" = "bash -c 'gunicorn --access-logfile - --bind 0.0.0.0:\$PORT --forwarded-allow-ips '\"'\"'*'\"'\"' app:app'" + integration-appjson: runs-on: ubuntu-latest needs: [test, build-image] @@ -156,7 +157,7 @@ jobs: run: | set -ex cat apppack.toml - test "$(python -c 'import tomllib; print(tomllib.load(open("apppack.toml", "rb"))["services"]["web"]["command"])')" = 'gunicorn --access-logfile - --bind 0.0.0.0:$PORT --forwarded-allow-ips '"'"'*'"' app:app" + test "$(python -c 'import tomllib; print(tomllib.load(open("apppack.toml", "rb"))["services"]["web"]["command"])')" = "bash -c 'gunicorn --access-logfile - --bind 0.0.0.0:\$PORT --forwarded-allow-ips '\"'\"'*'\"'\"' app:app'" integration-heroku20: runs-on: ubuntu-latest