From c3679c762fffb6426394ce32e688ee3a3d7cf8bb Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Sun, 12 Feb 2023 19:57:24 -0800 Subject: [PATCH] split up utility targets from main dockerfile We can not use bake with linked targets to split up ld, sdk-extras and llvm in own subdirectories. Signed-off-by: Tonis Tiigi --- .github/workflows/build.yml | 11 +- .github/workflows/ld.yml | 108 +++++++++ .github/workflows/llvm.yml | 52 +++++ .github/workflows/sdk-extras.yml | 37 +++ docker-bake.hcl | 121 ++++++++-- hack/dockerfiles/shellcheck.Dockerfile | 4 +- hack/dockerfiles/shfmt.Dockerfile | 2 +- src/Dockerfile | 110 +++++++++ {base => src}/fixtures/hello.c | 0 {base => src}/fixtures/hello.cc | 0 {base => src}/fixtures/hello.go | 0 {base => src}/fixtures/hello_cargo/Cargo.toml | 0 .../fixtures/hello_cargo/src/main.rs | 0 {base => src}/fixtures/hello_cgo.go | 0 {base => src/ld}/Dockerfile | 214 +++--------------- {base => src/ld}/ld64.signed | 0 {base => src/ld}/libtapi-cmake-args.patch | 0 {base => src/ld}/sigtool-static.patch | 0 src/llvm/Dockerfile | 39 ++++ src/sdk-extras/Dockerfile | 23 ++ {base => src}/test-apk.bats | 0 {base => src}/test-apt.bats | 0 {base => src}/test-cargo.bats | 0 {base => src}/test-clang.bats | 0 {base => src}/test-go.bats | 0 {base => src}/test-info-alpine.bats | 0 {base => src}/test-info-common.bats | 0 {base => src}/test-info-debian.bats | 0 {base => src}/test-info-rhel.bats | 0 {base => src}/test-verify.bats | 0 {base => src}/test-windres.bats | 0 {base => src}/test_helper.bash | 0 {base => src}/xx-apk | 0 {base => src}/xx-apt | 0 {base => src}/xx-cargo | 0 {base => src}/xx-cc | 0 {base => src}/xx-go | 0 {base => src}/xx-info | 0 {base => src}/xx-verify | 0 {base => src}/xx-windres | 0 40 files changed, 508 insertions(+), 213 deletions(-) create mode 100644 .github/workflows/ld.yml create mode 100644 .github/workflows/llvm.yml create mode 100644 .github/workflows/sdk-extras.yml create mode 100644 src/Dockerfile rename {base => src}/fixtures/hello.c (100%) rename {base => src}/fixtures/hello.cc (100%) rename {base => src}/fixtures/hello.go (100%) rename {base => src}/fixtures/hello_cargo/Cargo.toml (100%) rename {base => src}/fixtures/hello_cargo/src/main.rs (100%) rename {base => src}/fixtures/hello_cgo.go (100%) rename {base => src/ld}/Dockerfile (59%) rename {base => src/ld}/ld64.signed (100%) rename {base => src/ld}/libtapi-cmake-args.patch (100%) rename {base => src/ld}/sigtool-static.patch (100%) create mode 100644 src/llvm/Dockerfile create mode 100644 src/sdk-extras/Dockerfile rename {base => src}/test-apk.bats (100%) rename {base => src}/test-apt.bats (100%) rename {base => src}/test-cargo.bats (100%) rename {base => src}/test-clang.bats (100%) rename {base => src}/test-go.bats (100%) rename {base => src}/test-info-alpine.bats (100%) rename {base => src}/test-info-common.bats (100%) rename {base => src}/test-info-debian.bats (100%) rename {base => src}/test-info-rhel.bats (100%) rename {base => src}/test-verify.bats (100%) rename {base => src}/test-windres.bats (100%) rename {base => src}/test_helper.bash (100%) rename {base => src}/xx-apk (100%) rename {base => src}/xx-apt (100%) rename {base => src}/xx-cargo (100%) rename {base => src}/xx-cc (100%) rename {base => src}/xx-go (100%) rename {base => src}/xx-info (100%) rename {base => src}/xx-verify (100%) rename {base => src}/xx-windres (100%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fc4d74e..301d607 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -132,9 +132,18 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Test + if: ${{ matrix.typ != 'rhel' }} uses: docker/bake-action@v2 with: - targets: test-${{ matrix.typ }} + targets: test + set: | + test-${{ matrix.typ }}.args.TEST_BASE_IMAGE=${{ matrix.image }} + - + name: Test (info) + if: ${{ matrix.typ == 'rhel' }} + uses: docker/bake-action@v2 + with: + targets: test-info set: | test-${{ matrix.typ }}.args.TEST_BASE_IMAGE=${{ matrix.image }} diff --git a/.github/workflows/ld.yml b/.github/workflows/ld.yml new file mode 100644 index 0000000..9d4dc89 --- /dev/null +++ b/.github/workflows/ld.yml @@ -0,0 +1,108 @@ +name: ld + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_dispatch: + push: + branches: + - 'master' + tags: + - 'v*' + paths: + - 'src/ld/**' + pull_request: + branches: + - 'master' + paths: + - 'src/ld/**' + +jobs: + ld64: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - + name: Build + uses: docker/bake-action@v2 + with: + files: | + ./docker-bake.hcl + targets: ld64-tgz + + binutils: + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.allow-failure }} + strategy: + fail-fast: false + matrix: + include: + - target: linux-arm64 + - target: linux-amd64 + - target: linux-armv7 + - target: linux-armv6 + - target: linux-ppc64le + - target: linux-386 + - target: linux-riscv64 + - target: linux-s390x + - target: windows-amd64 + - target: windows-386 + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - + name: Build + uses: docker/bake-action@v2 + with: + targets: binutils-${{ matrix.target }}-alpine + + ld: + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.allow-failure }} + strategy: + fail-fast: false + matrix: + include: + - target: linux-arm64 + - target: linux-amd64 + - target: linux-armv7 + - target: linux-armv6 + - target: linux-ppc64le + - target: linux-386 + - target: linux-riscv64 + - target: linux-s390x + - target: windows-amd64 + - target: windows-386 + + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - + name: Build + uses: docker/bake-action@v2 + with: + targets: ld-${{ matrix.target }}-tgz \ No newline at end of file diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml new file mode 100644 index 0000000..6eab62b --- /dev/null +++ b/.github/workflows/llvm.yml @@ -0,0 +1,52 @@ +name: llvm + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_dispatch: + push: + branches: + - "master" + tags: + - "v*" + paths: + - "src/llvm/**" + pull_request: + branches: + - "master" + paths: + - "src/llvm/**" + +jobs: + compiler-rt: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build + uses: docker/bake-action@v2 + with: + files: | + ./docker-bake.hcl + targets: compiler-rt + libcxx: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build + uses: docker/bake-action@v2 + with: + files: | + ./docker-bake.hcl + targets: compiler-rt diff --git a/.github/workflows/sdk-extras.yml b/.github/workflows/sdk-extras.yml new file mode 100644 index 0000000..488c97b --- /dev/null +++ b/.github/workflows/sdk-extras.yml @@ -0,0 +1,37 @@ +name: sdk-extras + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_dispatch: + push: + branches: + - "master" + tags: + - "v*" + paths: + - "src/sdk-extras/**" + pull_request: + branches: + - "master" + paths: + - "src/sdk-extras/**" + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build + uses: docker/bake-action@v2 + with: + files: | + ./docker-bake.hcl + targets: sdk-extras diff --git a/docker-bake.hcl b/docker-bake.hcl index b6055ce..4421ebc 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -2,26 +2,25 @@ variable "XX_REPO" { default = "tonistiigi/xx" } +variable "TEST_BASE_TYPE" { + default = "alpine" +} + // Special target: https://github.com/docker/metadata-action#bake-definition target "meta-helper" { tags = ["${XX_REPO}:test"] } -group "test" { - targets = ["test-alpine", "test-debian", "test-rhel"] -} - -target "test-base" { - context = "base" - target = "test" +target "test-src" { + context = "src" } target "test-alpine" { - inherits = ["test-base"] + inherits = ["test-src"] } target "test-debian" { - inherits = ["test-base"] + inherits = ["test-src"] args = { APT_MIRROR = "cdn-fastly.deb.debian.org" TEST_BASE_TYPE = "debian" @@ -30,14 +29,64 @@ target "test-debian" { } target "test-rhel" { - inherits = ["test-base"] + inherits = ["test-src"] args = { TEST_BASE_TYPE = "rhel" TEST_BASE_IMAGE = "fedora:35" - TEST_CMDS = "info" } } +target "test-base" { + inherits = ["test-${TEST_BASE_TYPE}"] +} + +group "test" { + targets = [ + "test-info", + "test-apk", + "test-apt", + "test-verify", + "test-clang", + "test-go", + "test-cargo" + ] +} + +target "test-info" { + inherits = ["test-base"] + target = "test-info" +} + +target "test-apk" { + inherits = ["test-base"] + target = "test-apk" +} + +target "test-apt" { + inherits = ["test-base"] + target = "test-apt" +} + +target "test-verify" { + inherits = ["test-base"] + target = "test-verify" +} + +target "test-clang" { + inherits = ["test-base"] + target = "test-clang" +} + +target "test-go" { + inherits = ["test-base"] + target = "test-go" +} + +target "test-cargo" { + inherits = ["test-base"] + target = "test-cargo" +} + group "validate" { targets = ["shfmt-validate", "shellcheck"] } @@ -81,18 +130,18 @@ target "_all-platforms" { ] } -target "base" { +target "xx" { inherits = ["meta-helper"] - context = "base" - target = "base" + context = "src" + target = "xx" } -target "base-all" { - inherits = ["base", "_all-platforms"] +target "image-all" { + inherits = ["image", "_all-platforms"] } target "sdk-extras" { - context = "base" + context = "src/sdk-extras" target = "sdk-extras" tags = ["${XX_REPO}:sdk-extras"] platforms = [ @@ -119,15 +168,21 @@ target "sdk-extras" { ] } +target "_ld-base" { + context = "src/ld" + contexts = { + "tonistiigi/xx" = "target:xx" + } +} + target "ld64-tgz" { - context = "base" + inherits = ["_ld-base"] target = "ld64-tgz" output = ["./ld64-tgz"] platforms = [ "linux/386", "linux/amd64", "linux/arm64", - "linux/arm/v5", "linux/arm/v6", "linux/arm/v7" ] @@ -153,7 +208,7 @@ group "binutils" { } target "binutils-base" { - context = "base" + inherits = ["_ld-base"] target = "binutils" args = { BINUTILS_VERSION = BINUTILS_VERSION @@ -262,7 +317,7 @@ group "ld-static-tgz" { } target "ld-tgz-base" { - context = "base" + inherits = ["_ld-base"] target = "ld-static-tgz" args = { LD_VERSION = BINUTILS_VERSION @@ -357,3 +412,27 @@ target "ld-windows-386-tgz" { } cache-from = [join("", ["type=registry,ref=", binutilsTag(XX_REPO, BINUTILS_VERSION, "1", "windows-386")[0]])] } + +target "compiler-rt" { + context = "src/llvm" + contexts = { + "tonistiigi/xx" = "target:xx" + } + target = "compiler-rt" + platforms = [ + "linux/amd64", + "linux/arm64", + ] +} + +target "libcxx" { + context = "src/llvm" + contexts = { + "tonistiigi/xx" = "target:xx" + } + target = "libcxx" + platforms = [ + "linux/amd64", + "linux/arm64", + ] +} \ No newline at end of file diff --git a/hack/dockerfiles/shellcheck.Dockerfile b/hack/dockerfiles/shellcheck.Dockerfile index 150951f..eee5a35 100644 --- a/hack/dockerfiles/shellcheck.Dockerfile +++ b/hack/dockerfiles/shellcheck.Dockerfile @@ -1,5 +1,5 @@ -# syntax = docker/dockerfile:1.2 +# syntax = docker/dockerfile:1.5 FROM koalaman/shellcheck-alpine:v0.7.2 WORKDIR /src -COPY base . +COPY src . RUN shellcheck xx-* \ No newline at end of file diff --git a/hack/dockerfiles/shfmt.Dockerfile b/hack/dockerfiles/shfmt.Dockerfile index 86156dc..cc6556d 100644 --- a/hack/dockerfiles/shfmt.Dockerfile +++ b/hack/dockerfiles/shfmt.Dockerfile @@ -1,4 +1,4 @@ -# syntax = docker/dockerfile:1.2 +# syntax = docker/dockerfile:1.5 FROM mvdan/shfmt:v3.2.1-alpine AS shfmt WORKDIR /src ARG SHFMT_FLAGS="-i 2 -ci" diff --git a/src/Dockerfile b/src/Dockerfile new file mode 100644 index 0000000..aa85ba0 --- /dev/null +++ b/src/Dockerfile @@ -0,0 +1,110 @@ +# syntax=docker/dockerfile:1.5 + +ARG TEST_BASE_TYPE=alpine +ARG TEST_BASE_IMAGE=${TEST_BASE_TYPE} +ARG TEST_WITH_DARWIN=false + +# build prepares the sources +FROM --platform=$BUILDPLATFORM alpine AS build +COPY xx-* /out/ +RUN ln -s xx-cc /out/xx-clang && \ + ln -s xx-cc /out/xx-clang++ && \ + ln -s xx-cc /out/xx-c++ && \ + ln -s xx-apt /out/xx-apt-get + +# xx builds the xx image +FROM scratch AS xx +COPY --from=build /out/ /usr/bin/ + +FROM --platform=$BUILDPLATFORM tonistiigi/bats-assert AS bats-assert + +FROM ${TEST_BASE_IMAGE} AS test-base-alpine +RUN --mount=type=cache,target=/pkg-cache \ + ln -s /pkg-cache /etc/apk/cache && \ + apk add bats vim +WORKDIR /work + +FROM ${TEST_BASE_IMAGE} AS test-base-debian +ARG APT_MIRROR=deb.debian.org +RUN --mount=type=cache,target=/pkg-cache \ + rm -rf /var/cache/apt/archives && \ + ln -s /pkg-cache /var/cache/apt/archives && \ + rm /etc/apt/apt.conf.d/docker-clean && \ + echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/keep-downloads && \ + touch /etc/apt/sources.list && \ + sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list && \ + sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list && \ + apt update && apt install --no-install-recommends -y bats vim +WORKDIR /work + +FROM ${TEST_BASE_IMAGE} AS test-base-rhel +RUN </dev/null; then + if [ "$ID" != "fedora" ]; then + yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-${VERSION:0:1}.noarch.rpm + fi + fi +fi +EOT +RUN --mount=type=cache,target=/pkg-cache \ + rm -rf /var/cache/yum && \ + ln -s /pkg-cache /var/cache/yum && \ + yum update -y && yum -y install bats vim +WORKDIR /work + + +FROM test-base-${TEST_BASE_TYPE} AS test-base +ADD --chmod=755 https://raw.githubusercontent.com/fsaintjacques/semver-tool/3.4.0/src/semver /usr/bin/semver +COPY --from=bats-assert . . +COPY --from=xx / / + +FROM test-base AS test-base-fixtures +COPY fixtures fixtures + +FROM test-base AS test-info +ARG TEST_BASE_TYPE +COPY test-info-common.bats test-info-${TEST_BASE_TYPE}.bats ./ +RUN ./test-info-common.bats && ./test-info-${TEST_BASE_TYPE}.bats + +FROM test-base AS test-windres +COPY test-windres.bats test_helper.bash ./ +RUN --mount=type=cache,target=/pkg-cache,sharing=locked ./test-windres.bats + +FROM test-base AS test-apt +COPY test-apt.bats . +RUN --mount=type=cache,target=/pkg-cache,sharing=locked [ ! -f /etc/debian_version ] || ./test-apt.bats + +FROM test-base AS test-apk +COPY test-apk.bats . +RUN --mount=type=cache,target=/pkg-cache,sharing=locked [ ! -f /etc/alpine-release ] || ./test-apk.bats + +FROM test-base AS test-verify +COPY test-verify.bats . +RUN --mount=type=cache,target=/pkg-cache,sharing=locked ./test-verify.bats + +FROM test-base-fixtures AS test-clang +COPY test-clang.bats test_helper.bash ./ +RUN --mount=type=cache,target=/pkg-cache,sharing=locked ./test-clang.bats + +FROM test-base-fixtures AS test-go +COPY test-go.bats test_helper.bash ./ +RUN --mount=type=cache,target=/pkg-cache,sharing=locked --mount=type=cache,target=/root/.cache ./test-go.bats + +FROM test-base-fixtures AS test-cargo +COPY test-cargo.bats test_helper.bash ./ +RUN --mount=type=cache,target=/pkg-cache,sharing=locked ./test-cargo.bats + + +FROM scratch AS sdk-extras + +# dev can be used for debugging during development +FROM test-base AS dev +COPY --from=sdk-extras / / +COPY --from=xx / / +COPY fixtures fixtures +COPY *.bats test_helper.bash ./ + +# default is alias to build xx image +FROM xx as default diff --git a/base/fixtures/hello.c b/src/fixtures/hello.c similarity index 100% rename from base/fixtures/hello.c rename to src/fixtures/hello.c diff --git a/base/fixtures/hello.cc b/src/fixtures/hello.cc similarity index 100% rename from base/fixtures/hello.cc rename to src/fixtures/hello.cc diff --git a/base/fixtures/hello.go b/src/fixtures/hello.go similarity index 100% rename from base/fixtures/hello.go rename to src/fixtures/hello.go diff --git a/base/fixtures/hello_cargo/Cargo.toml b/src/fixtures/hello_cargo/Cargo.toml similarity index 100% rename from base/fixtures/hello_cargo/Cargo.toml rename to src/fixtures/hello_cargo/Cargo.toml diff --git a/base/fixtures/hello_cargo/src/main.rs b/src/fixtures/hello_cargo/src/main.rs similarity index 100% rename from base/fixtures/hello_cargo/src/main.rs rename to src/fixtures/hello_cargo/src/main.rs diff --git a/base/fixtures/hello_cgo.go b/src/fixtures/hello_cgo.go similarity index 100% rename from base/fixtures/hello_cgo.go rename to src/fixtures/hello_cgo.go diff --git a/base/Dockerfile b/src/ld/Dockerfile similarity index 59% rename from base/Dockerfile rename to src/ld/Dockerfile index f6dd89c..1b68c43 100644 --- a/base/Dockerfile +++ b/src/ld/Dockerfile @@ -1,87 +1,10 @@ -# syntax=docker/dockerfile:1.3-labs +#syntax=docker/dockerfile:1.5 -ARG TEST_BASE_TYPE=alpine -ARG TEST_BASE_IMAGE=${TEST_BASE_TYPE} -ARG TEST_WITH_DARWIN=false - -ARG SHADOW_VERSION=4.8.1 - -FROM --platform=$BUILDPLATFORM alpine AS scripts -COPY xx-* /out/ -RUN ln -s xx-cc /out/xx-clang && \ - ln -s xx-cc /out/xx-clang++ && \ - ln -s xx-cc /out/xx-c++ && \ - ln -s xx-apt /out/xx-apt-get - -FROM scratch AS base -COPY --from=scripts /out/ /usr/bin/ - -FROM --platform=$BUILDPLATFORM tonistiigi/bats-assert AS assert - -FROM ${TEST_BASE_IMAGE} AS test-base-alpine -RUN --mount=type=cache,target=/pkg-cache \ - ln -s /pkg-cache /etc/apk/cache && \ - apk add bats vim -WORKDIR /work - -FROM ${TEST_BASE_IMAGE} AS test-base-debian -ARG APT_MIRROR=deb.debian.org -RUN --mount=type=cache,target=/pkg-cache \ - rm -rf /var/cache/apt/archives && \ - ln -s /pkg-cache /var/cache/apt/archives && \ - rm /etc/apt/apt.conf.d/docker-clean && \ - echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/keep-downloads && \ - touch /etc/apt/sources.list && \ - sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list && \ - sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list && \ - apt update && apt install --no-install-recommends -y bats vim -WORKDIR /work - -FROM ${TEST_BASE_IMAGE} AS test-base-rhel -RUN </dev/null; then - if [ "$ID" != "fedora" ]; then - yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-${VERSION:0:1}.noarch.rpm - fi - fi -fi -EOT -RUN --mount=type=cache,target=/pkg-cache \ - rm -rf /var/cache/yum && \ - ln -s /pkg-cache /var/cache/yum && \ - yum update -y && yum -y install bats vim -WORKDIR /work - -FROM test-base-${TEST_BASE_TYPE} AS test -ADD --chmod=755 https://raw.githubusercontent.com/fsaintjacques/semver-tool/3.4.0/src/semver /usr/bin/semver -COPY --from=assert . . -COPY --from=base / / -COPY fixtures fixtures -COPY test-*.bats test_helper.bash . -ARG TEST_BASE_TYPE -ARG TEST_CMDS -RUN [ "${TEST_CMDS:-info}" = "${TEST_CMDS#*info}" ] && exit 0; ./test-info-common.bats && ./test-info-$(echo $TEST_BASE_TYPE | cut -d: -f1).bats -RUN --mount=type=cache,target=/pkg-cache \ - [ "${TEST_CMDS:-windres}" = "${TEST_CMDS#*windres}" ] && exit 0; ./test-windres.bats -RUN --mount=type=cache,target=/pkg-cache \ - [ "${TEST_CMDS:-apk}" = "${TEST_CMDS#*apk}" ] && exit 0; [ ! -f /etc/alpine-release ] || ./test-apk.bats -RUN --mount=type=cache,target=/pkg-cache \ - [ "${TEST_CMDS:-apt}" = "${TEST_CMDS#*apt}" ] && exit 0; [ ! -f /etc/debian_version ] || ./test-apt.bats -RUN --mount=type=cache,target=/pkg-cache \ - [ "${TEST_CMDS:-verify}" = "${TEST_CMDS#*verify}" ] && exit 0; ./test-verify.bats -RUN --mount=type=cache,target=/pkg-cache \ - [ "${TEST_CMDS:-clang}" = "${TEST_CMDS#*clang}" ] && exit 0; ./test-clang.bats -RUN --mount=type=cache,target=/pkg-cache \ - --mount=target=/root/.cache,type=cache \ - [ "${TEST_CMDS:-golang}" = "${TEST_CMDS#*golang}" ] && exit 0; ./test-go.bats -RUN --mount=type=cache,target=/pkg-cache \ - [ "${TEST_CMDS:-cargo}" = "${TEST_CMDS#*cargo}" ] && exit 0; ./test-cargo.bats +FROM --platform=$BUILDPLATFORM tonistiigi/xx AS xx FROM --platform=${BUILDPLATFORM} alpine AS libtapi-base RUN apk add --no-cache git clang lld cmake make python3 bash -COPY --from=base / / +COPY --from=xx / / ARG LIBTAPI_VERSION=1100.0.11 RUN git clone https://github.com/tpoechtrager/apple-libtapi --depth 1 -b ${LIBTAPI_VERSION} WORKDIR ./apple-libtapi @@ -111,13 +34,10 @@ RUN export INSTALLPREFIX=/opt/libtapi/ \ && sed -i s/SHARED/STATIC/g src/llvm/projects/libtapi/tools/libtapi/CMakeLists.txt && \ ./build.sh && cd build && make -j $(nproc) LLVMObject tapiCore LLVMSupport LLVMDemangle LLVMMC LLVMBinaryFormat install-tapi-headers && mkdir /opt/libtapi/lib && cp -a ./lib/*.a /opt/libtapi/lib/ && \ cd .. && rm -rf build - #xx-verify --static ./build/lib/libtapi.a - -#FROM --platform=${BUILDPLATFORM} tonistiigi/xx:binutils-2.36.1-${TARGETOS}-${TARGETARCH}${TARGETVARIANT}-alpine AS binutils-release FROM --platform=${BUILDPLATFORM} alpine AS cctools-base RUN apk add --no-cache git clang lld make llvm -COPY --from=base / / +COPY --from=xx / / WORKDIR /work ARG CCTOOLS_REPO=https://github.com/tpoechtrager/cctools-port ARG CCTOOLS_VERSION=949.0.1-ld64-530 @@ -177,11 +97,11 @@ ARG CCTOOLS_CXXFLAGS="-Wl,-s -Os" ARG CCTOOLS_CFLAGS="-Wl,-s -Os" RUN --mount=from=libtapi,source=/opt/libtapi,target=/opt/libtapi \ # copy to /usr/bin for clean rpath. TODO: try static build - cp -a /opt/libtapi/. /usr/ && \ + cp -a /opt/libtapi/. /usr/ && \ export CFLAGS=${CCTOOLS_CFLAGS} CXXFLAGS=${CCTOOLS_CXXFLAGS} && \ ./configure --prefix=/opt/cctools --with-libtapi=/opt/libtapi --host=$(xx-clang --print-target-triple) && \ make -j $(nproc) install && \ - xx-verify /opt/cctools/bin/ld + xx-verify /opt/cctools/bin/ld FROM scratch AS cctools COPY --from=libtapi /opt/libtapi/lib/*.so /usr/lib/ @@ -189,7 +109,7 @@ COPY --from=cctools-build /opt/cctools /usr FROM --platform=${BUILDPLATFORM} alpine AS sigtool-base RUN apk add --no-cache git clang lld cmake make pkgconf -COPY --from=base / / +COPY --from=xx / / WORKDIR /work RUN git clone https://github.com/CLIUtils/CLI11 && \ cd CLI11 && \ @@ -205,7 +125,7 @@ RUN --mount=target=/tmp/sigtool-static.patch,source=sigtool-static.patch \ ARG TARGETPLATFORM RUN xx-apk add --no-cache g++ openssl-dev openssl-libs-static ARG SIGTOOL_CXXFLAGS="-Wl,-s -Os" -RUN if xx-info is-cross; then cp -a /usr/include/CLI /$(xx-info triple)/usr/include/; fi && \ +RUN if xx-info is-cross; then cp -a /usr/include/CLI /$(xx-info triple)/usr/include/; fi && \ export CXXFLAGS=${SIGTOOL_CXXFLAGS} && \ mkdir build && cd build && \ cmake $(xx-clang --print-cmake-defines) -DCMAKE_EXE_LINKER_FLAGS=-static .. && \ @@ -216,22 +136,6 @@ FROM scratch AS sigtool COPY --from=codesign-static / / COPY --from=sigtool-base /work/sigtool/build/gensig /sigtool-gensig -FROM --platform=darwin/amd64 tonistiigi/osxcross:xx-sdk-11.1 AS sdk-extras-darwin-amd64 -FROM tonistiigi/osxcross:xx-sdk-11.1 AS sdk-extras-darwin - -FROM --platform=$BUILDPLATFORM alpine AS extras-base -RUN mkdir --p /out/Files/xx-sdk /out/xx-sdk - -FROM scratch AS sdk-extras-linux -COPY --from=extras-base /out/Files /xx-sdk - -FROM scratch AS sdk-extras-windows -COPY --from=extras-base /out / - -FROM sdk-extras-linux AS sdk-extras-freebsd - -FROM sdk-extras-${TARGETOS} AS sdk-extras - FROM scratch AS ld64-signed-static COPY --from=ld64-static / / COPY --from=sigtool / / @@ -246,34 +150,6 @@ RUN --mount=from=ld64-signed-static \ FROM scratch AS ld64-tgz COPY --from=ld64-tgz-build /out-tgz/ / -FROM test-base-${TEST_BASE_TYPE} AS test-base-darwintrue -COPY --from=sdk-extras-darwin-amd64 / / - -FROM test-base-${TEST_BASE_TYPE} AS test-base-darwinfalse - -FROM test-base-darwin${TEST_WITH_DARWIN} AS dev -COPY fixtures fixtures -COPY --from=base / / - -# newuidmap & newgidmap binaries (shadow-uidmap 4.7-r1) shipped with alpine cannot be executed without CAP_SYS_ADMIN, -# because the binaries are built without libcap-dev. -# So we need to build the binaries with libcap enabled. -FROM --platform=$BUILDPLATFORM alpine AS idmap -RUN apk add --no-cache git autoconf automake clang lld gettext-dev libtool make byacc binutils -COPY --from=base / / -ARG SHADOW_VERSION -RUN git clone https://github.com/shadow-maint/shadow.git /shadow && cd /shadow && git checkout $SHADOW_VERSION -WORKDIR /shadow -ARG TARGETPLATFORM -RUN xx-apk add --no-cache musl-dev gcc libcap-dev -RUN CC=$(xx-clang --print-target-triple)-clang ./autogen.sh --disable-nls --disable-man --without-audit --without-selinux --without-acl --without-attr --without-tcb --without-nscd --host $(xx-clang --print-target-triple) \ - && make -j $(nproc) \ - && xx-verify src/newuidmap src/newuidmap \ - && cp src/newuidmap src/newgidmap /usr/bin - -FROM scratch AS idmap-binaries -COPY --from=idmap /usr/bin/newuidmap /usr/bin/newgidmap / - FROM --platform=$BUILDPLATFORM alpine AS binutils-base0 RUN apk add --no-cache git clang lld zlib-dev gcc patch make musl-dev WORKDIR /work @@ -283,7 +159,7 @@ RUN git clone --depth 1 -b ${BINUTILS_PATCHES_VERSION} https://github.com/alpine mkdir patches && \ cp -a aports/main/binutils/*.patch patches/ && \ rm -rf aports -COPY --from=base / / +COPY --from=xx / / ARG BINUTILS_VERSION=2.36.1 RUN wget https://ftp.gnu.org/gnu/binutils/binutils-${BINUTILS_VERSION}.tar.gz ARG TARGETPLATFORM @@ -296,25 +172,7 @@ RUN export CC=xx-clang CXX=xx-clang++ LD=lld BINUTILS_TARGET=${TARGETPLATFORM} & make -j $(nproc) && \ make install && \ cd .. && rm -rf binutils-${BINUTILS_VERSION} - RUN xx-apk add --no-cache musl-dev gcc g++ zlib-dev -FROM binutils-base0 AS binutils-base -ARG TARGETOS TARGETARCH TARGETVARIANT -# BINUTILS_TARGET defines platform that binutils binaries will target when run -ARG BINUTILS_TARGET=${TARGETOS}-${TARGETARCH}${TARGETVARIANT} -# BINUTILS_CFLAGS defines C compiler flags when building binutils -ARG BINUTILS_CFLAGS="-Wl,-s -Os" -# BINUTILS_CONFIG defines extra options passed to binutils configure script -ARG BINUTILS_CONFIG= -RUN export CC=xx-clang CXX=xx-clang++ CFLAGS="$BINUTILS_CFLAGS" CXXFLAGS="$BINUTILS_CFLAGS" && \ - tar xf binutils-${BINUTILS_VERSION}.tar.gz && \ - cd binutils-${BINUTILS_VERSION} && \ - for f in ../patches/*; do patch -p1 < $f; done && \ - ./configure --disable-separate-code --libdir=/lib --prefix=/out --disable-multilib --enable-deterministic-archives --target=$(TARGETPLATFORM= TARGETPAIR=$BINUTILS_TARGET xx-info) --host $(xx-clang --print-target-triple) --disable-nls --enable-gold --enable-relro --enable-plugins --with-pic --with-mmap --with-system-zlib $BINUTILS_CONFIG && \ - make -j $(nproc) && \ - make install && \ - cd .. && rm -rf binutils-${BINUTILS_VERSION} && \ - for f in /out/bin/*; do xx-verify $f; done FROM binutils-base0 AS ld-base RUN xx-apk add --no-cache zlib-static @@ -349,43 +207,23 @@ COPY --from=ld-base /out / FROM scratch AS ld-static-tgz COPY --from=ld-tgz-base /out-tgz/ / -FROM --platform=$BUILDPLATFORM alpine AS compiler-rt-build -RUN apk add --no-cache git cmake clang lld make ninja python3 llvm -WORKDIR /work -COPY --from=base / / -ARG LLVM_VERSION=llvmorg-11.0.1 -RUN git clone --depth 1 -b ${LLVM_VERSION} https://github.com/llvm/llvm-project.git -WORKDIR llvm-project/compiler-rt -ARG TARGETPLATFORM -RUN xx-apk add gcc g++ -RUN mkdir build && cd build && \ - cmake $(xx-clang --print-cmake-defines) -DCOMPILER_RT_BUILD_SANITIZERS=OFF -DCOMPILER_RT_BUILD_XRAY=OFF -DCOMPILER_RT_BUILD_LIBFUZZER=OFF -DCOMPILER_RT_BUILD_PROFILE=OFF -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON -DCMAKE_SYSTEM_NAME=$(xx-info os | awk '{print toupper(substr($0,0,1))tolower(substr($0,2))}') -DCMAKE_LIPO=/usr/bin/llvm-lipo -G Ninja .. && \ - ninja && mkdir /out && cp -a lib/linux /out/ && \ - cd .. && rm -rf build - -FROM scratch AS compiler-rt -COPY --from=compiler-rt-build /out /usr/lib/clang/compiler-rt/ - -FROM --platform=$BUILDPLATFORM alpine AS libcxx-build -RUN apk add --no-cache git cmake clang lld make ninja python3 binutils -WORKDIR /work -COPY --from=base / / -ARG LLVM_VERSION=llvmorg-11.0.1 -RUN git clone --depth 1 -b ${LLVM_VERSION} https://github.com/llvm/llvm-project.git -WORKDIR llvm-project/libcxx -ARG LIBCXX_TARGET -ENV TARGETPLATFORM=${LIBCXX_TARGET} -RUN xx-apk -v add gcc g++ linux-headers -RUN mkdir build && cd build && \ - cmake $(xx-clang --print-cmake-defines) -DLIBCXX_HAS_MUSL_LIBC=ON -G Ninja .. && \ - ninja && mkdir /out && cp -a lib/libc++* /out/ && \ - xx-verify /out/libc++.so && \ - cd .. && rm -rf build - -FROM scratch AS libcxx -COPY --from=libcxx-build /out /usr/lib/ +FROM binutils-base0 AS binutils-base +ARG TARGETOS TARGETARCH TARGETVARIANT +# BINUTILS_TARGET defines platform that binutils binaries will target when run +ARG BINUTILS_TARGET=${TARGETOS}-${TARGETARCH}${TARGETVARIANT} +# BINUTILS_CFLAGS defines C compiler flags when building binutils +ARG BINUTILS_CFLAGS="-Wl,-s -Os" +# BINUTILS_CONFIG defines extra options passed to binutils configure script +ARG BINUTILS_CONFIG= +RUN export CC=xx-clang CXX=xx-clang++ CFLAGS="$BINUTILS_CFLAGS" CXXFLAGS="$BINUTILS_CFLAGS" && \ + tar xf binutils-${BINUTILS_VERSION}.tar.gz && \ + cd binutils-${BINUTILS_VERSION} && \ + for f in ../patches/*; do patch -p1 < $f; done && \ + ./configure --disable-separate-code --libdir=/lib --prefix=/out --disable-multilib --enable-deterministic-archives --target=$(TARGETPLATFORM= TARGETPAIR=$BINUTILS_TARGET xx-info) --host $(xx-clang --print-target-triple) --disable-nls --enable-gold --enable-relro --enable-plugins --with-pic --with-mmap --with-system-zlib $BINUTILS_CONFIG && \ + make -j $(nproc) && \ + make install && \ + cd .. && rm -rf binutils-${BINUTILS_VERSION} && \ + for f in /out/bin/*; do xx-verify $f; done FROM scratch AS binutils COPY --from=binutils-base /out /usr/ - -FROM base diff --git a/base/ld64.signed b/src/ld/ld64.signed similarity index 100% rename from base/ld64.signed rename to src/ld/ld64.signed diff --git a/base/libtapi-cmake-args.patch b/src/ld/libtapi-cmake-args.patch similarity index 100% rename from base/libtapi-cmake-args.patch rename to src/ld/libtapi-cmake-args.patch diff --git a/base/sigtool-static.patch b/src/ld/sigtool-static.patch similarity index 100% rename from base/sigtool-static.patch rename to src/ld/sigtool-static.patch diff --git a/src/llvm/Dockerfile b/src/llvm/Dockerfile new file mode 100644 index 0000000..588dc60 --- /dev/null +++ b/src/llvm/Dockerfile @@ -0,0 +1,39 @@ +#syntax=docker/dockerfile:1.5 + +FROM --platform=$BUILDPLATFORM tonistiigi/xx AS xx + +FROM --platform=$BUILDPLATFORM alpine AS compiler-rt-build +RUN apk add --no-cache git cmake clang lld make ninja python3 llvm +WORKDIR /work +COPY --from=xx / / +ARG LLVM_VERSION=llvmorg-11.0.1 +RUN git clone --depth 1 -b ${LLVM_VERSION} https://github.com/llvm/llvm-project.git +WORKDIR llvm-project/compiler-rt +ARG TARGETPLATFORM +RUN xx-apk add gcc g++ +RUN mkdir build && cd build && \ + cmake $(xx-clang --print-cmake-defines) -DCOMPILER_RT_BUILD_SANITIZERS=OFF -DCOMPILER_RT_BUILD_XRAY=OFF -DCOMPILER_RT_BUILD_LIBFUZZER=OFF -DCOMPILER_RT_BUILD_PROFILE=OFF -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON -DCMAKE_SYSTEM_NAME=$(xx-info os | awk '{print toupper(substr($0,0,1))tolower(substr($0,2))}') -DCMAKE_LIPO=/usr/bin/llvm-lipo -G Ninja .. && \ + ninja && mkdir /out && cp -a lib/linux /out/ && \ + cd .. && rm -rf build + +FROM scratch AS compiler-rt +COPY --from=compiler-rt-build /out /usr/lib/clang/compiler-rt/ + +FROM --platform=$BUILDPLATFORM alpine AS libcxx-build +RUN apk add --no-cache git cmake clang lld make ninja python3 binutils +WORKDIR /work +COPY --from=xx / / +ARG LLVM_VERSION=llvmorg-11.0.1 +RUN git clone --depth 1 -b ${LLVM_VERSION} https://github.com/llvm/llvm-project.git +WORKDIR llvm-project/libcxx +ARG LIBCXX_TARGET +ENV TARGETPLATFORM=${LIBCXX_TARGET} +RUN xx-apk -v add gcc g++ linux-headers +RUN mkdir build && cd build && \ + cmake $(xx-clang --print-cmake-defines) -DLIBCXX_HAS_MUSL_LIBC=ON -G Ninja .. && \ + ninja && mkdir /out && cp -a lib/libc++* /out/ && \ + xx-verify /out/libc++.so && \ + cd .. && rm -rf build + +FROM scratch AS libcxx +COPY --from=libcxx-build /out /usr/lib/ diff --git a/src/sdk-extras/Dockerfile b/src/sdk-extras/Dockerfile new file mode 100644 index 0000000..e42c12a --- /dev/null +++ b/src/sdk-extras/Dockerfile @@ -0,0 +1,23 @@ +ARG OSXCROSS_VERSION=12.3 + +# Make sure to read XCode license agreement before using this code +# same for all arch therefore hardcoded to arm64 +FROM --platform=darwin/arm64 crazymax/osxcross:${OSXCROSS_VERSION} AS osxcross + +FROM scratch AS sdk-extras-darwin +COPY --from=osxcross /osxsdk /xx-sdk/ + +# this dance is a hack to work aroung windows tarball being different from linux +# create directories in both levels so layers can be accessed from either OS +FROM --platform=$BUILDPLATFORM alpine AS extras-base +RUN mkdir -p /out/Files/xx-sdk /out/xx-sdk + +FROM scratch AS sdk-extras-linux +COPY --from=extras-base /out/Files /xx-sdk + +FROM scratch AS sdk-extras-windows +COPY --from=extras-base /out / + +FROM sdk-extras-linux AS sdk-extras-freebsd + +FROM sdk-extras-${TARGETOS} AS sdk-extras diff --git a/base/test-apk.bats b/src/test-apk.bats similarity index 100% rename from base/test-apk.bats rename to src/test-apk.bats diff --git a/base/test-apt.bats b/src/test-apt.bats similarity index 100% rename from base/test-apt.bats rename to src/test-apt.bats diff --git a/base/test-cargo.bats b/src/test-cargo.bats similarity index 100% rename from base/test-cargo.bats rename to src/test-cargo.bats diff --git a/base/test-clang.bats b/src/test-clang.bats similarity index 100% rename from base/test-clang.bats rename to src/test-clang.bats diff --git a/base/test-go.bats b/src/test-go.bats similarity index 100% rename from base/test-go.bats rename to src/test-go.bats diff --git a/base/test-info-alpine.bats b/src/test-info-alpine.bats similarity index 100% rename from base/test-info-alpine.bats rename to src/test-info-alpine.bats diff --git a/base/test-info-common.bats b/src/test-info-common.bats similarity index 100% rename from base/test-info-common.bats rename to src/test-info-common.bats diff --git a/base/test-info-debian.bats b/src/test-info-debian.bats similarity index 100% rename from base/test-info-debian.bats rename to src/test-info-debian.bats diff --git a/base/test-info-rhel.bats b/src/test-info-rhel.bats similarity index 100% rename from base/test-info-rhel.bats rename to src/test-info-rhel.bats diff --git a/base/test-verify.bats b/src/test-verify.bats similarity index 100% rename from base/test-verify.bats rename to src/test-verify.bats diff --git a/base/test-windres.bats b/src/test-windres.bats similarity index 100% rename from base/test-windres.bats rename to src/test-windres.bats diff --git a/base/test_helper.bash b/src/test_helper.bash similarity index 100% rename from base/test_helper.bash rename to src/test_helper.bash diff --git a/base/xx-apk b/src/xx-apk similarity index 100% rename from base/xx-apk rename to src/xx-apk diff --git a/base/xx-apt b/src/xx-apt similarity index 100% rename from base/xx-apt rename to src/xx-apt diff --git a/base/xx-cargo b/src/xx-cargo similarity index 100% rename from base/xx-cargo rename to src/xx-cargo diff --git a/base/xx-cc b/src/xx-cc similarity index 100% rename from base/xx-cc rename to src/xx-cc diff --git a/base/xx-go b/src/xx-go similarity index 100% rename from base/xx-go rename to src/xx-go diff --git a/base/xx-info b/src/xx-info similarity index 100% rename from base/xx-info rename to src/xx-info diff --git a/base/xx-verify b/src/xx-verify similarity index 100% rename from base/xx-verify rename to src/xx-verify diff --git a/base/xx-windres b/src/xx-windres similarity index 100% rename from base/xx-windres rename to src/xx-windres