Musl testing (#544)

Co-authored-by: Michael Graeb <[email protected]>
Co-authored-by: Waqar Ahmed Khan <[email protected]>

.builder/actions/aws_crt_java_test.py

@@ -6,10 +6,25 @@
 
 class AWSCrtJavaTest(Builder.Action):
 
-    def _run_java_tests(self, *cmd_args):
+    def _run_java_tests(self, *extra_args):
         if os.path.exists('log.txt'):
             os.remove('log.txt')
 
+        profiles = 'continuous-integration'
+
+        cmd_args = [
+            "mvn", "-B",
+            "-P", profiles,
+            "-DredirectTestOutputToFile=true",
+            "-DreuseForks=false",
+            "-Daws.crt.memory.tracing=2",
+            "-Daws.crt.debugnative=true",
+            "-Daws.crt.aws_trace_log_per_test",
+            "-Daws.crt.ci=true",
+        ]
+        cmd_args.extend(extra_args)
+        cmd_args.append("test")
+
         result = self.env.shell.exec(*cmd_args, check=False)
         if result.returncode:
             if os.path.exists('log.txt'):
@@ -23,29 +38,11 @@ def start_maven_tests(self, env):
         # tests must run with leak detection turned on
         env.shell.setenv('AWS_CRT_MEMORY_TRACING', '2')
 
-        self._run_java_tests(
-            "mvn", "-P", "continuous-integration", "-B", "test",
-            "-DredirectTestOutputToFile=true",
-            "-DreuseForks=false",
-            "-DrerunFailingTestsCount=5",
-            "-Daws.crt.memory.tracing=2",
-            "-Daws.crt.debugnative=true",
-            "-Daws.crt.aws_trace_log_per_test",
-            "-Daws.crt.ci=true",
-        )
+        self._run_java_tests("-DrerunFailingTestsCount=5")
 
         # run the ShutdownTest by itself
         env.shell.setenv('AWS_CRT_SHUTDOWN_TESTING', '1')
-        self._run_java_tests(
-            "mvn", "-P", "continuous-integration", "-B", "test",
-            "-DredirectTestOutputToFile=true",
-            "-DreuseForks=false",
-            "-Daws.crt.memory.tracing=2",
-            "-Daws.crt.debugnative=true",
-            "-Daws.crt.aws_trace_log_per_test",
-            "-Daws.crt.ci=true",
-            "-Dtest=ShutdownTest",
-        )
+        self._run_java_tests("-Dtest=ShutdownTest")
 
         # run the elasticurl integration tests
         python = sys.executable

.builder/actions/build-classifier.py

@@ -0,0 +1,15 @@
+
+import Builder
+import sys
+import os
+import tempfile
+from builder.core.host import current_host
+
+# This action is used by the musl-linux-build.sh script to build the binaries for release.
+class BuildClassifier(Builder.Action):
+
+
+    def run(self, env):
+        crt_classifier = os.getenv("CRT_CLASSIFIER")
+        if crt_classifier is not None:
+            env.shell.exec("mvn", "-B", "install", "-DskipTests", "-Dshared-lib.skip=true", f"-Dcrt.classifier={crt_classifier}", check=True)

.github/workflows/ci.yml

@@ -7,7 +7,7 @@ on:
       - 'docs'
 
 env:
-  BUILDER_VERSION: v0.9.45
+  BUILDER_VERSION: v0.9.46
   BUILDER_SOURCE: releases
   BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
   PACKAGE_NAME: aws-crt-java
@@ -79,6 +79,46 @@ jobs:
           python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
           chmod a+x builder
           ./builder build -p ${{ env.PACKAGE_NAME }} --target=linux-${{ matrix.arch }} --spec=downstream
+          
+  linux-musl:
+    runs-on: ubuntu-22.04 # latest
+    strategy:
+      matrix:
+        image:
+          - alpine-3.16-x64
+          - alpine-3.16-arm64
+          - openwrt-x64-openjdk8
+    steps:
+      - name: Install qemu/docker
+        run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+      - name: Checkout Sources
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Build ${{ env.PACKAGE_NAME }}
+        run: |
+          aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
+          ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }}
+
+  # armv7 needs its own action due to inability to differentiate at runtime armv6 vs armv7 in the jvm:
+  #
+  # At build time we can properly figure out that we're targeting armv7.
+  # At run time we have to force armv7 (via environment variable) in order to achieve proper resource path
+  # resolution.
+  linux-musl-armv7:
+    runs-on: ubuntu-20.04 # latest
+    steps:
+      - name: Install qemu/docker
+        run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+      - name: Checkout Sources
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Build ${{ env.PACKAGE_NAME }}
+        run: |
+          export AWS_CRT_ARCH=armv7
+          aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
+          ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-alpine-3.16-armv7 build -p ${{ env.PACKAGE_NAME }}
 
   raspberry:
     runs-on: ubuntu-22.04 # latest

CMakeLists.txt

@@ -105,6 +105,9 @@ endif()
 include(UseJava)
 include(AwsPlatformDetect)
 include(AwsSharedLibSetup)
+include(AwsCRuntime)
+
+aws_determine_local_c_runtime(AWS_C_RUNTIME)
 
 file(GLOB AWS_CRT_JAVA_HEADERS
         "include/aws/jni/*.h"
@@ -156,7 +159,7 @@ file(GLOB CRT_JAVA_SRC
 # figure out the target platform info and output the lib there
 aws_detect_target_platform(TARGET_OS TARGET_ARCH)
 message(STATUS "Compiling for ${TARGET_OS}-${TARGET_ARCH}")
-set(TARGET_LIB_DIR "${AWS_LIBRARY_OUTPUT_DIR}/${TARGET_OS}/${TARGET_ARCH}")
+set(TARGET_LIB_DIR "${AWS_LIBRARY_OUTPUT_DIR}/${TARGET_OS}/${TARGET_ARCH}/${AWS_C_RUNTIME}")
 
 # shared lib that contains the CRT and JNI bindings, to be loaded by java
 add_library(${PROJECT_NAME} SHARED ${CRT_JAVA_HEADERS} ${CRT_JAVA_SRC})

README.md

@@ -82,11 +82,14 @@ The `aws-crt` JAR in Maven Central is a large "uber" jar that contains compiled
 
 ### Available platform classifiers
 
-- linux-armv6
-- linux-armv7
+- linux-armv6 (no auto-detect)
+- linux-armv7 (no auto-detect)
 - linux-aarch_64
 - linux-x86_32
 - linux-x86_64
+- linux-x86_64-musl (no auto-detect)
+- linux-armv7-musl (no auto-detect)
+- linux-aarch_64-musl (no auto-detect)
 - osx-aarch_64
 - osx-x86_64
 - windows-x86_32
@@ -97,6 +100,8 @@ The `aws-crt` JAR in Maven Central is a large "uber" jar that contains compiled
 The [os-maven-plugin](https://github.com/trustin/os-maven-plugin) can automatically detect your platform's classifier at build time.
 
 **NOTES**: The auto-detected `linux-arm_32` platform classifier is not supported, you must specify `linux-armv6` or `linux-armv7`.
+Additionally, musl vs glibc detection is not supported either.  If you are deploying to a musl-based system and wish to use 
+a classifier-based jar, you must specify the classifier name yourself.
 
 ``` xml
 <build>

builder.json

@@ -78,6 +78,28 @@
                         "-DBUILD_TESTING=OFF"
                     ]
                 }
+            },
+            "hosts": {
+                "_comment": "Reset the overrides from targets/linux/architectures/arm* for Alpine",
+                "alpine": {
+                    "!packages": [
+                        "maven",
+                        "openjdk8"
+                    ],
+                    "!build_steps": [
+                        "mvn -P continuous-integration -B compile",
+                        "build-classifier"
+                    ],
+                    "!test_steps": [
+                        "aws-crt-java-test"
+                    ]
+                },
+                "openwrt": {
+                    "!packages": [],
+                    "!build_steps": [
+                        "mvn -P continuous-integration -B compile"
+                    ]
+                }
             }
         },
         "android": {

codebuild/cd/deploy-platform-specific-jars.sh

@@ -13,7 +13,7 @@ else
   DEPLOY_REPOSITORY_URL=https://aws.oss.sonatype.org:443/service/local/staging/deployByRepositoryId/${STAGING_REPO_ID}
 fi
 
-CLASSIFIERS_ARRAY=("linux-armv6" "linux-armv7" "linux-aarch_64" "linux-x86_32" "linux-x86_64" "osx-aarch_64" "osx-x86_64" "windows-x86_32" "windows-x86_64")
+CLASSIFIERS_ARRAY=("linux-armv6" "linux-armv7" "linux-aarch_64" "linux-x86_32" "linux-x86_64" "osx-aarch_64" "osx-x86_64" "windows-x86_32" "windows-x86_64" "linux-x86_64-musl" "linux-armv7-musl" "linux-aarch_64-musl")
 
 for str in ${CLASSIFIERS_ARRAY[@]}; do
   FILES="${FILES}target/aws-crt-1.0.0-SNAPSHOT-$str.jar,"

codebuild/cd/musl-arm64-build.yml

@@ -0,0 +1,22 @@
+version: 0.2
+phases:
+  install:
+    commands:
+  pre_build:
+    commands:
+      - export CC=gcc
+  build:
+    commands:
+      - cd aws-crt-java
+      - ./codebuild/cd/musl-linux-build.sh aws-crt-alpine-3.16-arm64 linux-aarch_64-musl
+
+  post_build:
+    commands:
+
+artifacts:
+  files:
+    - 'dist/**/*'
+
+cache:
+  paths:
+    - '/root/.m2/**/*'

codebuild/cd/musl-armv7-build.yml

@@ -0,0 +1,23 @@
+version: 0.2
+phases:
+  install:
+    commands:
+  pre_build:
+    commands:
+      - export CC=gcc
+  build:
+    commands:
+      - cd aws-crt-java
+      - export AWS_CRT_ARCH=armv7
+      - ./codebuild/cd/musl-linux-build.sh aws-crt-alpine-3.16-armv7 linux-armv7-musl
+
+  post_build:
+    commands:
+
+artifacts:
+  files:
+    - 'dist/**/*'
+
+cache:
+  paths:
+    - '/root/.m2/**/*'

codebuild/cd/musl-linux-build.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -ex
+
+IMAGE_NAME=$1
+shift
+CLASSIFIER=$1
+shift
+
+# Pry the builder version this CRT is using out of ci.yml
+BUILDER_VERSION=$(cat .github/workflows/ci.yml | grep 'BUILDER_VERSION:' | sed 's/\s*BUILDER_VERSION:\s*\(.*\)/\1/')
+echo "Using builder version ${BUILDER_VERSION}"
+
+aws ecr get-login-password | docker login 123124136734.dkr.ecr.us-east-1.amazonaws.com -u AWS --password-stdin
+export DOCKER_IMAGE=123124136734.dkr.ecr.us-east-1.amazonaws.com/${IMAGE_NAME}:${BUILDER_VERSION}
+export QEMU_IMAGE=123124136734.dkr.ecr.us-east-1.amazonaws.com/multiarch-qemu-user-static:latest
+docker run --rm --privileged ${QEMU_IMAGE} --reset -p yes
+
+
+export CRT_CLASSIFIER=${CLASSIFIER}
+export BRANCH_TAG=$(git describe --tags)
+docker run --mount type=bind,src=$(pwd),dst=/root/aws-crt-java --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env AWS_DEFAULT_REGION --env CXXFLAGS --env AWS_CRT_ARCH --env CRT_CLASSIFIER $DOCKER_IMAGE --version=${BUILDER_VERSION} build -p aws-crt-java --branch ${BRANCH_TAG} run_tests=false
+docker container prune -f
+
+# Upload the artifacts to S3
+export GIT_TAG=$(git describe --tags)
+
+aws s3 cp --recursive --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/lib
+aws s3 cp target/ s3://aws-crt-java-pipeline/${GIT_TAG}/jar/ --recursive --exclude "*" --include "aws-crt*.jar"

codebuild/cd/musl-x64-build.yml

@@ -0,0 +1,22 @@
+version: 0.2
+phases:
+  install:
+    commands:
+  pre_build:
+    commands:
+      - export CC=gcc
+  build:
+    commands:
+      - cd aws-crt-java
+      - ./codebuild/cd/musl-linux-build.sh aws-crt-alpine-3.16-x64 linux-x86_64-musl
+
+  post_build:
+    commands:
+
+artifacts:
+  files:
+    - 'dist/**/*'
+
+cache:
+  paths:
+    - '/root/.m2/**/*'

codebuild/cd/test-platform-specific-jar-snapshot.sh

@@ -2,7 +2,7 @@
 
 set -ex
 
-PLATFORM_ARRAY=("linux-armv6" "linux-armv7" "linux-aarch_64" "linux-x86_32" "linux-x86_64" "osx-aarch_64" "osx-x86_64" "windows-x86_32" "windows-x86_64")
+PLATFORM_ARRAY=("linux-armv6" "linux-armv7" "linux-aarch_64" "linux-x86_32" "linux-x86_64" "osx-aarch_64" "osx-x86_64" "windows-x86_32" "windows-x86_64" "linux-x86_64-musl" "linux-armv7-musl" "linux-aarch_64-musl")
 
 # test uber jar
 mvn -B dependency:get -DrepoUrl=https://aws.oss.sonatype.org/content/repositories/snapshots -Dartifact=software.amazon.awssdk.crt:aws-crt:${CRT_VERSION}-SNAPSHOT -Dtransitive=false

codebuild/macos_compatibility_check.py

@@ -25,7 +25,7 @@ def main():
     #   version 10.9
     #       sdk 12.1
     # Load command 9
-    otool_cmd = "otool -l target/cmake-build/lib/osx/{}/libaws-crt-jni.dylib | grep -A3 \'LC_VERSION_MIN_MACOSX\' | grep -E version | tr -s ' ' | cut -f3 -d' ' | tr -d '[:space:]'".format(arch)
+    otool_cmd = "otool -l target/cmake-build/lib/osx/{}/cruntime/libaws-crt-jni.dylib | grep -A3 \'LC_VERSION_MIN_MACOSX\' | grep -E version | tr -s ' ' | cut -f3 -d' ' | tr -d '[:space:]'".format(arch)
 
     if len(sys.argv) > 1:
         # Parsing the macos archtecture
@@ -39,7 +39,7 @@ def main():
         arch = "armv8"
         # The oldest version we can target on arm64 is 11.0
         supported_version = "11.0"
-        otool_cmd = "otool -l target/cmake-build/lib/osx/{}/libaws-crt-jni.dylib | grep -E minos | tr -s ' ' | cut -f3 -d' ' | tr -d ' ' | tr -d '[:space:]'".format(arch)
+        otool_cmd = "otool -l target/cmake-build/lib/osx/{}/cruntime/libaws-crt-jni.dylib | grep -E minos | tr -s ' ' | cut -f3 -d' ' | tr -d ' ' | tr -d '[:space:]'".format(arch)
 
     print("Start to validate the build binary for MacOS with architecture {}, expected min os version: {}".format(arch,supported_version))
     result = subprocess.check_output(otool_cmd, shell=True).decode("utf-8")