Update to NDK 23

finagolfin · Aug 24, 2021 · bf98fbb · bf98fbb
1 parent 3549660
commit bf98fbb
Show file tree

Hide file tree

Showing 8 changed files with 208 additions and 29 deletions.
diff --git a/.github/workflows/sdks.yml b/.github/workflows/sdks.yml
@@ -14,6 +14,15 @@ jobs:
       devel-version: ${{ steps.check.outputs.devel-tag }}
       trunk-version: ${{ steps.check.outputs.trunk-tag }}
     steps:
+      - name: Get cached NDK
+        id: cache-ndk
+        uses: actions/cache@v2
+        with:
+          path: ~/android-ndk-r23-linux.zip
+          key: ndk-23
+      - name: Get NDK 23 if not cached
+        if: ${{ steps.cache-ndk.outputs.cache-hit != 'true' }}
+        run: wget -q -O ~/android-ndk-r23-linux.zip https://dl.google.com/android/repository/android-ndk-r23-linux.zip
       - name: Check for latest Swift ${{ matrix.version }} toolchain
         id: check
         run: |
@@ -26,7 +35,9 @@ jobs:
             SWIFT_TAG="swift-5.5-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a"
             echo "::set-output name=devel-tag::$SWIFT_TAG"
           else
-            LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/releases | grep -m1 swift-DEV | cut -d- -f4-6)
+            # LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/releases | grep -m1 swift-DEV | cut -d- -f4-6)
+            # trunk toolchain is broken right now, so pin to Aug. 16
+            LATEST_TOOLCHAIN_VERSION="2021-08-16"
             SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a"
             echo "::set-output name=trunk-tag::$SWIFT_TAG"
           fi
@@ -82,12 +93,18 @@ jobs:
         with:
           path: ~/${{ steps.version.outputs.tag }}-ubuntu20.04.tar.gz
           key: ${{ steps.version.outputs.tag }}-toolchain
+      - name: Get cached NDK
+        id: cache-ndk
+        uses: actions/cache@v2
+        with:
+          path: ~/android-ndk-r23-linux.zip
+          key: ndk-23
       - name: Get cached SDK
         id: cache-sdk
         uses: actions/cache@v2
         with:
           path: ~/swift-${{ matrix.version }}*-android-${{ matrix.arch }}-*-sdk.tar.xz
-          key: ${{ matrix.version }}-${{ steps.version.outputs.latest }}-${{ matrix.arch }}-sdk
+          key: ${{ matrix.version }}-${{ steps.version.outputs.latest }}-${{ matrix.arch }}-23-sdk
       - name: Clone
         uses: actions/checkout@v2
         with:
@@ -106,29 +123,47 @@ jobs:
           SDK=`pwd`/$SDK_NAME
 
           tar xf ~/$SWIFT_TAG-ubuntu20.04.tar.gz
+          unzip -q ~/android-ndk-r23-linux.zip
+          NDK=`pwd`/android-ndk-r23
 
-          if [ ${{ matrix.version }} = 'trunk' ]; then
+          if [ ${{ matrix.version }} = '5.4' ]; then
+            sed -i "s%abort(void) %abort(void);// %" android-ndk-r23/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/stdlib.h
+          elif [ ${{ matrix.version }} = 'trunk' ]; then
             EXTRA_FLAGS="--skip-early-swift-driver"
           fi
 
-          ./swift/utils/build-script -RA --skip-build-cmark --build-llvm=0 --android --android-ndk $ANDROID_NDK_HOME --android-arch ${{ matrix.arch }} --android-api-level $ANDROID_API_LEVEL --android-icu-uc $SDK/usr/lib/libicuuc.so --android-icu-uc-include $SDK/usr/include --android-icu-i18n $SDK/usr/lib/libicui18n.so --android-icu-i18n-include $SDK/usr/include --android-icu-data $SDK/usr/lib/libicudata.so --build-swift-tools=0 --native-swift-tools-path=`pwd`/$SWIFT_TAG-ubuntu20.04/usr/bin --native-clang-tools-path=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin --cross-compile-hosts=android-${{ matrix.arch }} --cross-compile-deps-path=$SDK --skip-local-build --xctest --install-swift --install-libdispatch --install-foundation --install-xctest --install-destdir=$SDK --swift-install-components='clang-resource-dir-symlink;license;stdlib;sdk-overlay' $EXTRA_FLAGS
+          ./swift/utils/build-script -RA --skip-build-cmark --build-llvm=0 --android --android-ndk $NDK --android-arch ${{ matrix.arch }} --android-api-level $ANDROID_API_LEVEL --android-icu-uc $SDK/usr/lib/libicuuc.so --android-icu-uc-include $SDK/usr/include --android-icu-i18n $SDK/usr/lib/libicui18n.so --android-icu-i18n-include $SDK/usr/include --android-icu-data $SDK/usr/lib/libicudata.so --build-swift-tools=0 --native-swift-tools-path=`pwd`/$SWIFT_TAG-ubuntu20.04/usr/bin --native-clang-tools-path=$NDK/toolchains/llvm/prebuilt/linux-x86_64/bin --cross-compile-hosts=android-${{ matrix.arch }} --cross-compile-deps-path=$SDK --skip-local-build --xctest --install-swift --install-libdispatch --install-foundation --install-xctest --install-destdir=$SDK --swift-install-components='clang-resource-dir-symlink;license;stdlib;sdk-overlay' $EXTRA_FLAGS
 
           patchelf --set-rpath \$ORIGIN/../..:\$ORIGIN $SDK/usr/lib/swift/android/lib*.so
 
           tar cJf ~/$SDK_NAME.tar.xz $SDK_NAME
-          rm -rf $SDK_NAME
       - name: Patch SPM ${{ matrix.version }} config
         run: |
           cd sdk-config
 
-          tar xf ~/${{ steps.version.outputs.tag }}-ubuntu20.04.tar.gz
+          if [ ! -d android-ndk-r23 ]; then
+            unzip -q ~/android-ndk-r23-linux.zip
+            if [ ${{ matrix.version }} = '5.4' ]; then
+              sed -i "s%abort(void) %abort(void);// %" android-ndk-r23/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/stdlib.h
+            fi
+          fi
+
+          if [ ! -d ${{ steps.version.outputs.tag }}-ubuntu20.04 ]; then
+            tar xf ~/${{ steps.version.outputs.tag }}-ubuntu20.04.tar.gz
+          fi
           sed -i "s%/home/butta/swift-5.4.2-RELEASE%`pwd`/${{ steps.version.outputs.tag }}%" android-${{ matrix.arch }}.json
 
-          tar xf ~/swift-${{ matrix.version }}*-android-${{ matrix.arch }}*-${ANDROID_API_LEVEL}-sdk.tar.xz
+          ls
           SDK_NAME=$(ls | grep swift-${{ matrix.version }}.*-sdk)
+          df
+          if [[ $SDK_NAME = "" ]]; then
+            echo "foo"
+            tar xf ~/swift-${{ matrix.version }}*-android-${{ matrix.arch }}*-${ANDROID_API_LEVEL}-sdk.tar.xz
+          fi
+          echo "got before"
           sed -i "s%/home/butta/swift-5.4-android-${{ matrix.arch }}-${ANDROID_API_LEVEL}-sdk%`pwd`/${SDK_NAME}%" android-${{ matrix.arch }}.json
 
-          sed -i "s%/home/butta/src/android-ndk-r21e%${ANDROID_NDK_HOME}%" android-${{ matrix.arch }}.json
+          sed -i "s%/home/butta/src/android-ndk-r21e%`pwd`/android-ndk-r23%" android-${{ matrix.arch }}.json
       - name: Get Swift Argument Parser package
         uses: actions/checkout@v2
         with:
@@ -251,6 +286,13 @@ jobs:
         run: |
           mkdir -p pack/lib/swift/android
           cp swift-argument-parser/.build/x86_64-unknown-linux-android/debug/{math,repeat,roll,swift-argument-parserPackageTests.xctest} pack
+
+          wget -q https://raw.githubusercontent.com/termux/termux-elf-cleaner/master/termux-elf-cleaner.cpp
+          wget -q https://raw.githubusercontent.com/termux/termux-elf-cleaner/master/elf.h
+          clang -D__ANDROID_API__=$ANDROID_API_LEVEL -o termux-elf-cleaner termux-elf-cleaner.cpp
+
+          ./termux-elf-cleaner pack/{math,repeat,roll} || true
+
           cp swift-crypto/.build/x86_64-unknown-linux-android/debug/swift-cryptoPackageTests.xctest pack
           if [ ${{ matrix.version }} = '5.4' ]; then
             cp swift-nio/.build/x86_64-unknown-linux-android/debug/swift-nioPackageTests.xctest pack
@@ -268,7 +310,7 @@ jobs:
           cp yams/Tests/YamsTests/Fixtures/SourceKitten#289/debug.yaml pack
           cp sdk-config/swift-${{ matrix.version }}*-android-x86_64-*${ANDROID_API_LEVEL}-sdk/usr/lib/lib*so pack/lib
           cp sdk-config/swift-${{ matrix.version }}*-android-x86_64-*${ANDROID_API_LEVEL}-sdk/usr/lib/swift/android/lib*so pack/lib/swift/android
-          cp $ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/x86_64-linux-android/libc++_shared.so pack/lib/swift/android
+          cp sdk-config/android-ndk-r23/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/x86_64-linux-android/libc++_shared.so pack/lib/swift/android
       - name: Run tests on Android x86_64 emulator
         if: ${{ matrix.arch == 'x86_64' }}
         uses: reactivecircus/android-emulator-runner@v2

diff --git a/README.md b/README.md
@@ -3,21 +3,21 @@
 All patches used to build these SDKs are open source and listed below.
 
 To build with an SDK, first download [the latest Android LTS NDK
-21e](https://developer.android.com/ndk/downloads) and [Swift 5.4.2
+23](https://developer.android.com/ndk/downloads) and [Swift 5.4.2
 compiler](https://swift.org/download/#releases) (make sure to install the Swift
 compiler's dependencies listed there). Unpack these archives and the SDK.
 
 I will write up a Swift script to do this SDK configuration, but you will need
 to do it manually for now. [You can see how I do it on the CI for a concrete
-example](https://github.com/buttaface/swift-android-sdk/blob/main/.github/workflows/sdks.yml#L125).
+example](https://github.com/buttaface/swift-android-sdk/blob/main/.github/workflows/sdks.yml#L154).
 
 The SDK will need to be modified with the path to your NDK and Swift compiler
 in the following ways (I'll show aarch64 below, the same will need to be done
 for the armv7 or x86_64 SDKs):
 
 1. Change all paths in `swift-5.4-android-aarch64-24-sdk/usr/lib/swift/android/aarch64/glibc.modulemap`
 from `/home/butta/src/android-ndk-r21e` to the path to your NDK, ie something
-like `/home/yourname/android-ndk-r21e`.
+like `/home/yourname/android-ndk-r23`.
 
 2. There's a single line pointing to a header in the SDK itself, so change it
 from `/home/butta/swift-5.4-android-aarch64-24-sdk` in `glibc.modulemap` to the
@@ -33,7 +33,7 @@ swift-5.4-android-aarch64-24-sdk/usr/lib/swift/clang
 Finally, modify the cross-compilation JSON file in this repo similarly:
 
 1. All paths to the NDK should change from `/home/butta/src/android-ndk-r21e`
-to the path to your NDK, `/home/yourname/android-ndk-r21e`.
+to the path to your NDK, `/home/yourname/android-ndk-r23`.
 
 2. The path to the compiler should change from `/home/butta/swift-5.4.2-RELEASE-ubuntu20.04`
 to the path to your Swift compiler, `/home/yourname/swift-5.4.2-RELEASE-centos8`.
@@ -63,13 +63,13 @@ same build directory. Other packages use `#file` to point at test data in the
 repo: I've had success moving this data with the test runner, after modifying
 the test source so it has the path to this test data in the Android test
 environment. See [the example](https://github.com/buttaface/swift-android-sdk/blob/main/package-patches/swift-crypto-test-vectors.patch)
-of [swift-crypto on the CI](https://github.com/buttaface/swift-android-sdk/blob/main/.github/workflows/sdks.yml#L267).
+of [swift-crypto on the CI](https://github.com/buttaface/swift-android-sdk/blob/main/.github/workflows/sdks.yml#L305).
 
 You can copy these executables and the SDK to [an emulator or a USB
 debugging-enabled device with adb](https://github.com/apple/swift/blob/release/5.4/docs/Android.md#4-deploying-the-build-products-to-the-device),
 or put them on an Android device with [a terminal emulator app like Termux](https://termux.com).
 I test aarch64 with Termux so I'll show how to run the test runner there, but
-the process is similar with adb, [as can be seen on the CI](https://github.com/buttaface/swift-android-sdk/blob/main/.github/workflows/sdks.yml#L280).
+the process is similar with adb, [as can be seen on the CI](https://github.com/buttaface/swift-android-sdk/blob/main/.github/workflows/sdks.yml#L318).
 
 Copy the test executables to the same directory as the SDK:
 ```
@@ -188,14 +188,14 @@ After making sure [needed build tools like python 3, CMake, and ninja](https://g
 are installed, I ran the following `build-script` command:
 ```
 ./swift/utils/build-script -R --no-assertions --skip-build-cmark --skip-build-llvm --android
---android-ndk /home/butta/src/android-ndk-r21e/ --android-arch aarch64 --android-api-level 24
+--android-ndk /home/butta/src/android-ndk-r23/ --android-arch aarch64 --android-api-level 24
 --android-icu-uc /home/butta/swift-5.4-android-aarch64-24-sdk/usr/lib/libicuuc.so
 --android-icu-uc-include /home/butta/swift-5.4-android-aarch64-24-sdk/usr/include/
 --android-icu-i18n /home/butta/swift-5.4-android-aarch64-24-sdk/usr/lib/libicui18n.so
 --android-icu-i18n-include /home/butta/swift-5.4-android-aarch64-24-sdk/usr/include/
 --android-icu-data /home/butta/swift-5.4-android-aarch64-24-sdk/usr/lib/libicudata.so
 --build-swift-tools=0 --native-swift-tools-path=/home/butta/swift-5.4-RELEASE-ubuntu20.04/usr/bin/
---native-clang-tools-path=/home/butta/src/android-ndk-r21e/toolchains/llvm/prebuilt/linux-x86_64/bin
+--native-clang-tools-path=/home/butta/src/android-ndk-r23/toolchains/llvm/prebuilt/linux-x86_64/bin
 --host-cc=/usr/bin/clang-11 --host-cxx=/usr/bin/clang++-11
 --cross-compile-hosts=android-aarch64 --cross-compile-deps-path=/home/butta/swift-5.4-android-aarch64-24-sdk
 --skip-local-build --xctest --swift-install-components='clang-resource-dir-symlink;license;stdlib;sdk-overlay'

diff --git a/android-aarch64.json b/android-aarch64.json
@@ -9,7 +9,7 @@
     "extra-swiftc-flags": [
         "-resource-dir", "/home/butta/swift-5.4-android-aarch64-24-sdk/usr/lib/swift",
         "-tools-directory", "/home/butta/src/android-ndk-r21e/toolchains/llvm/prebuilt/linux-x86_64/bin",
-        "-Xclang-linker", "--target=aarch64-linux-android24",
+        "-Xclang-linker", "--target=aarch64-linux-android24", "-use-ld=lld",
         "-L/home/butta/src/android-ndk-r21e/toolchains/llvm/prebuilt/linux-x86_64/lib/gcc/aarch64-linux-android/4.9.x",
     ],
     "extra-cpp-flags": [

diff --git a/android-armv7.json b/android-armv7.json
@@ -9,7 +9,7 @@
     "extra-swiftc-flags": [
         "-resource-dir", "/home/butta/swift-5.4-android-armv7-24-sdk/usr/lib/swift",
         "-tools-directory", "/home/butta/src/android-ndk-r21e/toolchains/llvm/prebuilt/linux-x86_64/bin",
-        "-Xclang-linker", "--target=armv7-linux-android24",
+        "-Xclang-linker", "--target=armv7-linux-android24", "-use-ld=lld",
         "-L/home/butta/src/android-ndk-r21e/toolchains/llvm/prebuilt/linux-x86_64/lib/gcc/armv7-linux-androideabi/4.9.x/armv7-a",
     ],
     "extra-cpp-flags": [

diff --git a/android-x86_64.json b/android-x86_64.json
@@ -9,7 +9,7 @@
     "extra-swiftc-flags": [
         "-resource-dir", "/home/butta/swift-5.4-android-x86_64-24-sdk/usr/lib/swift",
         "-tools-directory", "/home/butta/src/android-ndk-r21e/toolchains/llvm/prebuilt/linux-x86_64/bin",
-        "-Xclang-linker", "--target=x86_64-linux-android24",
+        "-Xclang-linker", "--target=x86_64-linux-android24", "-use-ld=lld",
         "-L/home/butta/src/android-ndk-r21e/toolchains/llvm/prebuilt/linux-x86_64/lib/gcc/x86_64-linux-android/4.9.x",
     ],
     "extra-cpp-flags": [

diff --git a/swift-android-5.4.patch b/swift-android-5.4.patch
@@ -1,3 +1,16 @@
+diff --git a/swift/CMakeLists.txt b/swift/CMakeLists.txt
+index 6d0493d9e00..584890d655e 100644
+--- a/swift/CMakeLists.txt
++++ b/swift/CMakeLists.txt
+@@ -177,6 +177,8 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL Windows AND NOT CMAKE_HOST_SYSTEM_NAME STREQUA
+   set(SWIFT_USE_LINKER_default "lld")
+ elseif(CMAKE_SYSTEM_NAME STREQUAL Darwin)
+   set(SWIFT_USE_LINKER_default "")
++elseif(SWIFT_HOST_VARIANT_SDK STREQUAL ANDROID)
++  set(SWIFT_USE_LINKER_default "lld")
+ else()
+   set(SWIFT_USE_LINKER_default "gold")
+ endif()
 diff --git a/swift/stdlib/cmake/modules/AddSwiftStdlib.cmake b/swift/stdlib/cmake/modules/AddSwiftStdlib.cmake
 index 38191b01a64..1f851d3409c 100644
 --- a/swift/stdlib/cmake/modules/AddSwiftStdlib.cmake
@@ -28,6 +41,17 @@ diff --git a/swift/stdlib/public/Platform/bionic.modulemap.gyb b/swift/stdlib/pu
 index e44f9082653..9ed6719e17d 100644
 --- a/swift/stdlib/public/Platform/bionic.modulemap.gyb
 +++ b/swift/stdlib/public/Platform/bionic.modulemap.gyb
+@@ -82,6 +82,10 @@ module SwiftGlibc [system] {
+       header "${GLIBC_INCLUDE_PATH}/locale.h"
+       export *
+     }
++    module malloc {
++      header "${GLIBC_INCLUDE_PATH}/malloc.h"
++      export *
++    }
+     module math {
+       link "m"
+       header "${GLIBC_INCLUDE_PATH}/math.h"
 @@ -185,7 +185,7 @@ module SwiftGlibc [system] {
        export *
      }
@@ -83,7 +107,7 @@ diff --git a/swift/utils/swift_build_support/swift_build_support/targets.py b/sw
 index 69d71e4e630..e222a510d58 100644
 --- a/swift/utils/swift_build_support/swift_build_support/targets.py
 +++ b/swift/utils/swift_build_support/swift_build_support/targets.py
-@@ -152,16 +152,21 @@ class Target(object):
+@@ -152,16 +152,22 @@ class Target(object):
      def name(self):
          return "{}-{}".format(self.platform.name, self.arch)
 
@@ -107,6 +131,7 @@ index 69d71e4e630..e222a510d58 100644
 +                flags += '-resource-dir %s/swift-%s/lib/swift ' % (
 +                    args.build_root, self.name)
 +            flags += '-sdk %s/sysroot ' % (android_toolchain_path)
++            flags += '-use-ld=lld '
 +            flags += '-tools-directory %s/bin' % (android_toolchain_path)
 
          return flags
@@ -134,3 +159,19 @@ index 617e27e3..0e4c233b 100644
  set_target_properties(Foundation PROPERTIES
    INSTALL_RPATH "$ORIGIN"
    BUILD_RPATH "$<TARGET_FILE_DIR:swiftDispatch>"
+diff --git a/swift-corelibs-libdispatch/src/shims/atomic.h b/swift-corelibs-libdispatch/src/shims/atomic.h
+index c002e72..bc232f1 100644
+--- a/swift-corelibs-libdispatch/src/shims/atomic.h
++++ b/swift-corelibs-libdispatch/src/shims/atomic.h
+@@ -35,7 +35,11 @@
+ #if defined(__cplusplus)
+ #define _Bool bool
+ #endif
++#if defined(__ANDROID__) && __NDK_MAJOR__ >= 23
++#include <bits/stdatomic.h>
++#else
+ #include <stdatomic.h>
++#endif
+
+ #define memory_order_ordered    memory_order_seq_cst
+ #define memory_order_dependency memory_order_acquire