From 5f1dbaca121a5b86a42a2b5330c243814735661f Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Mon, 31 Mar 2025 14:28:15 -0400 Subject: [PATCH 01/25] Add Android workflow --- .github/workflows/pull_request.yml | 22 ++++++++++++++++++++++ .gitignore | 2 ++ 2 files changed, 24 insertions(+) create mode 100644 .gitignore diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 1ed0a799..ab7601c9 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -21,3 +21,25 @@ jobs: name: docker-logs path: | *.log + + static-linux-build: + name: Build Static Linux image + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Build Docker images + working-directory: swift-ci/sdks/static-linux + run: ./build + + android-build: + name: Build Android image + runs-on: ubuntu-latest + if: false + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Build Docker images + working-directory: swift-ci/sdks/android + run: ./build + diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..98bcc3db --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.*.swp +static-swift-linux-sources From 65650529c889d988eccab044dc15c2bcb3292fc4 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Tue, 22 Apr 2025 18:00:06 -0400 Subject: [PATCH 02/25] Build Android image (#1) * Build Android image * Checkout without ssh * Retry build if it fails * Swift 6.1 Release Dockerfiles (#456) * Change binutils-gold package dependency on Debian 12 to binutils (#457) * Update installed packages after nightly platform expansion (#458) * update nightly-6.1 dependencies * update nightly-main dependencies * fix ubuntu images * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Fedora 41 Dockerfile (#464) * Build Android image * Build Android image * Swift 6.1 Release Dockerfiles (#456) * Change binutils-gold package dependency on Debian 12 to binutils (#457) * Build Android image * Build Android image * Build Android image * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Android build * Swift Android build * Swift Android build * Swift Android build --------- Co-authored-by: Mishal Shah Co-authored-by: Chris McGee <87777443+cmcgee1024@users.noreply.github.com> Co-authored-by: Justice Adams <107649528+justice-adams-apple@users.noreply.github.com> Co-authored-by: Andrew Sukach <134116196+sookach@users.noreply.github.com> --- .github/workflows/pull_request.yml | 126 +++- swift-ci/sdks/android/README.md | 43 ++ swift-ci/sdks/android/build | 91 +++ swift-ci/sdks/android/scripts/build.sh | 536 ++++++++++++++++++ swift-ci/sdks/android/scripts/fetch-source.sh | 183 ++++++ 5 files changed, 974 insertions(+), 5 deletions(-) create mode 100644 swift-ci/sdks/android/README.md create mode 100755 swift-ci/sdks/android/build create mode 100755 swift-ci/sdks/android/scripts/build.sh create mode 100755 swift-ci/sdks/android/scripts/fetch-source.sh diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ab7601c9..ed578bc6 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -33,13 +33,129 @@ jobs: run: ./build android-build: - name: Build Android image - runs-on: ubuntu-latest - if: false + name: Build Android ${{ matrix.arch }} SDK + strategy: + fail-fast: false + matrix: + # blank arch builds all (aarch64,x86_64,armv7) + #arch: [''] + # builds only x86_64 to speed up the validation + #arch: ['x86_64'] + # build both the quick (x86_64) and complete (aarch64,x86_64,armv7) SDKs + arch: ['x86_64', ''] + runs-on: ubuntu-24.04 steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Build Docker images + - name: Install Dependencies + run: | + sudo apt install -q ninja-build patchelf + - name: Build Android SDK working-directory: swift-ci/sdks/android - run: ./build + run: | + TARGET_ARCHS=${{ matrix.arch }} ./build + - name: Get artifact info + id: info + shell: bash + run: | + set -ex + SWIFT_ROOT=$(dirname ${{ runner.temp }}/swift-android-sdk/host-toolchain/*/usr) + echo "swift-root=${SWIFT_ROOT}" >> $GITHUB_OUTPUT + echo "swift-path=${SWIFT_ROOT}/usr/bin/swift" >> $GITHUB_OUTPUT + + ARTIFACT_BUILD=$(realpath ${{ runner.temp }}/swift-android-sdk/build/*.artifactbundle) + ARTIFACT_PATH=$(realpath ${{ runner.temp }}/swift-android-sdk/products/*.artifactbundle.tar.gz) + echo "artifact-path=${ARTIFACT_PATH}" >> $GITHUB_OUTPUT + echo "sdk-id=x86_64-unknown-linux-android28" >> $GITHUB_OUTPUT + + ARTIFACT_EXT=".artifactbundle.tar.gz" + ARTIFACT_NAME="$(basename ${ARTIFACT_PATH} ${ARTIFACT_EXT})" + # artifacts need a unique name so we suffix with the matrix arch(s) + if [[ ! -z "${{ matrix.arch }}" ]]; then + ARTIFACT_NAME="${ARTIFACT_NAME}-$(echo ${{ matrix.arch }} | tr ',' '-')" + fi + ARTIFACT_NAME="${ARTIFACT_NAME}${ARTIFACT_EXT}" + + # There is no way to prevent even a single-file artifact from being zipped: + # https://github.com/actions/upload-artifact?tab=readme-ov-file#zip-archives + # so the actual artifact download will look like: + # swift-6.1-RELEASE_android-0.1-x86_64.artifactbundle.tar.gz.zip + echo "artifact-name=${ARTIFACT_NAME}" >> $GITHUB_OUTPUT + + # validate some required paths in the artifactbundle + pushd ${ARTIFACT_BUILD}/*/*/usr + ls lib/swift/android + ls lib/swift/android/* + ls lib/swift/android/*/swiftrt.o + ls lib/*-linux-android/*/crtbegin_dynamic.o + + ls lib/swift_static-* + ls lib/swift_static-*/android + ls lib/swift_static-*/android/libFoundationEssentials.a + + ls lib/*-linux-android/libFoundationEssentials.so + ls lib/*-linux-android/libFoundationNetworking.so + ls lib/*-linux-android/libFoundationInternationalization.so + ls lib/*-linux-android/lib_FoundationICU.so + ls lib/*-linux-android/libFoundationXML.so + ls lib/*-linux-android/libTesting.so + + ls lib/swift/clang/lib + ls lib/swift/clang/lib/linux + ls lib/swift/clang/lib/linux/*/libunwind.a + popd + - name: Upload SDK artifactbundle + uses: actions/upload-artifact@v4 + with: + compression-level: 0 + name: ${{ steps.info.outputs.artifact-name }} + path: ${{ steps.info.outputs.artifact-path }} + - name: Cleanup + run: | + # need to free up some space or else when installing we get: No space left on device + rm -rf ${{ runner.temp }}/swift-android-sdk/{build,src} + - name: Install artifactbundle + shell: bash + run: | + set -ex + ${{ steps.info.outputs.swift-path }} sdk install ${{ steps.info.outputs.artifact-path }} + ${{ steps.info.outputs.swift-path }} sdk configure --show-configuration $(${{ steps.info.outputs.swift-path }} sdk list | head -n 1) ${{ steps.info.outputs.sdk-id }} + # recent releases require that ANDROID_NDK_ROOT *not* be set + # see https://github.com/finagolfin/swift-android-sdk/issues/207 + echo "ANDROID_NDK_ROOT=" >> $GITHUB_ENV + + - name: Create Demo Project + run: | + cd ${{ runner.temp }} + mkdir DemoProject + cd DemoProject + ${{ steps.info.outputs.swift-path }} --version + ${{ steps.info.outputs.swift-path }} package init + echo 'import Foundation' >> Sources/DemoProject/DemoProject.swift + echo 'import FoundationEssentials' >> Sources/DemoProject/DemoProject.swift + echo 'import FoundationXML' >> Sources/DemoProject/DemoProject.swift + echo 'import FoundationNetworking' >> Sources/DemoProject/DemoProject.swift + echo 'import Dispatch' >> Sources/DemoProject/DemoProject.swift + echo 'import Android' >> Sources/DemoProject/DemoProject.swift + - name: Test Demo Project on Android + uses: skiptools/swift-android-action@v2 + with: + # only test for the complete arch SDK build to speed up CI + run-tests: ${{ matrix.arch == '' }} + package-path: ${{ runner.temp }}/DemoProject + installed-sdk: ${{ steps.info.outputs.sdk-id }} + installed-swift: ${{ steps.info.outputs.swift-root }} + + - name: Checkout swift-algorithms + uses: actions/checkout@v4 + with: + repository: apple/swift-algorithms + path: swift-algorithms + - name: Test swift-algorithms + uses: skiptools/swift-android-action@v2 + with: + run-tests: ${{ matrix.arch == '' }} + package-path: swift-algorithms + installed-sdk: ${{ steps.info.outputs.sdk-id }} + installed-swift: ${{ steps.info.outputs.swift-root }} diff --git a/swift-ci/sdks/android/README.md b/swift-ci/sdks/android/README.md new file mode 100644 index 00000000..531aa88a --- /dev/null +++ b/swift-ci/sdks/android/README.md @@ -0,0 +1,43 @@ +# Build scripts for Swift Android SDK + +This folder contains scripts to build a Swift Android SDK +in the form of an artifactbundle. + +## Running + +The top-level `./build` script installs a host toolchain and the +Android NDK, and then invokes `scripts/fetch-source.sh` which will +fetch tagged sources for libxml2, curl, boringssl, and swift. + +It then applies some patches and invokes `scripts/build.sh`, +which will build the sources for each of the specified +architectures. Finally, it combines the NDK and the newly built +SDKs into a single artifactbundle. + +## Specifying Architectures + +By default all the supported Android architectures +(`aarch64`, `x86_64`, `aarmv7`) +will be built, but this can be reduced in order to speed +up the build. This can be useful, e.g., as part of a CI that +validates a pull request, as building a single architecture +takes around 30 minutes on a standard ubuntu-24.04 GitHub runner, +whereas building for all the architectures takes over an hour. + +To build an artifactbundle for just the `x86_64` architecture, run: + +``` +TARGET_ARCHS=x86_64 ./build +``` + +## Installing and validating the SDK + +The `.github/workflows/pull_request.yml` workflow +will create and upload an installable SDK named something like: +`swift-6.1-RELEASE_android-0.1.artifactbundle.tar.gz` + +The workflow will also install the SDK locally and use +[swift-android-action](https://github.com/marketplace/actions/swift-android-action) +to build and test various Swift packages in an Android emulator. + + diff --git a/swift-ci/sdks/android/build b/swift-ci/sdks/android/build new file mode 100755 index 00000000..e35bd431 --- /dev/null +++ b/swift-ci/sdks/android/build @@ -0,0 +1,91 @@ +#!/bin/bash -e +# Swift Android SDK: Top-level Build Script + +# default architectures to build for +TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} + +ANDROID_NDK_VERSION=android-ndk-r27c +ANDROID_API=28 + +SDKROOT=${RUNNER_TEMP:-${TMPDIR:-'/tmp'}}/swift-android-sdk +mkdir -p ${SDKROOT} + +# Install a Swift host toolchain if it isn't already present +SWIFT_ROOT=${SDKROOT}/host-toolchain +HOST_OS=ubuntu$(lsb_release -sr) +SWIFT_VERSION=6.1 +SWIFT_TAG="swift-${SWIFT_VERSION}-RELEASE" +SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" +SWIFT_BASE=$SWIFT_TAG-$HOST_OS + +if [[ ! -d $SWIFT_ROOT ]]; then + mkdir -p $SWIFT_ROOT + pushd $SWIFT_ROOT + + SWIFT_URL="https://download.swift.org/$SWIFT_BRANCH/$(echo $HOST_OS | tr -d '.')/$SWIFT_TAG/$SWIFT_BASE.tar.gz" + wget -q $SWIFT_URL + tar xzf $SWIFT_BASE.tar.gz + + popd +fi + +#HOST_TOOLCHAIN=${HOST_TOOLCHAIN:-$(dirname $(dirname $(which swiftc)))} +HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr +$HOST_TOOLCHAIN/bin/swift --version + +# ensure the correct Swift is first in the PATH +export PATH=$HOST_TOOLCHAIN/bin:$PATH + +export ANDROID_NDK_HOME=${SDKROOT}/ndk/${ANDROID_NDK_VERSION} +export ANDROID_NDK=${ANDROID_NDK_HOME} + +if [[ ! -d ${ANDROID_NDK_HOME} ]]; then + mkdir -p $(dirname ${ANDROID_NDK_HOME}) + pushd $(dirname ${ANDROID_NDK_HOME}) + NDKFILE=$(basename $ANDROID_NDK_HOME)-linux.zip + wget -q https://dl.google.com/android/repository/${NDKFILE} + unzip -q ${NDKFILE} + popd +fi + +mkdir -p ${SDKROOT}/products + +# Check-out the sources +if [[ ! -d ${SDKROOT}/src ]]; then + scripts/fetch-source.sh --source-dir ${SDKROOT}/src --swift-tag ${SWIFT_TAG} +fi + +# fetch and apply the patches +PATCHDIR=${SDKROOT}/patches +if [[ ! -d ${PATCHDIR} ]]; then + git clone https://github.com/finagolfin/swift-android-sdk.git ${PATCHDIR} + + # TODO: need to selectively apply patches based on release or not release + pushd ${SDKROOT}/src/swift-project + echo "Applying patches" + + # patch the patch, which seems to only be needed for an API less than 28 + # https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 + perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $PATCHDIR/swift-android.patch + + # remove the need to link in android-execinfo + perl -pi -e 's/dispatch android-execinfo/dispatch/g' $PATCHDIR/swift-android.patch + + git apply -v $PATCHDIR/swift-android.patch + # swift-android-ci.patch is not needed, since it is only used for llbuild, etc. + #git apply -C1 $PATCHDIR/swift-android-ci.patch + #git apply -v $PATCHDIR/swift-android-ci-release.patch + git apply -v $PATCHDIR/swift-android-testing-release.patch + + perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift + # disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) + perl -pi -e 's/os\(Android\)/os\(AndroidDISABLED\)/g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift + + # need to un-apply libandroid-spawn since we don't need it for API28+ + perl -pi -e 's/MATCHES "Android"/MATCHES "AndroidDISABLED"/g' llbuild/lib/llvm/Support/CMakeLists.txt + perl -pi -e 's/ STREQUAL Android\)/ STREQUAL AndroidDISABLED\)/g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt + popd +fi + +./scripts/build.sh --products-dir ${SDKROOT}/products --source-dir ${SDKROOT}/src --build-dir ${SDKROOT}/build --ndk-home ${ANDROID_NDK_HOME} --android-api ${ANDROID_API} --host-toolchain ${HOST_TOOLCHAIN} --archs ${TARGET_ARCHS} + diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh new file mode 100755 index 00000000..843ff2db --- /dev/null +++ b/swift-ci/sdks/android/scripts/build.sh @@ -0,0 +1,536 @@ +#!/bin/bash +# Swift SDK for Android: Build Script +set -e + +# Docker sets TERM to xterm if using a pty; we probably want +# xterm-256color, otherwise we only get eight colors +if [ -t 1 ]; then + if [[ "$TERM" == "xterm" ]]; then + export TERM=xterm-256color + fi +fi + +if [[ -n "$TERM" ]]; then + bold="" + white="" + grey="" + reset="" +else + bold=$(tput bold) + white=$(tput setaf 15) + grey=$(tput setaf 8) + reset=$(tput sgr0) +fi + +function cleanup { + echo "${reset}" +} +trap cleanup EXIT + +function header { + local text="$1" + echo "" + echo "${white}${bold}*** ${text} ***${reset}${grey}" + echo "" +} + +function groupstart { + local text="$1" + if [[ ! -z "$CI" ]]; then + echo "::group::${text}" + fi + header $text +} + +function groupend { + if [[ ! -z "$CI" ]]; then + echo "::endgroup::" + fi +} + +function usage { + cat < --products-dir --ndk-home --host-toolchain + [--name ] [--version ] [--build-dir ] + [--archs [, ...]] + +Build the Swift Android SDK. + +Options: + + --name Specify the name of the SDK bundle. + --version Specify the version of the Android SDK. + --source-dir Specify the path in which the sources can be found. + --ndk-home Specify the path to the Android NDK + --host-toolchain Specify the path to the host Swift toolchain + --products-dir Specify the path in which the products should be written. + --build-dir Specify the path in which intermediates should be stored. + --android-api Specify the Android API level + (Default is ${android_api}). + --archs [, ...] + Specify the architectures for which we should build + the SDK. + (Default is ${archs}). + --build Specify the CMake build type to use (Release, Debug, + RelWithDebInfo). + (Default is ${build_type}). + -j + --jobs Specify the number of parallel jobs to run at a time. + (Default is ${parallel_jobs}.) +EOF +} + +# Declare all the packages we depend on +declare -a packages + +function declare_package +{ + local name=$1 + local userVisibleName=$2 + local license=$3 + local url=$4 + + local snake=$(echo ${name} | tr '_' '-') + + declare -g ${name}_snake="$snake" + declare -g ${name}_name="$userVisibleName" + declare -g ${name}_license="$license" + declare -g ${name}_url="$url" + + packages+=(${name}) +} + +declare_package android_sdk \ + "Swift SDK for Android" \ + "Apache-2.0" "https://swift.org/install" +declare_package swift "swift" "Apache-2.0" "https://swift.org" +declare_package libxml2 "libxml2" "MIT" \ + "https://github.com/GNOME/libxml2" +declare_package curl "curl" "MIT" "https://curl.se" +declare_package boringssl "boringssl" "OpenSSL AND ISC AND MIT" \ + "https://boringssl.googlesource.com/boringssl/" +declare_package zlib "zlib" "Zlib" "https://zlib.net" + +# Parse command line arguments +android_sdk_version=0.1 +sdk_name= +archs=aarch64,armv7,x86_64 +android_api=28 +build_type=Release +parallel_jobs=$(($(nproc --all) + 2)) +source_dir= +ndk_home=${ANDROID_NDK} +build_dir=$(pwd)/build +products_dir= + +while [ "$#" -gt 0 ]; do + case "$1" in + --source-dir) + source_dir="$2"; shift ;; + --ndk-home) + ndk_home="$2"; shift ;; + --host-toolchain) + host_toolchain="$2"; shift ;; + --build-dir) + build_dir="$2"; shift ;; + --android-api) + android_api="$2"; shift ;; + --products-dir) + products_dir="$2"; shift ;; + --name) + sdk_name="$2"; shift ;; + --archs) + archs="$2"; shift ;; + --version) + android_sdk_version="$2"; shift ;; + -j|--jobs) + parallel_jobs=$2; shift ;; + *) + echo "Unknown argument '$1'"; usage; exit 0 ;; + esac + shift +done + +# Change the commas for spaces +archs="${archs//,/ }" + +if [[ -z "$source_dir" || -z "$products_dir" || -z "$ndk_home" || -z "$host_toolchain" ]]; then + usage + exit 1 +fi + +if ! swiftc=$(which swiftc); then + echo "build.sh: Unable to find Swift compiler. You must have a Swift toolchain installed to build the Android SDK." + exit 1 +fi + +script_dir=$(dirname -- "${BASH_SOURCE[0]}") +resource_dir="${script_dir}/../resources" + +# Find the version numbers of the various dependencies +function describe { + pushd $1 >/dev/null 2>&1 + git describe --tags + popd >/dev/null 2>&1 +} +function versionFromTag { + desc=$(describe $1) + if [[ $desc == v* ]]; then + echo "${desc#v}" + else + echo "${desc}" + fi +} + +swift_version=$(describe ${source_dir}/swift-project/swift) +if [[ $swift_version == swift-* ]]; then + swift_version=${swift_version#swift-} +fi + +if [[ -z "$sdk_name" ]]; then + sdk_name=swift-${swift_version}_android-${android_sdk_version} +fi + +libxml2_version=$(versionFromTag ${source_dir}/libxml2) + +curl_desc=$(describe ${source_dir}/curl | tr '_' '.') +curl_version=${curl_desc#curl-} + +boringssl_version=$(describe ${source_dir}/boringssl) + +zlib_version=$(versionFromTag ${source_dir}/zlib) + +function quiet_pushd { + pushd "$1" >/dev/null 2>&1 +} +function quiet_popd { + popd >/dev/null 2>&1 +} + +header "Swift Android SDK build script" + +swift_dir=$(realpath $(dirname "$swiftc")/..) +HOST=linux-x86_64 +#HOST=$(uname -s -m | tr '[:upper:]' '[:lower:]' | tr ' ' '-') +ndk_toolchain=$ndk_home/toolchains/llvm/prebuilt/$HOST + + +echo "Swift found at ${swift_dir}" +echo "Host toolchain found at ${host_toolchain}" +${host_toolchain}/bin/swift --version +echo "Android NDK found at ${ndk_home}" +${ndk_toolchain}/bin/clang --version +echo "Building for ${archs}" +echo "Sources are in ${source_dir}" +echo "Build will happen in ${build_dir}" +echo "Products will be placed in ${products_dir}" +echo +echo "Building from:" +echo " - Swift ${swift_version}" +echo " - libxml2 ${libxml2_version}" +echo " - curl ${curl_version}" +echo " - BoringSSL ${boringssl_version}" +echo " - zlib ${zlib_version}" + +function run() { + echo "$@" + "$@" +} + +for arch in $archs; do + # enable short-circuiting the individual builds + if [[ ! -z "$SWIFT_ANDROID_ARCHIVEONLY" ]]; then + continue + fi + + case $arch in + armv7) target_host="arm-linux-androideabi"; compiler_target_host="armv7a-linux-androideabi$android_api"; android_abi="armeabi-v7a" ;; + aarch64) target_host="aarch64-linux-android"; compiler_target_host="$target_host$android_api"; android_abi="arm64-v8a" ;; + x86_64) target_host="x86_64-linux-android"; compiler_target_host="$target_host$android_api"; android_abi="x86_64" ;; + x86) target_host="x86-linux-android"; compiler_target_host="$target_host$android_api"; android_abi="x86" ;; + *) echo "Unknown architecture '$1'"; usage; exit 0 ;; + esac + + sdk_root=${build_dir}/sdk_root/${arch} + mkdir -p "$sdk_root" + + groupstart "Building libxml2 for $arch" + quiet_pushd ${source_dir}/libxml2 + run cmake \ + -G Ninja \ + -S ${source_dir}/libxml2 \ + -B ${build_dir}/$arch/libxml2 \ + -DANDROID_ABI=$android_abi \ + -DANDROID_PLATFORM=android-$android_api \ + -DCMAKE_TOOLCHAIN_FILE=$ndk_home/build/cmake/android.toolchain.cmake \ + -DCMAKE_BUILD_TYPE=$build_type \ + -DCMAKE_EXTRA_LINK_FLAGS="-rtlib=compiler-rt -unwindlib=libunwind -stdlib=libc++ -fuse-ld=lld -lc++ -lc++abi" \ + -DCMAKE_BUILD_TYPE=$build_type \ + -DCMAKE_INSTALL_PREFIX=$sdk_root/usr \ + -DLIBXML2_WITH_PYTHON=NO \ + -DLIBXML2_WITH_ICU=NO \ + -DLIBXML2_WITH_ICONV=NO \ + -DLIBXML2_WITH_LZMA=NO \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_STATIC_LIBS=ON + + quiet_pushd ${build_dir}/$arch/libxml2 + run ninja -j$parallel_jobs + quiet_popd + + header "Installing libxml2 for $arch" + quiet_pushd ${build_dir}/$arch/libxml2 + run ninja -j$parallel_jobs install + quiet_popd + quiet_popd + groupend + + groupstart "Building boringssl for ${compiler_target_host}" + quiet_pushd ${source_dir}/boringssl + run cmake \ + -GNinja \ + -B ${build_dir}/$arch/boringssl \ + -DANDROID_ABI=$android_abi \ + -DANDROID_PLATFORM=android-$android_api \ + -DCMAKE_TOOLCHAIN_FILE=$ndk_home/build/cmake/android.toolchain.cmake \ + -DCMAKE_BUILD_TYPE=$build_type \ + -DCMAKE_INSTALL_PREFIX=$sdk_root/usr \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_STATIC_LIBS=ON \ + -DBUILD_TESTING=OFF + + quiet_pushd ${build_dir}/$arch/boringssl + run ninja -j$parallel_jobs + quiet_popd + + header "Installing BoringSSL for $arch" + quiet_pushd ${build_dir}/$arch/boringssl + run ninja -j$parallel_jobs install + quiet_popd + quiet_popd + groupend + + groupstart "Building libcurl for ${compiler_target_host}" + quiet_pushd ${source_dir}/curl + run cmake \ + -G Ninja \ + -S ${source_dir}/curl \ + -B ${build_dir}/$arch/curl \ + -DANDROID_ABI=$android_abi \ + -DANDROID_PLATFORM=android-$android_api \ + -DCMAKE_TOOLCHAIN_FILE=$ndk_home/build/cmake/android.toolchain.cmake \ + -DCMAKE_BUILD_TYPE=$build_type \ + -DCMAKE_INSTALL_PREFIX=$sdk_root/usr \ + -DOPENSSL_ROOT_DIR=$sdk_root/usr \ + -DOPENSSL_INCLUDE_DIR=$sdk_root/usr/include \ + -DOPENSSL_SSL_LIBRARY=$sdk_root/usr/lib/libssl.a \ + -DOPENSSL_CRYPTO_LIBRARY=$sdk_root/usr/lib/libcrypto.a \ + -DCURL_USE_OPENSSL=ON \ + -DCURLSSLOPT_NATIVE_CA=ON \ + -DTHREADS_PREFER_PTHREAD_FLAG=OFF \ + -DCMAKE_THREAD_PREFER_PTHREAD=OFF \ + -DCMAKE_THREADS_PREFER_PTHREAD_FLAG=OFF \ + -DCMAKE_HAVE_LIBC_PTHREAD=YES \ + -DBUILD_CURL_EXE=NO \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_STATIC_LIBS=ON \ + -DCURL_BUILD_TESTS=OFF + + quiet_pushd ${build_dir}/$arch/curl + run ninja -j$parallel_jobs + quiet_popd + + header "Installing libcurl for $arch" + quiet_pushd ${build_dir}/$arch/curl + run ninja -j$parallel_jobs install + quiet_popd + quiet_popd + groupend + + groupstart "Building Android SDK for ${compiler_target_host}" + quiet_pushd ${source_dir}/swift-project + build_type_flag="--debug" + case $build_type in + Debug) build_type_flag="--debug" ;; + Release) build_type_flag="--release" ;; + RelWithDebInfo) build_type_flag="--release-debuginfo" ;; + esac + + # use an out-of-tree build folder, otherwise subsequent arch builds have conflicts + export SWIFT_BUILD_ROOT=${build_dir}/$arch/swift-project + + ./swift/utils/build-script \ + $build_type_flag \ + --reconfigure \ + --no-assertions \ + --android \ + --android-ndk=$ndk_home \ + --android-arch=$arch \ + --android-api-level=$android_api \ + --native-swift-tools-path=$host_toolchain/bin \ + --native-clang-tools-path=$host_toolchain/bin \ + --cross-compile-hosts=android-$arch \ + --cross-compile-deps-path=$sdk_root \ + --install-destdir=$sdk_root \ + --build-llvm=0 \ + --build-swift-tools=0 \ + --skip-build-cmark \ + --skip-local-build \ + --build-swift-static-stdlib \ + --install-swift \ + --install-libdispatch \ + --install-foundation \ + --xctest --install-xctest \ + --swift-testing --install-swift-testing \ + --cross-compile-append-host-target-to-destdir=False + + # need to remove symlink that gets created in the NDK to the previous arch's build + # or else we get errors like: + # error: could not find module '_Builtin_float' for target 'x86_64-unknown-linux-android'; found: aarch64-unknown-linux-android, at: /home/runner/work/_temp/swift-android-sdk/ndk/android-ndk-r27c/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/swift/android/_Builtin_float.swiftmodule + rm -f $ndk_toolchain/sysroot/usr/lib/swift + quiet_popd + groupend +done + +# Now generate the bundle +groupstart "Bundling SDK" + +sdk_base=swift-android +sdk_staging="sdk_staging" + +bundle="${sdk_name}.artifactbundle" + +rm -rf ${build_dir}/$bundle +mkdir -p ${build_dir}/$bundle +quiet_pushd ${build_dir}/$bundle + +# First the info.json, for SwiftPM +cat > info.json < $sysroot_path/SDKSettings.json < swift-sdk.json <> swift-sdk.json <> swift-sdk.json <> swift-sdk.json < swift-toolset.json <|--swift-tag + |--swift-version ] + [--curl-version ] + [--boringssl-version ] + [--clone-with-ssh] + [--source-dir ] + +Fetch all the sources required to build the fully statically linked Linux +SDK for Swift. Options are: + + --clone-with-ssh Use git-over-SSH rather than HTTPS where possible. + --source-dir Specify the path in which the sources should be checked + out. This directory will be created it if does not exist. + --swift-scheme + --swift-tag + --swift-version + Select the version of Swift to check out sources for. + If starts with "scheme:" or "tag:", it will + select a scheme or tag; otherwise it will be treated as + a version number. + --libxml2-version + --curl-version + --boringssl-version +EOF +} + +# Defaults +if [[ -z "${SWIFT_VERSION}" ]]; then + SWIFT_VERSION=scheme:release/6.1 +fi +if [[ -z "${LIBXML2_VERSION}" ]]; then + LIBXML2_VERSION=2.12.7 + #LIBXML2_VERSION=2.14.2 +fi +if [[ -z "${CURL_VERSION}" ]]; then + CURL_VERSION=8.7.1 + #CURL_VERSION=8.13.0 +fi +if [[ -z "${BORINGSSL_VERSION}" ]]; then + BORINGSSL_VERSION=fips-20220613 +fi + +clone_with_ssh=false +while [ "$#" -gt 0 ]; do + case "$1" in + --swift-scheme) + SWIFT_VERSION="scheme:$2"; shift ;; + --swift-tag) + SWIFT_VERSION="tag:$2"; shift ;; + --swift-version) + SWIFT_VERSION="$2"; shift ;; + --libxml2-version) + LIBXML2_VERSION="$2"; shift ;; + --curl-version) + CURL_VERSION="$2"; shift ;; + --boringssl-version) + BORINGSSL_VERSION="$2"; shift ;; + --clone-with-ssh) + clone_with_ssh=true ;; + --source-dir) + source_dir="$2"; shift ;; + *) + usage; exit 0 ;; + esac + shift +done + +if [[ ! -z "$source_dir" ]]; then + mkdir -p "$source_dir" +else + source_dir=. +fi + +if [[ "$clone_with_ssh" == "true" ]]; then + github=git@github.com: + clone_arg=--clone-with-ssh +else + github=https://github.com/ + clone_arg=--clone +fi + +cd "$source_dir" + +# Fetch Swift +mkdir -p swift-project + +groupstart "Fetching Swift" +pushd swift-project >/dev/null + +[[ -d swift ]] || git clone ${github}apple/swift.git +cd swift + +# Get its dependencies +header "Fetching Swift Dependencies" + +extra_args="--skip-history --all-repositories" +if [[ $SWIFT_VERSION == scheme:* ]]; then + utils/update-checkout ${clone_arg} --scheme ${SWIFT_VERSION#scheme:} ${extra_args} +elif [[ $SWIFT_VERSION == tag:* ]]; then + utils/update-checkout ${clone_arg} --tag ${SWIFT_VERSION#tag:} ${extra_args} +else + utils/update-checkout ${clone_arg} --tag swift-${SWIFT_VERSION}-RELEASE ${extra_args} +fi + +popd >/dev/null +groupend + +# Fetch libxml2 +groupstart "Fetching libxml2" +[[ -d libxml2 ]] || git clone ${github}GNOME/libxml2.git +pushd libxml2 >/dev/null 2>&1 +git checkout v${LIBXML2_VERSION} +popd >/dev/null 2>&1 +groupend + +# Fetch curl +groupstart "Fetching curl" +[[ -d curl ]] || git clone ${github}curl/curl.git +pushd curl >/dev/null 2>&1 +git checkout curl-$(echo ${CURL_VERSION} | tr '.' '_') +popd >/dev/null 2>&1 +groupend + +# Fetch BoringSSL +groupstart "Fetching BoringSSL" +[[ -d boringssl ]] || git clone https://boringssl.googlesource.com/boringssl +pushd boringssl >/dev/null 2>&1 +git checkout ${BORINGSSL_VERSION} +popd >/dev/null 2>&1 +groupend + From 40733f718438f1553fcbd359c10b53f8b5d150b7 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Wed, 30 Apr 2025 20:25:12 -0400 Subject: [PATCH 03/25] Swift Android build 6.2 (#2) * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Start splitting NDK out from the rest of the SDK * Start splitting NDK out from the rest of the SDK * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 --- .github/workflows/pull_request.yml | 50 ++++++------ swift-ci/sdks/android/build | 30 +++++-- swift-ci/sdks/android/scripts/build.sh | 103 ++++++++++++++++++------- 3 files changed, 123 insertions(+), 60 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ed578bc6..582b2b9b 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -8,6 +8,8 @@ env: jobs: build: name: Build Docker images + # disabled for CI testing + if: false runs-on: ubuntu-latest steps: - name: Checkout repository @@ -24,6 +26,8 @@ jobs: static-linux-build: name: Build Static Linux image + # disabled for CI testing + if: false runs-on: ubuntu-latest steps: - name: Checkout repository @@ -33,7 +37,7 @@ jobs: run: ./build android-build: - name: Build Android ${{ matrix.arch }} SDK + name: Build Android ${{ matrix.swift-version }} ${{ matrix.arch }} SDK strategy: fail-fast: false matrix: @@ -43,8 +47,18 @@ jobs: #arch: ['x86_64'] # build both the quick (x86_64) and complete (aarch64,x86_64,armv7) SDKs arch: ['x86_64', ''] + swift-version: ['release', 'devel', 'trunk'] runs-on: ubuntu-24.04 steps: + - name: Free Disk Space + run: | + df -h + # brings available space from 25G to 32G + # otherwise we sometimes run out of space during the build + sudo rm -rf /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/share/chromium /usr/local/share/powershell /usr/share/dotnet /opt/ghc /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + sudo docker builder prune -a + df -h - name: Checkout repository uses: actions/checkout@v4 - name: Install Dependencies @@ -53,7 +67,7 @@ jobs: - name: Build Android SDK working-directory: swift-ci/sdks/android run: | - TARGET_ARCHS=${{ matrix.arch }} ./build + BUILD_VERSION=${{ matrix.swift-version }} TARGET_ARCHS=${{ matrix.arch }} ./build - name: Get artifact info id: info shell: bash @@ -82,28 +96,8 @@ jobs: # swift-6.1-RELEASE_android-0.1-x86_64.artifactbundle.tar.gz.zip echo "artifact-name=${ARTIFACT_NAME}" >> $GITHUB_OUTPUT - # validate some required paths in the artifactbundle - pushd ${ARTIFACT_BUILD}/*/*/usr - ls lib/swift/android - ls lib/swift/android/* - ls lib/swift/android/*/swiftrt.o - ls lib/*-linux-android/*/crtbegin_dynamic.o - - ls lib/swift_static-* - ls lib/swift_static-*/android - ls lib/swift_static-*/android/libFoundationEssentials.a - - ls lib/*-linux-android/libFoundationEssentials.so - ls lib/*-linux-android/libFoundationNetworking.so - ls lib/*-linux-android/libFoundationInternationalization.so - ls lib/*-linux-android/lib_FoundationICU.so - ls lib/*-linux-android/libFoundationXML.so - ls lib/*-linux-android/libTesting.so - - ls lib/swift/clang/lib - ls lib/swift/clang/lib/linux - ls lib/swift/clang/lib/linux/*/libunwind.a - popd + # show an abridged tree + tree ${ARTIFACT_BUILD} --filesfirst --prune -P 'Android.swiftmodule' -P 'libswiftAndroid.*' -P 'libFoundation.*' -P 'swiftrt.o' -P 'swift*.json' -P 'info.json' -P 'api-level.h' -P 'android.modulemap' -P 'SwiftAndroidNDK.h' -P 'bridging.modulemap' -P 'linux' -P 'libclang*.a' -P 'libunwind.a' -P 'libclang_rt.builtins-*-android.a' - name: Upload SDK artifactbundle uses: actions/upload-artifact@v4 with: @@ -113,7 +107,9 @@ jobs: - name: Cleanup run: | # need to free up some space or else when installing we get: No space left on device + df -h rm -rf ${{ runner.temp }}/swift-android-sdk/{build,src} + df -h - name: Install artifactbundle shell: bash run: | @@ -138,10 +134,10 @@ jobs: echo 'import Dispatch' >> Sources/DemoProject/DemoProject.swift echo 'import Android' >> Sources/DemoProject/DemoProject.swift - name: Test Demo Project on Android - uses: skiptools/swift-android-action@v2 + uses: skiptools/swift-android-action@main with: # only test for the complete arch SDK build to speed up CI - run-tests: ${{ matrix.arch == '' }} + #run-tests: ${{ matrix.arch == '' }} package-path: ${{ runner.temp }}/DemoProject installed-sdk: ${{ steps.info.outputs.sdk-id }} installed-swift: ${{ steps.info.outputs.swift-root }} @@ -152,7 +148,7 @@ jobs: repository: apple/swift-algorithms path: swift-algorithms - name: Test swift-algorithms - uses: skiptools/swift-android-action@v2 + uses: skiptools/swift-android-action@main with: run-tests: ${{ matrix.arch == '' }} package-path: swift-algorithms diff --git a/swift-ci/sdks/android/build b/swift-ci/sdks/android/build index e35bd431..20b9fce0 100755 --- a/swift-ci/sdks/android/build +++ b/swift-ci/sdks/android/build @@ -13,12 +13,28 @@ mkdir -p ${SDKROOT} # Install a Swift host toolchain if it isn't already present SWIFT_ROOT=${SDKROOT}/host-toolchain HOST_OS=ubuntu$(lsb_release -sr) -SWIFT_VERSION=6.1 -SWIFT_TAG="swift-${SWIFT_VERSION}-RELEASE" -SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" + +#BUILD_VERSION='release' +#BUILD_VERSION='trunk' +BUILD_VERSION=${BUILD_VERSION:-'devel'} + +if [ "${BUILD_VERSION}" = 'release' ]; then + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/releases | grep -m1 swift-6.1 | cut -d- -f2) + SWIFT_TAG="swift-${LATEST_TOOLCHAIN_VERSION}-RELEASE" + SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" +elif [ "${BUILD_VERSION}" = 'devel' ]; then + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) + SWIFT_TAG="swift-6.2-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" +else + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) + SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + SWIFT_BRANCH="development" +fi + SWIFT_BASE=$SWIFT_TAG-$HOST_OS -if [[ ! -d $SWIFT_ROOT ]]; then +if [[ ! -d "$SWIFT_ROOT/$SWIFT_BASE" ]]; then mkdir -p $SWIFT_ROOT pushd $SWIFT_ROOT @@ -75,7 +91,11 @@ if [[ ! -d ${PATCHDIR} ]]; then # swift-android-ci.patch is not needed, since it is only used for llbuild, etc. #git apply -C1 $PATCHDIR/swift-android-ci.patch #git apply -v $PATCHDIR/swift-android-ci-release.patch - git apply -v $PATCHDIR/swift-android-testing-release.patch + if [ "${BUILD_VERSION}" = 'release' ]; then + git apply -v $PATCHDIR/swift-android-testing-release.patch + else + git apply -v $PATCHDIR/swift-android-testing-except-release.patch + fi perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift # disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 843ff2db..a592b2b4 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -1,6 +1,10 @@ #!/bin/bash # Swift SDK for Android: Build Script -set -e +set -ex + +# temporary for splitting out NDK installation from the rest of the SDK +#NDK_LOCATION=${NDK_LOCATION:-"merged"} +NDK_LOCATION=${NDK_LOCATION:-"external"} # Docker sets TERM to xterm if using a pty; we probably want # xterm-256color, otherwise we only get eight colors @@ -188,7 +192,7 @@ if [[ $swift_version == swift-* ]]; then fi if [[ -z "$sdk_name" ]]; then - sdk_name=swift-${swift_version}_android-${android_sdk_version} + sdk_name=swift-${swift_version}-android-${android_sdk_version} fi libxml2_version=$(versionFromTag ${source_dir}/libxml2) @@ -212,14 +216,13 @@ header "Swift Android SDK build script" swift_dir=$(realpath $(dirname "$swiftc")/..) HOST=linux-x86_64 #HOST=$(uname -s -m | tr '[:upper:]' '[:lower:]' | tr ' ' '-') -ndk_toolchain=$ndk_home/toolchains/llvm/prebuilt/$HOST - +ndk_installation=$ndk_home/toolchains/llvm/prebuilt/$HOST echo "Swift found at ${swift_dir}" echo "Host toolchain found at ${host_toolchain}" ${host_toolchain}/bin/swift --version echo "Android NDK found at ${ndk_home}" -${ndk_toolchain}/bin/clang --version +${ndk_installation}/bin/clang --version echo "Building for ${archs}" echo "Sources are in ${source_dir}" echo "Build will happen in ${build_dir}" @@ -387,7 +390,7 @@ for arch in $archs; do # need to remove symlink that gets created in the NDK to the previous arch's build # or else we get errors like: # error: could not find module '_Builtin_float' for target 'x86_64-unknown-linux-android'; found: aarch64-unknown-linux-android, at: /home/runner/work/_temp/swift-android-sdk/ndk/android-ndk-r27c/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/swift/android/_Builtin_float.swiftmodule - rm -f $ndk_toolchain/sysroot/usr/lib/swift + rm -f $ndk_installation/sysroot/usr/lib/swift quiet_popd groupend done @@ -425,12 +428,22 @@ EOF mkdir -p $sdk_base quiet_pushd $sdk_base -#sysroot_path="ndk-sysroot" -#sysroot_path="android-27c-sysroot" -sysroot_path="sysroot" -cp -a ${ndk_toolchain}/sysroot ${sysroot_path} +cp -a ${build_dir}/sdk_root ${sdk_staging} + +if [ "${NDK_LOCATION}" = "external" ]; then + swift_res_root="swift-resources" + ndk_sysroot="ndk-sysroot" + cp -a ${ndk_installation}/sysroot ${ndk_sysroot} +else + merged_sysroot_path="sysroot" + swift_res_root=${merged_sysroot_path} + ndk_sysroot=${merged_sysroot_path} + cp -a ${ndk_installation}/sysroot ${ndk_sysroot} +fi -cat > $sysroot_path/SDKSettings.json < $swift_res_root/SDKSettings.json < $sysroot_path/SDKSettings.json < swift-sdk.json <> swift-sdk.json < Date: Thu, 8 May 2025 12:15:51 -0400 Subject: [PATCH 04/25] Swift Android build 6.2 (#3) * Swift Android build 6.2 * Swift Android build 6.2 * Add post-install script to SDK bundle * Add post-install script to SDK bundle * Add post-install script to SDK bundle * Add post-install script to SDK bundle --- swift-ci/sdks/android/build | 2 +- swift-ci/sdks/android/scripts/build.sh | 113 +++++++++++++++++-------- 2 files changed, 79 insertions(+), 36 deletions(-) diff --git a/swift-ci/sdks/android/build b/swift-ci/sdks/android/build index 20b9fce0..a2834f71 100755 --- a/swift-ci/sdks/android/build +++ b/swift-ci/sdks/android/build @@ -107,5 +107,5 @@ if [[ ! -d ${PATCHDIR} ]]; then popd fi -./scripts/build.sh --products-dir ${SDKROOT}/products --source-dir ${SDKROOT}/src --build-dir ${SDKROOT}/build --ndk-home ${ANDROID_NDK_HOME} --android-api ${ANDROID_API} --host-toolchain ${HOST_TOOLCHAIN} --archs ${TARGET_ARCHS} +./scripts/build.sh --products-dir ${SDKROOT}/products --source-dir ${SDKROOT}/src --build-dir ${SDKROOT}/build --ndk-home ${ANDROID_NDK_HOME} --android-api ${ANDROID_API} --host-toolchain ${HOST_TOOLCHAIN} --archs ${TARGET_ARCHS} ${@} diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index a592b2b4..879aa1e4 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -1,10 +1,6 @@ #!/bin/bash # Swift SDK for Android: Build Script -set -ex - -# temporary for splitting out NDK installation from the rest of the SDK -#NDK_LOCATION=${NDK_LOCATION:-"merged"} -NDK_LOCATION=${NDK_LOCATION:-"external"} +set -e # Docker sets TERM to xterm if using a pty; we probably want # xterm-256color, otherwise we only get eight colors @@ -145,6 +141,8 @@ while [ "$#" -gt 0 ]; do sdk_name="$2"; shift ;; --archs) archs="$2"; shift ;; + --build) + build_type="$2"; shift ;; --version) android_sdk_version="$2"; shift ;; -j|--jobs) @@ -430,17 +428,7 @@ quiet_pushd $sdk_base cp -a ${build_dir}/sdk_root ${sdk_staging} -if [ "${NDK_LOCATION}" = "external" ]; then - swift_res_root="swift-resources" - ndk_sysroot="ndk-sysroot" - cp -a ${ndk_installation}/sysroot ${ndk_sysroot} -else - merged_sysroot_path="sysroot" - swift_res_root=${merged_sysroot_path} - ndk_sysroot=${merged_sysroot_path} - cp -a ${ndk_installation}/sysroot ${ndk_sysroot} -fi - +swift_res_root="swift-resources" mkdir -p ${swift_res_root} cat > $swift_res_root/SDKSettings.json < scripts/setup-android-sdk.sh <<'EOF' +#/bin/sh +# this script will setup the ndk-sysroot with links to the +# local installation indicated by ANDROID_NDK_HOME +set -e +if [ -z "${ANDROID_NDK_HOME}" ]; then + echo "$(basename $0): error: missing environment variable ANDROID_NDK_HOME" + exit 1 +fi +PREBUILT="${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt" +if [ ! -d "${PREBUILT}" ]; then + echo "$(basename $0): error: ANDROID_NDK_HOME not found: ${PREBUILT}" + exit 1 +fi +DESTINATION=$(dirname $(dirname $(realpath $0)))/ndk-sysroot +# clear out any previous NDK setup +rm -rf ${DESTINATION} + +# copy vs. link the NDK files +SWIFT_ANDROID_NDK_LINK=${SWIFT_ANDROID_NDK_LINK:-1} +if [[ "${SWIFT_ANDROID_NDK_LINK}" != 1 ]]; then + ANDROID_NDK_DESC="copied" + cp -a ${PREBUILT}/*/sysroot ${DESTINATION} +else + ANDROID_NDK_DESC="linked" + mkdir -p ${DESTINATION}/usr/lib + ln -s $(realpath ${PREBUILT}/*/sysroot/usr/include) ${DESTINATION}/usr/include + for triplePath in ${PREBUILT}/*/sysroot/usr/lib/*; do + triple=$(basename ${triplePath}) + ln -s $(realpath ${triplePath}) ${DESTINATION}/usr/lib/${triple} + done +fi + +# copy each architecture's swiftrt.o into the sysroot, +# working around https://github.com/swiftlang/swift/pull/79621 +for swiftrt in ${DESTINATION}/../swift-resources/usr/lib/swift-*/android/*/swiftrt.o; do + arch=$(basename $(dirname ${swiftrt})) + mkdir -p ${DESTINATION}/usr/lib/swift/android/${arch} + cp -a ${swiftrt} ${DESTINATION}/usr/lib/swift/android/${arch} +done + +echo "$(basename $0): success: ndk-sysroot ${ANDROID_NDK_DESC} to Android SDK" +EOF + chmod +x scripts/setup-android-sdk.sh +else + COPY_NDK_SYSROOT=${COPY_NDK_SYSROOT:-1} + if [[ ${COPY_NDK_SYSROOT} == 1 ]]; then + cp -a ${ndk_installation}/sysroot ${ndk_sysroot} + else + # rather than copying the sysroot, we can instead make links to + # the various sub-folders this won't work for the distribution, + # since the NDK is going to be located in different places + # for different machines + mkdir -p ${ndk_sysroot}/usr/lib + ln -sv $(realpath ${ndk_installation}/sysroot/usr/include) ${ndk_sysroot}/usr/include + for triplePath in ${ndk_installation}/sysroot/usr/lib/*; do + triple=$(basename ${triplePath}) + ln -sv $(realpath ${triplePath}) ${ndk_sysroot}/usr/lib/${triple} + ls ${ndk_sysroot}/usr/lib/${triple}/ + done + fi + # need to manually copy over swiftrt.o or else: # error: link command failed with exit code 1 (use -v to see invocation) # clang: error: no such file or directory: '${HOME}/.swiftpm/swift-sdks/swift-6.2-DEVELOPMENT-SNAPSHOT-2025-04-24-a-android-0.1.artifactbundle/swift-android/ndk-sysroot/usr/lib/swift/android/x86_64/swiftrt.o' # see: https://github.com/swiftlang/swift-driver/pull/1822#issuecomment-2762811807 + # should be fixed by: https://github.com/swiftlang/swift/pull/79621 for arch in $archs; do mkdir -p ${ndk_sysroot}/usr/lib/swift/android/${arch} ln -srv ${swift_res_root}/usr/lib/swift-${arch}/android/${arch}/swiftrt.o ${ndk_sysroot}/usr/lib/swift/android/${arch}/swiftrt.o done -else - rm -r ${swift_res_root}/usr/{include,lib}/{i686,riscv64}-linux-android - rm -r ${swift_res_root}/usr/lib/swift/clang/lib/linux/*{i[36]86,riscv64}* fi rm -r ${swift_res_root}/usr/share/{doc,man} @@ -535,11 +582,7 @@ for api in $(eval echo "{$android_api..36}"); do EOF fi - if [ "${NDK_LOCATION}" = "external" ]; then - SWIFT_RES_DIR="swift-${arch}" - else - SWIFT_RES_DIR="swift" - fi + SWIFT_RES_DIR="swift-${arch}" SWIFT_STATIC_RES_DIR="swift_static-${arch}" cat >> swift-sdk.json < Date: Sat, 17 May 2025 20:03:09 -0400 Subject: [PATCH 05/25] Build SDK in Docker container (#4) * Add static libraries to post-install script * Add post-install script to SDK bundle * Add post-install script to SDK bundle * Update submodules --- .github/workflows/pull_request.yml | 62 +++++-- .gitmodules | 3 + swift-ci/sdks/android/Dockerfile | 89 +++++++++ swift-ci/sdks/android/README.md | 43 ++++- swift-ci/sdks/android/build | 111 ------------ swift-ci/sdks/android/build-docker | 67 +++++++ swift-ci/sdks/android/build-local | 64 +++++++ swift-ci/sdks/android/resources/patches | 1 + swift-ci/sdks/android/scripts/build.sh | 169 +++++++++--------- swift-ci/sdks/android/scripts/install-ndk.sh | 35 ++++ .../sdks/android/scripts/install-swift.sh | 54 ++++++ .../sdks/android/scripts/patch-sources.sh | 87 +++++++++ .../sdks/android/scripts/toolchain-vars.sh | 44 +++++ 13 files changed, 611 insertions(+), 218 deletions(-) create mode 100644 .gitmodules create mode 100644 swift-ci/sdks/android/Dockerfile delete mode 100755 swift-ci/sdks/android/build create mode 100755 swift-ci/sdks/android/build-docker create mode 100755 swift-ci/sdks/android/build-local create mode 160000 swift-ci/sdks/android/resources/patches create mode 100755 swift-ci/sdks/android/scripts/install-ndk.sh create mode 100755 swift-ci/sdks/android/scripts/install-swift.sh create mode 100755 swift-ci/sdks/android/scripts/patch-sources.sh create mode 100644 swift-ci/sdks/android/scripts/toolchain-vars.sh diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 582b2b9b..13c57b39 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -8,8 +8,6 @@ env: jobs: build: name: Build Docker images - # disabled for CI testing - if: false runs-on: ubuntu-latest steps: - name: Checkout repository @@ -26,8 +24,6 @@ jobs: static-linux-build: name: Build Static Linux image - # disabled for CI testing - if: false runs-on: ubuntu-latest steps: - name: Checkout repository @@ -37,16 +33,18 @@ jobs: run: ./build android-build: - name: Build Android ${{ matrix.swift-version }} ${{ matrix.arch }} SDK + name: Build Android ${{ matrix.build-type }} ${{ matrix.swift-version }} ${{ matrix.arch }} SDK strategy: fail-fast: false matrix: + build-type: ['docker'] + #build-type: ['docker', 'local'] # blank arch builds all (aarch64,x86_64,armv7) - #arch: [''] + arch: [''] # builds only x86_64 to speed up the validation #arch: ['x86_64'] # build both the quick (x86_64) and complete (aarch64,x86_64,armv7) SDKs - arch: ['x86_64', ''] + #arch: ['x86_64', ''] swift-version: ['release', 'devel', 'trunk'] runs-on: ubuntu-24.04 steps: @@ -59,31 +57,60 @@ jobs: sudo docker image prune --all --force sudo docker builder prune -a df -h + - name: Setup + id: config + run: | + # these variabes are used by build-docker and build-local + # to determine which Swift version to build for + echo "BUILD_VERSION=${{ matrix.swift-version }}" >> $GITHUB_ENV + echo "TARGET_ARCHS=${{ matrix.arch }}" >> $GITHUB_ENV + echo "WORKDIR=${{ runner.temp }}/swift-android-sdk" >> $GITHUB_ENV - name: Checkout repository uses: actions/checkout@v4 - - name: Install Dependencies + with: + submodules: 'true' + - name: Build Android SDK (Local) + if: ${{ matrix.build-type == 'local' }} + working-directory: swift-ci/sdks/android run: | sudo apt install -q ninja-build patchelf - - name: Build Android SDK + ./build-local ${BUILD_VERSION} ${WORKDIR} + - name: Build Android SDK (Docker) + if: ${{ matrix.build-type == 'docker' }} + working-directory: swift-ci/sdks/android + run: | + ./build-docker ${BUILD_VERSION} ${WORKDIR} + - name: Install Host Toolchain + if: ${{ matrix.build-type == 'docker' }} working-directory: swift-ci/sdks/android run: | - BUILD_VERSION=${{ matrix.swift-version }} TARGET_ARCHS=${{ matrix.arch }} ./build + # when building in a Docker container, we don't have a local host toolchain, + # but we need one in order to run the SDK validation tests, so we install it now + HOST_OS=ubuntu$(lsb_release -sr) + source ./scripts/toolchain-vars.sh + mkdir -p ${WORKDIR}/host-toolchain + ./scripts/install-swift.sh ${WORKDIR}/host-toolchain/$SWIFT_BASE/usr + ls ${WORKDIR}/host-toolchain + ${WORKDIR}/host-toolchain/*/usr/bin/swift --version - name: Get artifact info id: info shell: bash run: | set -ex - SWIFT_ROOT=$(dirname ${{ runner.temp }}/swift-android-sdk/host-toolchain/*/usr) + SWIFT_ROOT=$(dirname ${WORKDIR}/host-toolchain/*/usr) echo "swift-root=${SWIFT_ROOT}" >> $GITHUB_OUTPUT echo "swift-path=${SWIFT_ROOT}/usr/bin/swift" >> $GITHUB_OUTPUT - ARTIFACT_BUILD=$(realpath ${{ runner.temp }}/swift-android-sdk/build/*.artifactbundle) - ARTIFACT_PATH=$(realpath ${{ runner.temp }}/swift-android-sdk/products/*.artifactbundle.tar.gz) + ARTIFACT_PATH=$(realpath ${WORKDIR}/products/*.artifactbundle.tar.gz) echo "artifact-path=${ARTIFACT_PATH}" >> $GITHUB_OUTPUT echo "sdk-id=x86_64-unknown-linux-android28" >> $GITHUB_OUTPUT ARTIFACT_EXT=".artifactbundle.tar.gz" ARTIFACT_NAME="$(basename ${ARTIFACT_PATH} ${ARTIFACT_EXT})" + # depending on whether we are building locally or in a container, add a maker to the name + if [[ "${{ matrix.build-type }}" == 'local' ]]; then + ARTIFACT_NAME="${ARTIFACT_NAME}-local" + fi # artifacts need a unique name so we suffix with the matrix arch(s) if [[ ! -z "${{ matrix.arch }}" ]]; then ARTIFACT_NAME="${ARTIFACT_NAME}-$(echo ${{ matrix.arch }} | tr ',' '-')" @@ -95,9 +122,6 @@ jobs: # so the actual artifact download will look like: # swift-6.1-RELEASE_android-0.1-x86_64.artifactbundle.tar.gz.zip echo "artifact-name=${ARTIFACT_NAME}" >> $GITHUB_OUTPUT - - # show an abridged tree - tree ${ARTIFACT_BUILD} --filesfirst --prune -P 'Android.swiftmodule' -P 'libswiftAndroid.*' -P 'libFoundation.*' -P 'swiftrt.o' -P 'swift*.json' -P 'info.json' -P 'api-level.h' -P 'android.modulemap' -P 'SwiftAndroidNDK.h' -P 'bridging.modulemap' -P 'linux' -P 'libclang*.a' -P 'libunwind.a' -P 'libclang_rt.builtins-*-android.a' - name: Upload SDK artifactbundle uses: actions/upload-artifact@v4 with: @@ -108,7 +132,9 @@ jobs: run: | # need to free up some space or else when installing we get: No space left on device df -h - rm -rf ${{ runner.temp }}/swift-android-sdk/{build,src} + rm -rf ${WORKDIR}/{build,source} + sudo docker image prune --all --force + sudo docker builder prune -a df -h - name: Install artifactbundle shell: bash @@ -117,7 +143,7 @@ jobs: ${{ steps.info.outputs.swift-path }} sdk install ${{ steps.info.outputs.artifact-path }} ${{ steps.info.outputs.swift-path }} sdk configure --show-configuration $(${{ steps.info.outputs.swift-path }} sdk list | head -n 1) ${{ steps.info.outputs.sdk-id }} # recent releases require that ANDROID_NDK_ROOT *not* be set - # see https://github.com/finagolfin/swift-android-sdk/issues/207 + # see https://github.com/swiftlang/swift-driver/pull/1879 echo "ANDROID_NDK_ROOT=" >> $GITHUB_ENV - name: Create Demo Project diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..f3724176 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "swift-ci/sdks/android/resources/patches"] + path = swift-ci/sdks/android/resources/patches + url = https://github.com/swift-android-sdk/swift-android-sdk diff --git a/swift-ci/sdks/android/Dockerfile b/swift-ci/sdks/android/Dockerfile new file mode 100644 index 00000000..2be3fc37 --- /dev/null +++ b/swift-ci/sdks/android/Dockerfile @@ -0,0 +1,89 @@ +# ===----------------------------------------------------------------------=== +# +# Swift Android SDK: Docker-based build +# +# ===----------------------------------------------------------------------=== + +FROM ubuntu:24.04 + +# Architecture to build on (empty means x86-64) +ARG OS_ARCH_SUFFIX= + +# the Swift toolchain URL to download +ARG SWIFT_TOOLCHAIN_URL= + +# ............................................................................ + +# Install development tools +RUN apt-get -q update \ + && DEBIAN_FRONTEND=noninteractive apt-get -q install -y \ + build-essential \ + cmake \ + ninja-build \ + python3 \ + golang \ + git \ + gnupg2 \ + libsqlite3-dev \ + libcurl4-openssl-dev \ + libedit-dev \ + libicu-dev \ + libncurses5-dev \ + libpython3-dev \ + libsqlite3-dev \ + libxml2-dev \ + rsync \ + uuid-dev \ + uuid-runtime \ + tzdata \ + curl \ + unzip \ + && rm -rf /var/lib/apt-lists/* + +# Install Swift +ARG SWIFT_SIGNING_KEY=E813C892820A6FA13755B268F167DF1ACF9CE069 +ARG SWIFT_PLATFORM=ubuntu +ARG OS_MAJOR_VER=24 +ARG OS_MINOR_VER=04 + +ENV SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \ + SWIFT_PLATFORM=$SWIFT_PLATFORM \ + OS_MAJOR_VER=$OS_MAJOR_VER \ + OS_MINOR_VER=$OS_MINOR_VER \ + OS_VER=$SWIFT_PLATFORM$OS_MAJOR_VER.$OS_MINOR_VER + +COPY scripts/install-swift.sh /scripts/install-swift.sh +RUN chmod ugo+x /scripts/install-swift.sh +RUN /scripts/install-swift.sh /usr/local/swift +ENV PATH="/usr/local/swift/bin:${PATH}" + +ARG ANDROID_NDK_VERSION= + +ENV ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION + +COPY scripts/install-ndk.sh /scripts/install-ndk.sh +RUN chmod ugo+x /scripts/install-ndk.sh +RUN /scripts/install-ndk.sh +ENV ANDROID_NDK_HOME="/usr/local/ndk/${ANDROID_NDK_VERSION}" + +ENV SWIFT_VERSION=$SWIFT_VERSION \ + LIBXML2_VERSION=$LIBXML2_VERSION \ + CURL_VERSION=$CURL_VERSION \ + BORINGSSL_VERSION=$BORINGSSL_VERSION \ + ICU_VERSION=$ICU_VERSION \ + ZLIB_VERSION=$ZLIB_VERSION + +ENV SWIFT_BUILD_DOCKER="1" + +COPY scripts /scripts +RUN chmod ugo+x /scripts/* + +COPY resources /resources + +# Create a user +RUN groupadd -g 998 build-user && \ + useradd -m -r -u 998 -g build-user build-user + +USER build-user + +WORKDIR /home/build-user diff --git a/swift-ci/sdks/android/README.md b/swift-ci/sdks/android/README.md index 531aa88a..10d4db3e 100644 --- a/swift-ci/sdks/android/README.md +++ b/swift-ci/sdks/android/README.md @@ -1,11 +1,39 @@ -# Build scripts for Swift Android SDK +# Dockerfile-based build for Swift Android SDK -This folder contains scripts to build a Swift Android SDK -in the form of an artifactbundle. +This is a Dockerfile-based build set-up for the Swift Android SDK. + +The top-level `./build-docker` script will create a +Docker container and install a host toolchain and the +Android NDK, and then invoke `scripts/fetch-source.sh` which will +fetch tagged sources for libxml2, curl, boringssl, and swift. + +It can be run with: + +``` +$ ./build-docker +``` + +for example: + +``` +$ ./build-docker release /tmp/android-sdk +``` + +This will create an Ubuntu 24.04 container with the necessary dependencies +to build the Android SDK, including a Swift host toolchain and the +Android NDK that will be used for cross-compilation. + +The `version` argument can be one of the following values: + +| version | Swift version | +| --- | --- | +| `release` | swift-6.1-RELEASE | +| `devel` | swift-6.2-DEVELOPMENT-SNAPSHOT-yyyy-mm-dd | +| `trunk` | swift-DEVELOPMENT-SNAPSHOT-yyyy-mm-dd | ## Running -The top-level `./build` script installs a host toolchain and the +The top-level `./build-docker` script installs a host toolchain and the Android NDK, and then invokes `scripts/fetch-source.sh` which will fetch tagged sources for libxml2, curl, boringssl, and swift. @@ -27,7 +55,7 @@ whereas building for all the architectures takes over an hour. To build an artifactbundle for just the `x86_64` architecture, run: ``` -TARGET_ARCHS=x86_64 ./build +TARGET_ARCHS=aarch64 ./build-docker release /tmp/android-sdk ``` ## Installing and validating the SDK @@ -38,6 +66,5 @@ will create and upload an installable SDK named something like: The workflow will also install the SDK locally and use [swift-android-action](https://github.com/marketplace/actions/swift-android-action) -to build and test various Swift packages in an Android emulator. - - +to build and test various Swift packages in an Android emulator using the +freshly-created SDK bundle. diff --git a/swift-ci/sdks/android/build b/swift-ci/sdks/android/build deleted file mode 100755 index a2834f71..00000000 --- a/swift-ci/sdks/android/build +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/bash -e -# Swift Android SDK: Top-level Build Script - -# default architectures to build for -TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} - -ANDROID_NDK_VERSION=android-ndk-r27c -ANDROID_API=28 - -SDKROOT=${RUNNER_TEMP:-${TMPDIR:-'/tmp'}}/swift-android-sdk -mkdir -p ${SDKROOT} - -# Install a Swift host toolchain if it isn't already present -SWIFT_ROOT=${SDKROOT}/host-toolchain -HOST_OS=ubuntu$(lsb_release -sr) - -#BUILD_VERSION='release' -#BUILD_VERSION='trunk' -BUILD_VERSION=${BUILD_VERSION:-'devel'} - -if [ "${BUILD_VERSION}" = 'release' ]; then - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/releases | grep -m1 swift-6.1 | cut -d- -f2) - SWIFT_TAG="swift-${LATEST_TOOLCHAIN_VERSION}-RELEASE" - SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" -elif [ "${BUILD_VERSION}" = 'devel' ]; then - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) - SWIFT_TAG="swift-6.2-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" - SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" -else - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) - SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" - SWIFT_BRANCH="development" -fi - -SWIFT_BASE=$SWIFT_TAG-$HOST_OS - -if [[ ! -d "$SWIFT_ROOT/$SWIFT_BASE" ]]; then - mkdir -p $SWIFT_ROOT - pushd $SWIFT_ROOT - - SWIFT_URL="https://download.swift.org/$SWIFT_BRANCH/$(echo $HOST_OS | tr -d '.')/$SWIFT_TAG/$SWIFT_BASE.tar.gz" - wget -q $SWIFT_URL - tar xzf $SWIFT_BASE.tar.gz - - popd -fi - -#HOST_TOOLCHAIN=${HOST_TOOLCHAIN:-$(dirname $(dirname $(which swiftc)))} -HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr -$HOST_TOOLCHAIN/bin/swift --version - -# ensure the correct Swift is first in the PATH -export PATH=$HOST_TOOLCHAIN/bin:$PATH - -export ANDROID_NDK_HOME=${SDKROOT}/ndk/${ANDROID_NDK_VERSION} -export ANDROID_NDK=${ANDROID_NDK_HOME} - -if [[ ! -d ${ANDROID_NDK_HOME} ]]; then - mkdir -p $(dirname ${ANDROID_NDK_HOME}) - pushd $(dirname ${ANDROID_NDK_HOME}) - NDKFILE=$(basename $ANDROID_NDK_HOME)-linux.zip - wget -q https://dl.google.com/android/repository/${NDKFILE} - unzip -q ${NDKFILE} - popd -fi - -mkdir -p ${SDKROOT}/products - -# Check-out the sources -if [[ ! -d ${SDKROOT}/src ]]; then - scripts/fetch-source.sh --source-dir ${SDKROOT}/src --swift-tag ${SWIFT_TAG} -fi - -# fetch and apply the patches -PATCHDIR=${SDKROOT}/patches -if [[ ! -d ${PATCHDIR} ]]; then - git clone https://github.com/finagolfin/swift-android-sdk.git ${PATCHDIR} - - # TODO: need to selectively apply patches based on release or not release - pushd ${SDKROOT}/src/swift-project - echo "Applying patches" - - # patch the patch, which seems to only be needed for an API less than 28 - # https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 - perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $PATCHDIR/swift-android.patch - - # remove the need to link in android-execinfo - perl -pi -e 's/dispatch android-execinfo/dispatch/g' $PATCHDIR/swift-android.patch - - git apply -v $PATCHDIR/swift-android.patch - # swift-android-ci.patch is not needed, since it is only used for llbuild, etc. - #git apply -C1 $PATCHDIR/swift-android-ci.patch - #git apply -v $PATCHDIR/swift-android-ci-release.patch - if [ "${BUILD_VERSION}" = 'release' ]; then - git apply -v $PATCHDIR/swift-android-testing-release.patch - else - git apply -v $PATCHDIR/swift-android-testing-except-release.patch - fi - - perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift - # disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) - perl -pi -e 's/os\(Android\)/os\(AndroidDISABLED\)/g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift - - # need to un-apply libandroid-spawn since we don't need it for API28+ - perl -pi -e 's/MATCHES "Android"/MATCHES "AndroidDISABLED"/g' llbuild/lib/llvm/Support/CMakeLists.txt - perl -pi -e 's/ STREQUAL Android\)/ STREQUAL AndroidDISABLED\)/g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt - popd -fi - -./scripts/build.sh --products-dir ${SDKROOT}/products --source-dir ${SDKROOT}/src --build-dir ${SDKROOT}/build --ndk-home ${ANDROID_NDK_HOME} --android-api ${ANDROID_API} --host-toolchain ${HOST_TOOLCHAIN} --archs ${TARGET_ARCHS} ${@} - diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker new file mode 100755 index 00000000..fb3b377a --- /dev/null +++ b/swift-ci/sdks/android/build-docker @@ -0,0 +1,67 @@ +#!/bin/bash -e +# +# ===----------------------------------------------------------------------=== +# +# Swift Android SDK: Docker Container Build Script +# +# ===----------------------------------------------------------------------=== + +# default architectures to build for +TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} + +ANDROID_NDK_VERSION=android-ndk-r27c +ANDROID_API=28 + +export BUILD_VERSION=${1} +# note that WORKDIR must not be under the current checkout or the patches will fail to apply +WORKDIR=${2} +if [[ "${WORKDIR}" == '' ]]; then + echo "Usage: $(basename $0) " + exit 1 +fi +mkdir -p ${WORKDIR} +WORKDIR=$(realpath ${WORKDIR}) + +HOST_OS=ubuntu24.04 +source ./scripts/toolchain-vars.sh + +# Check-out and patch the sources +./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} +./scripts/patch-sources.sh ${WORKDIR}/source + +mkdir -p ${WORKDIR}/products +chmod ugo+rwx ${WORKDIR}/products + +if [[ "$DOCKER" == "" ]]; then + DOCKER=docker +fi + +case $(arch) in + arm64|aarch64) + OS_ARCH_SUFFIX=-aarch64 + ;; + amd64|x86_64) + OS_ARCH_SUFFIX= + ;; + *) + echo "Unknown architecture $(arch)" + exit 1 + ;; +esac + +CONTAINER_NAME="swift-android" + +# Build the Docker image +$DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLCHAIN_URL=$SWIFT_TOOLCHAIN_URL --build-arg ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION -t ${CONTAINER_NAME} . + +$DOCKER run -i --rm \ + -v ${WORKDIR}/source:/source \ + -v ${WORKDIR}/products:/products:rw \ + ${CONTAINER_NAME} \ + /scripts/build.sh \ + --source-dir /source \ + --products-dir /products \ + --host-toolchain /usr/local/swift \ + --android-api ${ANDROID_API} \ + --ndk-home /usr/local/ndk/${ANDROID_NDK_VERSION} \ + --archs ${TARGET_ARCHS} diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local new file mode 100755 index 00000000..30bd4f92 --- /dev/null +++ b/swift-ci/sdks/android/build-local @@ -0,0 +1,64 @@ +#!/bin/bash -e +# +# ===----------------------------------------------------------------------=== +# +# Swift Android SDK: Local (non-Docker-containerized) Build Script +# +# ===----------------------------------------------------------------------=== + +# default architectures to build for +TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} + +ANDROID_NDK_VERSION=android-ndk-r27c +ANDROID_API=28 + +export BUILD_VERSION=${1} +# note that WORKDIR must not be under the current checkout or the patches will fail to apply +WORKDIR=${2} +if [[ "${WORKDIR}" == '' ]]; then + echo "Usage: $(basename $0) " + exit 1 +fi +mkdir -p ${WORKDIR} +WORKDIR=$(realpath ${WORKDIR}) + +HOST_OS=ubuntu$(lsb_release -sr) +source ./scripts/toolchain-vars.sh + +SWIFT_ROOT=${WORKDIR}/host-toolchain +HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr +if [[ ! -d "$HOST_TOOLCHAIN" ]]; then + ./scripts/install-swift.sh ${HOST_TOOLCHAIN} +fi + +$HOST_TOOLCHAIN/bin/swift --version + +# ensure the correct Swift is first in the PATH +export PATH=$HOST_TOOLCHAIN/bin:$PATH + +export ANDROID_NDK_HOME=${WORKDIR}/ndk/${ANDROID_NDK_VERSION} + +if [[ ! -d ${ANDROID_NDK_HOME} ]]; then + mkdir -p $(dirname ${ANDROID_NDK_HOME}) + pushd $(dirname ${ANDROID_NDK_HOME}) + NDKFILE=$(basename $ANDROID_NDK_HOME)-linux.zip + wget -q https://dl.google.com/android/repository/${NDKFILE} + unzip -q ${NDKFILE} + popd +fi + + +# Check-out and patch the sources +./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} +./scripts/patch-sources.sh ${WORKDIR}/source + +mkdir -p ${WORKDIR}/products + +./scripts/build.sh \ + --source-dir ${WORKDIR}/source \ + --products-dir ${WORKDIR}/products \ + --build-dir ${WORKDIR}/build \ + --host-toolchain ${HOST_TOOLCHAIN} \ + --android-api ${ANDROID_API} \ + --ndk-home ${ANDROID_NDK_HOME} \ + --archs ${TARGET_ARCHS} diff --git a/swift-ci/sdks/android/resources/patches b/swift-ci/sdks/android/resources/patches new file mode 160000 index 00000000..d0828093 --- /dev/null +++ b/swift-ci/sdks/android/resources/patches @@ -0,0 +1 @@ +Subproject commit d08280938ff70b790bbd0f559c0b420b518877ca diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 879aa1e4..20877272 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -109,7 +109,6 @@ declare_package libxml2 "libxml2" "MIT" \ declare_package curl "curl" "MIT" "https://curl.se" declare_package boringssl "boringssl" "OpenSSL AND ISC AND MIT" \ "https://boringssl.googlesource.com/boringssl/" -declare_package zlib "zlib" "Zlib" "https://zlib.net" # Parse command line arguments android_sdk_version=0.1 @@ -166,12 +165,14 @@ if ! swiftc=$(which swiftc); then exit 1 fi -script_dir=$(dirname -- "${BASH_SOURCE[0]}") -resource_dir="${script_dir}/../resources" - # Find the version numbers of the various dependencies function describe { pushd $1 >/dev/null 2>&1 + # this is needed for docker containers or else we get the error: + # fatal: detected dubious ownership in repository at '/source/curl' + if [[ "${SWIFT_BUILD_DOCKER}" == "1" ]]; then + git config --global --add safe.directory $(pwd) + fi git describe --tags popd >/dev/null 2>&1 } @@ -200,8 +201,6 @@ curl_version=${curl_desc#curl-} boringssl_version=$(describe ${source_dir}/boringssl) -zlib_version=$(versionFromTag ${source_dir}/zlib) - function quiet_pushd { pushd "$1" >/dev/null 2>&1 } @@ -214,8 +213,24 @@ header "Swift Android SDK build script" swift_dir=$(realpath $(dirname "$swiftc")/..) HOST=linux-x86_64 #HOST=$(uname -s -m | tr '[:upper:]' '[:lower:]' | tr ' ' '-') + +# in a Docker container, the pre-installed NDK is read-only, +# but the build script needs to write to it to work around +# https://github.com/swiftlang/swift-driver/pull/1822 +# so we copy it to a read-write location for the purposes of the build +# this can all be removed once that PR lands +mkdir -p ${build_dir}/ndk/ +ndk_home_tmp=${build_dir}/ndk/$(basename $ndk_home) +cp -a $ndk_home $ndk_home_tmp +ndk_home=$ndk_home_tmp + ndk_installation=$ndk_home/toolchains/llvm/prebuilt/$HOST +# ANDROID_NDK env needed by the swift-android.patch for: +# call ln -sf "${SWIFT_BUILD_PATH}/lib/swift" "${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib" +export ANDROID_NDK_HOME=$ndk_home +export ANDROID_NDK=$ndk_home + echo "Swift found at ${swift_dir}" echo "Host toolchain found at ${host_toolchain}" ${host_toolchain}/bin/swift --version @@ -231,7 +246,12 @@ echo " - Swift ${swift_version}" echo " - libxml2 ${libxml2_version}" echo " - curl ${curl_version}" echo " - BoringSSL ${boringssl_version}" -echo " - zlib ${zlib_version}" + +# make sure the products_dir is writeable +ls -lad $products_dir +touch $products_dir/products_dir_write_test.tmp +rm $products_dir/products_dir_write_test.tmp +#chown -R $(id -u):$(id -g) $products_dir function run() { echo "$@" @@ -239,11 +259,6 @@ function run() { } for arch in $archs; do - # enable short-circuiting the individual builds - if [[ ! -z "$SWIFT_ANDROID_ARCHIVEONLY" ]]; then - continue - fi - case $arch in armv7) target_host="arm-linux-androideabi"; compiler_target_host="armv7a-linux-androideabi$android_api"; android_abi="armeabi-v7a" ;; aarch64) target_host="aarch64-linux-android"; compiler_target_host="$target_host$android_api"; android_abi="arm64-v8a" ;; @@ -326,8 +341,9 @@ for arch in $archs; do -DOPENSSL_INCLUDE_DIR=$sdk_root/usr/include \ -DOPENSSL_SSL_LIBRARY=$sdk_root/usr/lib/libssl.a \ -DOPENSSL_CRYPTO_LIBRARY=$sdk_root/usr/lib/libcrypto.a \ - -DCURL_USE_OPENSSL=ON \ -DCURLSSLOPT_NATIVE_CA=ON \ + -DCURL_USE_OPENSSL=ON \ + -DCURL_USE_LIBSSH2=OFF \ -DTHREADS_PREFER_PTHREAD_FLAG=OFF \ -DCMAKE_THREAD_PREFER_PTHREAD=OFF \ -DCMAKE_THREADS_PREFER_PTHREAD_FLAG=OFF \ @@ -477,16 +493,17 @@ for arch in $archs; do rsync -a ${sdk_staging}/${arch}/usr ${swift_res_root} done +rm -r ${swift_res_root}/usr/share/{doc,man} +rm -r ${sdk_staging} + +# create an install script to set up the NDK links +#ANDROID_NDK_HOME="/opt/homebrew/share/android-ndk" +mkdir scripts/ + ndk_sysroot="ndk-sysroot" -# whether to include the ndk-sysroot in the SDK bundle -INCLUDE_NDK_SYSROOT=${INCLUDE_NDK_SYSROOT:-0} -if [[ ${INCLUDE_NDK_SYSROOT} != 1 ]]; then - # if we do not include the NDK, then create an install script - #ANDROID_NDK_HOME="/opt/homebrew/share/android-ndk" - mkdir scripts/ - cat > scripts/setup-android-sdk.sh <<'EOF' -#/bin/sh +cat > scripts/setup-android-sdk.sh <<'EOF' +#/bin/bash # this script will setup the ndk-sysroot with links to the # local installation indicated by ANDROID_NDK_HOME set -e @@ -494,72 +511,64 @@ if [ -z "${ANDROID_NDK_HOME}" ]; then echo "$(basename $0): error: missing environment variable ANDROID_NDK_HOME" exit 1 fi -PREBUILT="${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt" -if [ ! -d "${PREBUILT}" ]; then - echo "$(basename $0): error: ANDROID_NDK_HOME not found: ${PREBUILT}" + +ndk_prebuilt="${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt" +if [ ! -d "${ndk_prebuilt}" ]; then + echo "$(basename $0): error: ANDROID_NDK_HOME not found: ${ndk_prebuilt}" exit 1 fi -DESTINATION=$(dirname $(dirname $(realpath $0)))/ndk-sysroot -# clear out any previous NDK setup -rm -rf ${DESTINATION} -# copy vs. link the NDK files +#Pkg.Revision = 27.0.12077973 +#Pkg.Revision = 28.1.13356709 +ndk_version=$(grep '^Pkg.Revision = ' "${ANDROID_NDK_HOME}/source.properties" | cut -f3- -d' ' | cut -f 1 -d '.') +if [[ "${ndk_version}" -lt 27 ]]; then + echo "$(basename $0): error: minimum NDK version 27 required; found ${ndk_version} in ${ANDROID_NDK_HOME}/source.properties" + exit 1 +fi + +cd $(dirname $(dirname $(realpath -- "${BASH_SOURCE[0]}"))) +swift_resources=swift-resources +ndk_sysroot=ndk-sysroot + +if [[ -d "${ndk_sysroot}" ]]; then + # clear out any previous NDK setup + rm -rf ${ndk_sysroot} + ndk_re="re-" +fi + +# link vs. copy the NDK files SWIFT_ANDROID_NDK_LINK=${SWIFT_ANDROID_NDK_LINK:-1} -if [[ "${SWIFT_ANDROID_NDK_LINK}" != 1 ]]; then - ANDROID_NDK_DESC="copied" - cp -a ${PREBUILT}/*/sysroot ${DESTINATION} -else - ANDROID_NDK_DESC="linked" - mkdir -p ${DESTINATION}/usr/lib - ln -s $(realpath ${PREBUILT}/*/sysroot/usr/include) ${DESTINATION}/usr/include - for triplePath in ${PREBUILT}/*/sysroot/usr/lib/*; do +if [[ "${SWIFT_ANDROID_NDK_LINK}" == 1 ]]; then + ndk_action="${ndk_re}linked" + mkdir -p ${ndk_sysroot}/usr/lib + ln -s ${ndk_prebuilt}/*/sysroot/usr/include ${ndk_sysroot}/usr/include + for triplePath in ${ndk_prebuilt}/*/sysroot/usr/lib/*; do triple=$(basename ${triplePath}) - ln -s $(realpath ${triplePath}) ${DESTINATION}/usr/lib/${triple} + ln -s ${triplePath} ${ndk_sysroot}/usr/lib/${triple} done +else + ndk_action="${ndk_re}copied" + cp -a ${ndk_prebuilt}/*/sysroot ${ndk_sysroot} fi # copy each architecture's swiftrt.o into the sysroot, # working around https://github.com/swiftlang/swift/pull/79621 -for swiftrt in ${DESTINATION}/../swift-resources/usr/lib/swift-*/android/*/swiftrt.o; do - arch=$(basename $(dirname ${swiftrt})) - mkdir -p ${DESTINATION}/usr/lib/swift/android/${arch} - cp -a ${swiftrt} ${DESTINATION}/usr/lib/swift/android/${arch} +for folder in swift swift_static; do + for swiftrt in ${swift_resources}/usr/lib/${folder}-*/android/*/swiftrt.o; do + arch=$(basename $(dirname ${swiftrt})) + mkdir -p ${ndk_sysroot}/usr/lib/${folder}/android/${arch} + if [[ "${SWIFT_ANDROID_NDK_LINK}" == 1 ]]; then + ln -s ../../../../../../${swiftrt} ${ndk_sysroot}/usr/lib/${folder}/android/${arch}/ + else + cp -a ${swiftrt} ${ndk_sysroot}/usr/lib/${folder}/android/${arch}/ + fi + done done -echo "$(basename $0): success: ndk-sysroot ${ANDROID_NDK_DESC} to Android SDK" +echo "$(basename $0): success: ndk-sysroot ${ndk_action} to Android NDK at ${ndk_prebuilt}" EOF - chmod +x scripts/setup-android-sdk.sh -else - COPY_NDK_SYSROOT=${COPY_NDK_SYSROOT:-1} - if [[ ${COPY_NDK_SYSROOT} == 1 ]]; then - cp -a ${ndk_installation}/sysroot ${ndk_sysroot} - else - # rather than copying the sysroot, we can instead make links to - # the various sub-folders this won't work for the distribution, - # since the NDK is going to be located in different places - # for different machines - mkdir -p ${ndk_sysroot}/usr/lib - ln -sv $(realpath ${ndk_installation}/sysroot/usr/include) ${ndk_sysroot}/usr/include - for triplePath in ${ndk_installation}/sysroot/usr/lib/*; do - triple=$(basename ${triplePath}) - ln -sv $(realpath ${triplePath}) ${ndk_sysroot}/usr/lib/${triple} - ls ${ndk_sysroot}/usr/lib/${triple}/ - done - fi - # need to manually copy over swiftrt.o or else: - # error: link command failed with exit code 1 (use -v to see invocation) - # clang: error: no such file or directory: '${HOME}/.swiftpm/swift-sdks/swift-6.2-DEVELOPMENT-SNAPSHOT-2025-04-24-a-android-0.1.artifactbundle/swift-android/ndk-sysroot/usr/lib/swift/android/x86_64/swiftrt.o' - # see: https://github.com/swiftlang/swift-driver/pull/1822#issuecomment-2762811807 - # should be fixed by: https://github.com/swiftlang/swift/pull/79621 - for arch in $archs; do - mkdir -p ${ndk_sysroot}/usr/lib/swift/android/${arch} - ln -srv ${swift_res_root}/usr/lib/swift-${arch}/android/${arch}/swiftrt.o ${ndk_sysroot}/usr/lib/swift/android/${arch}/swiftrt.o - done -fi - -rm -r ${swift_res_root}/usr/share/{doc,man} -rm -r ${sdk_staging} +chmod +x scripts/setup-android-sdk.sh cat > swift-sdk.json </dev/null + +if [[ "${ANDROID_NDK_VERSION}" == "" ]]; then + echo "$0: Missing ANDROID_NDK_VERSION environment" + exit 1 +fi + + +NDKFILE=${ANDROID_NDK_VERSION}-linux.zip + +NDKURL="https://dl.google.com/android/repository/${NDKFILE}" +echo "Going to fetch ${NDKURL}" + +curl -fsSL "${NDKURL}" -o ${NDKFILE} + +echo "Extracting NDK" +unzip -q ${NDKFILE} + +rm ${NDKFILE} + +popd >/dev/null + diff --git a/swift-ci/sdks/android/scripts/install-swift.sh b/swift-ci/sdks/android/scripts/install-swift.sh new file mode 100755 index 00000000..0acd9e02 --- /dev/null +++ b/swift-ci/sdks/android/scripts/install-swift.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# +# ===----------------------------------------------------------------------=== +# +# Swift Android SDK: Install Swift +# +# ===----------------------------------------------------------------------=== + +set -e + +if [[ "${SWIFT_TOOLCHAIN_URL}" == "" ]]; then + echo "$0: Missing SWIFT_TOOLCHAIN_URL environment" + exit 1 +fi + +destination=$1 +if [[ "${destination}" == "" ]]; then + echo "$0: Usage: $(basename $0) " + exit 1 +fi + +echo "Installing Swift from: ${SWIFT_TOOLCHAIN_URL} into: ${destination}" + +# Make a temporary directory +tmpdir=$(mktemp -d) +function cleanup { + rm -rf "$tmpdir" +} +trap cleanup EXIT + +pushd "$tmpdir" >/dev/null +export GNUPGHOME="$tmpdir" + +# Fetch the toolchain and signature +echo "Going to fetch ${SWIFT_TOOLCHAIN_URL}" +curl -fsSL "${SWIFT_TOOLCHAIN_URL}" -o toolchain.tar.gz + +echo "Going to fetch ${SWIFT_TOOLCHAIN_URL}.sig" +curl -fsSL "${SWIFT_TOOLCHAIN_URL}.sig" -o toolchain.sig + +echo "Fetching keys" +curl -fsSL --compressed https://swift.org/keys/all-keys.asc | gpg --import - + +echo "Verifying signature" +gpg --batch --verify toolchain.sig toolchain.tar.gz + +# Extract and install the toolchain +echo "Extracting Swift" +mkdir -p ${destination} +tar -xzf toolchain.tar.gz --directory ${destination} --strip-components=2 +chmod -R o+r ${destination}/lib/swift + +popd >/dev/null + diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh new file mode 100755 index 00000000..ee425c6a --- /dev/null +++ b/swift-ci/sdks/android/scripts/patch-sources.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Swift Android SDK: Patch Sources +set -e + +source_dir=$1 + +if [[ ! -d "${source_dir}" ]]; then + echo "$0: source_dir ${source_dir} does not exist" + exit 1 +fi + +script_dir=$(dirname -- "${BASH_SOURCE[0]}") +resource_dir=$(realpath "${script_dir}/../resources") +patches_dir="${resource_dir}/patches" + +if [[ ! -d "${patches_dir}" ]]; then + echo "$0: patches_dir ${patches_dir} does not exist" + exit 1 +fi + +cd ${source_dir}/swift-project +swift_android_patch="${patches_dir}/swift-android.patch" + +ls -la swift/utils/build-script-impl +ls -la ${swift_android_patch} +ls -la swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift + +# patch the patch, which seems to only be needed for an API less than 28 +# https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 +perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $swift_android_patch + +# remove the need to link in android-execinfo +perl -pi -e 's;dispatch android-execinfo;dispatch;g' $swift_android_patch + +# debug symbolic link setup +perl -pi -e 's;call ln -sf;call ln -svf;g' $swift_android_patch +perl -pi -e 's%linux-x86_64/sysroot/usr/lib"%linux-x86_64/sysroot/usr/lib"; echo "VALIDATING SYMBOLIC LINK"; ls -la "\${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib"; ls -la "\${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/swift"; %g' $swift_android_patch + +case "${BUILD_VERSION}" in + release) + testing_patch="${patches_dir}/swift-android-testing-release.patch" + ;; + devel) + testing_patch="${patches_dir}/swift-android-testing-except-release.patch" + ;; + trunk) + testing_patch="${patches_dir}/swift-android-testing-except-release.patch" + dispatch_patch="${patches_dir}/swift-android-trunk-libdispatch.patch" + ;; + *) + echo "$0: invalid BUILD_VERSION=${BUILD_VERSION}" + exit 1 + ;; +esac + +for patch in "$swift_android_patch" "$testing_patch" "$dispatch_patch"; do + if [[ "${patch}" == "" ]]; then + continue + fi + + echo "applying patch $patch in $PWD…" + # first check to make sure the patches can apply and fail if not + git apply -v --check -C1 "$patch" + git apply --no-index -v -C1 "$patch" + + #if git apply -C1 --reverse --check "$patch" >/dev/null 2>&1 ; then + # echo "already patched" + #elif git apply -C1 "$patch" ; then + # echo "done" + #else + # echo "failed to apply patch $patch in $PWD" + # exit 1 + #fi +done + +perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift +# disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) +perl -pi -e 's;os\(Android\);os\(AndroidDISABLED\);g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift + +# need to un-apply libandroid-spawn since we don't need it for API28+ +perl -pi -e 's;MATCHES "Android";MATCHES "AndroidDISABLED";g' llbuild/lib/llvm/Support/CMakeLists.txt +perl -pi -e 's; STREQUAL Android\); STREQUAL AndroidDISABLED\);g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt + +# validate the patches +ls -la swift/utils/build-script-impl +grep 'VALIDATING SYMBOLIC LINK' swift/utils/build-script-impl + diff --git a/swift-ci/sdks/android/scripts/toolchain-vars.sh b/swift-ci/sdks/android/scripts/toolchain-vars.sh new file mode 100644 index 00000000..c1001931 --- /dev/null +++ b/swift-ci/sdks/android/scripts/toolchain-vars.sh @@ -0,0 +1,44 @@ +#!/bin/bash -e +# +# ===----------------------------------------------------------------------=== +# +# Swift Android SDK: Toolchain source variables +# +# ===----------------------------------------------------------------------=== + +# This script is meant to be sourced from another script that sets the +# BUILD_VERSION environment variable to one of "release", "devel", or "trunk" +# and will set check the latest builds for each build type in order +# to provide information about the Swift tag name in use and where to +# obtain the latest toolchain for building. + +# TODO: we could instead use the latest-build.yml files for this, like: +# https://download.swift.org/swift-6.2-branch/ubuntu2404/latest-build.yml +# https://download.swift.org/development/ubuntu2404/latest-build.yml +# but there doesn't seem to be one for the current release build. + +case "${BUILD_VERSION}" in + release) + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/swiftlang/swift/releases | grep -m1 swift-6.1 | cut -d- -f2) + SWIFT_TAG="swift-${LATEST_TOOLCHAIN_VERSION}-RELEASE" + SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" + ;; + devel) + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/swiftlang/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) + SWIFT_TAG="swift-6.2-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" + ;; + trunk) + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/swiftlang/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) + SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + SWIFT_BRANCH="development" + ;; + *) + echo "$0: invalid BUILD_VERSION=${BUILD_VERSION}" + exit 1 + ;; +esac + +SWIFT_BASE=$SWIFT_TAG-$HOST_OS +export SWIFT_TOOLCHAIN_URL="https://download.swift.org/$SWIFT_BRANCH/$(echo $HOST_OS | tr -d '.')/$SWIFT_TAG/$SWIFT_BASE.tar.gz" + From 4987bcc7e57590e81bd5594434310d379fb2510f Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sun, 18 May 2025 18:19:00 -0400 Subject: [PATCH 06/25] Checkout patches repo instead of using a git submodule --- .github/workflows/pull_request.yml | 2 -- .gitmodules | 3 --- swift-ci/sdks/android/resources/patches | 1 - swift-ci/sdks/android/scripts/fetch-source.sh | 12 +++++++++++- swift-ci/sdks/android/scripts/patch-sources.sh | 10 +--------- 5 files changed, 12 insertions(+), 16 deletions(-) delete mode 160000 swift-ci/sdks/android/resources/patches diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 13c57b39..d5610edb 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -67,8 +67,6 @@ jobs: echo "WORKDIR=${{ runner.temp }}/swift-android-sdk" >> $GITHUB_ENV - name: Checkout repository uses: actions/checkout@v4 - with: - submodules: 'true' - name: Build Android SDK (Local) if: ${{ matrix.build-type == 'local' }} working-directory: swift-ci/sdks/android diff --git a/.gitmodules b/.gitmodules index f3724176..e69de29b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "swift-ci/sdks/android/resources/patches"] - path = swift-ci/sdks/android/resources/patches - url = https://github.com/swift-android-sdk/swift-android-sdk diff --git a/swift-ci/sdks/android/resources/patches b/swift-ci/sdks/android/resources/patches deleted file mode 160000 index d0828093..00000000 --- a/swift-ci/sdks/android/resources/patches +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d08280938ff70b790bbd0f559c0b420b518877ca diff --git a/swift-ci/sdks/android/scripts/fetch-source.sh b/swift-ci/sdks/android/scripts/fetch-source.sh index 26c88d7a..3b7b7b36 100755 --- a/swift-ci/sdks/android/scripts/fetch-source.sh +++ b/swift-ci/sdks/android/scripts/fetch-source.sh @@ -91,6 +91,9 @@ fi if [[ -z "${BORINGSSL_VERSION}" ]]; then BORINGSSL_VERSION=fips-20220613 fi +if [[ -z "${SWIFT_ANDROID_PATCHES_VERSION}" ]]; then + SWIFT_ANDROID_PATCHES_VERSION=main +fi clone_with_ssh=false while [ "$#" -gt 0 ]; do @@ -139,7 +142,7 @@ mkdir -p swift-project groupstart "Fetching Swift" pushd swift-project >/dev/null -[[ -d swift ]] || git clone ${github}apple/swift.git +[[ -d swift ]] || git clone ${github}swiftlang/swift.git cd swift # Get its dependencies @@ -181,3 +184,10 @@ git checkout ${BORINGSSL_VERSION} popd >/dev/null 2>&1 groupend +# Fetch BoringSSL +groupstart "Fetching Patches" +[[ -d swift-android-patches ]] || git clone https://github.com/swift-android-sdk/swift-android-sdk swift-android-patches +pushd swift-android-patches >/dev/null 2>&1 +git checkout ${SWIFT_ANDROID_PATCHES_VERSION} +popd >/dev/null 2>&1 +groupend diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh index ee425c6a..1282c706 100755 --- a/swift-ci/sdks/android/scripts/patch-sources.sh +++ b/swift-ci/sdks/android/scripts/patch-sources.sh @@ -3,16 +3,12 @@ set -e source_dir=$1 - if [[ ! -d "${source_dir}" ]]; then echo "$0: source_dir ${source_dir} does not exist" exit 1 fi -script_dir=$(dirname -- "${BASH_SOURCE[0]}") -resource_dir=$(realpath "${script_dir}/../resources") -patches_dir="${resource_dir}/patches" - +patches_dir="${source_dir}/swift-android-patches" if [[ ! -d "${patches_dir}" ]]; then echo "$0: patches_dir ${patches_dir} does not exist" exit 1 @@ -21,10 +17,6 @@ fi cd ${source_dir}/swift-project swift_android_patch="${patches_dir}/swift-android.patch" -ls -la swift/utils/build-script-impl -ls -la ${swift_android_patch} -ls -la swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift - # patch the patch, which seems to only be needed for an API less than 28 # https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $swift_android_patch From ff3f274cca53d02f5b3f331ba64dde177fb27333 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sun, 18 May 2025 18:28:34 -0400 Subject: [PATCH 07/25] Update libcurl to 8.13.0 --- swift-ci/sdks/android/scripts/fetch-source.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swift-ci/sdks/android/scripts/fetch-source.sh b/swift-ci/sdks/android/scripts/fetch-source.sh index 3b7b7b36..93309130 100755 --- a/swift-ci/sdks/android/scripts/fetch-source.sh +++ b/swift-ci/sdks/android/scripts/fetch-source.sh @@ -85,8 +85,8 @@ if [[ -z "${LIBXML2_VERSION}" ]]; then #LIBXML2_VERSION=2.14.2 fi if [[ -z "${CURL_VERSION}" ]]; then - CURL_VERSION=8.7.1 - #CURL_VERSION=8.13.0 + #CURL_VERSION=8.7.1 + CURL_VERSION=8.13.0 fi if [[ -z "${BORINGSSL_VERSION}" ]]; then BORINGSSL_VERSION=fips-20220613 From 595efc35bdd9e2c96add429309be5397ca615b97 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sun, 18 May 2025 18:33:49 -0400 Subject: [PATCH 08/25] Remove resources that we no longer use --- swift-ci/sdks/android/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/swift-ci/sdks/android/Dockerfile b/swift-ci/sdks/android/Dockerfile index 2be3fc37..69522bb2 100644 --- a/swift-ci/sdks/android/Dockerfile +++ b/swift-ci/sdks/android/Dockerfile @@ -78,8 +78,6 @@ ENV SWIFT_BUILD_DOCKER="1" COPY scripts /scripts RUN chmod ugo+x /scripts/* -COPY resources /resources - # Create a user RUN groupadd -g 998 build-user && \ useradd -m -r -u 998 -g build-user build-user From 27b1bf444d1f07abfcf4b61aae58d7d259e35d36 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sun, 18 May 2025 18:53:47 -0400 Subject: [PATCH 09/25] Update libcurl to 8.13.0 --- swift-ci/sdks/android/scripts/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 20877272..75085411 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -344,6 +344,7 @@ for arch in $archs; do -DCURLSSLOPT_NATIVE_CA=ON \ -DCURL_USE_OPENSSL=ON \ -DCURL_USE_LIBSSH2=OFF \ + -DCURL_USE_LIBPSL=OFF \ -DTHREADS_PREFER_PTHREAD_FLAG=OFF \ -DCMAKE_THREAD_PREFER_PTHREAD=OFF \ -DCMAKE_THREADS_PREFER_PTHREAD_FLAG=OFF \ From b898129337799dd43c98859a9a540e790a0a23f1 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sun, 18 May 2025 18:54:26 -0400 Subject: [PATCH 10/25] Update libxml2 to 2.14.2 --- swift-ci/sdks/android/scripts/fetch-source.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/swift-ci/sdks/android/scripts/fetch-source.sh b/swift-ci/sdks/android/scripts/fetch-source.sh index 93309130..b42abd07 100755 --- a/swift-ci/sdks/android/scripts/fetch-source.sh +++ b/swift-ci/sdks/android/scripts/fetch-source.sh @@ -81,11 +81,9 @@ if [[ -z "${SWIFT_VERSION}" ]]; then SWIFT_VERSION=scheme:release/6.1 fi if [[ -z "${LIBXML2_VERSION}" ]]; then - LIBXML2_VERSION=2.12.7 - #LIBXML2_VERSION=2.14.2 + LIBXML2_VERSION=2.14.2 fi if [[ -z "${CURL_VERSION}" ]]; then - #CURL_VERSION=8.7.1 CURL_VERSION=8.13.0 fi if [[ -z "${BORINGSSL_VERSION}" ]]; then From f566b239b8e608d77ef5938a59259c3b639764d6 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sun, 18 May 2025 19:36:29 -0400 Subject: [PATCH 11/25] Build libxml2, libcurl, and boringssl with support for Android 16kb page sizes --- swift-ci/sdks/android/Dockerfile | 1 - swift-ci/sdks/android/scripts/build.sh | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/swift-ci/sdks/android/Dockerfile b/swift-ci/sdks/android/Dockerfile index 69522bb2..c1d8c6db 100644 --- a/swift-ci/sdks/android/Dockerfile +++ b/swift-ci/sdks/android/Dockerfile @@ -24,7 +24,6 @@ RUN apt-get -q update \ golang \ git \ gnupg2 \ - libsqlite3-dev \ libcurl4-openssl-dev \ libedit-dev \ libicu-dev \ diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 75085411..ad59c311 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -283,6 +283,7 @@ for arch in $archs; do -DCMAKE_EXTRA_LINK_FLAGS="-rtlib=compiler-rt -unwindlib=libunwind -stdlib=libc++ -fuse-ld=lld -lc++ -lc++abi" \ -DCMAKE_BUILD_TYPE=$build_type \ -DCMAKE_INSTALL_PREFIX=$sdk_root/usr \ + -DCMAKE_SHARED_LINKER_FLAGS="-Wl,-z,max-page-size=16384" \ -DLIBXML2_WITH_PYTHON=NO \ -DLIBXML2_WITH_ICU=NO \ -DLIBXML2_WITH_ICONV=NO \ @@ -311,6 +312,7 @@ for arch in $archs; do -DCMAKE_TOOLCHAIN_FILE=$ndk_home/build/cmake/android.toolchain.cmake \ -DCMAKE_BUILD_TYPE=$build_type \ -DCMAKE_INSTALL_PREFIX=$sdk_root/usr \ + -DCMAKE_SHARED_LINKER_FLAGS="-Wl,-z,max-page-size=16384" \ -DBUILD_SHARED_LIBS=OFF \ -DBUILD_STATIC_LIBS=ON \ -DBUILD_TESTING=OFF @@ -337,6 +339,7 @@ for arch in $archs; do -DCMAKE_TOOLCHAIN_FILE=$ndk_home/build/cmake/android.toolchain.cmake \ -DCMAKE_BUILD_TYPE=$build_type \ -DCMAKE_INSTALL_PREFIX=$sdk_root/usr \ + -DCMAKE_SHARED_LINKER_FLAGS="-Wl,-z,max-page-size=16384" \ -DOPENSSL_ROOT_DIR=$sdk_root/usr \ -DOPENSSL_INCLUDE_DIR=$sdk_root/usr/include \ -DOPENSSL_SSL_LIBRARY=$sdk_root/usr/lib/libssl.a \ From 529e3f19d1f927b2671c4cc0717ec342f7525d7b Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sun, 18 May 2025 19:50:32 -0400 Subject: [PATCH 12/25] Add build-script --extra-cmake-options=-DCMAKE_EXTRA_LINK_FLAGS=-Wl,-z,max-page-size=16384 --- swift-ci/sdks/android/scripts/build.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index ad59c311..1d9504e0 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -280,10 +280,9 @@ for arch in $archs; do -DANDROID_PLATFORM=android-$android_api \ -DCMAKE_TOOLCHAIN_FILE=$ndk_home/build/cmake/android.toolchain.cmake \ -DCMAKE_BUILD_TYPE=$build_type \ - -DCMAKE_EXTRA_LINK_FLAGS="-rtlib=compiler-rt -unwindlib=libunwind -stdlib=libc++ -fuse-ld=lld -lc++ -lc++abi" \ + -DCMAKE_EXTRA_LINK_FLAGS="-rtlib=compiler-rt -unwindlib=libunwind -stdlib=libc++ -fuse-ld=lld -lc++ -lc++abi -Wl,-z,max-page-size=16384" \ -DCMAKE_BUILD_TYPE=$build_type \ -DCMAKE_INSTALL_PREFIX=$sdk_root/usr \ - -DCMAKE_SHARED_LINKER_FLAGS="-Wl,-z,max-page-size=16384" \ -DLIBXML2_WITH_PYTHON=NO \ -DLIBXML2_WITH_ICU=NO \ -DLIBXML2_WITH_ICONV=NO \ @@ -312,7 +311,7 @@ for arch in $archs; do -DCMAKE_TOOLCHAIN_FILE=$ndk_home/build/cmake/android.toolchain.cmake \ -DCMAKE_BUILD_TYPE=$build_type \ -DCMAKE_INSTALL_PREFIX=$sdk_root/usr \ - -DCMAKE_SHARED_LINKER_FLAGS="-Wl,-z,max-page-size=16384" \ + -DCMAKE_EXTRA_LINK_FLAGS="-Wl,-z,max-page-size=16384" \ -DBUILD_SHARED_LIBS=OFF \ -DBUILD_STATIC_LIBS=ON \ -DBUILD_TESTING=OFF @@ -339,7 +338,7 @@ for arch in $archs; do -DCMAKE_TOOLCHAIN_FILE=$ndk_home/build/cmake/android.toolchain.cmake \ -DCMAKE_BUILD_TYPE=$build_type \ -DCMAKE_INSTALL_PREFIX=$sdk_root/usr \ - -DCMAKE_SHARED_LINKER_FLAGS="-Wl,-z,max-page-size=16384" \ + -DCMAKE_EXTRA_LINK_FLAGS="-Wl,-z,max-page-size=16384" \ -DOPENSSL_ROOT_DIR=$sdk_root/usr \ -DOPENSSL_INCLUDE_DIR=$sdk_root/usr/include \ -DOPENSSL_SSL_LIBRARY=$sdk_root/usr/lib/libssl.a \ @@ -403,7 +402,8 @@ for arch in $archs; do --install-foundation \ --xctest --install-xctest \ --swift-testing --install-swift-testing \ - --cross-compile-append-host-target-to-destdir=False + --cross-compile-append-host-target-to-destdir=False \ + --extra-cmake-options='-DCMAKE_EXTRA_LINK_FLAGS="-Wl,-z,max-page-size=16384"' # need to remove symlink that gets created in the NDK to the previous arch's build # or else we get errors like: From c82587dbb084b774ade6564ecb0f233e70ae9981 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sun, 18 May 2025 19:56:15 -0400 Subject: [PATCH 13/25] Add 16KB page size linker flags to linker flags in swift-toolset.json --- swift-ci/sdks/android/scripts/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 1d9504e0..02931862 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -620,6 +620,7 @@ cat > swift-toolset.json < Date: Sun, 18 May 2025 22:01:29 -0400 Subject: [PATCH 14/25] Add 16KB page size linker flags to linker flags in swift-toolset.json --- swift-ci/sdks/android/scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 02931862..b82c36ad 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -620,7 +620,7 @@ cat > swift-toolset.json < Date: Mon, 19 May 2025 08:32:20 -0400 Subject: [PATCH 15/25] Build with ndk-r28b --- swift-ci/sdks/android/build-docker | 2 +- swift-ci/sdks/android/build-local | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index fb3b377a..4dfa3c96 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -9,7 +9,7 @@ # default architectures to build for TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} -ANDROID_NDK_VERSION=android-ndk-r27c +ANDROID_NDK_VERSION=android-ndk-r28b ANDROID_API=28 export BUILD_VERSION=${1} diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 30bd4f92..97b7aa11 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -9,7 +9,7 @@ # default architectures to build for TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} -ANDROID_NDK_VERSION=android-ndk-r27c +ANDROID_NDK_VERSION=android-ndk-r28b ANDROID_API=28 export BUILD_VERSION=${1} From 8b18b5fafa51e9aa6fc4e2204c86193959dac8a6 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Mon, 19 May 2025 09:05:28 -0400 Subject: [PATCH 16/25] Revert to building with ndk-r27c --- swift-ci/sdks/android/build-docker | 2 +- swift-ci/sdks/android/build-local | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 4dfa3c96..fb3b377a 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -9,7 +9,7 @@ # default architectures to build for TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} -ANDROID_NDK_VERSION=android-ndk-r28b +ANDROID_NDK_VERSION=android-ndk-r27c ANDROID_API=28 export BUILD_VERSION=${1} diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 97b7aa11..30bd4f92 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -9,7 +9,7 @@ # default architectures to build for TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} -ANDROID_NDK_VERSION=android-ndk-r28b +ANDROID_NDK_VERSION=android-ndk-r27c ANDROID_API=28 export BUILD_VERSION=${1} From f67f9bf939247eba99dbf0a7b93a64a5a922f3aa Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Mon, 19 May 2025 14:59:18 -0400 Subject: [PATCH 17/25] Use official endpoints for discovering latest Swift release/devel/trunk tags --- .../sdks/android/scripts/toolchain-vars.sh | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/swift-ci/sdks/android/scripts/toolchain-vars.sh b/swift-ci/sdks/android/scripts/toolchain-vars.sh index c1001931..162689a9 100644 --- a/swift-ci/sdks/android/scripts/toolchain-vars.sh +++ b/swift-ci/sdks/android/scripts/toolchain-vars.sh @@ -12,25 +12,23 @@ # to provide information about the Swift tag name in use and where to # obtain the latest toolchain for building. -# TODO: we could instead use the latest-build.yml files for this, like: -# https://download.swift.org/swift-6.2-branch/ubuntu2404/latest-build.yml -# https://download.swift.org/development/ubuntu2404/latest-build.yml -# but there doesn't seem to be one for the current release build. +OS=$(echo $HOST_OS | tr -d '.') case "${BUILD_VERSION}" in release) - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/swiftlang/swift/releases | grep -m1 swift-6.1 | cut -d- -f2) - SWIFT_TAG="swift-${LATEST_TOOLCHAIN_VERSION}-RELEASE" - SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" + # e.g., "swift-6.1-RELEASE" + SWIFT_TAG=$(curl -fsSL https://www.swift.org/api/v1/install/releases.json | jq -r '.[-1].tag') + # e.g., "swift-6.1-release" + SWIFT_BRANCH=$(echo "${SWIFT_TAG}" | tr '[A-Z]' '[a-z]') ;; devel) - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/swiftlang/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) - SWIFT_TAG="swift-6.2-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + # e.g., swift-6.2-DEVELOPMENT-SNAPSHOT-2025-05-15-a + SWIFT_TAG=$(curl -fsSL https://download.swift.org/swift-6.2-branch/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ' SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" ;; trunk) - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/swiftlang/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) - SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + # e.g., swift-DEVELOPMENT-SNAPSHOT-2025-05-14-a + SWIFT_TAG=$(curl -fsSL https://download.swift.org/development/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ') SWIFT_BRANCH="development" ;; *) @@ -40,5 +38,5 @@ case "${BUILD_VERSION}" in esac SWIFT_BASE=$SWIFT_TAG-$HOST_OS -export SWIFT_TOOLCHAIN_URL="https://download.swift.org/$SWIFT_BRANCH/$(echo $HOST_OS | tr -d '.')/$SWIFT_TAG/$SWIFT_BASE.tar.gz" +export SWIFT_TOOLCHAIN_URL="https://download.swift.org/$SWIFT_BRANCH/$OS/$SWIFT_TAG/$SWIFT_BASE.tar.gz" From 7011a45c5dee830a184226c71c22ccde402a0baf Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Mon, 19 May 2025 15:21:38 -0400 Subject: [PATCH 18/25] Typo fix in version script --- swift-ci/sdks/android/scripts/toolchain-vars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift-ci/sdks/android/scripts/toolchain-vars.sh b/swift-ci/sdks/android/scripts/toolchain-vars.sh index 162689a9..5f8c5e8a 100644 --- a/swift-ci/sdks/android/scripts/toolchain-vars.sh +++ b/swift-ci/sdks/android/scripts/toolchain-vars.sh @@ -23,7 +23,7 @@ case "${BUILD_VERSION}" in ;; devel) # e.g., swift-6.2-DEVELOPMENT-SNAPSHOT-2025-05-15-a - SWIFT_TAG=$(curl -fsSL https://download.swift.org/swift-6.2-branch/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ' + SWIFT_TAG=$(curl -fsSL https://download.swift.org/swift-6.2-branch/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ') SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" ;; trunk) From 4c65a9385415d748f775bdd44751b585e9fb53cc Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Mon, 19 May 2025 18:08:57 -0400 Subject: [PATCH 19/25] Cleanup for PR --- .github/workflows/pull_request.yml | 10 ---------- .gitignore | 1 - .gitmodules | 0 swift-ci/sdks/android/README.md | 8 +++++--- 4 files changed, 5 insertions(+), 14 deletions(-) delete mode 100644 .gitmodules diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index d5610edb..95503da5 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -22,16 +22,6 @@ jobs: path: | *.log - static-linux-build: - name: Build Static Linux image - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Build Docker images - working-directory: swift-ci/sdks/static-linux - run: ./build - android-build: name: Build Android ${{ matrix.build-type }} ${{ matrix.swift-version }} ${{ matrix.arch }} SDK strategy: diff --git a/.gitignore b/.gitignore index 98bcc3db..a01ee289 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1 @@ .*.swp -static-swift-linux-sources diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e69de29b..00000000 diff --git a/swift-ci/sdks/android/README.md b/swift-ci/sdks/android/README.md index 10d4db3e..c3c91010 100644 --- a/swift-ci/sdks/android/README.md +++ b/swift-ci/sdks/android/README.md @@ -39,18 +39,20 @@ fetch tagged sources for libxml2, curl, boringssl, and swift. It then applies some patches and invokes `scripts/build.sh`, which will build the sources for each of the specified -architectures. Finally, it combines the NDK and the newly built -SDKs into a single artifactbundle. +architectures and then combines the SDKs into a single +artifactbundle with targetTriples for each of the supported +architectures (`aarch64`, `x86_64`, `aarmv7`) +and Android API levels (28-35). ## Specifying Architectures By default all the supported Android architectures -(`aarch64`, `x86_64`, `aarmv7`) will be built, but this can be reduced in order to speed up the build. This can be useful, e.g., as part of a CI that validates a pull request, as building a single architecture takes around 30 minutes on a standard ubuntu-24.04 GitHub runner, whereas building for all the architectures takes over an hour. +Building within a docker container increases this by about 50%. To build an artifactbundle for just the `x86_64` architecture, run: From d80c80b9a2d8ff54ecd6d3dfe10286dbbd595730 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Mon, 19 May 2025 18:17:25 -0400 Subject: [PATCH 20/25] Change BUILD_VERSION to BUILD_SCHEME and have it match release, swift-*-branch, or development --- .github/workflows/pull_request.yml | 8 ++++---- swift-ci/sdks/android/build-docker | 2 +- swift-ci/sdks/android/build-local | 2 +- swift-ci/sdks/android/scripts/patch-sources.sh | 8 ++++---- swift-ci/sdks/android/scripts/toolchain-vars.sh | 14 +++++++------- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 95503da5..1a4b5833 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -35,7 +35,7 @@ jobs: #arch: ['x86_64'] # build both the quick (x86_64) and complete (aarch64,x86_64,armv7) SDKs #arch: ['x86_64', ''] - swift-version: ['release', 'devel', 'trunk'] + swift-version: ['release', 'swift-6.2-branch', 'development'] runs-on: ubuntu-24.04 steps: - name: Free Disk Space @@ -52,7 +52,7 @@ jobs: run: | # these variabes are used by build-docker and build-local # to determine which Swift version to build for - echo "BUILD_VERSION=${{ matrix.swift-version }}" >> $GITHUB_ENV + echo "BUILD_SCHEME=${{ matrix.swift-version }}" >> $GITHUB_ENV echo "TARGET_ARCHS=${{ matrix.arch }}" >> $GITHUB_ENV echo "WORKDIR=${{ runner.temp }}/swift-android-sdk" >> $GITHUB_ENV - name: Checkout repository @@ -62,12 +62,12 @@ jobs: working-directory: swift-ci/sdks/android run: | sudo apt install -q ninja-build patchelf - ./build-local ${BUILD_VERSION} ${WORKDIR} + ./build-local ${BUILD_SCHEME} ${WORKDIR} - name: Build Android SDK (Docker) if: ${{ matrix.build-type == 'docker' }} working-directory: swift-ci/sdks/android run: | - ./build-docker ${BUILD_VERSION} ${WORKDIR} + ./build-docker ${BUILD_SCHEME} ${WORKDIR} - name: Install Host Toolchain if: ${{ matrix.build-type == 'docker' }} working-directory: swift-ci/sdks/android diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index fb3b377a..ef283152 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -12,7 +12,7 @@ TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} ANDROID_NDK_VERSION=android-ndk-r27c ANDROID_API=28 -export BUILD_VERSION=${1} +export BUILD_SCHEME=${1} # note that WORKDIR must not be under the current checkout or the patches will fail to apply WORKDIR=${2} if [[ "${WORKDIR}" == '' ]]; then diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 30bd4f92..3a5e448d 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -12,7 +12,7 @@ TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} ANDROID_NDK_VERSION=android-ndk-r27c ANDROID_API=28 -export BUILD_VERSION=${1} +export BUILD_SCHEME=${1} # note that WORKDIR must not be under the current checkout or the patches will fail to apply WORKDIR=${2} if [[ "${WORKDIR}" == '' ]]; then diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh index 1282c706..efced711 100755 --- a/swift-ci/sdks/android/scripts/patch-sources.sh +++ b/swift-ci/sdks/android/scripts/patch-sources.sh @@ -28,19 +28,19 @@ perl -pi -e 's;dispatch android-execinfo;dispatch;g' $swift_android_patch perl -pi -e 's;call ln -sf;call ln -svf;g' $swift_android_patch perl -pi -e 's%linux-x86_64/sysroot/usr/lib"%linux-x86_64/sysroot/usr/lib"; echo "VALIDATING SYMBOLIC LINK"; ls -la "\${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib"; ls -la "\${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/swift"; %g' $swift_android_patch -case "${BUILD_VERSION}" in +case "${BUILD_SCHEME}" in release) testing_patch="${patches_dir}/swift-android-testing-release.patch" ;; - devel) + swift-*-branch) testing_patch="${patches_dir}/swift-android-testing-except-release.patch" ;; - trunk) + development) testing_patch="${patches_dir}/swift-android-testing-except-release.patch" dispatch_patch="${patches_dir}/swift-android-trunk-libdispatch.patch" ;; *) - echo "$0: invalid BUILD_VERSION=${BUILD_VERSION}" + echo "$0: invalid BUILD_SCHEME=${BUILD_SCHEME}" exit 1 ;; esac diff --git a/swift-ci/sdks/android/scripts/toolchain-vars.sh b/swift-ci/sdks/android/scripts/toolchain-vars.sh index 5f8c5e8a..010f7dd2 100644 --- a/swift-ci/sdks/android/scripts/toolchain-vars.sh +++ b/swift-ci/sdks/android/scripts/toolchain-vars.sh @@ -7,32 +7,32 @@ # ===----------------------------------------------------------------------=== # This script is meant to be sourced from another script that sets the -# BUILD_VERSION environment variable to one of "release", "devel", or "trunk" +# BUILD_SCHEME environment variable to one of "release", "swift-VERSION-branch", or "development" # and will set check the latest builds for each build type in order # to provide information about the Swift tag name in use and where to # obtain the latest toolchain for building. OS=$(echo $HOST_OS | tr -d '.') -case "${BUILD_VERSION}" in +case "${BUILD_SCHEME}" in release) # e.g., "swift-6.1-RELEASE" SWIFT_TAG=$(curl -fsSL https://www.swift.org/api/v1/install/releases.json | jq -r '.[-1].tag') # e.g., "swift-6.1-release" SWIFT_BRANCH=$(echo "${SWIFT_TAG}" | tr '[A-Z]' '[a-z]') ;; - devel) + swift-*-branch) # e.g., swift-6.2-DEVELOPMENT-SNAPSHOT-2025-05-15-a - SWIFT_TAG=$(curl -fsSL https://download.swift.org/swift-6.2-branch/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ') - SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" + SWIFT_TAG=$(curl -fsSL https://download.swift.org/$BUILD_SCHEME/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ') + SWIFT_BRANCH=$BUILD_SCHEME ;; - trunk) + development) # e.g., swift-DEVELOPMENT-SNAPSHOT-2025-05-14-a SWIFT_TAG=$(curl -fsSL https://download.swift.org/development/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ') SWIFT_BRANCH="development" ;; *) - echo "$0: invalid BUILD_VERSION=${BUILD_VERSION}" + echo "$0: invalid BUILD_SCHEME=${BUILD_SCHEME}" exit 1 ;; esac From 2c07eef666993487e9eb70ccc9f28095abfc385b Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Mon, 19 May 2025 18:36:07 -0400 Subject: [PATCH 21/25] Update Android README --- swift-ci/sdks/android/README.md | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/swift-ci/sdks/android/README.md b/swift-ci/sdks/android/README.md index c3c91010..1e57edd1 100644 --- a/swift-ci/sdks/android/README.md +++ b/swift-ci/sdks/android/README.md @@ -25,11 +25,15 @@ Android NDK that will be used for cross-compilation. The `version` argument can be one of the following values: -| version | Swift version | +| version | Swift version example | | --- | --- | | `release` | swift-6.1-RELEASE | -| `devel` | swift-6.2-DEVELOPMENT-SNAPSHOT-yyyy-mm-dd | -| `trunk` | swift-DEVELOPMENT-SNAPSHOT-yyyy-mm-dd | +| `swift-6.2-branch` | swift-6.2-DEVELOPMENT-SNAPSHOT-yyyy-mm-dd | +| `development` | swift-DEVELOPMENT-SNAPSHOT-yyyy-mm-dd | + +> [!WARNING] +> The workdir argument must not be located in a git repository (e.g., it cannot be the +> current directory) ## Running @@ -52,12 +56,11 @@ up the build. This can be useful, e.g., as part of a CI that validates a pull request, as building a single architecture takes around 30 minutes on a standard ubuntu-24.04 GitHub runner, whereas building for all the architectures takes over an hour. -Building within a docker container increases this by about 50%. To build an artifactbundle for just the `x86_64` architecture, run: ``` -TARGET_ARCHS=aarch64 ./build-docker release /tmp/android-sdk +TARGET_ARCHS=x86_64 ./build-docker release /tmp/android-sdk ``` ## Installing and validating the SDK @@ -66,7 +69,17 @@ The `.github/workflows/pull_request.yml` workflow will create and upload an installable SDK named something like: `swift-6.1-RELEASE_android-0.1.artifactbundle.tar.gz` -The workflow will also install the SDK locally and use +The GitHub workflow will also install the SDK locally and use [swift-android-action](https://github.com/marketplace/actions/swift-android-action) to build and test various Swift packages in an Android emulator using the freshly-created SDK bundle. + +## Building locally + +Instead of building within a Docker container, the script can also +perform the build locally on an Ubuntu 24.04 machine with all the +build prerequisites already installed. This will generate +the same artifacts in approximately half the time, and +may be suitable to an already containerized envrionment (such as +a GitHub runner). + From 51d93f81fe5156b190616d0ed9e18a6a8d7cd3ab Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Wed, 21 May 2025 16:53:57 -0400 Subject: [PATCH 22/25] Update how patches are applied --- swift-ci/sdks/android/build-docker | 2 +- swift-ci/sdks/android/build-local | 2 +- .../sdks/android/scripts/patch-sources.sh | 79 ------------------- 3 files changed, 2 insertions(+), 81 deletions(-) delete mode 100755 swift-ci/sdks/android/scripts/patch-sources.sh diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index ef283152..47d067dd 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -27,7 +27,7 @@ source ./scripts/toolchain-vars.sh # Check-out and patch the sources ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} -./scripts/patch-sources.sh ${WORKDIR}/source +${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source mkdir -p ${WORKDIR}/products chmod ugo+rwx ${WORKDIR}/products diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 3a5e448d..e44b896a 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -50,7 +50,7 @@ fi # Check-out and patch the sources ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} -./scripts/patch-sources.sh ${WORKDIR}/source +${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source mkdir -p ${WORKDIR}/products diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh deleted file mode 100755 index efced711..00000000 --- a/swift-ci/sdks/android/scripts/patch-sources.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -# Swift Android SDK: Patch Sources -set -e - -source_dir=$1 -if [[ ! -d "${source_dir}" ]]; then - echo "$0: source_dir ${source_dir} does not exist" - exit 1 -fi - -patches_dir="${source_dir}/swift-android-patches" -if [[ ! -d "${patches_dir}" ]]; then - echo "$0: patches_dir ${patches_dir} does not exist" - exit 1 -fi - -cd ${source_dir}/swift-project -swift_android_patch="${patches_dir}/swift-android.patch" - -# patch the patch, which seems to only be needed for an API less than 28 -# https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 -perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $swift_android_patch - -# remove the need to link in android-execinfo -perl -pi -e 's;dispatch android-execinfo;dispatch;g' $swift_android_patch - -# debug symbolic link setup -perl -pi -e 's;call ln -sf;call ln -svf;g' $swift_android_patch -perl -pi -e 's%linux-x86_64/sysroot/usr/lib"%linux-x86_64/sysroot/usr/lib"; echo "VALIDATING SYMBOLIC LINK"; ls -la "\${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib"; ls -la "\${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/swift"; %g' $swift_android_patch - -case "${BUILD_SCHEME}" in - release) - testing_patch="${patches_dir}/swift-android-testing-release.patch" - ;; - swift-*-branch) - testing_patch="${patches_dir}/swift-android-testing-except-release.patch" - ;; - development) - testing_patch="${patches_dir}/swift-android-testing-except-release.patch" - dispatch_patch="${patches_dir}/swift-android-trunk-libdispatch.patch" - ;; - *) - echo "$0: invalid BUILD_SCHEME=${BUILD_SCHEME}" - exit 1 - ;; -esac - -for patch in "$swift_android_patch" "$testing_patch" "$dispatch_patch"; do - if [[ "${patch}" == "" ]]; then - continue - fi - - echo "applying patch $patch in $PWD…" - # first check to make sure the patches can apply and fail if not - git apply -v --check -C1 "$patch" - git apply --no-index -v -C1 "$patch" - - #if git apply -C1 --reverse --check "$patch" >/dev/null 2>&1 ; then - # echo "already patched" - #elif git apply -C1 "$patch" ; then - # echo "done" - #else - # echo "failed to apply patch $patch in $PWD" - # exit 1 - #fi -done - -perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift -# disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) -perl -pi -e 's;os\(Android\);os\(AndroidDISABLED\);g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift - -# need to un-apply libandroid-spawn since we don't need it for API28+ -perl -pi -e 's;MATCHES "Android";MATCHES "AndroidDISABLED";g' llbuild/lib/llvm/Support/CMakeLists.txt -perl -pi -e 's; STREQUAL Android\); STREQUAL AndroidDISABLED\);g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt - -# validate the patches -ls -la swift/utils/build-script-impl -grep 'VALIDATING SYMBOLIC LINK' swift/utils/build-script-impl - From 46a06df6d05f843b560fce8cc2cde1d742d79b2e Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Wed, 21 May 2025 17:40:18 -0400 Subject: [PATCH 23/25] Fix source directory for patch target --- swift-ci/sdks/android/build-docker | 2 +- swift-ci/sdks/android/build-local | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 47d067dd..17a18820 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -27,7 +27,7 @@ source ./scripts/toolchain-vars.sh # Check-out and patch the sources ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} -${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source +${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source/swift-project mkdir -p ${WORKDIR}/products chmod ugo+rwx ${WORKDIR}/products diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index e44b896a..8311ecb1 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -50,7 +50,7 @@ fi # Check-out and patch the sources ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} -${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source +${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source/swift-project mkdir -p ${WORKDIR}/products From afb291884f4180054d0d2910f5973d7e9ec6035e Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Wed, 21 May 2025 18:50:10 -0400 Subject: [PATCH 24/25] Harmonize timestamps in artifactbundle with the swift source tag date for build reproducibility --- .github/workflows/pull_request.yml | 8 ++++---- swift-ci/sdks/android/scripts/build.sh | 11 +++++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 1a4b5833..88a19bdb 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -27,14 +27,14 @@ jobs: strategy: fail-fast: false matrix: - build-type: ['docker'] - #build-type: ['docker', 'local'] + #build-type: ['docker'] + build-type: ['docker', 'local'] # blank arch builds all (aarch64,x86_64,armv7) - arch: [''] + #arch: [''] # builds only x86_64 to speed up the validation #arch: ['x86_64'] # build both the quick (x86_64) and complete (aarch64,x86_64,armv7) SDKs - #arch: ['x86_64', ''] + arch: ['x86_64', ''] swift-version: ['release', 'swift-6.2-branch', 'development'] runs-on: ubuntu-24.04 steps: diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index b82c36ad..6e7f1714 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -186,6 +186,8 @@ function versionFromTag { } swift_version=$(describe ${source_dir}/swift-project/swift) +swift_tag_date=$(git -C ${source_dir}/swift-project/swift log -1 --format=%ct 2>/dev/null) + if [[ $swift_version == swift-* ]]; then swift_version=${swift_version#swift-} fi @@ -631,8 +633,13 @@ header "Outputting compressed bundle" quiet_pushd "${build_dir}" mkdir -p "${products_dir}" - tar czf "${bundle}.tar.gz" "${bundle}" - mv "${bundle}.tar.gz" "${products_dir}" + # set the timestamps of every file in the artifact to the tag date for the swift repo for build reproducibility + touch_date=$(date -d "@$swift_tag_date" "+%Y%m%d%H%M.%S") + find "${bundle}" -exec touch -t "$touch_date" {} + + + bundle_archive="${products_dir}/${bundle}.tar.gz" + tar czf "${bundle_archive}" "${bundle}" + shasum -a 256 "${bundle_archive}" quiet_popd groupend From a469e852b25b029b9efe0cd909c62de1a752f05e Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Wed, 21 May 2025 21:21:26 -0400 Subject: [PATCH 25/25] Simplify toolchain-vars.sh --- swift-ci/sdks/android/scripts/toolchain-vars.sh | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/swift-ci/sdks/android/scripts/toolchain-vars.sh b/swift-ci/sdks/android/scripts/toolchain-vars.sh index 010f7dd2..39868951 100644 --- a/swift-ci/sdks/android/scripts/toolchain-vars.sh +++ b/swift-ci/sdks/android/scripts/toolchain-vars.sh @@ -17,19 +17,16 @@ OS=$(echo $HOST_OS | tr -d '.') case "${BUILD_SCHEME}" in release) # e.g., "swift-6.1-RELEASE" - SWIFT_TAG=$(curl -fsSL https://www.swift.org/api/v1/install/releases.json | jq -r '.[-1].tag') + # there is no latest-build.yml for releases, so we need to get it from the API + export SWIFT_TAG=$(curl -fsSL https://www.swift.org/api/v1/install/releases.json | jq -r '.[-1].tag') # e.g., "swift-6.1-release" - SWIFT_BRANCH=$(echo "${SWIFT_TAG}" | tr '[A-Z]' '[a-z]') + export SWIFT_BRANCH=$(echo "${SWIFT_TAG}" | tr '[A-Z]' '[a-z]') ;; - swift-*-branch) + development|swift-*-branch) # e.g., swift-6.2-DEVELOPMENT-SNAPSHOT-2025-05-15-a - SWIFT_TAG=$(curl -fsSL https://download.swift.org/$BUILD_SCHEME/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ') - SWIFT_BRANCH=$BUILD_SCHEME - ;; - development) # e.g., swift-DEVELOPMENT-SNAPSHOT-2025-05-14-a - SWIFT_TAG=$(curl -fsSL https://download.swift.org/development/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ') - SWIFT_BRANCH="development" + export SWIFT_TAG=$(curl -fsSL https://download.swift.org/$BUILD_SCHEME/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ') + export SWIFT_BRANCH=$BUILD_SCHEME ;; *) echo "$0: invalid BUILD_SCHEME=${BUILD_SCHEME}"