From e256508c442ac21e6260c6f0b0514918c1521a59 Mon Sep 17 00:00:00 2001 From: Nils Briggen Date: Sun, 14 Jun 2026 23:09:34 +0200 Subject: [PATCH] APK build: persistent ccache + Gradle cache + export-only target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Android CI job recompiled all native C++ (whisper.cpp, reanimated, worklets, gesture-handler) from scratch every push — ~41 min on the shared runner — because nothing survived the `COPY . .` layer invalidation, and `--load` exported the whole ~15GB image just to copy one APK out. Dockerfile (docker/android.Dockerfile): - Install ccache and patch the NDK CMake toolchain file so every module's externalNativeBuild routes C/C++ compiles + links through ccache (the third-party modules don't honor ccache on their own). ccache dir is a persistent BuildKit cache mount, so objects survive across builds. - Mount GRADLE_USER_HOME as a cache and pass --build-cache --parallel, so Kotlin/Java/resource/dex tasks and resolved deps persist too. - Cache-mount the bun install dir; move the nodejs install into the cached toolchain layer (was reinstalling on every source change). - New `apk` stage (FROM scratch) holding just the signed APK, for --output type=local extraction without loading the 15GB image. CI (.gitea/workflows/ci.yml): - build-apk now builds `--target apk --output type=local,dest=./out` and streams ./out/app-release.apk to /srv/wisp/wisp.apk — no --load, no docker create/cp, and the build image no longer piles up on the host. Validated locally (arm64, real sign+assemble path): cold gradle 5m5s -> warm gradle 57s; 61/61 C++ compiles served from ccache (100% hit); 226/578 Gradle tasks from cache; export-only build 60s wall, APK valid + v2/v3 signed. The first server build after this is still cold (~41 min) to populate the caches; pushes after that are much faster. Co-Authored-By: Claude Opus 4.8 --- .gitea/workflows/ci.yml | 29 ++++++++------- docker/android.Dockerfile | 76 ++++++++++++++++++++++++++++++++++----- 2 files changed, 84 insertions(+), 21 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 9aa87a3..39e81fd 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -68,9 +68,11 @@ jobs: # ---- build-apk: signed release APK, served at /wisp.apk ------------------ # Socket-safe: the whole Android build runs inside `docker buildx build` # (context streamed to the host BuildKit — no host bind-mounts), signed via - # BuildKit --secret from the ANDROID_* repo secrets. We then extract the APK - # with docker create/cp and copy it into the running `wisp` web container so - # it is downloadable at https://wisp.briggen.dev/wisp.apk. + # BuildKit --secret from the ANDROID_* repo secrets. The `apk` target + + # `--output type=local` write just the APK to ./out (no image load), which we + # stream into /srv/wisp/wisp.apk so it's downloadable at + # https://wisp.briggen.dev/wisp.apk. Speed comes from persistent BuildKit + # cache mounts (ccache for C++, Gradle build cache) declared in the Dockerfile. build-apk: # Auto-build the APK on every push to master (arm64-only keeps it ~minutes). # Runs after deploy-web so the web updates promptly, then the APK follows and @@ -89,23 +91,26 @@ jobs: printf '%s' "${{ secrets.ANDROID_KEYSTORE_PASSWORD }}" > "$RUNNER_TEMP/sec/ks_pass" printf '%s' "${{ secrets.ANDROID_KEY_ALIAS }}" > "$RUNNER_TEMP/sec/ks_alias" printf '%s' "${{ secrets.ANDROID_KEY_PASSWORD }}" > "$RUNNER_TEMP/sec/ks_keypass" - - name: Build signed APK image + - name: Build + extract the signed APK (export-only target, no image load) run: | + # `--target apk --output type=local` writes ONLY app-release.apk to + # ./out — no `--load` of the ~15GB build image (saves the multi-minute + # export/unpack each run and stops that image filling the host disk). + # The persistent ccache + Gradle build cache (BuildKit cache mounts in + # the Dockerfile) make C++ recompiles near-instant on warm runs. docker buildx build \ --secret id=ks_b64,src="$RUNNER_TEMP/sec/ks_b64" \ --secret id=ks_pass,src="$RUNNER_TEMP/sec/ks_pass" \ --secret id=ks_alias,src="$RUNNER_TEMP/sec/ks_alias" \ --secret id=ks_keypass,src="$RUNNER_TEMP/sec/ks_keypass" \ - -f docker/android.Dockerfile -t wisp-android:ci --load . - - name: Extract APK + publish to the host (served at /wisp.apk via the compose volume) + --target apk --output type=local,dest=./out \ + -f docker/android.Dockerfile . + ls -lh ./out/app-release.apk + - name: Publish APK to the host (served at /wisp.apk via the compose volume) run: | - cid=$(docker create wisp-android:ci) - docker cp "$cid:/workspace/android/app/build/outputs/apk/release/app-release.apk" ./app-release.apk - docker rm "$cid" - ls -lh app-release.apk # Stream the APK into a host-mounted dir so it survives web container # recreation (the wisp container mounts /srv/wisp/wisp.apk read-only). - docker run --rm -i -v /srv/wisp:/out alpine:3 sh -c 'mkdir -p /out && cat > /out/wisp.apk' < ./app-release.apk + docker run --rm -i -v /srv/wisp:/out alpine:3 sh -c 'mkdir -p /out && cat > /out/wisp.apk' < ./out/app-release.apk - name: Cleanup staged secrets if: always() run: rm -rf "$RUNNER_TEMP/sec" @@ -114,4 +119,4 @@ jobs: continue-on-error: true with: name: wisp-release-apk - path: app-release.apk + path: ./out/app-release.apk diff --git a/docker/android.Dockerfile b/docker/android.Dockerfile index ea33db4..d142589 100644 --- a/docker/android.Dockerfile +++ b/docker/android.Dockerfile @@ -66,14 +66,53 @@ RUN yes | sdkmanager --licenses > /dev/null \ "cmake;${ANDROID_CMAKE_VERSION}" > /dev/null \ && yes | sdkmanager --licenses > /dev/null +# Build accelerators, installed AFTER the (large, slow) SDK/NDK layer so that +# layer stays cached: +# - ccache : short-circuits recompiling unchanged C++ across builds. The bulk +# of the build is third-party native modules (whisper.cpp via +# whisper.rn, reanimated/worklets, gesture-handler); see the NDK +# toolchain patch below for how it's wired in globally. +# - nodejs : scripts/ci-android-sign.sh runs a small Node patch script (this +# image ships Bun, not Node). Lives here so it's cached, not +# reinstalled on every source change. +RUN apt-get update \ + && apt-get install -y --no-install-recommends ccache nodejs \ + && rm -rf /var/lib/apt/lists/* + +# Route EVERY NDK CMake C/C++ compile + link through ccache. AGP invokes each +# externalNativeBuild project with this NDK toolchain file, so setting the +# compiler launcher here applies ccache to all native modules at once — no need +# to patch each module's CMakeLists (third-party ones don't honor ccache on +# their own). Guarded on ccache being present; a no-op otherwise. +RUN printf '\n%s\n' \ + '# --- WISP: route NDK CMake compiles/links through ccache ---' \ + 'find_program(WISP_CCACHE_PROGRAM ccache)' \ + 'if(WISP_CCACHE_PROGRAM)' \ + ' set(CMAKE_C_COMPILER_LAUNCHER "${WISP_CCACHE_PROGRAM}")' \ + ' set(CMAKE_CXX_COMPILER_LAUNCHER "${WISP_CCACHE_PROGRAM}")' \ + 'endif()' \ + >> "${ANDROID_HOME}/ndk/${ANDROID_NDK_VERSION}/build/cmake/android.toolchain.cmake" + +# ccache tuning. CCACHE_DIR is mounted as a persistent BuildKit cache on the +# gradle RUN below, so hits survive across builds. COMPILERCHECK=content keeps +# hits valid across image rebuilds; the sloppiness flags raise the hit rate for +# RN's headers without risking incorrect cache hits for release builds. +ENV CCACHE_DIR=/root/.ccache \ + CCACHE_MAXSIZE=5G \ + CCACHE_COMPILERCHECK=content \ + CCACHE_SLOPPINESS=time_macros,include_file_mtime,include_file_ctime,file_macro \ + CCACHE_BASEDIR=/workspace + WORKDIR /workspace # --------------------------------------------------------------------------- FROM toolchain AS build -# Install JS deps first (cached unless package.json/bun.lock change). +# Install JS deps first (cached unless package.json/bun.lock change). The bun +# cache mount keeps re-installs fast even when the lockfile DOES change. COPY package.json bun.lock ./ -RUN bun install --frozen-lockfile +RUN --mount=type=cache,target=/root/.bun/install/cache \ + bun install --frozen-lockfile # Bring in the rest of the source (.dockerignore excludes node_modules, android/, ios/, dist...). COPY . . @@ -81,16 +120,22 @@ COPY . . # Generate the native android/ project (autolinks whisper.rn, reanimated, etc.). RUN bunx expo prebuild --platform android --no-install -# Node is required by scripts/ci-android-sign.sh (this image ships Bun, not Node). -# Installed AFTER prebuild so the cached toolchain + prebuild layers are reused. -RUN apt-get update && apt-get install -y --no-install-recommends nodejs && rm -rf /var/lib/apt/lists/* - # Decode the keystore + wire release signing, then assemble the signed APK. # Secrets are mounted as files under /run/secrets and exported only for this RUN. +# +# Two persistent BuildKit cache mounts make rebuilds fast (they survive the +# `COPY . .` layer invalidation that happens on every source change): +# - /root/.ccache : compiled C++ objects (the big win — whisper.cpp etc.). +# - /opt/gradle : GRADLE_USER_HOME — the Gradle build cache (--build-cache), +# resolved dependencies, and the downloaded wrapper dist. +# --build-cache + --parallel let unchanged Kotlin/Java/resource/dex tasks be +# served from cache and independent module tasks run concurrently. RUN --mount=type=secret,id=ks_b64 \ --mount=type=secret,id=ks_pass \ --mount=type=secret,id=ks_alias \ --mount=type=secret,id=ks_keypass \ + --mount=type=cache,target=/root/.ccache \ + --mount=type=cache,target=/opt/gradle \ ANDROID_KEYSTORE_BASE64="$(cat /run/secrets/ks_b64)" \ ANDROID_KEYSTORE_PASSWORD="$(cat /run/secrets/ks_pass)" \ ANDROID_KEY_ALIAS="$(cat /run/secrets/ks_alias)" \ @@ -99,7 +144,20 @@ RUN --mount=type=secret,id=ks_b64 \ && cd android \ && ./gradlew assembleRelease \ -PreactNativeArchitectures=arm64-v8a \ - --no-daemon --max-workers=2 -x lint --stacktrace + --build-cache --parallel \ + --no-daemon --max-workers=2 -x lint --stacktrace \ + && ccache --show-stats --verbose 2>/dev/null | grep -iE 'cacheable|hit|miss|cache size' || true -# The signed APK (handy for `docker create` + `docker cp` extraction in CI). -# /workspace/android/app/build/outputs/apk/release/app-release.apk +# The signed APK lands at: +# /workspace/android/app/build/outputs/apk/release/app-release.apk + +# --------------------------------------------------------------------------- +# apk : a tiny export-only stage holding JUST the signed APK. +# +# CI builds this stage with `--target apk --output type=local,dest=...`, which +# writes only app-release.apk to the host — NO `--load` of the ~15GB build image +# (saves the multi-minute image export/unpack on every push AND avoids that +# image piling up on the server's disk). The COPY pulls from `build`, so the +# whole build still runs; only the output is slimmed. +FROM scratch AS apk +COPY --from=build /workspace/android/app/build/outputs/apk/release/app-release.apk /app-release.apk