ci: faster CI — stop rebuilding the test app during packaging, single-ABI emulator tests, ccache, CMake 3.31

.github/workflows/npm_release.yml

@@ -16,6 +16,7 @@ env:
   NPM_TAG: "next"
   EMULATOR_NAME: "runtime-emu"
   NDK_VERSION: r29
+  CMAKE_VERSION: "3.31.6"
   ANDROID_API: 33
   ANDROID_ABI: x86_64
   NDK_ARCH: linux
@@ -53,7 +54,7 @@ jobs:
         uses: android-actions/setup-android@651bceb6f9ca583f16b8d75b62c36ded2ae6fc9c # v4.0.0
       - name: Setup NDK
         run: |
-          echo "y" | sdkmanager "cmake;3.6.4111459"
+          echo "y" | sdkmanager "cmake;$CMAKE_VERSION"
           wget -q https://dl.google.com/android/repository/android-ndk-$NDK_VERSION-$NDK_ARCH.zip
           chmod +x android-ndk-$NDK_VERSION-$NDK_ARCH.zip
           unzip -q android-ndk-$NDK_VERSION-$NDK_ARCH.zip
@@ -62,6 +63,23 @@ jobs:
           echo ANDROID_NDK_HOME=${ANDROID_NDK_HOME} >> $GITHUB_ENV
           echo ANDROID_NDK=${ANDROID_NDK_HOME} >> $GITHUB_ENV
           echo ${ANDROID_NDK_HOME} >> $GITHUB_PATH
+      - name: Setup ccache
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.ccache
+          key: ccache-build-${{ github.sha }}
+          restore-keys: ccache-build-
+      - name: Configure build caches
+        run: |
+          command -v ccache || sudo apt-get install -y ccache
+          ccache --version
+          echo "CCACHE_DIR=$HOME/.ccache" >> $GITHUB_ENV
+          echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV
+          # the NDK is unzipped fresh every run, so mtime-based compiler
+          # identification (the default) would invalidate the whole cache
+          echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV
+          mkdir -p ~/.gradle
+          printf 'org.gradle.parallel=true\norg.gradle.caching=true\n' >> ~/.gradle/gradle.properties
       - name: Install Dependencies
         run: |
           npm install
@@ -96,7 +114,9 @@ jobs:
       - name: Fetch prebuilt V8
         run: ./download_v8.sh
       - name: Build npm package
-        run: ./gradlew -PgitCommitVersion=${{ github.sha }} -PnoCCache --stacktrace
+        run: ./gradlew -PgitCommitVersion=${{ github.sha }} --stacktrace
+      - name: ccache stats
+        run: ccache -s
       - name: Upload npm package artifact
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
@@ -133,7 +153,7 @@ jobs:
         uses: android-actions/setup-android@651bceb6f9ca583f16b8d75b62c36ded2ae6fc9c # v4.0.0
       - name: Setup NDK
         run: |
-          echo "y" | sdkmanager "cmake;3.6.4111459"
+          echo "y" | sdkmanager "cmake;$CMAKE_VERSION"
           wget -q https://dl.google.com/android/repository/android-ndk-$NDK_VERSION-$NDK_ARCH.zip
           chmod +x android-ndk-$NDK_VERSION-$NDK_ARCH.zip
           unzip -q android-ndk-$NDK_VERSION-$NDK_ARCH.zip
@@ -142,6 +162,26 @@ jobs:
           echo ANDROID_NDK_HOME=${ANDROID_NDK_HOME} >> $GITHUB_ENV
           echo ANDROID_NDK=${ANDROID_NDK_HOME} >> $GITHUB_ENV
           echo ${ANDROID_NDK_HOME} >> $GITHUB_PATH
+      - name: Setup ccache
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.ccache
+          key: ccache-test-${{ github.sha }}
+          restore-keys: ccache-test-
+      - name: Configure build caches
+        # no org.gradle.caching here: the SBG AST tests spawn nested `gradlew -b`
+        # builds that fail with "Build cache controller already set" when the
+        # build cache is on
+        run: |
+          command -v ccache || sudo apt-get install -y ccache
+          ccache --version
+          echo "CCACHE_DIR=$HOME/.ccache" >> $GITHUB_ENV
+          echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV
+          # the NDK is unzipped fresh every run, so mtime-based compiler
+          # identification (the default) would invalidate the whole cache
+          echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV
+          mkdir -p ~/.gradle
+          printf 'org.gradle.parallel=true\n' >> ~/.gradle/gradle.properties
       - name: Install Dependencies
         run: |
           npm install
@@ -162,7 +202,8 @@ jobs:
           # this is needed on API 30+
           #target: google_apis
           arch: ${{env.ANDROID_ABI}}
-          script: ./gradlew runtestsAndVerifyResults --stacktrace
+          # only build native code for the ABI the emulator actually runs
+          script: ./gradlew runtestsAndVerifyResults -Pabis=${{env.ANDROID_ABI}} --stacktrace
       - name: Upload Test Results
         if: ${{ !cancelled() }} # run this step even if previous step failed
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

.github/workflows/pull_request.yml

@@ -6,13 +6,18 @@ env:
   NPM_TAG: "pr"
   EMULATOR_NAME: "runtime-emu"
   NDK_VERSION: r29
+  CMAKE_VERSION: "3.31.6"
   ANDROID_API: 33
   ANDROID_ABI: x86_64
   NDK_ARCH: linux
 
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
     name: Build
@@ -38,7 +43,7 @@ jobs:
         uses: android-actions/setup-android@651bceb6f9ca583f16b8d75b62c36ded2ae6fc9c # v4.0.0
       - name: Setup NDK
         run: |
-          echo "y" | sdkmanager "cmake;3.6.4111459"
+          echo "y" | sdkmanager "cmake;$CMAKE_VERSION"
           wget -q https://dl.google.com/android/repository/android-ndk-$NDK_VERSION-$NDK_ARCH.zip
           chmod +x android-ndk-$NDK_VERSION-$NDK_ARCH.zip
           unzip -q android-ndk-$NDK_VERSION-$NDK_ARCH.zip
@@ -47,6 +52,23 @@ jobs:
           echo ANDROID_NDK_HOME=${ANDROID_NDK_HOME} >> $GITHUB_ENV
           echo ANDROID_NDK=${ANDROID_NDK_HOME} >> $GITHUB_ENV
           echo ${ANDROID_NDK_HOME} >> $GITHUB_PATH
+      - name: Setup ccache
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.ccache
+          key: ccache-build-${{ github.sha }}
+          restore-keys: ccache-build-
+      - name: Configure build caches
+        run: |
+          command -v ccache || sudo apt-get install -y ccache
+          ccache --version
+          echo "CCACHE_DIR=$HOME/.ccache" >> $GITHUB_ENV
+          echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV
+          # the NDK is unzipped fresh every run, so mtime-based compiler
+          # identification (the default) would invalidate the whole cache
+          echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV
+          mkdir -p ~/.gradle
+          printf 'org.gradle.parallel=true\norg.gradle.caching=true\n' >> ~/.gradle/gradle.properties
       - name: Install Dependencies
         run: |
           npm install
@@ -70,7 +92,9 @@ jobs:
       - name: Fetch prebuilt V8
         run: ./download_v8.sh
       - name: Build npm package
-        run: ./gradlew -PgitCommitVersion=${{ github.sha }} -PnoCCache --stacktrace
+        run: ./gradlew -PgitCommitVersion=${{ github.sha }} --stacktrace
+      - name: ccache stats
+        run: ccache -s
       - name: Upload npm package artifact
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
@@ -110,6 +134,26 @@ jobs:
           echo ANDROID_NDK_HOME=${ANDROID_NDK_HOME} >> $GITHUB_ENV
           echo ANDROID_NDK=${ANDROID_NDK_HOME} >> $GITHUB_ENV
           echo ${ANDROID_NDK_HOME} >> $GITHUB_PATH
+      - name: Setup ccache
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.ccache
+          key: ccache-test-${{ github.sha }}
+          restore-keys: ccache-test-
+      - name: Configure build caches
+        # no org.gradle.caching here: the SBG AST tests spawn nested `gradlew -b`
+        # builds that fail with "Build cache controller already set" when the
+        # build cache is on
+        run: |
+          command -v ccache || sudo apt-get install -y ccache
+          ccache --version
+          echo "CCACHE_DIR=$HOME/.ccache" >> $GITHUB_ENV
+          echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV
+          # the NDK is unzipped fresh every run, so mtime-based compiler
+          # identification (the default) would invalidate the whole cache
+          echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV
+          mkdir -p ~/.gradle
+          printf 'org.gradle.parallel=true\n' >> ~/.gradle/gradle.properties
       - name: Install Dependencies
         run: |
           npm install
@@ -130,7 +174,8 @@ jobs:
           # this is needed on API 30+
           #target: google_apis
           arch: ${{env.ANDROID_ABI}}
-          script: ./gradlew runtestsAndVerifyResults --stacktrace
+          # only build native code for the ABI the emulator actually runs
+          script: ./gradlew runtestsAndVerifyResults -Pabis=${{env.ANDROID_ABI}} --stacktrace
       - name: Upload Test Results
         if: ${{ !cancelled() }} # run this step even if previous step failed
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

build.gradle

@@ -17,6 +17,7 @@ import groovy.json.JsonOutput
 def onlyX86 = project.hasProperty("onlyX86")
 def useCCache = !project.hasProperty("noCCache")
 def hasNdkVersion = project.hasProperty("ndkVersion")
+def selectedAbis = project.hasProperty("abis") ? project.property("abis") : null
 
 if (hasNdkVersion) {
     println "Using NDK version " + ndkVersion
@@ -464,6 +465,9 @@ def getRunTestsBuildArguments = { taskName ->
     if (useCCache) {
         arguments.add("-PuseCCache")
     }
+    if (selectedAbis != null) {
+        arguments.add("-Pabis=${selectedAbis}")
+    }
     arguments += ["--warning-mode", "all"]
     return arguments
 }

test-app/app/build.gradle

@@ -747,6 +747,13 @@ task buildMetadata(type: BuildToolTask) {
 
 
     android.applicationVariants.all { variant ->
+        // The generator only reads the selected build type's classes (see doFirst),
+        // so depending on the other variant's tasks would just force a second full
+        // app build without affecting the produced metadata.
+        if (variant.buildType.name != project.ext.selectedBuildType) {
+            return
+        }
+
         def buildTypeName = variant.buildType.name.capitalize()
         def mergeShadersTaskName = "merge${buildTypeName}Shaders"
         def mergeShadersTask = tasks.findByName(mergeShadersTaskName)
@@ -1303,18 +1310,19 @@ tasks.configureEach({ DefaultTask currentTask ->
 
 })
 
-rootProject.subprojects.forEach {
-    it.tasks.configureEach({ DefaultTask currentTask ->
-        if (currentTask =~ /.+bundleLibCompileToJar.*/) {
-            currentTask.finalizedBy(cleanupAllJars)
-        }
-
-        if (currentTask =~ /bundleLibRuntimeToDir.*/) {
-            currentTask.finalizedBy(buildMetadata)
-        }
-
-        if (currentTask =~ /compile.*LibraryResources/) {
-            currentTask.finalizedBy(buildMetadata)
+// Do not attach app metadata tasks onto other projects' library tasks: any build
+// that actually packages this app reaches buildMetadata through merge*Assets, and
+// hooking library tasks forces a full app build into library-only invocations
+// (e.g. :runtime:assembleRelease).
+//
+// Library subprojects' classes.jar is an input of the jar-extraction pipeline
+// (see processJar), so its producer must be ordered before cleanupAllJars —
+// only for the selected build type; depending on the whole compile classpath
+// instead creates a cycle through this app's own compile tasks.
+rootProject.subprojects.forEach { sub ->
+    if (sub.path != project.path) {
+        sub.tasks.matching { it.name == "bundleLibCompileToJar${project.selectedBuildType.capitalize()}" }.all { producer ->
+            cleanupAllJars.dependsOn(producer)
         }
-    })
+    }
 }

test-app/runtests.gradle

@@ -4,6 +4,7 @@ def runOnDeviceOrEmulator = runOnDevice ? "-d" : "-e"
 
 def onlyX86 = project.hasProperty("onlyX86")
 def useCCache = project.hasProperty("useCCache")
+def selectedAbis = project.hasProperty("abis") ? project.property("abis") : null
 
 // task deleteDist(type: Delete) {
 //     doFirst {
@@ -37,6 +38,9 @@ def getBuildArguments = { ->
     if (useCCache) {
         arguments.add("-PuseCCache")
     }
+    if (selectedAbis != null) {
+        arguments.add("-Pabis=${selectedAbis}")
+    }
 
     arguments.add("-PuseKotlin=true")
 

test-app/runtime/build.gradle

@@ -115,12 +115,10 @@ android {
                 if (optimizedWithInspector) {
                     arguments.add("-DOPTIMIZED_WITH_INSPECTOR_BUILD=true")
                 }
-//
-//                if (useCCache) {
-//                    arguments.add("-DUSE_CCACHE=true")
-//                }
-//
-//                arguments "-DANDROID_TOOLCHAIN=clang", "-DANDROID_STL=c++_static", "-DANDROID_NDK_ROOT=${NDK_PATH}"
+
+                if (useCCache) {
+                    arguments.add("-DUSE_CCACHE=true")
+                }
 
                 cppFlags "-std=c++20"
                 arguments "-DANDROID_STL=c++_static", "-DANDROID_NDK_ROOT=${NDK_PATH}", "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON"
@@ -154,7 +152,7 @@ android {
     }
     externalNativeBuild {
         cmake {
-            // version "3.18.1"
+            version "3.31.6"
             path "CMakeLists.txt"
         }
     }
@@ -247,73 +245,9 @@ tasks.configureEach { task ->
         task.finalizedBy createPackageConfigFileTask(taskName)
     }
 
-    if (task =~ /configureCMake.*/) {
-        task.finalizedBy(":app:buildMetadata")
-    }
-
-    if (task =~ /buildCMake.*/) {
-        task.finalizedBy(":app:buildMetadata")
-    }
-
-    if (taskName.contains("syncReleaseLibJars") || taskName.contains("syncDebugLibJars")) {
-        task.finalizedBy(":app:buildMetadata")
-    }
-
-    if (taskName.contains("mergeReleaseJniLibFolders") || taskName.contains("mergeDebugJniLibFolders")) {
-        task.finalizedBy(":app:buildMetadata")
-    }
-
-    if (taskName.contains("mergeReleaseShaders") || taskName.contains("mergeDebugShaders")) {
-        task.finalizedBy(":app:buildMetadata")
-    }
-
-    if (taskName.contains("packageReleaseAssets") || taskName.contains("packageDebugAssets")) {
-        task.finalizedBy(":app:buildMetadata")
-    }
-
-    if (taskName.contains("copyReleaseJniLibsProjectOnly") || taskName.contains("copyDebugJniLibsProjectOnly")) {
-        task.finalizedBy(":app:buildMetadata")
-    }
-
-    if (taskName.contains("copyReleaseJniLibsProjectAndLocalJars") || taskName.contains("copyDebugJniLibsProjectAndLocalJars")) {
-        task.finalizedBy(":app:buildMetadata")
-    }
-
-    if (taskName.contains("generateReleaseLintVitalModel") || taskName.contains("generateDebugLintVitalModel")) {
-        task.finalizedBy(":app:buildMetadata")
-    }
-
-    if (taskName.contains("lintVitalAnalyzeRelease") || taskName.contains("lintVitalAnalyzeDebug")) {
-        task.finalizedBy(":app:buildMetadata")
-    }
-
-    if (task =~ /lintAnalyze.+AndroidTest/) {
-        task.finalizedBy(":app:buildMetadata")
-    }
-
-    if (task =~ /compile.+UnitTestJavaWithJavac/) {
-        task.finalizedBy(":app:buildMetadata")
-    }
-
-    if (task =~ /generate.+LintModel/) {
-        task.finalizedBy(":app:buildMetadata")
-    }
-
-    if (task =~ /process.+Manifest/) {
-        task.finalizedBy(":app:buildMetadata")
-    }
-
-    if (task =~ /merge.+Resources/) {
-        task.finalizedBy(":app:buildMetadata")
-    }
-
-    if (task =~ /verify.+Resources/) {
-        task.finalizedBy(":app:buildMetadata")
-    }
-
-    if (task =~ /test.+UnitTest/) {
-        task.finalizedBy(":app:buildMetadata")
-    }
+    // Do not wire :app:buildMetadata onto runtime tasks: the app declares its own
+    // dependency on it (merge*Assets), and forcing it here drags a full :app build
+    // (both variants) into every standalone :runtime:assembleRelease.
 }
 
 task 'setPackageVersion' {