From 43e57c93571fa4fa53dbb26bb7a62fc8e7a5401f Mon Sep 17 00:00:00 2001 From: kalarina Date: Sun, 25 Oct 2026 12:13:31 +0200 Subject: [PATCH] Add emulator benchmark that compares two revisions --- .github/workflows/bench.yml | 64 +++++++++++++ scripts/bench-build.sh | 29 ++++++ scripts/bench.sh | 179 ++++++++++++++++++++++++++++++++++++ 3 files changed, 272 insertions(+) create mode 100644 .github/workflows/bench.yml create mode 100755 scripts/bench-build.sh create mode 100755 scripts/bench.sh diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..d347fb5 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,64 @@ +name: Benchmark + +# Compares two revisions on one emulator: cold start, memory, background CPU, scroll jank, APK size. +# Run it from the Actions tab, or push the commit to measure to a bench/ branch. +on: + workflow_dispatch: + inputs: + base: + description: Baseline revision + default: origin/main + rounds: + description: Rounds per APK (each has 10 cold starts) + default: "2" + push: + branches: ["bench/**"] + +permissions: + contents: read + +jobs: + bench: + if: github.server_url == 'https://github.com' + runs-on: ubuntu-latest + timeout-minutes: 150 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + + - uses: gradle/actions/setup-gradle@v4 + + - name: Build base and head APKs + run: bash scripts/bench-build.sh "${{ inputs.base || 'origin/main' }}" "${{ github.sha }}" "$RUNNER_TEMP/bench" + + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + - name: Measure on the emulator + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 34 + target: google_apis + arch: x86_64 + profile: pixel_6 + disable-animations: false + emulator-options: -no-snapshot -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + script: bash scripts/bench.sh "$RUNNER_TEMP/bench/base.apk" "$RUNNER_TEMP/bench/head.apk" "${{ inputs.rounds || '2' }}" + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: bench-results + path: | + ${{ runner.temp }}/bench/*.csv + ${{ runner.temp }}/bench/*.md + if-no-files-found: ignore diff --git a/scripts/bench-build.sh b/scripts/bench-build.sh new file mode 100755 index 0000000..bc09c40 --- /dev/null +++ b/scripts/bench-build.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Builds signed release APKs of two revisions for scripts/bench.sh. +# Usage: scripts/bench-build.sh +set -euo pipefail + +base="$(git rev-parse "$1")" +head="$(git rev-parse "$2")" +out="$(mkdir -p "$3" && cd "$3" && pwd)" +work="${RUNNER_TEMP:-/tmp}/bench-src" + +keystore="$out/bench.jks" +[[ -f "$keystore" ]] || keytool -genkeypair -keystore "$keystore" -storepass benchpass -keypass benchpass \ + -alias bench -keyalg RSA -keysize 2048 -validity 365 -dname "CN=bench" >/dev/null 2>&1 +apksigner="$(ls -d "$ANDROID_HOME"/build-tools/*/ | sort -V | tail -n 1)apksigner" + +build() { + local rev="$1" name="$2" dir="$work-$2" apk + rm -rf "$dir" + git worktree add -f --detach "$dir" "$rev" >/dev/null + (cd "$dir" && ./gradlew :app:assembleRelease -q) + apk="$(ls "$dir"/app/build/outputs/apk/release/*.apk | head -n 1)" + echo "$name,apk_bytes,$(stat -c %s "$apk")" >> "$out/sizes.csv" + echo "::notice title=APK size ($name ${rev:0:7})::$(stat -c %s "$apk") bytes" + "$apksigner" sign --ks "$keystore" --ks-pass pass:benchpass --out "$out/$name.apk" "$apk" +} + +rm -f "$out/sizes.csv" +build "$base" base +build "$head" head diff --git a/scripts/bench.sh b/scripts/bench.sh new file mode 100755 index 0000000..a30702c --- /dev/null +++ b/scripts/bench.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Benchmarks two APKs of SparkleDNS on one device or emulator and prints medians side by side: +# cold start, memory, CPU while in the background and scroll jank on the Apps tab. +# Rounds alternate the order of the APKs so device drift affects both alike. +# Needs a rooted device (an emulator image without Play Store) to drop the page cache. +# Usage: scripts/bench.sh [rounds] +set -uo pipefail + +PKG=cc.skysparkle.sparkledns +ACT="$PKG/.MainActivity" +BASE_APK="$1" +HEAD_APK="$2" +ROUNDS="${3:-2}" +STARTS="${STARTS:-10}" +OUT="${OUT:-$(dirname "$BASE_APK")}" +RAW="$OUT/raw.csv" +: > "$RAW" + +adb root >/dev/null 2>&1 && sleep 3 +adb wait-for-device +read -r W H < <(adb shell wm size | tr -d '\r' | awk -F'[ x]' '/Physical/ {print $3, $4}') + +record() { echo "$1,$2,$3" >> "$RAW"; } # name,metric,value + +install() { + adb uninstall "$PKG" >/dev/null 2>&1 + adb install -g "$1" >/dev/null + # The first run lets ProfileInstaller store the profile, then the app is compiled + # the way the device does it in the background after an install + adb shell am start -W -n "$ACT" >/dev/null + sleep 12 + adb shell am force-stop "$PKG" + adb shell cmd package compile -f -m speed-profile "$PKG" >/dev/null +} + +launch() { + adb shell am start-activity -W -n "$ACT" | tr -d '\r' +} + +cold_starts() { + local t + for _ in $(seq "$STARTS"); do + adb shell am force-stop "$PKG" + adb shell 'sync; echo 3 > /proc/sys/vm/drop_caches' + sleep 2 + t="$(launch | awk '/TotalTime/ {print $2}')" + [[ -n "$t" ]] && record "$1" cold_start_ms "$t" + sleep 3 + done +} + +memory() { + local pss + for _ in 1 2 3; do + adb shell am force-stop "$PKG" + launch >/dev/null + sleep 10 + pss="$(adb shell dumpsys meminfo "$PKG" | tr -d '\r' | awk '/TOTAL PSS:/ {print $3; exit}')" + [[ -n "$pss" ]] && record "$1" pss_kb "$pss" + done +} + +# CPU time and scheduler wakeups of all app threads during 60 s on the home screen +background_cpu() { + local pid a b + adb shell am force-stop "$PKG" + launch >/dev/null + sleep 5 + adb shell input keyevent 3 + sleep 5 + pid="$(adb shell pidof "$PKG" | tr -d '\r')" + [[ -z "$pid" ]] && return + a="$(adb shell "cat /proc/$pid/task/*/schedstat" | awk '{t += $1; n += $3} END {print t, n}')" + sleep 60 + b="$(adb shell "cat /proc/$pid/task/*/schedstat" | awk '{t += $1; n += $3} END {print t, n}')" + read -r t1 n1 <<< "$a" + read -r t2 n2 <<< "$b" + record "$1" bg_cpu_ms_per_min "$(( (t2 - t1) / 1000000 ))" + record "$1" bg_wakeups_per_min "$(( n2 - n1 ))" +} + +tap_text() { + local xy + for _ in 1 2 3 4 5; do + adb shell uiautomator dump /sdcard/ui.xml >/dev/null 2>&1 + xy="$(adb shell cat /sdcard/ui.xml | python3 -c ' +import re, sys +xml, text = sys.stdin.read(), sys.argv[1] +for node in re.findall(r"]*>", xml): + if re.search(r"(text|content-desc)=\"%s\"" % re.escape(text), node): + x1, y1, x2, y2 = map(int, re.search(r"bounds=\"\[(\d+),(\d+)\]\[(\d+),(\d+)\]\"", node).groups()) + print((x1 + x2) // 2, (y1 + y2) // 2) + break +' "$1")" + if [[ -n "$xy" ]]; then + adb shell input tap $xy + return 0 + fi + sleep 1 + done + echo "::warning::'$1' not found on screen" + return 1 +} + +jank() { + local g total janky + adb shell am force-stop "$PKG" + launch >/dev/null + sleep 3 + tap_text "Apps" || return + sleep 2 + tap_text "System" # the only group collapsed by default + sleep 3 + adb shell dumpsys gfxinfo "$PKG" reset >/dev/null + for dir in down down down down down up up up up up; do + if [[ $dir == down ]]; then + adb shell input swipe $((W / 2)) $((H * 3 / 4)) $((W / 2)) $((H / 4)) 200 + else + adb shell input swipe $((W / 2)) $((H / 4)) $((W / 2)) $((H * 3 / 4)) 200 + fi + sleep 1.2 + done + g="$(adb shell dumpsys gfxinfo "$PKG" | tr -d '\r')" + total="$(awk -F': ' '/Total frames rendered/ {print $2; exit}' <<< "$g")" + janky="$(awk '/Janky frames:/ {print $3; exit}' <<< "$g")" + if [[ -n "$total" && "$total" -gt 0 ]]; then + record "$1" frames "$total" + record "$1" janky_pct "$(awk -v j="$janky" -v t="$total" 'BEGIN {printf "%.2f", j * 100 / t}')" + for p in 50 90 99; do + record "$1" "frame_p${p}_ms" "$(awk "/${p}th percentile/ {gsub(\"ms\", \"\", \$3); print \$3; exit}" <<< "$g")" + done + fi +} + +measure() { + echo "::group::$1" + install "$2" + cold_starts "$1" + memory "$1" + background_cpu "$1" + jank "$1" + jank "$1" + echo "::endgroup::" +} + +for r in $(seq "$ROUNDS"); do + if (( r % 2 )); then + measure base "$BASE_APK" + measure head "$HEAD_APK" + else + measure head "$HEAD_APK" + measure base "$BASE_APK" + fi +done + +[[ -f "$OUT/sizes.csv" ]] && cat "$OUT/sizes.csv" >> "$RAW" + +python3 - "$RAW" > "$OUT/results.md" <<'EOF' +import csv, statistics, sys +data = {} +for name, metric, value in csv.reader(open(sys.argv[1])): + if value: + data.setdefault(metric, {}).setdefault(name, []).append(float(value)) +print("| Metric (median) | base | head | change | base min–max | head min–max | n |") +print("|---|---|---|---|---|---|---|") +for metric, by in data.items(): + b, h = by.get("base", []), by.get("head", []) + mb = statistics.median(b) if b else None + mh = statistics.median(h) if h else None + change = f"{(mh - mb) / mb * 100:+.1f}%" if mb and mh is not None else "n/a" + fmt = lambda v: "n/a" if v is None else (f"{v:.0f}" if v >= 100 else f"{v:.2f}") + rng = lambda v: f"{fmt(min(v))}–{fmt(max(v))}" if v else "n/a" + print(f"| {metric} | {fmt(mb)} | {fmt(mh)} | {change} | {rng(b)} | {rng(h)} | {len(b)}/{len(h)} |") +EOF + +cat "$OUT/results.md" +[[ -n "${GITHUB_STEP_SUMMARY:-}" ]] && cat "$OUT/results.md" >> "$GITHUB_STEP_SUMMARY" +msg="$(cat "$OUT/results.md")" +echo "::notice title=Benchmark::${msg//$'\n'/'%0A'}"