From 8a462f377bafc869a3337204cba9c6a66fcbe2ef Mon Sep 17 00:00:00 2001 From: Davide Depau Date: Sun, 16 Aug 2026 14:35:19 +0200 Subject: [PATCH] feat(appium-tests): size test images from a measured write speed Emulated USB speed varies by orders of magnitude between hosts, and fixed image sizes made the timing-sensitive tests flaky: on a fast host the write finished before the harness could click the skip verification button. Measure the drive's throughput once per session with a small app-driven speedtest write, timed from the app's own logcat timestamps so UI latency cannot skew it, and size each generic test's image so the write phase lasts TARGET_WRITE_SECONDS (env-tunable). The calibration flow retries once, absorbing a known first-boot file picker flake that would otherwise error every test in the session. --- appium-tests/etchdroid/config.py | 5 ++ appium-tests/etchdroid/fixtures.py | 107 ++++++++++++++++++++++++++++- appium-tests/etchdroid/utils.py | 15 ++++ appium-tests/tests/test_generic.py | 28 +++++--- 4 files changed, 143 insertions(+), 12 deletions(-) diff --git a/appium-tests/etchdroid/config.py b/appium-tests/etchdroid/config.py index ce6281b..8142b19 100644 --- a/appium-tests/etchdroid/config.py +++ b/appium-tests/etchdroid/config.py @@ -14,6 +14,11 @@ class Config: QEMU_USB_BUS = os.environ.get("QEMU_USB_BUS", "ehci.0") QEMU_USB_SLOW_BUS = os.environ.get("QEMU_USB_SLOW_BUS", "ehci.0") QEMU_USB_DEV_ID = os.environ.get("QEMU_USB_DEV_ID", "usbstick") + # How long a test image's write phase should last, so there is time to interact with + # the UI (skip button, mid-write unplug) regardless of how fast the host emulates USB. + # Image sizes are derived from this and the measured drive speed, see + # usb_write_speed_mbps in fixtures.py. + TARGET_WRITE_SECONDS = float(os.environ.get("TARGET_WRITE_SECONDS", "15")) # Every Appium element wait is multiplied by this. CI has no host GPU, so the guest # software-renders (see appium-tests/scripts/vm-config.sh) and first-frame latency diff --git a/appium-tests/etchdroid/fixtures.py b/appium-tests/etchdroid/fixtures.py index de9f7e0..7097714 100644 --- a/appium-tests/etchdroid/fixtures.py +++ b/appium-tests/etchdroid/fixtures.py @@ -3,14 +3,22 @@ import os import pytest import signal import subprocess +import time import traceback +from selenium.common import WebDriverException from appium.options.android import UiAutomator2Options from appium.webdriver.appium_service import AppiumService from appium.webdriver.client_config import AppiumClientConfig from etchdroid import package_name from etchdroid.config import Config from etchdroid.qemu import QEMUController -from etchdroid.utils import denoise_logcat, execute_script, get_adb_udid, write_app_filtered_logcat +from etchdroid.utils import ( + denoise_logcat, + device_temp_sparse_file, + execute_script, + get_adb_udid, + write_app_filtered_logcat, +) from pathlib import Path from typing import Generator @@ -137,6 +145,103 @@ def driver(appium_service, request) -> Generator[appium.webdriver.Remote, None, _driver.quit() +CALIBRATION_IMAGE_MB = 256 + + +def _job_duration_from_logcat(driver) -> float | None: + """ + Duration of the last completed write+verify job, from the app's own logcat + timestamps. Unlike wall-clock timing around UI waits, this can't be skewed by how + long the harness took to notice the job started or finished. + """ + from datetime import datetime + + from etchdroid.utils import run_adb_command + + # noinspection PyBroadException + try: + out = run_adb_command(driver, "logcat", "-d", "-v", "threadtime", "-t", "3000", timeout=30)["stdout"] + except Exception: + traceback.print_exc() + return None + + def ts(line: str) -> datetime: + return datetime.strptime(line[:18], "%m-%d %H:%M:%S.%f") + + start = duration = None + for line in out.splitlines(): + try: + if "Writing image to USB drive" in line: + start = ts(line) + elif "broadcast.FINISHED" in line and start is not None: + duration = (ts(line) - start).total_seconds() + start = None + except ValueError: + continue + return duration + + +@pytest.fixture(scope="session") +def usb_write_speed_mbps(appium_service) -> float: + """ + Measure the emulated drive's write throughput once per session with a small + app-driven "speedtest" write. Tests size their images from this via + utils.scaled_image_mb() so the write phase lasts long enough to interact with, + no matter how fast the host emulates USB. + """ + # Imported here: actions is a consumer of driver sessions, keep fixtures importable + # without it at collection time. + from etchdroid import actions as app + + options = UiAutomator2Options() + options.app_package = package_name + options.app_activity = ".ui.MainActivity" + client_config = AppiumClientConfig(remote_server_addr=f"http://{Config.APPIUM_HOST}:{Config.APPIUM_PORT}") + print("\n[speedtest] Measuring USB write speed...") + _driver = appium.webdriver.Remote(options=options, client_config=client_config) + try: + # The very first flow after a VM boot occasionally flakes in the file picker; + # since this fixture always runs first, absorb that here instead of erroring + # every test that needs a speed measurement. + for attempt in range(2): + # noinspection PyBroadException + try: + execute_script(_driver, "mobile: clearApp", {"appId": package_name}) + except Exception: + traceback.print_exc() + execute_script(_driver, "mobile: startActivity", {"component": f"{package_name}/.ui.MainActivity"}) + + try: + with device_temp_sparse_file( + _driver, "etchdroid_speedtest_", ".iso", f"{CALIBRATION_IMAGE_MB}M" + ) as image: + start = time.monotonic() + app.basic_flow(_driver, image.filename) + app.wait_for_success(_driver, timeout=600) + # Wall-clock (includes UI latency) is only the fallback. + elapsed = _job_duration_from_logcat(_driver) or max(time.monotonic() - start, 0.001) + break + except WebDriverException: + if attempt == 1: + raise + print("[speedtest] Flow flaked, retrying once...") + + # noinspection PyBroadException + try: + _driver.terminate_app(package_name) + except Exception: + pass + finally: + _driver.quit() + + # The job writes the image and then verifies it, so it moves ~2x the data. The UI + # latency baked into `elapsed` underestimates the speed a little, which errs toward + # larger images -- the safe direction. + speed = 2 * CALIBRATION_IMAGE_MB / elapsed + print(f"[speedtest] {CALIBRATION_IMAGE_MB} MB write+verify took {elapsed:.1f}s -> {speed:.1f} MB/s") + return speed + + @pytest.fixture(scope="session") def qemu() -> Generator[QEMUController, None, None]: if not os.path.exists(Config.QEMU_QMP_PATH) or not os.path.exists(Config.QEMU_MONITOR_PATH): diff --git a/appium-tests/etchdroid/utils.py b/appium-tests/etchdroid/utils.py index b04e499..11cc734 100644 --- a/appium-tests/etchdroid/utils.py +++ b/appium-tests/etchdroid/utils.py @@ -127,6 +127,21 @@ def grant_permissions(driver: appium.webdriver.Remote, permissions: list[str]) - ) +def scaled_image_mb( + speed_mbps: float, + seconds: float | None = None, + min_mb: int = 128, + max_mb: int = 6000, +) -> int: + """ + Size (in MB) of a test image whose write phase lasts ~`seconds` at the measured + drive speed (see the usb_write_speed_mbps fixture). Clamped to stay well under the + emulated stick's capacity (VM_USB_SIZE in scripts/vm-config.sh). + """ + seconds = Config.TARGET_WRITE_SECONDS if seconds is None else seconds + return max(min_mb, min(max_mb, int(speed_mbps * seconds))) + + PathFilenamePair = namedtuple("PathFilenamePair", ["path", "filename"]) diff --git a/appium-tests/tests/test_generic.py b/appium-tests/tests/test_generic.py index 85bfe29..37bf11d 100644 --- a/appium-tests/tests/test_generic.py +++ b/appium-tests/tests/test_generic.py @@ -3,20 +3,23 @@ from time import sleep import appium.webdriver from etchdroid import actions as app -from etchdroid.fixtures import appium_service, driver -from etchdroid.utils import device_temp_sparse_file, used, wait_for_element +from etchdroid.config import Config +from etchdroid.fixtures import appium_service, driver, usb_write_speed_mbps +from etchdroid.utils import device_temp_sparse_file, scaled_image_mb, used, wait_for_element -used(appium_service) +used(appium_service, usb_write_speed_mbps) -def test_regular_flow(driver: appium.webdriver.Remote): - with device_temp_sparse_file(driver, "etchdroid_test_regular_flow_", ".iso", "1000M") as image: +def test_regular_flow(driver: appium.webdriver.Remote, usb_write_speed_mbps: float): + size = f"{scaled_image_mb(usb_write_speed_mbps)}M" + with device_temp_sparse_file(driver, "etchdroid_test_regular_flow_", ".iso", size) as image: app.basic_flow(driver, image.filename) app.wait_for_success(driver) -def test_skip_verification(driver: appium.webdriver.Remote): - with device_temp_sparse_file(driver, "etchdroid_test_skip_verification_", ".iso", "1000M") as image: +def test_skip_verification(driver: appium.webdriver.Remote, usb_write_speed_mbps: float): + size = f"{scaled_image_mb(usb_write_speed_mbps)}M" + with device_temp_sparse_file(driver, "etchdroid_test_skip_verification_", ".iso", size) as image: app.basic_flow(driver, image.filename) app.skip_lay_flat_sheet(driver) @@ -26,8 +29,9 @@ def test_skip_verification(driver: appium.webdriver.Remote): app.wait_for_success(driver) -def test_accept_notifications(driver: appium.webdriver.Remote): - with device_temp_sparse_file(driver, "etchdroid_test_accept_notifications_", ".iso", "1000M") as image: +def test_accept_notifications(driver: appium.webdriver.Remote, usb_write_speed_mbps: float): + size = f"{scaled_image_mb(usb_write_speed_mbps)}M" + with device_temp_sparse_file(driver, "etchdroid_test_accept_notifications_", ".iso", size) as image: app.basic_flow(driver, image.filename) sure_btn = wait_for_element( driver, @@ -48,8 +52,10 @@ def test_accept_notifications(driver: appium.webdriver.Remote): app.wait_for_success(driver) -def test_accept_then_deny_notifications(driver: appium.webdriver.Remote): - with device_temp_sparse_file(driver, "etchdroid_test_accept_then_deny_notifications_", ".iso", "1800M") as image: +def test_accept_then_deny_notifications(driver: appium.webdriver.Remote, usb_write_speed_mbps: float): + # Double the usual window: this flow takes a detour through the notification settings. + size = f"{scaled_image_mb(usb_write_speed_mbps, seconds=2 * Config.TARGET_WRITE_SECONDS)}M" + with device_temp_sparse_file(driver, "etchdroid_test_accept_then_deny_notifications_", ".iso", size) as image: app.basic_flow(driver, image.filename) sure_btn = wait_for_element( driver,