feat(appium-tests): size test images from a measured write speed

Emulated USB speed varies by orders of magnitude between hosts, and
fixed image sizes made the timing-sensitive tests flaky: on a fast
host the write finished before the harness could click the skip
verification button.

Measure the drive's throughput once per session with a small
app-driven speedtest write, timed from the app's own logcat
timestamps so UI latency cannot skew it, and size each generic test's
image so the write phase lasts TARGET_WRITE_SECONDS (env-tunable).
The calibration flow retries once, absorbing a known first-boot file
picker flake that would otherwise error every test in the session.
This commit is contained in:
Davide Depau committed 2026-08-16 14:35:19 +02:00
1 parent 2e932651de
commit 8a462f377b
4 files changed
+143 -12

No files matched your search

+5
View File
@@ -14,6 +14,11 @@ class Config:
QEMU_USB_BUS = os.environ.get("QEMU_USB_BUS", "ehci.0")
QEMU_USB_SLOW_BUS = os.environ.get("QEMU_USB_SLOW_BUS", "ehci.0")
QEMU_USB_DEV_ID = os.environ.get("QEMU_USB_DEV_ID", "usbstick")
# How long a test image's write phase should last, so there is time to interact with
# the UI (skip button, mid-write unplug) regardless of how fast the host emulates USB.
# Image sizes are derived from this and the measured drive speed, see
# usb_write_speed_mbps in fixtures.py.
TARGET_WRITE_SECONDS = float(os.environ.get("TARGET_WRITE_SECONDS", "15"))
# Every Appium element wait is multiplied by this. CI has no host GPU, so the guest
# software-renders (see appium-tests/scripts/vm-config.sh) and first-frame latency
+106 -1
View File
@@ -3,14 +3,22 @@ import os
import pytest
import signal
import subprocess
import time
import traceback
from selenium.common import WebDriverException
from appium.options.android import UiAutomator2Options
from appium.webdriver.appium_service import AppiumService
from appium.webdriver.client_config import AppiumClientConfig
from etchdroid import package_name
from etchdroid.config import Config
from etchdroid.qemu import QEMUController
from etchdroid.utils import denoise_logcat, execute_script, get_adb_udid, write_app_filtered_logcat
from etchdroid.utils import (
denoise_logcat,
device_temp_sparse_file,
execute_script,
get_adb_udid,
write_app_filtered_logcat,
)
from pathlib import Path
from typing import Generator
@@ -137,6 +145,103 @@ def driver(appium_service, request) -> Generator[appium.webdriver.Remote, None,
_driver.quit()
CALIBRATION_IMAGE_MB = 256
def _job_duration_from_logcat(driver) -> float | None:
"""
Duration of the last completed write+verify job, from the app's own logcat
timestamps. Unlike wall-clock timing around UI waits, this can't be skewed by how
long the harness took to notice the job started or finished.
"""
from datetime import datetime
from etchdroid.utils import run_adb_command
# noinspection PyBroadException
try:
out = run_adb_command(driver, "logcat", "-d", "-v", "threadtime", "-t", "3000", timeout=30)["stdout"]
except Exception:
traceback.print_exc()
return None
def ts(line: str) -> datetime:
return datetime.strptime(line[:18], "%m-%d %H:%M:%S.%f")
start = duration = None
for line in out.splitlines():
try:
if "Writing image to USB drive" in line:
start = ts(line)
elif "broadcast.FINISHED" in line and start is not None:
duration = (ts(line) - start).total_seconds()
start = None
except ValueError:
continue
return duration
@pytest.fixture(scope="session")
def usb_write_speed_mbps(appium_service) -> float:
"""
Measure the emulated drive's write throughput once per session with a small
app-driven "speedtest" write. Tests size their images from this via
utils.scaled_image_mb() so the write phase lasts long enough to interact with,
no matter how fast the host emulates USB.
"""
# Imported here: actions is a consumer of driver sessions, keep fixtures importable
# without it at collection time.
from etchdroid import actions as app
options = UiAutomator2Options()
options.app_package = package_name
options.app_activity = ".ui.MainActivity"
client_config = AppiumClientConfig(remote_server_addr=f"http://{Config.APPIUM_HOST}:{Config.APPIUM_PORT}")
print("\n[speedtest] Measuring USB write speed...")
_driver = appium.webdriver.Remote(options=options, client_config=client_config)
try:
# The very first flow after a VM boot occasionally flakes in the file picker;
# since this fixture always runs first, absorb that here instead of erroring
# every test that needs a speed measurement.
for attempt in range(2):
# noinspection PyBroadException
try:
execute_script(_driver, "mobile: clearApp", {"appId": package_name})
except Exception:
traceback.print_exc()
execute_script(_driver, "mobile: startActivity", {"component": f"{package_name}/.ui.MainActivity"})
try:
with device_temp_sparse_file(
_driver, "etchdroid_speedtest_", ".iso", f"{CALIBRATION_IMAGE_MB}M"
) as image:
start = time.monotonic()
app.basic_flow(_driver, image.filename)
app.wait_for_success(_driver, timeout=600)
# Wall-clock (includes UI latency) is only the fallback.
elapsed = _job_duration_from_logcat(_driver) or max(time.monotonic() - start, 0.001)
break
except WebDriverException:
if attempt == 1:
raise
print("[speedtest] Flow flaked, retrying once...")
# noinspection PyBroadException
try:
_driver.terminate_app(package_name)
except Exception:
pass
finally:
_driver.quit()
# The job writes the image and then verifies it, so it moves ~2x the data. The UI
# latency baked into `elapsed` underestimates the speed a little, which errs toward
# larger images -- the safe direction.
speed = 2 * CALIBRATION_IMAGE_MB / elapsed
print(f"[speedtest] {CALIBRATION_IMAGE_MB} MB write+verify took {elapsed:.1f}s -> {speed:.1f} MB/s")
return speed
@pytest.fixture(scope="session")
def qemu() -> Generator[QEMUController, None, None]:
if not os.path.exists(Config.QEMU_QMP_PATH) or not os.path.exists(Config.QEMU_MONITOR_PATH):
+15
View File
@@ -127,6 +127,21 @@ def grant_permissions(driver: appium.webdriver.Remote, permissions: list[str]) -
)
def scaled_image_mb(
speed_mbps: float,
seconds: float | None = None,
min_mb: int = 128,
max_mb: int = 6000,
) -> int:
"""
Size (in MB) of a test image whose write phase lasts ~`seconds` at the measured
drive speed (see the usb_write_speed_mbps fixture). Clamped to stay well under the
emulated stick's capacity (VM_USB_SIZE in scripts/vm-config.sh).
"""
seconds = Config.TARGET_WRITE_SECONDS if seconds is None else seconds
return max(min_mb, min(max_mb, int(speed_mbps * seconds)))
PathFilenamePair = namedtuple("PathFilenamePair", ["path", "filename"])
+17 -11
View File
@@ -3,20 +3,23 @@ from time import sleep
import appium.webdriver
from etchdroid import actions as app
from etchdroid.fixtures import appium_service, driver
from etchdroid.utils import device_temp_sparse_file, used, wait_for_element
from etchdroid.config import Config
from etchdroid.fixtures import appium_service, driver, usb_write_speed_mbps
from etchdroid.utils import device_temp_sparse_file, scaled_image_mb, used, wait_for_element
used(appium_service)
used(appium_service, usb_write_speed_mbps)
def test_regular_flow(driver: appium.webdriver.Remote):
with device_temp_sparse_file(driver, "etchdroid_test_regular_flow_", ".iso", "1000M") as image:
def test_regular_flow(driver: appium.webdriver.Remote, usb_write_speed_mbps: float):
size = f"{scaled_image_mb(usb_write_speed_mbps)}M"
with device_temp_sparse_file(driver, "etchdroid_test_regular_flow_", ".iso", size) as image:
app.basic_flow(driver, image.filename)
app.wait_for_success(driver)
def test_skip_verification(driver: appium.webdriver.Remote):
with device_temp_sparse_file(driver, "etchdroid_test_skip_verification_", ".iso", "1000M") as image:
def test_skip_verification(driver: appium.webdriver.Remote, usb_write_speed_mbps: float):
size = f"{scaled_image_mb(usb_write_speed_mbps)}M"
with device_temp_sparse_file(driver, "etchdroid_test_skip_verification_", ".iso", size) as image:
app.basic_flow(driver, image.filename)
app.skip_lay_flat_sheet(driver)
@@ -26,8 +29,9 @@ def test_skip_verification(driver: appium.webdriver.Remote):
app.wait_for_success(driver)
def test_accept_notifications(driver: appium.webdriver.Remote):
with device_temp_sparse_file(driver, "etchdroid_test_accept_notifications_", ".iso", "1000M") as image:
def test_accept_notifications(driver: appium.webdriver.Remote, usb_write_speed_mbps: float):
size = f"{scaled_image_mb(usb_write_speed_mbps)}M"
with device_temp_sparse_file(driver, "etchdroid_test_accept_notifications_", ".iso", size) as image:
app.basic_flow(driver, image.filename)
sure_btn = wait_for_element(
driver,
@@ -48,8 +52,10 @@ def test_accept_notifications(driver: appium.webdriver.Remote):
app.wait_for_success(driver)
def test_accept_then_deny_notifications(driver: appium.webdriver.Remote):
with device_temp_sparse_file(driver, "etchdroid_test_accept_then_deny_notifications_", ".iso", "1800M") as image:
def test_accept_then_deny_notifications(driver: appium.webdriver.Remote, usb_write_speed_mbps: float):
# Double the usual window: this flow takes a detour through the notification settings.
size = f"{scaled_image_mb(usb_write_speed_mbps, seconds=2 * Config.TARGET_WRITE_SECONDS)}M"
with device_temp_sparse_file(driver, "etchdroid_test_accept_then_deny_notifications_", ".iso", size) as image:
app.basic_flow(driver, image.filename)
sure_btn = wait_for_element(
driver,