feat(appium-tests): size test images from a measured write speed
Emulated USB speed varies by orders of magnitude between hosts, and fixed image sizes made the timing-sensitive tests flaky: on a fast host the write finished before the harness could click the skip verification button. Measure the drive's throughput once per session with a small app-driven speedtest write, timed from the app's own logcat timestamps so UI latency cannot skew it, and size each generic test's image so the write phase lasts TARGET_WRITE_SECONDS (env-tunable). The calibration flow retries once, absorbing a known first-boot file picker flake that would otherwise error every test in the session.
This commit is contained in:
4 files changed
+143
-12
No files matched your search
@@ -14,6 +14,11 @@ class Config:
|
||||
QEMU_USB_BUS = os.environ.get("QEMU_USB_BUS", "ehci.0")
|
||||
QEMU_USB_SLOW_BUS = os.environ.get("QEMU_USB_SLOW_BUS", "ehci.0")
|
||||
QEMU_USB_DEV_ID = os.environ.get("QEMU_USB_DEV_ID", "usbstick")
|
||||
# How long a test image's write phase should last, so there is time to interact with
|
||||
# the UI (skip button, mid-write unplug) regardless of how fast the host emulates USB.
|
||||
# Image sizes are derived from this and the measured drive speed, see
|
||||
# usb_write_speed_mbps in fixtures.py.
|
||||
TARGET_WRITE_SECONDS = float(os.environ.get("TARGET_WRITE_SECONDS", "15"))
|
||||
|
||||
# Every Appium element wait is multiplied by this. CI has no host GPU, so the guest
|
||||
# software-renders (see appium-tests/scripts/vm-config.sh) and first-frame latency
|
||||
|
||||
@@ -3,14 +3,22 @@ import os
|
||||
import pytest
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
import traceback
|
||||
from selenium.common import WebDriverException
|
||||
from appium.options.android import UiAutomator2Options
|
||||
from appium.webdriver.appium_service import AppiumService
|
||||
from appium.webdriver.client_config import AppiumClientConfig
|
||||
from etchdroid import package_name
|
||||
from etchdroid.config import Config
|
||||
from etchdroid.qemu import QEMUController
|
||||
from etchdroid.utils import denoise_logcat, execute_script, get_adb_udid, write_app_filtered_logcat
|
||||
from etchdroid.utils import (
|
||||
denoise_logcat,
|
||||
device_temp_sparse_file,
|
||||
execute_script,
|
||||
get_adb_udid,
|
||||
write_app_filtered_logcat,
|
||||
)
|
||||
from pathlib import Path
|
||||
from typing import Generator
|
||||
|
||||
@@ -137,6 +145,103 @@ def driver(appium_service, request) -> Generator[appium.webdriver.Remote, None,
|
||||
_driver.quit()
|
||||
|
||||
|
||||
CALIBRATION_IMAGE_MB = 256
|
||||
|
||||
|
||||
def _job_duration_from_logcat(driver) -> float | None:
|
||||
"""
|
||||
Duration of the last completed write+verify job, from the app's own logcat
|
||||
timestamps. Unlike wall-clock timing around UI waits, this can't be skewed by how
|
||||
long the harness took to notice the job started or finished.
|
||||
"""
|
||||
from datetime import datetime
|
||||
|
||||
from etchdroid.utils import run_adb_command
|
||||
|
||||
# noinspection PyBroadException
|
||||
try:
|
||||
out = run_adb_command(driver, "logcat", "-d", "-v", "threadtime", "-t", "3000", timeout=30)["stdout"]
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
return None
|
||||
|
||||
def ts(line: str) -> datetime:
|
||||
return datetime.strptime(line[:18], "%m-%d %H:%M:%S.%f")
|
||||
|
||||
start = duration = None
|
||||
for line in out.splitlines():
|
||||
try:
|
||||
if "Writing image to USB drive" in line:
|
||||
start = ts(line)
|
||||
elif "broadcast.FINISHED" in line and start is not None:
|
||||
duration = (ts(line) - start).total_seconds()
|
||||
start = None
|
||||
except ValueError:
|
||||
continue
|
||||
return duration
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def usb_write_speed_mbps(appium_service) -> float:
|
||||
"""
|
||||
Measure the emulated drive's write throughput once per session with a small
|
||||
app-driven "speedtest" write. Tests size their images from this via
|
||||
utils.scaled_image_mb() so the write phase lasts long enough to interact with,
|
||||
no matter how fast the host emulates USB.
|
||||
"""
|
||||
# Imported here: actions is a consumer of driver sessions, keep fixtures importable
|
||||
# without it at collection time.
|
||||
from etchdroid import actions as app
|
||||
|
||||
options = UiAutomator2Options()
|
||||
options.app_package = package_name
|
||||
options.app_activity = ".ui.MainActivity"
|
||||
client_config = AppiumClientConfig(remote_server_addr=f"http://{Config.APPIUM_HOST}:{Config.APPIUM_PORT}")
|
||||
print("\n[speedtest] Measuring USB write speed...")
|
||||
_driver = appium.webdriver.Remote(options=options, client_config=client_config)
|
||||
try:
|
||||
# The very first flow after a VM boot occasionally flakes in the file picker;
|
||||
# since this fixture always runs first, absorb that here instead of erroring
|
||||
# every test that needs a speed measurement.
|
||||
for attempt in range(2):
|
||||
# noinspection PyBroadException
|
||||
try:
|
||||
execute_script(_driver, "mobile: clearApp", {"appId": package_name})
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
execute_script(_driver, "mobile: startActivity", {"component": f"{package_name}/.ui.MainActivity"})
|
||||
|
||||
try:
|
||||
with device_temp_sparse_file(
|
||||
_driver, "etchdroid_speedtest_", ".iso", f"{CALIBRATION_IMAGE_MB}M"
|
||||
) as image:
|
||||
start = time.monotonic()
|
||||
app.basic_flow(_driver, image.filename)
|
||||
app.wait_for_success(_driver, timeout=600)
|
||||
# Wall-clock (includes UI latency) is only the fallback.
|
||||
elapsed = _job_duration_from_logcat(_driver) or max(time.monotonic() - start, 0.001)
|
||||
break
|
||||
except WebDriverException:
|
||||
if attempt == 1:
|
||||
raise
|
||||
print("[speedtest] Flow flaked, retrying once...")
|
||||
|
||||
# noinspection PyBroadException
|
||||
try:
|
||||
_driver.terminate_app(package_name)
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
_driver.quit()
|
||||
|
||||
# The job writes the image and then verifies it, so it moves ~2x the data. The UI
|
||||
# latency baked into `elapsed` underestimates the speed a little, which errs toward
|
||||
# larger images -- the safe direction.
|
||||
speed = 2 * CALIBRATION_IMAGE_MB / elapsed
|
||||
print(f"[speedtest] {CALIBRATION_IMAGE_MB} MB write+verify took {elapsed:.1f}s -> {speed:.1f} MB/s")
|
||||
return speed
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qemu() -> Generator[QEMUController, None, None]:
|
||||
if not os.path.exists(Config.QEMU_QMP_PATH) or not os.path.exists(Config.QEMU_MONITOR_PATH):
|
||||
|
||||
@@ -127,6 +127,21 @@ def grant_permissions(driver: appium.webdriver.Remote, permissions: list[str]) -
|
||||
)
|
||||
|
||||
|
||||
def scaled_image_mb(
|
||||
speed_mbps: float,
|
||||
seconds: float | None = None,
|
||||
min_mb: int = 128,
|
||||
max_mb: int = 6000,
|
||||
) -> int:
|
||||
"""
|
||||
Size (in MB) of a test image whose write phase lasts ~`seconds` at the measured
|
||||
drive speed (see the usb_write_speed_mbps fixture). Clamped to stay well under the
|
||||
emulated stick's capacity (VM_USB_SIZE in scripts/vm-config.sh).
|
||||
"""
|
||||
seconds = Config.TARGET_WRITE_SECONDS if seconds is None else seconds
|
||||
return max(min_mb, min(max_mb, int(speed_mbps * seconds)))
|
||||
|
||||
|
||||
PathFilenamePair = namedtuple("PathFilenamePair", ["path", "filename"])
|
||||
|
||||
|
||||
|
||||
@@ -3,20 +3,23 @@ from time import sleep
|
||||
import appium.webdriver
|
||||
|
||||
from etchdroid import actions as app
|
||||
from etchdroid.fixtures import appium_service, driver
|
||||
from etchdroid.utils import device_temp_sparse_file, used, wait_for_element
|
||||
from etchdroid.config import Config
|
||||
from etchdroid.fixtures import appium_service, driver, usb_write_speed_mbps
|
||||
from etchdroid.utils import device_temp_sparse_file, scaled_image_mb, used, wait_for_element
|
||||
|
||||
used(appium_service)
|
||||
used(appium_service, usb_write_speed_mbps)
|
||||
|
||||
|
||||
def test_regular_flow(driver: appium.webdriver.Remote):
|
||||
with device_temp_sparse_file(driver, "etchdroid_test_regular_flow_", ".iso", "1000M") as image:
|
||||
def test_regular_flow(driver: appium.webdriver.Remote, usb_write_speed_mbps: float):
|
||||
size = f"{scaled_image_mb(usb_write_speed_mbps)}M"
|
||||
with device_temp_sparse_file(driver, "etchdroid_test_regular_flow_", ".iso", size) as image:
|
||||
app.basic_flow(driver, image.filename)
|
||||
app.wait_for_success(driver)
|
||||
|
||||
|
||||
def test_skip_verification(driver: appium.webdriver.Remote):
|
||||
with device_temp_sparse_file(driver, "etchdroid_test_skip_verification_", ".iso", "1000M") as image:
|
||||
def test_skip_verification(driver: appium.webdriver.Remote, usb_write_speed_mbps: float):
|
||||
size = f"{scaled_image_mb(usb_write_speed_mbps)}M"
|
||||
with device_temp_sparse_file(driver, "etchdroid_test_skip_verification_", ".iso", size) as image:
|
||||
app.basic_flow(driver, image.filename)
|
||||
app.skip_lay_flat_sheet(driver)
|
||||
|
||||
@@ -26,8 +29,9 @@ def test_skip_verification(driver: appium.webdriver.Remote):
|
||||
app.wait_for_success(driver)
|
||||
|
||||
|
||||
def test_accept_notifications(driver: appium.webdriver.Remote):
|
||||
with device_temp_sparse_file(driver, "etchdroid_test_accept_notifications_", ".iso", "1000M") as image:
|
||||
def test_accept_notifications(driver: appium.webdriver.Remote, usb_write_speed_mbps: float):
|
||||
size = f"{scaled_image_mb(usb_write_speed_mbps)}M"
|
||||
with device_temp_sparse_file(driver, "etchdroid_test_accept_notifications_", ".iso", size) as image:
|
||||
app.basic_flow(driver, image.filename)
|
||||
sure_btn = wait_for_element(
|
||||
driver,
|
||||
@@ -48,8 +52,10 @@ def test_accept_notifications(driver: appium.webdriver.Remote):
|
||||
app.wait_for_success(driver)
|
||||
|
||||
|
||||
def test_accept_then_deny_notifications(driver: appium.webdriver.Remote):
|
||||
with device_temp_sparse_file(driver, "etchdroid_test_accept_then_deny_notifications_", ".iso", "1800M") as image:
|
||||
def test_accept_then_deny_notifications(driver: appium.webdriver.Remote, usb_write_speed_mbps: float):
|
||||
# Double the usual window: this flow takes a detour through the notification settings.
|
||||
size = f"{scaled_image_mb(usb_write_speed_mbps, seconds=2 * Config.TARGET_WRITE_SECONDS)}M"
|
||||
with device_temp_sparse_file(driver, "etchdroid_test_accept_then_deny_notifications_", ".iso", size) as image:
|
||||
app.basic_flow(driver, image.filename)
|
||||
sure_btn = wait_for_element(
|
||||
driver,
|
||||
|
||||
Reference in new issue
Block a user