#!/bin/sh
# test-stream.sh -- the unit measurement: drive the stream into the terminal and
# report what it received, one CSV line per 30-frame block.
#
#   sudo ./test-stream.sh <iface> <mac> [rows] [stretch]
#   sudo ./test-stream.sh <iface> <mac> --tiles
#
#   sudo ./test-stream.sh enp197s0f3u1 08:00:2b:16:4b:5a 128 8
#   sudo ./test-stream.sh enp197s0f3u1 ff:ff:ff:ff:ff:ff 128 8
#   sudo ./test-stream.sh enp197s0f3u1 08:00:2b:16:4b:5a --tiles
#
# <iface> is the host NIC the terminal is plugged into (a dedicated port or a
# hub -- the terminal answers with broadcast status frames on the same wire),
# <mac> the destination address frames are sent to: the terminal's own
# 08:00:2b:16:4b:5a, or broadcast, which its LANCE accepts with an empty filter.
# [rows] and [stretch] are the strip geometry (default 128 x 8 = the whole
# screen, ruling R3); rows*stretch must be 1024 or the encoder refuses, and rows
# may not exceed 144 (the terminal's 16-datagram ring).
#
# --tiles sends the tile stream instead: the 32 KB dictionary in 23 chunks,
# then one TILES_FULL/TILES_DELTA per clip frame from the cache
# make_tiles.py wrote (TILES_DIR, default harness/udpstream/tiles).  The clip
# loops when FRAMES is longer than the cache; the keyframe cadence keeps
# counting across the loop, and a status frame that reports drops forces a full
# frame at the next opportunity.  FRAMES defaults to 300 here and 6300 is the
# plan's 3.5-minute soak.
#
# ===========================================================================
# BEFORE ANY OF THIS: THE PAYLOAD MUST ALREADY BE RUNNING ON THE TERMINAL.
# ===========================================================================
# This script drives the wire; it installs nothing.  Getting vidstream onto the
# unit is the operator's step, and it is the loader route this tree actually
# runs (dwt/notes/LOADER.md, dwt/notes/LOADER_CROSS.md):
#
#   1. Build the linked image the loader consumes:
#        harness/udpstream/build-stream.sh
#      -> harness/udpstream/build/vidstream.coff   (plus .bin, .map, .env)
#
#   2. Put it in the terminal's RAM and start it as a process, over the
#      terminal's own debugger service (rdebug, TCP 771):
#        python3 dwt/loader/hostload.py --host <terminal-ip> \
#            --coff harness/udpstream/build/vidstream.coff
#      hostload.py mallocs a buffer, pokes the COFF into it, pokes the loader at
#      0x01880000 and jumps there; the loader places the sections from their COFF
#      headers and calls create_process(entry, "LOADER", ...), so the payload
#      becomes a scheduled process.  The TFTP variant fetches the same image
#      instead of poking it -- the terminal pulls the file, the host serves it:
#        sudo python3 -u dwt/tcpip/serve_probe.py <host-ip> --bootp \
#             --file harness/udpstream/build/vidstream.coff
#        python3 dwt/loader/hostload.py --host <terminal-ip> --tftp \
#            --coff harness/udpstream/build/vidstream.coff
#
#   3. THE LOAD IS ONE SHOT.  The payload's lance_start() STOPs the LANCE, loads
#      its own init block, INITs and STRTs it -- the chip belongs to the payload
#      from then on, so the OS's network stack (and with it the rdebug session
#      that loaded it) is dead.  If the jump fails there is no second attempt
#      over the wire, only a power cycle, and nothing about the payload can be
#      inspected over rdebug afterwards.
#
#   4. What stays observable is the wire.  The payload answers with a
#      KIND_STATUS frame every 64 accepted frames, so this script's PREFLIGHT
#      sends PROBE frames (default 70) and refuses to run the measurement unless
#      one comes back.  No status frame means the payload is not running (or the
#      cable/interface is wrong), and the script exits 4 saying that instead of
#      sending 300 frames into nothing and reporting "unmeasured" later.
#
# THE malloc CONTEXT IS UNPROVEN ON THE UNIT.  Every run to date enters the
# payload by a debugger/loader jump, and hostload.py's own malloc call is made in
# that jumped context; whether malloc_() from inside the payload's
# create_process child sees the same free list has not been shown on the
# hardware.  The markers that would answer it are POOL_PTR, POOL_FREEB and
# POOL_FREEA (bit 0x01810100, 0x01810120, 0x01810140), read WHILE the payload
# runs -- that page is unowned heap and a long soak can reuse it (frame_parse.h).
#
# THE REPORTING LAG.  The terminal emits a status frame every 64 ACCEPTED frames
# (STATUS_EVERY in vidstream.c), and its `frame_no` field is that report's own
# sequence number (0, 1, 2, ...), NOT a frame counter.  So a status read at any
# instant describes a terminal that is 0..63 accepted frames behind what the host
# has already sent, and a plain frames_rx/sent ratio is confounded by that: a
# 300-frame run whose last report is the 256th reads ~85 % even when not one frame
# was lost.  This script therefore reports three things instead:
#
#   * the CSV, one row per 30 frames (stdout): the host's `sent` count, the
#     status's own `status_no` (its frame_no), its `frames_rx`, the `gap`
#     (sent - frames_rx) and `delivery_lo_pct` = 100 * frames_rx / sent.  That
#     last column is a LOWER BOUND only -- it can never exceed the terminal's
#     own progress, and it is as loose as the 63-frame cadence allows;
#   * an interval table pairing CONSECUTIVE status frames (stderr):
#     delivery = 100 * (frames_rx_k - frames_rx_k-1) / (sent_k - sent_k-1).
#     Both terms are deltas between the same two report arrivals, so the
#     reporting lag cancels instead of biasing the number: a lost frame shows up
#     as the host having sent more than the 64 frames the terminal accepted;
#   * a final line (stderr) that compares the end-of-run gap with the lag budget:
#       63 frames of report cadence + 2 frames in flight = 65.
#     Within it, no loss is measured; beyond it, at least (gap - 65) frames were
#     lost and the script exits 1.
#
# The sender is run with --report-every 1 so a status frame can be paired with
# the host's sent count to within one frame; only every 30th row is printed.
#
# Exit status: 0 only if the preflight saw the payload, a status arrived, at
# least two reports could be paired, the end-of-run gap is inside the lag
# budget, and vidsend.py itself passed (it fails when the last status reports
# drops or crc_err).  2/3 mean a missing prerequisite (arguments, interface,
# root, sender); 4 means the payload did not answer the preflight -- it is not
# running, and nothing was measured.
#
# WHAT THIS CANNOT SHOW FROM HERE: OS_TOUCH (spec 8.9's VRAM canary, marker bit
# 0x01810220) and the terminal's free heap before the run are on the terminal,
# not in the status frame.  Read them on the terminal with its own debugger
# alongside a run and record them with this CSV; an OS repaint during the run
# also shows up here as a fall in the interval delivery, not as a cadence effect.
#
# The three unit runs (plan Task 7 step 6):
#
#   1. sudo ./test-stream.sh <iface> 08:00:2b:16:4b:5a 128 8
#   2. the same with the blit forced to PIXBLT L,XY instead of B,XY
#      (harness/udpstream/vram.asm's vram_blit_ vs vram_b.asm's vram_blit_b_:
#      swap the entry point in vidstream.c's blit_row and rebuild), to measure
#      what the unit costs per destination word
#   3. a 6,300-frame soak (~3.5 min at 30 fps): FRAMES=6300
#
# and, for the tile stream (plan 2, Task 6):
#
#   4. sudo ./test-stream.sh <iface> 08:00:2b:16:4b:5a --tiles
#   5. FRAMES=6300 sudo ./test-stream.sh <iface> 08:00:2b:16:4b:5a --tiles
#
# These need the physical terminal (ruling R4, R16) and have NOT been run.
set -eu

HERE=$(cd "$(dirname "$0")" && pwd)
SENDER="$HERE/vidsend.py"

IFACE=${1:-}
MAC=${2:-}
ROWS=${3:-128}
STRETCH=${4:-8}
TILES=0
if [ "${3:-}" = "--tiles" ]; then
    TILES=1
    ROWS=128
    STRETCH=8
fi
TILES_DIR=${TILES_DIR:-$HERE/tiles}
FRAMES=${FRAMES:-300}
FPS=${FPS:-30}
REPORT=${REPORT:-30}

if [ -z "$IFACE" ] || [ -z "$MAC" ]; then
    echo "usage: sudo $0 <iface> <mac> [rows] [stretch]" >&2
    echo "       sudo $0 <iface> <mac> --tiles" >&2
    echo "  e.g. sudo $0 enp197s0f3u1 08:00:2b:16:4b:5a 128 8" >&2
    exit 2
fi
if [ "$TILES" = 1 ] && [ ! -f "$TILES_DIR/stream.bin" ]; then
    echo "test-stream.sh: no tile cache at $TILES_DIR (build it first:" >&2
    echo "  python3 harness/udpstream/make_tiles.py --out $TILES_DIR)" >&2
    exit 3
fi

# Fail loudly rather than send nothing and call it a run.
if [ ! -f "$SENDER" ]; then
    echo "test-stream.sh: no sender at $SENDER (this script drives vidsend.py)" >&2
    exit 3
fi
if [ ! -d "/sys/class/net/$IFACE" ]; then
    echo "test-stream.sh: no interface '$IFACE' on this host (have: $(ls /sys/class/net | tr '\n' ' '))" >&2
    exit 3
fi
if [ "$(id -u)" -ne 0 ]; then
    echo "test-stream.sh: raw Ethernet needs CAP_NET_RAW; re-run with sudo" >&2
    exit 3
fi
case "$MAC" in
    *:*:*:*:*:*) ;;
    *) echo "test-stream.sh: '$MAC' is not a MAC address (aa:bb:cc:dd:ee:ff)" >&2
       exit 2 ;;
esac

OUT=${OUT:-/tmp/opencode/test-stream.log}
mkdir -p "$(dirname "$OUT")"
if [ "$TILES" = 1 ]; then
    echo "test-stream.sh: $IFACE -> $MAC, tile stream from $TILES_DIR," \
         "$FRAMES frames at $FPS fps" >&2
else
    echo "test-stream.sh: $IFACE -> $MAC, $ROWS rows x $STRETCH," \
         "$FRAMES frames at $FPS fps" >&2
fi
echo "test-stream.sh: capture $OUT" >&2

# ---------------------------------------------------------------------------
# Preflight: is the payload running at all?  Its only wire-visible trace is the
# status frame it builds every 64 accepted frames, so the check is a short run
# of PROBE frames: STATUS_EVERY + 6 covers the cadence and the mid-run report.
# Without this a 300-frame run against a powered-off payload would spend ten
# seconds to report "no status frame arrived", which reads like a measurement.
# ---------------------------------------------------------------------------
PROBE=${PROBE:-70}
PROBE_LOG=$OUT.preflight
echo "test-stream.sh: preflight: $PROBE frames, looking for a status frame" >&2
set +e
if [ "$TILES" = 1 ]; then
    python3 "$SENDER" --iface "$IFACE" --dst "$MAC" --tiles \
        --tiles-dir "$TILES_DIR" --fps "$FPS" --frames "$PROBE" \
        --report-every 0 --status-wait 5 > "$PROBE_LOG" 2>&1
else
    python3 "$SENDER" --iface "$IFACE" --dst "$MAC" --rows "$ROWS" \
        --stretch "$STRETCH" --fps "$FPS" --frames "$PROBE" \
        --report-every 0 --status-wait 5 > "$PROBE_LOG" 2>&1
fi
PROBE_STATUS=$?
set -e
if ! grep -q '^status ' "$PROBE_LOG"; then
    echo "test-stream.sh: PAYLOAD NOT RUNNING: $PROBE frames went out and no" \
         "status frame came back, so nothing on the wire is running vidstream" \
         "(or the NIC/MAC is wrong -- check '$IFACE' and '$MAC')." >&2
    echo "  Load it first: harness/udpstream/build-stream.sh, then" >&2
    echo "  python3 dwt/loader/hostload.py --host <terminal-ip> --coff" \
         "harness/udpstream/build/vidstream.coff (see this script's header);" >&2
    echo "  the load is one shot because the payload takes the LANCE over." >&2
    echo "  preflight log: $PROBE_LOG" >&2
    sed -n '1,5p' "$PROBE_LOG" >&2
    exit 4
fi
if [ "$PROBE_STATUS" -ne 0 ]; then
    echo "test-stream.sh: the payload answered but the preflight already" \
         "failed (see $PROBE_LOG): a run that starts with drops or crc_err" \
         "cannot measure anything." >&2
    grep -E '^(FAIL|PASS|status )' "$PROBE_LOG" | tail -5 >&2
    exit 1
fi
echo "test-stream.sh: preflight: the payload is running" >&2

set +e
if [ "$TILES" = 1 ]; then
    python3 "$SENDER" --iface "$IFACE" --dst "$MAC" --tiles \
        --tiles-dir "$TILES_DIR" --fps "$FPS" --frames "$FRAMES" \
        --report-every 1 > "$OUT" 2>&1
else
    python3 "$SENDER" --iface "$IFACE" --dst "$MAC" --rows "$ROWS" \
        --stretch "$STRETCH" --fps "$FPS" --frames "$FRAMES" \
        --report-every 1 > "$OUT" 2>&1
fi
STATUS=$?
set -e
if [ "$STATUS" -ne 0 ] && [ ! -s "$OUT" ]; then
    echo "test-stream.sh: vidsend.py failed with status $STATUS and wrote nothing" >&2
    exit "$STATUS"
fi

# Merge the sender's CSV rows with the terminal's status reports, in the order
# they arrived.  Both are printed line by line with a flush, so the order in
# this one file IS the chronological order; no timestamps are needed.  vidsend
# runs with --report-every 1, so `sent` is exact to within the one frame that is
# in flight when a status is read.
set +e
python3 - "$OUT" "$REPORT" "$PROBE" <<'PY'
import re, sys

# The third argument is the number of frames the PREFLIGHT sent.  The terminal's
# frames_rx keeps counting across it, so the CSV's `sent` column has to start
# from the same place: without the offset the first rows would compare a
# main-run `sent` against a frames_rx that already includes 70 probe frames and
# print deliveries well over 100 %.  Both columns are then cumulative over
# probe + run, so every gap and every interval delta is measured between the
# same two events, which is all the pairing needs.
path, every, sent0 = sys.argv[1], int(sys.argv[2]), int(sys.argv[3])
# The tile CSV's first column is the FRAME NUMBER (from 0), not a count: the
# row describes the frame whose number it carries.  The strip CSV's is the
# count sent.  One frame apart, and both columns have to line up with
# frames_rx, which counts accepted frames from 1.
tiles = len(sys.argv) > 4 and sys.argv[4] == "1"
row = re.compile(r"^(\d+),([\d.]+),([\d.]+),(\d+)(?:,|$)")
status = re.compile(
    r"^status (\d+): frame_no=(\d+) frames_rx=(\d+) crc_err=(\d+) "
    r"drops=(\d+) csr0=0x([0-9A-Fa-f]+) heap_free_kb=(\d+)$")

# The terminal reports every 64 accepted frames, so its last report can be 63
# frames old; two more cover the frame on the wire and the one on the screen.
LAG = 63
INFLIGHT = 2
BUDGET = LAG + INFLIGHT

sent = 0
last = None
events = []
rows = []
with open(path, "r", encoding="utf-8", errors="replace") as f:
    for line in f:
        line = line.rstrip("\n")
        m = row.match(line)
        if m:
            n = int(m.group(1))
            sent = n + sent0 + (1 if tiles else 0)
            # The 30-frame cadence is the SENDER's frame number, not the
            # cumulative one: with a probe offset the cumulative value is never
            # a multiple of 30, and no row would ever print.
            if every > 0 and n % every == 0:
                rows.append((sent, m.group(2), m.group(3), m.group(4), last))
            continue
        m = status.match(line)
        if not m:
            continue
        last = dict(status_no=int(m.group(2)), frames_rx=int(m.group(3)),
                    crc_err=int(m.group(4)), drops=int(m.group(5)),
                    csr0=int(m.group(6), 16), sent=sent)
        events.append(last)

print("frame,elapsed_s,fps,bytes,sent,status_no,frames_rx,gap,delivery_lo_pct")
for sent_n, elapsed, fps, nbytes, ev in rows:
    if ev is None:
        print("%d,%s,%s,%s,%d,,,,," % (sent_n, elapsed, fps, nbytes, sent_n))
        continue
    gap = sent_n - ev["frames_rx"]
    lo = 100.0 * ev["frames_rx"] / sent_n if sent_n else 0.0
    print("%d,%s,%s,%s,%d,%d,%d,%d,%.1f"
          % (sent_n, elapsed, fps, nbytes, sent_n, ev["status_no"],
             ev["frames_rx"], gap, lo))

print("", file=sys.stderr)
if len(events) == 0:
    print("test-stream: UNMEASURED: no status frame arrived, so delivery and "
          "drops are unknown (the terminal said nothing)", file=sys.stderr)
    sys.exit(1)
if len(events) < 2:
    print("test-stream: UNMEASURED: only one status frame arrived (status_no="
          "%d, frames_rx=%d); delivery is a paired ratio and needs two reports"
          % (events[0]["status_no"], events[0]["frames_rx"]), file=sys.stderr)
    sys.exit(1)

print("interval,status_no_from,status_no_to,accepted,sent,delivery_pct",
      file=sys.stderr)
i = 0
for prev, cur in zip(events, events[1:]):
    i += 1
    d_acc = cur["frames_rx"] - prev["frames_rx"]
    d_sent = cur["sent"] - prev["sent"]
    if d_sent <= 0:
        # Two reports with no send between them: the sender stalled (or the
        # status pair straddles the last frame).  No ratio, said so.
        print("%d,%d,%d,%d,%d,?" % (i, prev["status_no"], cur["status_no"],
                                    d_acc, d_sent), file=sys.stderr)
        continue
    pct = 100.0 * d_acc / d_sent
    print("%d,%d,%d,%d,%d,%.1f" % (i, prev["status_no"], cur["status_no"],
                                   d_acc, d_sent, pct), file=sys.stderr)

first, final = events[0], events[-1]
d_acc = final["frames_rx"] - first["frames_rx"]
d_sent = final["sent"] - first["sent"]
paired = ("%.1f%%" % (100.0 * d_acc / d_sent)) if d_sent > 0 else "?"
gap = sent - final["frames_rx"]
if gap <= BUDGET:
    verdict = "WITHIN REPORTING LAG (%d allowed): no loss measured" % BUDGET
    code = 0
else:
    verdict = ("BEYOND REPORTING LAG: at least %d frame(s) lost"
               % (gap - BUDGET))
    code = 1
print("test-stream: FINAL sent=%d accepted=%d (status_no=%d) gap=%d budget=%d "
      "paired_delivery=%s -> %s"
      % (sent, final["frames_rx"], final["status_no"], gap, BUDGET, paired,
         verdict), file=sys.stderr)
sys.exit(code)
PY
MERGE=$?
set -e

echo "test-stream.sh: vidsend.py status $STATUS, merge status $MERGE" >&2
echo "test-stream.sh: OS_TOUCH and the terminal's heap are NOT in this table:" \
     "read marker bit 0x01810220 and POOL_FREE* on the terminal and record them" \
     "with this CSV." >&2
if [ "$STATUS" -ne 0 ] || [ "$MERGE" -ne 0 ]; then
    exit 1
fi
exit 0