sv6160: read RX aggregates the firmware's size, not a quarter of it

Root cause of the TX-completion stall, and of the throughput that never went
anywhere: SV6160_MAX_RX_SIZE was 4096.  The vendor's limit is 33280.

The firmware chains RX aggregates and advertises the NEXT chunk's size in the
current packet header's upper 16 bits.  Under real traffic it builds aggregates
bigger than 4096; our chain walk then hit `rx_next_len > SV6160_MAX_RX_SIZE`,
gave up, and left that data sitting in the firmware.  The firmware waits for the
host to collect it and never asserts host-wake again -- so RX, TX_DONE and
command replies stop together, permanently.  That is why TX descriptors looked
like they were leaking: their completions were queued behind bytes we refused to
read.

Caught by the per-chain-link trace added here (trace=1):

  RXLINK d=125 c=0 pktlen=2048 id=0 raw=0x1a000044 next=6656
  RXEND  d=125 chain_done=0 recvlen=2048 irqs=125     <- nothing, ever again

0x1a00 is 6656.  In every stalled trial gpio_irq_count == drains exactly and
both simply stop, with the last RECV_LEN always large (2048 or 1536).

The right value is the vendor's MAX_RX_SERIAS_BYTES verbatim:
MAX_RX_SERIAS_COUNT(16) * MAX_DATA_SIZE_2K(2048) + 512 = 33280 (rk915
inc/if_io.h:22-23, kzalloc'd in init.c:145, handed to rk915_serias_read as
max_len).  One chain "packet" is an aggregate of up to 16 sub-frames, so it is
not bounded by the 2304-byte 802.11 MTU -- which was the wrong assumption behind
4096.  The TX worker's scratch buffer was sized against the same constant for no
reason (it only ever holds the 581-byte descriptor), so it gets its own.

Measured on the harness that had just produced 4 of 4 STALLED: 6 of 6 CLEAN,
tx_frames == tx_done_events exactly, oversize 0.  Sustained 6 MB each way over
ssh: device RX 970 KB/s, device TX 351 KB/s.  A 6391-frame run completed all
6391 with every descriptor sent == seen == done and nothing stuck.
pkts_per_wake rose from ~1.3 to 2.47 -- the firmware's aggregation now works
instead of being truncated.

tx_stall_ms (the whole-pool resync from 672fdf5) stays as defence in depth, but
it is no longer what carries the link.  rx_stats gains an `oversize` counter so
a future cap hit is loud rather than silent, and the chain walk now warns when
it drops a chunk, because doing so is fatal.

Also adds the instrument that found this: trace=1 logs an ordered
CMDLOG/EVLOG/TXLOG/RXLINK/RXEND stream, bounded so it cannot flood the 1.5 Mbaud
serial console.  Off by default.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Moos7zcsHdpW8DKMq2KM3G
This commit is contained in:
Paul KilarandClaude Opus 5 committed 2026-09-11 21:40:43 -04:00
1 parent 672fdf57c8
commit bc8b7361fc
2 files changed
+83 -4

No files matched your search

+34 -1
View File
@@ -76,7 +76,27 @@ enum sv6160_fw_state {
#define SV6160_NUM_CHANNELS 14
#define SV6160_MAX_SCAN_CHANNELS 50
#define SV6160_RF_PARAMS_SIZE 369
#define SV6160_MAX_RX_SIZE 4096
/* Largest single RX read the firmware can ask us to perform.
*
* This is the vendor's MAX_RX_SERIAS_BYTES verbatim:
* MAX_RX_SERIAS_COUNT(16) * MAX_DATA_SIZE_2K(2048) + 512 = 33280
* (rk915 inc/if_io.h:22-23, allocated once in init.c:145 and passed to
* rk915_serias_read() as max_len). One "packet" in the chain is really an
* aggregate of up to 16 sub-frames, so it is NOT bounded by the 2304-byte
* 802.11 MTU.
*
* We had 4096 here, and that single number was the driver's worst bug. Under
* real traffic the firmware aggregates more than that and advertises the size
* in the previous packet header's upper 16 bits; the chain walk then hit
* `rx_next_len > SV6160_MAX_RX_SIZE`, gave up, and left the data queued in the
* firmware. The firmware waits for the host to collect it and NEVER ASSERTS
* HOST-WAKE AGAIN -- so RX, TX_DONE and command replies all stop together,
* permanently, which is what made TX descriptors look like they were leaking.
* Captured at the moment of death:
* RXLINK d=125 c=0 pktlen=2048 id=0 raw=0x1a000044 next=6656
* RXEND d=125 chain_done=0 recvlen=2048 irqs=125
* 6656 > 4096, so we dropped it; irqs and drains both stop at 125. */
#define SV6160_MAX_RX_SIZE 33280
/* ===================================================================
* Firmware Packet Dispatch Architecture
@@ -556,6 +576,11 @@ struct rpu_cmd_txq_params {
/* Frames we put in one TX descriptor. The firmware allows up to
* SV6160_MAX_TX_CMDS; we send one, so a completion only ever describes one. */
#define SV6160_TX_FRAMES_PER_DESC 1
/* Scratch buffer the TX worker builds a descriptor in. Only the 581-byte
* cmd_tx_ctrl goes here -- the frame body is written straight from skb->data in
* the second half of the same SDIO transaction -- so this does not need to
* track the RX aggregate size, which it used to be sized against. */
#define SV6160_TX_CMD_BUF 4096
/*
* TX control header — RPU_CMD_TX (id=7) from gamely BSP cmd_tx_ctrl.
@@ -903,6 +928,14 @@ struct sv6160_softc {
u32 txs_done_short;
u16 txs_done_short_len;
#define SV6160_CMD_CENSUS_MAX 112
/* Sequence numbers for the ordered trace (mod_trace). */
#define SV6160_TRACE_EVENTS 260
#define SV6160_TRACE_CMDS 120
#define SV6160_TRACE_TX 80
#define SV6160_TRACE_DRAINS 200
u32 rxs_oversize;
u32 evs_seq;
u32 cmds_seq;
u32 cmds_by_id[SV6160_CMD_CENSUS_MAX];
u32 cmds_out_of_range;
@@ -403,6 +403,14 @@ static int mod_descs_per_ac = SV6160_TX_DESCS_PER_AC;
module_param_named(descs_per_ac, mod_descs_per_ac, int, 0644);
MODULE_PARM_DESC(descs_per_ac, "Max descriptors in flight per AC (default 2 = vendor; 1 = strictly serialised)");
/* Ordered dev_info trace of the first N commands, events and TX frames, so the
* seconds around a TX-completion stall can be replayed and diffed against a
* healthy run. Bounded so it cannot flood the 1.5 Mbaud serial console, which
* has wedged this board before. */
static int mod_trace;
module_param_named(trace, mod_trace, int, 0644);
MODULE_PARM_DESC(trace, "Log an ordered CMDLOG/EVLOG/TXLOG trace of the first few hundred bus transactions: 0=off (default)");
static int mod_system_rev = 0x494D47;
module_param_named(system_rev, mod_system_rev, int, 0644);
MODULE_PARM_DESC(system_rev, "RESET.system_rev value (default 0x494D47, as the vendor sends; 0 = old behaviour)");
@@ -972,6 +980,11 @@ int sv6160_rpu_send_cmd(struct sv6160_softc *sc, void *cmd, size_t len,
/* Census of what we actually ask the firmware to do, so an unexpected
* command storm shows up next to the event census rather than having
* to be inferred from a COMMAND_PROC_DONE total. */
sc->cmds_seq++;
if (mod_trace && sc->cmds_seq <= SV6160_TRACE_CMDS)
dev_info(sc->dev, "CMDLOG %u id=%u len=%zu\n",
sc->cmds_seq, id, len);
if (id < SV6160_CMD_CENSUS_MAX)
sc->cmds_by_id[id]++;
else
@@ -2637,6 +2650,14 @@ void sv6160_rpu_process_event(struct sv6160_softc *sc, struct sk_buff *skb)
* it arrived shorter than sizeof(struct rpu_event_tx_done) is
* indistinguishable, from the descriptor ledger alone, from one the
* firmware never sent; this tells the two apart. */
/* Ordered trace of the first events, so a stalled run can be diffed
* against a healthy one. The censuses give counts; only an ordering
* shows what the firmware was doing when completions stopped. */
sc->evs_seq++;
if (mod_trace && sc->evs_seq <= SV6160_TRACE_EVENTS)
dev_info(sc->dev, "EVLOG %u id=%u len=%u\n",
sc->evs_seq, event, skb->len);
if (event < SV6160_EV_CENSUS_MAX) {
sc->evs_count[event]++;
if (!sc->evs_minlen[event] || skb->len < sc->evs_minlen[event])
@@ -3230,6 +3251,7 @@ void sv6160_rx_drain(struct sv6160_softc *sc)
u32 raw_len;
u16 rx_next_len;
bool is_serias;
u32 id_dbg = 0xffff, skb_len_dbg = 0;
/* Let the firmware finish DMAing the first packet into the
* FIFO before the CMD53 read. Our threaded IRQ services the
@@ -3320,6 +3342,8 @@ void sv6160_rx_drain(struct sv6160_softc *sc)
is_serias = le32_to_cpu(hdr->id) == RPU_EVENT_RX_SERIAS;
hdr->length = cpu_to_le32(raw_len & 0x0000FFFF);
id_dbg = le32_to_cpu(hdr->id);
skb_len_dbg = skb->len;
sv6160_rpu_process_event(sc, skb); /* consumes skb */
sc->rxs_pkts++;
if (chain)
@@ -3328,6 +3352,13 @@ void sv6160_rx_drain(struct sv6160_softc *sc)
rx_next_len = sc->serias_next_len;
sc->last_activity = jiffies;
if (mod_trace && sc->rxs_drains <= SV6160_TRACE_DRAINS)
dev_info(sc->dev,
"RXLINK d=%u c=%d pktlen=%u id=%u raw=0x%08x next=%u%s\n",
sc->rxs_drains, chain, skb_len_dbg,
id_dbg, raw_len, rx_next_len,
is_serias ? " serias" : "");
/* End of chain, or a bad next-length — stop and ACK. */
if (!mod_rx_chain) {
chain_done = rx_next_len == 0;
@@ -3337,8 +3368,17 @@ void sv6160_rx_drain(struct sv6160_softc *sc)
chain_done = true;
break;
}
if (rx_next_len > SV6160_MAX_RX_SIZE)
/* Must not happen now the cap matches the vendor's, and
* it is fatal when it does: the data stays queued in the
* firmware, which then stops asserting host-wake for
* good. Shout rather than fail quietly. */
if (rx_next_len > SV6160_MAX_RX_SIZE) {
sc->rxs_oversize++;
dev_warn(sc->dev,
"RX chain advertised %u bytes, over the %u cap - dropping it will stall the firmware\n",
rx_next_len, SV6160_MAX_RX_SIZE);
break;
}
pkt_len = rx_next_len;
}
}
@@ -3362,6 +3402,11 @@ clear_and_done:
* every TX after it went out with no TX_DONE. Clear the interrupt
* regardless, or the level line just re-fires, but keep the doorbell
* for a chain we actually finished. */
if (mod_trace && sc->rxs_drains <= SV6160_TRACE_DRAINS)
dev_info(sc->dev, "RXEND d=%u chain_done=%d recvlen=%u irqs=%u\n",
sc->rxs_drains, chain_done, sc->last_recv_len,
sc->gpio_irq_count);
sv6160_rawb_w(sc, SDIO_REG_INT_ADDR, SDIO_REG_INT_CLR_VAL, &err);
if (chain_done || !mod_notify_gate)
sv6160_rawb_w(sc, SDIO_REG_NOTIFY_ADDR, 1, &err);
@@ -4589,7 +4634,7 @@ static void sv6160_tx_worker(struct work_struct *work)
u32 fw_queue;
tx_buf = kmalloc(SV6160_TX_CMD_SIZE + SV6160_MAX_RX_SIZE, GFP_KERNEL);
tx_buf = kmalloc(SV6160_TX_CMD_SIZE + SV6160_TX_CMD_BUF, GFP_KERNEL);
if (!tx_buf)
return;
@@ -4744,7 +4789,7 @@ static void sv6160_tx_worker(struct work_struct *work)
* which descriptor, which AC, protected or not, aggregated or
* not. Bounded so it cannot flood the console. */
sc->txs_attempts++;
if (sc->txs_attempts <= 60)
if (mod_trace && sc->txs_attempts <= SV6160_TRACE_TX)
dev_info(sc->dev,
"TXLOG #%u desc=%u q=%u fc=0x%04x %s enc=%u iv=%u ampdu=%u len=%u\n",
sc->txs_attempts, desc_id, fw_queue,
@@ -5672,6 +5717,7 @@ static int sv6160_debugfs_rx_stats_show(struct seq_file *s, void *data)
seq_printf(s, "drains %u\n", sc->rxs_drains);
seq_printf(s, "pkts %u\n", sc->rxs_pkts);
seq_printf(s, "nodata %u\n", sc->rxs_nodata);
seq_printf(s, "oversize %u\n", sc->rxs_oversize);
seq_printf(s, "chained %u\n", sc->rxs_chained);
seq_printf(s, "pkts_per_wake %u.%02u\n",
sc->rxs_drains ? sc->rxs_pkts / sc->rxs_drains : 0,