scripts: add a pcap analyser to find the BBA throughput limit
Swiss dumps a disc over the adapter at 740-780 KB/s, which is ~23% of the
27 Mbit/s EXI ceiling (3375 KB/s), and no published real-BBA figure exists to
compare against. Rather than reason about where the rest goes, measure it.
./scripts/analyse-pcap.py /tmp/bba.pcap --gc-ip 10.0.0.64 reports, per
direction: throughput as a fraction of the EXI ceiling, MEAN FRAME SIZE,
retransmissions, the GC's advertised TCP window, zero-window spans, and the
largest gaps between data frames. Each one discriminates a different
bottleneck:
mean frame far below 1514 -> per-frame overhead (interrupts, EXI
transactions, socket bookkeeping) dominates
retransmissions -> we are dropping frames, e.g. the 15-page RX
ring overflowing (RBFI)
small advertised window -> capped at window/RTT regardless of the wire
zero-window spans -> the GC cannot drain: CPU/stack bound
large gaps, little sending -> the limit is upstream of the wire entirely
(disc read, CPU, stack), not the adapter
Classic pcap only, stdlib only, no root. pcapng is detected and refused with
a conversion hint rather than misparsed.
scripts/test-analyse-pcap.py synthesises captures with known properties and
checks each conclusion is reached: 1000 full frames in 2 s reports ~22% of the
ceiling, 200-byte payloads raise the overhead warning, a repeated seq counts 9
retransmissions, one 300 ms hole in 100 frames is found and attributed, a
zero-window span is called CPU/stack bound, and a pcapng header is rejected.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Executable
+192
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Work out what is limiting BBA throughput, from a capture.
|
||||
|
||||
sudo tcpdump -i enp0s31f6 -s0 -w /tmp/bba.pcap host 10.0.0.64
|
||||
# ... run the Swiss dump for ~20 s, then Ctrl-C ...
|
||||
./scripts/analyse-pcap.py /tmp/bba.pcap --gc-ip 10.0.0.64
|
||||
|
||||
Answers, per direction:
|
||||
* throughput, and what fraction of the 27 Mbit/s EXI ceiling it is
|
||||
* MEAN FRAME SIZE -- if this is far below 1514 the cost is per-frame
|
||||
overhead (interrupts, EXI transactions, socket bookkeeping), not bandwidth
|
||||
* retransmissions -- frames we dropped would show up here, e.g. from the
|
||||
15-page RX ring overflowing (RBFI)
|
||||
* the GC's advertised TCP window, and how long it sat at zero
|
||||
* the largest gaps in the stream, which is where the stall actually is
|
||||
|
||||
Classic pcap only (what `tcpdump -w` writes). Stdlib only, no root needed.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
|
||||
PCAP_MAGICS = {0xA1B2C3D4: (">", 1_000_000), 0xD4C3B2A1: ("<", 1_000_000),
|
||||
0xA1B23C4D: (">", 1_000_000_000), 0x4D3CB2A1: ("<", 1_000_000_000)}
|
||||
EXI_BYTES_PER_S = 27_000_000 / 8 # the hard ceiling: 27 MHz, 1 bit/clock
|
||||
|
||||
|
||||
def packets(path):
|
||||
with open(path, "rb") as f:
|
||||
hdr = f.read(24)
|
||||
if len(hdr) < 24:
|
||||
sys.exit("not a pcap file (too short)")
|
||||
magic = struct.unpack(">I", hdr[:4])[0]
|
||||
if magic == 0x0A0D0D0A:
|
||||
sys.exit("this is pcapng; re-capture with `tcpdump -w ... ` "
|
||||
"(libpcap writes classic pcap) or convert with "
|
||||
"`editcap -F pcap in.pcapng out.pcap`")
|
||||
if magic not in PCAP_MAGICS:
|
||||
sys.exit(f"unknown pcap magic {magic:#010x}")
|
||||
endian, divisor = PCAP_MAGICS[magic]
|
||||
linktype = struct.unpack(endian + "I", hdr[20:24])[0]
|
||||
if linktype != 1:
|
||||
sys.exit(f"linktype {linktype}, expected 1 (Ethernet)")
|
||||
while True:
|
||||
ph = f.read(16)
|
||||
if len(ph) < 16:
|
||||
return
|
||||
ts_s, ts_f, incl, orig = struct.unpack(endian + "IIII", ph)
|
||||
data = f.read(incl)
|
||||
if len(data) < incl:
|
||||
return
|
||||
yield ts_s + ts_f / divisor, data, orig
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("pcap")
|
||||
ap.add_argument("--gc-ip", default="10.0.0.64")
|
||||
ap.add_argument("--top-gaps", type=int, default=5)
|
||||
a = ap.parse_args()
|
||||
gc = socket.inet_aton(a.gc_ip)
|
||||
|
||||
stats = {d: dict(pkts=0, wire=0, payload=0, retx=0, sizes=[], zerowin=0)
|
||||
for d in ("from_gc", "to_gc")}
|
||||
seen = {d: set() for d in ("from_gc", "to_gc")}
|
||||
t0 = t1 = None
|
||||
wins, gaps, last_ts = [], [], None
|
||||
zero_win_spans, zw_start = [], None
|
||||
other = collections.Counter()
|
||||
|
||||
for ts, frame, orig in packets(a.pcap):
|
||||
if len(frame) < 34:
|
||||
continue
|
||||
etype = struct.unpack("!H", frame[12:14])[0]
|
||||
if etype != 0x0800:
|
||||
other[etype] += 1
|
||||
continue
|
||||
ip = frame[14:]
|
||||
ihl = (ip[0] & 0x0F) * 4
|
||||
src, dst = ip[12:16], ip[16:20]
|
||||
if src == gc:
|
||||
d = "from_gc"
|
||||
elif dst == gc:
|
||||
d = "to_gc"
|
||||
else:
|
||||
continue
|
||||
if t0 is None:
|
||||
t0 = ts
|
||||
t1 = ts
|
||||
s = stats[d]
|
||||
s["pkts"] += 1
|
||||
s["wire"] += orig
|
||||
s["sizes"].append(orig)
|
||||
if ip[9] != socket.IPPROTO_TCP:
|
||||
continue
|
||||
tcp = ip[ihl:]
|
||||
if len(tcp) < 20:
|
||||
continue
|
||||
sport, dport, seq = struct.unpack("!HHI", tcp[:8])
|
||||
doff = (tcp[12] >> 4) * 4
|
||||
win = struct.unpack("!H", tcp[14:16])[0]
|
||||
plen = struct.unpack("!H", ip[2:4])[0] - ihl - doff
|
||||
if plen > 0:
|
||||
s["payload"] += plen
|
||||
key = (sport, dport, seq, plen)
|
||||
if key in seen[d]:
|
||||
s["retx"] += 1
|
||||
else:
|
||||
seen[d].add(key)
|
||||
# Gaps measured on the DATA direction only: a gap between data
|
||||
# frames is a stall, a gap before an ACK is just latency.
|
||||
if d == "from_gc":
|
||||
if last_ts is not None:
|
||||
gaps.append((ts - last_ts, ts))
|
||||
last_ts = ts
|
||||
if d == "from_gc":
|
||||
wins.append(win)
|
||||
if d == "to_gc":
|
||||
if win == 0 and zw_start is None:
|
||||
zw_start = ts
|
||||
elif win != 0 and zw_start is not None:
|
||||
zero_win_spans.append(ts - zw_start)
|
||||
zw_start = None
|
||||
|
||||
if t0 is None:
|
||||
sys.exit(f"no IPv4 traffic involving {a.gc_ip} in the capture")
|
||||
dur = max(t1 - t0, 1e-9)
|
||||
print(f"capture: {dur:.2f} s involving {a.gc_ip}\n")
|
||||
|
||||
for d, label in (("from_gc", "GC -> PC (disc dump)"), ("to_gc", "PC -> GC")):
|
||||
s = stats[d]
|
||||
if not s["pkts"]:
|
||||
print(f"{label}: nothing\n")
|
||||
continue
|
||||
mean = s["wire"] / s["pkts"]
|
||||
bps = s["wire"] / dur
|
||||
pps = s["pkts"] / dur
|
||||
print(f"{label}")
|
||||
print(f" {s['pkts']} frames, {s['wire']/1e6:.2f} MB on the wire, "
|
||||
f"{s['payload']/1e6:.2f} MB TCP payload")
|
||||
print(f" throughput {bps/1024:.0f} KB/s ({bps*8/1e6:.2f} Mbit/s, "
|
||||
f"{100*bps/EXI_BYTES_PER_S:.1f}% of the 27 Mbit/s EXI ceiling)")
|
||||
print(f" {pps:.0f} frames/s, mean frame {mean:.0f} B "
|
||||
f"(max {max(s['sizes'])} B)")
|
||||
if s["payload"]:
|
||||
print(f" retransmissions {s['retx']} "
|
||||
f"({100*s['retx']/max(s['pkts'],1):.2f}% of frames)")
|
||||
if mean < 1000 and s["payload"]:
|
||||
print(f" ⚠️ mean frame {mean:.0f} B is well under 1514: the cost "
|
||||
f"is PER-FRAME overhead,")
|
||||
print(f" not bandwidth. {pps:.0f} frames/s each needing an "
|
||||
f"interrupt + EXI")
|
||||
print(f" transactions + W5100 socket bookkeeping.")
|
||||
print()
|
||||
|
||||
if wins:
|
||||
print(f"GC advertised TCP window: min {min(wins)} max {max(wins)} "
|
||||
f"last {wins[-1]} B")
|
||||
if max(wins) < 8192:
|
||||
print(f" ⚠️ a {max(wins)} B window caps a single TCP stream at "
|
||||
f"window/RTT; at 1 ms RTT that is "
|
||||
f"{max(wins)/1e-3/1024:.0f} KB/s")
|
||||
if zero_win_spans:
|
||||
print(f"PC saw zero-window from the GC {len(zero_win_spans)} times, "
|
||||
f"total {sum(zero_win_spans)*1000:.0f} ms "
|
||||
f"({100*sum(zero_win_spans)/dur:.1f}% of the capture)")
|
||||
print(" -> the GC could not drain fast enough: CPU/stack bound")
|
||||
if other:
|
||||
print("non-IPv4 frames: " +
|
||||
", ".join(f"{e:#06x}×{n}" for e, n in other.most_common(4)))
|
||||
|
||||
if gaps:
|
||||
gaps.sort(reverse=True)
|
||||
total_stall = sum(g for g, _ in gaps if g > 0.005)
|
||||
print(f"\nlargest gaps between GC data frames "
|
||||
f"(top {a.top_gaps}):")
|
||||
for g, ts in gaps[:a.top_gaps]:
|
||||
print(f" {g*1000:7.2f} ms at t+{ts-t0:.3f}s")
|
||||
print(f" gaps >5 ms total {total_stall*1000:.0f} ms "
|
||||
f"({100*total_stall/dur:.1f}% of the capture)")
|
||||
if total_stall / dur > 0.2:
|
||||
print(" -> most of the time is spent NOT sending: the limit is "
|
||||
"upstream of the wire")
|
||||
print(" (disc read, CPU, or stack), not the adapter")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user