Roflin
4e412c8b9b
Streaming measured 509 KB/s, BELOW the menu's 850 KB/s TCP dump, and the
arithmetic says exactly why.
I had this wrong: QUEUE_SIZE 2 is the number of queued GAME reads, not
concurrent network requests. fsp_read_queued() sends ONE fsp_get_file and
arms a 1 s alarm; fsp_pop_queue() issues the next only once the response
arrives. Streaming is therefore STRICTLY ONE REQUEST IN FLIGHT, so
throughput = bytes_per_reply / round_trip
The server clamped replies to FSP_SPACE (1024), giving 1024 B / ~2.01 ms =
509 KB/s -- the measured figure almost exactly. But the in-game client asks
for pmtu - 40 (1460 at a 1500 MTU) and accepts it; only fsplib has the 1024
limit, and fsplib never asks for more than that. So honouring the requested
size is safe for both clients, and at 1440 B (the 1500 MTU budget rounded
down to 32) the same round trip gives ~1.41x, about 700 KB/s.
Replies stay a multiple of 32 unless they are the file tail, because the
client computes data_size = MIN(requested, OSRoundDown32B(reply_len)) and a
non-aligned mid-file reply would be silently truncated.
Also instruments the round trip: with one request in flight the observed
interval IS the RTT, so the report now splits it into total and this server's
median service time and says SERVER-bound or console-bound. Without that,
"it is slow" has no owner. And the raw socket now binds ETH_P_IP rather than
ETH_P_ALL, so our own outgoing frames are no longer copied to userspace and
parsed in Python on the path we are trying to shorten.
T13 checks a 1460-byte request yields 1440 (32-aligned) while an fsplib-sized
1024 request still yields exactly 1024.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
255 lines
11 KiB
Python
Executable File
255 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Self-test for fsp-server.py, using a client that packs requests the way
|
|
Swiss's fsplib/tcpip.c do (path\\npassword\\0, 2-byte xlen, v2 checksum)."""
|
|
import os, socket, struct, subprocess, sys, tempfile, time
|
|
|
|
HSIZE = 12
|
|
CC_VERSION, CC_ERR, CC_GET_DIR, CC_GET_FILE, CC_STAT = 0x10, 0x40, 0x41, 0x42, 0x4D
|
|
PW = "alltheroms"
|
|
fails = []
|
|
|
|
|
|
def csum(pkt):
|
|
"""fsp_pkt_write / tcpip.c: INCLUDES the length. For requests."""
|
|
t = len(pkt) + sum(pkt)
|
|
t += t >> 8
|
|
return t & 0xFF
|
|
|
|
|
|
def fsplib_verify(pkt):
|
|
"""fsp_pkt_read, transcribed literally -- this is what Swiss's menu client
|
|
accepts, and it does NOT add the length:
|
|
|
|
mysum = -ptr[SUM]; for(i..recv_len) mysum += ptr[i];
|
|
mysum = (mysum + (mysum >> 8)) & 0xff;
|
|
ok if mysum == ptr[SUM]
|
|
|
|
⚠️ Verify replies with THIS, never with the server's own function: an
|
|
earlier version of this test used the server's symmetric checksum, passed
|
|
every assertion, and the real Swiss client rejected every reply."""
|
|
mysum = -pkt[1]
|
|
for b in pkt:
|
|
mysum += b
|
|
mysum = (mysum + (mysum >> 8)) & 0xFF
|
|
return mysum == pkt[1]
|
|
|
|
|
|
def req(cmd, seq, pos, path, pw=PW, extra=b""):
|
|
data = path.encode() + (b"\n" + pw.encode() if pw else b"") + b"\0"
|
|
body = struct.pack("!BBHHHI", cmd, 0, 0x1234, seq, len(data), pos) + data + extra
|
|
return body[:1] + bytes([csum(body)]) + body[2:]
|
|
|
|
|
|
def parse(pkt):
|
|
cmd, cs, key, seq, ln, pos = struct.unpack("!BBHHHI", pkt[:HSIZE])
|
|
# Replies are checked with FSPLIB's verifier, which is the only opinion
|
|
# that matters -- not with csum() above, which is the request formula.
|
|
return dict(cmd=cmd, seq=seq, pos=pos, data=pkt[HSIZE:HSIZE + ln],
|
|
csum_ok=fsplib_verify(pkt))
|
|
|
|
|
|
tmp = tempfile.mkdtemp()
|
|
PAYLOAD = bytes((i * 7 + 3) & 0xFF for i in range(8192))
|
|
open(os.path.join(tmp, "dvd.iso"), "wb").write(PAYLOAD)
|
|
os.mkdir(os.path.join(tmp, "subdir"))
|
|
|
|
srv = subprocess.Popen(
|
|
[sys.executable, "-I", "scripts/fsp-server.py", "--root", tmp,
|
|
"--password", PW, "--bind", "127.0.0.1", "--port", "2121",
|
|
"--quiet-reads", "--writable"],
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
|
|
time.sleep(1.0)
|
|
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
s.settimeout(3.0)
|
|
A = ("127.0.0.1", 2121)
|
|
|
|
|
|
def rt(pkt):
|
|
s.sendto(pkt, A)
|
|
raw = s.recv(2048)
|
|
if not fsplib_verify(raw):
|
|
fails.append(f"reply to cmd {pkt[0]:#04x} FAILS fsplib's verifier "
|
|
f"-- Swiss would drop it and retry")
|
|
return parse(raw)
|
|
|
|
try:
|
|
# T1 checksum both ways
|
|
r = rt(req(CC_VERSION, 1, 0, ""))
|
|
if r["cmd"] != CC_VERSION or not r["csum_ok"]:
|
|
fails.append(f"T1 VERSION: cmd {r['cmd']:#04x} csum_ok {r['csum_ok']}")
|
|
else:
|
|
print("T1 VERSION answered, reply checksum verifies")
|
|
|
|
# T2 STAT returns the real size
|
|
r = rt(req(CC_STAT, 2, 0, "/dvd.iso"))
|
|
t, sz, ty = struct.unpack("!IIB", r["data"][:9])
|
|
if sz != len(PAYLOAD) or ty != 0x01:
|
|
fails.append(f"T2 STAT size {sz} type {ty}, want {len(PAYLOAD)} / 1")
|
|
else:
|
|
print(f"T2 STAT -> size {sz}, type FILE")
|
|
|
|
# T3 GET_DIR lists the file and the subdir, and parses as fsplib does
|
|
r = rt(req(CC_GET_DIR, 3, 0, "/"))
|
|
d, i, found = r["data"], 0, {}
|
|
while i + 9 <= len(d):
|
|
ty = d[i + 8]
|
|
if ty == 0x00:
|
|
break
|
|
sz = struct.unpack("!I", d[i + 4:i + 8])[0]
|
|
end = d.index(b"\0", i + 9)
|
|
found[d[i + 9:end].decode()] = (ty, sz)
|
|
i = end + 1
|
|
i += (-i) % 4
|
|
if found.get("dvd.iso") != (0x01, len(PAYLOAD)) or found.get("subdir", (0,))[0] != 0x02:
|
|
fails.append(f"T3 GET_DIR entries wrong: {found}")
|
|
else:
|
|
print(f"T3 GET_DIR -> {found}")
|
|
if (len(r['data'])) % 4:
|
|
fails.append("T3 dir block not 4-byte aligned (fsplib rejects dirpos%4)")
|
|
|
|
# T4 second GET_DIR page must be EMPTY so the client stops looping
|
|
r2 = rt(req(CC_GET_DIR, 4, len(r["data"]), "/"))
|
|
if len(r2["data"]) != 0:
|
|
fails.append(f"T4 second dir page returned {len(r2['data'])} B, want 0")
|
|
else:
|
|
print("T4 second GET_DIR page empty -> client terminates")
|
|
|
|
# T5 GET_FILE: right bytes, and sequence+position ECHOED (the in-game
|
|
# client discards a reply whose seq or pos does not match)
|
|
for pos, want in ((0, 1024), (4096, 1024), (8000, 1024)):
|
|
r = rt(req(CC_GET_FILE, 10 + pos, pos, "/dvd.iso",
|
|
extra=struct.pack("!H", want)))
|
|
exp = PAYLOAD[pos:pos + want]
|
|
if r["data"] != exp:
|
|
fails.append(f"T5 GET_FILE pos={pos}: {len(r['data'])} B, "
|
|
f"want {len(exp)}, match={r['data']==exp}")
|
|
elif r["seq"] != 10 + pos or r["pos"] != pos:
|
|
fails.append(f"T5 seq/pos not echoed: seq {r['seq']} pos {r['pos']}")
|
|
print("T5 GET_FILE at 0 / 4096 / EOF-straddle: bytes exact, seq+pos echoed")
|
|
|
|
# T6 a 32-aligned request must give a 32-aligned reply, or the in-game
|
|
# client's OSRoundDown32B silently truncates and the transfer stalls
|
|
r = rt(req(CC_GET_FILE, 99, 0, "/dvd.iso", extra=struct.pack("!H", 1024)))
|
|
if len(r["data"]) % 32:
|
|
fails.append(f"T6 reply {len(r['data'])} B is not a multiple of 32")
|
|
else:
|
|
print(f"T6 1024 B request -> {len(r['data'])} B, 32-aligned")
|
|
|
|
# T7 wrong password must be refused
|
|
r = rt(req(CC_STAT, 7, 0, "/dvd.iso", pw="wrong"))
|
|
if r["cmd"] != CC_ERR:
|
|
fails.append(f"T7 wrong password accepted (cmd {r['cmd']:#04x})")
|
|
else:
|
|
print("T7 wrong password -> CC_ERR")
|
|
|
|
# T8 path traversal must not escape the root
|
|
r = rt(req(CC_STAT, 8, 0, "/../../etc/passwd"))
|
|
t, sz, ty = struct.unpack("!IIB", (r["data"] + b"\0" * 9)[:9])
|
|
if r["cmd"] != CC_ERR and ty != 0:
|
|
fails.append(f"T8 traversal escaped the root: type {ty} size {sz}")
|
|
else:
|
|
print("T8 ../../etc/passwd refused")
|
|
# T9 GET_PRO: flags go in the EXTRA region at buf[len], and pos must
|
|
# equal FSP_PRO_BYTES (1) or fsplib returns ENOMSG.
|
|
CC_GET_PRO = 0x47
|
|
r = rt(req(CC_GET_PRO, 20, 0, ""))
|
|
if r["cmd"] != CC_GET_PRO:
|
|
fails.append(f"T9 GET_PRO cmd {r['cmd']:#04x}")
|
|
elif r["pos"] != 1:
|
|
fails.append(f"T9 GET_PRO pos {r['pos']}, want 1 (FSP_PRO_BYTES) "
|
|
f"- fsplib returns ENOMSG otherwise")
|
|
else:
|
|
s.sendto(req(CC_GET_PRO, 21, 0, ""), A)
|
|
raw = s.recv(2048)
|
|
ln = struct.unpack("!H", raw[8:10])[0]
|
|
pro = raw[HSIZE + ln] # buf[len] -- the first extra byte
|
|
if not (pro & 0x10) or not (pro & 0x40):
|
|
fails.append(f"T9 GET_PRO flags {pro:#04x} lack GET|LIST")
|
|
elif not (pro & 0x01):
|
|
fails.append(f"T9 GET_PRO flags {pro:#04x} lack OWNER, but the "
|
|
f"server was started --writable, so Swiss would "
|
|
f"disable FEAT_PATCHES and refuse to boot")
|
|
else:
|
|
print(f"T9 GET_PRO -> pos=1, flags {pro:#04x} "
|
|
f"(OWNER|GET|LIST, --writable) at buf[len]")
|
|
|
|
# T10 a missing directory must be CC_ERR, so fsplib reports ENOENT and
|
|
# Swiss falls back from "swiss/patches" to "swiss" to "".
|
|
r = rt(req(CC_GET_PRO, 22, 0, "swiss/patches"))
|
|
if r["cmd"] != CC_ERR:
|
|
fails.append(f"T10 GET_PRO on a missing dir returned {r['cmd']:#04x}, "
|
|
f"want CC_ERR -- Swiss's fallback chain needs ENOENT")
|
|
else:
|
|
print("T10 GET_PRO 'swiss/patches' -> CC_ERR (Swiss falls back to root)")
|
|
|
|
# T13 reply size must follow the REQUEST, not FSP_SPACE. Clamping to
|
|
# 1024 is what held streaming to 509 KB/s: one request in flight means
|
|
# throughput is payload / RTT.
|
|
r = rt(req(CC_GET_FILE, 60, 0, "/dvd.iso", extra=struct.pack("!H", 1460)))
|
|
if len(r["data"]) != 1440:
|
|
fails.append(f"T13 asked 1460 -> got {len(r['data'])} B, want 1440 "
|
|
f"(1500 MTU budget, rounded down to 32)")
|
|
elif len(r["data"]) % 32:
|
|
fails.append(f"T13 reply {len(r['data'])} B not 32-aligned")
|
|
else:
|
|
print(f"T13 request 1460 -> 1440 B reply (32-aligned, 1.41x of 1024)")
|
|
|
|
# ...and a 1024-byte request (what fsplib asks) still gets exactly 1024,
|
|
# so the menu client is unaffected by the larger cap.
|
|
r = rt(req(CC_GET_FILE, 61, 0, "/dvd.iso", extra=struct.pack("!H", 1024)))
|
|
if len(r["data"]) != 1024:
|
|
fails.append(f"T13 fsplib-sized request got {len(r['data'])} B, "
|
|
f"want 1024 - must not exceed what was asked")
|
|
else:
|
|
print("T13 request 1024 -> 1024 B (fsplib unaffected)")
|
|
|
|
# T11 the write path Swiss needs for patches: MAKE_DIR, then UP_LOAD
|
|
# (which carries NO filename -- data is raw bytes at pos), then INSTALL
|
|
# to name the destination. Without this Swiss reports "failed to write
|
|
# file" and then "Failed to setup the file (too fragmented?)".
|
|
CC_UP_LOAD, CC_INSTALL, CC_MAKE_DIR = 0x43, 0x44, 0x49
|
|
r = rt(req(CC_MAKE_DIR, 30, 0, "swiss/patches"))
|
|
if r["cmd"] != CC_MAKE_DIR:
|
|
fails.append(f"T11 MAKE_DIR -> {r['cmd']:#04x}")
|
|
elif not os.path.isdir(os.path.join(tmp, "swiss/patches")):
|
|
fails.append("T11 MAKE_DIR did not create the directory")
|
|
else:
|
|
print("T11 MAKE_DIR swiss/patches")
|
|
|
|
BLOB = bytes((i * 11 + 5) & 0xFF for i in range(2048))
|
|
for off in range(0, len(BLOB), 1024):
|
|
chunk = BLOB[off:off + 1024]
|
|
# UP_LOAD: no filename, data is the bytes, pos is the offset
|
|
body = struct.pack("!BBHHHI", CC_UP_LOAD, 0, 0x1234, 40 + off // 1024,
|
|
len(chunk), off) + chunk
|
|
pkt = body[:1] + bytes([csum(body)]) + body[2:]
|
|
rr = rt(pkt)
|
|
if rr["cmd"] != CC_UP_LOAD or rr["pos"] != off:
|
|
fails.append(f"T11 UP_LOAD off={off} -> cmd {rr['cmd']:#04x} "
|
|
f"pos {rr['pos']} (pos IS checked for UP_LOAD)")
|
|
r = rt(req(CC_INSTALL, 45, 0, "swiss/patches/test.img"))
|
|
got = os.path.join(tmp, "swiss/patches/test.img")
|
|
if r["cmd"] != CC_INSTALL:
|
|
fails.append(f"T11 INSTALL -> {r['cmd']:#04x}")
|
|
elif not os.path.exists(got) or open(got, "rb").read() != BLOB:
|
|
fails.append("T11 INSTALL did not commit the uploaded bytes")
|
|
else:
|
|
print(f"T11 UP_LOAD 2x1024 B + INSTALL -> {len(BLOB)} B committed, "
|
|
f"byte-exact")
|
|
|
|
# T12 writes must stay inside --root
|
|
r = rt(req(CC_MAKE_DIR, 46, 0, "../escaped"))
|
|
if r["cmd"] != 0x40:
|
|
fails.append(f"T12 MAKE_DIR ../escaped -> {r['cmd']:#04x}, want CC_ERR")
|
|
elif os.path.exists(os.path.join(os.path.dirname(tmp), "escaped")):
|
|
fails.append("T12 MAKE_DIR escaped the root")
|
|
else:
|
|
print("T12 MAKE_DIR ../escaped refused")
|
|
|
|
finally:
|
|
srv.terminate()
|
|
|
|
print("\nFAILURES:\n " + "\n ".join(fails) if fails else
|
|
"\nAll fsp-server self-tests passed.")
|
|
sys.exit(1 if fails else 0)
|