#!/usr/bin/env python3 """A read-only FSP (File Service Protocol) server, for ROM streaming to Swiss. sudo ./scripts/fsp-server.py --root /workspace --password alltheroms Swiss streams games over FSP/UDP, not TCP -- see cube/patches/bba/tcpip.c. This serves a directory read-only so Swiss can browse it, stat a disc image and stream it while a game runs. It also reports the achieved read rate, which is the number we actually want to know. Protocol taken from Swiss's own two clients, not from memory: * cube/swiss/source/devices/fsp/fsplib.c (the menu: browse, stat, read) * cube/patches/bba/tcpip.c (in-game streaming reads) Wire format (v2, all multi-byte fields BIG-endian): cmd(1) checksum(1) key(2) sequence(2) len(2) pos(4) then data[len] extra[xlen] checksum: sum = total_size; sum += every byte (with the checksum byte zeroed); sum += sum >> 8; take the low 8 bits. password: the data field is "path\\npassword\\0" -- the '\\n' REPLACES the path's terminating NUL (fsplib buildfilename). dir entry: time(4) size(4) type(1) name(NUL-terminated), 4-byte aligned; type 0x00 END, 0x01 FILE, 0x02 DIR, 0x2A SKIP. stat reply: the same 9-byte time/size/type prefix. ⚠️ GET_FILE replies must be a MULTIPLE OF 32 BYTES unless they are the last piece of the request: the in-game client does data_size = MIN(requested, OSRoundDown32B(reply_len)) so a non-32-aligned reply is silently truncated down and the transfer stalls. """ import argparse import os import socket import struct import sys import time ETH_P_ALL, ETH_P_IP = 0x0003, 0x0800 def ip_csum(data): if len(data) % 2: data += b"\0" t = sum(struct.unpack("!%dH" % (len(data) // 2), data)) while t >> 16: t = (t & 0xFFFF) + (t >> 16) return (~t) & 0xFFFF class RawTransport: """Reply with raw ethernet frames, swapping the addresses from the request. ⚠️ This exists because the UDP-socket version did not work and the raw one did. With a normal socket the reply depends on the kernel having a route to the console, an ARP entry for it, and picking the right source address; mini-dhcpd.py bypassed all three with AF_PACKET and got a DHCP lease on the first try, while this server's replies never arrived at all (proven by fsplib never adopting the key from any reply). Mirroring the request's own addresses removes routing, ARP and source selection as variables in one go. """ def __init__(self, iface, port): self.port = port # ETH_P_IP, not ETH_P_ALL: with ALL, every frame on the link -- # including our own outgoing ones -- is copied to userspace and # parsed in Python, which adds latency to the very round trip we are # trying to shorten. self.s = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(ETH_P_IP)) self.s.bind((iface, 0)) self.our_mac = self.s.getsockname()[4][:6] def recv(self): """Return (payload, reply_ctx) for a UDP datagram aimed at our port.""" while True: frame = self.s.recv(65535) if len(frame) < 34: continue ip = frame[14:] ihl = (ip[0] & 0x0F) * 4 if ip[0] >> 4 != 4 or ip[9] != socket.IPPROTO_UDP: continue udp = ip[ihl:] if len(udp) < 8: continue sport, dport, ulen, _ = struct.unpack("!HHHH", udp[:8]) if dport != self.port: continue payload = udp[8:max(ulen, 8)] ctx = dict(dst_mac=frame[6:12], src_mac=frame[0:6], dst_ip=ip[12:16], src_ip=ip[16:20], dst_port=sport, src_port=dport) return payload, ctx def send(self, payload, ctx): udp = struct.pack("!HHHH", ctx["src_port"], ctx["dst_port"], 8 + len(payload), 0) + payload ip = struct.pack("!BBHHHBBH4s4s", 0x45, 0, 20 + len(udp), 0, 0, 64, socket.IPPROTO_UDP, 0, ctx["src_ip"], ctx["dst_ip"]) ip = ip[:10] + struct.pack("!H", ip_csum(ip)) + ip[12:] frame = (ctx["dst_mac"] + ctx["src_mac"] + struct.pack("!H", ETH_P_IP) + ip + udp) if len(frame) < 60: frame += b"\0" * (60 - len(frame)) self.s.send(frame) def peer(self, ctx): return (socket.inet_ntoa(ctx["dst_ip"]), ctx["dst_port"]) class UdpTransport: """Ordinary UDP socket. Simpler, but needs routing and ARP to cooperate.""" def __init__(self, bind, port): self.s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.s.bind((bind, port)) def recv(self): pkt, peer = self.s.recvfrom(FSP_SPACE + HSIZE + 64) return pkt, peer def send(self, payload, ctx): self.s.sendto(payload, ctx) def peer(self, ctx): return ctx CC_VERSION, CC_INFO = 0x10, 0x11 CC_ERR, CC_GET_DIR, CC_GET_FILE = 0x40, 0x41, 0x42 CC_BYE, CC_STAT = 0x4A, 0x4D CC_GET_PRO = 0x47 CC_UP_LOAD, CC_INSTALL, CC_DEL_FILE, CC_DEL_DIR, CC_MAKE_DIR = ( 0x43, 0x44, 0x45, 0x46, 0x49) # Directory protection bits (fsplib.h). Swiss uses these to decide whether # the device supports patches: # if ((pro & (MKDIR|ADD|DEL|GET)) == (MKDIR|ADD|DEL) || (pro & OWNER)) # features |= FEAT_PATCHES; emulable |= EMU_ETHERNET; # so a plain read-only answer makes Swiss DISABLE patch support here. PRO_OWNER, PRO_DEL, PRO_ADD, PRO_MKDIR = 0x01, 0x02, 0x04, 0x08 PRO_GET, PRO_README, PRO_LIST, PRO_RENAME = 0x10, 0x20, 0x40, 0x80 PRO_BYTES = 1 RDTYPE_END, RDTYPE_FILE, RDTYPE_DIR, RDTYPE_SKIP = 0x00, 0x01, 0x02, 0x2A HSIZE, FSP_SPACE = 12, 1024 SERVER_KEY = 0x1234 # ⚠️ FSP HAS TWO DIFFERENT CHECKSUM FORMULAS AND YOU MUST USE BOTH. # # fsplib's generator and its verifier disagree by the packet length: # # fsp_pkt_write (what a CLIENT sends): # checksum = sum(bytes, with the sum byte zeroed) + used # sum_byte = checksum + (checksum >> 8) # # fsp_pkt_read (what a CLIENT accepts): # mysum = sum(bytes) - sum_byte # i.e. no length term at all # ok if ((mysum + (mysum >> 8)) & 0xff) == sum_byte # # cube/patches/bba/tcpip.c's fsp_checksum matches the WRITE form (it starts # `sum = size`), and the in-game client never verifies replies at all. # # So: replies must be built the way fsp_pkt_read VERIFIES -- without the # length -- or Swiss's menu client silently drops every one. That is exactly # what happened: fsplib retried, made no progress, and Swiss hung on # "setting up device" while the server logged perfectly good-looking replies. def checksum_reply(pkt): """Match fsp_pkt_read's verifier: no length term. pkt has sum byte = 0.""" total = sum(pkt) total += total >> 8 return total & 0xFF def checksum_request(pkt): """Match fsp_pkt_write / tcpip.c: includes the length. pkt has sum = 0.""" total = len(pkt) + sum(pkt) total += total >> 8 return total & 0xFF def pack(cmd, key, seq, pos, data=b"", extra=b""): body = struct.pack("!BBHHHI", cmd, 0, key, seq, len(data), pos) + data + extra return body[:1] + bytes([checksum_reply(body)]) + body[2:] def unpack(pkt): if len(pkt) < HSIZE: return None cmd, csum, key, seq, ln, pos = struct.unpack("!BBHHHI", pkt[:HSIZE]) if len(pkt) < HSIZE + ln: return None data = pkt[HSIZE:HSIZE + ln] extra = pkt[HSIZE + ln:] zeroed = pkt[:1] + b"\0" + pkt[2:] # Accept EITHER formula on the way in: fsplib and tcpip.c both send the # with-length form, but being permissive costs nothing and a rejected # request is much harder to diagnose than a tolerated one. ok = csum in (checksum_request(zeroed), checksum_reply(zeroed)) return dict(cmd=cmd, key=key, seq=seq, pos=pos, data=data, extra=extra, csum_ok=ok) def split_path(data, password): """Return (path, password_ok). fsplib sends "path\\npassword\\0".""" s = data.split(b"\0")[0].decode("latin-1", "replace") if "\n" in s: path, _, pw = s.partition("\n") else: path, pw = s, "" return path, (password is None or pw == password) def safe_join(root, path): p = os.path.normpath(os.path.join(root, path.lstrip("/"))) if not (p == root or p.startswith(root + os.sep)): return None return p def dir_block(real): """One FSP directory block: entries 4-byte aligned, then an END marker.""" out = bytearray() try: names = sorted(os.listdir(real)) except OSError: names = [] for nm in names: full = os.path.join(real, nm) try: st = os.stat(full) except OSError: continue isdir = os.path.isdir(full) ent = struct.pack("!IIB", int(st.st_mtime) & 0xFFFFFFFF, st.st_size & 0xFFFFFFFF, RDTYPE_DIR if isdir else RDTYPE_FILE) ent += nm.encode("latin-1", "replace") + b"\0" while len(ent) % 4: ent += b"\0" # Keep the whole listing inside one block: the client treats # "fewer than 9 bytes left in this block" as SKIP. if len(out) + len(ent) + 9 > FSP_SPACE: break out += ent out += struct.pack("!IIB", 0, 0, RDTYPE_END) while len(out) % 4: out += b"\0" return bytes(out) def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--root", default=".", help="directory to serve") ap.add_argument("--password", default=None, help="require this password (Swiss sends it with every request)") ap.add_argument("--iface", default=None, help="reply with RAW ethernet frames on this interface, " "mirroring the request's own addresses. Use this if " "replies are not arriving: it removes routing, ARP " "and source-address selection from the picture, " "which is what made mini-dhcpd work.") ap.add_argument("--bind", default="0.0.0.0") ap.add_argument("--port", type=int, default=21) ap.add_argument("--max-reply", type=int, default=1440, help="largest GET_FILE payload, 32-ALIGNED (default 1440 " "= 1500 MTU - 20 IP - 8 UDP - 12 FSP, rounded down " "to 32). Streaming is strictly ONE request in " "flight, so throughput is this divided by the round " "trip: at ~2 ms, 1024 B gives ~509 KB/s and 1440 B " "gives ~700 KB/s. fsplib never asks for more than " "its own FSP_SPACE (1024) so the menu is unaffected.") ap.add_argument("--quiet-reads", action="store_true", help="only log the rate, not every read") ap.add_argument("--trace", action="store_true", help="hexdump every request and reply, with the sequence " "split into transaction id and retry counter") ap.add_argument("--writable", "--claim-owner", dest="writable", action="store_true", help="allow writes (UP_LOAD / INSTALL / MAKE_DIR / " "DEL_FILE) and report FSP_DIR_OWNER on GET_PRO so " "Swiss enables FEAT_PATCHES and EMU_ETHERNET. " "Booting a game needs this: Swiss writes patch files " "to swiss/patches/ and fails with 'failed to write " "file' otherwise. Writes are confined to --root.") a = ap.parse_args() root = os.path.realpath(a.root) try: if a.iface: tr = RawTransport(a.iface, a.port) else: tr = UdpTransport(a.bind, a.port) except PermissionError: sys.exit(f"needs root (port {a.port} / AF_PACKET); run with sudo") s = tr # everything below talks through the transport where = (f"raw on {a.iface}" if a.iface else f"{a.bind}") + f":{a.port}" print(f"fsp-server {where} root {root}") print(f" password {'set' if a.password else 'NONE'}") print(f" {'WRITABLE (patches allowed)' if a.writable else 'read-only'}") print(f" point Swiss's FSP device at this host and port\n") def reply(ctx_, pkt_, what): tr.send(pkt_, ctx_) if a.trace: print(f" -> {len(pkt_):4d} B {what}") print(f" {pkt_[:48].hex()}") uploads = {} # peer -> staged bytes, committed by CC_INSTALL service = [] # per-request service time, to split the round trip served = 0 t_first = None t_report = time.time() reads = 0 fh_path, fh = None, None while True: pkt, ctx = tr.recv() peer = tr.peer(ctx) r = unpack(pkt) if r is None: print(f" short packet from {peer[0]} ({len(pkt)} B)") continue if not r["csum_ok"]: print(f" ⚠️ bad checksum from {peer[0]}, cmd {r['cmd']:#04x}") cmd, seq, pos = r["cmd"], r["seq"], r["pos"] # fsp_transaction puts a RANDOM value in seq & 0xfff8 per transaction # and the RETRY COUNTER in seq & 0x7. So repeated requests with the # same high bits and climbing low bits mean our reply was rejected or # lost; different high bits mean the client gave up and started a new # transaction. Without this you cannot tell those apart, which is # exactly the hole that made "stuck on setting up device" opaque. txn, try_n = seq & 0xFFF8, seq & 0x7 tag = f"seq={seq:#06x}[txn {txn:#06x} try {try_n}]" if a.trace: print(f" <- {len(pkt):4d} B cmd {cmd:#04x} {tag} " f"key={r['key']:#06x} pos={pos} len={len(r['data'])} " f"xlen={len(r['extra'])} from {peer[0]}:{peer[1]}") print(f" {pkt[:48].hex()}") if cmd == CC_VERSION: reply(ctx, pack(CC_VERSION, SERVER_KEY, seq, 0, b"fsp-server (rebbarb)\0"), "VERSION") print(f"[{peer[0]}] VERSION") continue if cmd == CC_BYE: reply(ctx, pack(CC_BYE, SERVER_KEY, seq, 0), "BYE") print(f"[{peer[0]}] BYE") continue # ⚠️ UP_LOAD must be handled BEFORE the path/password parse: its # data field is RAW FILE BYTES, not "path\npassword\0", so running # it through split_path() rejects every chunk as a bad password. # fsplib only calls buildfilename() on the read branch; the password # is checked on CC_INSTALL, which does carry the path. if cmd == CC_UP_LOAD: # ⚠️ UP_LOAD carries NO FILENAME -- fsplib only calls # buildfilename() on the read branch. The data is raw file bytes # at `pos`, written to a per-session temp file, and CC_INSTALL # names the destination afterwards. `pos` IS checked by # fsp_transaction for UP_LOAD, so it must be echoed. key = peer up = uploads.get(key) if up is None: up = uploads[key] = bytearray() end = pos + len(r["data"]) if len(up) < end: up.extend(b"\0" * (end - len(up))) up[pos:end] = r["data"] reply(ctx, pack(CC_UP_LOAD, SERVER_KEY, seq, pos), "UP_LOAD") if len(r["data"]) == 0: print(f"[{peer[0]}] UP_LOAD eof, {len(up)} B staged") continue path, pw_ok = split_path(r["data"], a.password) real = safe_join(root, path) if not pw_ok: print(f"[{peer[0]}] {cmd:#04x} {path!r} -> WRONG PASSWORD") reply(ctx, pack(CC_ERR, SERVER_KEY, seq, pos, b"bad password\0"), "ERR") continue if real is None: reply(ctx, pack(CC_ERR, SERVER_KEY, seq, pos, b"bad path\0"), "ERR") continue if cmd == CC_STAT: if os.path.exists(real): st = os.stat(real) body = struct.pack("!IIB", int(st.st_mtime) & 0xFFFFFFFF, st.st_size & 0xFFFFFFFF, RDTYPE_DIR if os.path.isdir(real) else RDTYPE_FILE) print(f"[{peer[0]}] STAT {path!r} -> {st.st_size} B") else: body = struct.pack("!IIB", 0, 0, 0) # type 0 == ENOENT print(f"[{peer[0]}] STAT {path!r} -> not found") reply(ctx, pack(CC_STAT, SERVER_KEY, seq, pos, body), "STAT") continue if cmd == CC_GET_PRO: # Reply shape (fsplib fsp_getpro): pos MUST equal FSP_PRO_BYTES, # and the flags byte sits at buf[len] -- i.e. the first byte of # the EXTRA region, after the echoed name, not inside data. # A non-existent directory must be CC_ERR so fsplib reports # ENOENT: Swiss probes "swiss/patches", then "swiss", then "". if not os.path.isdir(real): reply(ctx, pack(CC_ERR, SERVER_KEY, seq, pos, b"no such dir\0"), "ERR") print(f"[{peer[0]}] GET_PRO {path!r} {tag} -> ENOENT (expected; " f"Swiss falls back)") continue pro = PRO_GET | PRO_LIST | (PRO_OWNER if a.writable else 0) reply(ctx, pack(CC_GET_PRO, SERVER_KEY, seq, PRO_BYTES, path.encode("latin-1", "replace"), bytes([pro])), "GET_PRO") print(f"[{peer[0]}] GET_PRO {path!r} {tag} -> {pro:#04x}" + (" (OWNER: Swiss will enable patches)" if a.writable else " (read-only: Swiss disables patches here)")) continue if cmd in (CC_UP_LOAD, CC_INSTALL, CC_MAKE_DIR, CC_DEL_FILE, CC_DEL_DIR) and not a.writable: print(f"[{peer[0]}] {cmd:#04x} refused: server is read-only " f"(pass --writable)") reply(ctx, pack(CC_ERR, SERVER_KEY, seq, pos, b"read-only server\0"), "ERR") continue if cmd == CC_INSTALL: up = uploads.pop(peer, None) if up is None: reply(ctx, pack(CC_ERR, SERVER_KEY, seq, pos, b"nothing uploaded\0"), "ERR") continue try: os.makedirs(os.path.dirname(real) or root, exist_ok=True) with open(real, "wb") as fo: fo.write(up) reply(ctx, pack(CC_INSTALL, SERVER_KEY, seq, pos), "INSTALL") print(f"[{peer[0]}] INSTALL {path!r} <- {len(up)} B") except OSError as e: reply(ctx, pack(CC_ERR, SERVER_KEY, seq, pos, f"{e}\0".encode()), "ERR") print(f"[{peer[0]}] INSTALL {path!r} FAILED: {e}") continue if cmd == CC_MAKE_DIR: try: os.makedirs(real, exist_ok=True) reply(ctx, pack(CC_MAKE_DIR, SERVER_KEY, seq, pos), "MAKE_DIR") print(f"[{peer[0]}] MAKE_DIR {path!r}") except OSError as e: reply(ctx, pack(CC_ERR, SERVER_KEY, seq, pos, f"{e}\0".encode()), "ERR") continue if cmd in (CC_DEL_FILE, CC_DEL_DIR): try: os.rmdir(real) if cmd == CC_DEL_DIR else os.remove(real) reply(ctx, pack(cmd, SERVER_KEY, seq, pos), "DEL") print(f"[{peer[0]}] DEL {path!r}") except OSError as e: reply(ctx, pack(CC_ERR, SERVER_KEY, seq, pos, f"{e}\0".encode()), "ERR") continue if cmd == CC_GET_DIR: if not os.path.isdir(real): reply(ctx, pack(CC_ERR, SERVER_KEY, seq, pos, b"not a dir\0"), "ERR") continue blk = dir_block(real) # The client reads blocks until one comes back shorter than the # first; so the whole listing, then an empty reply. out = blk if pos == 0 else b"" reply(ctx, pack(CC_GET_DIR, SERVER_KEY, seq, pos, out), "GET_DIR") print(f"[{peer[0]}] GET_DIR {path!r} pos={pos} -> {len(out)} B") continue if cmd == CC_GET_FILE: want = struct.unpack("!H", r["extra"][:2])[0] if len(r["extra"]) >= 2 else 1024 # ⚠️ Do NOT clamp to FSP_SPACE (1024). The in-game client asks # for pmtu - 40 (1460 at a 1500 MTU) and accepts it; only # fsplib has the 1024 limit, and fsplib never asks for more than # that, so honouring the request is safe for both. Clamping here # is what held streaming to 509 KB/s: one request in flight means # throughput is payload / RTT, nothing else. want = min(want, a.max_reply) # Keep it a multiple of 32 unless it is the file tail: the client # does data_size = MIN(requested, OSRoundDown32B(reply_len)), so a # non-aligned mid-file reply is silently truncated. if want >= 32: want &= ~31 if fh_path != real: if fh: fh.close() try: fh = open(real, "rb") fh_path = real except OSError as e: reply(ctx, pack(CC_ERR, SERVER_KEY, seq, pos, f"{e}\0".encode()), "ERR") continue t_req = time.time() fh.seek(pos) data = fh.read(want) reply(ctx, pack(CC_GET_FILE, SERVER_KEY, seq, pos, data), "GET_FILE") served += len(data) reads += 1 now = time.time() service.append(now - t_req) if t_first is None: t_first = now print(f"[{peer[0]}] GET_FILE {path!r} streaming started") if not a.quiet_reads and reads <= 3: print(f"[{peer[0]}] GET_FILE {path!r} pos={pos} " f"want={want} -> {len(data)} B") if now - t_report >= 2.0: el = max(now - t_first, 1e-9) svc = sorted(service) med = svc[len(svc) // 2] * 1000 if svc else 0.0 rtt = (el / reads) * 1000 if reads else 0.0 print(f" streaming {served/el/1024:7.0f} KB/s " f"{served/2**20:8.1f} MiB in {el:6.1f}s " f"{reads/el:5.0f} reads/s {len(data)} B/reply") # One request in flight, so the observed interval IS the round # trip. Splitting it shows whether WE are the slow part. print(f" round trip {rtt:5.2f} ms, of which this server " f"{med:5.2f} ms (median) -> " f"{'SERVER-bound' if med > rtt/2 else 'console-bound'}") service.clear() t_report = now continue print(f"[{peer[0]}] unsupported command {cmd:#04x} {path!r}") reply(ctx, pack(CC_ERR, SERVER_KEY, seq, pos, b"unsupported\0"), "ERR") if __name__ == "__main__": try: main() except KeyboardInterrupt: print("\nstopped")