Read and patch UE3 packages and soundbanks

NOTES.md

@@ -16,8 +16,10 @@
   (music heard in game ncc 1.0, reverts byte-identical, speech line apply/revert byte-identical).
 - Mission 3 (2026-10-09, brief: `docs/history/mission3_audio_text_prompt.md`): full-game audio, then subtitle
   and menu text editing. Plan (one commit per stage; a stage = code + tests + docs + this status):
-  1. [ ] `ackit/lzo.py`, `ackit/upk.py` (UE3 package summary, compressed chunks, block-level read/patch),
+  1. [x] `ackit/lzo.py`, `ackit/upk.py` (UE3 package summary, compressed chunks, block-level read/patch),
-     `ackit/bnk.py` (bank parsing: DIDX/DATA/HIRC sources, prefetch). Unit + real-game read-only tests.
+     `ackit/bnk.py` (bank parsing: DIDX/DATA/HIRC sources). Docs: formats/upk.md, banks.md. Findings: pure-Python
+     LZO ~47 MB/s (== decompress.exe on every block); 15,800 media IDs in 96,011 copies, 396 are prefetch heads
+     of WAD sounds (2,101 copies) -> 15,404 in-memory sounds; copies identical except 1C898731.
   2. [ ] Catalog of in-memory sounds (archive `BANK`): build from loose banks + packages, location index,
      names/categories, prefetch handling; `ackit verify --catalog` cross-check (missing/extra/duplicate/
      character/category/language) + test.

ackit/__init__.py

@@ -1,7 +1,8 @@
 """ackit - Batman: Arkham City (PC) audio toolkit.
 
 Library layers (low -> high):
-  formats   wad, pack, wem, ogg, bits, codebooks, wwvorbis, allocsize, builder, convert, known_setups
+  formats   wad, pack, bnk (soundbanks), upk + lzo (UE3 packages), wem, ogg, bits, codebooks, wwvorbis,
+            allocsize, builder, convert, known_setups
   audio     audio (load/convert/decode), encoder (PCM -> game WEM)
   game      paths, game (install discovery), catalog + classify (names/categories/characters),
             inject (WAD writing), originals (store), workspace (apply/revert), modpack (share)

ackit/bnk.py

@@ -0,0 +1,108 @@
+"""Wwise soundbanks (bank version 56): chunks, in-memory media (DIDX/DATA) and HIRC sound sources.
+
+Works on a buffer that may hold a bank anywhere (a loose .bnk file, a BKPK pack's embedded bank, or
+an uncompressed .upk package with many banks). Offsets are relative to the start of the bank.
+Format: docs/formats/banks.md.
+"""
+import re
+import struct
+from collections import namedtuple
+
+CHUNKS = (b"BKHD", b"DIDX", b"DATA", b"HIRC", b"STID", b"STMG", b"ENVS", b"FXPR")
+SOUND = 2                         # HIRC object type of a Sound
+IN_MEMORY, STREAMED, PREFETCH = 0, 1, 2
+
+Media = namedtuple("Media", "id off size")                     # off: from bank start
+Source = namedtuple("Source", "object stream media bank off size pos")  # pos: of `off` field, from bank start
+
+
+class BankError(ValueError):
+    pass
+
+
+class Bank:
+    def __init__(self, buf, at=0):
+        self.buf, self.at = buf, at
+        self.chunks = {}
+        p = at
+        while p + 8 <= len(buf) and bytes(buf[p:p + 4]) in CHUNKS:
+            tag = bytes(buf[p:p + 4]).decode()
+            n = struct.unpack_from("<I", buf, p + 4)[0]
+            if p + 8 + n > len(buf) or tag in self.chunks:
+                break
+            self.chunks[tag] = (p + 8 - at, n)
+            p += 8 + n
+        if "BKHD" not in self.chunks or self.chunks["BKHD"][1] < 8:
+            raise BankError("no bank at 0x%X" % at)
+        self.size = p - at
+        o = at + self.chunks["BKHD"][0]
+        self.version, self.id = struct.unpack_from("<II", buf, o)
+
+    def _abs(self, tag):
+        o, n = self.chunks[tag]
+        return self.at + o, n
+
+    def media(self):
+        """In-memory WEMs: [Media(id, offset from bank start, size)]."""
+        if "DIDX" not in self.chunks or "DATA" not in self.chunks:
+            return []
+        o, n = self._abs("DIDX")
+        data = self.chunks["DATA"][0]
+        out = []
+        for i in range(n // 12):
+            mid, off, size = struct.unpack_from("<III", self.buf, o + 12 * i)
+            out.append(Media(mid, data + off, size))
+        return out
+
+    def sources(self):
+        """Sources of the HIRC Sound objects: [Source]. In-memory sources carry offset/size."""
+        if "HIRC" not in self.chunks:
+            return []
+        o, n = self._abs("HIRC")
+        end = o + n
+        count = struct.unpack_from("<I", self.buf, o)[0]
+        p = o + 4
+        out = []
+        for _ in range(count):
+            if p + 5 > end:
+                raise BankError("HIRC overruns its chunk in bank %08X" % self.id)
+            t = self.buf[p]
+            ln = struct.unpack_from("<I", self.buf, p + 1)[0]
+            if t == SOUND and ln >= 28:
+                obj, _plugin, stream, mid, bank = struct.unpack_from("<5I", self.buf, p + 5)
+                off, size = struct.unpack_from("<II", self.buf, p + 25)
+                out.append(Source(obj, stream, mid, bank, off, size, p + 25 - self.at))
+            p += 5 + ln
+        return out
+
+    def names(self):
+        """STID: {bank id: name}."""
+        if "STID" not in self.chunks:
+            return {}
+        o, n = self._abs("STID")
+        count = struct.unpack_from("<I", self.buf, o + 4)[0]
+        p = o + 8
+        out = {}
+        for _ in range(count):
+            bid, ln = struct.unpack_from("<IB", self.buf, p)
+            out[bid] = bytes(self.buf[p + 5:p + 5 + ln]).decode("latin-1")
+            p += 5 + ln
+        return out
+
+
+def find_banks(buf, start=0):
+    """Every bank in `buf` (e.g. an uncompressed package): yields Bank."""
+    for m in re.finditer(b"BKHD", buf[start:] if start else buf):
+        try:
+            yield Bank(buf, m.start() + start)
+        except (BankError, struct.error):
+            continue
+
+
+def fnv1(name):
+    """Wwise ID of a name (bank, event, ...): FNV-1 32 of the lower-cased name."""
+    h = 2166136261
+    for c in name.lower().encode("utf-8"):
+        h = (h * 16777619) & 0xFFFFFFFF
+        h ^= c
+    return h

ackit/lzo.py

@@ -0,0 +1,142 @@
+"""LZO1X, as used by the game's compressed UE3 packages (docs/formats/upk.md).
+
+`decompress` is a pure-Python LZO1X decoder (same semantics as lzo1x_decompress_safe; the packages
+use 128 KiB blocks, so speed is fine for block-level access). `compress_stored` writes a valid LZO1X
+stream that holds the data as one literal run (no matches): bigger than real compression, but any
+LZO1X decoder (the game's included) reads it back exactly, and nothing else has to change.
+"""
+M2_MAX_OFFSET = 0x0800
+
+
+class LzoError(ValueError):
+    pass
+
+
+def _copy_match(out, dist, n):
+    start = len(out) - dist
+    if start < 0:
+        raise LzoError("match distance beyond output start")
+    if dist >= n:
+        out += out[start:start + n]
+    else:  # overlapping: repeat the last `dist` bytes
+        pat = bytes(out[start:])
+        out += (pat * (n // dist + 1))[:n]
+
+
+def decompress(src, out_len=None):
+    """LZO1X block -> bytes. `out_len` (if given) is checked."""
+    src = bytes(src)
+    n = len(src)
+    out = bytearray()
+    ip = 0
+
+    def ext(t, base):
+        nonlocal ip
+        while src[ip] == 0:
+            t += 255
+            ip += 1
+        t += base + src[ip]
+        ip += 1
+        return t
+
+    try:
+        state = "loop"
+        if src[0] > 17:
+            t = src[0] - 17
+            ip = 1
+            if t < 4:
+                state = "match_next"
+            else:
+                out += src[ip:ip + t]
+                ip += t
+                state = "first_literal_run"
+        while True:
+            if state == "loop":
+                t = src[ip]
+                ip += 1
+                if t >= 16:
+                    state = "match"
+                    continue
+                if t == 0:
+                    t = ext(0, 15)
+                out += src[ip:ip + t + 3]
+                ip += t + 3
+                state = "first_literal_run"
+                continue
+            if state == "first_literal_run":
+                t = src[ip]
+                ip += 1
+                if t >= 16:
+                    state = "match"
+                    continue
+                dist = 1 + M2_MAX_OFFSET + (t >> 2) + (src[ip] << 2)
+                ip += 1
+                _copy_match(out, dist, 3)
+                state = "match_done"
+                continue
+            if state == "match_next":
+                out += src[ip:ip + t]
+                ip += t
+                t = src[ip]
+                ip += 1
+                state = "match"
+                continue
+            if state == "match":
+                if t >= 64:                       # M2
+                    dist = 1 + ((t >> 2) & 7) + (src[ip] << 3)
+                    ip += 1
+                    _copy_match(out, dist, (t >> 5) + 1)
+                elif t >= 32:                     # M3
+                    t &= 31
+                    if t == 0:
+                        t = ext(0, 31)
+                    dist = 1 + ((src[ip] | (src[ip + 1] << 8)) >> 2)
+                    ip += 2
+                    _copy_match(out, dist, t + 2)
+                elif t >= 16:                     # M4 (distance 0 = end of stream)
+                    hi = (t & 8) << 11
+                    t &= 7
+                    if t == 0:
+                        t = ext(0, 7)
+                    dist = hi + ((src[ip] | (src[ip + 1] << 8)) >> 2)
+                    ip += 2
+                    if dist == 0:
+                        break
+                    _copy_match(out, dist + 0x4000, t + 2)
+                else:                             # M1 after a match: 2 bytes
+                    dist = 1 + (t >> 2) + (src[ip] << 2)
+                    ip += 1
+                    _copy_match(out, dist, 2)
+                state = "match_done"
+                continue
+            # match_done: the low 2 bits of the instruction's last byte = trailing literals
+            t = src[ip - 2] & 3
+            if t == 0:
+                state = "loop"
+            else:
+                state = "match_next"
+    except IndexError:
+        raise LzoError("truncated LZO stream") from None
+    if ip != n:
+        raise LzoError("%d bytes after the end of the LZO stream" % (n - ip))
+    if out_len is not None and len(out) != out_len:
+        raise LzoError("LZO block decoded to %d bytes, expected %d" % (len(out), out_len))
+    return bytes(out)
+
+
+EOF = b"\x11\x00\x00"
+
+
+def compress_stored(data):
+    """bytes -> LZO1X stream made of one literal run (decodes back to `data`)."""
+    data = bytes(data)
+    n = len(data)
+    if n == 0:
+        return EOF
+    if n <= 238:
+        return bytes([n + 17]) + data + EOF
+    # long literal run: 0x00, k zero bytes (255 each), last byte; length = 18 + 255*k + last
+    r = n - 18
+    k = (r - 1) // 255
+    last = r - 255 * k
+    return b"\x00" + b"\x00" * k + bytes([last]) + data + EOF

ackit/upk.py

@@ -0,0 +1,250 @@
+"""The game's UE3 packages (.upk): summary, LZO-compressed chunks, block-level reads and patches.
+
+Offsets used by the rest of ackit are positions in the *uncompressed* package (what Gildor's
+decompress.exe writes), so a sound's location does not depend on how the file is compressed.
+Format and rules: docs/formats/upk.md.
+
+Patching rewrites only the blocks that contain changed bytes (stored as one LZO literal run, see
+lzo.compress_stored) and moves the chunks after them; every other byte of the file stays identical.
+"""
+import struct
+from collections import OrderedDict
+
+from . import lzo
+
+TAG = 0x9E2A83C1
+COMPRESS_LZO = 2
+
+
+class PackageError(ValueError):
+    pass
+
+
+class Block:
+    __slots__ = ("chunk", "uoff", "usize", "coff", "csize")
+
+    def __init__(self, chunk, uoff, usize, coff, csize):
+        self.chunk, self.uoff, self.usize, self.coff, self.csize = chunk, uoff, usize, coff, csize
+
+
+class Package:
+    """Read access to one package file. `blocks` covers the uncompressed range [header_size, size)."""
+
+    def __init__(self, path, cache_blocks=8):
+        self.path = path
+        with open(path, "rb") as f:
+            head = f.read(0x10000)
+            self._parse_summary(head, f)
+            self.file_size = f.seek(0, 2)
+        self._cache = OrderedDict()
+        self._cache_n = cache_blocks
+
+    # summary -----------------------------------------------------------------------------------
+    def _parse_summary(self, b, f):
+        if len(b) < 64 or struct.unpack_from("<I", b, 0)[0] != TAG:
+            raise PackageError("%s is not an Unreal package" % self.path)
+        self.version, self.licensee = struct.unpack_from("<HH", b, 4)
+        o = 12
+        n = struct.unpack_from("<i", b, o)[0]
+        o += 4 + max(n, 0)
+        # PackageFlags, NameCount/Offset, ExportCount/Offset, ImportCount/Offset, DependsOffset,
+        # ImportExportGuidsOffset, ImportGuidsCount, ExportGuidsCount, ThumbnailTableOffset
+        f12 = struct.unpack_from("<12i", b, o)
+        self.name_count, self.name_offset = f12[1], f12[2]
+        self.export_count, self.export_offset = f12[3], f12[4]
+        o += 48
+        self.guid = b[o:o + 16]
+        o += 16
+        ng = struct.unpack_from("<i", b, o)[0]
+        o += 4 + 12 * ng
+        self.engine, self.cooker, self.compression = struct.unpack_from("<3I", b, o)
+        o += 12
+        nc = struct.unpack_from("<i", b, o)[0]
+        o += 4
+        if nc and len(b) < o + 16 * nc:
+            f.seek(0)
+            b = f.read(o + 16 * nc)
+        self.chunk_table_offset = o
+        self.chunks = [list(struct.unpack_from("<4i", b, o + 16 * i)) for i in range(nc)]  # uoff usize coff csize
+        if nc and self.compression != COMPRESS_LZO:
+            raise PackageError("%s: unsupported compression %d" % (self.path, self.compression))
+        self.blocks = []
+        if not nc:
+            return
+        for ci, (uoff, usize, coff, csize) in enumerate(self.chunks):
+            f.seek(coff)
+            h = f.read(16)
+            tag, bsize, sum_c, sum_u = struct.unpack("<4I", h)
+            if tag != TAG or sum_u != usize:
+                raise PackageError("%s: bad chunk header at 0x%X" % (self.path, coff))
+            nb = (sum_u + bsize - 1) // bsize
+            tbl = f.read(8 * nb)
+            p = coff + 16 + 8 * nb
+            u = uoff
+            for i in range(nb):
+                c, uu = struct.unpack_from("<2I", tbl, 8 * i)
+                self.blocks.append(Block(ci, u, uu, p, c))
+                p += c
+                u += uu
+            if p != coff + csize or u != uoff + usize:
+                raise PackageError("%s: inconsistent chunk %d" % (self.path, ci))
+        self.block_size = bsize
+
+    @property
+    def compressed(self):
+        return bool(self.chunks)
+
+    @property
+    def header_size(self):
+        """Uncompressed size of the summary area (= start of the first chunk's data)."""
+        return self.chunks[0][0] if self.chunks else 0
+
+    @property
+    def size(self):
+        """Uncompressed package size."""
+        if not self.chunks:
+            return self.file_size
+        return self.chunks[-1][0] + self.chunks[-1][1]
+
+    def fingerprint(self):
+        """Cheap identity of the package contents: guid + chunk table (changes with any re-cook/patch)."""
+        import hashlib
+        h = hashlib.sha1(self.guid)
+        h.update(struct.pack("<q", self.file_size))
+        for c in self.chunks:
+            h.update(struct.pack("<4i", *c))
+        return h.hexdigest()[:16]
+
+    # reading -----------------------------------------------------------------------------------
+    def _block(self, i, f=None):
+        if i in self._cache:
+            self._cache.move_to_end(i)
+            return self._cache[i]
+        bl = self.blocks[i]
+        own = f is None
+        f = f or open(self.path, "rb")
+        try:
+            f.seek(bl.coff)
+            data = lzo.decompress(f.read(bl.csize), bl.usize)
+        finally:
+            if own:
+                f.close()
+        self._cache[i] = data
+        if len(self._cache) > self._cache_n:
+            self._cache.popitem(last=False)
+        return data
+
+    def _block_index(self, off):
+        lo, hi = 0, len(self.blocks) - 1
+        while lo <= hi:
+            mid = (lo + hi) // 2
+            bl = self.blocks[mid]
+            if off < bl.uoff:
+                hi = mid - 1
+            elif off >= bl.uoff + bl.usize:
+                lo = mid + 1
+            else:
+                return mid
+        raise PackageError("offset 0x%X is outside the package data" % off)
+
+    def read(self, off, n):
+        """Uncompressed bytes [off, off+n)."""
+        if not self.chunks:
+            with open(self.path, "rb") as f:
+                f.seek(off)
+                return f.read(n)
+        if off < self.header_size:
+            raise PackageError("reading the package summary area is not supported")
+        out = bytearray()
+        i = self._block_index(off)
+        with open(self.path, "rb") as f:
+            while len(out) < n:
+                bl = self.blocks[i]
+                d = self._block(i, f)
+                s = off + len(out) - bl.uoff
+                out += d[s:s + n - len(out)]
+                i += 1
+        return bytes(out)
+
+    def iter_data(self):
+        """(uncompressed offset, bytes) per block, in order (whole-package scans)."""
+        if not self.chunks:
+            with open(self.path, "rb") as f:
+                yield 0, f.read()
+            return
+        with open(self.path, "rb") as f:
+            for bl in self.blocks:
+                f.seek(bl.coff)
+                yield bl.uoff, lzo.decompress(f.read(bl.csize), bl.usize)
+
+    def read_all(self):
+        """The whole uncompressed data area as one bytes object (offset `data_start`)."""
+        parts = list(self.iter_data())
+        return (parts[0][0] if parts else 0), b"".join(p for _, p in parts)
+
+    # patching ----------------------------------------------------------------------------------
+    def patched(self, edits):
+        """edits: [(uncompressed offset, bytes)] (same length as what they overwrite) -> new file bytes.
+
+        Uncompressed packages are patched in place. For compressed ones the touched blocks are
+        re-encoded and the chunk headers and the summary's chunk table are updated."""
+        with open(self.path, "rb") as f:
+            raw = f.read()
+        if not self.chunks:
+            out = bytearray(raw)
+            for off, data in edits:
+                if off + len(data) > len(out):
+                    raise PackageError("patch beyond the end of %s" % self.path)
+                out[off:off + len(data)] = data
+            return bytes(out)
+        touched = {}
+        for off, data in edits:
+            if off < self.header_size or off + len(data) > self.size:
+                raise PackageError("patch outside the package data (0x%X)" % off)
+            pos = 0
+            while pos < len(data):
+                i = self._block_index(off + pos)
+                bl = self.blocks[i]
+                if i not in touched:
+                    touched[i] = bytearray(lzo.decompress(raw[bl.coff:bl.coff + bl.csize], bl.usize))
+                s = off + pos - bl.uoff
+                k = min(len(data) - pos, bl.usize - s)
+                touched[i][s:s + k] = data[pos:pos + k]
+                pos += k
+        # rebuild chunk by chunk
+        head = bytearray(raw[:self.chunks[0][2]])
+        body = bytearray()
+        by_chunk = {}
+        for i, bl in enumerate(self.blocks):
+            by_chunk.setdefault(bl.chunk, []).append(i)
+        for ci, (uoff, usize, coff, csize) in enumerate(self.chunks):
+            new_coff = len(head) + len(body)
+            idx = by_chunk[ci]
+            if not any(i in touched for i in idx):
+                body += raw[coff:coff + csize]
+                self._set_chunk(head, ci, uoff, usize, new_coff, csize)
+                continue
+            datas = []
+            for i in idx:
+                bl = self.blocks[i]
+                if i in touched:
+                    datas.append((lzo.compress_stored(touched[i]), bl.usize))
+                else:
+                    datas.append((raw[bl.coff:bl.coff + bl.csize], bl.usize))
+            hdr = struct.pack("<4I", TAG, self.block_size, sum(len(d) for d, _ in datas), usize)
+            hdr += b"".join(struct.pack("<2I", len(d), u) for d, u in datas)
+            chunk = hdr + b"".join(d for d, _ in datas)
+            body += chunk
+            self._set_chunk(head, ci, uoff, usize, new_coff, len(chunk))
+        return bytes(head + body)
+
+    def _set_chunk(self, head, ci, uoff, usize, coff, csize):
+        struct.pack_into("<4i", head, self.chunk_table_offset + 16 * ci, uoff, usize, coff, csize)
+
+
+def is_package(path):
+    try:
+        with open(path, "rb") as f:
+            return struct.unpack("<I", f.read(4))[0] == TAG
+    except (OSError, struct.error):
+        return False

docs/README.md

@@ -8,6 +8,7 @@
 | [building.md](building.md) | building the editor and its installer (Windows, Linux) |
 | [formats/wad.md](formats/wad.md) | WWAD archives, grouped entries, BKPK stream packs |
 | [formats/banks.md](formats/banks.md) | soundbanks and the in-memory sounds not covered yet (footsteps, foley, effects) |
+| [formats/upk.md](formats/upk.md) | UE3 packages: summary, LZO chunks, how ackit reads and patches them |
 | [formats/wem.md](formats/wem.md) | WEM layout the game accepts, setup stripping, hash and decode-alloc formulas |
 | [reverse-engineering/decoder.md](reverse-engineering/decoder.md) | BatmanAC.exe decoder functions and addresses |
 | [reverse-engineering/findings.md](reverse-engineering/findings.md) | root cause, findings, in-game experiment log (incl. failed hypotheses) |

docs/formats/banks.md

@@ -12,8 +12,10 @@ Survey of the installed game (buildid 3718300) on 2026-10-09, read-only.
 | loose `SFX\*.bnk` (Batman/Catwoman footsteps, 32 surfaces each, + `TST_Prep_Media`) | 65 | 1,117 | 0 |
 | banks embedded in `.upk` packages (in 1,432 of the 2,650 packages) | 3,340 | 15,641 (~276 MB) | 0 |
 
-The two sets share 119 media IDs: about **15,800 unique sounds** in total. The WADs, by comparison, are fully
+The two sets share 119 media IDs: 15,800 unique media IDs in 96,011 places (bank copies). 396 of them are
-covered: the BKPK tables in the packages reference 2,543 WAD entries, all of them in the catalog.
+**prefetch heads** of streamed WAD sounds (below), so there are **15,404 in-memory sounds**. All copies of a
+media ID are byte-identical except `1C898731` (two variants). The WADs, by comparison, are fully covered: the
+BKPK tables in the packages reference 2,543 WAD entries, all of them in the catalog.
 
 Embedded WEMs by bank name prefix: `Speech` 7,872, `CHR` (character foley, combat, bosses) 3,896, `PSO` 781,
 `WPN` 664, `NPC` 475, `OBJ` 439, `SEQ` 265, `GUI` 208, `PSA` 203, `MTN` 196, `PX` 195, `DVC` 100, `GEN` 80,
@@ -40,14 +42,17 @@ HIRC Sound object (type 2) payload: u32 object id, u32 plugin id (`0x00040001` =
 (0 = in memory, 1 = streamed, 2 = prefetch), u32 media id, u32 bank id of the media; for in-memory sources
 then u32 offset, u32 size, where offset = `DATA` payload position (from the start of the bank) + DIDX offset
 and size = DIDX size (checked on every in-memory source of the footstep banks and of `BmGame.upk`).
-A few sources use media of another bank. Every streamed media ID found is in a BKPK table, i.e. a WAD sound
+Source kinds over all bank copies: 115,861 in-memory sources of the bank's own media; 2,233 in-memory sources
-(`wad.md`).
+with bank id 0 whose media is not in the same bank (loaded by another bank); 133,075 streamed sources;
+2,101 **prefetch** sources. Every streamed or prefetched media ID is in a BKPK table, i.e. a WAD sound
+(`wad.md`). A prefetch source's media is also in the bank's DIDX: that in-memory copy is the **head** of the
+streamed WEM (its size = the source's size), which the game plays while the rest is read from the WAD. 396
+WAD sounds have such heads (2,101 copies); replacing one of them must rewrite the heads too.
 
 ## Banks inside packages
-- The packages are fully LZO-compressed (package version 805, licensee 101). For the survey they were unpacked
+- The packages are LZO-compressed UE3 packages (`upk.md`); ackit decompresses them itself and searches the
-  with Gildor's `decompress.exe` (gildor.org; used from the user's old modding folder, not part of this
+  uncompressed data for `BKHD` (banks) and `BKPK` (WAD indexes). (The first survey used Gildor's
-  repository) and searched for `BKHD` (banks) and `BKPK` (WAD indexes). `UnrealX.upk` (512 bytes) is not a
+  decompress.exe; `ackit/upk.py` gives identical data.)
-  readable package (wrong tag, probably encrypted).
 - Bank blobs in packages: 41,798 for 3,340 bank IDs. Many banks are copied into several packages (e.g.
   `CHR_CatwomanFoleySuit` 31 times, `CHR_BatmanFoleyBelt` 26). 302 bank IDs occur as several blobs with
   *different* media subsets (often one WEM each, several per package), so a bank's media set is the union

docs/formats/upk.md

@@ -0,0 +1,38 @@
+# UE3 packages (`*.upk`)
+
+Arkham City's 2,650 packages in `BmGame\CookedPCConsole` are Unreal Engine 3 packages, version 805,
+licensee 101. 2,647 are fully LZO-compressed, 2 are stored uncompressed, and `UnrealX.upk` (512 bytes)
+is not a readable package. ackit reads and patches them with `ackit/upk.py` and `ackit/lzo.py` (no external
+tools). Soundbanks inside them: `banks.md`.
+
+## Summary (file start, little-endian)
+
+| field | notes |
+|-------|-------|
+| u32 tag `0x9E2A83C1`, u16 version 805, u16 licensee 101, i32 TotalHeaderSize | |
+| FString FolderName | i32 length incl. NUL, then bytes (`None`) |
+| 12 x i32 | PackageFlags, NameCount, NameOffset, ExportCount, ExportOffset, ImportCount, ImportOffset, DependsOffset, ImportExportGuidsOffset, ImportGuidsCount, ExportGuidsCount, ThumbnailTableOffset |
+| 16 bytes Guid | |
+| i32 N + N x {ExportCount, NameCount, NetObjectCount} | generations (N = 1) |
+| u32 EngineVersion 7748, u32 CookerVersion, u32 CompressionFlags | 2 = LZO, 0 = none |
+| i32 C + C x {i32 UncompressedOffset, UncompressedSize, CompressedOffset, CompressedSize} | chunk table |
+| u32 PackageSource, ... | not used by ackit |
+
+All offsets in the summary (names, exports, ...) refer to the **uncompressed** package. The uncompressed
+package = the summary without the chunk table, followed by the chunks' data at their UncompressedOffset
+(the first chunk starts right after the summary, at NameOffset).
+
+## Compressed chunks
+Each chunk (at CompressedOffset): `u32 tag, u32 block size (0x20000), u32 compressed sum, u32 uncompressed
+sum`, then one `{u32 compressed size, u32 uncompressed size}` per block, then the blocks. Every block is an
+independent LZO1X stream. In all game packages the chunks follow each other without gaps, start right
+after the summary and end at the end of the file (checked for every package).
+
+## How ackit reads and writes them
+- Positions of sounds are offsets in the uncompressed package, the same as in a decompressed copy
+  (e.g. Gildor's decompress.exe output). Reading decompresses only the blocks that are needed.
+- Writing (`Package.patched`): every changed block is stored again as a single LZO literal run (a valid
+  LZO1X stream without matches, `lzo.compress_stored`), the chunk headers of the changed chunks and the
+  CompressedOffset/CompressedSize of all following chunks are updated; every other byte is copied, so a
+  package without changes is rebuilt byte-for-byte. Replacements never change the uncompressed layout
+  (sounds are replaced in place), so names, exports and their offsets stay valid.

tests/synth.py

@@ -89,6 +89,72 @@ def pack(media):
     return bytes(hdr) + bank, bytes(payload)
 
 
+def bank(bank_id, media, name=None, extra_sources=()):
+    """A v56 soundbank holding `media` {id: wem} in memory (DIDX/DATA, 16-byte aligned) with one HIRC
+    Sound object per media; extra_sources: [(media id, bank id, off, size)] for sources of other banks."""
+    data = bytearray()
+    didx = b""
+    for m in sorted(media):
+        data += bytes(_a(len(data), 16) - len(data))
+        didx += struct.pack("<III", m, len(data), len(media[m]))
+        data += media[m]
+    bkhd = struct.pack("<II", 56, bank_id) + bytes(8)
+    head = b"BKHD" + struct.pack("<I", len(bkhd)) + bkhd + b"DIDX" + struct.pack("<I", len(didx)) + didx
+    data_pos = len(head) + 8                       # DATA payload, from the start of the bank
+    objs = []
+    for i, m in enumerate(sorted(media)):
+        off = struct.unpack_from("<I", didx, 12 * i + 4)[0]
+        objs.append((m, bank_id, data_pos + off, len(media[m])))
+    objs += list(extra_sources)
+    hirc = struct.pack("<I", len(objs))
+    for i, (m, b, off, size) in enumerate(objs):
+        body = struct.pack("<7I", 0x100 + i, 0x00040001, 0, m, b, off, size) + bytes(6)
+        hirc += bytes([2]) + struct.pack("<I", len(body)) + body
+    out = head + b"DATA" + struct.pack("<I", len(data)) + bytes(data) + b"HIRC" + struct.pack("<I", len(hirc)) + hirc
+    if name:
+        nm = name.encode()
+        stid = struct.pack("<II", 1, 1) + struct.pack("<IB", bank_id, len(nm)) + nm
+        out += b"STID" + struct.pack("<I", len(stid)) + stid
+    return out
+
+
+def package(path, body, block_size=0x1000, chunk_blocks=3):
+    """A fully LZO-compressed UE3 package (docs/formats/upk.md) whose uncompressed data area is `body`.
+    Small blocks so that tests cross block and chunk boundaries."""
+    from ackit import lzo
+    folder = b"None\x00"
+    nblocks = max(1, _a(len(body), block_size) // block_size)
+    nchunks = _a(nblocks, chunk_blocks) // chunk_blocks
+
+    def summary(chunks):
+        s = struct.pack("<IHHi", 0x9E2A83C1, 805, 101, 0) + struct.pack("<i", len(folder)) + folder
+        s += struct.pack("<12i", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + b"GUID-synthetic!!"
+        s += struct.pack("<i3i", 1, 0, 0, 0) + struct.pack("<3I", 7748, 0, 2)
+        s += struct.pack("<i", len(chunks)) + b"".join(struct.pack("<4i", *c) for c in chunks)
+        return s + struct.pack("<Ii", 0x12345678, 0)
+    header_u = len(summary([])) + 16                       # uncompressed header (some spare bytes)
+    chunks, blobs = [], []
+    coff = len(summary([(0, 0, 0, 0)] * nchunks)) + 16
+    u = header_u
+    for c in range(nchunks):
+        part = body[(u - header_u):(u - header_u) + block_size * chunk_blocks]
+        blocks = [part[i:i + block_size] for i in range(0, len(part), block_size)] or [b""]
+        comp = [lzo.compress_stored(b) for b in blocks]
+        hdr = struct.pack("<4I", 0x9E2A83C1, block_size, sum(map(len, comp)), len(part))
+        hdr += b"".join(struct.pack("<2I", len(cb), len(b)) for cb, b in zip(comp, blocks))
+        blob = hdr + b"".join(comp)
+        chunks.append((u, len(part), coff, len(blob)))
+        blobs.append(blob)
+        coff += len(blob)
+        u += len(part)
+    s = summary(chunks)
+    out = s + bytes(16) + b"".join(blobs)
+    assert len(s) + 16 == chunks[0][2]
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    open(path, "wb").write(out)
+    return header_u
+
+
 def make_game(root):
     """Create the folder; returns the dict of original WEMs by target."""
     cooked = os.path.join(root, "BmGame", "CookedPCConsole")

tests/test_containers.py

@@ -0,0 +1,110 @@
+"""LZO, UE3 packages and soundbanks (ackit/lzo.py, upk.py, bnk.py)."""
+import os
+
+import pytest
+
+import synth
+from ackit import lzo
+from ackit.bnk import Bank, find_banks, fnv1, IN_MEMORY
+from ackit.upk import Package
+
+
+# --- LZO ----------------------------------------------------------------------------------------
+@pytest.mark.parametrize("n", [0, 1, 3, 4, 17, 18, 19, 238, 239, 273, 274, 529, 0x20000])
+def test_lzo_stored_roundtrip(n):
+    data = os.urandom(n)
+    assert lzo.decompress(lzo.compress_stored(data), n) == data
+
+
+def test_lzo_matches():
+    # "abc" + M3 match (9 bytes at distance 3) + end
+    assert lzo.decompress(bytes([20]) + b"abc" + bytes([39, 0x08, 0x00]) + lzo.EOF) == b"abc" * 4
+    # "abcd" + M2 match (4 bytes at distance 4) + end
+    assert lzo.decompress(bytes([21]) + b"abcd" + bytes([108, 0]) + lzo.EOF) == b"abcd" * 2
+
+
+def test_lzo_errors():
+    with pytest.raises(lzo.LzoError):
+        lzo.decompress(bytes([25]) + b"abc")                       # truncated
+    with pytest.raises(lzo.LzoError):
+        lzo.decompress(lzo.compress_stored(b"hello") + b"x")        # trailing bytes
+    with pytest.raises(lzo.LzoError):
+        lzo.decompress(lzo.compress_stored(b"hello"), 6)             # wrong size
+
+
+# --- packages -----------------------------------------------------------------------------------
+@pytest.fixture
+def pkg(tmp_path):
+    body = bytes(range(256)) * 80                                     # 20480 bytes: 5 blocks, 2 chunks
+    p = str(tmp_path / "Synth.upk")
+    start = synth.package(p, body)
+    return p, body, start
+
+
+def test_package_read(pkg):
+    p, body, start = pkg
+    pk = Package(p)
+    assert pk.compressed and pk.version == 805 and pk.header_size == start
+    assert pk.size == start + len(body) and len(pk.chunks) == 2 and len(pk.blocks) == 5
+    assert pk.read(start + 4000, 300) == body[4000:4300]                # crosses a block boundary
+    assert pk.read_all() == (start, body)
+
+
+def test_package_patch(pkg):
+    p, body, start = pkg
+    raw = open(p, "rb").read()
+    pk = Package(p)
+    assert pk.patched([]) == raw                                       # nothing changed -> same bytes
+    new = pk.patched([(start + 16334, b"X" * 100)])                    # second chunk, across two blocks
+    q = p + ".new"
+    open(q, "wb").write(new)
+    pk2 = Package(q)
+    exp = bytearray(body)
+    exp[16334:16434] = b"X" * 100
+    assert pk2.read_all()[1] == bytes(exp)
+    # the first chunk is copied verbatim
+    c0 = pk.chunks[0]
+    assert new[c0[2]:c0[2] + c0[3]] == raw[c0[2]:c0[2] + c0[3]]
+    with pytest.raises(ValueError):
+        pk.patched([(start + len(body) - 10, b"too long" * 4)])
+
+
+# --- banks --------------------------------------------------------------------------------------
+def test_bank_parse():
+    media = {0x111: b"RIFF" + bytes(41), 0x222: b"RIFF" + bytes(100)}
+    b = synth.bank(fnv1("CHR_Test"), media, name="CHR_Test")
+    bk = Bank(b)
+    assert bk.id == fnv1("CHR_Test") and bk.version == 56 and bk.size == len(b)
+    assert bk.names() == {bk.id: "CHR_Test"}
+    m = {x.id: x for x in bk.media()}
+    assert b[m[0x222].off:m[0x222].off + m[0x222].size] == media[0x222]
+    assert m[0x222].off % 16 == (bk.chunks["DATA"][0]) % 16
+    src = {s.media: s for s in bk.sources()}
+    assert src[0x111].stream == IN_MEMORY and (src[0x111].off, src[0x111].size) == (m[0x111].off, m[0x111].size)
+
+
+def test_find_banks_in_buffer():
+    a = synth.bank(1, {5: b"x" * 20})
+    b = synth.bank(2, {6: b"y" * 30})
+    buf = b"junkBKHDjunk" + a + bytes(7) + b
+    assert [x.id for x in find_banks(buf)] == [1, 2]
+
+
+def test_fnv1():
+    assert fnv1("Init") == 0x50C63A23 and fnv1("Music") == 0xEDF036D6
+
+
+# --- real game (read-only) ----------------------------------------------------------------------
+def test_real_package_blocks(real_game_dir):
+    from ackit.game import Game
+    p = os.path.join(Game(real_game_dir).cooked, "BmGame.upk")
+    pk = Package(p)
+    assert pk.compressed and pk.patched([]) == open(p, "rb").read()
+    start, data = pk.read_all()
+    banks = list(find_banks(data))
+    assert len(banks) > 20
+    bk = banks[0]
+    m = bk.media()[0]
+    off = start + bk.at + m.off
+    assert pk.read(off, m.size) == data[bk.at + m.off:bk.at + m.off + m.size]
+    assert pk.read(off, 4) == b"RIFF"