Read and patch UE3 packages and soundbanks
This commit is contained in:
@@ -1,7 +1,8 @@
|
||||
"""ackit - Batman: Arkham City (PC) audio toolkit.
|
||||
|
||||
Library layers (low -> high):
|
||||
formats wad, pack, wem, ogg, bits, codebooks, wwvorbis, allocsize, builder, convert, known_setups
|
||||
formats wad, pack, bnk (soundbanks), upk + lzo (UE3 packages), wem, ogg, bits, codebooks, wwvorbis,
|
||||
allocsize, builder, convert, known_setups
|
||||
audio audio (load/convert/decode), encoder (PCM -> game WEM)
|
||||
game paths, game (install discovery), catalog + classify (names/categories/characters),
|
||||
inject (WAD writing), originals (store), workspace (apply/revert), modpack (share)
|
||||
|
||||
108
ackit/bnk.py
Normal file
108
ackit/bnk.py
Normal file
@@ -0,0 +1,108 @@
|
||||
"""Wwise soundbanks (bank version 56): chunks, in-memory media (DIDX/DATA) and HIRC sound sources.
|
||||
|
||||
Works on a buffer that may hold a bank anywhere (a loose .bnk file, a BKPK pack's embedded bank, or
|
||||
an uncompressed .upk package with many banks). Offsets are relative to the start of the bank.
|
||||
Format: docs/formats/banks.md.
|
||||
"""
|
||||
import re
|
||||
import struct
|
||||
from collections import namedtuple
|
||||
|
||||
CHUNKS = (b"BKHD", b"DIDX", b"DATA", b"HIRC", b"STID", b"STMG", b"ENVS", b"FXPR")
|
||||
SOUND = 2 # HIRC object type of a Sound
|
||||
IN_MEMORY, STREAMED, PREFETCH = 0, 1, 2
|
||||
|
||||
Media = namedtuple("Media", "id off size") # off: from bank start
|
||||
Source = namedtuple("Source", "object stream media bank off size pos") # pos: of `off` field, from bank start
|
||||
|
||||
|
||||
class BankError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class Bank:
|
||||
def __init__(self, buf, at=0):
|
||||
self.buf, self.at = buf, at
|
||||
self.chunks = {}
|
||||
p = at
|
||||
while p + 8 <= len(buf) and bytes(buf[p:p + 4]) in CHUNKS:
|
||||
tag = bytes(buf[p:p + 4]).decode()
|
||||
n = struct.unpack_from("<I", buf, p + 4)[0]
|
||||
if p + 8 + n > len(buf) or tag in self.chunks:
|
||||
break
|
||||
self.chunks[tag] = (p + 8 - at, n)
|
||||
p += 8 + n
|
||||
if "BKHD" not in self.chunks or self.chunks["BKHD"][1] < 8:
|
||||
raise BankError("no bank at 0x%X" % at)
|
||||
self.size = p - at
|
||||
o = at + self.chunks["BKHD"][0]
|
||||
self.version, self.id = struct.unpack_from("<II", buf, o)
|
||||
|
||||
def _abs(self, tag):
|
||||
o, n = self.chunks[tag]
|
||||
return self.at + o, n
|
||||
|
||||
def media(self):
|
||||
"""In-memory WEMs: [Media(id, offset from bank start, size)]."""
|
||||
if "DIDX" not in self.chunks or "DATA" not in self.chunks:
|
||||
return []
|
||||
o, n = self._abs("DIDX")
|
||||
data = self.chunks["DATA"][0]
|
||||
out = []
|
||||
for i in range(n // 12):
|
||||
mid, off, size = struct.unpack_from("<III", self.buf, o + 12 * i)
|
||||
out.append(Media(mid, data + off, size))
|
||||
return out
|
||||
|
||||
def sources(self):
|
||||
"""Sources of the HIRC Sound objects: [Source]. In-memory sources carry offset/size."""
|
||||
if "HIRC" not in self.chunks:
|
||||
return []
|
||||
o, n = self._abs("HIRC")
|
||||
end = o + n
|
||||
count = struct.unpack_from("<I", self.buf, o)[0]
|
||||
p = o + 4
|
||||
out = []
|
||||
for _ in range(count):
|
||||
if p + 5 > end:
|
||||
raise BankError("HIRC overruns its chunk in bank %08X" % self.id)
|
||||
t = self.buf[p]
|
||||
ln = struct.unpack_from("<I", self.buf, p + 1)[0]
|
||||
if t == SOUND and ln >= 28:
|
||||
obj, _plugin, stream, mid, bank = struct.unpack_from("<5I", self.buf, p + 5)
|
||||
off, size = struct.unpack_from("<II", self.buf, p + 25)
|
||||
out.append(Source(obj, stream, mid, bank, off, size, p + 25 - self.at))
|
||||
p += 5 + ln
|
||||
return out
|
||||
|
||||
def names(self):
|
||||
"""STID: {bank id: name}."""
|
||||
if "STID" not in self.chunks:
|
||||
return {}
|
||||
o, n = self._abs("STID")
|
||||
count = struct.unpack_from("<I", self.buf, o + 4)[0]
|
||||
p = o + 8
|
||||
out = {}
|
||||
for _ in range(count):
|
||||
bid, ln = struct.unpack_from("<IB", self.buf, p)
|
||||
out[bid] = bytes(self.buf[p + 5:p + 5 + ln]).decode("latin-1")
|
||||
p += 5 + ln
|
||||
return out
|
||||
|
||||
|
||||
def find_banks(buf, start=0):
|
||||
"""Every bank in `buf` (e.g. an uncompressed package): yields Bank."""
|
||||
for m in re.finditer(b"BKHD", buf[start:] if start else buf):
|
||||
try:
|
||||
yield Bank(buf, m.start() + start)
|
||||
except (BankError, struct.error):
|
||||
continue
|
||||
|
||||
|
||||
def fnv1(name):
|
||||
"""Wwise ID of a name (bank, event, ...): FNV-1 32 of the lower-cased name."""
|
||||
h = 2166136261
|
||||
for c in name.lower().encode("utf-8"):
|
||||
h = (h * 16777619) & 0xFFFFFFFF
|
||||
h ^= c
|
||||
return h
|
||||
142
ackit/lzo.py
Normal file
142
ackit/lzo.py
Normal file
@@ -0,0 +1,142 @@
|
||||
"""LZO1X, as used by the game's compressed UE3 packages (docs/formats/upk.md).
|
||||
|
||||
`decompress` is a pure-Python LZO1X decoder (same semantics as lzo1x_decompress_safe; the packages
|
||||
use 128 KiB blocks, so speed is fine for block-level access). `compress_stored` writes a valid LZO1X
|
||||
stream that holds the data as one literal run (no matches): bigger than real compression, but any
|
||||
LZO1X decoder (the game's included) reads it back exactly, and nothing else has to change.
|
||||
"""
|
||||
M2_MAX_OFFSET = 0x0800
|
||||
|
||||
|
||||
class LzoError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _copy_match(out, dist, n):
|
||||
start = len(out) - dist
|
||||
if start < 0:
|
||||
raise LzoError("match distance beyond output start")
|
||||
if dist >= n:
|
||||
out += out[start:start + n]
|
||||
else: # overlapping: repeat the last `dist` bytes
|
||||
pat = bytes(out[start:])
|
||||
out += (pat * (n // dist + 1))[:n]
|
||||
|
||||
|
||||
def decompress(src, out_len=None):
|
||||
"""LZO1X block -> bytes. `out_len` (if given) is checked."""
|
||||
src = bytes(src)
|
||||
n = len(src)
|
||||
out = bytearray()
|
||||
ip = 0
|
||||
|
||||
def ext(t, base):
|
||||
nonlocal ip
|
||||
while src[ip] == 0:
|
||||
t += 255
|
||||
ip += 1
|
||||
t += base + src[ip]
|
||||
ip += 1
|
||||
return t
|
||||
|
||||
try:
|
||||
state = "loop"
|
||||
if src[0] > 17:
|
||||
t = src[0] - 17
|
||||
ip = 1
|
||||
if t < 4:
|
||||
state = "match_next"
|
||||
else:
|
||||
out += src[ip:ip + t]
|
||||
ip += t
|
||||
state = "first_literal_run"
|
||||
while True:
|
||||
if state == "loop":
|
||||
t = src[ip]
|
||||
ip += 1
|
||||
if t >= 16:
|
||||
state = "match"
|
||||
continue
|
||||
if t == 0:
|
||||
t = ext(0, 15)
|
||||
out += src[ip:ip + t + 3]
|
||||
ip += t + 3
|
||||
state = "first_literal_run"
|
||||
continue
|
||||
if state == "first_literal_run":
|
||||
t = src[ip]
|
||||
ip += 1
|
||||
if t >= 16:
|
||||
state = "match"
|
||||
continue
|
||||
dist = 1 + M2_MAX_OFFSET + (t >> 2) + (src[ip] << 2)
|
||||
ip += 1
|
||||
_copy_match(out, dist, 3)
|
||||
state = "match_done"
|
||||
continue
|
||||
if state == "match_next":
|
||||
out += src[ip:ip + t]
|
||||
ip += t
|
||||
t = src[ip]
|
||||
ip += 1
|
||||
state = "match"
|
||||
continue
|
||||
if state == "match":
|
||||
if t >= 64: # M2
|
||||
dist = 1 + ((t >> 2) & 7) + (src[ip] << 3)
|
||||
ip += 1
|
||||
_copy_match(out, dist, (t >> 5) + 1)
|
||||
elif t >= 32: # M3
|
||||
t &= 31
|
||||
if t == 0:
|
||||
t = ext(0, 31)
|
||||
dist = 1 + ((src[ip] | (src[ip + 1] << 8)) >> 2)
|
||||
ip += 2
|
||||
_copy_match(out, dist, t + 2)
|
||||
elif t >= 16: # M4 (distance 0 = end of stream)
|
||||
hi = (t & 8) << 11
|
||||
t &= 7
|
||||
if t == 0:
|
||||
t = ext(0, 7)
|
||||
dist = hi + ((src[ip] | (src[ip + 1] << 8)) >> 2)
|
||||
ip += 2
|
||||
if dist == 0:
|
||||
break
|
||||
_copy_match(out, dist + 0x4000, t + 2)
|
||||
else: # M1 after a match: 2 bytes
|
||||
dist = 1 + (t >> 2) + (src[ip] << 2)
|
||||
ip += 1
|
||||
_copy_match(out, dist, 2)
|
||||
state = "match_done"
|
||||
continue
|
||||
# match_done: the low 2 bits of the instruction's last byte = trailing literals
|
||||
t = src[ip - 2] & 3
|
||||
if t == 0:
|
||||
state = "loop"
|
||||
else:
|
||||
state = "match_next"
|
||||
except IndexError:
|
||||
raise LzoError("truncated LZO stream") from None
|
||||
if ip != n:
|
||||
raise LzoError("%d bytes after the end of the LZO stream" % (n - ip))
|
||||
if out_len is not None and len(out) != out_len:
|
||||
raise LzoError("LZO block decoded to %d bytes, expected %d" % (len(out), out_len))
|
||||
return bytes(out)
|
||||
|
||||
|
||||
EOF = b"\x11\x00\x00"
|
||||
|
||||
|
||||
def compress_stored(data):
|
||||
"""bytes -> LZO1X stream made of one literal run (decodes back to `data`)."""
|
||||
data = bytes(data)
|
||||
n = len(data)
|
||||
if n == 0:
|
||||
return EOF
|
||||
if n <= 238:
|
||||
return bytes([n + 17]) + data + EOF
|
||||
# long literal run: 0x00, k zero bytes (255 each), last byte; length = 18 + 255*k + last
|
||||
r = n - 18
|
||||
k = (r - 1) // 255
|
||||
last = r - 255 * k
|
||||
return b"\x00" + b"\x00" * k + bytes([last]) + data + EOF
|
||||
250
ackit/upk.py
Normal file
250
ackit/upk.py
Normal file
@@ -0,0 +1,250 @@
|
||||
"""The game's UE3 packages (.upk): summary, LZO-compressed chunks, block-level reads and patches.
|
||||
|
||||
Offsets used by the rest of ackit are positions in the *uncompressed* package (what Gildor's
|
||||
decompress.exe writes), so a sound's location does not depend on how the file is compressed.
|
||||
Format and rules: docs/formats/upk.md.
|
||||
|
||||
Patching rewrites only the blocks that contain changed bytes (stored as one LZO literal run, see
|
||||
lzo.compress_stored) and moves the chunks after them; every other byte of the file stays identical.
|
||||
"""
|
||||
import struct
|
||||
from collections import OrderedDict
|
||||
|
||||
from . import lzo
|
||||
|
||||
TAG = 0x9E2A83C1
|
||||
COMPRESS_LZO = 2
|
||||
|
||||
|
||||
class PackageError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class Block:
|
||||
__slots__ = ("chunk", "uoff", "usize", "coff", "csize")
|
||||
|
||||
def __init__(self, chunk, uoff, usize, coff, csize):
|
||||
self.chunk, self.uoff, self.usize, self.coff, self.csize = chunk, uoff, usize, coff, csize
|
||||
|
||||
|
||||
class Package:
|
||||
"""Read access to one package file. `blocks` covers the uncompressed range [header_size, size)."""
|
||||
|
||||
def __init__(self, path, cache_blocks=8):
|
||||
self.path = path
|
||||
with open(path, "rb") as f:
|
||||
head = f.read(0x10000)
|
||||
self._parse_summary(head, f)
|
||||
self.file_size = f.seek(0, 2)
|
||||
self._cache = OrderedDict()
|
||||
self._cache_n = cache_blocks
|
||||
|
||||
# summary -----------------------------------------------------------------------------------
|
||||
def _parse_summary(self, b, f):
|
||||
if len(b) < 64 or struct.unpack_from("<I", b, 0)[0] != TAG:
|
||||
raise PackageError("%s is not an Unreal package" % self.path)
|
||||
self.version, self.licensee = struct.unpack_from("<HH", b, 4)
|
||||
o = 12
|
||||
n = struct.unpack_from("<i", b, o)[0]
|
||||
o += 4 + max(n, 0)
|
||||
# PackageFlags, NameCount/Offset, ExportCount/Offset, ImportCount/Offset, DependsOffset,
|
||||
# ImportExportGuidsOffset, ImportGuidsCount, ExportGuidsCount, ThumbnailTableOffset
|
||||
f12 = struct.unpack_from("<12i", b, o)
|
||||
self.name_count, self.name_offset = f12[1], f12[2]
|
||||
self.export_count, self.export_offset = f12[3], f12[4]
|
||||
o += 48
|
||||
self.guid = b[o:o + 16]
|
||||
o += 16
|
||||
ng = struct.unpack_from("<i", b, o)[0]
|
||||
o += 4 + 12 * ng
|
||||
self.engine, self.cooker, self.compression = struct.unpack_from("<3I", b, o)
|
||||
o += 12
|
||||
nc = struct.unpack_from("<i", b, o)[0]
|
||||
o += 4
|
||||
if nc and len(b) < o + 16 * nc:
|
||||
f.seek(0)
|
||||
b = f.read(o + 16 * nc)
|
||||
self.chunk_table_offset = o
|
||||
self.chunks = [list(struct.unpack_from("<4i", b, o + 16 * i)) for i in range(nc)] # uoff usize coff csize
|
||||
if nc and self.compression != COMPRESS_LZO:
|
||||
raise PackageError("%s: unsupported compression %d" % (self.path, self.compression))
|
||||
self.blocks = []
|
||||
if not nc:
|
||||
return
|
||||
for ci, (uoff, usize, coff, csize) in enumerate(self.chunks):
|
||||
f.seek(coff)
|
||||
h = f.read(16)
|
||||
tag, bsize, sum_c, sum_u = struct.unpack("<4I", h)
|
||||
if tag != TAG or sum_u != usize:
|
||||
raise PackageError("%s: bad chunk header at 0x%X" % (self.path, coff))
|
||||
nb = (sum_u + bsize - 1) // bsize
|
||||
tbl = f.read(8 * nb)
|
||||
p = coff + 16 + 8 * nb
|
||||
u = uoff
|
||||
for i in range(nb):
|
||||
c, uu = struct.unpack_from("<2I", tbl, 8 * i)
|
||||
self.blocks.append(Block(ci, u, uu, p, c))
|
||||
p += c
|
||||
u += uu
|
||||
if p != coff + csize or u != uoff + usize:
|
||||
raise PackageError("%s: inconsistent chunk %d" % (self.path, ci))
|
||||
self.block_size = bsize
|
||||
|
||||
@property
|
||||
def compressed(self):
|
||||
return bool(self.chunks)
|
||||
|
||||
@property
|
||||
def header_size(self):
|
||||
"""Uncompressed size of the summary area (= start of the first chunk's data)."""
|
||||
return self.chunks[0][0] if self.chunks else 0
|
||||
|
||||
@property
|
||||
def size(self):
|
||||
"""Uncompressed package size."""
|
||||
if not self.chunks:
|
||||
return self.file_size
|
||||
return self.chunks[-1][0] + self.chunks[-1][1]
|
||||
|
||||
def fingerprint(self):
|
||||
"""Cheap identity of the package contents: guid + chunk table (changes with any re-cook/patch)."""
|
||||
import hashlib
|
||||
h = hashlib.sha1(self.guid)
|
||||
h.update(struct.pack("<q", self.file_size))
|
||||
for c in self.chunks:
|
||||
h.update(struct.pack("<4i", *c))
|
||||
return h.hexdigest()[:16]
|
||||
|
||||
# reading -----------------------------------------------------------------------------------
|
||||
def _block(self, i, f=None):
|
||||
if i in self._cache:
|
||||
self._cache.move_to_end(i)
|
||||
return self._cache[i]
|
||||
bl = self.blocks[i]
|
||||
own = f is None
|
||||
f = f or open(self.path, "rb")
|
||||
try:
|
||||
f.seek(bl.coff)
|
||||
data = lzo.decompress(f.read(bl.csize), bl.usize)
|
||||
finally:
|
||||
if own:
|
||||
f.close()
|
||||
self._cache[i] = data
|
||||
if len(self._cache) > self._cache_n:
|
||||
self._cache.popitem(last=False)
|
||||
return data
|
||||
|
||||
def _block_index(self, off):
|
||||
lo, hi = 0, len(self.blocks) - 1
|
||||
while lo <= hi:
|
||||
mid = (lo + hi) // 2
|
||||
bl = self.blocks[mid]
|
||||
if off < bl.uoff:
|
||||
hi = mid - 1
|
||||
elif off >= bl.uoff + bl.usize:
|
||||
lo = mid + 1
|
||||
else:
|
||||
return mid
|
||||
raise PackageError("offset 0x%X is outside the package data" % off)
|
||||
|
||||
def read(self, off, n):
|
||||
"""Uncompressed bytes [off, off+n)."""
|
||||
if not self.chunks:
|
||||
with open(self.path, "rb") as f:
|
||||
f.seek(off)
|
||||
return f.read(n)
|
||||
if off < self.header_size:
|
||||
raise PackageError("reading the package summary area is not supported")
|
||||
out = bytearray()
|
||||
i = self._block_index(off)
|
||||
with open(self.path, "rb") as f:
|
||||
while len(out) < n:
|
||||
bl = self.blocks[i]
|
||||
d = self._block(i, f)
|
||||
s = off + len(out) - bl.uoff
|
||||
out += d[s:s + n - len(out)]
|
||||
i += 1
|
||||
return bytes(out)
|
||||
|
||||
def iter_data(self):
|
||||
"""(uncompressed offset, bytes) per block, in order (whole-package scans)."""
|
||||
if not self.chunks:
|
||||
with open(self.path, "rb") as f:
|
||||
yield 0, f.read()
|
||||
return
|
||||
with open(self.path, "rb") as f:
|
||||
for bl in self.blocks:
|
||||
f.seek(bl.coff)
|
||||
yield bl.uoff, lzo.decompress(f.read(bl.csize), bl.usize)
|
||||
|
||||
def read_all(self):
|
||||
"""The whole uncompressed data area as one bytes object (offset `data_start`)."""
|
||||
parts = list(self.iter_data())
|
||||
return (parts[0][0] if parts else 0), b"".join(p for _, p in parts)
|
||||
|
||||
# patching ----------------------------------------------------------------------------------
|
||||
def patched(self, edits):
|
||||
"""edits: [(uncompressed offset, bytes)] (same length as what they overwrite) -> new file bytes.
|
||||
|
||||
Uncompressed packages are patched in place. For compressed ones the touched blocks are
|
||||
re-encoded and the chunk headers and the summary's chunk table are updated."""
|
||||
with open(self.path, "rb") as f:
|
||||
raw = f.read()
|
||||
if not self.chunks:
|
||||
out = bytearray(raw)
|
||||
for off, data in edits:
|
||||
if off + len(data) > len(out):
|
||||
raise PackageError("patch beyond the end of %s" % self.path)
|
||||
out[off:off + len(data)] = data
|
||||
return bytes(out)
|
||||
touched = {}
|
||||
for off, data in edits:
|
||||
if off < self.header_size or off + len(data) > self.size:
|
||||
raise PackageError("patch outside the package data (0x%X)" % off)
|
||||
pos = 0
|
||||
while pos < len(data):
|
||||
i = self._block_index(off + pos)
|
||||
bl = self.blocks[i]
|
||||
if i not in touched:
|
||||
touched[i] = bytearray(lzo.decompress(raw[bl.coff:bl.coff + bl.csize], bl.usize))
|
||||
s = off + pos - bl.uoff
|
||||
k = min(len(data) - pos, bl.usize - s)
|
||||
touched[i][s:s + k] = data[pos:pos + k]
|
||||
pos += k
|
||||
# rebuild chunk by chunk
|
||||
head = bytearray(raw[:self.chunks[0][2]])
|
||||
body = bytearray()
|
||||
by_chunk = {}
|
||||
for i, bl in enumerate(self.blocks):
|
||||
by_chunk.setdefault(bl.chunk, []).append(i)
|
||||
for ci, (uoff, usize, coff, csize) in enumerate(self.chunks):
|
||||
new_coff = len(head) + len(body)
|
||||
idx = by_chunk[ci]
|
||||
if not any(i in touched for i in idx):
|
||||
body += raw[coff:coff + csize]
|
||||
self._set_chunk(head, ci, uoff, usize, new_coff, csize)
|
||||
continue
|
||||
datas = []
|
||||
for i in idx:
|
||||
bl = self.blocks[i]
|
||||
if i in touched:
|
||||
datas.append((lzo.compress_stored(touched[i]), bl.usize))
|
||||
else:
|
||||
datas.append((raw[bl.coff:bl.coff + bl.csize], bl.usize))
|
||||
hdr = struct.pack("<4I", TAG, self.block_size, sum(len(d) for d, _ in datas), usize)
|
||||
hdr += b"".join(struct.pack("<2I", len(d), u) for d, u in datas)
|
||||
chunk = hdr + b"".join(d for d, _ in datas)
|
||||
body += chunk
|
||||
self._set_chunk(head, ci, uoff, usize, new_coff, len(chunk))
|
||||
return bytes(head + body)
|
||||
|
||||
def _set_chunk(self, head, ci, uoff, usize, coff, csize):
|
||||
struct.pack_into("<4i", head, self.chunk_table_offset + 16 * ci, uoff, usize, coff, csize)
|
||||
|
||||
|
||||
def is_package(path):
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
return struct.unpack("<I", f.read(4))[0] == TAG
|
||||
except (OSError, struct.error):
|
||||
return False
|
||||
Reference in New Issue
Block a user