gora/UnheardCity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version 3.1.0: voice lines linked across all languages from the game data

Browse Source
This commit is contained in:
Cristian Gora
2026-10-10 14:10:03 +02:00
parent ea3e44000a
commit fbde809370
56 changed files with 74728 additions and 73360 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
ackit/banks.py
View File

@@ -27,7 +27,8 @@ SOUND, MEMORY, PREFETCH_HEAD = 2, "m", "p"
EVENT, ACTION = 4, 3
CONTAINERS = (5, 6, 7, 9) # random/sequence, switch, actor-mixer, layer
Location = namedtuple("Location", "media file offset size kind sound", defaults=("",))
# lang: language of the bank holding this copy ('' = language-neutral bank); known after a scan only
Location = namedtuple("Location", "media file offset size kind sound lang", defaults=("", ""))
# --- files ----------------------------------------------------------------------------------------
@@ -137,6 +138,10 @@ def _children(payload, known):
# --- scan ---------------------------------------------------------------------------------------
def _by_language():
return defaultdict(dict)
class Scan:
"""Everything found in the game's banks (built by `scan`)."""
@@ -152,20 +157,23 @@ class Scan:
self.actions = {} # id -> target id
self.sound_media = defaultdict(set) # sound object id -> media ids
self.prefetch = set() # media ids whose in-memory copy is a prefetch head
self.packs = defaultdict(dict) # WAD entry id -> {media id: (offset in entry, size)} (BKPK tables)
self.lang_media = defaultdict(set) # language -> media ids named by Sound objects of that language's banks
# BKPK tables: WAD entry id -> {language: {media id: (offset in entry, size)}}. A table belongs to the
# language its localized AkBank export declares ('' = outside such an export). The same media ID can be
# used by several languages (at a different position in each language's WAD), so tables never merge.
self.packs = defaultdict(_by_language)
self.issues = [] # (kind, detail)
_LANG = re.compile(rb"([\x01-\x10])\x00\x00\x00\x04\x00\x00\x00[A-Z]{3}\x00")
def bank_languages(data, start, imports, exports, banks, issues=None, rel=""):
"""{bank offset in data: language} for localized banks. A localized AkBank export starts with its list of
languages (FStrings, e.g. INT FRA ITA DEU ESN) and holds one bank per language, in that order
(docs/formats/banks.md). Banks of other exports are language-neutral (not in the result)."""
out = {}
def localized_exports(data, start, imports, exports):
"""[(begin, end, [languages])] of the localized AkBank exports of a package (offsets in data). Such an
export starts with its list of languages (FStrings, e.g. INT FRA ITA DEU ESN) and then holds one block per
language, in that order: a bank and/or the BKPK table of the speech WAD entry (docs/formats/banks.md)."""
out = []
akbank = {-(i + 1) for i, (c, o, _) in enumerate(imports) if c == "Class" and o == "AkBank"}
starts = sorted(bk.at for bk in banks)
for x in exports:
if x.cls not in akbank or x.size <= 0:
continue
@@ -179,27 +187,67 @@ def bank_languages(data, start, imports, exports, banks, issues=None, rel=""):
ln = struct.unpack_from("<i", data, o)[0]
langs.append(bytes(data[o + 4:o + 3 + ln]).decode("latin-1"))
o += 4 + ln
inside = [a for a in starts if o0 <= a < o0 + x.size]
out.append((o0, o0 + x.size, langs, x.name))
return out
def _assign(positions, regions, issues, rel, what):
"""{position: language} for the blocks (banks or BKPK tables) inside localized exports: the k-th block of
an export belongs to its k-th language. An export whose block count differs is reported, not guessed."""
out = {}
for o0, o1, langs, name in regions:
inside = [a for a in positions if o0 <= a < o1]
if not inside:
continue
if len(inside) != len(langs):
if issues is not None:
issues.append(("languages", "%s: %s has %d banks for %d languages" % (rel, x.name, len(inside), len(langs))))
issues.append(("languages", "%s: %s has %d %s for %d languages" % (rel, name, len(inside), what, len(langs))))
continue
out.update(zip(inside, langs))
return out
def _bkpk_tables(data, packs):
"""BKPK tables (docs/formats/wad.md) embedded in a package: WAD entry -> media offsets/sizes."""
def bank_languages(data, start, imports, exports, banks, issues=None, rel=""):
"""{bank offset in data: language} for localized banks (one bank per declared language, in that order).
Banks of other exports are language-neutral (not in the result)."""
return _assign(sorted(bk.at for bk in banks), localized_exports(data, start, imports, exports), issues, rel, "banks")
def bkpk_positions(data):
"""Offsets of the BKPK tables (docs/formats/wad.md) embedded in a package."""
out = []
for m in re.finditer(b"BKPK", data):
p = m.start()
if p + 0x48 > len(data) or struct.unpack_from("<I", data, p + 0x1C)[0] != 0x48:
continue
entry, n = struct.unpack_from("<I", data, p + 8)[0], struct.unpack_from("<I", data, p + 0x10)[0]
if n > 100000 or p + 0x48 + 12 * n > len(data):
continue
for i in range(n):
mid, size, off = struct.unpack_from("<III", data, p + 0x48 + 12 * i)
packs[entry][mid] = (off, size)
n = struct.unpack_from("<I", data, p + 0x10)[0]
if n <= 100000 and p + 0x48 + 12 * n <= len(data):
out.append(p)
return out
def read_bkpk(data, p):
"""(WAD entry id, {media id: (offset in entry, size)}) of the BKPK table at p."""
entry, n = struct.unpack_from("<I", data, p + 8)[0], struct.unpack_from("<I", data, p + 0x10)[0]
table = {}
for i in range(n):
mid, size, off = struct.unpack_from("<III", data, p + 0x48 + 12 * i)
table[mid] = (off, size)
return entry, table
def _bkpk_tables(data, packs, regions, issues=None, rel=""):
"""Add a package's BKPK tables to `packs` under the language their export declares. Copies of a table
in several packages must agree (reported otherwise)."""
pos = bkpk_positions(data)
lang_at = _assign(pos, regions, issues, rel, "BKPK tables")
for p in pos:
entry, table = read_bkpk(data, p)
lang = lang_at.get(p, "")
have = packs[entry].get(lang)
if have and have != table and issues is not None:
issues.append(("bkpk", "%s: table of %08X (%s) differs from another copy" % (rel, entry, lang or "-")))
packs[entry][lang] = table
def scan(cooked, progress=None, cooked_banks=None, files=None, visit=None):
@@ -226,11 +274,12 @@ def scan(cooked, progress=None, cooked_banks=None, files=None, visit=None):
names, imports, exports = r.tables(data, start)
sc.names.update(names) # base names ...
sc.names.update(x.name for x in exports) # ... and numbered object names (Speech_X_04)
_bkpk_tables(data, sc.packs)
regions = localized_exports(data, start, imports, exports)
_bkpk_tables(data, sc.packs, regions, sc.issues, rel)
if visit:
visit(rel, r, start, data)
banks = list(find_banks(data))
lang_of = bank_languages(data, start, imports, exports, banks, sc.issues, rel)
lang_of = _assign(sorted(bk.at for bk in banks), regions, sc.issues, rel, "banks")
else:
try:
banks = [Bank(data, 0)]
@@ -256,7 +305,7 @@ def scan(cooked, progress=None, cooked_banks=None, files=None, visit=None):
e["sha1"].add(hashlib.sha1(w).hexdigest()[:16])
e["banks"].add(bk.id)
e["kinds"].add(kind)
sc.locations.append(Location(m.id, rel, start + bk.at + m.off, m.size, kind))
sc.locations.append(Location(m.id, rel, start + bk.at + m.off, m.size, kind, "", lang_of.get(bk.at, "")))
for t, oid, pl in _hirc_objects(data, bk):
sc.objects[oid] = t
if t == EVENT and len(pl) >= 8:
@@ -267,17 +316,20 @@ def scan(cooked, progress=None, cooked_banks=None, files=None, visit=None):
sc.actions[oid] = struct.unpack_from("<I", pl, 8)[0]
elif t in CONTAINERS:
sc.containers[oid] = pl
for s in bk.sources():
if s.stream != PREFETCH:
sc.sound_media[s.object].add(s.media)
for s in bk.sources(): # every source, prefetched ones too (streamed lines whose start
sc.sound_media[s.object].add(s.media) # is kept in the bank: the game-over taunts)
if lang_of.get(bk.at):
sc.lang_media[lang_of[bk.at]].add(s.media)
if progress:
progress(len(files), len(files))
return sc
# --- names --------------------------------------------------------------------------------------
def event_media(sc):
"""{event name: [media ids in traversal order]} for every event whose name is known."""
def event_objects(sc):
"""{event name: [sound object ids in traversal order]} for every event whose name is known. Events,
actions, containers and sound objects have the same IDs in every language (only the media differ), so
this order is the same in every language."""
known = set(sc.objects)
kids = {cid: _children(pl, known) for cid, pl in sc.containers.items()}
by_hash = {}
@@ -296,15 +348,26 @@ def event_media(sc):
continue
seen.add(o)
if o in sc.sound_media:
for m in sorted(sc.sound_media[o]):
if m not in order:
order.append(m)
order.append(o)
stack.extend(list(kids.get(o, ()))[::-1])
if order:
out[name] = order
return out
def event_media(sc):
"""{event name: [media ids in traversal order]} for every event whose name is known."""
out = {}
for name, objs in event_objects(sc).items():
order = []
for o in objs:
for m in sorted(sc.sound_media[o]):
if m not in order:
order.append(m)
out[name] = order
return out
def media_language(sc, m):
"""Language of a localized in-memory media ('' = language-neutral)."""
langs = sc.media[m].get("langs", set()) - {""}
@@ -321,7 +384,7 @@ def media_names(sc):
for name, ms in ev.items():
groups = defaultdict(list)
for m in ms:
if m in sc.media:
if m in sc.media and PREFETCH_HEAD not in sc.media[m]["kinds"]: # heads belong to WAD lines
groups[media_language(sc, m)].append(m)
for g in groups.values():
rank = (0 if name.lower().startswith("play") else 1, len(g), name.lower())