gora/UnheardCity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version 3.1.0: voice lines linked across all languages from the game data

Browse Source
This commit is contained in:
Cristian Gora
2026-10-10 14:10:03 +02:00
parent ea3e44000a
commit fbde809370
56 changed files with 74728 additions and 73360 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ackit/__init__.py
View File

@@ -8,4 +8,4 @@ Library layers (low -> high):
inject (WAD writing), originals (store), workspace (apply/revert), modpack (share)
tools cli (`ackit` command), verify/ (offline self-check, in-game test on Windows)
"""
__version__ = "3.0.0"
__version__ = "3.1.0"

123
ackit/banks.py
View File

@@ -27,7 +27,8 @@ SOUND, MEMORY, PREFETCH_HEAD = 2, "m", "p"
EVENT, ACTION = 4, 3
CONTAINERS = (5, 6, 7, 9) # random/sequence, switch, actor-mixer, layer
Location = namedtuple("Location", "media file offset size kind sound", defaults=("",))
# lang: language of the bank holding this copy ('' = language-neutral bank); known after a scan only
Location = namedtuple("Location", "media file offset size kind sound lang", defaults=("", ""))
# --- files ----------------------------------------------------------------------------------------
@@ -137,6 +138,10 @@ def _children(payload, known):
# --- scan ---------------------------------------------------------------------------------------
def _by_language():
return defaultdict(dict)
class Scan:
"""Everything found in the game's banks (built by `scan`)."""
@@ -152,20 +157,23 @@ class Scan:
self.actions = {} # id -> target id
self.sound_media = defaultdict(set) # sound object id -> media ids
self.prefetch = set() # media ids whose in-memory copy is a prefetch head
self.packs = defaultdict(dict) # WAD entry id -> {media id: (offset in entry, size)} (BKPK tables)
self.lang_media = defaultdict(set) # language -> media ids named by Sound objects of that language's banks
# BKPK tables: WAD entry id -> {language: {media id: (offset in entry, size)}}. A table belongs to the
# language its localized AkBank export declares ('' = outside such an export). The same media ID can be
# used by several languages (at a different position in each language's WAD), so tables never merge.
self.packs = defaultdict(_by_language)
self.issues = [] # (kind, detail)
_LANG = re.compile(rb"([\x01-\x10])\x00\x00\x00\x04\x00\x00\x00[A-Z]{3}\x00")
def bank_languages(data, start, imports, exports, banks, issues=None, rel=""):
"""{bank offset in data: language} for localized banks. A localized AkBank export starts with its list of
languages (FStrings, e.g. INT FRA ITA DEU ESN) and holds one bank per language, in that order
(docs/formats/banks.md). Banks of other exports are language-neutral (not in the result)."""
out = {}
def localized_exports(data, start, imports, exports):
"""[(begin, end, [languages])] of the localized AkBank exports of a package (offsets in data). Such an
export starts with its list of languages (FStrings, e.g. INT FRA ITA DEU ESN) and then holds one block per
language, in that order: a bank and/or the BKPK table of the speech WAD entry (docs/formats/banks.md)."""
out = []
akbank = {-(i + 1) for i, (c, o, _) in enumerate(imports) if c == "Class" and o == "AkBank"}
starts = sorted(bk.at for bk in banks)
for x in exports:
if x.cls not in akbank or x.size <= 0:
continue
@@ -179,27 +187,67 @@ def bank_languages(data, start, imports, exports, banks, issues=None, rel=""):
ln = struct.unpack_from("<i", data, o)[0]
langs.append(bytes(data[o + 4:o + 3 + ln]).decode("latin-1"))
o += 4 + ln
inside = [a for a in starts if o0 <= a < o0 + x.size]
out.append((o0, o0 + x.size, langs, x.name))
return out
def _assign(positions, regions, issues, rel, what):
"""{position: language} for the blocks (banks or BKPK tables) inside localized exports: the k-th block of
an export belongs to its k-th language. An export whose block count differs is reported, not guessed."""
out = {}
for o0, o1, langs, name in regions:
inside = [a for a in positions if o0 <= a < o1]
if not inside:
continue
if len(inside) != len(langs):
if issues is not None:
issues.append(("languages", "%s: %s has %d banks for %d languages" % (rel, x.name, len(inside), len(langs))))
issues.append(("languages", "%s: %s has %d %s for %d languages" % (rel, name, len(inside), what, len(langs))))
continue
out.update(zip(inside, langs))
return out
def _bkpk_tables(data, packs):
"""BKPK tables (docs/formats/wad.md) embedded in a package: WAD entry -> media offsets/sizes."""
def bank_languages(data, start, imports, exports, banks, issues=None, rel=""):
"""{bank offset in data: language} for localized banks (one bank per declared language, in that order).
Banks of other exports are language-neutral (not in the result)."""
return _assign(sorted(bk.at for bk in banks), localized_exports(data, start, imports, exports), issues, rel, "banks")
def bkpk_positions(data):
"""Offsets of the BKPK tables (docs/formats/wad.md) embedded in a package."""
out = []
for m in re.finditer(b"BKPK", data):
p = m.start()
if p + 0x48 > len(data) or struct.unpack_from("<I", data, p + 0x1C)[0] != 0x48:
continue
entry, n = struct.unpack_from("<I", data, p + 8)[0], struct.unpack_from("<I", data, p + 0x10)[0]
if n > 100000 or p + 0x48 + 12 * n > len(data):
continue
for i in range(n):
mid, size, off = struct.unpack_from("<III", data, p + 0x48 + 12 * i)
packs[entry][mid] = (off, size)
n = struct.unpack_from("<I", data, p + 0x10)[0]
if n <= 100000 and p + 0x48 + 12 * n <= len(data):
out.append(p)
return out
def read_bkpk(data, p):
"""(WAD entry id, {media id: (offset in entry, size)}) of the BKPK table at p."""
entry, n = struct.unpack_from("<I", data, p + 8)[0], struct.unpack_from("<I", data, p + 0x10)[0]
table = {}
for i in range(n):
mid, size, off = struct.unpack_from("<III", data, p + 0x48 + 12 * i)
table[mid] = (off, size)
return entry, table
def _bkpk_tables(data, packs, regions, issues=None, rel=""):
"""Add a package's BKPK tables to `packs` under the language their export declares. Copies of a table
in several packages must agree (reported otherwise)."""
pos = bkpk_positions(data)
lang_at = _assign(pos, regions, issues, rel, "BKPK tables")
for p in pos:
entry, table = read_bkpk(data, p)
lang = lang_at.get(p, "")
have = packs[entry].get(lang)
if have and have != table and issues is not None:
issues.append(("bkpk", "%s: table of %08X (%s) differs from another copy" % (rel, entry, lang or "-")))
packs[entry][lang] = table
def scan(cooked, progress=None, cooked_banks=None, files=None, visit=None):
@@ -226,11 +274,12 @@ def scan(cooked, progress=None, cooked_banks=None, files=None, visit=None):
names, imports, exports = r.tables(data, start)
sc.names.update(names) # base names ...
sc.names.update(x.name for x in exports) # ... and numbered object names (Speech_X_04)
_bkpk_tables(data, sc.packs)
regions = localized_exports(data, start, imports, exports)
_bkpk_tables(data, sc.packs, regions, sc.issues, rel)
if visit:
visit(rel, r, start, data)
banks = list(find_banks(data))
lang_of = bank_languages(data, start, imports, exports, banks, sc.issues, rel)
lang_of = _assign(sorted(bk.at for bk in banks), regions, sc.issues, rel, "banks")
else:
try:
banks = [Bank(data, 0)]
@@ -256,7 +305,7 @@ def scan(cooked, progress=None, cooked_banks=None, files=None, visit=None):
e["sha1"].add(hashlib.sha1(w).hexdigest()[:16])
e["banks"].add(bk.id)
e["kinds"].add(kind)
sc.locations.append(Location(m.id, rel, start + bk.at + m.off, m.size, kind))
sc.locations.append(Location(m.id, rel, start + bk.at + m.off, m.size, kind, "", lang_of.get(bk.at, "")))
for t, oid, pl in _hirc_objects(data, bk):
sc.objects[oid] = t
if t == EVENT and len(pl) >= 8:
@@ -267,17 +316,20 @@ def scan(cooked, progress=None, cooked_banks=None, files=None, visit=None):
sc.actions[oid] = struct.unpack_from("<I", pl, 8)[0]
elif t in CONTAINERS:
sc.containers[oid] = pl
for s in bk.sources():
if s.stream != PREFETCH:
sc.sound_media[s.object].add(s.media)
for s in bk.sources(): # every source, prefetched ones too (streamed lines whose start
sc.sound_media[s.object].add(s.media) # is kept in the bank: the game-over taunts)
if lang_of.get(bk.at):
sc.lang_media[lang_of[bk.at]].add(s.media)
if progress:
progress(len(files), len(files))
return sc
# --- names --------------------------------------------------------------------------------------
def event_media(sc):
"""{event name: [media ids in traversal order]} for every event whose name is known."""
def event_objects(sc):
"""{event name: [sound object ids in traversal order]} for every event whose name is known. Events,
actions, containers and sound objects have the same IDs in every language (only the media differ), so
this order is the same in every language."""
known = set(sc.objects)
kids = {cid: _children(pl, known) for cid, pl in sc.containers.items()}
by_hash = {}
@@ -296,15 +348,26 @@ def event_media(sc):
continue
seen.add(o)
if o in sc.sound_media:
for m in sorted(sc.sound_media[o]):
if m not in order:
order.append(m)
order.append(o)
stack.extend(list(kids.get(o, ()))[::-1])
if order:
out[name] = order
return out
def event_media(sc):
"""{event name: [media ids in traversal order]} for every event whose name is known."""
out = {}
for name, objs in event_objects(sc).items():
order = []
for o in objs:
for m in sorted(sc.sound_media[o]):
if m not in order:
order.append(m)
out[name] = order
return out
def media_language(sc, m):
"""Language of a localized in-memory media ('' = language-neutral)."""
langs = sc.media[m].get("langs", set()) - {""}
@@ -321,7 +384,7 @@ def media_names(sc):
for name, ms in ev.items():
groups = defaultdict(list)
for m in ms:
if m in sc.media:
if m in sc.media and PREFETCH_HEAD not in sc.media[m]["kinds"]: # heads belong to WAD lines
groups[media_language(sc, m)].append(m)
for g in groups.values():
rank = (0 if name.lower().startswith("play") else 1, len(g), name.lower())

256
ackit/catalog.py
View File

@@ -19,15 +19,17 @@ import struct
from dataclasses import dataclass, asdict
from . import banks, classify, subtitles
from . import lines as lines_mod
from .game import wad_fingerprint
from .lines import bank_langs
from .inject import pack_bnk_path
from .pack import Pack
from .paths import catalog_dir, package_data, user_dir
from .wad import Wad
FIELDS = ["archive", "target", "name", "aliases", "group", "category", "bank", "language", "character",
FIELDS = ["archive", "target", "name", "aliases", "group", "category", "bank", "language", "line", "character",
"characters_inferred", "seconds", "rate", "channels", "size", "sha1", "notes"]
SCAN_VERSION = 2
SCAN_VERSION = 3
@dataclass
@@ -39,7 +41,8 @@ class Sound:
group: str = "Unknown"
category: str = "Unknown"
bank: str = "" # BANK sounds: the soundbank holding it (name); WAD sounds: ''
language: str = "" # BANK sounds: language of a localized speech line ('' = any language)
language: str = "" # BANK sounds: language(s) of a localized speech line, ';'-separated ('' = every language)
line: str = "" # speech: the game's identity of the line in every language (ackit/lines.py)
character: str = "" # verified speaker(s), '; '-separated (curated catalog)
characters_inferred: str = "" # characters/roles mentioned by the name (hint)
seconds: float = 0.0
@@ -65,6 +68,14 @@ class Sound:
def names(self):
return (self.name,) + tuple(self.aliases)
@property
def languages(self):
"""Voice languages of a localized in-memory sound (() = every language)."""
return tuple(self.language.split(";")) if self.language else ()
def in_language(self, lang):
return not self.language or lang in self.languages
# --- static data ---------------------------------------------------------------------------
_names = None
@@ -86,8 +97,8 @@ def hash_names():
def load_characters(path=None, lang=None):
"""{sound name (lower): 'Speaker' or 'A; B'} from data/catalog/characters.csv: the rows valid in every
language (keyed by the game's line names) plus the rows of `lang` (lines without a line name, labelled
by their position in that language's WAD)."""
language (keyed by line names, the same line everywhere) plus the rows of `lang` (a sound of that
language only; none at present)."""
path = path or os.path.join(catalog_dir(), "characters.csv")
out = {}
if os.path.exists(path):
@@ -98,13 +109,9 @@ def load_characters(path=None, lang=None):
def character_of(s, characters):
"""Verified speaker of a sound from a load_characters() map: by its names, and for the k-th take of a
line shown as NAME#k (an event playing several WEMs of one language) by the line name NAME."""
for n in s.names() + (s.name.split("#")[0],):
c = characters.get(n.lower())
if c:
return c
return ""
"""Verified speaker of a sound from a load_characters() map, by its name only: names are unique in a
language and a line has the same name in every language, while aliases (positions) are local."""
return characters.get(s.name.lower(), "")
def read_csv(path):
@@ -113,6 +120,7 @@ def read_csv(path):
out.append(Sound(archive=r["archive"], target=r["target"], name=r["name"],
aliases=tuple(a for a in r["aliases"].split("|") if a), group=r["group"],
category=r["category"], bank=r.get("bank") or "", language=r.get("language") or "",
line=r.get("line") or "",
character=r["character"],
characters_inferred=r["characters_inferred"], seconds=float(r["seconds"] or 0),
rate=int(r["rate"] or 0), channels=int(r["channels"] or 0), size=int(r["size"] or 0),
@@ -221,17 +229,16 @@ def bank_sounds(sc, characters=None):
"""[Sound] of the in-memory sounds found by `banks.scan` (prefetch heads belong to WAD sounds)."""
characters = characters if characters is not None else load_characters()
names = banks.media_names(sc)
first = {}
for loc in sorted(sc.locations, key=lambda l: (l.media, l.file, l.offset)):
first.setdefault(loc.media, loc)
every = set().union(*(e.get("langs", set()) for e in sc.media.values())) - {""}
out = []
for m, e in sc.media.items():
if banks.PREFETCH_HEAD in e["kinds"]:
continue
name, bank = names[m]
rate, ch, secs = _hdr(e["head"])
ls = bank_langs(sc, m) # localized: the languages whose banks hold it
s = Sound(banks.ARCHIVE, "%08X" % m, name, ("%s/%08X" % (bank, m),), bank=bank,
language=banks.media_language(sc, m), seconds=secs, rate=rate, channels=ch,
language="" if set(ls) == every else ";".join(ls), seconds=secs, rate=rate, channels=ch,
size=min(l.size for l in sc.locations if l.media == m) if len(e["sha1"]) > 1 else e["size"],
sha1=next(iter(e["sha1"])) if len(e["sha1"]) == 1 else "")
s.group, s.category = classify.classify(name, bank=bank)
@@ -242,127 +249,134 @@ def bank_sounds(sc, characters=None):
return out
def head_owners(game, sc, archives=None):
"""{prefetch media id: WAD sound key} -- the speech line (one language) each prefetch head belongs to."""
return media_owners(game, sc, {m for m, e in sc.media.items() if banks.PREFETCH_HEAD in e["kinds"]}, archives)
def media_owners(game, sc, want, archives=None):
"""{media id: WAD sound key} for streamed speech media, found through the per-language BKPK tables of
the packages (docs/formats/banks.md): every language has its own media IDs for the same WAD entry."""
out = {}
for a in archives or [x for x in game.archives if x != "SFX"]:
wad = Wad(game.wad(a))
by_hash = {e.hash: e for e in wad.entries}
with open(game.wad(a), "rb") as f:
for ent, table in sc.packs.items():
mine = [m for m in table if m in want and m not in out]
e = by_hash.get(ent)
if not mine or not e:
continue
wems = _walk_entry(f, e.offset, e.size)
for m in mine:
off, size = table[m]
for i, (o, ln, _) in enumerate(wems):
if (o, ln) == (off, size):
out[m] = "%s:%08X" % (a, ent) + (":%d" % i if len(wems) > 1 else "")
return out
def build_lines(game, sc):
"""The game's speech lines across languages (ackit/lines.py)."""
return lines_mod.build(game, sc, _walk_entry)
def scan_banks(game, progress=None, characters=None, visit=None):
"""Full scan of the game's banks -> ([Sound], [Location] with WAD keys on prefetch heads, Scan)."""
"""Full scan of the game's banks -> ([Sound], [Location] with WAD keys on prefetch heads, Scan, Lines).
Localized in-memory speech lines carry their line identity and the name shared by every language."""
sc = banks.scan(game.cooked, progress, visit=visit)
owners = head_owners(game, sc)
locs = [l._replace(sound=owners.get(l.media, "")) if l.kind == banks.PREFETCH_HEAD else l for l in sc.locations]
return bank_sounds(sc, characters), locs, sc
snd, locs, lines = derive_banks(game, sc, characters)
return snd, locs, sc, lines
def speech_line_names(game, sc):
"""{WAD sound key: the game's name of that line} for every speech sound some event plays. The name is the
speech event (e.g. Speech_CH_ZERO_CH0_fight_intro_01): the same line has the same name in every language,
while the position of a variation inside a WAD entry differs between languages (WEMs are stored in
media-ID order and every language has its own media IDs). If several events reach a sound, the most
specific one (fewest sounds in that language) names it."""
ev = {n: ms for n, ms in banks.event_media(sc).items() if n.lower().startswith("speech_")}
owners = media_owners(game, sc, {m for ms in ev.values() for m in ms})
best = {}
for n, ms in ev.items():
keys = collections.defaultdict(set)
for m in ms:
if m in owners:
k = owners[m]
keys[k.split(":")[0]].add(k)
for lang, ks in keys.items():
for k in ks:
rank = (len(ks), n.lower())
if k not in best or rank < best[k][0]:
best[k] = (rank, n)
return {k: n for k, (_, n) in best.items()}
def derive_banks(game, sc, characters=None):
"""(bank sounds, locations, Lines) from a Scan: the lines, the prefetch heads' lines, the in-memory sounds
with their line identities and names."""
lines = build_lines(game, sc)
locs = [l._replace(sound=lines.head_key(l)) if l.kind == banks.PREFETCH_HEAD else l for l in sc.locations]
snd = bank_sounds(sc, characters)
lines.names = line_names(game, sc, lines, {s.key: s.name for s in snd})
apply_lines(snd, lines, lines.names)
return snd, locs, lines
def apply_line_names(sounds, line_names):
"""Rename WAD speech sounds after the game's line names (see speech_line_names). The position-based name
stays available as alias NAME#i (single-WEM entries keep their entry name as alias)."""
used = collections.Counter()
def line_names(game, sc, lines, bank_names=None):
"""{line id: name} (lines.line_names). Lines no event names keep their English recording's name: the
position-based name in the INT WAD (`NAME_i`, or `NAME#i` where that is taken) or the bank-numbered name."""
pos = {}
for L in {k.split(":")[0] for per in lines.members.values() for k in per.values()} - {banks.ARCHIVE}:
for s in scan_archive(L, game.wad(L), characters={}):
pos[s.key] = s
bank_names = bank_names if bank_names is not None else {}
def fallback(k):
return pos[k].name if k in pos else bank_names.get(k, k)
out = lines_mod.line_names(sc, lines, fallback)
taken = collections.Counter(n.lower() for n in out.values())
for lid, n in out.items(): # a fallback name equal to an event-based one -> NAME#i of the INT WEM
if taken[n.lower()] > 1:
k = lines.members[lid].get("INT") or next(iter(lines.members[lid].values()))
alt = [a for a in (pos[k].names() if k in pos else ()) if "#" in a]
if alt and alt[0] != n:
out[lid] = alt[0]
return out
def apply_lines(sounds, lines, names):
"""Give every speech sound of one archive (a WAD language or BANK) its line identity and the line's name,
the same in every language. A WAD sound keeps its position-based names as aliases (NAME#i: position in
this language's WAD; single-WEM entries: the entry name). Names stay unique in the archive (and, for
BANK, per language together with the language-neutral sounds): a clash is resolved by the sound that
is not the line's own name falling back to its position name / media-ID name."""
for s in sounds:
ev = line_names.get(s.key)
if not ev:
lid = lines.line_of.get(s.key)
if not lid:
continue
used[ev.lower()] += 1
name = ev if used[ev.lower()] == 1 else "%s#%d" % (ev, used[ev.lower()])
if ":" in s.target: # variation: keep NAME#i (position in this language's WAD)
pos = [a for a in s.names() if "#" in a]
s.aliases = tuple(pos)
else: # single WEM: the entry name stays an alias
s.aliases = (s.name,) + tuple(a for a in s.aliases if a != s.name)
s.line = lid
if s.key in lines.copy_of: # a second place of a recording: keeps its position name, written with it
continue
name = names[lid]
if s.archive != banks.ARCHIVE:
if ":" in s.target:
s.aliases = tuple(a for a in s.names() if "#" in a)
else:
s.aliases = (s.name,) + tuple(a for a in s.aliases if a != s.name)
if name in s.aliases:
s.aliases = tuple(a for a in s.aliases if a != name)
s.name = name
taken = collections.Counter(s.name.lower() for s in sounds)
for s in sounds: # a positional name equal to another sound's line name -> NAME#i
if taken[s.name.lower()] > 1 and s.key not in line_names and ":" in s.target:
pos = [a for a in s.aliases if "#" in a]
if pos:
s.aliases, s.name = (s.name,), pos[0]
neutral, by_lang = set(), collections.defaultdict(set) # lower-case names taken
def clash(s, n):
if s.archive != banks.ARCHIVE or not s.languages:
return n in neutral or any(n in v for v in by_lang.values())
return n in neutral or any(n in by_lang[L] for L in s.languages)
# a line's own recordings first: they keep the line's name in a clash (copies and others give way)
for s in sorted(sounds, key=lambda s: (not s.line or s.key in lines.copy_of, s.key)):
if clash(s, s.name.lower()):
if s.archive == banks.ARCHIVE:
s.name = "%s_%s" % (s.name, s.target)
else:
alt = [a for a in s.aliases if "#" in a and a != s.name]
s.aliases, s.name = (s.name,), alt[0] if alt else "%s@%s" % (s.name, s.target)
if s.archive == banks.ARCHIVE and s.languages:
for L in s.languages:
by_lang[L].add(s.name.lower())
else:
neutral.add(s.name.lower())
for s in sounds:
s.group, s.category = classify.classify(s.name)
s.characters_inferred = "; ".join(classify.infer_characters(s.name))
if s.archive != banks.ARCHIVE:
s.group, s.category = classify.classify(s.name)
s.characters_inferred = "; ".join(classify.infer_characters(s.name))
return sounds
def subtitle_links(game, sc, speech_names):
"""{speech event name: {language: sound key}}: event -> actions -> containers -> sounds -> the
per-language media of the line (exact, no name matching). The media is a streamed WAD sound, or an
in-memory BANK sound of a localized bank."""
ev = banks.event_media(sc)
want = {m for n in speech_names for m in ev.get(n, ())}
owners = {m: (k.split(":")[0], k) for m, k in media_owners(game, sc, want).items()} # m -> (lang, key)
for m in want - set(owners):
lang = banks.media_language(sc, m) if m in sc.media else ""
if lang:
owners[m] = (lang, "%s:%08X" % (banks.ARCHIVE, m))
def subtitle_links(sc, lines, speech_names):
"""{speech event name: {language: sound keys}}: event -> actions -> containers -> Sound objects (the same
in every language) -> that language's recording of each line (a WAD line or a localized in-memory one).
An event that plays several lines (two versions) links its subtitle to all of them: 'k1;k2', in order."""
ev = banks.event_objects(sc)
out = {}
for n in speech_names:
keys = {}
for m in ev.get(n, ()):
if m in owners:
lang, key = owners[m]
keys.setdefault(lang, key)
keys = collections.defaultdict(list)
for o in ev.get(n, ()):
for L, k in lines.obj_keys.get(o, {}).items():
if k not in keys[L]:
keys[L].append(k)
if keys:
out[n] = keys
out[n] = {L: ";".join(ks) for L, ks in keys.items()}
return out
def scan_text(game, progress=None, characters=None):
"""One pass over every package: (bank sounds, bank locations, Scan, {lang: [SubLine]}, [SubLoc], issues,
{WAD sound key: game line name})."""
Lines)."""
col = subtitles.Collector()
snd, locs, sc = scan_banks(game, progress, characters, visit=col)
line_names = speech_line_names(game, sc)
names = {n for n in sc.names if n.lower().startswith("speech_")} | set(line_names.values())
names |= {s.name for s in snd if s.name.lower().startswith("speech_")}
sc = banks.scan(game.cooked, progress, visit=col)
names = {n for n in sc.names if n.lower().startswith("speech_")}
names |= {n for n, _ in banks.media_names(sc).values() if n.lower().startswith("speech_")}
col.resolve(names)
links = subtitle_links(game, sc, list(col.dialogues))
# the subtitle objects name their speech events too: an event ID is the FNV-1 hash of its name, so these
# names find events no package name table lists (e.g. Vicki Vale's radio reports)
sc.names.update(col.dialogues)
snd, locs, lines = derive_banks(game, sc, characters)
links = subtitle_links(sc, lines, list(col.dialogues))
texts, sublocs = subtitles.rows(col, links)
return snd, locs, sc, texts, sublocs, col.issues, line_names
return snd, locs, sc, texts, sublocs, col.issues, lines
def load_subtitles(game, lang, progress=None):
@@ -396,9 +410,13 @@ def load_banks(game, progress=None):
return read_csv(csv_path), banks.read_locations(loc_path)
cache = os.path.join(user_dir("cache"), "scan_%s_%s_v%d" % (banks.ARCHIVE, fp, SCAN_VERSION))
if not (os.path.exists(cache + ".csv") and os.path.exists(cache + ".locations.csv.gz")):
sounds, locs, _ = scan_banks(game, progress)
sounds, locs, _, lines = scan_banks(game, progress)
for s in sounds:
s.sha1 = "" # unknown build: no trusted original checksums
# the speech WADs of this build with the lines found now (load_archive reads these caches)
for a in game.languages:
snd = apply_lines(scan_archive(a, game.wad(a), progress=progress), lines, lines.names)
write_csv(_wad_cache(game, a), snd)
banks.write_locations(cache + ".locations.csv.gz", locs)
write_csv(cache + ".csv", sounds)
return read_csv(cache + ".csv"), banks.read_locations(cache + ".locations.csv.gz")
@@ -421,6 +439,10 @@ def is_shipped(game, archive):
return shipped.get("fingerprint") == wad_fingerprint(game.wad(archive))
def _wad_cache(game, archive):
return os.path.join(user_dir("cache"), "scan_%s_%s_v%d.csv" % (archive, wad_fingerprint(game.wad(archive)), SCAN_VERSION))
def load_archive(game, archive, progress=None):
"""[Sound] for one archive of a game (shipped CSV, user-cache scan, or fresh scan)."""
path = game.wad(archive)
@@ -429,7 +451,7 @@ def load_archive(game, archive, progress=None):
csv_path = os.path.join(catalog_dir(), "%s.csv" % archive)
if shipped.get("fingerprint") == fp and os.path.exists(csv_path):
return read_csv(csv_path)
cache = os.path.join(user_dir("cache"), "scan_%s_%s_v%d.csv" % (archive, fp, SCAN_VERSION))
cache = _wad_cache(game, archive)
if os.path.exists(cache):
return read_csv(cache)
sounds = scan_archive(archive, path, progress=progress)
@@ -463,10 +485,16 @@ class Catalog:
self.heads.setdefault(l.sound, []).append(l)
# localized in-memory speech: only the languages loaded here (all of them if none is)
langs = [a for a in self.archives if a in game.languages] or None
bank_sounds_ = [s for s in bank_sounds_ if not s.language or not langs or s.language in langs]
bank_sounds_ = [s for s in bank_sounds_ if not langs or any(s.in_language(L) for L in langs)]
for a in self.archives:
self.sounds += bank_sounds_ if a == banks.ARCHIVE else load_archive(game, a, progress)
self.by_key = {s.key: s for s in self.sounds}
# copies: one recording stored in two places of a WAD (same line, same archive); written together
same = collections.defaultdict(list)
for s in self.sounds:
if s.line and s.archive != banks.ARCHIVE:
same[(s.archive, s.line)].append(s)
self.copies = {s.key: [c for c in ss if c is not s] for ss in same.values() if len(ss) > 1 for s in ss}
self._by_name = {}
for s in self.sounds:
for n in s.names():

2
ackit/cli.py
View File

@@ -300,6 +300,8 @@ def cmd_verify(a):
for arch, kind, key, detail in rep.issues[:a.show]:
print(" %s %s %s %s" % (arch, kind, key, detail))
print("checked: %s" % ", ".join("%s %d" % kv for kv in sorted(rep.checked.items())))
for f in rep.facts:
print("fact: " + f)
print("catalog OK" if rep.ok else "%d issue(s)" % len(rep.issues))
sys.exit(0 if rep.ok else 1)
from .verify import selfcheck

222
ackit/lines.py Normal file
View File

@@ -0,0 +1,222 @@
"""Speech lines across languages, derived only from the game's own data (docs/formats/banks.md, "Lines").
A voice recording exists once per language. The game ties them together itself:
- every recording of a speech WAD has exactly one row in its language's BKPK table (media ID, offset, size);
the table's language is the one its localized AkBank export declares (banks.py: localized_exports);
- every recording, streamed, prefetched or in memory, is the source of a Wwise Sound object. Sound objects
(and the events, actions and containers above them) have the same IDs in every language; only the media
differ (a few media IDs are shared by several languages, each at its own position in each WAD).
So a *line* is the set of recordings whose media are the sources of the same Sound objects: one recording per
language. Its identity (`Sound.line`) is the smallest Sound object ID (8 hex digits), its name is chosen once
for all languages (`line_names`). A media stored in two entries of one WAD is one recording with a copy
(`copy_of`), written together. Nothing here depends on file positions or on names.
"""
import collections
from dataclasses import dataclass, field
from . import banks
from .wad import Wad
@dataclass
class Lines:
langs: list # voice languages (speech WADs)
owners: dict = field(default_factory=dict) # lang -> {media id: sound key} (WAD line or BANK sound)
line_of: dict = field(default_factory=dict) # sound key -> line id
members: dict = field(default_factory=dict) # line id -> {lang: sound key}
objects: dict = field(default_factory=dict) # line id -> sorted Sound object ids
obj_keys: dict = field(default_factory=dict) # Sound object id -> {lang: sound key}
names: dict = field(default_factory=dict) # line id -> name (catalog.line_names)
copy_of: dict = field(default_factory=dict) # WAD key -> key of the same media stored in another entry
issues: list = field(default_factory=list) # (kind, detail): broken one-to-one relations
stats: dict = field(default_factory=dict)
_keys: dict = field(default_factory=dict, repr=False) # lang -> set of sound keys
def key_langs(self, key):
return [L for L in self.langs if key in self._keys.get(L, ())]
def head_key(self, loc):
"""Speech line (sound key) of a prefetch-head copy: the line of the copy's bank language that has
this media. A head in a language-neutral bank belongs to the one line that uses the media."""
if loc.lang:
return self.owners.get(loc.lang, {}).get(loc.media, "")
ks = {d[loc.media] for d in self.owners.values() if loc.media in d}
return ks.pop() if len(ks) == 1 else ""
def wad_owners(game, sc, langs, walk):
"""({lang: {media id: WAD sound key}}, {copy key: key}, issues, WEM counts) through each language's own BKPK
tables. Issues: a WEM without a table row, a row without a WEM. A media stored in two entries of one
language's WAD (a few lines, byte-identical) has one key; the other place is a copy written with it.
`walk(f, offset, size)` -> [(offset, length, header)] (catalog._walk_entry)."""
owners, issues, wems = {}, [], collections.Counter()
places = {}
for L in langs:
owners[L] = {}
places[L] = collections.defaultdict(list)
wad = Wad(game.wad(L))
with open(game.wad(L), "rb") as f:
for e in wad.entries:
tables = sc.packs.get(e.hash, {})
table = tables.get(L) if L in tables else tables.get("")
ws = walk(f, e.offset, e.size)
wems[L] += len(ws)
if table is None:
issues.append(("no table", "%s:%08X: %d WEMs, no BKPK table" % (L, e.hash, len(ws))))
continue
at = {(o, ln): i for i, (o, ln, _) in enumerate(ws)}
used = set()
for m, v in table.items():
if v not in at:
issues.append(("row without WEM", "%s:%08X: media %08X" % (L, e.hash, m)))
continue
used.add(at[v])
places[L][m].append("%s:%08X" % (L, e.hash) + (":%d" % at[v] if len(ws) > 1 else ""))
for i in sorted(set(range(len(ws))) - used):
issues.append(("WEM without row", "%s:%08X:%d" % (L, e.hash, i)))
copy_of = {}
for L in langs:
for m, ks in places[L].items():
ks = sorted(ks)
owners[L][m] = ks[0]
for k in ks[1:]:
copy_of[k] = ks[0]
return owners, copy_of, issues, dict(wems)
def bank_langs(sc, m):
"""Voice languages of a localized in-memory media: the languages of the banks holding it ('' = neutral)."""
ls = sc.media[m].get("langs", set())
return () if "" in ls else tuple(sorted(ls))
def build(game, sc, walk):
"""Lines of every voice language of the game (see module doc)."""
in_banks = sorted(set().union(*(e.get("langs", set()) for e in sc.media.values())) - {""} - set(game.languages))
langs = list(game.languages) + in_banks # voice languages: speech WADs + localized banks
lines = Lines(langs)
owners, copy_of, issues, wems = wad_owners(game, sc, game.languages, walk)
for L in in_banks:
owners[L] = {}
lines.issues += issues
lines.copy_of = copy_of
for m, e in sc.media.items(): # localized in-memory recordings (not prefetch heads)
if banks.PREFETCH_HEAD in e["kinds"]:
continue
for L in bank_langs(sc, m):
if L in owners:
owners[L].setdefault(m, "%s:%08X" % (banks.ARCHIVE, m))
lines.owners = owners
lines._keys = {L: set(d.values()) for L, d in owners.items()}
# Sound object -> its recording in each language (exactly one, or the relation is broken)
for o, ms in sc.sound_media.items():
per = {}
for L in langs:
ks = {owners[L][m] for m in ms if m in owners[L]}
if len(ks) > 1:
lines.issues.append(("object", "Sound %08X has %d recordings in %s: %s" % (o, len(ks), L, sorted(ks))))
elif ks:
per[L] = ks.pop()
if per:
lines.obj_keys[o] = per
# lines = connected recordings (union-find over the objects' recordings)
parent = {}
def find(x):
while parent.get(x, x) != x:
parent[x] = parent.get(parent[x], parent[x])
x = parent[x]
return x
for o, per in lines.obj_keys.items():
ks = sorted(set(per.values()))
for k in ks[1:]:
a, b = find(ks[0]), find(k)
if a != b:
parent[max(a, b)] = min(a, b)
groups = collections.defaultdict(set)
objs = collections.defaultdict(set)
for o, per in lines.obj_keys.items():
for k in per.values():
groups[find(k)].add(k)
objs[find(k)].add(o)
for root, ks in groups.items():
per = {}
ok = True
for k in ks:
for L in lines.key_langs(k):
if L in per and per[L] != k:
ok = False
per.setdefault(L, k)
if not ok:
lines.issues.append(("line", "recordings %s share Sound objects but several are in one language" % sorted(ks)))
continue
lid = "%08X" % min(objs[root])
lines.members[lid] = per
lines.objects[lid] = sorted(objs[root])
for k in ks:
lines.line_of[k] = lid
for k, primary in copy_of.items(): # a second place of the same recording: the same line
if primary in lines.line_of:
lines.line_of[k] = lines.line_of[primary]
all_keys = set().union(*lines._keys.values()) if lines._keys else set()
all_keys |= set(copy_of)
full = sum(1 for per in lines.members.values() if len(per) == len(langs))
lines.stats = {"languages": langs, "wad_wems": wems, "recordings": len(all_keys),
"in_a_line": len(lines.line_of), "lines": len(lines.members), "lines_in_every_language": full,
"copies": sorted(copy_of),
"lines_missing_a_language": {lid: sorted(set(langs) - set(per))
for lid, per in lines.members.items() if len(per) < len(langs)},
"without_line": sorted(all_keys - set(lines.line_of))}
return lines
def line_names(sc, lines, fallback):
"""{line id: name}, the same in every language. A line is named after the game event that plays it: for
WAD lines the most specific `Speech_` event (fewest lines), for in-memory lines the most specific event
(`Play_` first), as the in-memory sounds always were. When an event plays several lines they are numbered
in the event's own order (the same in every language): NAME, NAME#2, ... (WAD) or NAME_1, NAME_2, ... (in
memory). A line no named event plays keeps the name its English recording has (`fallback(key)`: the
position-based name in the INT WAD, or the bank-numbered name), else the first voice language's."""
obj_line = {o: lid for lid, os_ in lines.objects.items() for o in os_}
is_wad = {lid: any(not k.startswith(banks.ARCHIVE + ":") for k in per.values()) for lid, per in lines.members.items()}
best = {}
reach = {}
for name, objs in banks.event_objects(sc).items():
order = []
for o in objs:
lid = obj_line.get(o)
if lid and lid not in order:
order.append(lid)
if order:
reach[name] = order
for name, order in reach.items():
low = name.lower()
for lid in order:
if is_wad[lid]:
if not low.startswith("speech_"):
continue
rank = (len(order), low)
else:
rank = (0 if low.startswith("play") else 1, len(order), low)
if lid not in best or rank < best[lid][0]:
best[lid] = (rank, name)
named = collections.defaultdict(list) # event -> lines it names, in the event's order
for name, order in reach.items():
for lid in order:
if lid in best and best[lid][1] == name:
named[name].append(lid)
out = {}
for name, lids in named.items():
for i, lid in enumerate(lids):
if is_wad[lid]:
out[lid] = name if i == 0 else "%s#%d" % (name, i + 1)
else:
out[lid] = name if len(lids) == 1 else "%s_%d" % (name, i + 1)
for lid, per in lines.members.items():
if lid not in out:
k = per.get("INT") or per[next(L for L in lines.langs if L in per)]
out[lid] = fallback(k)
return out

29
ackit/modpack.py
View File

@@ -5,7 +5,8 @@ A package is a zip:
items/NNN.wem the exact WEM that was applied (used as-is when the target matches)
items/NNN.flac the same audio, decoded (used to re-encode when the target differs, e.g. another
language whose original has a different size)
Each item records archive, target, name, group and the SHA-1 of the original it replaced.
Each item records archive, target, name, group, the SHA-1 of the original it replaced and, for a voice line,
its line identity (the same in every language, ackit/lines.py): that is how another language finds it.
"texts": [{"key", "text", "original"}] edited subtitles / menu texts (keys: ackit/texts.py); optional,
so packages without texts stay valid
"""
@@ -49,6 +50,7 @@ def export_mod(ws, path, keys, name="", author="", description="", texts=()):
z.writestr(base + ".wem", wem)
z.writestr(base + ".flac", buf.getvalue())
items.append({"archive": e["archive"], "target": e["target"], "name": s.name if s else "",
"line": s.line if s else "",
"group": s.group if s else "", "original_sha1": e["orig_sha1"],
"wem": base + ".wem", "audio": base + ".flac", "meta": e.get("meta", {})})
z.writestr("manifest.json", json.dumps({"format": FORMAT, "name": name, "author": author,
@@ -63,7 +65,7 @@ def export_mod(ws, path, keys, name="", author="", description="", texts=()):
class ImportItem:
item: dict
sound: object = None # matched Sound or None
how: str = "" # 'exact' | 'by name' | 'missing'
how: str = "" # 'exact' | 'same line' | 'by name' | 'missing'
wem: bytes = None # ready-to-apply bytes (after `prepare_import`)
note: str = ""
@@ -80,19 +82,34 @@ def match_mod(ws, path):
"""Match package items to sounds of this workspace. -> (manifest, [ImportItem])"""
z, man = read_mod(path)
out = []
by_line = {}
for c in ws.catalog.sounds:
if c.line:
by_line.setdefault(c.line, []).append(c)
for it in man["items"]:
key = "%s:%s" % (it["archive"], it["target"])
s = ws.catalog.by_key.get(key)
if s and it.get("name") and s.name != it["name"]:
s = None
if s:
# the very same recording: same position and the same name or the same original bytes
if s and (s.name == it.get("name") or (s.sha1 and s.sha1 == it.get("original_sha1"))):
out.append(ImportItem(it, s, "exact"))
continue
if it.get("line"): # a voice line: its recording in the language loaded here
cands = sorted(by_line.get(it["line"], []), key=lambda c: (c.name != it.get("name"), c.key))
if cands: # several only for one recording stored twice (written together)
out.append(ImportItem(it, cands[0], "same line"))
else:
out.append(ImportItem(it, None, "missing", note="the line %r has no recording in the language "
"loaded here" % it.get("name")))
continue
try:
cands = [c for c in ws.catalog.search(it["name"]) if c.name == it["name"]] if it.get("name") else []
except LookupError:
cands = []
if len(cands) == 1:
if len(cands) == 1 and cands[0].line:
# a voice line from a package without line identities (older version): names alone are not proof
out.append(ImportItem(it, None, "missing", note="%r is a voice line from a package of an older "
"version; export the mod again to move it to another language" % it["name"]))
elif len(cands) == 1:
out.append(ImportItem(it, cands[0], "by name"))
else:
out.append(ImportItem(it, None, "missing", note="no sound named %r in this game/language" % it.get("name")))

24
ackit/subtitles.py
View File

@@ -19,7 +19,6 @@ import re
import struct
from collections import defaultdict, namedtuple
from .upk import Package
CLASS = "RDialogueEvent"
SUBTITLE_CLASS, LOOKUP_CLASS = "RSubtitle", "RSubtitleLookup" # scripted radio/encryption subtitles
@@ -29,7 +28,8 @@ T_STRUCT, T_STRING, T_OBJECT = 0x09, 0x0D, 0x10
# one subtitle string inside a package: pos = absolute uncompressed offset of the FString length field,
# size = bytes of the whole FString (length field included), enc 'a' (cp1252) or 'w' (UTF-16)
Field = namedtuple("Field", "lang index text pos size enc")
Dialogue = namedtuple("Dialogue", "speech export fields") # speech: name of the referenced speech event
# speech: name of the referenced speech event; speaker: the object's speaker field (RDialogueEvent only)
Dialogue = namedtuple("Dialogue", "speech export fields speaker", defaults=("",))
class SubtitleError(ValueError):
@@ -148,6 +148,15 @@ def parse_dialogue(b, base, names_idx):
return fields
def _speaker(b):
"""The speaker FString of a dialogue object (property 0x0D at field offset 0x34), '' if none."""
m = re.search(rb"\x0d\x00\x34\x00", b)
try:
return read_fstring(b, m.start() + 4)[0] if m else ""
except (struct.error, UnicodeDecodeError, ValueError):
return ""
def _speech_ref(b, exports, imports):
"""Name of the object referenced by the dialogue's first object property (its speech event)."""
m = re.search(rb"\x10\x00\x2c\x00", b)
@@ -197,7 +206,14 @@ class Collector:
# --- catalog files: SUB_<LANG>.csv (texts) + SUB_locations.csv.gz (every copy) ------------------
LOCATIONS = "SUB_locations.csv.gz"
TEXT_FIELDS = ["speech", "line", "text", "room", "enc", "sound"]
SubLine = namedtuple("SubLine", "lang speech line text room enc sound") # sound: linked WAD sound key
class SubLine(namedtuple("SubLine", "lang speech line text room enc sound")):
"""A subtitle string. sound: the voice recording(s) its speech event plays in that language (sound keys,
';'-separated in the event's order: an event can play one of two versions of a line)."""
__slots__ = ()
@property
def sounds(self):
return tuple(self.sound.split(";")) if self.sound else ()
SubLoc = namedtuple("SubLoc", "lang speech line file pos size enc")
@@ -312,5 +328,5 @@ def scan_package(pkg, data=None, start=None, tables=None):
speech = UNRESOLVED % keys[e.index]
else:
speech = e.name # not in a lookup table: keep the object's name
out.append(Dialogue(speech, e.name, fields))
out.append(Dialogue(speech, e.name, fields, _speaker(b) if kind == CLASS else ""))
return out

4
ackit/texts.py
View File

@@ -55,10 +55,10 @@ class Texts:
def for_sound(self, sound, lang=None):
"""Subtitle lines of a speech sound, in `lang` (default: the sound's own language). Languages without
their own voices link their subtitles to the INT sounds, so e.g. JPN subtitles are found for INT lines."""
lang = lang or sound.language or sound.archive
lang = lang or (sound.languages[0] if sound.languages else sound.archive)
if lang not in self.subtitle_languages():
return []
return [s for s in self.subtitles(lang) if s.sound == sound.key]
return [s for s in self.subtitles(lang) if sound.key in s.sounds]
def menu(self, lang):
"""[loctext.Entry] of the game's current files (the values the game shows now)."""

271
ackit/verify/catalogcheck.py
View File

@@ -17,8 +17,8 @@ import os
import random
from .. import banks, classify, subtitles
from ..catalog import (Catalog, available_archives, bank_sounds, character_of, hash_names, head_owners, is_shipped,
load_banks, load_characters, read_csv, scan_archive)
from ..catalog import (Catalog, apply_lines, available_archives, bank_sounds, build_lines, character_of, hash_names,
is_shipped, line_names, load_archive, load_banks, load_characters, read_csv, scan_archive)
from ..inject import read_target
from ..paths import catalog_dir
from ..wad import Wad
@@ -31,6 +31,7 @@ class Report:
def __init__(self):
self.checked = collections.Counter()
self.issues = [] # (archive, kind, key, detail)
self.facts = [] # what was established about the game's relations (full check)
def add(self, archive, kind, key, detail=""):
self.issues.append((archive, kind, key, detail))
@@ -73,19 +74,30 @@ def _check_rows(rep, archive, rows, characters):
rep.add(archive, "character", s.key, "%r, curated list says %r" % (s.character, want))
def check_wad(rep, game, archive, rows, characters):
def check_wad(rep, game, archive, rows, characters, lines=None):
"""Rows vs a fresh scan of the WAD. With `lines` (full check) the fresh sounds get the game's line
identities and names, so names, aliases and lines are compared too."""
path = game.wad(archive)
if archive != "SFX":
folder = os.path.basename(os.path.dirname(path)).upper()
if FOLDER_LANG.get(folder, archive) != archive:
rep.add(archive, "language", archive, "WAD in folder %s" % folder)
_check_rows(rep, archive, rows, characters)
fresh = {s.target: s for s in scan_archive(archive, path, characters=characters)}
if is_shipped(game, archive): # shipped names are the game's line names (built from the banks)
fresh_list = scan_archive(archive, path, characters=characters)
if lines is not None and archive in lines.langs:
apply_lines(fresh_list, lines, lines.names)
fresh = {s.target: s for s in fresh_list}
if lines is None and archive in game.languages: # quick check: line names need the banks, not rescanned
byk = {s.target: s for s in rows}
for t, s in fresh.items():
if t in byk:
s.name, s.aliases = byk[t].name, byk[t].aliases
s.name, s.aliases, s.line = byk[t].name, byk[t].aliases, byk[t].line
byk = {s.target: s for s in rows}
for t in sorted(set(fresh) & set(byk)):
f, s = fresh[t], byk[t]
if (f.name, tuple(f.aliases), f.line) != (s.name, tuple(s.aliases), s.line):
rep.add(archive, "name", s.key, "catalog %s %s line %s, game %s %s line %s"
% (s.name, list(s.aliases), s.line or "-", f.name, list(f.aliases), f.line or "-"))
cat = {s.target: s for s in rows}
for t in sorted(set(fresh) - set(cat)):
rep.add(archive, "missing", "%s:%s" % (archive, t), fresh[t].name)
@@ -103,6 +115,8 @@ def check_wad(rep, game, archive, rows, characters):
def check_banks(rep, game, rows, locations, characters, quick=False, sample=40, progress=None):
"""BANK rows, copies, prefetch heads and subtitles vs a scan of the packages. The full check (not quick)
also derives the game's lines from that scan and returns them (else None)."""
_check_rows(rep, banks.ARCHIVE, rows, characters)
files = banks.bank_files(game.cooked)
if quick:
@@ -118,7 +132,10 @@ def check_banks(rep, game, rows, locations, characters, quick=False, sample=40,
known.update(n.split("#")[0] for s in read_csv(p) for n in s.names())
col.resolve({n for n in sc.names | known if n.lower().startswith("speech_")})
check_subtitles(rep, game, col, set(files), quick)
owners = head_owners(game, sc)
lines = None
if not quick:
sc.names.update(col.dialogues) # as the build does: subtitle objects name speech events too
lines = build_lines(game, sc)
index = collections.defaultdict(set)
for m, ls in locations.items():
for l in ls:
@@ -126,13 +143,22 @@ def check_banks(rep, game, rows, locations, characters, quick=False, sample=40,
index[(l.media, l.kind)].add((l.file, l.offset, l.size, l.sound if l.kind == banks.PREFETCH_HEAD else ""))
found = collections.defaultdict(set)
for l in sc.locations:
found[(l.media, l.kind)].add((l.file, l.offset, l.size, owners.get(l.media, "") if l.kind == banks.PREFETCH_HEAD else ""))
head = l.kind == banks.PREFETCH_HEAD
found[(l.media, l.kind)].add((l.file, l.offset, l.size, lines.head_key(l) if head and lines else ""))
if lines is None: # quick: a subset of the packages cannot resolve the heads' lines
index = {k: {(f, o, s, "") for f, o, s, _ in v} for k, v in index.items()}
for k in sorted(set(found) | set(index)):
if found.get(k, set()) != index.get(k, set()):
rep.add(banks.ARCHIVE, "copies", "%08X" % k[0], "%s: index %d, game %d"
% ("head" if k[1] == banks.PREFETCH_HEAD else "copies", len(index.get(k, ())), len(found.get(k, ()))))
cat = {s.target: s for s in rows}
fresh = {s.target: s for s in bank_sounds(sc, characters)}
fresh_list = bank_sounds(sc, characters)
if lines is not None:
lines.names = line_names(game, sc, lines, {s.key: s.name for s in fresh_list})
apply_lines(fresh_list, lines, lines.names)
for s in fresh_list:
s.character = character_of(s, characters)
fresh = {s.target: s for s in fresh_list}
for t in sorted(set(fresh) - set(cat)):
rep.add(banks.ARCHIVE, "missing", "BANK:" + t, fresh[t].bank)
if not quick:
@@ -144,10 +170,14 @@ def check_banks(rep, game, rows, locations, characters, quick=False, sample=40,
rep.add(banks.ARCHIVE, "sha1", s.key, "game file differs from the catalog's original")
if f.size < s.size:
rep.add(banks.ARCHIVE, "size", s.key, "catalog %d, game %d" % (s.size, f.size))
if lines is not None and (f.name, f.language, f.line) != (s.name, s.language, s.line):
rep.add(banks.ARCHIVE, "name", s.key, "catalog %s [%s] line %s, game %s [%s] line %s"
% (s.name, s.language, s.line or "-", f.name, f.language, f.line or "-"))
for m, ls in locations.items():
for l in ls:
if l.kind == banks.PREFETCH_HEAD and not l.sound:
rep.add(banks.ARCHIVE, "copies", "%08X" % m, "prefetch head without a speech line")
return sc, lines, col
def check_subtitles(rep, game, col, files, quick):
@@ -174,6 +204,7 @@ def check_subtitles(rep, game, col, files, quick):
langs = sorted(_shipped_index().get("SUB", {}).get("lines", {}))
bank_lang = {s.key: s.language for s in read_csv(os.path.join(catalog_dir(), "BANK.csv"))}
for lang in langs:
voice = lang if lang in game.languages else "INT" # languages without own voices: INT voices
p = os.path.join(catalog_dir(), "SUB_%s.csv" % lang)
rows = subtitles.read_texts(p, lang)
rep.checked["SUB_" + lang] += len(rows)
@@ -185,12 +216,200 @@ def check_subtitles(rep, game, col, files, quick):
seen.add(k)
if k in texts and texts[k] != s.text:
rep.add("SUB", "text", k, "catalog %r, game %r" % (s.text[:40], texts[k][:40]))
if s.sound.startswith(banks.ARCHIVE + ":"):
b = bank_lang.get(s.sound)
if b is not None and b not in (lang, "INT"): # INT: languages without own voices
rep.add("SUB", "language", k, "linked to %s (%s)" % (s.sound, b or "language-neutral"))
elif s.sound and not s.sound.startswith(lang + ":") and not s.sound.startswith("INT:"):
rep.add("SUB", "language", k, "linked to %s" % s.sound)
for snd in s.sounds:
if snd.startswith(banks.ARCHIVE + ":"):
b = bank_lang.get(snd)
if b and voice not in b.split(";"): # '' = stored for every language
rep.add("SUB", "language", k, "linked to %s (%s)" % (snd, b))
elif not snd.startswith(voice + ":"):
rep.add("SUB", "language", k, "linked to %s, not a %s voice line" % (snd, voice))
def check_lines(rep, game, sc, lines, locations):
"""The game's relations, re-derived from the game files (ackit/lines.py) and compared with the catalogs:
links a broken one-to-one relation in the game data (a WEM without its table row, a Sound object with
two recordings in one language, a line without a recording in some language, ...)
line a catalog row whose line identity / name differs from the game's
subtitle a subtitle linked to a different line in one language than in INT
Facts (rep.facts) give the counts, prefetch heads compared byte by byte with the start of their WAD
recording, and the length agreement of a line's recordings across languages (independent evidence)."""
for kind, detail in lines.issues:
rep.add("LINES", "links", kind, detail)
st = lines.stats
for lid, miss in sorted(st["lines_missing_a_language"].items()):
rep.facts.append("line %s (%s) has no recording in %s" % (lid, lines.names.get(lid, "?"), ", ".join(miss)))
for k in st["without_line"]:
rep.add("LINES", "links", k, "a recording of no line")
# independent of the BKPK tables: each WAD recording is the source of a Sound object in a bank of the
# language the tables give it (localized banks: one per declared language)
named = total = 0
for L in lines.langs:
for m, k in lines.owners.get(L, {}).items():
if not k.startswith(banks.ARCHIVE + ":"):
total += 1
if m in sc.lang_media.get(L, ()):
named += 1
else:
rep.add("LINES", "links", k, "media %08X is not a source in any %s bank" % (m, L))
rep.facts.append("WAD recordings named as a source by a Sound object of their own language's bank: %d of %d"
% (named, total))
rep.facts.append("voice languages %s: %d recordings (WAD %s + localized in-memory), %d lines, %d of them "
"with exactly one recording in every language"
% ("/".join(lines.langs), st["recordings"],
"/".join(str(st["wad_wems"].get(L, 0)) for L in lines.langs), st["lines"],
st["lines_in_every_language"]))
# catalog rows vs the game's lines
rows = {}
for L in lines.langs + [banks.ARCHIVE]:
p = os.path.join(catalog_dir(), "%s.csv" % L)
if os.path.exists(p):
for s in read_csv(p):
rows[s.key] = s
for lid, per in lines.members.items():
for L, k in per.items():
s = rows.get(k)
if s is None:
rep.add("LINES", "line", k, "recording of line %s is not in the catalog" % lid)
elif s.line != lid or s.name != lines.names[lid]:
rep.add("LINES", "line", k, "catalog %s/%s, game %s/%s" % (s.line, s.name, lid, lines.names[lid]))
for k, s in rows.items():
if s.line and lines.line_of.get(k) != s.line:
rep.add("LINES", "line", k, "catalog says line %s, the game data %s" % (s.line, lines.line_of.get(k)))
# prefetch heads: every stored head is the start of the WAD recording it is mapped to
wads = {L: (game.wad(L), Wad(game.wad(L))) for L in lines.langs}
heads = [l for ls in locations.values() for l in ls if l.kind == banks.PREFETCH_HEAD]
by_file = collections.defaultdict(list)
for l in heads:
by_file[l.file].append(l)
same = 0
for rel, ls in by_file.items():
r = banks.open_file(banks.file_path(game.cooked, rel))
for l in ls:
L = l.sound.split(":")[0]
if L not in wads:
rep.add("LINES", "head", "%08X" % l.media, "head in %s maps to %r" % (rel, l.sound))
continue
w = read_target(wads[L][0], l.sound.split(":", 1)[1], wads[L][1])
if r.read(l.offset, l.size) == (w + bytes(l.size))[:l.size]:
same += 1
else:
rep.add("LINES", "head", "%08X" % l.media, "head in %s is not the start of %s" % (rel, l.sound))
same_copies = 0
for k, primary in sorted(lines.copy_of.items()): # one recording stored in two WAD entries
L = k.split(":")[0]
a = read_target(wads[L][0], k.split(":", 1)[1], wads[L][1])
b = read_target(wads[L][0], primary.split(":", 1)[1], wads[L][1])
if a == b:
same_copies += 1
else:
rep.add("LINES", "copies", k, "differs from %s (same media ID)" % primary)
rep.facts.append("recordings stored twice in a WAD (same media ID, written together): %d, byte-identical: %d"
% (len(lines.copy_of), same_copies))
rep.facts.append("prefetch heads: %d of %d stored copies are byte-identical to the start of their WAD recording"
% (same, len(heads)))
# independent evidence: the recordings of a line have similar lengths in every language
ratios = []
for lid, per in lines.members.items():
secs = [rows[k].seconds for k in per.values() if k in rows and rows[k].seconds > 0]
if len(secs) > 1:
ratios.append((max(secs) / min(secs), lid))
if ratios:
ratios.sort()
within = sum(1 for r, _ in ratios if r <= 1.5)
rep.facts.append("lengths across languages: %d of %d lines within a factor 1.5, median %.2f, longest ratio "
"%.1f (%s)" % (within, len(ratios), ratios[len(ratios) // 2][0], ratios[-1][0],
lines.names.get(ratios[-1][1], "")))
return rows
def check_subtitle_lines(rep, rows):
"""A subtitle line (speech event + line index) must point to the same line in every language."""
from ..catalog import _shipped_index
langs = sorted(_shipped_index().get("SUB", {}).get("lines", {}))
ref = {}
p = os.path.join(catalog_dir(), "SUB_INT.csv")
if os.path.exists(p):
for s in subtitles.read_texts(p, "INT"):
if s.sounds and all(k in rows for k in s.sounds):
ref[(s.speech, s.line)] = [rows[k].line for k in s.sounds]
linked = total = 0
for lang in langs:
for s in subtitles.read_texts(os.path.join(catalog_dir(), "SUB_%s.csv" % lang), lang):
total += 1
if not s.sound:
continue
linked += 1
unknown = [k for k in s.sounds if k not in rows]
want = ref.get((s.speech, s.line))
if unknown:
rep.add("SUB", "link", subtitles.key(lang, s.speech, s.line), "linked to unknown %s" % unknown)
elif want and [rows[k].line for k in s.sounds] != want:
rep.add("SUB", "link", subtitles.key(lang, s.speech, s.line),
"lines %s, INT subtitle is on lines %s" % ([rows[k].line for k in s.sounds], want))
rep.facts.append("subtitles: %d strings in %d languages, %d linked to a voice line, each to the same line as in INT"
% (total, len(langs), linked))
SAME_PERSON = {("mr freeze", "mister freeze"), ("batman", "bruce wayne"), ("two face", "twoface"),
("azrael", "mysterious stranger")}
def _person(s):
return " ".join(s.lower().replace("'", "").replace(".", "").replace("_", " ").replace("-", " ").split())
def check_speakers(rep, rows, col):
"""Independent evidence for the curated speaker labels: the dialogue object of a subtitled line names its
speaker. Facts only (the lists are curated by listening; differences are for a person to judge)."""
speaker = {}
for speech, copies in col.dialogues.items():
for _, d in copies:
if d.speaker:
speaker.setdefault(speech, d.speaker)
p = os.path.join(catalog_dir(), "SUB_INT.csv")
if not os.path.exists(p):
return
agree, unknown, differ = 0, 0, []
for s in subtitles.read_texts(p, "INT"):
spk = speaker.get(s.speech, "")
if s.line != 0 or not spk:
continue
for k in s.sounds:
r = rows.get(k)
if not r or not r.character:
continue
who = [_person(c) for c in r.character.split("; ")]
g = _person(spk)
if any(g in w or w in g or (w, g) in SAME_PERSON for w in who):
agree += 1
elif who == ["unknown"]:
unknown += 1
else:
differ.append("%s (list: %s, game: %s)" % (r.name, r.character, spk))
rep.facts.append("speakers: %d labelled voice lines agree with the speaker field of their dialogue object, "
"%d listed as Unknown are named there, %d differ%s"
% (agree, unknown, len(differ), (": " + "; ".join(differ)) if differ else ""))
def check_menu(rep, game):
"""Menu texts: the same keys in every language (facts; differences are the game's own)."""
from .. import loctext
root = os.path.join(game.root, "BmGame", "Localization")
if not os.path.isdir(root):
return
keys = {}
for lang in loctext.languages(root):
try:
keys[lang] = {(os.path.splitext(e.file)[0].upper(), e.section, e.name, e.occurrence)
for e in loctext.load(root, lang)}
except Exception as e: # unreadable folder: report, do not stop
rep.add("MENU", "read", lang, str(e))
if "INT" not in keys:
return
for lang, ks in sorted(keys.items()):
extra, miss = ks - keys["INT"], keys["INT"] - ks
rep.facts.append("menu text %s: %d keys%s" % (lang, len(ks), "" if not (extra or miss) else
" (%d not in INT, %d of INT missing)" % (len(extra), len(miss))))
def run(game, archives=None, quick=False, log=print, progress=None):
@@ -201,14 +420,24 @@ def run(game, archives=None, quick=False, log=print, progress=None):
by_arch = collections.defaultdict(list)
for s in cat.sounds:
by_arch[s.archive].append(s)
lines = None
if banks.ARCHIVE in archives: # first: the full check derives the game's lines from this scan
log("checking %s ..." % banks.ARCHIVE)
rows = load_banks(game, progress)[0] # every language, not only the loaded one
sc, lines, col = check_banks(rep, game, rows, cat.locations, load_characters(lang=None), quick=quick,
progress=progress)
for a in archives:
log("checking %s ..." % a)
characters = load_characters(lang=None if a == banks.ARCHIVE else a)
if a == banks.ARCHIVE:
rows = load_banks(game, progress)[0] # every language, not only the loaded one
check_banks(rep, game, rows, cat.locations, characters, quick=quick, progress=progress)
else:
check_wad(rep, game, a, by_arch[a], characters)
continue
log("checking %s ..." % a)
rows_a = by_arch[a] if a in cat.archives else load_archive(game, a, progress)
check_wad(rep, game, a, rows_a, load_characters(lang=a), lines)
if lines is not None and is_shipped(game, banks.ARCHIVE):
log("checking the relations ...")
rows_all = check_lines(rep, game, sc, lines, cat.locations)
check_subtitle_lines(rep, rows_all)
check_speakers(rep, rows_all, col)
check_menu(rep, game)
characters = load_characters(lang=game.language())
known = {n.lower() for s in cat.sounds for n in s.names()}
# speaker names are the same in every language; only meaningful for the build the lists were made for

70
ackit/verify/ingame.py
View File

@@ -69,18 +69,18 @@ def run(ws, key=DEFAULT_TARGET, wem=None, mode="modded", seconds=40.0, threshold
report = dict(target=s.key, name=s.name, mode=mode)
before = {f: _sha1_file(f) for f in files} if mode == "modded" else {}
if mode == "modded":
ws.apply([(s, wem, {"source": "ingame-test"})], progress=log)
expect = ws.current_bytes(s)
orig = ws.original_bytes(s)
from ..wem import Wem
report["expect_hash"] = "%08x" % Wem.parse(expect).vorb["codebook_hash"]
ref_expect, ref_orig = _mono48(expect), _mono48(orig)
rec = LoopbackRecorder(seconds + 10)
rec.start()
runner = Run(ws.game.exe, logp)
try:
rec = runner = None
try: # from the first write on: whatever happens, the finally block reverts
if mode == "modded":
ws.apply([(s, wem, {"source": "ingame-test"})], progress=log)
expect = ws.current_bytes(s)
orig = ws.original_bytes(s)
from ..wem import Wem
report["expect_hash"] = "%08x" % Wem.parse(expect).vorb["codebook_hash"]
ref_expect, ref_orig = _mono48(expect), _mono48(orig)
rec = LoopbackRecorder(seconds + 10)
rec.start()
runner = Run(ws.game.exe, logp)
report.update(runner.start())
while time.time() - runner.t0 < seconds and runner.alive():
time.sleep(0.5)
@@ -88,9 +88,11 @@ def run(ws, key=DEFAULT_TARGET, wem=None, mode="modded", seconds=40.0, threshold
break
report["alive_at_end"] = runner.alive()
finally:
runner.stop()
rec.stop_flag = True
rec.join(5)
if runner:
runner.stop()
if rec:
rec.stop_flag = True
rec.join(5)
if mode == "modded" and revert_after:
ws.revert([s], progress=log)
report["reverted_byte_identical"] = all(_sha1_file(f) == h for f, h in before.items())
@@ -123,10 +125,11 @@ def run(ws, key=DEFAULT_TARGET, wem=None, mode="modded", seconds=40.0, threshold
# --- texts --------------------------------------------------------------------------------------
def run_texts(ws, seconds=45.0, menu_files=("FRONTEND", "GFXUI"), n_menu=12, n_subs=12, logdir=None, log=print):
"""Write markers into menu texts of the game's language (front-end files, loaded at startup) and into some
subtitles, start the game, then check: no crash, the menu markers are in the game's memory (it loaded the
edited localization files), and after the revert every touched file is byte-identical. Subtitles are not
loaded on the title screen, so for them the check is that the patched packages do not break the game."""
"""Write markers into menu texts of the game's language (front-end files, loaded at startup), into the
subtitles stored in Startup.upk (RSubtitle objects, loaded at startup) and into some other subtitles, start
the game, then check: no crash, the menu and Startup subtitle markers are in the game's memory (it loaded the
edited files), and after the revert every touched file is byte-identical. The other subtitles are not loaded
on the title screen; for them the check is that the patched packages do not break the game."""
import random
from .debugger import Run, kill_game, search_memory
tx = ws.texts
@@ -136,21 +139,31 @@ def run_texts(ws, seconds=45.0, menu_files=("FRONTEND", "GFXUI"), n_menu=12, n_s
picks = rnd.sample(menu, min(n_menu, len(menu)))
changes = {"MENU:" + e.key: "ACKIT%02d %s" % (i, e.value) for i, e in enumerate(picks)}
subs = [s for s in tx.subtitles(lang) if s.room >= 12] if lang in tx.subtitle_languages() else []
for i, s in enumerate(rnd.sample(subs, min(n_subs, len(subs)))):
startup = [s for s in subs if {l.file for l in tx.subtitle_locations("SUB:%s:%s:%d" % (lang, s.speech, s.line))}
== {"Startup.upk"}]
for i, s in enumerate(startup):
changes["SUB:%s:%s:%d" % (lang, s.speech, s.line)] = ("AKST%02d " % i + s.text)[:s.room]
others = [s for s in subs if s not in startup]
for i, s in enumerate(rnd.sample(others, min(n_subs, len(others)))):
changes["SUB:%s:%s:%d" % (lang, s.speech, s.line)] = ("AKS%02d " % i + s.text)[:s.room]
files = sorted({p for k in changes for p in tx.files(k)})
logdir = logdir or os.path.join(os.getcwd(), "logs")
os.makedirs(logdir, exist_ok=True)
logp = os.path.join(logdir, "ingame_texts_%s.log" % time.strftime("%Y%m%d_%H%M%S"))
report = dict(mode="texts", lang=lang, menu_texts=len(picks), subtitles=len(changes) - len(picks),
files=len(files))
report = dict(mode="texts", lang=lang, menu_texts=len(picks), startup_subtitles=len(startup),
subtitles=len(changes) - len(picks), files=len(files))
kill_game()
before = {f: _sha1_file(f) for f in files}
tx.apply(changes, progress=log)
needles = {("ACKIT%02d " % i).encode("utf-16-le") for i in range(len(picks))}
sub_needles = {} # marker -> its byte forms (the game may keep cp1252 or UTF-16)
for i in range(len(startup)):
m = "AKST%02d " % i
sub_needles[m] = {m.encode("utf-16-le"), m.encode("cp1252")}
needles |= {n for v in sub_needles.values() for n in v}
runner = Run(ws.game.exe, logp)
found = set()
try:
try: # from the first write on: the finally block reverts
tx.apply(changes, progress=log)
report.update(runner.start())
while time.time() - runner.t0 < seconds and runner.alive():
time.sleep(0.5)
@@ -164,9 +177,10 @@ def run_texts(ws, seconds=45.0, menu_files=("FRONTEND", "GFXUI"), n_menu=12, n_s
tx.revert(list(changes), progress=log)
report["reverted_byte_identical"] = all(_sha1_file(f) == h for f, h in before.items())
report["crash"] = bool(re.search(r"^ACKIT_CRASH", runner.log(), re.M))
report["menu_markers_in_memory"] = len(found)
report["menu_markers_in_memory"] = sum(1 for i in range(len(picks)) if ("ACKIT%02d " % i).encode("utf-16-le") in found)
report["startup_subtitles_in_memory"] = sum(1 for v in sub_needles.values() if v & found)
ok = (not report["crash"] and report["alive_at_end"] and report["menu_markers_in_memory"] > 0
and report["reverted_byte_identical"])
and (not startup or report["startup_subtitles_in_memory"] > 0) and report["reverted_byte_identical"])
report["result"] = "PASS" if ok else "FAIL"
json.dump(report, open(logp.replace(".log", ".json"), "w"), indent=1)
return report
@@ -221,7 +235,6 @@ def run_banks(ws, keys=None, seconds=45.0, logdir=None, log=print):
if len(w) > 400:
needles.add(w[len(w) // 2:len(w) // 2 + 32])
report.update(probe_hashes=sorted(hashes), needles=len(needles))
ws.apply([(s, w, {"source": "ingame-test"}) for s, w in probes], progress=log)
from .debugger import ADDR
# also log every setup parse (in-memory sources may not go through the stream setup cache)
extra = 'bp %#x ".printf \\"ACKIT_PARSE2 hash=%%08x\\\\n\\", poi(poi(@esp+4)+0x9c); gc"' % ADDR["parse_setup"]
@@ -229,7 +242,8 @@ def run_banks(ws, keys=None, seconds=45.0, logdir=None, log=print):
found = set()
rec = LoopbackRecorder(seconds + 10)
rec.start()
try:
try: # from the first write on: the finally block reverts
ws.apply([(s, w, {"source": "ingame-test"}) for s, w in probes], progress=log)
report.update(runner.start())
pid = report["pid"]
while time.time() - runner.t0 < seconds and runner.alive():

14
ackit/workspace.py
View File

@@ -73,6 +73,10 @@ class Workspace:
return self.catalog.locations.get(int(target, 16), [])
return self.catalog.heads.get(key, [])
def copies(self, s):
"""Other places of the same recording in the WAD (written and reverted together with `s`)."""
return self.catalog.copies.get(s.key, [])
def files(self, s):
"""Absolute paths of the game files that writing `s` changes."""
out = []
@@ -80,7 +84,7 @@ class Workspace:
out.append(self.wad_path(s))
if s.is_pack_media:
out.append(pack_bnk_path(self.wad_path(s), parse_target(s.target)[0]))
out += sorted({self.bank_path(l.file) for l in self.locations(s)})
out += sorted({self.bank_path(l.file) for x in [s] + self.copies(s) for l in self.locations(x)})
return out
def current_bytes(self, s):
@@ -286,7 +290,11 @@ class Workspace:
self.sync_files(files, progress)
def apply(self, changes, force=False, progress=None):
"""changes: [(sound, wem_bytes, meta dict)]. One atomic rewrite per WAD / bank file."""
"""changes: [(sound, wem_bytes, meta dict)]. One atomic rewrite per WAD / bank file. A recording stored
in two places of a WAD gets the same bytes in both (Workspace.copies)."""
keys = {s.key for s, _, _ in changes}
changes = list(changes) + [(c, data, meta) for s, data, meta in changes for c in self.copies(s)
if c.key not in keys]
conflicts = []
for s, _, _ in changes:
if self.state(s) in (CHANGED, FOREIGN):
@@ -306,6 +314,8 @@ class Workspace:
"""Put the stored originals back (sounds without a record are skipped). Returns reverted."""
by_arch = defaultdict(list)
done = []
keys = {s.key for s in sounds}
sounds = list(sounds) + [c for s in sounds for c in self.copies(s) if c.key not in keys]
for s in sounds:
orig = self.store.original(s.key)
if orig is None: