225 lines
11 KiB
Python
225 lines
11 KiB
Python
"""Speech lines across languages, derived only from the game's own data (docs/formats/banks.md, "Lines").
|
|
|
|
A voice recording exists once per language. The game ties them together itself:
|
|
- every recording of a speech WAD has exactly one row in its language's BKPK table (media ID, offset, size);
|
|
the table's language is the one its localized AkBank export declares (banks.py: localized_exports);
|
|
- every recording, streamed, prefetched or in memory, is the source of a Wwise Sound object. Sound objects
|
|
(and the events, actions and containers above them) have the same IDs in every language; only the media
|
|
differ (a few media IDs are shared by several languages, each at its own position in each WAD).
|
|
So a *line* is the set of recordings whose media are the sources of the same Sound objects: one recording per
|
|
language. Its identity (`Sound.line`) is the smallest Sound object ID (8 hex digits), its name is chosen once
|
|
for all languages (`line_names`). A media stored in two entries of one WAD is one recording with a copy
|
|
(`copy_of`), written together. Nothing here depends on file positions or on names.
|
|
"""
|
|
import collections
|
|
from dataclasses import dataclass, field
|
|
|
|
from . import banks
|
|
from .wad import Wad
|
|
|
|
|
|
@dataclass
|
|
class Lines:
|
|
langs: list # voice languages (speech WADs)
|
|
owners: dict = field(default_factory=dict) # lang -> {media id: sound key} (WAD line or BANK sound)
|
|
line_of: dict = field(default_factory=dict) # sound key -> line id
|
|
members: dict = field(default_factory=dict) # line id -> {lang: sound key}
|
|
objects: dict = field(default_factory=dict) # line id -> sorted Sound object ids
|
|
obj_keys: dict = field(default_factory=dict) # Sound object id -> {lang: sound key}
|
|
names: dict = field(default_factory=dict) # line id -> name (catalog.line_names)
|
|
unnamed: set = field(default_factory=set) # line ids no named event plays (named after a recording)
|
|
copy_of: dict = field(default_factory=dict) # WAD key -> key of the same media stored in another entry
|
|
issues: list = field(default_factory=list) # (kind, detail): broken one-to-one relations
|
|
stats: dict = field(default_factory=dict)
|
|
_keys: dict = field(default_factory=dict, repr=False) # lang -> set of sound keys
|
|
|
|
def key_langs(self, key):
|
|
return [L for L in self.langs if key in self._keys.get(L, ())]
|
|
|
|
def head_key(self, loc):
|
|
"""Speech line (sound key) of a prefetch-head copy: the line of the copy's bank language that has
|
|
this media. A head in a language-neutral bank belongs to the one line that uses the media."""
|
|
if loc.lang:
|
|
return self.owners.get(loc.lang, {}).get(loc.media, "")
|
|
ks = {d[loc.media] for d in self.owners.values() if loc.media in d}
|
|
return ks.pop() if len(ks) == 1 else ""
|
|
|
|
|
|
def wad_owners(game, sc, langs, walk):
|
|
"""({lang: {media id: WAD sound key}}, {copy key: key}, issues, WEM counts) through each language's own BKPK
|
|
tables. Issues: a WEM without a table row, a row without a WEM. A media stored in two entries of one
|
|
language's WAD (a few lines, byte-identical) has one key; the other place is a copy written with it.
|
|
`walk(f, offset, size)` -> [(offset, length, header)] (catalog._walk_entry)."""
|
|
owners, issues, wems = {}, [], collections.Counter()
|
|
places = {}
|
|
for L in langs:
|
|
owners[L] = {}
|
|
places[L] = collections.defaultdict(list)
|
|
wad = Wad(game.wad(L))
|
|
with open(game.wad(L), "rb") as f:
|
|
for e in wad.entries:
|
|
tables = sc.packs.get(e.hash, {})
|
|
table = tables.get(L) if L in tables else tables.get("")
|
|
ws = walk(f, e.offset, e.size)
|
|
wems[L] += len(ws)
|
|
if table is None:
|
|
issues.append(("no table", "%s:%08X: %d WEMs, no BKPK table" % (L, e.hash, len(ws))))
|
|
continue
|
|
at = {(o, ln): i for i, (o, ln, _) in enumerate(ws)}
|
|
used = set()
|
|
for m, v in table.items():
|
|
if v not in at:
|
|
issues.append(("row without WEM", "%s:%08X: media %08X" % (L, e.hash, m)))
|
|
continue
|
|
used.add(at[v])
|
|
places[L][m].append("%s:%08X" % (L, e.hash) + (":%d" % at[v] if len(ws) > 1 else ""))
|
|
for i in sorted(set(range(len(ws))) - used):
|
|
issues.append(("WEM without row", "%s:%08X:%d" % (L, e.hash, i)))
|
|
copy_of = {}
|
|
for L in langs:
|
|
for m, ks in places[L].items():
|
|
ks = sorted(ks)
|
|
owners[L][m] = ks[0]
|
|
for k in ks[1:]:
|
|
copy_of[k] = ks[0]
|
|
return owners, copy_of, issues, dict(wems)
|
|
|
|
|
|
def bank_langs(sc, m):
|
|
"""Voice languages of a localized in-memory media: the languages of the banks holding it ('' = neutral)."""
|
|
ls = sc.media[m].get("langs", set())
|
|
return () if "" in ls else tuple(sorted(ls))
|
|
|
|
|
|
def build(game, sc, walk):
|
|
"""Lines of every voice language of the game (see module doc)."""
|
|
in_banks = sorted(set().union(*(e.get("langs", set()) for e in sc.media.values())) - {""} - set(game.languages))
|
|
langs = list(game.languages) + in_banks # voice languages: speech WADs + localized banks
|
|
lines = Lines(langs)
|
|
owners, copy_of, issues, wems = wad_owners(game, sc, game.languages, walk)
|
|
for L in in_banks:
|
|
owners[L] = {}
|
|
lines.issues += issues
|
|
lines.copy_of = copy_of
|
|
for m, e in sc.media.items(): # localized in-memory recordings (not prefetch heads)
|
|
if banks.PREFETCH_HEAD in e["kinds"]:
|
|
continue
|
|
for L in bank_langs(sc, m):
|
|
if L in owners:
|
|
owners[L].setdefault(m, "%s:%08X" % (banks.ARCHIVE, m))
|
|
lines.owners = owners
|
|
lines._keys = {L: set(d.values()) for L, d in owners.items()}
|
|
|
|
# Sound object -> its recording in each language (exactly one, or the relation is broken)
|
|
for o, ms in sc.sound_media.items():
|
|
per = {}
|
|
for L in langs:
|
|
ks = {owners[L][m] for m in ms if m in owners[L]}
|
|
if len(ks) > 1:
|
|
lines.issues.append(("object", "Sound %08X has %d recordings in %s: %s" % (o, len(ks), L, sorted(ks))))
|
|
elif ks:
|
|
per[L] = ks.pop()
|
|
if per:
|
|
lines.obj_keys[o] = per
|
|
|
|
# lines = connected recordings (union-find over the objects' recordings)
|
|
parent = {}
|
|
|
|
def find(x):
|
|
while parent.get(x, x) != x:
|
|
parent[x] = parent.get(parent[x], parent[x])
|
|
x = parent[x]
|
|
return x
|
|
|
|
for o, per in lines.obj_keys.items():
|
|
ks = sorted(set(per.values()))
|
|
for k in ks[1:]:
|
|
a, b = find(ks[0]), find(k)
|
|
if a != b:
|
|
parent[max(a, b)] = min(a, b)
|
|
groups = collections.defaultdict(set)
|
|
objs = collections.defaultdict(set)
|
|
for o, per in lines.obj_keys.items():
|
|
for k in per.values():
|
|
groups[find(k)].add(k)
|
|
objs[find(k)].add(o)
|
|
for root, ks in groups.items():
|
|
per = {}
|
|
ok = True
|
|
for k in ks:
|
|
for L in lines.key_langs(k):
|
|
if L in per and per[L] != k:
|
|
ok = False
|
|
per.setdefault(L, k)
|
|
if not ok:
|
|
lines.issues.append(("line", "recordings %s share Sound objects but several are in one language" % sorted(ks)))
|
|
continue
|
|
lid = "%08X" % min(objs[root])
|
|
lines.members[lid] = per
|
|
lines.objects[lid] = sorted(objs[root])
|
|
for k in ks:
|
|
lines.line_of[k] = lid
|
|
for k, primary in copy_of.items(): # a second place of the same recording: the same line
|
|
if primary in lines.line_of:
|
|
lines.line_of[k] = lines.line_of[primary]
|
|
recordings = set().union(*lines._keys.values()) if lines._keys else set()
|
|
all_keys = recordings | set(copy_of) # + the second places of recordings stored twice
|
|
full = sum(1 for per in lines.members.values() if len(per) == len(langs))
|
|
lines.stats = {"languages": langs, "wad_wems": wems, "recordings": len(recordings),
|
|
"in_a_line": len(lines.line_of), "lines": len(lines.members), "lines_in_every_language": full,
|
|
"copies": sorted(copy_of),
|
|
"lines_missing_a_language": {lid: sorted(set(langs) - set(per))
|
|
for lid, per in lines.members.items() if len(per) < len(langs)},
|
|
"without_line": sorted(all_keys - set(lines.line_of))}
|
|
return lines
|
|
|
|
|
|
def line_names(sc, lines, fallback):
|
|
"""{line id: name}, the same in every language. A line is named after the game event that plays it: for
|
|
WAD lines the most specific `Speech_` event (fewest lines), for in-memory lines the most specific event
|
|
(`Play_` first), as the in-memory sounds always were. When an event plays several lines they are numbered
|
|
in the event's own order (the same in every language): NAME, NAME#2, ... (WAD) or NAME_1, NAME_2, ... (in
|
|
memory). A line no named event plays keeps the name its English recording has (`fallback(key)`: the
|
|
position-based name in the INT WAD, or the bank-numbered name), else the first voice language's."""
|
|
obj_line = {o: lid for lid, os_ in lines.objects.items() for o in os_}
|
|
is_wad = {lid: any(not k.startswith(banks.ARCHIVE + ":") for k in per.values()) for lid, per in lines.members.items()}
|
|
best = {}
|
|
reach = {}
|
|
for name, objs in banks.event_objects(sc).items():
|
|
order = []
|
|
for o in objs:
|
|
lid = obj_line.get(o)
|
|
if lid and lid not in order:
|
|
order.append(lid)
|
|
if order:
|
|
reach[name] = order
|
|
for name, order in reach.items():
|
|
low = name.lower()
|
|
for lid in order:
|
|
if is_wad[lid]:
|
|
if not low.startswith("speech_"):
|
|
continue
|
|
rank = (len(order), low)
|
|
else:
|
|
rank = (0 if low.startswith("play") else 1, len(order), low)
|
|
if lid not in best or rank < best[lid][0]:
|
|
best[lid] = (rank, name)
|
|
named = collections.defaultdict(list) # event -> lines it names, in the event's order
|
|
for name, order in reach.items():
|
|
for lid in order:
|
|
if lid in best and best[lid][1] == name:
|
|
named[name].append(lid)
|
|
out = {}
|
|
for name, lids in named.items():
|
|
for i, lid in enumerate(lids):
|
|
if is_wad[lid]:
|
|
out[lid] = name if i == 0 else "%s#%d" % (name, i + 1)
|
|
else:
|
|
out[lid] = name if len(lids) == 1 else "%s_%d" % (name, i + 1)
|
|
lines.unnamed = set(lines.members) - set(out)
|
|
for lid in lines.unnamed:
|
|
per = lines.members[lid]
|
|
k = per.get("INT") or per[next(L for L in lines.langs if L in per)]
|
|
out[lid] = fallback(k)
|
|
return out
|