"""Speech lines across languages, derived only from the game's own data (docs/formats/banks.md, "Lines"). A voice recording exists once per language. The game ties them together itself: - every recording of a speech WAD has exactly one row in its language's BKPK table (media ID, offset, size); the table's language is the one its localized AkBank export declares (banks.py: localized_exports); - every recording, streamed, prefetched or in memory, is the source of a Wwise Sound object. Sound objects (and the events, actions and containers above them) have the same IDs in every language; only the media differ (a few media IDs are shared by several languages, each at its own position in each WAD). So a *line* is the set of recordings whose media are the sources of the same Sound objects: one recording per language. Its identity (`Sound.line`) is the smallest Sound object ID (8 hex digits), its name is chosen once for all languages (`line_names`). A media stored in two entries of one WAD is one recording with a copy (`copy_of`), written together. Nothing here depends on file positions or on names. """ import collections from dataclasses import dataclass, field from . import banks from .wad import Wad @dataclass class Lines: langs: list # voice languages (speech WADs) owners: dict = field(default_factory=dict) # lang -> {media id: sound key} (WAD line or BANK sound) line_of: dict = field(default_factory=dict) # sound key -> line id members: dict = field(default_factory=dict) # line id -> {lang: sound key} objects: dict = field(default_factory=dict) # line id -> sorted Sound object ids obj_keys: dict = field(default_factory=dict) # Sound object id -> {lang: sound key} names: dict = field(default_factory=dict) # line id -> name (catalog.line_names) copy_of: dict = field(default_factory=dict) # WAD key -> key of the same media stored in another entry issues: list = field(default_factory=list) # (kind, detail): broken one-to-one relations stats: dict = field(default_factory=dict) _keys: dict = field(default_factory=dict, repr=False) # lang -> set of sound keys def key_langs(self, key): return [L for L in self.langs if key in self._keys.get(L, ())] def head_key(self, loc): """Speech line (sound key) of a prefetch-head copy: the line of the copy's bank language that has this media. A head in a language-neutral bank belongs to the one line that uses the media.""" if loc.lang: return self.owners.get(loc.lang, {}).get(loc.media, "") ks = {d[loc.media] for d in self.owners.values() if loc.media in d} return ks.pop() if len(ks) == 1 else "" def wad_owners(game, sc, langs, walk): """({lang: {media id: WAD sound key}}, {copy key: key}, issues, WEM counts) through each language's own BKPK tables. Issues: a WEM without a table row, a row without a WEM. A media stored in two entries of one language's WAD (a few lines, byte-identical) has one key; the other place is a copy written with it. `walk(f, offset, size)` -> [(offset, length, header)] (catalog._walk_entry).""" owners, issues, wems = {}, [], collections.Counter() places = {} for L in langs: owners[L] = {} places[L] = collections.defaultdict(list) wad = Wad(game.wad(L)) with open(game.wad(L), "rb") as f: for e in wad.entries: tables = sc.packs.get(e.hash, {}) table = tables.get(L) if L in tables else tables.get("") ws = walk(f, e.offset, e.size) wems[L] += len(ws) if table is None: issues.append(("no table", "%s:%08X: %d WEMs, no BKPK table" % (L, e.hash, len(ws)))) continue at = {(o, ln): i for i, (o, ln, _) in enumerate(ws)} used = set() for m, v in table.items(): if v not in at: issues.append(("row without WEM", "%s:%08X: media %08X" % (L, e.hash, m))) continue used.add(at[v]) places[L][m].append("%s:%08X" % (L, e.hash) + (":%d" % at[v] if len(ws) > 1 else "")) for i in sorted(set(range(len(ws))) - used): issues.append(("WEM without row", "%s:%08X:%d" % (L, e.hash, i))) copy_of = {} for L in langs: for m, ks in places[L].items(): ks = sorted(ks) owners[L][m] = ks[0] for k in ks[1:]: copy_of[k] = ks[0] return owners, copy_of, issues, dict(wems) def bank_langs(sc, m): """Voice languages of a localized in-memory media: the languages of the banks holding it ('' = neutral).""" ls = sc.media[m].get("langs", set()) return () if "" in ls else tuple(sorted(ls)) def build(game, sc, walk): """Lines of every voice language of the game (see module doc).""" in_banks = sorted(set().union(*(e.get("langs", set()) for e in sc.media.values())) - {""} - set(game.languages)) langs = list(game.languages) + in_banks # voice languages: speech WADs + localized banks lines = Lines(langs) owners, copy_of, issues, wems = wad_owners(game, sc, game.languages, walk) for L in in_banks: owners[L] = {} lines.issues += issues lines.copy_of = copy_of for m, e in sc.media.items(): # localized in-memory recordings (not prefetch heads) if banks.PREFETCH_HEAD in e["kinds"]: continue for L in bank_langs(sc, m): if L in owners: owners[L].setdefault(m, "%s:%08X" % (banks.ARCHIVE, m)) lines.owners = owners lines._keys = {L: set(d.values()) for L, d in owners.items()} # Sound object -> its recording in each language (exactly one, or the relation is broken) for o, ms in sc.sound_media.items(): per = {} for L in langs: ks = {owners[L][m] for m in ms if m in owners[L]} if len(ks) > 1: lines.issues.append(("object", "Sound %08X has %d recordings in %s: %s" % (o, len(ks), L, sorted(ks)))) elif ks: per[L] = ks.pop() if per: lines.obj_keys[o] = per # lines = connected recordings (union-find over the objects' recordings) parent = {} def find(x): while parent.get(x, x) != x: parent[x] = parent.get(parent[x], parent[x]) x = parent[x] return x for o, per in lines.obj_keys.items(): ks = sorted(set(per.values())) for k in ks[1:]: a, b = find(ks[0]), find(k) if a != b: parent[max(a, b)] = min(a, b) groups = collections.defaultdict(set) objs = collections.defaultdict(set) for o, per in lines.obj_keys.items(): for k in per.values(): groups[find(k)].add(k) objs[find(k)].add(o) for root, ks in groups.items(): per = {} ok = True for k in ks: for L in lines.key_langs(k): if L in per and per[L] != k: ok = False per.setdefault(L, k) if not ok: lines.issues.append(("line", "recordings %s share Sound objects but several are in one language" % sorted(ks))) continue lid = "%08X" % min(objs[root]) lines.members[lid] = per lines.objects[lid] = sorted(objs[root]) for k in ks: lines.line_of[k] = lid for k, primary in copy_of.items(): # a second place of the same recording: the same line if primary in lines.line_of: lines.line_of[k] = lines.line_of[primary] all_keys = set().union(*lines._keys.values()) if lines._keys else set() all_keys |= set(copy_of) full = sum(1 for per in lines.members.values() if len(per) == len(langs)) lines.stats = {"languages": langs, "wad_wems": wems, "recordings": len(all_keys), "in_a_line": len(lines.line_of), "lines": len(lines.members), "lines_in_every_language": full, "copies": sorted(copy_of), "lines_missing_a_language": {lid: sorted(set(langs) - set(per)) for lid, per in lines.members.items() if len(per) < len(langs)}, "without_line": sorted(all_keys - set(lines.line_of))} return lines def line_names(sc, lines, fallback): """{line id: name}, the same in every language. A line is named after the game event that plays it: for WAD lines the most specific `Speech_` event (fewest lines), for in-memory lines the most specific event (`Play_` first), as the in-memory sounds always were. When an event plays several lines they are numbered in the event's own order (the same in every language): NAME, NAME#2, ... (WAD) or NAME_1, NAME_2, ... (in memory). A line no named event plays keeps the name its English recording has (`fallback(key)`: the position-based name in the INT WAD, or the bank-numbered name), else the first voice language's.""" obj_line = {o: lid for lid, os_ in lines.objects.items() for o in os_} is_wad = {lid: any(not k.startswith(banks.ARCHIVE + ":") for k in per.values()) for lid, per in lines.members.items()} best = {} reach = {} for name, objs in banks.event_objects(sc).items(): order = [] for o in objs: lid = obj_line.get(o) if lid and lid not in order: order.append(lid) if order: reach[name] = order for name, order in reach.items(): low = name.lower() for lid in order: if is_wad[lid]: if not low.startswith("speech_"): continue rank = (len(order), low) else: rank = (0 if low.startswith("play") else 1, len(order), low) if lid not in best or rank < best[lid][0]: best[lid] = (rank, name) named = collections.defaultdict(list) # event -> lines it names, in the event's order for name, order in reach.items(): for lid in order: if lid in best and best[lid][1] == name: named[name].append(lid) out = {} for name, lids in named.items(): for i, lid in enumerate(lids): if is_wad[lid]: out[lid] = name if i == 0 else "%s#%d" % (name, i + 1) else: out[lid] = name if len(lids) == 1 else "%s_%d" % (name, i + 1) for lid, per in lines.members.items(): if lid not in out: k = per.get("INT") or per[next(L for L in lines.langs if L in per)] out[lid] = fallback(k) return out