gora/UnheardCity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version 3.1.0: voice lines linked across all languages from the game data

Browse Source
This commit is contained in:
Cristian Gora
2026-10-10 14:10:03 +02:00
parent ea3e44000a
commit fbde809370
56 changed files with 74728 additions and 73360 deletions
Download Patch File Download Diff File Expand all files Collapse all files

256
ackit/catalog.py
View File

@@ -19,15 +19,17 @@ import struct
from dataclasses import dataclass, asdict
from . import banks, classify, subtitles
from . import lines as lines_mod
from .game import wad_fingerprint
from .lines import bank_langs
from .inject import pack_bnk_path
from .pack import Pack
from .paths import catalog_dir, package_data, user_dir
from .wad import Wad
FIELDS = ["archive", "target", "name", "aliases", "group", "category", "bank", "language", "character",
FIELDS = ["archive", "target", "name", "aliases", "group", "category", "bank", "language", "line", "character",
"characters_inferred", "seconds", "rate", "channels", "size", "sha1", "notes"]
SCAN_VERSION = 2
SCAN_VERSION = 3
@dataclass
@@ -39,7 +41,8 @@ class Sound:
group: str = "Unknown"
category: str = "Unknown"
bank: str = "" # BANK sounds: the soundbank holding it (name); WAD sounds: ''
language: str = "" # BANK sounds: language of a localized speech line ('' = any language)
language: str = "" # BANK sounds: language(s) of a localized speech line, ';'-separated ('' = every language)
line: str = "" # speech: the game's identity of the line in every language (ackit/lines.py)
character: str = "" # verified speaker(s), '; '-separated (curated catalog)
characters_inferred: str = "" # characters/roles mentioned by the name (hint)
seconds: float = 0.0
@@ -65,6 +68,14 @@ class Sound:
def names(self):
return (self.name,) + tuple(self.aliases)
@property
def languages(self):
"""Voice languages of a localized in-memory sound (() = every language)."""
return tuple(self.language.split(";")) if self.language else ()
def in_language(self, lang):
return not self.language or lang in self.languages
# --- static data ---------------------------------------------------------------------------
_names = None
@@ -86,8 +97,8 @@ def hash_names():
def load_characters(path=None, lang=None):
"""{sound name (lower): 'Speaker' or 'A; B'} from data/catalog/characters.csv: the rows valid in every
language (keyed by the game's line names) plus the rows of `lang` (lines without a line name, labelled
by their position in that language's WAD)."""
language (keyed by line names, the same line everywhere) plus the rows of `lang` (a sound of that
language only; none at present)."""
path = path or os.path.join(catalog_dir(), "characters.csv")
out = {}
if os.path.exists(path):
@@ -98,13 +109,9 @@ def load_characters(path=None, lang=None):
def character_of(s, characters):
"""Verified speaker of a sound from a load_characters() map: by its names, and for the k-th take of a
line shown as NAME#k (an event playing several WEMs of one language) by the line name NAME."""
for n in s.names() + (s.name.split("#")[0],):
c = characters.get(n.lower())
if c:
return c
return ""
"""Verified speaker of a sound from a load_characters() map, by its name only: names are unique in a
language and a line has the same name in every language, while aliases (positions) are local."""
return characters.get(s.name.lower(), "")
def read_csv(path):
@@ -113,6 +120,7 @@ def read_csv(path):
out.append(Sound(archive=r["archive"], target=r["target"], name=r["name"],
aliases=tuple(a for a in r["aliases"].split("|") if a), group=r["group"],
category=r["category"], bank=r.get("bank") or "", language=r.get("language") or "",
line=r.get("line") or "",
character=r["character"],
characters_inferred=r["characters_inferred"], seconds=float(r["seconds"] or 0),
rate=int(r["rate"] or 0), channels=int(r["channels"] or 0), size=int(r["size"] or 0),
@@ -221,17 +229,16 @@ def bank_sounds(sc, characters=None):
"""[Sound] of the in-memory sounds found by `banks.scan` (prefetch heads belong to WAD sounds)."""
characters = characters if characters is not None else load_characters()
names = banks.media_names(sc)
first = {}
for loc in sorted(sc.locations, key=lambda l: (l.media, l.file, l.offset)):
first.setdefault(loc.media, loc)
every = set().union(*(e.get("langs", set()) for e in sc.media.values())) - {""}
out = []
for m, e in sc.media.items():
if banks.PREFETCH_HEAD in e["kinds"]:
continue
name, bank = names[m]
rate, ch, secs = _hdr(e["head"])
ls = bank_langs(sc, m) # localized: the languages whose banks hold it
s = Sound(banks.ARCHIVE, "%08X" % m, name, ("%s/%08X" % (bank, m),), bank=bank,
language=banks.media_language(sc, m), seconds=secs, rate=rate, channels=ch,
language="" if set(ls) == every else ";".join(ls), seconds=secs, rate=rate, channels=ch,
size=min(l.size for l in sc.locations if l.media == m) if len(e["sha1"]) > 1 else e["size"],
sha1=next(iter(e["sha1"])) if len(e["sha1"]) == 1 else "")
s.group, s.category = classify.classify(name, bank=bank)
@@ -242,127 +249,134 @@ def bank_sounds(sc, characters=None):
return out
def head_owners(game, sc, archives=None):
"""{prefetch media id: WAD sound key} -- the speech line (one language) each prefetch head belongs to."""
return media_owners(game, sc, {m for m, e in sc.media.items() if banks.PREFETCH_HEAD in e["kinds"]}, archives)
def media_owners(game, sc, want, archives=None):
"""{media id: WAD sound key} for streamed speech media, found through the per-language BKPK tables of
the packages (docs/formats/banks.md): every language has its own media IDs for the same WAD entry."""
out = {}
for a in archives or [x for x in game.archives if x != "SFX"]:
wad = Wad(game.wad(a))
by_hash = {e.hash: e for e in wad.entries}
with open(game.wad(a), "rb") as f:
for ent, table in sc.packs.items():
mine = [m for m in table if m in want and m not in out]
e = by_hash.get(ent)
if not mine or not e:
continue
wems = _walk_entry(f, e.offset, e.size)
for m in mine:
off, size = table[m]
for i, (o, ln, _) in enumerate(wems):
if (o, ln) == (off, size):
out[m] = "%s:%08X" % (a, ent) + (":%d" % i if len(wems) > 1 else "")
return out
def build_lines(game, sc):
"""The game's speech lines across languages (ackit/lines.py)."""
return lines_mod.build(game, sc, _walk_entry)
def scan_banks(game, progress=None, characters=None, visit=None):
"""Full scan of the game's banks -> ([Sound], [Location] with WAD keys on prefetch heads, Scan)."""
"""Full scan of the game's banks -> ([Sound], [Location] with WAD keys on prefetch heads, Scan, Lines).
Localized in-memory speech lines carry their line identity and the name shared by every language."""
sc = banks.scan(game.cooked, progress, visit=visit)
owners = head_owners(game, sc)
locs = [l._replace(sound=owners.get(l.media, "")) if l.kind == banks.PREFETCH_HEAD else l for l in sc.locations]
return bank_sounds(sc, characters), locs, sc
snd, locs, lines = derive_banks(game, sc, characters)
return snd, locs, sc, lines
def speech_line_names(game, sc):
"""{WAD sound key: the game's name of that line} for every speech sound some event plays. The name is the
speech event (e.g. Speech_CH_ZERO_CH0_fight_intro_01): the same line has the same name in every language,
while the position of a variation inside a WAD entry differs between languages (WEMs are stored in
media-ID order and every language has its own media IDs). If several events reach a sound, the most
specific one (fewest sounds in that language) names it."""
ev = {n: ms for n, ms in banks.event_media(sc).items() if n.lower().startswith("speech_")}
owners = media_owners(game, sc, {m for ms in ev.values() for m in ms})
best = {}
for n, ms in ev.items():
keys = collections.defaultdict(set)
for m in ms:
if m in owners:
k = owners[m]
keys[k.split(":")[0]].add(k)
for lang, ks in keys.items():
for k in ks:
rank = (len(ks), n.lower())
if k not in best or rank < best[k][0]:
best[k] = (rank, n)
return {k: n for k, (_, n) in best.items()}
def derive_banks(game, sc, characters=None):
"""(bank sounds, locations, Lines) from a Scan: the lines, the prefetch heads' lines, the in-memory sounds
with their line identities and names."""
lines = build_lines(game, sc)
locs = [l._replace(sound=lines.head_key(l)) if l.kind == banks.PREFETCH_HEAD else l for l in sc.locations]
snd = bank_sounds(sc, characters)
lines.names = line_names(game, sc, lines, {s.key: s.name for s in snd})
apply_lines(snd, lines, lines.names)
return snd, locs, lines
def apply_line_names(sounds, line_names):
"""Rename WAD speech sounds after the game's line names (see speech_line_names). The position-based name
stays available as alias NAME#i (single-WEM entries keep their entry name as alias)."""
used = collections.Counter()
def line_names(game, sc, lines, bank_names=None):
"""{line id: name} (lines.line_names). Lines no event names keep their English recording's name: the
position-based name in the INT WAD (`NAME_i`, or `NAME#i` where that is taken) or the bank-numbered name."""
pos = {}
for L in {k.split(":")[0] for per in lines.members.values() for k in per.values()} - {banks.ARCHIVE}:
for s in scan_archive(L, game.wad(L), characters={}):
pos[s.key] = s
bank_names = bank_names if bank_names is not None else {}
def fallback(k):
return pos[k].name if k in pos else bank_names.get(k, k)
out = lines_mod.line_names(sc, lines, fallback)
taken = collections.Counter(n.lower() for n in out.values())
for lid, n in out.items(): # a fallback name equal to an event-based one -> NAME#i of the INT WEM
if taken[n.lower()] > 1:
k = lines.members[lid].get("INT") or next(iter(lines.members[lid].values()))
alt = [a for a in (pos[k].names() if k in pos else ()) if "#" in a]
if alt and alt[0] != n:
out[lid] = alt[0]
return out
def apply_lines(sounds, lines, names):
"""Give every speech sound of one archive (a WAD language or BANK) its line identity and the line's name,
the same in every language. A WAD sound keeps its position-based names as aliases (NAME#i: position in
this language's WAD; single-WEM entries: the entry name). Names stay unique in the archive (and, for
BANK, per language together with the language-neutral sounds): a clash is resolved by the sound that
is not the line's own name falling back to its position name / media-ID name."""
for s in sounds:
ev = line_names.get(s.key)
if not ev:
lid = lines.line_of.get(s.key)
if not lid:
continue
used[ev.lower()] += 1
name = ev if used[ev.lower()] == 1 else "%s#%d" % (ev, used[ev.lower()])
if ":" in s.target: # variation: keep NAME#i (position in this language's WAD)
pos = [a for a in s.names() if "#" in a]
s.aliases = tuple(pos)
else: # single WEM: the entry name stays an alias
s.aliases = (s.name,) + tuple(a for a in s.aliases if a != s.name)
s.line = lid
if s.key in lines.copy_of: # a second place of a recording: keeps its position name, written with it
continue
name = names[lid]
if s.archive != banks.ARCHIVE:
if ":" in s.target:
s.aliases = tuple(a for a in s.names() if "#" in a)
else:
s.aliases = (s.name,) + tuple(a for a in s.aliases if a != s.name)
if name in s.aliases:
s.aliases = tuple(a for a in s.aliases if a != name)
s.name = name
taken = collections.Counter(s.name.lower() for s in sounds)
for s in sounds: # a positional name equal to another sound's line name -> NAME#i
if taken[s.name.lower()] > 1 and s.key not in line_names and ":" in s.target:
pos = [a for a in s.aliases if "#" in a]
if pos:
s.aliases, s.name = (s.name,), pos[0]
neutral, by_lang = set(), collections.defaultdict(set) # lower-case names taken
def clash(s, n):
if s.archive != banks.ARCHIVE or not s.languages:
return n in neutral or any(n in v for v in by_lang.values())
return n in neutral or any(n in by_lang[L] for L in s.languages)
# a line's own recordings first: they keep the line's name in a clash (copies and others give way)
for s in sorted(sounds, key=lambda s: (not s.line or s.key in lines.copy_of, s.key)):
if clash(s, s.name.lower()):
if s.archive == banks.ARCHIVE:
s.name = "%s_%s" % (s.name, s.target)
else:
alt = [a for a in s.aliases if "#" in a and a != s.name]
s.aliases, s.name = (s.name,), alt[0] if alt else "%s@%s" % (s.name, s.target)
if s.archive == banks.ARCHIVE and s.languages:
for L in s.languages:
by_lang[L].add(s.name.lower())
else:
neutral.add(s.name.lower())
for s in sounds:
s.group, s.category = classify.classify(s.name)
s.characters_inferred = "; ".join(classify.infer_characters(s.name))
if s.archive != banks.ARCHIVE:
s.group, s.category = classify.classify(s.name)
s.characters_inferred = "; ".join(classify.infer_characters(s.name))
return sounds
def subtitle_links(game, sc, speech_names):
"""{speech event name: {language: sound key}}: event -> actions -> containers -> sounds -> the
per-language media of the line (exact, no name matching). The media is a streamed WAD sound, or an
in-memory BANK sound of a localized bank."""
ev = banks.event_media(sc)
want = {m for n in speech_names for m in ev.get(n, ())}
owners = {m: (k.split(":")[0], k) for m, k in media_owners(game, sc, want).items()} # m -> (lang, key)
for m in want - set(owners):
lang = banks.media_language(sc, m) if m in sc.media else ""
if lang:
owners[m] = (lang, "%s:%08X" % (banks.ARCHIVE, m))
def subtitle_links(sc, lines, speech_names):
"""{speech event name: {language: sound keys}}: event -> actions -> containers -> Sound objects (the same
in every language) -> that language's recording of each line (a WAD line or a localized in-memory one).
An event that plays several lines (two versions) links its subtitle to all of them: 'k1;k2', in order."""
ev = banks.event_objects(sc)
out = {}
for n in speech_names:
keys = {}
for m in ev.get(n, ()):
if m in owners:
lang, key = owners[m]
keys.setdefault(lang, key)
keys = collections.defaultdict(list)
for o in ev.get(n, ()):
for L, k in lines.obj_keys.get(o, {}).items():
if k not in keys[L]:
keys[L].append(k)
if keys:
out[n] = keys
out[n] = {L: ";".join(ks) for L, ks in keys.items()}
return out
def scan_text(game, progress=None, characters=None):
"""One pass over every package: (bank sounds, bank locations, Scan, {lang: [SubLine]}, [SubLoc], issues,
{WAD sound key: game line name})."""
Lines)."""
col = subtitles.Collector()
snd, locs, sc = scan_banks(game, progress, characters, visit=col)
line_names = speech_line_names(game, sc)
names = {n for n in sc.names if n.lower().startswith("speech_")} | set(line_names.values())
names |= {s.name for s in snd if s.name.lower().startswith("speech_")}
sc = banks.scan(game.cooked, progress, visit=col)
names = {n for n in sc.names if n.lower().startswith("speech_")}
names |= {n for n, _ in banks.media_names(sc).values() if n.lower().startswith("speech_")}
col.resolve(names)
links = subtitle_links(game, sc, list(col.dialogues))
# the subtitle objects name their speech events too: an event ID is the FNV-1 hash of its name, so these
# names find events no package name table lists (e.g. Vicki Vale's radio reports)
sc.names.update(col.dialogues)
snd, locs, lines = derive_banks(game, sc, characters)
links = subtitle_links(sc, lines, list(col.dialogues))
texts, sublocs = subtitles.rows(col, links)
return snd, locs, sc, texts, sublocs, col.issues, line_names
return snd, locs, sc, texts, sublocs, col.issues, lines
def load_subtitles(game, lang, progress=None):
@@ -396,9 +410,13 @@ def load_banks(game, progress=None):
return read_csv(csv_path), banks.read_locations(loc_path)
cache = os.path.join(user_dir("cache"), "scan_%s_%s_v%d" % (banks.ARCHIVE, fp, SCAN_VERSION))
if not (os.path.exists(cache + ".csv") and os.path.exists(cache + ".locations.csv.gz")):
sounds, locs, _ = scan_banks(game, progress)
sounds, locs, _, lines = scan_banks(game, progress)
for s in sounds:
s.sha1 = "" # unknown build: no trusted original checksums
# the speech WADs of this build with the lines found now (load_archive reads these caches)
for a in game.languages:
snd = apply_lines(scan_archive(a, game.wad(a), progress=progress), lines, lines.names)
write_csv(_wad_cache(game, a), snd)
banks.write_locations(cache + ".locations.csv.gz", locs)
write_csv(cache + ".csv", sounds)
return read_csv(cache + ".csv"), banks.read_locations(cache + ".locations.csv.gz")
@@ -421,6 +439,10 @@ def is_shipped(game, archive):
return shipped.get("fingerprint") == wad_fingerprint(game.wad(archive))
def _wad_cache(game, archive):
return os.path.join(user_dir("cache"), "scan_%s_%s_v%d.csv" % (archive, wad_fingerprint(game.wad(archive)), SCAN_VERSION))
def load_archive(game, archive, progress=None):
"""[Sound] for one archive of a game (shipped CSV, user-cache scan, or fresh scan)."""
path = game.wad(archive)
@@ -429,7 +451,7 @@ def load_archive(game, archive, progress=None):
csv_path = os.path.join(catalog_dir(), "%s.csv" % archive)
if shipped.get("fingerprint") == fp and os.path.exists(csv_path):
return read_csv(csv_path)
cache = os.path.join(user_dir("cache"), "scan_%s_%s_v%d.csv" % (archive, fp, SCAN_VERSION))
cache = _wad_cache(game, archive)
if os.path.exists(cache):
return read_csv(cache)
sounds = scan_archive(archive, path, progress=progress)
@@ -463,10 +485,16 @@ class Catalog:
self.heads.setdefault(l.sound, []).append(l)
# localized in-memory speech: only the languages loaded here (all of them if none is)
langs = [a for a in self.archives if a in game.languages] or None
bank_sounds_ = [s for s in bank_sounds_ if not s.language or not langs or s.language in langs]
bank_sounds_ = [s for s in bank_sounds_ if not langs or any(s.in_language(L) for L in langs)]
for a in self.archives:
self.sounds += bank_sounds_ if a == banks.ARCHIVE else load_archive(game, a, progress)
self.by_key = {s.key: s for s in self.sounds}
# copies: one recording stored in two places of a WAD (same line, same archive); written together
same = collections.defaultdict(list)
for s in self.sounds:
if s.line and s.archive != banks.ARCHIVE:
same[(s.archive, s.line)].append(s)
self.copies = {s.key: [c for c in ss if c is not s] for ss in same.values() if len(ss) > 1 for s in ss}
self._by_name = {}
for s in self.sounds:
for n in s.names():