gora/UnheardCity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version 3.1.0: voice lines linked across all languages from the game data

Browse Source
This commit is contained in:
Cristian Gora
2026-10-10 14:10:03 +02:00
parent ea3e44000a
commit fbde809370
56 changed files with 74728 additions and 73360 deletions
Download Patch File Download Diff File Expand all files Collapse all files

271
ackit/verify/catalogcheck.py
View File

@@ -17,8 +17,8 @@ import os
import random
from .. import banks, classify, subtitles
from ..catalog import (Catalog, available_archives, bank_sounds, character_of, hash_names, head_owners, is_shipped,
load_banks, load_characters, read_csv, scan_archive)
from ..catalog import (Catalog, apply_lines, available_archives, bank_sounds, build_lines, character_of, hash_names,
is_shipped, line_names, load_archive, load_banks, load_characters, read_csv, scan_archive)
from ..inject import read_target
from ..paths import catalog_dir
from ..wad import Wad
@@ -31,6 +31,7 @@ class Report:
def __init__(self):
self.checked = collections.Counter()
self.issues = [] # (archive, kind, key, detail)
self.facts = [] # what was established about the game's relations (full check)
def add(self, archive, kind, key, detail=""):
self.issues.append((archive, kind, key, detail))
@@ -73,19 +74,30 @@ def _check_rows(rep, archive, rows, characters):
rep.add(archive, "character", s.key, "%r, curated list says %r" % (s.character, want))
def check_wad(rep, game, archive, rows, characters):
def check_wad(rep, game, archive, rows, characters, lines=None):
"""Rows vs a fresh scan of the WAD. With `lines` (full check) the fresh sounds get the game's line
identities and names, so names, aliases and lines are compared too."""
path = game.wad(archive)
if archive != "SFX":
folder = os.path.basename(os.path.dirname(path)).upper()
if FOLDER_LANG.get(folder, archive) != archive:
rep.add(archive, "language", archive, "WAD in folder %s" % folder)
_check_rows(rep, archive, rows, characters)
fresh = {s.target: s for s in scan_archive(archive, path, characters=characters)}
if is_shipped(game, archive): # shipped names are the game's line names (built from the banks)
fresh_list = scan_archive(archive, path, characters=characters)
if lines is not None and archive in lines.langs:
apply_lines(fresh_list, lines, lines.names)
fresh = {s.target: s for s in fresh_list}
if lines is None and archive in game.languages: # quick check: line names need the banks, not rescanned
byk = {s.target: s for s in rows}
for t, s in fresh.items():
if t in byk:
s.name, s.aliases = byk[t].name, byk[t].aliases
s.name, s.aliases, s.line = byk[t].name, byk[t].aliases, byk[t].line
byk = {s.target: s for s in rows}
for t in sorted(set(fresh) & set(byk)):
f, s = fresh[t], byk[t]
if (f.name, tuple(f.aliases), f.line) != (s.name, tuple(s.aliases), s.line):
rep.add(archive, "name", s.key, "catalog %s %s line %s, game %s %s line %s"
% (s.name, list(s.aliases), s.line or "-", f.name, list(f.aliases), f.line or "-"))
cat = {s.target: s for s in rows}
for t in sorted(set(fresh) - set(cat)):
rep.add(archive, "missing", "%s:%s" % (archive, t), fresh[t].name)
@@ -103,6 +115,8 @@ def check_wad(rep, game, archive, rows, characters):
def check_banks(rep, game, rows, locations, characters, quick=False, sample=40, progress=None):
"""BANK rows, copies, prefetch heads and subtitles vs a scan of the packages. The full check (not quick)
also derives the game's lines from that scan and returns them (else None)."""
_check_rows(rep, banks.ARCHIVE, rows, characters)
files = banks.bank_files(game.cooked)
if quick:
@@ -118,7 +132,10 @@ def check_banks(rep, game, rows, locations, characters, quick=False, sample=40,
known.update(n.split("#")[0] for s in read_csv(p) for n in s.names())
col.resolve({n for n in sc.names | known if n.lower().startswith("speech_")})
check_subtitles(rep, game, col, set(files), quick)
owners = head_owners(game, sc)
lines = None
if not quick:
sc.names.update(col.dialogues) # as the build does: subtitle objects name speech events too
lines = build_lines(game, sc)
index = collections.defaultdict(set)
for m, ls in locations.items():
for l in ls:
@@ -126,13 +143,22 @@ def check_banks(rep, game, rows, locations, characters, quick=False, sample=40,
index[(l.media, l.kind)].add((l.file, l.offset, l.size, l.sound if l.kind == banks.PREFETCH_HEAD else ""))
found = collections.defaultdict(set)
for l in sc.locations:
found[(l.media, l.kind)].add((l.file, l.offset, l.size, owners.get(l.media, "") if l.kind == banks.PREFETCH_HEAD else ""))
head = l.kind == banks.PREFETCH_HEAD
found[(l.media, l.kind)].add((l.file, l.offset, l.size, lines.head_key(l) if head and lines else ""))
if lines is None: # quick: a subset of the packages cannot resolve the heads' lines
index = {k: {(f, o, s, "") for f, o, s, _ in v} for k, v in index.items()}
for k in sorted(set(found) | set(index)):
if found.get(k, set()) != index.get(k, set()):
rep.add(banks.ARCHIVE, "copies", "%08X" % k[0], "%s: index %d, game %d"
% ("head" if k[1] == banks.PREFETCH_HEAD else "copies", len(index.get(k, ())), len(found.get(k, ()))))
cat = {s.target: s for s in rows}
fresh = {s.target: s for s in bank_sounds(sc, characters)}
fresh_list = bank_sounds(sc, characters)
if lines is not None:
lines.names = line_names(game, sc, lines, {s.key: s.name for s in fresh_list})
apply_lines(fresh_list, lines, lines.names)
for s in fresh_list:
s.character = character_of(s, characters)
fresh = {s.target: s for s in fresh_list}
for t in sorted(set(fresh) - set(cat)):
rep.add(banks.ARCHIVE, "missing", "BANK:" + t, fresh[t].bank)
if not quick:
@@ -144,10 +170,14 @@ def check_banks(rep, game, rows, locations, characters, quick=False, sample=40,
rep.add(banks.ARCHIVE, "sha1", s.key, "game file differs from the catalog's original")
if f.size < s.size:
rep.add(banks.ARCHIVE, "size", s.key, "catalog %d, game %d" % (s.size, f.size))
if lines is not None and (f.name, f.language, f.line) != (s.name, s.language, s.line):
rep.add(banks.ARCHIVE, "name", s.key, "catalog %s [%s] line %s, game %s [%s] line %s"
% (s.name, s.language, s.line or "-", f.name, f.language, f.line or "-"))
for m, ls in locations.items():
for l in ls:
if l.kind == banks.PREFETCH_HEAD and not l.sound:
rep.add(banks.ARCHIVE, "copies", "%08X" % m, "prefetch head without a speech line")
return sc, lines, col
def check_subtitles(rep, game, col, files, quick):
@@ -174,6 +204,7 @@ def check_subtitles(rep, game, col, files, quick):
langs = sorted(_shipped_index().get("SUB", {}).get("lines", {}))
bank_lang = {s.key: s.language for s in read_csv(os.path.join(catalog_dir(), "BANK.csv"))}
for lang in langs:
voice = lang if lang in game.languages else "INT" # languages without own voices: INT voices
p = os.path.join(catalog_dir(), "SUB_%s.csv" % lang)
rows = subtitles.read_texts(p, lang)
rep.checked["SUB_" + lang] += len(rows)
@@ -185,12 +216,200 @@ def check_subtitles(rep, game, col, files, quick):
seen.add(k)
if k in texts and texts[k] != s.text:
rep.add("SUB", "text", k, "catalog %r, game %r" % (s.text[:40], texts[k][:40]))
if s.sound.startswith(banks.ARCHIVE + ":"):
b = bank_lang.get(s.sound)
if b is not None and b not in (lang, "INT"): # INT: languages without own voices
rep.add("SUB", "language", k, "linked to %s (%s)" % (s.sound, b or "language-neutral"))
elif s.sound and not s.sound.startswith(lang + ":") and not s.sound.startswith("INT:"):
rep.add("SUB", "language", k, "linked to %s" % s.sound)
for snd in s.sounds:
if snd.startswith(banks.ARCHIVE + ":"):
b = bank_lang.get(snd)
if b and voice not in b.split(";"): # '' = stored for every language
rep.add("SUB", "language", k, "linked to %s (%s)" % (snd, b))
elif not snd.startswith(voice + ":"):
rep.add("SUB", "language", k, "linked to %s, not a %s voice line" % (snd, voice))
def check_lines(rep, game, sc, lines, locations):
"""The game's relations, re-derived from the game files (ackit/lines.py) and compared with the catalogs:
links a broken one-to-one relation in the game data (a WEM without its table row, a Sound object with
two recordings in one language, a line without a recording in some language, ...)
line a catalog row whose line identity / name differs from the game's
subtitle a subtitle linked to a different line in one language than in INT
Facts (rep.facts) give the counts, prefetch heads compared byte by byte with the start of their WAD
recording, and the length agreement of a line's recordings across languages (independent evidence)."""
for kind, detail in lines.issues:
rep.add("LINES", "links", kind, detail)
st = lines.stats
for lid, miss in sorted(st["lines_missing_a_language"].items()):
rep.facts.append("line %s (%s) has no recording in %s" % (lid, lines.names.get(lid, "?"), ", ".join(miss)))
for k in st["without_line"]:
rep.add("LINES", "links", k, "a recording of no line")
# independent of the BKPK tables: each WAD recording is the source of a Sound object in a bank of the
# language the tables give it (localized banks: one per declared language)
named = total = 0
for L in lines.langs:
for m, k in lines.owners.get(L, {}).items():
if not k.startswith(banks.ARCHIVE + ":"):
total += 1
if m in sc.lang_media.get(L, ()):
named += 1
else:
rep.add("LINES", "links", k, "media %08X is not a source in any %s bank" % (m, L))
rep.facts.append("WAD recordings named as a source by a Sound object of their own language's bank: %d of %d"
% (named, total))
rep.facts.append("voice languages %s: %d recordings (WAD %s + localized in-memory), %d lines, %d of them "
"with exactly one recording in every language"
% ("/".join(lines.langs), st["recordings"],
"/".join(str(st["wad_wems"].get(L, 0)) for L in lines.langs), st["lines"],
st["lines_in_every_language"]))
# catalog rows vs the game's lines
rows = {}
for L in lines.langs + [banks.ARCHIVE]:
p = os.path.join(catalog_dir(), "%s.csv" % L)
if os.path.exists(p):
for s in read_csv(p):
rows[s.key] = s
for lid, per in lines.members.items():
for L, k in per.items():
s = rows.get(k)
if s is None:
rep.add("LINES", "line", k, "recording of line %s is not in the catalog" % lid)
elif s.line != lid or s.name != lines.names[lid]:
rep.add("LINES", "line", k, "catalog %s/%s, game %s/%s" % (s.line, s.name, lid, lines.names[lid]))
for k, s in rows.items():
if s.line and lines.line_of.get(k) != s.line:
rep.add("LINES", "line", k, "catalog says line %s, the game data %s" % (s.line, lines.line_of.get(k)))
# prefetch heads: every stored head is the start of the WAD recording it is mapped to
wads = {L: (game.wad(L), Wad(game.wad(L))) for L in lines.langs}
heads = [l for ls in locations.values() for l in ls if l.kind == banks.PREFETCH_HEAD]
by_file = collections.defaultdict(list)
for l in heads:
by_file[l.file].append(l)
same = 0
for rel, ls in by_file.items():
r = banks.open_file(banks.file_path(game.cooked, rel))
for l in ls:
L = l.sound.split(":")[0]
if L not in wads:
rep.add("LINES", "head", "%08X" % l.media, "head in %s maps to %r" % (rel, l.sound))
continue
w = read_target(wads[L][0], l.sound.split(":", 1)[1], wads[L][1])
if r.read(l.offset, l.size) == (w + bytes(l.size))[:l.size]:
same += 1
else:
rep.add("LINES", "head", "%08X" % l.media, "head in %s is not the start of %s" % (rel, l.sound))
same_copies = 0
for k, primary in sorted(lines.copy_of.items()): # one recording stored in two WAD entries
L = k.split(":")[0]
a = read_target(wads[L][0], k.split(":", 1)[1], wads[L][1])
b = read_target(wads[L][0], primary.split(":", 1)[1], wads[L][1])
if a == b:
same_copies += 1
else:
rep.add("LINES", "copies", k, "differs from %s (same media ID)" % primary)
rep.facts.append("recordings stored twice in a WAD (same media ID, written together): %d, byte-identical: %d"
% (len(lines.copy_of), same_copies))
rep.facts.append("prefetch heads: %d of %d stored copies are byte-identical to the start of their WAD recording"
% (same, len(heads)))
# independent evidence: the recordings of a line have similar lengths in every language
ratios = []
for lid, per in lines.members.items():
secs = [rows[k].seconds for k in per.values() if k in rows and rows[k].seconds > 0]
if len(secs) > 1:
ratios.append((max(secs) / min(secs), lid))
if ratios:
ratios.sort()
within = sum(1 for r, _ in ratios if r <= 1.5)
rep.facts.append("lengths across languages: %d of %d lines within a factor 1.5, median %.2f, longest ratio "
"%.1f (%s)" % (within, len(ratios), ratios[len(ratios) // 2][0], ratios[-1][0],
lines.names.get(ratios[-1][1], "")))
return rows
def check_subtitle_lines(rep, rows):
"""A subtitle line (speech event + line index) must point to the same line in every language."""
from ..catalog import _shipped_index
langs = sorted(_shipped_index().get("SUB", {}).get("lines", {}))
ref = {}
p = os.path.join(catalog_dir(), "SUB_INT.csv")
if os.path.exists(p):
for s in subtitles.read_texts(p, "INT"):
if s.sounds and all(k in rows for k in s.sounds):
ref[(s.speech, s.line)] = [rows[k].line for k in s.sounds]
linked = total = 0
for lang in langs:
for s in subtitles.read_texts(os.path.join(catalog_dir(), "SUB_%s.csv" % lang), lang):
total += 1
if not s.sound:
continue
linked += 1
unknown = [k for k in s.sounds if k not in rows]
want = ref.get((s.speech, s.line))
if unknown:
rep.add("SUB", "link", subtitles.key(lang, s.speech, s.line), "linked to unknown %s" % unknown)
elif want and [rows[k].line for k in s.sounds] != want:
rep.add("SUB", "link", subtitles.key(lang, s.speech, s.line),
"lines %s, INT subtitle is on lines %s" % ([rows[k].line for k in s.sounds], want))
rep.facts.append("subtitles: %d strings in %d languages, %d linked to a voice line, each to the same line as in INT"
% (total, len(langs), linked))
SAME_PERSON = {("mr freeze", "mister freeze"), ("batman", "bruce wayne"), ("two face", "twoface"),
("azrael", "mysterious stranger")}
def _person(s):
return " ".join(s.lower().replace("'", "").replace(".", "").replace("_", " ").replace("-", " ").split())
def check_speakers(rep, rows, col):
"""Independent evidence for the curated speaker labels: the dialogue object of a subtitled line names its
speaker. Facts only (the lists are curated by listening; differences are for a person to judge)."""
speaker = {}
for speech, copies in col.dialogues.items():
for _, d in copies:
if d.speaker:
speaker.setdefault(speech, d.speaker)
p = os.path.join(catalog_dir(), "SUB_INT.csv")
if not os.path.exists(p):
return
agree, unknown, differ = 0, 0, []
for s in subtitles.read_texts(p, "INT"):
spk = speaker.get(s.speech, "")
if s.line != 0 or not spk:
continue
for k in s.sounds:
r = rows.get(k)
if not r or not r.character:
continue
who = [_person(c) for c in r.character.split("; ")]
g = _person(spk)
if any(g in w or w in g or (w, g) in SAME_PERSON for w in who):
agree += 1
elif who == ["unknown"]:
unknown += 1
else:
differ.append("%s (list: %s, game: %s)" % (r.name, r.character, spk))
rep.facts.append("speakers: %d labelled voice lines agree with the speaker field of their dialogue object, "
"%d listed as Unknown are named there, %d differ%s"
% (agree, unknown, len(differ), (": " + "; ".join(differ)) if differ else ""))
def check_menu(rep, game):
"""Menu texts: the same keys in every language (facts; differences are the game's own)."""
from .. import loctext
root = os.path.join(game.root, "BmGame", "Localization")
if not os.path.isdir(root):
return
keys = {}
for lang in loctext.languages(root):
try:
keys[lang] = {(os.path.splitext(e.file)[0].upper(), e.section, e.name, e.occurrence)
for e in loctext.load(root, lang)}
except Exception as e: # unreadable folder: report, do not stop
rep.add("MENU", "read", lang, str(e))
if "INT" not in keys:
return
for lang, ks in sorted(keys.items()):
extra, miss = ks - keys["INT"], keys["INT"] - ks
rep.facts.append("menu text %s: %d keys%s" % (lang, len(ks), "" if not (extra or miss) else
" (%d not in INT, %d of INT missing)" % (len(extra), len(miss))))
def run(game, archives=None, quick=False, log=print, progress=None):
@@ -201,14 +420,24 @@ def run(game, archives=None, quick=False, log=print, progress=None):
by_arch = collections.defaultdict(list)
for s in cat.sounds:
by_arch[s.archive].append(s)
lines = None
if banks.ARCHIVE in archives: # first: the full check derives the game's lines from this scan
log("checking %s ..." % banks.ARCHIVE)
rows = load_banks(game, progress)[0] # every language, not only the loaded one
sc, lines, col = check_banks(rep, game, rows, cat.locations, load_characters(lang=None), quick=quick,
progress=progress)
for a in archives:
log("checking %s ..." % a)
characters = load_characters(lang=None if a == banks.ARCHIVE else a)
if a == banks.ARCHIVE:
rows = load_banks(game, progress)[0] # every language, not only the loaded one
check_banks(rep, game, rows, cat.locations, characters, quick=quick, progress=progress)
else:
check_wad(rep, game, a, by_arch[a], characters)
continue
log("checking %s ..." % a)
rows_a = by_arch[a] if a in cat.archives else load_archive(game, a, progress)
check_wad(rep, game, a, rows_a, load_characters(lang=a), lines)
if lines is not None and is_shipped(game, banks.ARCHIVE):
log("checking the relations ...")
rows_all = check_lines(rep, game, sc, lines, cat.locations)
check_subtitle_lines(rep, rows_all)
check_speakers(rep, rows_all, col)
check_menu(rep, game)
characters = load_characters(lang=game.language())
known = {n.lower() for s in cat.sounds for n in s.names()}
# speaker names are the same in every language; only meaningful for the build the lists were made for