gora/UnheardCity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Game line names across languages; RSubtitle; editor in three columns

Browse Source
This commit is contained in:
Cristian Gora
2026-10-10 02:14:06 +02:00
parent f29af40b37
commit ea3e44000a
40 changed files with 54018 additions and 53559 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
ackit/catalog.py
View File

@@ -84,16 +84,29 @@ def hash_names():
return _names
def load_characters(path=None):
"""{sound name (lower): 'Speaker' or 'A; B'} from data/catalog/characters.csv."""
def load_characters(path=None, lang=None):
"""{sound name (lower): 'Speaker' or 'A; B'} from data/catalog/characters.csv: the rows valid in every
language (keyed by the game's line names) plus the rows of `lang` (lines without a line name, labelled
by their position in that language's WAD)."""
path = path or os.path.join(catalog_dir(), "characters.csv")
out = {}
if os.path.exists(path):
for row in csv.DictReader(open(path, encoding="utf-8")):
out[row["name"].lower()] = row["character"]
if not row.get("language") or row["language"] == lang:
out[row["name"].lower()] = row["character"]
return out
def character_of(s, characters):
"""Verified speaker of a sound from a load_characters() map: by its names, and for the k-th take of a
line shown as NAME#k (an event playing several WEMs of one language) by the line name NAME."""
for n in s.names() + (s.name.split("#")[0],):
c = characters.get(n.lower())
if c:
return c
return ""
def read_csv(path):
out = []
for r in csv.DictReader(open(path, encoding="utf-8", newline="")):
@@ -160,7 +173,7 @@ def _walk_entry(f, offset, size):
def scan_archive(archive, wad_path, characters=None, progress=None):
"""Header scan of one WAD -> [Sound]."""
names = hash_names()
characters = characters if characters is not None else load_characters()
characters = characters if characters is not None else load_characters(lang=archive)
wad = Wad(wad_path)
out = []
with open(wad_path, "rb") as f:
@@ -197,7 +210,7 @@ def scan_archive(archive, wad_path, characters=None, progress=None):
s.name, s.aliases = s.aliases[0], (s.name,)
for s in out:
s.characters_inferred = "; ".join(classify.infer_characters(s.name))
s.character = next((characters[a.lower()] for a in s.names() if a.lower() in characters), "")
s.character = character_of(s, characters)
return out
@@ -223,7 +236,7 @@ def bank_sounds(sc, characters=None):
sha1=next(iter(e["sha1"])) if len(e["sha1"]) == 1 else "")
s.group, s.category = classify.classify(name, bank=bank)
s.characters_inferred = "; ".join(classify.infer_characters(_CAMEL.sub("_", name + "_" + bank)))
s.character = next((characters[a.lower()] for a in s.names() if a.lower() in characters), "")
s.character = character_of(s, characters)
out.append(s)
out.sort(key=lambda s: (s.bank.lower(), s.name.lower()))
return out
@@ -264,6 +277,57 @@ def scan_banks(game, progress=None, characters=None, visit=None):
return bank_sounds(sc, characters), locs, sc
def speech_line_names(game, sc):
"""{WAD sound key: the game's name of that line} for every speech sound some event plays. The name is the
speech event (e.g. Speech_CH_ZERO_CH0_fight_intro_01): the same line has the same name in every language,
while the position of a variation inside a WAD entry differs between languages (WEMs are stored in
media-ID order and every language has its own media IDs). If several events reach a sound, the most
specific one (fewest sounds in that language) names it."""
ev = {n: ms for n, ms in banks.event_media(sc).items() if n.lower().startswith("speech_")}
owners = media_owners(game, sc, {m for ms in ev.values() for m in ms})
best = {}
for n, ms in ev.items():
keys = collections.defaultdict(set)
for m in ms:
if m in owners:
k = owners[m]
keys[k.split(":")[0]].add(k)
for lang, ks in keys.items():
for k in ks:
rank = (len(ks), n.lower())
if k not in best or rank < best[k][0]:
best[k] = (rank, n)
return {k: n for k, (_, n) in best.items()}
def apply_line_names(sounds, line_names):
"""Rename WAD speech sounds after the game's line names (see speech_line_names). The position-based name
stays available as alias NAME#i (single-WEM entries keep their entry name as alias)."""
used = collections.Counter()
for s in sounds:
ev = line_names.get(s.key)
if not ev:
continue
used[ev.lower()] += 1
name = ev if used[ev.lower()] == 1 else "%s#%d" % (ev, used[ev.lower()])
if ":" in s.target: # variation: keep NAME#i (position in this language's WAD)
pos = [a for a in s.names() if "#" in a]
s.aliases = tuple(pos)
else: # single WEM: the entry name stays an alias
s.aliases = (s.name,) + tuple(a for a in s.aliases if a != s.name)
s.name = name
taken = collections.Counter(s.name.lower() for s in sounds)
for s in sounds: # a positional name equal to another sound's line name -> NAME#i
if taken[s.name.lower()] > 1 and s.key not in line_names and ":" in s.target:
pos = [a for a in s.aliases if "#" in a]
if pos:
s.aliases, s.name = (s.name,), pos[0]
for s in sounds:
s.group, s.category = classify.classify(s.name)
s.characters_inferred = "; ".join(classify.infer_characters(s.name))
return sounds
def subtitle_links(game, sc, speech_names):
"""{speech event name: {language: sound key}}: event -> actions -> containers -> sounds -> the
per-language media of the line (exact, no name matching). The media is a streamed WAD sound, or an
@@ -288,12 +352,17 @@ def subtitle_links(game, sc, speech_names):
def scan_text(game, progress=None, characters=None):
"""One pass over every package: (bank sounds, bank locations, Scan, {lang: [SubLine]}, [SubLoc], issues)."""
"""One pass over every package: (bank sounds, bank locations, Scan, {lang: [SubLine]}, [SubLoc], issues,
{WAD sound key: game line name})."""
col = subtitles.Collector()
snd, locs, sc = scan_banks(game, progress, characters, visit=col)
line_names = speech_line_names(game, sc)
names = {n for n in sc.names if n.lower().startswith("speech_")} | set(line_names.values())
names |= {s.name for s in snd if s.name.lower().startswith("speech_")}
col.resolve(names)
links = subtitle_links(game, sc, list(col.dialogues))
texts, sublocs = subtitles.rows(col, links)
return snd, locs, sc, texts, sublocs, col.issues
return snd, locs, sc, texts, sublocs, col.issues, line_names
def load_subtitles(game, lang, progress=None):
@@ -307,7 +376,7 @@ def load_subtitles(game, lang, progress=None):
return subtitles.read_texts(t, lang), subtitles.read_locations(loc, lang)
cache = os.path.join(user_dir("cache"), "scan_SUB_%s_v%d" % (fp, SCAN_VERSION))
if not os.path.exists(cache + ".locations.csv.gz"):
_, _, _, texts, sublocs, _ = scan_text(game, progress)
_, _, _, texts, sublocs, _, _ = scan_text(game, progress)
for lg, rows in texts.items():
subtitles.write_texts(cache + "_%s.csv" % lg, rows)
subtitles.write_locations(cache + ".locations.csv.gz", sublocs)

81
ackit/subtitles.py
View File

@@ -22,6 +22,8 @@ from collections import defaultdict, namedtuple
from .upk import Package
CLASS = "RDialogueEvent"
SUBTITLE_CLASS, LOOKUP_CLASS = "RSubtitle", "RSubtitleLookup" # scripted radio/encryption subtitles
UNRESOLVED = "#lookup:%08X" # speech of an RSubtitle until its lookup key is matched to a line name
T_STRUCT, T_STRING, T_OBJECT = 0x09, 0x0D, 0x10
# one subtitle string inside a package: pos = absolute uncompressed offset of the FString length field,
@@ -109,9 +111,14 @@ def _parse_lines(b, o, end, base, lang):
def parse_dialogue(b, base, names_idx):
"""Fields of one RDialogueEvent export (bytes `b` starting at uncompressed offset `base`)."""
ls, sub = names_idx
i = b.find(struct.pack("<II", ls, 0))
"""Fields of one dialogue/subtitle export (bytes `b` starting at uncompressed offset `base`).
names_idx: (name indices of the per-language array: LocalizedSubtitles or ArraySubtitles, of Subtitles)."""
arrays, sub = names_idx
i = -1
for ls in arrays:
i = b.find(struct.pack("<II", ls, 0))
if i >= 4 and struct.unpack_from("<H", b, i - 4)[0] == T_STRUCT:
break
if i < 4 or struct.unpack_from("<H", b, i - 4)[0] != T_STRUCT:
return []
size = struct.unpack_from("<i", b, i + 8)[0]
@@ -168,6 +175,24 @@ class Collector:
except (SubtitleError, struct.error, ValueError, IndexError) as e:
self.issues.append(("subtitles", "%s: %s" % (rel, e)))
PREFIXES = ("", "RADIO_SCAN_", "RADIO_", "TV_", "CH_ZERO_", "GCPD_", "CHAR_", "SEQUENCE_")
def resolve(self, line_names):
"""Give RSubtitle objects their speech line name: the name whose lookup_key equals the object's key,
from `line_names` or built from the object's own name (Speech_ + prefix + name; exact hash match)."""
by_key = {lookup_key(n): n for n in line_names}
for sp in [s for s in self.dialogues if s.startswith("#lookup:")]:
key = int(sp.split(":")[1], 16)
name = by_key.get(key)
if not name:
exp = self.dialogues[sp][0][1].export
name = next(("Speech_" + p + exp for p in self.PREFIXES if lookup_key("Speech_" + p + exp) == key), None)
copies = self.dialogues.pop(sp)
if not name:
self.issues.append(("subtitles", "no speech line for subtitle %s" % copies[0][1].export))
name = copies[0][1].export
self.dialogues[name].extend((rel, d._replace(speech=name)) for rel, d in copies)
# --- catalog files: SUB_<LANG>.csv (texts) + SUB_locations.csv.gz (every copy) ------------------
LOCATIONS = "SUB_locations.csv.gz"
@@ -236,20 +261,56 @@ def read_locations(path, lang=None):
return dict(out)
def lookup_key(line_name):
"""Key of a speech line in the RSubtitleLookup table: FNV-1 of its name without 'Speech_'."""
from .bnk import fnv1
return fnv1(line_name[7:] if line_name.lower().startswith("speech_") else line_name)
def _lookup_table(b):
"""RSubtitleLookup export data -> {export index: key}: array of {int key @0, object @4}."""
count = struct.unpack_from("<I", b, 24)[0]
out, o = {}, 28
for _ in range(count):
if b[o:o + 4] != b"\x02\x00\x00\x00" or b[o + 8:o + 12] != b"\x10\x00\x04\x00":
raise SubtitleError("unexpected subtitle lookup layout")
key, ref = struct.unpack_from("<Ii", b, o + 4)[0], struct.unpack_from("<i", b, o + 12)[0]
out[ref] = key
o += 18
return out
def scan_package(pkg, data=None, start=None, tables=None):
"""[Dialogue] of one package (empty if it has none)."""
"""[Dialogue] of one package (empty if it has none): RDialogueEvent objects (speech = the referenced
speech event) and RSubtitle objects (speech = UNRESOLVED % lookup key, see Collector.resolve)."""
if data is None:
start, data = pkg.read_all()
names, imports, exports = tables or pkg.tables(data, start)
cls = {-(i + 1) for i, (c, o, _) in enumerate(imports) if c == "Class" and o == CLASS}
if not cls or "LocalizedSubtitles" not in names or "Subtitles" not in names:
by_class = {}
for i, (c, o, _) in enumerate(imports):
if c == "Class" and o in (CLASS, SUBTITLE_CLASS, LOOKUP_CLASS):
by_class[-(i + 1)] = o
arrays = [names.index(a) for a in ("LocalizedSubtitles", "ArraySubtitles") if a in names]
if not by_class or not arrays or "Subtitles" not in names:
return []
idx = (names.index("LocalizedSubtitles"), names.index("Subtitles"))
idx = (arrays, names.index("Subtitles"))
keys = {}
for e in exports:
if by_class.get(e.cls) == LOOKUP_CLASS and e.size > 0:
keys.update(_lookup_table(data[e.offset - start:e.offset - start + e.size]))
out = []
for e in exports:
if e.cls in cls and e.size > 0:
kind = by_class.get(e.cls)
if kind in (CLASS, SUBTITLE_CLASS) and e.size > 0:
b = data[e.offset - start:e.offset - start + e.size]
fields = parse_dialogue(b, e.offset, idx)
if fields:
out.append(Dialogue(_speech_ref(b, exports, imports), e.name, fields))
if not fields:
continue
if kind == CLASS:
speech = _speech_ref(b, exports, imports)
elif e.index in keys:
speech = UNRESOLVED % keys[e.index]
else:
speech = e.name # not in a lookup table: keep the object's name
out.append(Dialogue(speech, e.name, fields))
return out

19
ackit/verify/catalogcheck.py
View File

@@ -17,9 +17,10 @@ import os
import random
from .. import banks, classify, subtitles
from ..catalog import (Catalog, available_archives, bank_sounds, hash_names, head_owners, is_shipped,
from ..catalog import (Catalog, available_archives, bank_sounds, character_of, hash_names, head_owners, is_shipped,
load_banks, load_characters, read_csv, scan_archive)
from ..inject import read_target
from ..paths import catalog_dir
from ..wad import Wad
FOLDER_LANG = {"ENGLISH(US)": "INT", "ITALIAN": "ITA", "FRENCH(FRANCE)": "FRA", "GERMAN": "DEU",
@@ -67,7 +68,7 @@ def _check_rows(rep, archive, rows, characters):
g, c = _expected_category(s)
if (s.group, s.category) != (g, c):
rep.add(archive, "category", s.key, "%s/%s, rules give %s/%s" % (s.group, s.category, g, c))
want = next((characters[a.lower()] for a in s.names() if a.lower() in characters), "")
want = character_of(s, characters)
if s.character != want:
rep.add(archive, "character", s.key, "%r, curated list says %r" % (s.character, want))
@@ -80,6 +81,11 @@ def check_wad(rep, game, archive, rows, characters):
rep.add(archive, "language", archive, "WAD in folder %s" % folder)
_check_rows(rep, archive, rows, characters)
fresh = {s.target: s for s in scan_archive(archive, path, characters=characters)}
if is_shipped(game, archive): # shipped names are the game's line names (built from the banks)
byk = {s.target: s for s in rows}
for t, s in fresh.items():
if t in byk:
s.name, s.aliases = byk[t].name, byk[t].aliases
cat = {s.target: s for s in rows}
for t in sorted(set(fresh) - set(cat)):
rep.add(archive, "missing", "%s:%s" % (archive, t), fresh[t].name)
@@ -105,6 +111,12 @@ def check_banks(rep, game, rows, locations, characters, quick=False, sample=40,
files = [f for f in files if f.startswith("SFX/")] + sorted(rnd.sample(pk, min(sample, len(pk))))
col = subtitles.Collector()
sc = banks.scan(game.cooked, progress, files=files, visit=col)
known = {n.split("#")[0] for s in rows for n in s.names()} # BANK names
for a in game.languages: # + speech line names of the catalogs
p = os.path.join(catalog_dir(), "%s.csv" % a)
if os.path.exists(p):
known.update(n.split("#")[0] for s in read_csv(p) for n in s.names())
col.resolve({n for n in sc.names | known if n.lower().startswith("speech_")})
check_subtitles(rep, game, col, set(files), quick)
owners = head_owners(game, sc)
index = collections.defaultdict(set)
@@ -184,7 +196,6 @@ def check_subtitles(rep, game, col, files, quick):
def run(game, archives=None, quick=False, log=print, progress=None):
"""Check the given archives (default: all). Returns a Report."""
rep = Report()
characters = load_characters()
archives = archives or available_archives(game)
cat = Catalog(game, archives, progress)
by_arch = collections.defaultdict(list)
@@ -192,11 +203,13 @@ def run(game, archives=None, quick=False, log=print, progress=None):
by_arch[s.archive].append(s)
for a in archives:
log("checking %s ..." % a)
characters = load_characters(lang=None if a == banks.ARCHIVE else a)
if a == banks.ARCHIVE:
rows = load_banks(game, progress)[0] # every language, not only the loaded one
check_banks(rep, game, rows, cat.locations, characters, quick=quick, progress=progress)
else:
check_wad(rep, game, a, by_arch[a], characters)
characters = load_characters(lang=game.language())
known = {n.lower() for s in cat.sounds for n in s.names()}
# speaker names are the same in every language; only meaningful for the build the lists were made for
if any(a in game.languages and is_shipped(game, a) for a in archives):