gora/UnheardCity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version 3.1.0: voice lines linked across all languages from the game data

Browse Source
This commit is contained in:
Cristian Gora
2026-10-10 14:10:03 +02:00
parent ea3e44000a
commit fbde809370
56 changed files with 74728 additions and 73360 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
tools/build_catalog.py
View File

@@ -4,13 +4,15 @@
Outputs
data/catalog/characters.csv name -> verified speaker, from data/catalog/source/<lang>/*.txt
(speech sounds carry the game's line identity and one name in every language: ackit/lines.py)
data/catalog/<ARCHIVE>.csv every sound of one archive (schema: data/catalog/README.md)
data/catalog/BANK.csv in-memory sounds (soundbanks in SFX/*.bnk and the .upk packages)
data/catalog/BANK_locations.csv.gz every stored copy of them + the prefetch heads of speech lines
data/catalog/SUB_<LANG>.csv subtitles of the speech lines (one file per language)
data/catalog/SUB_locations.csv.gz every stored copy of each subtitle string
data/catalog/index.json fingerprints the CSVs were built from + counts
Run it only on clean game files (`ackit status` must say nothing is modified).
Run it only on clean game files (`ackit status` must say nothing is modified). It stops before writing
anything if a sound differs from the current catalog (a modified game); after a game update use --accept-changes.
"""
import argparse
import collections
@@ -23,7 +25,7 @@ import time
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from ackit import banks, settings, subtitles # noqa: E402
from ackit.catalog import (apply_line_names, character_of, hash_names, scan_archive, scan_text, # noqa: E402
from ackit.catalog import (apply_lines, character_of, hash_names, read_csv, scan_archive, scan_text, # noqa: E402
write_csv)
from ackit.classify import USER_LABELS # noqa: E402
from ackit.game import Game, default_game_dir, wad_fingerprint # noqa: E402
@@ -51,17 +53,28 @@ def read_user_lists(src_dir):
return out
def build_characters(sounds, src_root):
"""Map the user's wemscan-style names onto catalog names (name or alias)."""
by_alias = collections.defaultdict(list)
for s in sounds:
for a in s.names():
by_alias[a.lower()].append(s)
rows, unmatched, ambiguous = {}, [], []
for lang in sorted(os.listdir(src_root)):
d = os.path.join(src_root, lang)
def build_characters(game, src_root, final):
"""Map the curated lists onto the catalog. Each list folder (source/<lang>/) names sounds of THAT language's
WAD by position (wemscan names: NAME = single entry, NAME_i = i-th WEM of the entry). Positions differ
between languages, so a listed recording is found by its position in that language and the label goes to
its line (`final`: {key: Sound} with line identities and names), i.e. to the same recording in every
language; a recording without a line keeps the label in that language only.
-> ({(name, language or ''): set(characters)}, unmatched names, ambiguous names, skipped folders)"""
rows = collections.defaultdict(set)
unmatched, ambiguous, skipped = [], [], []
for folder in sorted(os.listdir(src_root)):
d = os.path.join(src_root, folder)
if not os.path.isdir(d):
continue
lang = folder.upper()
if lang not in game.archives:
skipped.append(folder) # no speech WAD of that language: positions cannot be resolved
continue
ref = scan_archive(lang, game.wad(lang), characters={}) # position names of this language
by_alias = collections.defaultdict(list)
for s in ref:
for a in s.names():
by_alias[a.lower()].append(s)
for uname, who in read_user_lists(d).items():
hits = list({id(s): s for s in by_alias.get(uname, [])}.values())
if not hits:
@@ -70,9 +83,27 @@ def build_characters(sounds, src_root):
if len(hits) > 1:
ambiguous.append(uname) # wemscan name collision: both candidates get the label
for s in hits:
cur = rows.setdefault(s.name, set())
cur |= who
return rows, unmatched, ambiguous
f = final[s.key]
rows[(f.name, "" if f.line else lang)] |= who
return rows, unmatched, ambiguous, skipped
def changed_since_catalog(game, out_dir, bank_snd):
"""Keys whose original checksum in the existing catalog differs from the game files now."""
out = []
for arch, path in game.archives.items():
p = os.path.join(out_dir, "%s.csv" % arch)
if not os.path.exists(p):
continue
wad = Wad(path)
for s in read_csv(p):
if s.sha1 and s.sha1 != file_sha1(path, s.target, wad):
out.append(s.key)
p = os.path.join(out_dir, "%s.csv" % banks.ARCHIVE)
if os.path.exists(p):
old = {s.key: s.sha1 for s in read_csv(p)}
out += [s.key for s in bank_snd if old.get(s.key) and s.sha1 and old[s.key] != s.sha1]
return out
def file_sha1(path, target, wad):
@@ -82,6 +113,8 @@ def file_sha1(path, target, wad):
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--game")
ap.add_argument("--accept-changes", action="store_true",
help="the game was updated: accept sounds whose bytes differ from the current catalog")
a = ap.parse_args()
game = Game(a.game or default_game_dir(settings.load()))
out_dir = catalog_dir()
@@ -90,33 +123,42 @@ def main():
# 1. every package once: in-memory sounds, prefetch heads, subtitles, events (a few minutes)
t0 = time.time()
bank_snd, locs, sc, texts, sublocs, sub_issues, line_names = scan_text(
bank_snd, locs, sc, texts, sublocs, sub_issues, lines = scan_text(
game, progress=lambda k, n: print(" banks + subtitles: %d/%d files" % (k, n), flush=True)
if k % 500 == 0 else None, characters={})
print("packages scanned (%.0fs); %d speech sounds named by their game line" % (time.time() - t0, len(line_names)))
st = lines.stats
print("packages scanned (%.0fs); %d recordings in %d lines, %d of them in every language (%s); "
"%d recordings without a line; %d relation issues"
% (time.time() - t0, st["in_a_line"], st["lines"], st["lines_in_every_language"], "/".join(lines.langs),
len(st["without_line"]), len(lines.issues)))
for k, d in lines.issues[:10]:
print(" issue:", k, d)
# 2. characters from the curated lists. They name ITA sounds by position (wemscan); the position of a
# variation differs between languages, so labels go through the game's line name (valid everywhere);
# sounds without one keep an ITA-only label.
ref_lang = "ITA" if "ITA" in game.archives else game.languages[0]
ref = scan_archive(ref_lang, game.wad(ref_lang), characters={})
chars, unmatched, ambiguous = build_characters(ref, src_root)
by_pos = {s.name: s for s in ref}
rows = collections.defaultdict(set) # (name, language) -> characters
for pos_name, who in chars.items():
s = by_pos[pos_name]
if s.key in line_names:
rows[(line_names[s.key], "")] |= who
else:
rows[(pos_name, ref_lang)] |= who
# safety: the catalog records ORIGINAL bytes. A sound that differs from the current catalog means a modified
# game (an applied change, another tool) unless the game itself was updated -> stop before writing anything.
changed = changed_since_catalog(game, out_dir, bank_snd)
if changed and not a.accept_changes:
print("STOP: %d sound(s) differ from the current catalog, e.g. %s." % (len(changed), changed[:5]))
print("Revert all changes (ackit status / ackit revert --all) and run again; after a game update use "
"--accept-changes.")
sys.exit(2)
# 2. every speech sound with its line identity and name, then the curated speaker lists through them
wad_sounds = {}
for arch, path in game.archives.items():
sounds = scan_archive(arch, path, characters={})
wad_sounds[arch] = apply_lines(sounds, lines, lines.names) if arch in lines.langs else sounds
final = {s.key: s for ss in wad_sounds.values() for s in ss}
rows, unmatched, ambiguous, skipped = build_characters(game, src_root, final)
with open(os.path.join(out_dir, "characters.csv"), "w", encoding="utf-8", newline="") as f:
w = csv.writer(f, lineterminator="\n")
w.writerow(["name", "character", "language"])
for (n, lg) in sorted(rows, key=lambda r: (r[0].lower(), r[1])):
w.writerow([n, "; ".join(sorted(rows[(n, lg)])), lg])
print("characters.csv: %d sounds labelled (%d by line name, %d %s-only); %d unmatched names %s; %d ambiguous %s"
% (len(rows), sum(1 for _, lg in rows if not lg), sum(1 for _, lg in rows if lg), ref_lang,
len(unmatched), unmatched[:3], len(ambiguous), ambiguous[:3]))
print("characters.csv: %d sounds labelled (%d by line name, %d language-only); %d unmatched names %s; "
"%d ambiguous %s; skipped list folders without a speech WAD: %s"
% (len(rows), sum(1 for _, lg in rows if not lg), sum(1 for _, lg in rows if lg),
len(unmatched), unmatched[:3], len(ambiguous), ambiguous[:3], skipped))
def charmap(lang):
return {n.lower(): "; ".join(sorted(c)) for (n, lg), c in rows.items() if not lg or lg == lang}
@@ -124,7 +166,7 @@ def main():
index = {"built_from": "unmodified game files", "archives": {}}
for arch, path in game.archives.items():
cm = charmap(arch)
sounds = apply_line_names(scan_archive(arch, path, characters={}), line_names)
sounds = wad_sounds[arch]
for s in sounds:
s.character = character_of(s, cm)
wad = Wad(path)
@@ -170,6 +212,11 @@ def main():
"speech_lines": len({s.speech for s in any_lang}),
"linked": {lang: sum(1 for s in rows if s.sound) for lang, rows in sorted(texts.items())},
"copies": len(sublocs), "issues": ["%s: %s" % i for i in sub_issues]}
index["archives"]["LINES"] = {
"languages": lines.langs, "recordings": st["recordings"], "lines": st["lines"],
"lines_in_every_language": st["lines_in_every_language"],
"lines_missing_a_language": st["lines_missing_a_language"], "without_line": st["without_line"],
"issues": ["%s: %s" % i for i in lines.issues]}
print("SUB_*.csv: %d languages, %d subtitle strings, %d copies"
% (len(texts), sum(len(r) for r in texts.values()), len(sublocs)))
# flatten for catalog.load_archive