gora/UnheardCity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version 3.1.0: voice lines linked across all languages from the game data

Browse Source
This commit is contained in:
Cristian Gora
2026-10-10 14:10:03 +02:00
parent ea3e44000a
commit fbde809370
56 changed files with 74728 additions and 73360 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
tools/build_catalog.py
View File

@@ -4,13 +4,15 @@
Outputs
data/catalog/characters.csv name -> verified speaker, from data/catalog/source/<lang>/*.txt
(speech sounds carry the game's line identity and one name in every language: ackit/lines.py)
data/catalog/<ARCHIVE>.csv every sound of one archive (schema: data/catalog/README.md)
data/catalog/BANK.csv in-memory sounds (soundbanks in SFX/*.bnk and the .upk packages)
data/catalog/BANK_locations.csv.gz every stored copy of them + the prefetch heads of speech lines
data/catalog/SUB_<LANG>.csv subtitles of the speech lines (one file per language)
data/catalog/SUB_locations.csv.gz every stored copy of each subtitle string
data/catalog/index.json fingerprints the CSVs were built from + counts
Run it only on clean game files (`ackit status` must say nothing is modified).
Run it only on clean game files (`ackit status` must say nothing is modified). It stops before writing
anything if a sound differs from the current catalog (a modified game); after a game update use --accept-changes.
"""
import argparse
import collections
@@ -23,7 +25,7 @@ import time
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from ackit import banks, settings, subtitles # noqa: E402
from ackit.catalog import (apply_line_names, character_of, hash_names, scan_archive, scan_text, # noqa: E402
from ackit.catalog import (apply_lines, character_of, hash_names, read_csv, scan_archive, scan_text, # noqa: E402
write_csv)
from ackit.classify import USER_LABELS # noqa: E402
from ackit.game import Game, default_game_dir, wad_fingerprint # noqa: E402
@@ -51,17 +53,28 @@ def read_user_lists(src_dir):
return out
def build_characters(sounds, src_root):
"""Map the user's wemscan-style names onto catalog names (name or alias)."""
by_alias = collections.defaultdict(list)
for s in sounds:
for a in s.names():
by_alias[a.lower()].append(s)
rows, unmatched, ambiguous = {}, [], []
for lang in sorted(os.listdir(src_root)):
d = os.path.join(src_root, lang)
def build_characters(game, src_root, final):
"""Map the curated lists onto the catalog. Each list folder (source/<lang>/) names sounds of THAT language's
WAD by position (wemscan names: NAME = single entry, NAME_i = i-th WEM of the entry). Positions differ
between languages, so a listed recording is found by its position in that language and the label goes to
its line (`final`: {key: Sound} with line identities and names), i.e. to the same recording in every
language; a recording without a line keeps the label in that language only.
-> ({(name, language or ''): set(characters)}, unmatched names, ambiguous names, skipped folders)"""
rows = collections.defaultdict(set)
unmatched, ambiguous, skipped = [], [], []
for folder in sorted(os.listdir(src_root)):
d = os.path.join(src_root, folder)
if not os.path.isdir(d):
continue
lang = folder.upper()
if lang not in game.archives:
skipped.append(folder) # no speech WAD of that language: positions cannot be resolved
continue
ref = scan_archive(lang, game.wad(lang), characters={}) # position names of this language
by_alias = collections.defaultdict(list)
for s in ref:
for a in s.names():
by_alias[a.lower()].append(s)
for uname, who in read_user_lists(d).items():
hits = list({id(s): s for s in by_alias.get(uname, [])}.values())
if not hits:
@@ -70,9 +83,27 @@ def build_characters(sounds, src_root):
if len(hits) > 1:
ambiguous.append(uname) # wemscan name collision: both candidates get the label
for s in hits:
cur = rows.setdefault(s.name, set())
cur |= who
return rows, unmatched, ambiguous
f = final[s.key]
rows[(f.name, "" if f.line else lang)] |= who
return rows, unmatched, ambiguous, skipped
def changed_since_catalog(game, out_dir, bank_snd):
"""Keys whose original checksum in the existing catalog differs from the game files now."""
out = []
for arch, path in game.archives.items():
p = os.path.join(out_dir, "%s.csv" % arch)
if not os.path.exists(p):
continue
wad = Wad(path)
for s in read_csv(p):
if s.sha1 and s.sha1 != file_sha1(path, s.target, wad):
out.append(s.key)
p = os.path.join(out_dir, "%s.csv" % banks.ARCHIVE)
if os.path.exists(p):
old = {s.key: s.sha1 for s in read_csv(p)}
out += [s.key for s in bank_snd if old.get(s.key) and s.sha1 and old[s.key] != s.sha1]
return out
def file_sha1(path, target, wad):
@@ -82,6 +113,8 @@ def file_sha1(path, target, wad):
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--game")
ap.add_argument("--accept-changes", action="store_true",
help="the game was updated: accept sounds whose bytes differ from the current catalog")
a = ap.parse_args()
game = Game(a.game or default_game_dir(settings.load()))
out_dir = catalog_dir()
@@ -90,33 +123,42 @@ def main():
# 1. every package once: in-memory sounds, prefetch heads, subtitles, events (a few minutes)
t0 = time.time()
bank_snd, locs, sc, texts, sublocs, sub_issues, line_names = scan_text(
bank_snd, locs, sc, texts, sublocs, sub_issues, lines = scan_text(
game, progress=lambda k, n: print(" banks + subtitles: %d/%d files" % (k, n), flush=True)
if k % 500 == 0 else None, characters={})
print("packages scanned (%.0fs); %d speech sounds named by their game line" % (time.time() - t0, len(line_names)))
st = lines.stats
print("packages scanned (%.0fs); %d recordings in %d lines, %d of them in every language (%s); "
"%d recordings without a line; %d relation issues"
% (time.time() - t0, st["in_a_line"], st["lines"], st["lines_in_every_language"], "/".join(lines.langs),
len(st["without_line"]), len(lines.issues)))
for k, d in lines.issues[:10]:
print(" issue:", k, d)
# 2. characters from the curated lists. They name ITA sounds by position (wemscan); the position of a
# variation differs between languages, so labels go through the game's line name (valid everywhere);
# sounds without one keep an ITA-only label.
ref_lang = "ITA" if "ITA" in game.archives else game.languages[0]
ref = scan_archive(ref_lang, game.wad(ref_lang), characters={})
chars, unmatched, ambiguous = build_characters(ref, src_root)
by_pos = {s.name: s for s in ref}
rows = collections.defaultdict(set) # (name, language) -> characters
for pos_name, who in chars.items():
s = by_pos[pos_name]
if s.key in line_names:
rows[(line_names[s.key], "")] |= who
else:
rows[(pos_name, ref_lang)] |= who
# safety: the catalog records ORIGINAL bytes. A sound that differs from the current catalog means a modified
# game (an applied change, another tool) unless the game itself was updated -> stop before writing anything.
changed = changed_since_catalog(game, out_dir, bank_snd)
if changed and not a.accept_changes:
print("STOP: %d sound(s) differ from the current catalog, e.g. %s." % (len(changed), changed[:5]))
print("Revert all changes (ackit status / ackit revert --all) and run again; after a game update use "
"--accept-changes.")
sys.exit(2)
# 2. every speech sound with its line identity and name, then the curated speaker lists through them
wad_sounds = {}
for arch, path in game.archives.items():
sounds = scan_archive(arch, path, characters={})
wad_sounds[arch] = apply_lines(sounds, lines, lines.names) if arch in lines.langs else sounds
final = {s.key: s for ss in wad_sounds.values() for s in ss}
rows, unmatched, ambiguous, skipped = build_characters(game, src_root, final)
with open(os.path.join(out_dir, "characters.csv"), "w", encoding="utf-8", newline="") as f:
w = csv.writer(f, lineterminator="\n")
w.writerow(["name", "character", "language"])
for (n, lg) in sorted(rows, key=lambda r: (r[0].lower(), r[1])):
w.writerow([n, "; ".join(sorted(rows[(n, lg)])), lg])
print("characters.csv: %d sounds labelled (%d by line name, %d %s-only); %d unmatched names %s; %d ambiguous %s"
% (len(rows), sum(1 for _, lg in rows if not lg), sum(1 for _, lg in rows if lg), ref_lang,
len(unmatched), unmatched[:3], len(ambiguous), ambiguous[:3]))
print("characters.csv: %d sounds labelled (%d by line name, %d language-only); %d unmatched names %s; "
"%d ambiguous %s; skipped list folders without a speech WAD: %s"
% (len(rows), sum(1 for _, lg in rows if not lg), sum(1 for _, lg in rows if lg),
len(unmatched), unmatched[:3], len(ambiguous), ambiguous[:3], skipped))
def charmap(lang):
return {n.lower(): "; ".join(sorted(c)) for (n, lg), c in rows.items() if not lg or lg == lang}
@@ -124,7 +166,7 @@ def main():
index = {"built_from": "unmodified game files", "archives": {}}
for arch, path in game.archives.items():
cm = charmap(arch)
sounds = apply_line_names(scan_archive(arch, path, characters={}), line_names)
sounds = wad_sounds[arch]
for s in sounds:
s.character = character_of(s, cm)
wad = Wad(path)
@@ -170,6 +212,11 @@ def main():
"speech_lines": len({s.speech for s in any_lang}),
"linked": {lang: sum(1 for s in rows if s.sound) for lang, rows in sorted(texts.items())},
"copies": len(sublocs), "issues": ["%s: %s" % i for i in sub_issues]}
index["archives"]["LINES"] = {
"languages": lines.langs, "recordings": st["recordings"], "lines": st["lines"],
"lines_in_every_language": st["lines_in_every_language"],
"lines_missing_a_language": st["lines_missing_a_language"], "without_line": st["without_line"],
"issues": ["%s: %s" % i for i in lines.issues]}
print("SUB_*.csv: %d languages, %d subtitle strings, %d copies"
% (len(texts), sum(len(r) for r in texts.values()), len(sublocs)))
# flatten for catalog.load_archive

27
tools/e2e_editor.py
View File

@@ -29,9 +29,6 @@ def sha(p):
def main():
from PySide6.QtWidgets import QApplication
from ackit import audio
from ackit.inject import pack_bnk_path, parse_target
from ackit.verify import ingame
from app.mainwindow import MainWindow
from app.workers import Runner
@@ -45,6 +42,21 @@ def main():
ws = w.ws
assert ws, "game not loaded"
ok = True
try:
ok = run(w, ws, errors)
finally: # never leave a change behind, whatever failed
ws.revert_all()
ws.texts.revert_all()
print("E2E EDITOR %s" % ("PASS" if ok else "FAIL"))
w.close()
return 0 if ok else 1
def run(w, ws, errors):
from ackit import audio
from ackit.inject import pack_bnk_path, parse_target
from ackit.verify import ingame
ok = True
with tempfile.TemporaryDirectory() as td:
# --- music (stream pack, resized) ---
s = ws.sound("Music_393BC53A")
@@ -71,10 +83,11 @@ def main():
ok &= same
# --- speech line (in place) ---
t = ws.sound("Speech_TAPE_JOKER_1_4")
t = ws.sound("Speech_TAPE_JOKER_1#4") # 4th WEM of that entry in this language's WAD
twad = ws.wad_path(t)
tb = sha(twad)
w.select_name("Speech_TAPE_JOKER_1_4") or (w.clear_filters(), w.select_name("Speech_TAPE_JOKER_1_4"))
w.clear_filters()
assert w.select_name(t.name)
y, ysr = audio.decode_wem(ws.original_bytes(t))
line = os.path.join(td, "line.wav")
audio.save_wav(line, y[::-1], ysr) # reversed speech, same length
@@ -86,9 +99,7 @@ def main():
same2 = sha(twad) == tb
print("speech line applied (WAD changed: %s), reverted byte-identical: %s" % (changed, same2))
ok &= changed and same2
print("E2E EDITOR %s" % ("PASS" if ok else "FAIL"))
w.close()
return 0 if ok else 1
return ok
if __name__ == "__main__":

248
tools/e2e_features.py Normal file
View File

@@ -0,0 +1,248 @@
"""Real-game write round trips for the mission-3 features (no game launch; reverts everything itself).
python tools/e2e_features.py [--game DIR] [--lang ITA,INT,... | --lang all]
Runs the checks for each language (default: the game's language; `all` = every subtitle/menu language). A
language without its own voices (ESM, POL, POR, RUS, JPN, KOR) uses the INT voices, like the game and the editor.
For every check: the SHA-1 of each touched file is taken, the change is applied through Workspace (the code path
of the editor and the CLI), read back from the game files, reverted, and the files must be byte-identical again.
1. in-memory sound with the most copies (every copy written; once, it is shared by all languages)
2. localized in-memory speech line of the language's voices
3. speech line of the language's voices with prefetch heads (WAD + every head copy)
4. subtitle with several copies (RDialogueEvent), and a scripted subtitle (RSubtitle, Startup.upk)
5. menu text
6. sound + subtitle in the same package, reverted one at a time
7. mod package made with the language's voices and imported into another voice language (INT; FRA for INT):
matched by the game's line identity, so the same line (not the same WAD position) is replaced; a text key
names its language, so the edited subtitle stays that language's subtitle (the other language's is untouched)
Uses temporary originals stores (the user's store and settings are not touched), deleted at the end when every
change was reverted (kept, with a warning, otherwise). Whatever happens (a failed check, an exception), every
change is reverted before the tool exits; a sound that is not in its original state is never written.
"""
import argparse
import hashlib
import os
import shutil
import sys
import tempfile
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT)
import numpy as np # noqa: E402
from ackit import banks, settings, subtitles # noqa: E402
from ackit.game import Game, default_game_dir # noqa: E402
from ackit.modpack import export_mod, import_mod, match_mod # noqa: E402
from ackit.workspace import Workspace, MODIFIED, ORIGINAL # noqa: E402
RESULTS = []
SPACES = [] # every Workspace created: reverted in main()'s finally block
def sha(p):
h = hashlib.sha1()
with open(p, "rb") as f:
for c in iter(lambda: f.read(1 << 22), b""):
h.update(c)
return h.hexdigest()
def check(name, ok, detail=""):
RESULTS.append((name, bool(ok)))
print("%-62s %s %s" % (name, "PASS" if ok else "FAIL", detail))
def tone(seconds, freq):
t = np.arange(int(seconds * 48000)) / 48000.0
return (0.3 * np.sin(2 * np.pi * freq * t)).astype(np.float32)[:, None], 48000
def snapshot(paths):
return {p: sha(p) for p in sorted(set(paths))}
def copies_of(ws, s):
out = []
for l in ws.locations(s):
b = banks.open_file(ws.bank_path(l.file)).read(l.offset, l.size)
out.append((l, b))
return out
def sound_round_trip(ws, s, label):
if ws.state(s) != ORIGINAL:
check(label, False, "(%s is not original: %s; nothing written)" % (s.name, ws.state(s)))
return
wem, _ = ws.prepare(s, *tone(min(max(s.seconds, 0.3), 1.0), 700))
files = ws.files(s)
before = snapshot(files)
ws.apply([(s, wem, {"source": "e2e"})])
ok = ws.state(s) == MODIFIED and ws.current_bytes(s) == wem
for l, b in copies_of(ws, s):
if l.kind == banks.MEMORY:
ok &= b == wem + bytes(l.size - len(wem))
else: # prefetch head of a WAD line (stored zero-padded to its size)
full = wem + bytes(max(0, ws.store.get(s.key)["orig_size"] - len(wem)))
ok &= b == (full + bytes(l.size))[:l.size]
ws.revert([s])
ok &= ws.state(s) == ORIGINAL and snapshot(files) == before and not ws.store.entries
check(label, ok, "(%s, %d files, %d copies/heads)" % (s.name, len(files), len(ws.locations(s))))
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--game")
ap.add_argument("--lang", help="comma-separated languages, or 'all' (default: the game's language)")
a = ap.parse_args()
game = Game(a.game or default_game_dir(settings.load()))
td = tempfile.mkdtemp()
try:
first = True
for lang in languages(game, a.lang, td):
voices = lang if lang in game.languages else "INT"
ws = Workspace(game, os.path.join(td, "store_" + lang), [voices, "SFX", "BANK"])
SPACES.append(ws)
print("\n== %s (voices %s): %d sounds" % (lang, voices, len(ws.catalog.sounds)))
run_checks(game, lang, voices, td, ws, first)
first = False
finally:
clean = True
for w in SPACES: # never leave a change behind
w.revert_all()
w.texts.revert_all()
if w.store.entries or w.texts.records():
clean = False
print("WARNING: could not revert everything; store: %s" % w.store.root)
if clean: # the stores hold full copies of game files (GBs): drop them
shutil.rmtree(td, ignore_errors=True)
print("\n%d/%d PASS" % (sum(ok for _, ok in RESULTS), len(RESULTS)))
return 0 if RESULTS and all(ok for _, ok in RESULTS) else 1
def languages(game, arg, td):
if not arg:
return [game.language() or "INT"]
if arg.lower() != "all":
return [x.strip().upper() for x in arg.split(",") if x.strip()]
tx = Workspace(game, os.path.join(td, "store_list"), [game.languages[0]]).texts
return sorted(set(game.languages) | set(tx.subtitle_languages()) | set(tx.menu_languages()))
def run_checks(game, lang, voices, td, ws, first):
bank = [s for s in ws.catalog.sounds if s.archive == "BANK" and s.size >= 2500]
if first:
sound_round_trip(ws, max((s for s in bank if not s.language), key=lambda s: len(ws.locations(s))),
"1 in-memory sound, every copy")
loc = [s for s in bank if s.language == voices]
sound_round_trip(ws, max(loc, key=lambda s: len(ws.locations(s))),
"2 %s localized in-memory speech line (%s)" % (lang, voices))
heads = [ws.catalog.by_key[k] for k in ws.catalog.heads if k in ws.catalog.by_key
and ws.catalog.by_key[k].archive == voices]
sound_round_trip(ws, max(heads, key=lambda s: len(ws.locations(s))),
"3 %s speech line with prefetch heads" % lang)
tx = ws.texts
subs = tx.subtitles(lang)
def text_round_trip(key, new, label):
files = tx.files(key)
before = snapshot(files)
tx.apply({key: new})
ok = tx.current(key) == new and tx.state(key) == MODIFIED
for l in tx.subtitle_locations(key) if key.startswith("SUB:") else []:
b = banks.open_file(ws.bank_path(l.file)).read(l.pos, l.size)
ok &= subtitles.read_fstring(b, 0)[0] == new
tx.revert([key])
ok &= snapshot(files) == before and not tx.records()
check(label, ok, "(%s, %d files)" % (key[:60], len(files)))
multi = max((s for s in subs if s.room >= 8), key=lambda s: len(tx.subtitle_locations(subtitles.key(lang, s.speech, s.line))))
text_round_trip(subtitles.key(lang, multi.speech, multi.line), "E2E test.", "4a %s subtitle with several copies" % lang)
rsub = next((s for s in subs if s.room >= 10 and
{l.file for l in tx.subtitle_locations(subtitles.key(lang, s.speech, s.line))} == {"Startup.upk"}), None)
if rsub:
text_round_trip(subtitles.key(lang, rsub.speech, rsub.line), "E2E radio.", "4b %s scripted subtitle (RSubtitle)" % lang)
else:
check("4b %s scripted subtitle (RSubtitle)" % lang, False, "no scripted subtitle found in Startup.upk")
e = next(x for x in tx.menu(lang) if x.file.upper().startswith("FRONTEND"))
text_round_trip("MENU:" + e.key, "E2E " + e.value, "5 %s menu text" % lang)
# 6. sound + subtitle in one package
sub_files = {}
for s in subs:
if s.room < len("Shared package."):
continue
for l in tx.subtitle_locations(subtitles.key(lang, s.speech, s.line)):
sub_files.setdefault(l.file, subtitles.key(lang, s.speech, s.line))
pair = next(((s, sub_files[l.file]) for s in bank if not s.language for l in ws.locations(s)
if l.file in sub_files and len(ws.locations(s)) <= 3), None)
if pair:
s, key = pair
files = set(ws.files(s)) | set(tx.files(key))
before = snapshot(files)
wem, _ = ws.prepare(s, *tone(0.3, 900))
ws.apply([(s, wem, {})])
tx.apply({key: "Shared package."})
ok = ws.state(s) == MODIFIED and tx.current(key) == "Shared package."
ws.revert([s])
ok &= ws.state(s) == ORIGINAL and tx.current(key) == "Shared package."
tx.revert([key])
ok &= snapshot(files) == before
check("6 %s sound + subtitle in one package, reverted separately" % lang, ok, "(%s + %s)" % (s.name, key[:40]))
else:
check("6 %s sound + subtitle in one package" % lang, False, "no such package found")
# 7. cross-language mod: the same game line, not the same WAD position
from ackit.catalog import read_csv
from ackit.paths import catalog_dir
other = "FRA" if voices == "INT" else "INT"
other_target = {s.name: s.target for s in read_csv(os.path.join(catalog_dir(), other + ".csv"))}
other_subs = {(s.speech, s.line) for s in tx.subtitles(other)}
by_sound = {}
for s in subs:
for k in s.sounds:
by_sound.setdefault(k, s)
line = next((s for s in ws.catalog.sounds if s.archive == voices and ":" in s.target
and s.key in by_sound and by_sound[s.key].room >= 9
and (by_sound[s.key].speech, by_sound[s.key].line) in other_subs # the line exists there too
and other_target.get(s.name, s.target) != s.target), None) # at another position there
if line is None:
check("7 mod %s -> %s" % (lang, other), False, "no line found at another position in %s" % other)
return
sub = by_sound[line.key]
sub_key = subtitles.key(lang, sub.speech, sub.line)
if ws.state(line) != ORIGINAL or tx.state(sub_key) != ORIGINAL:
check("7 mod %s -> %s" % (lang, other), False, "(%s is not original; nothing written)" % line.name)
return
wem, _ = ws.prepare(line, *tone(1.0, 500))
ws.apply([(line, wem, {})])
tx.apply({sub_key: "Mod text."})
pkg = os.path.join(td, "e2e_%s.acmod" % lang)
export_mod(ws, pkg, ws.modified_keys(), "e2e", texts=tx.modified())
ws.revert_all()
tx.revert_all()
wo = Workspace(game, os.path.join(td, "store_%s_%s" % (lang, other)), [other, "SFX", "BANK"])
SPACES.append(wo)
_, items = match_mod(wo, pkg)
target = items[0].sound
same_line = target is not None and target.name == line.name
moved = target is not None and target.target != line.target # positions differ between languages
other_sub = subtitles.key(other, sub.speech, sub.line)
files = set(wo.files(target)) if target else set()
files |= set(wo.texts.files(other_sub)) | set(wo.texts.files(sub_key))
before = snapshot(files)
other_text = wo.texts.current(other_sub)
res, texts = import_mod(wo, pkg)
ok = same_line and wo.state(target) == MODIFIED
ok &= [k for k, _, p in texts if not p] == [sub_key] and wo.texts.current(sub_key) == "Mod text."
ok &= wo.texts.current(other_sub) == other_text
wo.revert_all()
wo.texts.revert_all()
ok &= snapshot(files) == before
check("7 mod %s -> %s: same game line%s" % (lang, other, " (another WAD position)" if moved else ""), ok,
"(%s: %s:%s -> %s:%s)" % (line.name, voices, line.target, other, target.target if target else "-"))
if __name__ == "__main__":
sys.exit(main())