gora/UnheardCity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

One language setting; localized in-memory speech per language

Browse Source
This commit is contained in:
Cristian Gora
2026-10-09 23:09:07 +02:00
parent 98ec2fe9c4
commit f29af40b37
40 changed files with 77414 additions and 77240 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
ackit/banks.py
View File

@@ -14,6 +14,7 @@ import gzip
import hashlib
import io
import os
import re
import struct
from collections import defaultdict, namedtuple
@@ -155,9 +156,40 @@ class Scan:
self.issues = [] # (kind, detail)
_LANG = re.compile(rb"([\x01-\x10])\x00\x00\x00\x04\x00\x00\x00[A-Z]{3}\x00")
def bank_languages(data, start, imports, exports, banks, issues=None, rel=""):
"""{bank offset in data: language} for localized banks. A localized AkBank export starts with its list of
languages (FStrings, e.g. INT FRA ITA DEU ESN) and holds one bank per language, in that order
(docs/formats/banks.md). Banks of other exports are language-neutral (not in the result)."""
out = {}
akbank = {-(i + 1) for i, (c, o, _) in enumerate(imports) if c == "Class" and o == "AkBank"}
starts = sorted(bk.at for bk in banks)
for x in exports:
if x.cls not in akbank or x.size <= 0:
continue
o0 = x.offset - start
m = _LANG.search(data, o0, min(o0 + 256, o0 + x.size))
if not m:
continue
n = m.group(1)[0]
langs, o = [], m.start() + 4
for _ in range(n):
ln = struct.unpack_from("<i", data, o)[0]
langs.append(bytes(data[o + 4:o + 3 + ln]).decode("latin-1"))
o += 4 + ln
inside = [a for a in starts if o0 <= a < o0 + x.size]
if len(inside) != len(langs):
if issues is not None:
issues.append(("languages", "%s: %s has %d banks for %d languages" % (rel, x.name, len(inside), len(langs))))
continue
out.update(zip(inside, langs))
return out
def _bkpk_tables(data, packs):
"""BKPK tables (docs/formats/wad.md) embedded in a package: WAD entry -> media offsets/sizes."""
import re
for m in re.finditer(b"BKPK", data):
p = m.start()
if p + 0x48 > len(data) or struct.unpack_from("<I", data, p + 0x1C)[0] != 0x48:
@@ -189,14 +221,16 @@ def scan(cooked, progress=None, cooked_banks=None, files=None, visit=None):
except (PackageError, OSError) as e:
sc.issues.append(("unreadable", "%s: %s" % (rel, e)))
continue
lang_of = {}
if isinstance(r, Package):
names, _, exports = r.tables(data, start)
names, imports, exports = r.tables(data, start)
sc.names.update(names) # base names ...
sc.names.update(x.name for x in exports) # ... and numbered object names (Speech_X_04)
_bkpk_tables(data, sc.packs)
if visit:
visit(rel, r, start, data)
banks = list(find_banks(data))
lang_of = bank_languages(data, start, imports, exports, banks, sc.issues, rel)
else:
try:
banks = [Bank(data, 0)]
@@ -217,7 +251,8 @@ def scan(cooked, progress=None, cooked_banks=None, files=None, visit=None):
if kind == PREFETCH_HEAD:
sc.prefetch.add(m.id)
e = sc.media.setdefault(m.id, {"size": m.size, "sha1": set(), "head": w[:0x60], "banks": set(),
"kinds": set()})
"kinds": set(), "langs": set()})
e["langs"].add(lang_of.get(bk.at, ""))
e["sha1"].add(hashlib.sha1(w).hexdigest()[:16])
e["banks"].add(bk.id)
e["kinds"].add(kind)
@@ -270,39 +305,57 @@ def event_media(sc):
return out
def media_language(sc, m):
"""Language of a localized in-memory media ('' = language-neutral)."""
langs = sc.media[m].get("langs", set()) - {""}
return next(iter(langs)) if len(langs) == 1 else ""
def media_names(sc):
"""{media id: (display name, primary bank name)}: named after the most specific 'Play' event that
reaches the media, otherwise after its bank (<Bank>_<n>)."""
reaches the media, otherwise after its bank (<Bank>_<n>). Localized speech has one media per language
for the same line: they share the name, variations are numbered within a language. Names are unique
among the language-neutral media plus any one language (what a catalog shows at a time)."""
ev = event_media(sc)
best = {}
for name, ms in ev.items():
rank = (0 if name.lower().startswith("play") else 1, len(ms), name.lower())
for i, m in enumerate(ms):
if m in sc.media and (m not in best or rank < best[m][0]):
best[m] = (rank, name, i, len(ms))
groups = defaultdict(list)
for m in ms:
if m in sc.media:
groups[media_language(sc, m)].append(m)
for g in groups.values():
rank = (0 if name.lower().startswith("play") else 1, len(g), name.lower())
for i, m in enumerate(g):
if m not in best or rank < best[m][0]:
best[m] = (rank, name, i, len(g))
out = {}
taken = set()
for m, (_, name, i, n) in sorted(best.items(), key=lambda x: (x[1][1].lower(), x[1][2])):
nm = name if n == 1 else "%s_%d" % (name, i + 1)
if nm.lower() in taken:
nm = "%s_%08X" % (nm, m)
taken.add(nm.lower())
out[m] = nm
# the rest: by bank, numbered in media id order
per_bank = defaultdict(list)
taken = set() # (language, lower-case name)
def free(lang, nm):
k = nm.lower()
if lang:
return (lang, k) not in taken and ("", k) not in taken
return not any(n == k for _, n in taken)
def take(lang, nm, m, base):
if not free(lang, nm):
nm = "%s_%08X" % (base, m)
taken.add((lang, nm.lower()))
return nm
for m, (_, name, i, n) in sorted(best.items(), key=lambda x: (x[1][1].lower(), x[1][2], x[0])):
out[m] = take(media_language(sc, m), name if n == 1 else "%s_%d" % (name, i + 1), m, name)
# the rest: by bank (and language), numbered in media id order
groups = defaultdict(list)
for m, e in sc.media.items():
bank = sorted(sc.bank_names.get(b, "%08X" % b) for b in e["banks"])[0]
per_bank[bank].append(m)
if m in out:
out[m] = (out[m], bank)
for bank, ms in per_bank.items():
rest = sorted(m for m in ms if m not in out)
for i, m in enumerate(rest):
nm = "%s_%d" % (bank, i + 1)
if nm.lower() in taken:
nm = "%s_%08X" % (bank, m)
taken.add(nm.lower())
out[m] = (nm, bank)
else:
groups[(bank, media_language(sc, m))].append(m)
for (bank, lang), ms in sorted(groups.items()):
for i, m in enumerate(sorted(ms)):
out[m] = (take(lang, "%s_%d" % (bank, i + 1), m, bank), bank)
return out

28
ackit/catalog.py
View File

@@ -25,7 +25,7 @@ from .pack import Pack
from .paths import catalog_dir, package_data, user_dir
from .wad import Wad
FIELDS = ["archive", "target", "name", "aliases", "group", "category", "bank", "character",
FIELDS = ["archive", "target", "name", "aliases", "group", "category", "bank", "language", "character",
"characters_inferred", "seconds", "rate", "channels", "size", "sha1", "notes"]
SCAN_VERSION = 2
@@ -39,6 +39,7 @@ class Sound:
group: str = "Unknown"
category: str = "Unknown"
bank: str = "" # BANK sounds: the soundbank holding it (name); WAD sounds: ''
language: str = "" # BANK sounds: language of a localized speech line ('' = any language)
character: str = "" # verified speaker(s), '; '-separated (curated catalog)
characters_inferred: str = "" # characters/roles mentioned by the name (hint)
seconds: float = 0.0
@@ -98,7 +99,8 @@ def read_csv(path):
for r in csv.DictReader(open(path, encoding="utf-8", newline="")):
out.append(Sound(archive=r["archive"], target=r["target"], name=r["name"],
aliases=tuple(a for a in r["aliases"].split("|") if a), group=r["group"],
category=r["category"], bank=r.get("bank") or "", character=r["character"],
category=r["category"], bank=r.get("bank") or "", language=r.get("language") or "",
character=r["character"],
characters_inferred=r["characters_inferred"], seconds=float(r["seconds"] or 0),
rate=int(r["rate"] or 0), channels=int(r["channels"] or 0), size=int(r["size"] or 0),
sha1=r["sha1"], notes=r["notes"]))
@@ -216,7 +218,7 @@ def bank_sounds(sc, characters=None):
name, bank = names[m]
rate, ch, secs = _hdr(e["head"])
s = Sound(banks.ARCHIVE, "%08X" % m, name, ("%s/%08X" % (bank, m),), bank=bank,
seconds=secs, rate=rate, channels=ch,
language=banks.media_language(sc, m), seconds=secs, rate=rate, channels=ch,
size=min(l.size for l in sc.locations if l.media == m) if len(e["sha1"]) > 1 else e["size"],
sha1=next(iter(e["sha1"])) if len(e["sha1"]) == 1 else "")
s.group, s.category = classify.classify(name, bank=bank)
@@ -263,18 +265,23 @@ def scan_banks(game, progress=None, characters=None, visit=None):
def subtitle_links(game, sc, speech_names):
"""{speech event name: {language: WAD sound key}}: event -> actions -> containers -> sounds -> the
per-language media of the streamed speech line (exact, no name matching)."""
"""{speech event name: {language: sound key}}: event -> actions -> containers -> sounds -> the
per-language media of the line (exact, no name matching). The media is a streamed WAD sound, or an
in-memory BANK sound of a localized bank."""
ev = banks.event_media(sc)
want = {m for n in speech_names for m in ev.get(n, ())}
owners = media_owners(game, sc, want)
owners = {m: (k.split(":")[0], k) for m, k in media_owners(game, sc, want).items()} # m -> (lang, key)
for m in want - set(owners):
lang = banks.media_language(sc, m) if m in sc.media else ""
if lang:
owners[m] = (lang, "%s:%08X" % (banks.ARCHIVE, m))
out = {}
for n in speech_names:
keys = {}
for m in ev.get(n, ()):
k = owners.get(m)
if k:
keys.setdefault(k.split(":")[0], k)
if m in owners:
lang, key = owners[m]
keys.setdefault(lang, key)
if keys:
out[n] = keys
return out
@@ -385,6 +392,9 @@ class Catalog:
for l in ls:
if l.kind == banks.PREFETCH_HEAD and l.sound:
self.heads.setdefault(l.sound, []).append(l)
# localized in-memory speech: only the languages loaded here (all of them if none is)
langs = [a for a in self.archives if a in game.languages] or None
bank_sounds_ = [s for s in bank_sounds_ if not s.language or not langs or s.language in langs]
for a in self.archives:
self.sounds += bank_sounds_ if a == banks.ARCHIVE else load_archive(game, a, progress)
self.by_key = {s.key: s for s in self.sounds}

4
ackit/settings.py
View File

@@ -4,10 +4,10 @@ import os
from .paths import user_dir
DEFAULTS = {"game_dir": None, "store_root": None, "full_backup": True, "language": None,
DEFAULTS = {"game_dir": None, "store_root": None, "full_backup": True,
"input_device": None, "output_device": None, "last_dir": None,
"background_enabled": True, "background_opacity": 0.1}
OBSOLETE = ("theme", "background_image")
OBSOLETE = ("theme", "background_image", "language") # the editor starts in the game's language
def path():

9
ackit/texts.py
View File

@@ -52,10 +52,11 @@ class Texts:
self.subtitles(lang)
return self._subs[lang][1].get(key, [])
def for_sound(self, sound):
"""Subtitle lines of a speech sound in its own language (or the INT line for other languages)."""
lang = sound.archive if sound.archive in self.subtitle_languages() else None
if not lang:
def for_sound(self, sound, lang=None):
"""Subtitle lines of a speech sound, in `lang` (default: the sound's own language). Languages without
their own voices link their subtitles to the INT sounds, so e.g. JPN subtitles are found for INT lines."""
lang = lang or sound.language or sound.archive
if lang not in self.subtitle_languages():
return []
return [s for s in self.subtitles(lang) if s.sound == sound.key]

21
ackit/verify/catalogcheck.py
View File

@@ -17,8 +17,8 @@ import os
import random
from .. import banks, classify, subtitles
from ..catalog import (Catalog, available_archives, bank_sounds, hash_names, head_owners, is_shipped, load_characters,
scan_archive)
from ..catalog import (Catalog, available_archives, bank_sounds, hash_names, head_owners, is_shipped,
load_banks, load_characters, read_csv, scan_archive)
from ..inject import read_target
from ..wad import Wad
@@ -60,9 +60,10 @@ def _check_rows(rep, archive, rows, characters):
if s.target in seen_t:
rep.add(archive, "duplicate", s.key, "target")
seen_t.add(s.target)
if s.name.lower() in seen_n:
rep.add(archive, "duplicate", s.key, "name %s (also %s)" % (s.name, seen_n[s.name.lower()]))
seen_n[s.name.lower()] = s.key
n = (s.language, s.name.lower()) # BANK: names are unique per language
if n in seen_n or (s.language and ("", n[1]) in seen_n):
rep.add(archive, "duplicate", s.key, "name %s (also %s)" % (s.name, seen_n.get(n) or seen_n[("", n[1])]))
seen_n[n] = s.key
g, c = _expected_category(s)
if (s.group, s.category) != (g, c):
rep.add(archive, "category", s.key, "%s/%s, rules give %s/%s" % (s.group, s.category, g, c))
@@ -159,6 +160,7 @@ def check_subtitles(rep, game, col, files, quick):
if found.get(k, set()) != index.get(k, set()):
rep.add("SUB", "copies", k, "index %d, game %d" % (len(index.get(k, ())), len(found.get(k, ()))))
langs = sorted(_shipped_index().get("SUB", {}).get("lines", {}))
bank_lang = {s.key: s.language for s in read_csv(os.path.join(catalog_dir(), "BANK.csv"))}
for lang in langs:
p = os.path.join(catalog_dir(), "SUB_%s.csv" % lang)
rows = subtitles.read_texts(p, lang)
@@ -171,7 +173,11 @@ def check_subtitles(rep, game, col, files, quick):
seen.add(k)
if k in texts and texts[k] != s.text:
rep.add("SUB", "text", k, "catalog %r, game %r" % (s.text[:40], texts[k][:40]))
if s.sound and not s.sound.startswith(lang + ":") and not s.sound.startswith("INT:"):
if s.sound.startswith(banks.ARCHIVE + ":"):
b = bank_lang.get(s.sound)
if b is not None and b not in (lang, "INT"): # INT: languages without own voices
rep.add("SUB", "language", k, "linked to %s (%s)" % (s.sound, b or "language-neutral"))
elif s.sound and not s.sound.startswith(lang + ":") and not s.sound.startswith("INT:"):
rep.add("SUB", "language", k, "linked to %s" % s.sound)
@@ -187,7 +193,8 @@ def run(game, archives=None, quick=False, log=print, progress=None):
for a in archives:
log("checking %s ..." % a)
if a == banks.ARCHIVE:
check_banks(rep, game, by_arch[a], cat.locations, characters, quick=quick, progress=progress)
rows = load_banks(game, progress)[0] # every language, not only the loaded one
check_banks(rep, game, rows, cat.locations, characters, quick=quick, progress=progress)
else:
check_wad(rep, game, a, by_arch[a], characters)
known = {n.lower() for s in cat.sounds for n in s.names()}