Edit subtitles and menu text

This commit is contained in:
2026-10-09 22:36:46 +02:00
parent 5d6c2447f7
commit ef68edbe45
40 changed files with 105877 additions and 1291 deletions

View File

@@ -9,6 +9,7 @@ Batman: Arkham City (PC) audio toolkit + editor. Read `NOTES.md` first (status,
Speech/ambience: in place, never larger than the original WEM; music packs: any size (`docs/formats/wad.md`).
In-memory sounds (archive BANK, loose `.bnk` + `.upk` packages): in place into every copy; a bank file is always
rebuilt from the store's original copy + the applied replacements (`docs/formats/banks.md`, `upk.md`).
Texts go through `Workspace.texts` (subtitles in packages, in place and NUL-padded; menu `.int` files).
- Never use Steam "verify integrity". Keep game files original after tests (`ackit status`, `ackit revert --all`).
- Real-game checks: `ackit verify` (offline), `ackit verify --catalog [--quick]` (catalog vs game files),
`ackit verify --ingame [--banks]` (Windows; needs Steam running and logged in, WinDbg; ~1.5 min; reverts and

View File

@@ -27,7 +27,11 @@
rebuilt from a mandatory full original copy); editor shows bank + copies. `ackit verify` PASS (300/300
BANK WEMs rebuilt identically); `ackit verify --ingame --banks` PASS (loaded, no crash, revert identical;
audible check not possible without input: findings 18).
4. [ ] Subtitles + menu text: find/document formats, `ackit` parse/write with backup/revert.
4. [x] Subtitles = RDialogueEvent objects in packages (9,016 lines x 11 languages, 168k copies; parser
`ackit/subtitles.py`, catalog SUB_<LANG>.csv + SUB_locations.csv.gz, 92 % linked to speech sounds via
events); menu text = Localization/<LANG>/*.int (`ackit/loctext.py`, ~7,000 strings/language).
`Workspace.texts` (ackit/texts.py): states, apply/revert, conflicts, mod packages; CLI `ackit text`.
Subtitles in place (NUL-padded, limit = original length); decision + reasons in decisions.md 4c.
5. [ ] Editor: Subtitles section linked to audio, separate Menu text section, revert, mod export/import.
6. [ ] Final e2e (audio + text), packaged build, REPORT.md.
- OPEN: channel order for >2 channels unverified; speech and in-memory sounds *heard* in-game (need gameplay;
@@ -105,3 +109,7 @@
runs (no compressor needed). Bank-file conflicts use the whole-file SHA-1 (the cheap guid+chunk-table
fingerprint missed same-size edits in tests). Name collisions in the WAD catalogs (2 per language) are shown as
NAME#i. `soundcard` comes from the `verify` extra (was missing from the venv).
- 2026-10-09 (mission 3, stage 4): subtitle links by name were wrong for most multi-variation entries (WEMs are
stored in media-ID order, not _NN order): links come only from events. Numbered event names (Speech_X_04)
live in export tables, not in name tables: the bank scan now collects export names (+40 named BANK sounds,
links 71 % -> 92 %). Version*.int (build IDs) are not treated as UI text.

View File

@@ -33,13 +33,16 @@ ackit find joker tape # search by name, category or c
ackit extract Speech_TAPE_JOKER_1_4 orig.wav # listen to the original
ackit replace Speech_TAPE_JOKER_1_4 my_line.wav --match-loudness
ackit revert Speech_TAPE_JOKER_1_4 # or: ackit revert --all
ackit text find --lang ITA protocollo # subtitles and menu texts (keys SUB:... / MENU:...)
ackit text set "MENU:ITA:FRONTEND.ITA:Menu:Start" "Premi Invio"
```
More: `ackit status`, `ackit list --character Joker`, `ackit export-mod my.acmod` / `ackit import-mod`,
`ackit verify` (offline self-check), `ackit game` (detected install, language, store). `ackit -h` lists all
commands. The game folder is found through Steam (Windows, Linux, Flatpak, Proton); otherwise pass `--game DIR`.
Rules worth knowing: speech/effects replacements must fit in the original's size (the tool squeezes or refuses
with the maximum length); music can be any length. Close the game before writing.
with the maximum length); music can be any length. Subtitles must fit the original text's length (menu text:
any length). Close the game before writing.
## Where things are
```

View File

@@ -155,23 +155,6 @@ class Scan:
self.issues = [] # (kind, detail)
def _package_names(pkg, start, data):
"""Name table of an uncompressed package."""
out = []
o = pkg.name_offset - start
for _ in range(pkg.name_count):
n = struct.unpack_from("<i", data, o)[0]
o += 4
if n > 0:
out.append(bytes(data[o:o + n - 1]).decode("latin-1"))
o += n
else:
out.append(bytes(data[o:o - 2 * n - 2]).decode("utf-16-le", "replace"))
o += -2 * n
o += 8
return out
def _bkpk_tables(data, packs):
"""BKPK tables (docs/formats/wad.md) embedded in a package: WAD entry -> media offsets/sizes."""
import re
@@ -187,8 +170,9 @@ def _bkpk_tables(data, packs):
packs[entry][mid] = (off, size)
def scan(cooked, progress=None, cooked_banks=None, files=None):
"""Scan every loose bank and package of the game (minutes), or only `files`. Returns a `Scan`."""
def scan(cooked, progress=None, cooked_banks=None, files=None, visit=None):
"""Scan every loose bank and package of the game (minutes), or only `files`. Returns a `Scan`.
`visit(rel, package, start, data)` is called for every package (other scans share the decompression)."""
sc = Scan()
files = bank_files(cooked) if files is None else files
if cooked_banks is None:
@@ -206,8 +190,12 @@ def scan(cooked, progress=None, cooked_banks=None, files=None):
sc.issues.append(("unreadable", "%s: %s" % (rel, e)))
continue
if isinstance(r, Package):
sc.names.update(_package_names(r, start, data))
names, _, exports = r.tables(data, start)
sc.names.update(names) # base names ...
sc.names.update(x.name for x in exports) # ... and numbered object names (Speech_X_04)
_bkpk_tables(data, sc.packs)
if visit:
visit(rel, r, start, data)
banks = list(find_banks(data))
else:
try:

View File

@@ -18,7 +18,7 @@ import re
import struct
from dataclasses import dataclass, asdict
from . import banks, classify
from . import banks, classify, subtitles
from .game import wad_fingerprint
from .inject import pack_bnk_path
from .pack import Pack
@@ -228,9 +228,13 @@ def bank_sounds(sc, characters=None):
def head_owners(game, sc, archives=None):
"""{prefetch media id: WAD sound key} -- the speech line (one language) each prefetch head belongs to,
found through the per-language BKPK tables of the packages (docs/formats/banks.md)."""
want = {m for m, e in sc.media.items() if banks.PREFETCH_HEAD in e["kinds"]}
"""{prefetch media id: WAD sound key} -- the speech line (one language) each prefetch head belongs to."""
return media_owners(game, sc, {m for m, e in sc.media.items() if banks.PREFETCH_HEAD in e["kinds"]}, archives)
def media_owners(game, sc, want, archives=None):
"""{media id: WAD sound key} for streamed speech media, found through the per-language BKPK tables of
the packages (docs/formats/banks.md): every language has its own media IDs for the same WAD entry."""
out = {}
for a in archives or [x for x in game.archives if x != "SFX"]:
wad = Wad(game.wad(a))
@@ -250,14 +254,61 @@ def head_owners(game, sc, archives=None):
return out
def scan_banks(game, progress=None, characters=None):
def scan_banks(game, progress=None, characters=None, visit=None):
"""Full scan of the game's banks -> ([Sound], [Location] with WAD keys on prefetch heads, Scan)."""
sc = banks.scan(game.cooked, progress)
sc = banks.scan(game.cooked, progress, visit=visit)
owners = head_owners(game, sc)
locs = [l._replace(sound=owners.get(l.media, "")) if l.kind == banks.PREFETCH_HEAD else l for l in sc.locations]
return bank_sounds(sc, characters), locs, sc
def subtitle_links(game, sc, speech_names):
"""{speech event name: {language: WAD sound key}}: event -> actions -> containers -> sounds -> the
per-language media of the streamed speech line (exact, no name matching)."""
ev = banks.event_media(sc)
want = {m for n in speech_names for m in ev.get(n, ())}
owners = media_owners(game, sc, want)
out = {}
for n in speech_names:
keys = {}
for m in ev.get(n, ()):
k = owners.get(m)
if k:
keys.setdefault(k.split(":")[0], k)
if keys:
out[n] = keys
return out
def scan_text(game, progress=None, characters=None):
"""One pass over every package: (bank sounds, bank locations, Scan, {lang: [SubLine]}, [SubLoc], issues)."""
col = subtitles.Collector()
snd, locs, sc = scan_banks(game, progress, characters, visit=col)
links = subtitle_links(game, sc, list(col.dialogues))
texts, sublocs = subtitles.rows(col, links)
return snd, locs, sc, texts, sublocs, col.issues
def load_subtitles(game, lang, progress=None):
"""([SubLine], {SUB key: [SubLoc]}) of one language: shipped catalog when the game matches it, else a
cached or fresh scan of the packages (several minutes)."""
fp = banks.game_fingerprint(game.cooked)
shipped = _shipped_index().get("SUB", {})
t = os.path.join(catalog_dir(), "SUB_%s.csv" % lang)
loc = os.path.join(catalog_dir(), subtitles.LOCATIONS)
if shipped.get("fingerprint") == fp and os.path.exists(t) and os.path.exists(loc):
return subtitles.read_texts(t, lang), subtitles.read_locations(loc, lang)
cache = os.path.join(user_dir("cache"), "scan_SUB_%s_v%d" % (fp, SCAN_VERSION))
if not os.path.exists(cache + ".locations.csv.gz"):
_, _, _, texts, sublocs, _ = scan_text(game, progress)
for lg, rows in texts.items():
subtitles.write_texts(cache + "_%s.csv" % lg, rows)
subtitles.write_locations(cache + ".locations.csv.gz", sublocs)
t = cache + "_%s.csv" % lang
return (subtitles.read_texts(t, lang) if os.path.exists(t) else []), \
subtitles.read_locations(cache + ".locations.csv.gz", lang)
def load_banks(game, progress=None):
"""([Sound], {media: [Location]}) of the in-memory sounds: shipped catalog when the game's bank set
matches it, else a cached or fresh scan (several minutes)."""

View File

@@ -171,7 +171,9 @@ def cmd_revert(a):
ws = _workspace(a)
if a.all:
done = ws.revert_all(progress=print)
else:
texts = ws.texts.revert_all(progress=print)
print("reverted %d sound(s) and %d text(s)" % (len(done), len(texts)))
return
done = ws.revert([ws.sound(n) for n in a.names], progress=print)
print("reverted %d sound(s)" % len(done))
@@ -179,29 +181,72 @@ def cmd_revert(a):
def cmd_status(a):
ws = _workspace(a)
keys = ws.modified_keys()
if not keys:
print("no sounds modified; game audio is original")
texts = ws.texts.modified()
if not keys and not texts:
print("no sounds or texts modified; game files are original")
return
for k in keys:
s = ws.catalog.by_key.get(k)
st = ws.state(s) if s else "(other language)"
print("%-50s %s" % (s.name if s else k, st))
for k in texts:
print("%-50s %s" % (k, ws.texts.state(k)))
def cmd_export_mod(a):
from .modpack import export_mod
ws = _workspace(a)
keys = [ws.sound(n).key for n in a.names] if a.names else ws.modified_keys()
texts = [] if a.names else ws.texts.modified()
n = export_mod(ws, a.out, keys, a.title or os.path.splitext(os.path.basename(a.out))[0], a.author or "",
a.description or "")
print("wrote %s (%d sounds)" % (a.out, n))
a.description or "", texts=texts)
print("wrote %s (%d sounds, %d texts)" % (a.out, len(keys), n - len(keys)))
def cmd_import_mod(a):
from .modpack import import_mod
ws = _workspace(a)
for im in import_mod(ws, a.file, force=a.force, progress=print):
items, texts = import_mod(ws, a.file, force=a.force, progress=print)
for im in items:
print("%-8s %-46s %s" % (im.how, im.item.get("name"), im.note))
for k, _, problem in texts:
print("%-8s %-46s %s" % ("text", k, problem or "applied"))
def cmd_text(a):
"""Subtitles and menu text: find, show, set, revert."""
ws = _workspace(a)
tx = ws.texts
lang = (a.lang or ws.game.language()).upper()
if a.action == "find":
words = [w.lower() for w in a.args]
rows = []
if a.kind in ("all", "subs"):
rows += [(subtitles_key(s), s.text) for s in tx.subtitles(lang)]
if a.kind in ("all", "menu"):
rows += [("MENU:" + e.key, e.value) for e in tx.menu(lang)]
hits = [(k, v) for k, v in rows if all(w in (k + " " + v).lower() for w in words)]
for k, v in hits[:a.limit]:
print("%s\n %s" % (k, v))
print("%d match(es)%s" % (len(hits), "" if len(hits) <= a.limit else " (first %d shown)" % a.limit))
elif a.action == "show":
for k in a.args:
print("%s\n now: %s\n original: %s\n state: %s%s" % (
k, tx.current(k), tx.original(k), tx.state(k),
"" if tx.limit(k) is None else "\n room: %d characters" % tx.limit(k)))
elif a.action == "set":
if len(a.args) != 2:
raise GameError("usage: ackit text set KEY \"new text\"")
tx.apply({a.args[0]: a.args[1]}, force=a.force, progress=print)
print("set %s" % a.args[0])
elif a.action == "revert":
done = tx.revert_all(progress=print) if a.all else tx.revert(a.args, progress=print)
print("reverted %d text(s)" % len(done))
def subtitles_key(s):
from .subtitles import key
return key(s.lang, s.speech, s.line)
# --- advanced ---------------------------------------------------------------------------------
@@ -321,6 +366,12 @@ def build_parser():
s = add("revert", cmd_revert, "put original sounds back", aliases=["restore"])
s.add_argument("names", nargs="*"); s.add_argument("--all", action="store_true")
add("status", cmd_status, "list the sounds you modified")
s = add("text", cmd_text, "subtitles and menu text: find WORDS | show KEY | set KEY TEXT | revert KEY/--all")
s.add_argument("action", choices=["find", "show", "set", "revert"])
s.add_argument("args", nargs="*")
s.add_argument("--kind", choices=["all", "subs", "menu"], default="all", help="with find: what to search")
s.add_argument("--limit", type=int, default=40); s.add_argument("--all", action="store_true")
s.add_argument("--force", action="store_true", help="overwrite texts changed outside this tool")
s = add("export-mod", cmd_export_mod, "save your changes as a shareable .acmod package")
s.add_argument("out"); s.add_argument("names", nargs="*", help="default: everything you modified")
s.add_argument("--title"); s.add_argument("--author"); s.add_argument("--description")

110
ackit/loctext.py Normal file
View File

@@ -0,0 +1,110 @@
"""Menu and UI text: the game's localization files BmGame/Localization/<LANG>/<File>.<lang>
(UE3 .ini format; docs/formats/text.md).
Files are UTF-16LE with a BOM and CRLF line ends (two small ones are plain ASCII). Lines are `[Section]`,
`Key=Value` (the value may be in double quotes) or comments (`;`). A key can repeat inside a section, so an
entry is identified by (file, section, key, occurrence). Writing changes only the value part of the edited
lines; every other byte of the file (BOM, encoding, quotes, line ends, comments) stays as it was.
"""
import os
import re
from collections import namedtuple
# key: "<LANG>:<File>:<Section>:<Key>" + ("#n" for the n-th repeat, n >= 2)
Entry = namedtuple("Entry", "key lang file section name occurrence value quoted line")
LANG_DIRS = ("INT", "ITA", "FRA", "DEU", "ESN", "ESM", "POL", "POR", "RUS", "JPN", "KOR")
class TextError(ValueError):
pass
def _decode(raw):
if raw[:2] == b"\xff\xfe":
return raw[2:].decode("utf-16-le"), "utf-16"
if raw[:3] == b"\xef\xbb\xbf":
return raw[3:].decode("utf-8"), "utf-8"
return raw.decode("cp1252"), "cp1252"
def _encode(text, enc):
if enc == "utf-16":
return b"\xff\xfe" + text.encode("utf-16-le")
if enc == "utf-8":
return b"\xef\xbb\xbf" + text.encode("utf-8")
return text.encode("cp1252")
_KV = re.compile(r"^([^=;\[\s][^=]*?)=(.*)$")
def parse(raw, lang, fname):
"""Entries of one file's bytes."""
text, _ = _decode(raw)
out = []
section = ""
seen = {}
for i, line in enumerate(re.split(r"(?<=\n)", text)):
body = line.rstrip("\r\n")
s = body.strip()
if s.startswith("[") and s.endswith("]"):
section = s[1:-1]
continue
m = _KV.match(body)
if not m or s.startswith(";"):
continue
name, value = m.group(1), m.group(2)
quoted = len(value) >= 2 and value[0] == '"' and value[-1] == '"'
if quoted:
value = value[1:-1]
k = (section.lower(), name.lower())
seen[k] = seen.get(k, 0) + 1
n = seen[k]
key = "%s:%s:%s:%s" % (lang, fname, section, name) + ("#%d" % n if n > 1 else "")
out.append(Entry(key, lang, fname, section, name, n, value, quoted, i))
return out
def write(raw, lang, fname, values):
"""New file bytes with {entry key: new value} applied (keys not in the file raise TextError)."""
text, enc = _decode(raw)
lines = re.split(r"(?<=\n)", text)
by_key = {e.key: e for e in parse(raw, lang, fname)}
for key, value in values.items():
e = by_key.get(key)
if e is None:
raise TextError("no text %s in %s" % (key, fname))
if "\r" in value or "\n" in value:
raise TextError("a menu text cannot contain line breaks (use the game's own markers, e.g. \\n)")
line = lines[e.line]
end = line[len(line.rstrip("\r\n")):]
head = line[:line.index("=") + 1]
lines[e.line] = head + ('"%s"' % value if e.quoted else value) + end
out = "".join(lines)
try:
return _encode(out, enc)
except UnicodeEncodeError:
raise TextError("%s is stored as %s and cannot hold these characters" % (fname, enc)) from None
def files(loc_root, lang):
"""{file name: path} of one language folder (case-insensitive): localization files, without the
Version*.int build IDs (not UI text)."""
d = next((os.path.join(loc_root, x) for x in os.listdir(loc_root) if x.upper() == lang.upper()), None)
if not d:
return {}
return {f: os.path.join(d, f) for f in sorted(os.listdir(d))
if os.path.splitext(f)[1].lower() == "." + lang.lower() and not f.lower().startswith("version")}
def languages(loc_root):
return sorted(x.upper() for x in os.listdir(loc_root) if os.path.isdir(os.path.join(loc_root, x)))
def load(loc_root, lang):
"""All entries of a language: [Entry]."""
out = []
for f, p in files(loc_root, lang).items():
out += parse(open(p, "rb").read(), lang.upper(), f)
return out

View File

@@ -6,6 +6,8 @@ A package is a zip:
items/NNN.flac the same audio, decoded (used to re-encode when the target differs, e.g. another
language whose original has a different size)
Each item records archive, target, name, group and the SHA-1 of the original it replaced.
"texts": [{"key", "text", "original"}] edited subtitles / menu texts (keys: ackit/texts.py); optional,
so packages without texts stay valid
"""
import io
import json
@@ -22,9 +24,17 @@ from .originals import sha1_16
FORMAT = 1
def export_mod(ws, path, keys, name="", author="", description=""):
"""Write the applied versions of `keys` (from the originals store) into a package."""
def export_mod(ws, path, keys, name="", author="", description="", texts=()):
"""Write the applied versions of sound `keys` and text keys `texts` (from the originals store) into a
package. Returns the number of entries."""
items = []
recs = ws.texts.records()
text_items = []
for k in texts:
r = recs.get(k)
if not r or r.get("applied") is None:
raise ValueError("%s is not modified" % k)
text_items.append({"key": k, "text": r["applied"], "original": r["orig"]})
with zipfile.ZipFile(path + ".tmp", "w", zipfile.ZIP_DEFLATED) as z:
for i, key in enumerate(keys):
e = ws.store.get(key)
@@ -43,9 +53,10 @@ def export_mod(ws, path, keys, name="", author="", description=""):
"wem": base + ".wem", "audio": base + ".flac", "meta": e.get("meta", {})})
z.writestr("manifest.json", json.dumps({"format": FORMAT, "name": name, "author": author,
"description": description, "created": time.time(),
"game": "Batman: Arkham City (PC)", "items": items}, indent=1))
"game": "Batman: Arkham City (PC)", "items": items,
"texts": text_items}, indent=1, ensure_ascii=False))
os.replace(path + ".tmp", path)
return len(items)
return len(items) + len(text_items)
@dataclass
@@ -89,6 +100,20 @@ def match_mod(ws, path):
return man, out
def match_texts(ws, man):
"""[(key, text, problem or '')] for the package's texts in this game (problem: why it cannot be applied)."""
out = []
for it in man.get("texts", []):
k, text = it["key"], it["text"]
try:
ws.texts.current(k)
ws.texts.check(k, text)
out.append((k, text, ""))
except Exception as e: # missing key, too long, characters the file cannot hold
out.append((k, text, str(e)))
return out
def prepare_import(ws, path, items, progress=None):
"""Fill `wem` for matched items: reuse the packaged WEM when it targets the very same original,
otherwise re-encode the packaged audio for the local sound."""
@@ -110,11 +135,15 @@ def prepare_import(ws, path, items, progress=None):
def import_mod(ws, path, force=False, progress=None):
"""Match, prepare and apply a package. Returns the list of ImportItem (see .how/.note)."""
_, items = match_mod(ws, path)
"""Match, prepare and apply a package. Returns ([ImportItem] (see .how/.note), [(key, text, problem)])."""
man, items = match_mod(ws, path)
prepare_import(ws, path, items, progress)
changes = [(im.sound, im.wem, {"source": "mod:" + path.replace("\\", "/").split("/")[-1]})
for im in items if im.wem]
source = {"source": "mod:" + path.replace("\\", "/").split("/")[-1]}
changes = [(im.sound, im.wem, source) for im in items if im.wem]
if changes:
ws.apply(changes, force=force, progress=progress)
return items
texts = match_texts(ws, man)
ok = {k: v for k, v, problem in texts if not problem}
if ok:
ws.texts.apply(ok, force=force, progress=progress, meta=source)
return items, texts

255
ackit/subtitles.py Normal file
View File

@@ -0,0 +1,255 @@
"""Subtitles: the RDialogueEvent objects inside the game's packages (docs/formats/subtitles.md).
Each dialogue object references its speech event (an export named like the speech line, e.g.
'Speech_CHURCH_B1_Ch2_POST_Explosion_BM_Cash_04') and holds the subtitle text of every language:
LocalizedSubtitles: array of {Language: FString, Subtitles: array of {Text: FString, (float @0x0c)}}
in Rocksteady's compact tagged-property format (u16 type, u16 field offset, value; structs and arrays with
a byte size; struct elements end with a u16 0).
A dialogue object can be stored in several packages; a subtitle edit is written into every copy, in place:
the new text replaces the bytes of the old FString and is padded with NUL characters (the game stops at the
first NUL), so no size field, export or package layout changes. The text must therefore fit the original
string's bytes (Latin-1/cp1252 strings: one byte per character; UTF-16 strings: two).
"""
import csv
import gzip
import io
import os
import re
import struct
from collections import defaultdict, namedtuple
from .upk import Package
CLASS = "RDialogueEvent"
T_STRUCT, T_STRING, T_OBJECT = 0x09, 0x0D, 0x10
# one subtitle string inside a package: pos = absolute uncompressed offset of the FString length field,
# size = bytes of the whole FString (length field included), enc 'a' (cp1252) or 'w' (UTF-16)
Field = namedtuple("Field", "lang index text pos size enc")
Dialogue = namedtuple("Dialogue", "speech export fields") # speech: name of the referenced speech event
class SubtitleError(ValueError):
pass
def read_fstring(b, o):
"""-> (text, end offset, enc)"""
n = struct.unpack_from("<i", b, o)[0]
if n >= 0:
raw = bytes(b[o + 4:o + 4 + n])
return raw.split(b"\0", 1)[0].decode("cp1252", "replace"), o + 4 + n, "a"
raw = bytes(b[o + 4:o + 4 - 2 * n])
return raw.decode("utf-16-le", "replace").split("\0", 1)[0], o + 4 - 2 * n, "w"
def encode_fstring(text, size, enc):
"""The FString bytes (length field included) for `text`, exactly `size` bytes long (NUL padded).
Raises SubtitleError when the text does not fit."""
if "\0" in text:
raise SubtitleError("the text must not contain NUL characters")
if enc == "a":
try:
raw = text.encode("cp1252")
except UnicodeEncodeError:
raise SubtitleError("this subtitle is stored as Latin-1 text; '%s' cannot be written"
% next(c for c in text if not _cp1252(c))) from None
n = size - 4
if len(raw) + 1 > n:
raise SubtitleError("the text is %d characters long but this subtitle has room for %d"
% (len(raw), n - 1))
return struct.pack("<i", n) + raw + bytes(n - len(raw))
raw = text.encode("utf-16-le")
n = (size - 4) // 2
if len(raw) // 2 + 1 > n:
raise SubtitleError("the text is %d characters long but this subtitle has room for %d" % (len(raw) // 2, n - 1))
return struct.pack("<i", -n) + raw + bytes(2 * n - len(raw))
def _cp1252(c):
try:
c.encode("cp1252")
return True
except UnicodeEncodeError:
return False
def room(field):
"""Maximum number of characters for a subtitle string."""
return (field.size - 4 - 1) if field.enc == "a" else (field.size - 4) // 2 - 1
def _parse_lines(b, o, end, base, lang):
"""Subtitles array content [o, end) -> [Field]."""
count = struct.unpack_from("<I", b, o)[0]
o += 4
out = []
for i in range(count):
# element: props until u16 0; the text is the string at field offset 0
while True:
t, fo = struct.unpack_from("<HH", b, o)
if t == 0:
o += 2
break
if t == T_STRING:
text, e, enc = read_fstring(b, o + 4)
if fo == 0:
out.append(Field(lang, i, text, base + o + 4, e - (o + 4), enc))
o = e
elif t == 0x04: # 4-byte value (the line's time)
o += 8
else:
raise SubtitleError("unexpected property type %#x in a subtitle line" % t)
if o > end:
raise SubtitleError("subtitle line overruns its array")
if o != end:
raise SubtitleError("subtitle array size mismatch")
return out
def parse_dialogue(b, base, names_idx):
"""Fields of one RDialogueEvent export (bytes `b` starting at uncompressed offset `base`)."""
ls, sub = names_idx
i = b.find(struct.pack("<II", ls, 0))
if i < 4 or struct.unpack_from("<H", b, i - 4)[0] != T_STRUCT:
return []
size = struct.unpack_from("<i", b, i + 8)[0]
o, end = i + 16, i + 16 + size
count = struct.unpack_from("<I", b, o)[0]
o += 4
fields = []
for _ in range(count):
lang = None
while True:
t, fo = struct.unpack_from("<HH", b, o)
if t == 0:
o += 2
break
if t == T_STRING:
lang, o, _ = read_fstring(b, o + 4)
elif t == T_STRUCT:
nm, _nn, ssize, _ai = struct.unpack_from("<iiii", b, o + 4)
c0 = o + 20
if nm == sub:
fields += _parse_lines(b, c0, c0 + ssize, base, lang)
o = c0 + ssize
else:
raise SubtitleError("unexpected property type %#x in a localized subtitle" % t)
if o != end:
raise SubtitleError("localized subtitle size mismatch")
return fields
def _speech_ref(b, exports, imports):
"""Name of the object referenced by the dialogue's first object property (its speech event)."""
m = re.search(rb"\x10\x00\x2c\x00", b)
if not m:
return ""
ref = struct.unpack_from("<i", b, m.start() + 4)[0]
if ref > 0 and ref <= len(exports):
return exports[ref - 1].name
if ref < 0 and -ref <= len(imports):
return imports[-ref - 1][1]
return ""
class Collector:
"""`visit` hook for banks.scan: collects the dialogue objects of every package."""
def __init__(self):
self.dialogues = defaultdict(list) # speech name -> [(file, Dialogue)]
self.issues = []
def __call__(self, rel, pkg, start, data):
try:
for d in scan_package(pkg, data, start):
self.dialogues[d.speech].append((rel, d))
except (SubtitleError, struct.error, ValueError, IndexError) as e:
self.issues.append(("subtitles", "%s: %s" % (rel, e)))
# --- catalog files: SUB_<LANG>.csv (texts) + SUB_locations.csv.gz (every copy) ------------------
LOCATIONS = "SUB_locations.csv.gz"
TEXT_FIELDS = ["speech", "line", "text", "room", "enc", "sound"]
SubLine = namedtuple("SubLine", "lang speech line text room enc sound") # sound: linked WAD sound key
SubLoc = namedtuple("SubLoc", "lang speech line file pos size enc")
def key(lang, speech, line):
return "SUB:%s:%s:%d" % (lang, speech, line)
def rows(collector, links):
"""({lang: [SubLine]}, [SubLoc]) from a Collector; links: {speech: {lang: sound key}}."""
texts = defaultdict(list)
locs = []
for speech in sorted(collector.dialogues):
copies = collector.dialogues[speech]
rel0, d0 = copies[0]
for f in d0.fields:
ln = links.get(speech, {})
snd = ln.get(f.lang) or ln.get("INT", "")
texts[f.lang].append(SubLine(f.lang, speech, f.index, f.text, room(f), f.enc, snd))
for rel, d in copies:
for f in d.fields:
locs.append(SubLoc(f.lang, speech, f.index, rel, f.pos, f.size, f.enc))
return dict(texts), locs
def write_texts(path, lines):
with open(path + ".tmp", "w", encoding="utf-8", newline="") as fh:
w = csv.writer(fh, lineterminator="\n")
w.writerow(TEXT_FIELDS)
for s in lines:
w.writerow([s.speech, s.line, s.text, s.room, s.enc, s.sound])
os.replace(path + ".tmp", path)
def read_texts(path, lang):
with open(path, encoding="utf-8", newline="") as fh:
return [SubLine(lang, r["speech"], int(r["line"]), r["text"], int(r["room"]), r["enc"], r["sound"])
for r in csv.DictReader(fh)]
def write_locations(path, locs):
raw = io.StringIO()
w = csv.writer(raw, lineterminator="\n")
w.writerow(["lang", "speech", "line", "file", "pos", "size", "enc"])
for l in sorted(locs, key=lambda l: (l.lang, l.speech, l.line, l.file, l.pos)):
w.writerow([l.lang, l.speech, l.line, l.file, "%X" % l.pos, l.size, l.enc])
with open(path + ".tmp", "wb") as fh:
with gzip.GzipFile(fileobj=fh, mode="wb", mtime=0) as gz:
gz.write(raw.getvalue().encode("utf-8"))
os.replace(path + ".tmp", path)
def read_locations(path, lang=None):
"""{SUB key: [SubLoc]} (only `lang` if given)."""
out = defaultdict(list)
with gzip.open(path, "rt", encoding="utf-8", newline="") as fh:
for r in csv.DictReader(fh):
if lang and r["lang"] != lang:
continue
l = SubLoc(r["lang"], r["speech"], int(r["line"]), r["file"], int(r["pos"], 16), int(r["size"]), r["enc"])
out[key(l.lang, l.speech, l.line)].append(l)
return dict(out)
def scan_package(pkg, data=None, start=None, tables=None):
"""[Dialogue] of one package (empty if it has none)."""
if data is None:
start, data = pkg.read_all()
names, imports, exports = tables or pkg.tables(data, start)
cls = {-(i + 1) for i, (c, o, _) in enumerate(imports) if c == "Class" and o == CLASS}
if not cls or "LocalizedSubtitles" not in names or "Subtitles" not in names:
return []
idx = (names.index("LocalizedSubtitles"), names.index("Subtitles"))
out = []
for e in exports:
if e.cls in cls and e.size > 0:
b = data[e.offset - start:e.offset - start + e.size]
fields = parse_dialogue(b, e.offset, idx)
if fields:
out.append(Dialogue(_speech_ref(b, exports, imports), e.name, fields))
return out

222
ackit/texts.py Normal file
View File

@@ -0,0 +1,222 @@
"""Subtitles and menu text: reading, states, apply and revert (shared by the CLI and the editor).
Keys (also used in mod packages):
SUB:<LANG>:<speech event>:<line> a subtitle string (docs/formats/subtitles.md)
MENU:<LANG>:<File>:<Section>:<Key>[#n] a menu/UI text (docs/formats/text.md)
Records live in the originals store (data["texts"]): {key: {"orig", "applied", "files", ...}}. The files
holding the texts are derived files of the Workspace: each is rebuilt from its original copy plus every edit
applied now (sounds and texts), so reverting the last edit restores it byte-for-byte.
"""
import os
import time
from . import loctext, subtitles
from .catalog import load_subtitles
from .game import GameError
ORIGINAL, MODIFIED, LOST, CHANGED, FOREIGN = "original", "modified", "lost", "changed", "foreign"
class TextConflictError(GameError):
def __init__(self, keys):
self.keys = keys
super().__init__("These texts were changed outside this tool (Steam update or 'verify', or another mod "
"tool): %s.\nRevert them or confirm to overwrite." % ", ".join(keys[:5])
+ (" ..." if len(keys) > 5 else ""))
class Texts:
def __init__(self, ws):
self.ws = ws
self._subs = {} # lang -> ([SubLine], {key: [SubLoc]})
self._menu = {} # lang -> [Entry]
self.loc_root = _find_dir(ws.game.root, "BmGame", "Localization")
# --- languages and lists ---------------------------------------------------------------------
def menu_languages(self):
return loctext.languages(self.loc_root) if self.loc_root else []
def subtitle_languages(self):
from .catalog import _shipped_index
langs = sorted(_shipped_index().get("SUB", {}).get("lines", {}))
return langs or self.menu_languages()
def subtitles(self, lang, progress=None):
"""[SubLine] of a language (original texts from the catalog; use `current` for the game's text)."""
if lang not in self._subs:
self._subs[lang] = load_subtitles(self.ws.game, lang, progress)
return self._subs[lang][0]
def subtitle_locations(self, key):
lang = key.split(":")[1]
self.subtitles(lang)
return self._subs[lang][1].get(key, [])
def for_sound(self, sound):
"""Subtitle lines of a speech sound in its own language (or the INT line for other languages)."""
lang = sound.archive if sound.archive in self.subtitle_languages() else None
if not lang:
return []
return [s for s in self.subtitles(lang) if s.sound == sound.key]
def menu(self, lang):
"""[loctext.Entry] of the game's current files (the values the game shows now)."""
if lang not in self._menu:
self._menu[lang] = loctext.load(self.loc_root, lang) if self.loc_root else []
return self._menu[lang]
def is_menu_file(self, path):
return bool(self.loc_root) and os.path.normcase(os.path.abspath(path)).startswith(
os.path.normcase(os.path.abspath(self.loc_root)) + os.sep)
# --- values and states ----------------------------------------------------------------------
def records(self):
return self.ws.store.data.setdefault("texts", {})
def modified(self):
return [k for k, r in self.records().items() if r.get("applied") is not None]
def _menu_entry(self, key):
lang = key.split(":")[1]
raw = key[len("MENU:"):]
for e in loctext.load(self.loc_root, lang):
if e.key == raw:
return e
raise GameError("no menu text %s" % raw)
def files(self, key):
"""Absolute paths of the files holding a text."""
if key.startswith("SUB:"):
return sorted({self.ws.bank_path(l.file) for l in self.subtitle_locations(key)})
e = self._menu_entry(key)
return [loctext.files(self.loc_root, e.lang)[e.file]]
def current(self, key):
"""The text in the game files now."""
if key.startswith("SUB:"):
locs = self.subtitle_locations(key)
if not locs:
raise GameError("no subtitle %s" % key)
from .banks import open_file
l = locs[0]
b = open_file(self.ws.bank_path(l.file)).read(l.pos, l.size)
return subtitles.read_fstring(b, 0)[0]
return self._menu_entry(key).value
def catalog_text(self, key):
"""Original text according to the shipped catalog (subtitles only), or None."""
if not key.startswith("SUB:"):
return None
_, lang, rest = key.split(":", 2)
speech, line = rest.rsplit(":", 1)
for s in self.subtitles(lang):
if s.speech == speech and s.line == int(line):
return s.text
return None
def original(self, key):
r = self.records().get(key)
return r["orig"] if r else self.current(key)
def state(self, key, current=None):
cur = self.current(key) if current is None else current
r = self.records().get(key)
if r:
if r.get("applied") is not None and cur == r["applied"]:
return MODIFIED
if cur == r["orig"]:
return LOST if r.get("applied") is not None else ORIGINAL
return CHANGED
cat = self.catalog_text(key)
if cat is not None and cur != cat:
return FOREIGN
return ORIGINAL
def limit(self, key):
"""Maximum characters for a subtitle (None = no limit: menu text)."""
if not key.startswith("SUB:"):
return None
return min((subtitles.room(subtitles.Field(l.lang, l.line, "", l.pos, l.size, l.enc))
for l in self.subtitle_locations(key)), default=0)
def check(self, key, text):
"""Raise SubtitleError / TextError if `text` cannot be written for `key`."""
if key.startswith("SUB:"):
for l in self.subtitle_locations(key):
subtitles.encode_fstring(text, l.size, l.enc)
else:
e = self._menu_entry(key)
path = loctext.files(self.loc_root, e.lang)[e.file]
loctext.write(open(path, "rb").read(), e.lang, e.file, {e.key: text})
# --- writing --------------------------------------------------------------------------------
def apply(self, changes, force=False, progress=None, meta=None):
"""changes: {key: new text}. Every file holding one of them is rebuilt once."""
for k, v in changes.items():
self.check(k, v)
conflicts = [k for k in changes if self.state(k) in (CHANGED, FOREIGN)]
paths = {p for k in changes for p in self.files(k)}
changed = set(self.ws.changed_files(paths))
conflicts += [k for k in changes if k not in conflicts and set(self.files(k)) & changed]
if conflicts and not force:
raise TextConflictError(conflicts)
for p in paths:
self.ws._keep_original(p)
recs = self.records()
now = time.time()
for k, v in changes.items():
if k not in recs:
recs[k] = {"orig": self.current(k), "files": [self.ws.game.rel(p) for p in self.files(k)],
"first_modified": now, "applied": None}
recs[k].update(applied=v, applied_at=now, meta=meta or {})
self.ws.store.save()
self._sync(paths, progress)
def revert(self, keys, progress=None):
recs = self.records()
keys = [k for k in keys if k in recs]
paths = {p for k in keys for p in self.files(k)}
for k in keys:
recs.pop(k)
self.ws.store.save()
self._sync(paths, progress)
return keys
def revert_all(self, progress=None):
return self.revert(list(self.records()), progress)
def _sync(self, paths, progress):
self._menu.clear()
self.ws.sync_files(paths, progress)
# --- derived-file hooks (called by Workspace.sync_files) ------------------------------------
def subtitle_edits(self, rel):
out = []
for k, r in self.records().items():
if r.get("applied") is None or not k.startswith("SUB:"):
continue
for l in self.subtitle_locations(k):
if l.file == rel:
out.append((l.pos, subtitles.encode_fstring(r["applied"], l.size, l.enc)))
return out
def menu_content(self, path, orig_raw):
fname = os.path.basename(path)
lang = os.path.basename(os.path.dirname(path)).upper()
values = {}
for k, r in self.records().items():
if r.get("applied") is None or not k.startswith("MENU:"):
continue
_, lg, f, _rest = k.split(":", 3)
if lg == lang and f == fname:
values[k[len("MENU:"):]] = r["applied"]
return loctext.write(orig_raw, lang, fname, values) if values else orig_raw
def _find_dir(root, *parts):
d = root
for p in parts:
if not d or not os.path.isdir(d):
return None
d = next((os.path.join(d, x) for x in os.listdir(d) if x.lower() == p.lower()), None)
return d if d and os.path.isdir(d) else None

View File

@@ -8,7 +8,7 @@ Patching rewrites only the blocks that contain changed bytes (stored as one LZO
lzo.compress_stored) and moves the chunks after them; every other byte of the file stays identical.
"""
import struct
from collections import OrderedDict
from collections import OrderedDict, namedtuple
from . import lzo
@@ -20,6 +20,9 @@ class PackageError(ValueError):
pass
Export = namedtuple("Export", "index cls outer name size offset entry") # entry: table position (uncompressed)
class Block:
__slots__ = ("chunk", "uoff", "usize", "coff", "csize")
@@ -52,6 +55,7 @@ class Package:
f12 = struct.unpack_from("<12i", b, o)
self.name_count, self.name_offset = f12[1], f12[2]
self.export_count, self.export_offset = f12[3], f12[4]
self.import_count, self.import_offset = f12[5], f12[6]
o += 48
self.guid = b[o:o + 16]
o += 16
@@ -106,6 +110,42 @@ class Package:
return self.file_size
return self.chunks[-1][0] + self.chunks[-1][1]
# tables (need the uncompressed data: `read_all`) ---------------------------------------------
def tables(self, data=None, start=None):
"""(names, imports, exports) of the package. imports: [(class name, object name, outer)],
exports: [Export]. Layouts: docs/formats/upk.md (licensee 101 adds one u32 to each export)."""
if data is None:
start, data = self.read_all()
buf = memoryview(data)
names = []
o = self.name_offset - start
for _ in range(self.name_count):
n = struct.unpack_from("<i", buf, o)[0]
o += 4
if n >= 0:
names.append(bytes(buf[o:o + n - 1]).decode("latin-1"))
o += n
else:
names.append(bytes(buf[o:o - 2 * n - 2]).decode("utf-16-le", "replace"))
o += -2 * n
o += 8 # name flags
imports = []
o = self.import_offset - start
for _ in range(self.import_count):
_pkg, _pn, cls, _cn, outer, nm, nn = struct.unpack_from("<7i", buf, o)
imports.append((names[cls], names[nm], outer))
o += 28
exports = []
o = self.export_offset - start
for i in range(self.export_count):
cls, sup, outer, nm, nn, arch, _x = struct.unpack_from("<7i", buf, o)
size, off = struct.unpack_from("<ii", buf, o + 36)
ngen = struct.unpack_from("<i", buf, o + 48)[0]
exports.append(Export(i + 1, cls, outer, names[nm] + ("_%d" % (nn - 1) if nn else ""), size, off,
o + start))
o += 72 + 4 * ngen
return names, imports, exports
def fingerprint(self):
"""Cheap identity of the package contents: guid + chunk table (changes with any re-cook/patch)."""
import hashlib

View File

@@ -16,7 +16,7 @@ import hashlib
import os
import random
from .. import banks, classify
from .. import banks, classify, subtitles
from ..catalog import (Catalog, available_archives, bank_sounds, hash_names, head_owners, is_shipped, load_characters,
scan_archive)
from ..inject import read_target
@@ -102,7 +102,9 @@ def check_banks(rep, game, rows, locations, characters, quick=False, sample=40,
pk = [f for f in files if f.lower().endswith(".upk")]
rnd = random.Random(1)
files = [f for f in files if f.startswith("SFX/")] + sorted(rnd.sample(pk, min(sample, len(pk))))
sc = banks.scan(game.cooked, progress, files=files)
col = subtitles.Collector()
sc = banks.scan(game.cooked, progress, files=files, visit=col)
check_subtitles(rep, game, col, set(files), quick)
owners = head_owners(game, sc)
index = collections.defaultdict(set)
for m, ls in locations.items():
@@ -135,6 +137,44 @@ def check_banks(rep, game, rows, locations, characters, quick=False, sample=40,
rep.add(banks.ARCHIVE, "copies", "%08X" % m, "prefetch head without a speech line")
def check_subtitles(rep, game, col, files, quick):
"""Subtitle copies found in the scanned packages vs SUB_locations + the SUB_<LANG>.csv texts."""
from ..catalog import _shipped_index, catalog_dir
loc_path = os.path.join(catalog_dir(), subtitles.LOCATIONS)
if not os.path.exists(loc_path) or not is_shipped(game, banks.ARCHIVE):
return
index = collections.defaultdict(set)
for k, ls in subtitles.read_locations(loc_path).items():
for l in ls:
if l.file in files:
index[k].add((l.file, l.pos, l.size))
found, texts = collections.defaultdict(set), {}
for speech, copies in col.dialogues.items():
for rel, d in copies:
for f in d.fields:
k = subtitles.key(f.lang, speech, f.index)
found[k].add((rel, f.pos, f.size))
texts[k] = f.text
for k in sorted(set(found) | set(index)):
if found.get(k, set()) != index.get(k, set()):
rep.add("SUB", "copies", k, "index %d, game %d" % (len(index.get(k, ())), len(found.get(k, ()))))
langs = sorted(_shipped_index().get("SUB", {}).get("lines", {}))
for lang in langs:
p = os.path.join(catalog_dir(), "SUB_%s.csv" % lang)
rows = subtitles.read_texts(p, lang)
rep.checked["SUB_" + lang] += len(rows)
seen = set()
for s in rows:
k = subtitles.key(lang, s.speech, s.line)
if k in seen:
rep.add("SUB", "duplicate", k)
seen.add(k)
if k in texts and texts[k] != s.text:
rep.add("SUB", "text", k, "catalog %r, game %r" % (s.text[:40], texts[k][:40]))
if s.sound and not s.sound.startswith(lang + ":") and not s.sound.startswith("INT:"):
rep.add("SUB", "language", k, "linked to %s" % s.sound)
def run(game, archives=None, quick=False, log=print, progress=None):
"""Check the given archives (default: all). Returns a Report."""
rep = Report()

View File

@@ -22,6 +22,7 @@ from .game import Game, GameError
from .inject import inject, read_target, parse_target, pack_bnk_path, SizeLimitError
from .originals import OriginalsStore, sha256, sha1_16
from .pack import Pack
from .texts import Texts
from .paths import user_dir
from .wad import Wad
@@ -51,6 +52,7 @@ class Workspace:
self.store = OriginalsStore(store_root or default_store_root(self.game), self.game.root)
self.catalog = Catalog(self.game, archives, progress)
self.full_backup = full_backup
self.texts = Texts(self)
# reading -----------------------------------------------------------------------------------
def sound(self, name_or_key):
@@ -153,9 +155,9 @@ class Workspace:
"""items: [(sound, new_bytes, meta, is_revert)] all in one archive."""
if archive == banks.ARCHIVE:
return self._write_banks(items, progress)
heads = sorted({l.file for s, _, _, _ in items for l in self.locations(s)})
for rel in heads:
self._keep_original_file(rel)
heads = sorted({self.bank_path(l.file) for s, _, _, _ in items for l in self.locations(s)})
for path in heads:
self._keep_original(path)
wad = self.game.wad(archive)
self.game.check_writable(wad)
packs = {parse_target(s.target)[0] for s, _, _, _ in items if s.is_pack_media}
@@ -181,43 +183,44 @@ class Workspace:
else:
self.store.record_applied(s.key, data, meta)
self.store.save()
self._sync_bank_files(heads, progress)
self.sync_files(heads, progress)
# bank files (in-memory sounds, prefetch heads) ----------------------------------------------
def _bank_files(self):
return self.store.data.setdefault("bank_files", {})
# derived files: bank files and packages (in-memory sounds, prefetch heads, subtitles) and the
# localization files (menu text). Each is rebuilt from its original copy + every edit applied now.
def _derived(self):
return self.store.data.setdefault("derived", {})
def _fingerprint(self, path):
"""Content hash of a bank file (whole file: any change by another tool is detected)."""
"""Content hash of a file (whole file: any change by another tool is detected)."""
h = hashlib.sha1()
with open(path, "rb") as f:
for c in iter(lambda: f.read(1 << 22), b""):
h.update(c)
return h.hexdigest()
def _keep_original_file(self, rel):
"""Before the first change to a bank file: verified full copy + its fingerprint."""
rec = self._bank_files().get(rel)
def _keep_original(self, path):
"""Before the first change to a derived file: verified full copy + its fingerprint."""
rel = self.game.rel(path)
rec = self._derived().get(rel)
if rec:
return rec
path = self.bank_path(rel)
self.game.check_writable(path)
self.store.full_backup(path, self.game.rel(path))
rec = self._bank_files()[rel] = {"orig_fp": self._fingerprint(path), "written_fp": None}
self.store.full_backup(path, rel)
rec = self._derived()[rel] = {"orig_fp": self._fingerprint(path), "written_fp": None}
self.store.save()
return rec
def changed_files(self, sounds):
"""Bank files of `sounds` changed since we last wrote them (Steam update/verify, other tool)."""
def changed_files(self, paths):
"""Derived files changed since we last wrote them (Steam update/verify, another tool)."""
out = []
for rel in sorted({l.file for s in sounds for l in self.locations(s)}):
rec = self._bank_files().get(rel)
if rec and self._fingerprint(self.bank_path(rel)) not in (rec["orig_fp"], rec["written_fp"]):
out.append(rel)
for path in sorted(set(paths)):
rec = self._derived().get(self.game.rel(path))
if rec and self._fingerprint(path) not in (rec["orig_fp"], rec["written_fp"]):
out.append(path)
return out
def _edits(self, rel):
"""[(offset, bytes)] that the replacements applied now need in bank file `rel`."""
def _audio_edits(self, rel):
"""[(offset, bytes)] that the sound replacements applied now need in bank file `rel` (cooked-relative)."""
out = []
for key, e in self.store.entries.items():
if not e.get("applied"):
@@ -235,24 +238,31 @@ class Workspace:
out.append((l.offset, payload))
return out
def _sync_bank_files(self, rels, progress=None):
"""Rebuild each bank file from its original copy and the replacements applied now."""
for rel in rels:
path = self.bank_path(rel)
rec = self._keep_original_file(rel)
def _build(self, path, orig):
"""Content of derived file `path` from its original copy `orig`."""
if self.texts.is_menu_file(path):
return self.texts.menu_content(path, open(orig, "rb").read())
rel = os.path.relpath(path, self.game.cooked).replace(os.sep, "/")
edits = self._audio_edits(rel) + self.texts.subtitle_edits(rel)
return banks.open_file(orig).patched(edits) if edits else open(orig, "rb").read()
def sync_files(self, paths, progress=None):
"""Rebuild each derived file from its original copy and the edits applied now (atomic writes)."""
for path in sorted(set(paths)):
rec = self._keep_original(path)
orig = self.store.full_copy(self.game.rel(path))
if orig is None:
raise GameError("the original copy of %s is missing from the store" % rel)
raise GameError("the original copy of %s is missing from the store" % self.game.rel(path))
if progress:
progress("writing %s" % rel)
edits = self._edits(rel)
data = banks.open_file(orig).patched(edits) if edits else open(orig, "rb").read()
progress("writing %s" % os.path.basename(path))
data = self._build(path, orig)
self.game.check_writable(path)
tmp = path + ".ackit-tmp"
with open(tmp, "wb") as f:
f.write(data)
os.replace(tmp, path)
rec["written_fp"] = self._fingerprint(path) if edits else None
same = data == open(orig, "rb").read()
rec["written_fp"] = None if same else self._fingerprint(path)
self.store.save()
def _write_banks(self, items, progress=None):
@@ -262,9 +272,9 @@ class Workspace:
raise GameError("no copy of %s was found in the game's banks" % s.name)
if not is_revert and len(data) > min(l.size for l in locs):
raise SizeLimitError(s.target, min(l.size for l in locs), len(data))
files = sorted({l.file for s, _, _, _ in items for l in self.locations(s)})
for rel in files:
self._keep_original_file(rel)
files = sorted({self.bank_path(l.file) for s, _, _, _ in items for l in self.locations(s)})
for path in files:
self._keep_original(path)
for s, data, meta, is_revert in items:
if is_revert:
self.store.forget(s.key)
@@ -273,7 +283,7 @@ class Workspace:
self.current_bytes(s))
self.store.record_applied(s.key, data, meta)
self.store.save() # the store says what the files must contain before they change
self._sync_bank_files(files, progress)
self.sync_files(files, progress)
def apply(self, changes, force=False, progress=None):
"""changes: [(sound, wem_bytes, meta dict)]. One atomic rewrite per WAD / bank file."""
@@ -281,9 +291,9 @@ class Workspace:
for s, _, _ in changes:
if self.state(s) in (CHANGED, FOREIGN):
conflicts.append(s)
changed = set(self.changed_files([s for s, _, _ in changes]))
changed = set(self.changed_files([self.bank_path(l.file) for s, _, _ in changes for l in self.locations(s)]))
conflicts += [s for s, _, _ in changes
if s not in conflicts and {l.file for l in self.locations(s)} & changed]
if s not in conflicts and {self.bank_path(l.file) for l in self.locations(s)} & changed]
if conflicts and not force:
raise ConflictError(conflicts)
by_arch = defaultdict(list)
@@ -328,8 +338,9 @@ class Workspace:
shutil.copy2(src, tmp)
os.replace(tmp, dst)
out.append(rel)
for rec in self._bank_files().values():
for rec in self._derived().values():
rec["written_fp"] = None
self.store.data["texts"] = {}
self.store.data["entries"] = {}
self.store.save()
self.store.gc()

File diff suppressed because it is too large Load Diff

View File

@@ -4,6 +4,8 @@
|------|---------|
| `<ARCHIVE>.csv` | every sound of one audio archive: `ITA`, `INT`, `FRA`, `DEU`, `ESN` (speech WADs, one per language), `SFX` (music, ambience and effects streamed from the SFX WAD) and `BANK` (in-memory sounds stored inside soundbanks: footsteps, foley, combat, weapons, UI, efforts; `docs/formats/banks.md`) |
| `BANK_locations.csv.gz` | every stored copy of the `BANK` sounds, and the prefetch heads of speech lines (below) |
| `SUB_<LANG>.csv` | subtitles of the speech lines, one file per language (11 languages; `docs/formats/subtitles.md`) |
| `SUB_locations.csv.gz` | every stored copy of each subtitle string (dialogue objects in the packages) |
| `characters.csv` | `name,character`: verified speaker per sound name (language independent) |
| `index.json` | per archive: fingerprint of the game files the CSV was built from (`ackit/game.py: wad_fingerprint`, `ackit/banks.py: game_fingerprint`) and counts |
| `source/<lang>/*.txt` | **the curated original lists** (one file per character, one wemscan-style file name per line, e.g. `Speech_TAPE_JOKER_1_4.wem`). `Files.txt` = all files, `Stats.txt` = counts, `Checkpoint.txt` = progress marker. Never edited by tools. |
@@ -40,6 +42,19 @@ All CSVs are UTF-8 with a header row. They contain only names and metadata (no g
A replacement is written into every `m` copy of a BANK sound; replacing a speech line also rewrites its `p`
heads (`docs/formats/banks.md`).
## `SUB_<LANG>.csv` columns
| column | meaning |
|--------|---------|
| `speech` | the speech event of the line (e.g. `Speech_CHURCH_B1_Ch2_POST_Explosion_BM_Cash_04`) |
| `line` | 0-based line of a multi-line subtitle |
| `text` | original subtitle text |
| `room` | maximum length of a replacement in characters (it is written in place) |
| `enc` | `a` = cp1252 string, `w` = UTF-16 string |
| `sound` | key of the linked speech sound (this language's WAD, or `INT:` for languages without speech WADs); empty when no event reaches a sound |
`SUB_locations.csv.gz`: `lang, speech, line, file, pos` (hex, uncompressed offset of the FString), `size`, `enc`.
Menu text has no catalog: the localization files are read directly (`docs/formats/text.md`).
The editor uses these files directly when the installed game matches `index.json`; for another game build it
scans the game itself and applies the same rules (no checksums then; the bank scan takes a few minutes).
`ackit verify --catalog` cross-checks the catalog in use against the game files (`--quick` samples packages).

9639
data/catalog/SUB_DEU.csv Normal file

File diff suppressed because it is too large Load Diff

9477
data/catalog/SUB_ESM.csv Normal file

File diff suppressed because it is too large Load Diff

9469
data/catalog/SUB_ESN.csv Normal file

File diff suppressed because it is too large Load Diff

9541
data/catalog/SUB_FRA.csv Normal file

File diff suppressed because it is too large Load Diff

9380
data/catalog/SUB_INT.csv Normal file

File diff suppressed because it is too large Load Diff

9353
data/catalog/SUB_ITA.csv Normal file

File diff suppressed because it is too large Load Diff

9017
data/catalog/SUB_JPN.csv Normal file

File diff suppressed because it is too large Load Diff

9230
data/catalog/SUB_KOR.csv Normal file

File diff suppressed because it is too large Load Diff

9353
data/catalog/SUB_POL.csv Normal file

File diff suppressed because it is too large Load Diff

9451
data/catalog/SUB_POR.csv Normal file

File diff suppressed because it is too large Load Diff

9470
data/catalog/SUB_RUS.csv Normal file

File diff suppressed because it is too large Load Diff

Binary file not shown.

View File

@@ -73,7 +73,7 @@
},
"copies": 93910,
"files": 1499,
"named_by_event": 6416,
"named_by_event": 6456,
"prefetch_heads": {
"lines": 396,
"copies": 2101,
@@ -81,5 +81,37 @@
},
"issues": []
},
"SUB": {
"fingerprint": "3f9766af425b8eef",
"lines": {
"DEU": 9638,
"ESM": 9476,
"ESN": 9468,
"FRA": 9540,
"INT": 9379,
"ITA": 9352,
"JPN": 9016,
"KOR": 9229,
"POL": 9352,
"POR": 9450,
"RUS": 9469
},
"speech_lines": 9016,
"linked": {
"DEU": 8884,
"ESM": 8712,
"ESN": 8741,
"FRA": 8790,
"INT": 8637,
"ITA": 8623,
"JPN": 8318,
"KOR": 8510,
"POL": 8613,
"POR": 8700,
"RUS": 8724
},
"copies": 168122,
"issues": []
},
"_info": "unmodified game files"
}

View File

@@ -8,6 +8,8 @@
| [building.md](building.md) | building the editor and its installer (Windows, Linux) |
| [formats/wad.md](formats/wad.md) | WWAD archives, grouped entries, BKPK stream packs |
| [formats/banks.md](formats/banks.md) | soundbanks and the in-memory sounds not covered yet (footsteps, foley, effects) |
| [formats/subtitles.md](formats/subtitles.md) | subtitles: dialogue objects in the packages, link to the speech sounds |
| [formats/text.md](formats/text.md) | menu and UI text: the localization files |
| [formats/upk.md](formats/upk.md) | UE3 packages: summary, LZO chunks, how ackit reads and patches them |
| [formats/wem.md](formats/wem.md) | WEM layout the game accepts, setup stripping, hash and decode-alloc formulas |
| [reverse-engineering/decoder.md](reverse-engineering/decoder.md) | BatmanAC.exe decoder functions and addresses |

View File

@@ -79,6 +79,19 @@ tables): high risk for little gain, not done.
packages' name tables (FNV-1 of the name = event ID), following event -> action -> containers -> sounds
(6,416 sounds); the rest are numbered per bank. Categories use the bank name.
## 4c. Subtitles and menu text
- **Subtitles** are written in place into every copy of the dialogue object, NUL-padded inside the original
string (the game stops at the first NUL), so nothing in the package moves; the limit is the original
string's length ("room"). Moving the object to the end of the package (new export offset/size and an extra
chunk) would lift the limit, but no dialogue is loaded on the title screen, so the result could not be
checked automatically in the game: not done.
- **Links** subtitle -> speech sound come from the soundbanks (event -> ... -> per-language media -> WAD), not
from names: the `_NN` suffixes do not follow the WEM order inside WAD entries.
- **Menu text** files are plain UTF-16 text: no size limit; only the edited values change.
- Both use the originals store like sounds: texts are recorded as strings (`texts` in store.json) and every
file is rebuilt from its original copy + the applied edits (sound and text edits combine in one package).
- Keys (`SUB:<LANG>:<speech>:<line>`, `MENU:<LANG>:<File>:<Section>:<Key>`) are stable and used in mod packages.
## 5. Catalog
- Shipped CSVs per archive (names, categories, characters, original size + checksum) make loading instant and
enable change detection; any other build is scanned at runtime (header reads only, cached).

46
docs/formats/subtitles.md Normal file
View File

@@ -0,0 +1,46 @@
# Subtitles (`RDialogueEvent` objects in the packages)
There are no subtitle files: every speech line has a dialogue object (class `RDialogueEvent`, Rocksteady) inside
the UE3 packages (`upk.md`), holding the subtitle text of all 11 languages. ackit reads them with
`ackit/subtitles.py` and writes them through `Workspace.texts` (`ackit/texts.py`).
## Inventory (buildid 3718300)
- 14,743 dialogue objects in the packages for 9,016 speech lines (a line's object is copied into every package
that uses it; all copies are identical).
- 11 languages: INT, FRA, ITA, DEU, ESN, ESM, POL, POR, RUS, JPN, KOR (the speech WADs exist only for INT, FRA,
ITA, DEU, ESN; the other languages play the INT voices with their own subtitles).
- ~9,400 subtitle strings per language: 94 % of the subtitles have one line, the rest up to 5 lines.
- Strings are stored as cp1252 (positive FString length) or UTF-16 (negative length): RUS, POL, JPN, KOR
and some INT/ESN/FRA/DEU strings are UTF-16.
## Object layout (compact tagged properties)
The export data is a list of properties `u16 type, u16 field offset, value`, ended by `u16 0`. Types seen:
`0x10` object reference (i32 export/import index), `0x0D` FString, `0x09` struct/array (name i32+i32 = struct
or property name from the name table, i32 byte size, i32 array index, then `size` bytes), `0x04` 4-byte value,
`0x03` bool, `0x01` enum. Relevant properties:
| field | content |
|-------|---------|
| `0x10` @ `0x2C` | the speech event this line plays, e.g. export `Speech_CHURCH_B1_Ch2_POST_Explosion_BM_Cash_04` |
| `0x0D` @ `0x34` | speaker, e.g. `Batman` |
| `0x09 LocalizedSubtitles` | u32 count, then per language: `0D 00 00 00` FString language code, `09 00 0C 00 Subtitles` struct, `00 00` |
| `Subtitles` | u32 line count, then per line: `0D 00 00 00` FString text, `04 00 0C 00` 4-byte value, `00 00` |
The sizes add up exactly in all 14,743 objects (checked by the parser: `SubtitleError` otherwise).
## Link to the speech sound
The speech event name is hashed (FNV-1) to the Wwise event ID, followed through the soundbanks (event ->
action -> containers -> Sound) to the streamed media, and that media ID is looked up in the per-language
BKPK tables (`banks.md`) to find the WAD entry and variation of each language. This is exact; the `_NN`
suffix of the event name does **not** match the variation order inside the WAD entry (WEMs are stored in
media-ID order). Linked: ~92 % of the strings (e.g. INT 8,637 / 9,379); the rest have no event that reaches a
streamed media (unused or placeholder lines such as `..._TEMP_29`) and stay editable without a sound link.
Event names with a number (`Speech_X_04`) exist only as export names, not in the name table.
## How ackit writes them
- In place, into **every copy** (`data/catalog/SUB_locations.csv.gz`): the new text replaces the FString's
bytes and is padded with NUL characters, keeping the length field. The game stops at the first NUL, so no
size field, export or package layout changes. Limit: the new text must fit the original string's bytes
(cp1252: one byte per character; UTF-16: two) - shown as "room" in the CLI and the editor.
- A cp1252 string can only hold cp1252 characters; UTF-16 strings can hold any character.
- Packages are rebuilt from the store's original copy + all applied sound and text edits (`docs/decisions.md`).

19
docs/formats/text.md Normal file
View File

@@ -0,0 +1,19 @@
# Menu and UI text (`BmGame\Localization\<LANG>\*.<lang>`)
The menus, HUD, character bios, riddles, credits and other UI strings are UE3 localization files, one folder
per language (INT, ITA, FRA, DEU, ESN, ESM, POL, POR, RUS, JPN, KOR), ~37 files and ~7,000 strings each
(`FRONTEND`, `GFxUI`, `Bios`, `Credits`, `PC`, `RIDDLER`, ...). ackit reads and writes them with
`ackit/loctext.py` through `Workspace.texts`.
## Format
- UTF-16LE with a BOM (`FF FE`), CRLF line ends; the first line is empty. (`INT\Version.int` and
`VersionData.int` are plain ASCII build IDs, not UI text: ackit ignores them.)
- `[Section]` lines, `Key=Value` lines, `;` comments. A few values are in double quotes (`Quit="Quit game"`).
- File extension = language code (`FRONTEND.int`, `FRONTEND.ITA`); case varies, lookups are case-insensitive.
- Markers inside values (e.g. `\n`, button tags) are the game's own; ackit keeps them as plain text.
## Keys and writing
- Key: `MENU:<LANG>:<File>:<Section>:<Key>` (`#n` for a repeated key; none in the shipped files).
- Writing replaces only the value part of the edited lines (quotes kept); BOM, encoding, other lines and line
ends stay byte-identical. Values cannot contain line breaks; characters must fit the file's encoding.
- The files are rebuilt from the store's original copy + the applied edits, so reverting restores them exactly.

View File

@@ -89,10 +89,11 @@ def pack(media):
return bytes(hdr) + bank, bytes(payload)
def bank(bank_id, media, name=None, extra_sources=(), prefetch=()):
def bank(bank_id, media, name=None, extra_sources=(), prefetch=(), events=()):
"""A v56 soundbank holding `media` {id: wem} in memory (DIDX/DATA, 16-byte aligned) with one HIRC
Sound object per media; extra_sources: [(media id, bank id, off, size)] for sources of other banks;
media ids in `prefetch` get a prefetch source (their DIDX data is the head of a streamed WEM)."""
media ids in `prefetch` get a prefetch source (their DIDX data is the head of a streamed WEM);
events: [(event name, streamed media id)] -> event -> action -> streamed Sound object."""
data = bytearray()
didx = b""
for m in sorted(media):
@@ -108,10 +109,18 @@ def bank(bank_id, media, name=None, extra_sources=(), prefetch=()):
objs.append((m, bank_id if m not in prefetch else 0x1234, data_pos + off, len(media[m]),
2 if m in prefetch else 0))
objs += [x + (0,) for x in extra_sources]
hirc = struct.pack("<I", len(objs))
from ackit.bnk import fnv1
hirc_objs = []
for i, (m, b, off, size, stream) in enumerate(objs):
body = struct.pack("<7I", 0x100 + i, 0x00040001, stream, m, b, off, size) + bytes(6)
hirc += bytes([2]) + struct.pack("<I", len(body)) + body
hirc_objs.append((2, struct.pack("<7I", 0x100 + i, 0x00040001, stream, m, b, off, size) + bytes(6)))
for i, (ev, m) in enumerate(events):
sid, aid = 0x300 + i, 0x400 + i
hirc_objs.append((2, struct.pack("<7I", sid, 0x00040001, 1, m, 0, 0, 0) + bytes(6)))
hirc_objs.append((3, struct.pack("<III", aid, 0x4011, sid) + bytes(8)))
hirc_objs.append((4, struct.pack("<III", fnv1(ev), 1, aid)))
hirc = struct.pack("<I", len(hirc_objs))
for t, body in hirc_objs:
hirc += bytes([t]) + struct.pack("<I", len(body)) + body
out = head + b"DATA" + struct.pack("<I", len(data)) + bytes(data) + b"HIRC" + struct.pack("<I", len(hirc)) + hirc
if name:
nm = name.encode()
@@ -120,9 +129,10 @@ def bank(bank_id, media, name=None, extra_sources=(), prefetch=()):
return out
def package(path, body, block_size=0x1000, chunk_blocks=3):
def package(path, body, block_size=0x1000, chunk_blocks=3, tables=(0, 0, 0, 0, 0, 0)):
"""A fully LZO-compressed UE3 package (docs/formats/upk.md) whose uncompressed data area is `body`.
Small blocks so that tests cross block and chunk boundaries."""
Small blocks so that tests cross block and chunk boundaries. `tables`: (NameCount, NameOffset,
ExportCount, ExportOffset, ImportCount, ImportOffset) as absolute uncompressed offsets."""
from ackit import lzo
folder = b"None\x00"
nblocks = max(1, _a(len(body), block_size) // block_size)
@@ -130,7 +140,7 @@ def package(path, body, block_size=0x1000, chunk_blocks=3):
def summary(chunks):
s = struct.pack("<IHHi", 0x9E2A83C1, 805, 101, 0) + struct.pack("<i", len(folder)) + folder
s += struct.pack("<12i", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + b"GUID-synthetic!!"
s += struct.pack("<12i", 0, *tables, 0, 0, 0, 0, 0) + b"GUID-synthetic!!"
s += struct.pack("<i3i", 1, 0, 0, 0) + struct.pack("<3I", 7748, 0, 2)
s += struct.pack("<i", len(chunks)) + b"".join(struct.pack("<4i", *c) for c in chunks)
return s + struct.pack("<Ii", 0x12345678, 0)
@@ -157,6 +167,69 @@ def package(path, body, block_size=0x1000, chunk_blocks=3):
return header_u
HEADER_U = None # uncompressed summary size of synthetic packages (fixed; computed on first use)
def _header_u():
global HEADER_U
if HEADER_U is None:
import tempfile
with tempfile.TemporaryDirectory() as d:
HEADER_U = package(os.path.join(d, "x.upk"), b"\0" * 16)
return HEADER_U
def ue_package(path, objects, prefix=b""):
"""A package with name/import/export tables. objects: [(class name, object name, bytes or callable)]:
a callable gets {name: index} and the export index map and returns the object's bytes.
Returns {object name: (absolute offset, size)}."""
classes = sorted({c for c, _, _ in objects})
names = ["None", "Core", "Class", "LocalizedSubtitles", "Subtitles"] + classes + [n for _, n, _ in objects]
idx = {n: i for i, n in enumerate(names)}
exp_index = {n: i + 1 for i, (_, n, _) in enumerate(objects)}
blobs = [o(idx, exp_index) if callable(o) else o for _, _, o in objects]
nt = b"".join(struct.pack("<i", len(n) + 1) + n.encode() + b"\0" + bytes(8) for n in names)
it = b"".join(struct.pack("<7i", idx["Core"], 0, idx["Class"], 0, 0, idx[c], 0) for c in classes)
base = _header_u() + len(prefix)
name_off, imp_off = base, base + len(nt)
exp_off = imp_off + len(it)
data_off = exp_off + 72 * len(objects)
et, pos, offs = b"", data_off, {}
for (c, n, _), b in zip(objects, blobs):
et += struct.pack("<7i", -(classes.index(c) + 1), 0, 0, idx[n], 0, 0, 0) + bytes(8)
et += struct.pack("<ii", len(b), pos) + bytes(4 + 4 + 16 + 4)
offs[n] = (pos, len(b))
pos += len(b)
body = prefix + nt + it + et + b"".join(blobs) + bytes(64)
package(path, body, tables=(len(names), name_off, len(objects), exp_off, len(classes), imp_off))
return offs
def _fstr(text, wide=False):
if wide:
return struct.pack("<i", -(len(text) + 1)) + text.encode("utf-16-le") + b"\0\0"
return struct.pack("<i", len(text) + 1) + text.encode("cp1252") + b"\0"
def dialogue(speech_object, texts, wide=()):
"""An RDialogueEvent export (compact tagged properties, docs/formats/subtitles.md) referencing the
export `speech_object`; texts: {lang: [line, ...]}; languages in `wide` are stored as UTF-16."""
def build(idx, exp_index):
content = struct.pack("<I", len(texts))
for lang, lines in texts.items():
sub = struct.pack("<I", len(lines))
for ln in lines:
sub += struct.pack("<HH", 0x0D, 0) + _fstr(ln, lang in wide) + struct.pack("<HHf", 4, 0x0C, 1.5)
sub += struct.pack("<H", 0)
content += struct.pack("<HH", 0x0D, 0) + _fstr(lang)
content += struct.pack("<HHiiii", 9, 0x0C, idx["Subtitles"], 0, len(sub), 0) + sub + struct.pack("<H", 0)
b = bytes(4) + struct.pack("<HHi", 0x10, 0x2C, exp_index[speech_object])
b += struct.pack("<HH", 0x0D, 0x34) + _fstr("Batman")
b += struct.pack("<HHiiii", 9, 0x9C, idx["LocalizedSubtitles"], 0, len(content), 0) + content
return b + struct.pack("<H", 0)
return build
def make_game(root):
"""Create the folder; returns the dict of original WEMs by target."""
cooked = os.path.join(root, "BmGame", "CookedPCConsole")
@@ -175,6 +248,7 @@ def make_game(root):
originals["%08X" % SINGLE_ENTRY] = single
originals.update({"%08X/%08X" % (PACK_ID, m): w for m, w in music.items()})
originals.update(make_banks(cooked, v))
make_localization(root)
return originals
@@ -213,5 +287,35 @@ def make_banks(cooked, speech_wems):
synth_b = filler[:1000] + bank(fnv1(names[3]), gui) + filler + bank(fnv1(names[2]), foley) + filler
package(os.path.join(cooked, "Synth_A.upk"), synth_a)
package(os.path.join(cooked, "Synth_B.upk"), synth_b)
# dialogue (subtitles) in two packages; the first one also holds a bank with an in-memory sound and the
# event that links the dialogue's speech event to INT variation 1 of SPEECH_ENTRY
dlg = dialogue(SPEECH_EVENT, SUBTITLES, wide=("RUS",))
dlg_wem = wem(0.3, 1400)
ev_bank = bank(fnv1("NPC_SynthDialogue"), {DLG_MEDIA: dlg_wem}, name="NPC_SynthDialogue",
events=[(SPEECH_EVENT, LINE_MEDIA[1])])
objs = [("AkEvent", SPEECH_EVENT, bytes(8)), ("RDialogueEvent", "Radio_02", dlg)]
ue_package(os.path.join(cooked, "Synth_Dialogue.upk"), objs + [("AkBank", "NPC_SynthDialogue", ev_bank)])
ue_package(os.path.join(cooked, "Synth_Dialogue2.upk"), objs, prefix=bytes(3000))
with open(os.path.join(cooked, "CookedBanks.txt"), "a", newline="\r\n") as f:
f.write("NPC_SynthDialogue\n")
out = {"%08X" % m: w for m, w in list(foot.items()) + list(foley.items()) + list(gui.items())}
out["%08X" % DLG_MEDIA] = dlg_wem
return out
SPEECH_EVENT = "Speech_Synth_Radio_02"
SUBTITLES = {"INT": ["Scanner line two.", "Second part."], "ITA": ["Riga due dello scanner."],
"RUS": ["Строка два"]}
DLG_MEDIA = 0x0E000001
MENU = {"INT": ("FRONTEND.int", "\r\n[Menu]\r\nStart=Press any key\r\nQuit=\"Quit game\"\r\n; comment\r\n"
"[Tips]\r\nTip=Use the grapnel\r\nTip=Glide over the city\r\n"),
"ITA": ("FRONTEND.ITA", "\r\n[Menu]\r\nStart=Premi un tasto\r\nQuit=\"Esci\"\r\n"
"[Tips]\r\nTip=Usa il rampino\r\nTip=Plana sulla citta\r\n")}
def make_localization(root):
for lang, (fname, text) in MENU.items():
d = os.path.join(root, "BmGame", "Localization", lang)
os.makedirs(d, exist_ok=True)
open(os.path.join(d, fname), "wb").write(b"\xff\xfe" + text.encode("utf-16-le"))
open(os.path.join(root, "BmGame", "Localization", "INT", "Version.int"), "wb").write(b"[ID]\r\nUnreal=SYNTH\r\n")

View File

@@ -97,7 +97,7 @@ def test_bank_sound_apply_revert_in_editor(win, tmp_path):
w, root, originals = win
before = files(root)
w.cmb_arch.setCurrentText("BANK")
assert w.proxy.rowCount() == 5
assert w.proxy.rowCount() == 6
assert w.select_name("CHR_SynthFoley_1")
ed = w.editor
assert "bank CHR_SynthFoley (2 copies)" in ed.meta.text()

View File

@@ -32,7 +32,7 @@ def test_bank_catalog(synth_game, tmp_path):
root, originals = synth_game
ws = _ws(root, tmp_path)
bank = {s.target: s for s in ws.catalog.sounds if s.archive == "BANK"}
assert set(bank) == {"%08X" % m for m in synth.FOOT + synth.FOLEY + synth.GUI}
assert set(bank) == {"%08X" % m for m in synth.FOOT + synth.FOLEY + synth.GUI + (synth.DLG_MEDIA,)}
foot = bank["%08X" % synth.FOOT[0]]
assert foot.bank == "CHR_SynthFootsteps_Media" and foot.name == "CHR_SynthFootsteps_Media_1"
assert (foot.group, foot.category) == ("SFX", "Footsteps")

View File

@@ -57,7 +57,13 @@ def test_characters_csv_covers_user_lists():
def test_shipped_csvs_are_consistent():
from ackit import subtitles
for f in os.listdir(catalog_dir()):
if f.startswith("SUB_") and f.endswith(".csv"):
lines = subtitles.read_texts(os.path.join(catalog_dir(), f), f[4:7])
assert len({(s.speech, s.line) for s in lines}) == len(lines), f
assert all(len(s.text) <= s.room for s in lines), f
continue
if f.endswith(".csv") and f != "characters.csv":
rows = read_csv(os.path.join(catalog_dir(), f))
keys = {r.key for r in rows}

View File

@@ -12,7 +12,7 @@ def test_clean_synthetic_catalog_passes(synth_game):
root, _ = synth_game
rep = catalogcheck.run(Game(root), log=lambda *a: None)
assert rep.ok, rep.issues
assert rep.checked["BANK"] == 5 and rep.checked["INT"] == 4
assert rep.checked["BANK"] == 6 and rep.checked["INT"] == 4
def test_corrupted_catalog_is_reported(synth_game):

140
tests/test_texts.py Normal file
View File

@@ -0,0 +1,140 @@
"""Subtitles (dialogue objects in packages) and menu text (localization files): parse, apply, revert."""
import hashlib
import os
import pytest
import synth
from ackit import loctext, subtitles
from ackit.game import Game
from ackit.texts import MODIFIED, ORIGINAL, TextConflictError
from ackit.workspace import Workspace
SUB_KEY = "SUB:INT:%s:0" % synth.SPEECH_EVENT
def _sha(p):
return hashlib.sha1(open(p, "rb").read()).hexdigest()
def _ws(root, tmp_path):
return Workspace(root, store_root=str(tmp_path / "store"), archives=["INT", "SFX", "BANK"])
def _files(root):
d = os.path.join(root, "BmGame")
return {os.path.join(a, f): _sha(os.path.join(a, f)) for a, _, fs in os.walk(d) for f in fs}
# --- formats ------------------------------------------------------------------------------------
def test_fstring_encoding():
b = subtitles.encode_fstring("Hi", 10, "a")
assert b == b"\x06\x00\x00\x00Hi\0\0\0\0" and subtitles.read_fstring(b, 0)[0] == "Hi"
w = subtitles.encode_fstring("Привет", 4 + 2 * 8, "w")
assert subtitles.read_fstring(w, 0) == ("Привет", 20, "w")
with pytest.raises(subtitles.SubtitleError):
subtitles.encode_fstring("too long", 10, "a")
with pytest.raises(subtitles.SubtitleError):
subtitles.encode_fstring("→", 10, "a") # not in cp1252
def test_menu_parse_and_write():
fname, text = synth.MENU["INT"]
raw = b"\xff\xfe" + text.encode("utf-16-le")
es = {e.key: e for e in loctext.parse(raw, "INT", fname)}
assert es["INT:FRONTEND.int:Menu:Start"].value == "Press any key"
assert es["INT:FRONTEND.int:Menu:Quit"].quoted and es["INT:FRONTEND.int:Tips:Tip#2"].value == "Glide over the city"
assert loctext.write(raw, "INT", fname, {}) == raw
new = loctext.write(raw, "INT", fname, {"INT:FRONTEND.int:Menu:Quit": "Leave", "INT:FRONTEND.int:Tips:Tip#2": "Fly"})
assert new.decode("utf-16") == text.replace('"Quit game"', '"Leave"').replace("Glide over the city", "Fly")
with pytest.raises(loctext.TextError):
loctext.write(raw, "INT", fname, {"INT:FRONTEND.int:Menu:Start": "two\nlines"})
# --- subtitles through the workspace ------------------------------------------------------------
def test_subtitle_catalog_and_link(synth_game, tmp_path):
root, _ = synth_game
ws = _ws(root, tmp_path)
lines = ws.texts.subtitles("INT")
assert [(s.speech, s.line, s.text) for s in lines] == [(synth.SPEECH_EVENT, 0, "Scanner line two."),
(synth.SPEECH_EVENT, 1, "Second part.")]
line = ws.catalog.by_key["INT:%08X:1" % synth.SPEECH_ENTRY]
assert [s.text for s in ws.texts.for_sound(line)] == ["Scanner line two.", "Second part."]
assert ws.texts.subtitles("ITA")[0].sound == line.key # no ITA speech WAD here: INT line
assert ws.texts.subtitles("RUS")[0].enc == "w"
assert sorted(os.path.basename(p) for p in ws.texts.files(SUB_KEY)) == ["Synth_Dialogue.upk", "Synth_Dialogue2.upk"]
assert ws.texts.limit(SUB_KEY) == len("Scanner line two.")
def test_subtitle_apply_revert_every_copy(synth_game, tmp_path):
root, _ = synth_game
ws = _ws(root, tmp_path)
before = _files(root)
ws.texts.apply({SUB_KEY: "Line two.", "SUB:RUS:%s:0" % synth.SPEECH_EVENT: "Две"})
assert ws.texts.current(SUB_KEY) == "Line two." and ws.texts.state(SUB_KEY) == MODIFIED
for p in ws.texts.files(SUB_KEY): # every copy
ws2 = _ws(root, tmp_path / "other")
assert ws2.texts.current(SUB_KEY) == "Line two."
with pytest.raises(subtitles.SubtitleError):
ws.texts.apply({SUB_KEY: "This line is far too long for the room"})
ws.texts.revert([SUB_KEY])
assert ws.texts.current(SUB_KEY) == "Scanner line two." and ws.texts.current("SUB:RUS:%s:0" % synth.SPEECH_EVENT) == "Две"
ws.texts.revert_all()
assert _files(root) == before and ws.texts.state(SUB_KEY) == ORIGINAL
def test_subtitle_and_sound_in_one_package(synth_game, tmp_path):
root, _ = synth_game
ws = _ws(root, tmp_path)
before = _files(root)
s = ws.catalog.by_key["BANK:%08X" % synth.DLG_MEDIA]
ws.apply([(s, synth.wem(0.1, 1900), {})])
ws.texts.apply({SUB_KEY: "Both."})
assert ws.texts.current(SUB_KEY) == "Both." and ws.state(s) == "modified"
ws.revert([s])
assert ws.texts.current(SUB_KEY) == "Both." and ws.state(s) == "original"
ws.texts.revert_all()
assert _files(root) == before
# --- menu text through the workspace ------------------------------------------------------------
def test_menu_apply_revert(synth_game, tmp_path):
root, _ = synth_game
ws = _ws(root, tmp_path)
before = _files(root)
assert "INT" in ws.texts.menu_languages()
keys = {e.key: e for e in ws.texts.menu("INT")}
assert "INT:Version.int:ID:Unreal" not in keys
k = "MENU:INT:FRONTEND.int:Menu:Start"
ws.texts.apply({k: "Press Enter"})
assert ws.texts.current(k) == "Press Enter" and ws.texts.state(k) == MODIFIED
assert {e.key: e.value for e in ws.texts.menu("INT")}["INT:FRONTEND.int:Menu:Start"] == "Press Enter"
ws.texts.revert([k])
assert _files(root) == before
def test_menu_changed_underneath(synth_game, tmp_path):
root, _ = synth_game
ws = _ws(root, tmp_path)
k = "MENU:INT:FRONTEND.int:Tips:Tip#2"
ws.texts.apply({k: "Glide"})
path = ws.texts.files(k)[0]
raw = open(path, "rb").read()
open(path, "wb").write(loctext.write(raw, "INT", "FRONTEND.int", {"INT:FRONTEND.int:Menu:Start": "Other tool"}))
with pytest.raises(TextConflictError):
ws.texts.apply({k: "Soar"})
ws.texts.apply({k: "Soar"}, force=True)
assert ws.texts.current(k) == "Soar"
ws.texts.revert_all()
assert open(path, "rb").read() == open(ws.store.full_copy(ws.game.rel(path)), "rb").read()
# --- real game (read-only) ----------------------------------------------------------------------
def test_real_subtitles_and_menu(real_game_dir, tmp_path):
ws = Workspace(Game(real_game_dir), store_root=str(tmp_path / "store"), archives=["ITA"])
lines = ws.texts.subtitles("ITA")
assert len(lines) > 9000
s = next(x for x in lines if x.sound)
assert ws.texts.current(subtitles.key("ITA", s.speech, s.line)) == s.text
menu = ws.texts.menu("ITA")
assert len(menu) > 7000 and all(e.lang == "ITA" for e in menu)

View File

@@ -136,7 +136,7 @@ def test_mod_export_import_roundtrip(synth_game, synth_template, tmp_path):
ws2 = ws_for(other, store_root=str(tmp_path / "store2"))
_, items = match_mod(ws2, pkg)
assert [i.how for i in items] == ["exact", "exact"]
res = import_mod(ws2, pkg)
res, _ = import_mod(ws2, pkg)
assert all(i.note == "exact copy" for i in res)
assert ws2.current_bytes(ws2.catalog.by_key[a.key]) == wa
assert ws2.current_bytes(ws2.catalog.by_key[b.key]) == wb

View File

@@ -7,6 +7,8 @@ Outputs
data/catalog/<ARCHIVE>.csv every sound of one archive (schema: data/catalog/README.md)
data/catalog/BANK.csv in-memory sounds (soundbanks in SFX/*.bnk and the .upk packages)
data/catalog/BANK_locations.csv.gz every stored copy of them + the prefetch heads of speech lines
data/catalog/SUB_<LANG>.csv subtitles of the speech lines (one file per language)
data/catalog/SUB_locations.csv.gz every stored copy of each subtitle string
data/catalog/index.json fingerprints the CSVs were built from + counts
Run it only on clean game files (`ackit status` must say nothing is modified).
"""
@@ -20,8 +22,8 @@ import sys
import time
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from ackit import banks, settings # noqa: E402
from ackit.catalog import scan_archive, scan_banks, write_csv, hash_names # noqa: E402
from ackit import banks, settings, subtitles # noqa: E402
from ackit.catalog import scan_archive, scan_text, write_csv, hash_names # noqa: E402
from ackit.classify import USER_LABELS # noqa: E402
from ackit.game import Game, default_game_dir, wad_fingerprint # noqa: E402
from ackit.inject import read_target # noqa: E402
@@ -113,7 +115,8 @@ def main():
# in-memory sounds: loose SFX\*.bnk + banks inside the .upk packages (a few minutes)
t0 = time.time()
sounds, locs, sc = scan_banks(game, progress=lambda k, n: print(" banks: %d/%d files" % (k, n), flush=True)
sounds, locs, sc, texts, sublocs, sub_issues = scan_text(
game, progress=lambda k, n: print(" banks + subtitles: %d/%d files" % (k, n), flush=True)
if k % 500 == 0 else None, characters=charmap)
write_csv(os.path.join(out_dir, "%s.csv" % banks.ARCHIVE), sounds)
banks.write_locations(os.path.join(out_dir, banks.LOCATIONS), locs)
@@ -128,6 +131,23 @@ def main():
"issues": ["%s: %s" % i for i in sc.issues]}
print("%s.csv: %d sounds %s, %d copies in %d files, %d prefetch heads (%.0fs)"
% (banks.ARCHIVE, len(sounds), dict(g), len(locs) - len(heads), len(sc.files), len(heads), time.time() - t0))
# subtitles (dialogue objects in the packages), one CSV per language + every stored copy
for f in os.listdir(out_dir):
if f.startswith("SUB_") and f.endswith(".csv"):
os.remove(os.path.join(out_dir, f))
for lang, rows in texts.items():
subtitles.write_texts(os.path.join(out_dir, "SUB_%s.csv" % lang), rows)
subtitles.write_locations(os.path.join(out_dir, subtitles.LOCATIONS), sublocs)
any_lang = next(iter(texts.values()), [])
index["archives"]["SUB"] = {
"fingerprint": banks.game_fingerprint(game.cooked),
"lines": {lang: len(rows) for lang, rows in sorted(texts.items())},
"speech_lines": len({s.speech for s in any_lang}),
"linked": {lang: sum(1 for s in rows if s.sound) for lang, rows in sorted(texts.items())},
"copies": len(sublocs), "issues": ["%s: %s" % i for i in sub_issues]}
print("SUB_*.csv: %d languages, %d subtitle strings, %d copies"
% (len(texts), sum(len(r) for r in texts.values()), len(sublocs)))
# flatten for catalog.load_archive
flat = {a: v for a, v in index["archives"].items()}
json.dump(dict(flat, _info=index["built_from"]), open(os.path.join(out_dir, "index.json"), "w", newline="\n"), indent=1)