Edit subtitles and menu text
This commit is contained in:
@@ -9,6 +9,7 @@ Batman: Arkham City (PC) audio toolkit + editor. Read `NOTES.md` first (status,
|
||||
Speech/ambience: in place, never larger than the original WEM; music packs: any size (`docs/formats/wad.md`).
|
||||
In-memory sounds (archive BANK, loose `.bnk` + `.upk` packages): in place into every copy; a bank file is always
|
||||
rebuilt from the store's original copy + the applied replacements (`docs/formats/banks.md`, `upk.md`).
|
||||
Texts go through `Workspace.texts` (subtitles in packages, in place and NUL-padded; menu `.int` files).
|
||||
- Never use Steam "verify integrity". Keep game files original after tests (`ackit status`, `ackit revert --all`).
|
||||
- Real-game checks: `ackit verify` (offline), `ackit verify --catalog [--quick]` (catalog vs game files),
|
||||
`ackit verify --ingame [--banks]` (Windows; needs Steam running and logged in, WinDbg; ~1.5 min; reverts and
|
||||
|
||||
10
NOTES.md
10
NOTES.md
@@ -27,7 +27,11 @@
|
||||
rebuilt from a mandatory full original copy); editor shows bank + copies. `ackit verify` PASS (300/300
|
||||
BANK WEMs rebuilt identically); `ackit verify --ingame --banks` PASS (loaded, no crash, revert identical;
|
||||
audible check not possible without input: findings 18).
|
||||
4. [ ] Subtitles + menu text: find/document formats, `ackit` parse/write with backup/revert.
|
||||
4. [x] Subtitles = RDialogueEvent objects in packages (9,016 lines x 11 languages, 168k copies; parser
|
||||
`ackit/subtitles.py`, catalog SUB_<LANG>.csv + SUB_locations.csv.gz, 92 % linked to speech sounds via
|
||||
events); menu text = Localization/<LANG>/*.int (`ackit/loctext.py`, ~7,000 strings/language).
|
||||
`Workspace.texts` (ackit/texts.py): states, apply/revert, conflicts, mod packages; CLI `ackit text`.
|
||||
Subtitles in place (NUL-padded, limit = original length); decision + reasons in decisions.md 4c.
|
||||
5. [ ] Editor: Subtitles section linked to audio, separate Menu text section, revert, mod export/import.
|
||||
6. [ ] Final e2e (audio + text), packaged build, REPORT.md.
|
||||
- OPEN: channel order for >2 channels unverified; speech and in-memory sounds *heard* in-game (need gameplay;
|
||||
@@ -105,3 +109,7 @@
|
||||
runs (no compressor needed). Bank-file conflicts use the whole-file SHA-1 (the cheap guid+chunk-table
|
||||
fingerprint missed same-size edits in tests). Name collisions in the WAD catalogs (2 per language) are shown as
|
||||
NAME#i. `soundcard` comes from the `verify` extra (was missing from the venv).
|
||||
- 2026-10-09 (mission 3, stage 4): subtitle links by name were wrong for most multi-variation entries (WEMs are
|
||||
stored in media-ID order, not _NN order): links come only from events. Numbered event names (Speech_X_04)
|
||||
live in export tables, not in name tables: the bank scan now collects export names (+40 named BANK sounds,
|
||||
links 71 % -> 92 %). Version*.int (build IDs) are not treated as UI text.
|
||||
|
||||
@@ -33,13 +33,16 @@ ackit find joker tape # search by name, category or c
|
||||
ackit extract Speech_TAPE_JOKER_1_4 orig.wav # listen to the original
|
||||
ackit replace Speech_TAPE_JOKER_1_4 my_line.wav --match-loudness
|
||||
ackit revert Speech_TAPE_JOKER_1_4 # or: ackit revert --all
|
||||
ackit text find --lang ITA protocollo # subtitles and menu texts (keys SUB:... / MENU:...)
|
||||
ackit text set "MENU:ITA:FRONTEND.ITA:Menu:Start" "Premi Invio"
|
||||
```
|
||||
More: `ackit status`, `ackit list --character Joker`, `ackit export-mod my.acmod` / `ackit import-mod`,
|
||||
`ackit verify` (offline self-check), `ackit game` (detected install, language, store). `ackit -h` lists all
|
||||
commands. The game folder is found through Steam (Windows, Linux, Flatpak, Proton); otherwise pass `--game DIR`.
|
||||
|
||||
Rules worth knowing: speech/effects replacements must fit in the original's size (the tool squeezes or refuses
|
||||
with the maximum length); music can be any length. Close the game before writing.
|
||||
with the maximum length); music can be any length. Subtitles must fit the original text's length (menu text:
|
||||
any length). Close the game before writing.
|
||||
|
||||
## Where things are
|
||||
```
|
||||
|
||||
@@ -155,23 +155,6 @@ class Scan:
|
||||
self.issues = [] # (kind, detail)
|
||||
|
||||
|
||||
def _package_names(pkg, start, data):
|
||||
"""Name table of an uncompressed package."""
|
||||
out = []
|
||||
o = pkg.name_offset - start
|
||||
for _ in range(pkg.name_count):
|
||||
n = struct.unpack_from("<i", data, o)[0]
|
||||
o += 4
|
||||
if n > 0:
|
||||
out.append(bytes(data[o:o + n - 1]).decode("latin-1"))
|
||||
o += n
|
||||
else:
|
||||
out.append(bytes(data[o:o - 2 * n - 2]).decode("utf-16-le", "replace"))
|
||||
o += -2 * n
|
||||
o += 8
|
||||
return out
|
||||
|
||||
|
||||
def _bkpk_tables(data, packs):
|
||||
"""BKPK tables (docs/formats/wad.md) embedded in a package: WAD entry -> media offsets/sizes."""
|
||||
import re
|
||||
@@ -187,8 +170,9 @@ def _bkpk_tables(data, packs):
|
||||
packs[entry][mid] = (off, size)
|
||||
|
||||
|
||||
def scan(cooked, progress=None, cooked_banks=None, files=None):
|
||||
"""Scan every loose bank and package of the game (minutes), or only `files`. Returns a `Scan`."""
|
||||
def scan(cooked, progress=None, cooked_banks=None, files=None, visit=None):
|
||||
"""Scan every loose bank and package of the game (minutes), or only `files`. Returns a `Scan`.
|
||||
`visit(rel, package, start, data)` is called for every package (other scans share the decompression)."""
|
||||
sc = Scan()
|
||||
files = bank_files(cooked) if files is None else files
|
||||
if cooked_banks is None:
|
||||
@@ -206,8 +190,12 @@ def scan(cooked, progress=None, cooked_banks=None, files=None):
|
||||
sc.issues.append(("unreadable", "%s: %s" % (rel, e)))
|
||||
continue
|
||||
if isinstance(r, Package):
|
||||
sc.names.update(_package_names(r, start, data))
|
||||
names, _, exports = r.tables(data, start)
|
||||
sc.names.update(names) # base names ...
|
||||
sc.names.update(x.name for x in exports) # ... and numbered object names (Speech_X_04)
|
||||
_bkpk_tables(data, sc.packs)
|
||||
if visit:
|
||||
visit(rel, r, start, data)
|
||||
banks = list(find_banks(data))
|
||||
else:
|
||||
try:
|
||||
|
||||
@@ -18,7 +18,7 @@ import re
|
||||
import struct
|
||||
from dataclasses import dataclass, asdict
|
||||
|
||||
from . import banks, classify
|
||||
from . import banks, classify, subtitles
|
||||
from .game import wad_fingerprint
|
||||
from .inject import pack_bnk_path
|
||||
from .pack import Pack
|
||||
@@ -228,9 +228,13 @@ def bank_sounds(sc, characters=None):
|
||||
|
||||
|
||||
def head_owners(game, sc, archives=None):
|
||||
"""{prefetch media id: WAD sound key} -- the speech line (one language) each prefetch head belongs to,
|
||||
found through the per-language BKPK tables of the packages (docs/formats/banks.md)."""
|
||||
want = {m for m, e in sc.media.items() if banks.PREFETCH_HEAD in e["kinds"]}
|
||||
"""{prefetch media id: WAD sound key} -- the speech line (one language) each prefetch head belongs to."""
|
||||
return media_owners(game, sc, {m for m, e in sc.media.items() if banks.PREFETCH_HEAD in e["kinds"]}, archives)
|
||||
|
||||
|
||||
def media_owners(game, sc, want, archives=None):
|
||||
"""{media id: WAD sound key} for streamed speech media, found through the per-language BKPK tables of
|
||||
the packages (docs/formats/banks.md): every language has its own media IDs for the same WAD entry."""
|
||||
out = {}
|
||||
for a in archives or [x for x in game.archives if x != "SFX"]:
|
||||
wad = Wad(game.wad(a))
|
||||
@@ -250,14 +254,61 @@ def head_owners(game, sc, archives=None):
|
||||
return out
|
||||
|
||||
|
||||
def scan_banks(game, progress=None, characters=None):
|
||||
def scan_banks(game, progress=None, characters=None, visit=None):
|
||||
"""Full scan of the game's banks -> ([Sound], [Location] with WAD keys on prefetch heads, Scan)."""
|
||||
sc = banks.scan(game.cooked, progress)
|
||||
sc = banks.scan(game.cooked, progress, visit=visit)
|
||||
owners = head_owners(game, sc)
|
||||
locs = [l._replace(sound=owners.get(l.media, "")) if l.kind == banks.PREFETCH_HEAD else l for l in sc.locations]
|
||||
return bank_sounds(sc, characters), locs, sc
|
||||
|
||||
|
||||
def subtitle_links(game, sc, speech_names):
|
||||
"""{speech event name: {language: WAD sound key}}: event -> actions -> containers -> sounds -> the
|
||||
per-language media of the streamed speech line (exact, no name matching)."""
|
||||
ev = banks.event_media(sc)
|
||||
want = {m for n in speech_names for m in ev.get(n, ())}
|
||||
owners = media_owners(game, sc, want)
|
||||
out = {}
|
||||
for n in speech_names:
|
||||
keys = {}
|
||||
for m in ev.get(n, ()):
|
||||
k = owners.get(m)
|
||||
if k:
|
||||
keys.setdefault(k.split(":")[0], k)
|
||||
if keys:
|
||||
out[n] = keys
|
||||
return out
|
||||
|
||||
|
||||
def scan_text(game, progress=None, characters=None):
|
||||
"""One pass over every package: (bank sounds, bank locations, Scan, {lang: [SubLine]}, [SubLoc], issues)."""
|
||||
col = subtitles.Collector()
|
||||
snd, locs, sc = scan_banks(game, progress, characters, visit=col)
|
||||
links = subtitle_links(game, sc, list(col.dialogues))
|
||||
texts, sublocs = subtitles.rows(col, links)
|
||||
return snd, locs, sc, texts, sublocs, col.issues
|
||||
|
||||
|
||||
def load_subtitles(game, lang, progress=None):
|
||||
"""([SubLine], {SUB key: [SubLoc]}) of one language: shipped catalog when the game matches it, else a
|
||||
cached or fresh scan of the packages (several minutes)."""
|
||||
fp = banks.game_fingerprint(game.cooked)
|
||||
shipped = _shipped_index().get("SUB", {})
|
||||
t = os.path.join(catalog_dir(), "SUB_%s.csv" % lang)
|
||||
loc = os.path.join(catalog_dir(), subtitles.LOCATIONS)
|
||||
if shipped.get("fingerprint") == fp and os.path.exists(t) and os.path.exists(loc):
|
||||
return subtitles.read_texts(t, lang), subtitles.read_locations(loc, lang)
|
||||
cache = os.path.join(user_dir("cache"), "scan_SUB_%s_v%d" % (fp, SCAN_VERSION))
|
||||
if not os.path.exists(cache + ".locations.csv.gz"):
|
||||
_, _, _, texts, sublocs, _ = scan_text(game, progress)
|
||||
for lg, rows in texts.items():
|
||||
subtitles.write_texts(cache + "_%s.csv" % lg, rows)
|
||||
subtitles.write_locations(cache + ".locations.csv.gz", sublocs)
|
||||
t = cache + "_%s.csv" % lang
|
||||
return (subtitles.read_texts(t, lang) if os.path.exists(t) else []), \
|
||||
subtitles.read_locations(cache + ".locations.csv.gz", lang)
|
||||
|
||||
|
||||
def load_banks(game, progress=None):
|
||||
"""([Sound], {media: [Location]}) of the in-memory sounds: shipped catalog when the game's bank set
|
||||
matches it, else a cached or fresh scan (several minutes)."""
|
||||
|
||||
63
ackit/cli.py
63
ackit/cli.py
@@ -171,7 +171,9 @@ def cmd_revert(a):
|
||||
ws = _workspace(a)
|
||||
if a.all:
|
||||
done = ws.revert_all(progress=print)
|
||||
else:
|
||||
texts = ws.texts.revert_all(progress=print)
|
||||
print("reverted %d sound(s) and %d text(s)" % (len(done), len(texts)))
|
||||
return
|
||||
done = ws.revert([ws.sound(n) for n in a.names], progress=print)
|
||||
print("reverted %d sound(s)" % len(done))
|
||||
|
||||
@@ -179,29 +181,72 @@ def cmd_revert(a):
|
||||
def cmd_status(a):
|
||||
ws = _workspace(a)
|
||||
keys = ws.modified_keys()
|
||||
if not keys:
|
||||
print("no sounds modified; game audio is original")
|
||||
texts = ws.texts.modified()
|
||||
if not keys and not texts:
|
||||
print("no sounds or texts modified; game files are original")
|
||||
return
|
||||
for k in keys:
|
||||
s = ws.catalog.by_key.get(k)
|
||||
st = ws.state(s) if s else "(other language)"
|
||||
print("%-50s %s" % (s.name if s else k, st))
|
||||
for k in texts:
|
||||
print("%-50s %s" % (k, ws.texts.state(k)))
|
||||
|
||||
|
||||
def cmd_export_mod(a):
|
||||
from .modpack import export_mod
|
||||
ws = _workspace(a)
|
||||
keys = [ws.sound(n).key for n in a.names] if a.names else ws.modified_keys()
|
||||
texts = [] if a.names else ws.texts.modified()
|
||||
n = export_mod(ws, a.out, keys, a.title or os.path.splitext(os.path.basename(a.out))[0], a.author or "",
|
||||
a.description or "")
|
||||
print("wrote %s (%d sounds)" % (a.out, n))
|
||||
a.description or "", texts=texts)
|
||||
print("wrote %s (%d sounds, %d texts)" % (a.out, len(keys), n - len(keys)))
|
||||
|
||||
|
||||
def cmd_import_mod(a):
|
||||
from .modpack import import_mod
|
||||
ws = _workspace(a)
|
||||
for im in import_mod(ws, a.file, force=a.force, progress=print):
|
||||
items, texts = import_mod(ws, a.file, force=a.force, progress=print)
|
||||
for im in items:
|
||||
print("%-8s %-46s %s" % (im.how, im.item.get("name"), im.note))
|
||||
for k, _, problem in texts:
|
||||
print("%-8s %-46s %s" % ("text", k, problem or "applied"))
|
||||
|
||||
|
||||
def cmd_text(a):
|
||||
"""Subtitles and menu text: find, show, set, revert."""
|
||||
ws = _workspace(a)
|
||||
tx = ws.texts
|
||||
lang = (a.lang or ws.game.language()).upper()
|
||||
if a.action == "find":
|
||||
words = [w.lower() for w in a.args]
|
||||
rows = []
|
||||
if a.kind in ("all", "subs"):
|
||||
rows += [(subtitles_key(s), s.text) for s in tx.subtitles(lang)]
|
||||
if a.kind in ("all", "menu"):
|
||||
rows += [("MENU:" + e.key, e.value) for e in tx.menu(lang)]
|
||||
hits = [(k, v) for k, v in rows if all(w in (k + " " + v).lower() for w in words)]
|
||||
for k, v in hits[:a.limit]:
|
||||
print("%s\n %s" % (k, v))
|
||||
print("%d match(es)%s" % (len(hits), "" if len(hits) <= a.limit else " (first %d shown)" % a.limit))
|
||||
elif a.action == "show":
|
||||
for k in a.args:
|
||||
print("%s\n now: %s\n original: %s\n state: %s%s" % (
|
||||
k, tx.current(k), tx.original(k), tx.state(k),
|
||||
"" if tx.limit(k) is None else "\n room: %d characters" % tx.limit(k)))
|
||||
elif a.action == "set":
|
||||
if len(a.args) != 2:
|
||||
raise GameError("usage: ackit text set KEY \"new text\"")
|
||||
tx.apply({a.args[0]: a.args[1]}, force=a.force, progress=print)
|
||||
print("set %s" % a.args[0])
|
||||
elif a.action == "revert":
|
||||
done = tx.revert_all(progress=print) if a.all else tx.revert(a.args, progress=print)
|
||||
print("reverted %d text(s)" % len(done))
|
||||
|
||||
|
||||
def subtitles_key(s):
|
||||
from .subtitles import key
|
||||
return key(s.lang, s.speech, s.line)
|
||||
|
||||
|
||||
# --- advanced ---------------------------------------------------------------------------------
|
||||
@@ -321,6 +366,12 @@ def build_parser():
|
||||
s = add("revert", cmd_revert, "put original sounds back", aliases=["restore"])
|
||||
s.add_argument("names", nargs="*"); s.add_argument("--all", action="store_true")
|
||||
add("status", cmd_status, "list the sounds you modified")
|
||||
s = add("text", cmd_text, "subtitles and menu text: find WORDS | show KEY | set KEY TEXT | revert KEY/--all")
|
||||
s.add_argument("action", choices=["find", "show", "set", "revert"])
|
||||
s.add_argument("args", nargs="*")
|
||||
s.add_argument("--kind", choices=["all", "subs", "menu"], default="all", help="with find: what to search")
|
||||
s.add_argument("--limit", type=int, default=40); s.add_argument("--all", action="store_true")
|
||||
s.add_argument("--force", action="store_true", help="overwrite texts changed outside this tool")
|
||||
s = add("export-mod", cmd_export_mod, "save your changes as a shareable .acmod package")
|
||||
s.add_argument("out"); s.add_argument("names", nargs="*", help="default: everything you modified")
|
||||
s.add_argument("--title"); s.add_argument("--author"); s.add_argument("--description")
|
||||
|
||||
110
ackit/loctext.py
Normal file
110
ackit/loctext.py
Normal file
@@ -0,0 +1,110 @@
|
||||
"""Menu and UI text: the game's localization files BmGame/Localization/<LANG>/<File>.<lang>
|
||||
(UE3 .ini format; docs/formats/text.md).
|
||||
|
||||
Files are UTF-16LE with a BOM and CRLF line ends (two small ones are plain ASCII). Lines are `[Section]`,
|
||||
`Key=Value` (the value may be in double quotes) or comments (`;`). A key can repeat inside a section, so an
|
||||
entry is identified by (file, section, key, occurrence). Writing changes only the value part of the edited
|
||||
lines; every other byte of the file (BOM, encoding, quotes, line ends, comments) stays as it was.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
from collections import namedtuple
|
||||
|
||||
# key: "<LANG>:<File>:<Section>:<Key>" + ("#n" for the n-th repeat, n >= 2)
|
||||
Entry = namedtuple("Entry", "key lang file section name occurrence value quoted line")
|
||||
|
||||
LANG_DIRS = ("INT", "ITA", "FRA", "DEU", "ESN", "ESM", "POL", "POR", "RUS", "JPN", "KOR")
|
||||
|
||||
|
||||
class TextError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _decode(raw):
|
||||
if raw[:2] == b"\xff\xfe":
|
||||
return raw[2:].decode("utf-16-le"), "utf-16"
|
||||
if raw[:3] == b"\xef\xbb\xbf":
|
||||
return raw[3:].decode("utf-8"), "utf-8"
|
||||
return raw.decode("cp1252"), "cp1252"
|
||||
|
||||
|
||||
def _encode(text, enc):
|
||||
if enc == "utf-16":
|
||||
return b"\xff\xfe" + text.encode("utf-16-le")
|
||||
if enc == "utf-8":
|
||||
return b"\xef\xbb\xbf" + text.encode("utf-8")
|
||||
return text.encode("cp1252")
|
||||
|
||||
|
||||
_KV = re.compile(r"^([^=;\[\s][^=]*?)=(.*)$")
|
||||
|
||||
|
||||
def parse(raw, lang, fname):
|
||||
"""Entries of one file's bytes."""
|
||||
text, _ = _decode(raw)
|
||||
out = []
|
||||
section = ""
|
||||
seen = {}
|
||||
for i, line in enumerate(re.split(r"(?<=\n)", text)):
|
||||
body = line.rstrip("\r\n")
|
||||
s = body.strip()
|
||||
if s.startswith("[") and s.endswith("]"):
|
||||
section = s[1:-1]
|
||||
continue
|
||||
m = _KV.match(body)
|
||||
if not m or s.startswith(";"):
|
||||
continue
|
||||
name, value = m.group(1), m.group(2)
|
||||
quoted = len(value) >= 2 and value[0] == '"' and value[-1] == '"'
|
||||
if quoted:
|
||||
value = value[1:-1]
|
||||
k = (section.lower(), name.lower())
|
||||
seen[k] = seen.get(k, 0) + 1
|
||||
n = seen[k]
|
||||
key = "%s:%s:%s:%s" % (lang, fname, section, name) + ("#%d" % n if n > 1 else "")
|
||||
out.append(Entry(key, lang, fname, section, name, n, value, quoted, i))
|
||||
return out
|
||||
|
||||
|
||||
def write(raw, lang, fname, values):
|
||||
"""New file bytes with {entry key: new value} applied (keys not in the file raise TextError)."""
|
||||
text, enc = _decode(raw)
|
||||
lines = re.split(r"(?<=\n)", text)
|
||||
by_key = {e.key: e for e in parse(raw, lang, fname)}
|
||||
for key, value in values.items():
|
||||
e = by_key.get(key)
|
||||
if e is None:
|
||||
raise TextError("no text %s in %s" % (key, fname))
|
||||
if "\r" in value or "\n" in value:
|
||||
raise TextError("a menu text cannot contain line breaks (use the game's own markers, e.g. \\n)")
|
||||
line = lines[e.line]
|
||||
end = line[len(line.rstrip("\r\n")):]
|
||||
head = line[:line.index("=") + 1]
|
||||
lines[e.line] = head + ('"%s"' % value if e.quoted else value) + end
|
||||
out = "".join(lines)
|
||||
try:
|
||||
return _encode(out, enc)
|
||||
except UnicodeEncodeError:
|
||||
raise TextError("%s is stored as %s and cannot hold these characters" % (fname, enc)) from None
|
||||
|
||||
|
||||
def files(loc_root, lang):
|
||||
"""{file name: path} of one language folder (case-insensitive): localization files, without the
|
||||
Version*.int build IDs (not UI text)."""
|
||||
d = next((os.path.join(loc_root, x) for x in os.listdir(loc_root) if x.upper() == lang.upper()), None)
|
||||
if not d:
|
||||
return {}
|
||||
return {f: os.path.join(d, f) for f in sorted(os.listdir(d))
|
||||
if os.path.splitext(f)[1].lower() == "." + lang.lower() and not f.lower().startswith("version")}
|
||||
|
||||
|
||||
def languages(loc_root):
|
||||
return sorted(x.upper() for x in os.listdir(loc_root) if os.path.isdir(os.path.join(loc_root, x)))
|
||||
|
||||
|
||||
def load(loc_root, lang):
|
||||
"""All entries of a language: [Entry]."""
|
||||
out = []
|
||||
for f, p in files(loc_root, lang).items():
|
||||
out += parse(open(p, "rb").read(), lang.upper(), f)
|
||||
return out
|
||||
@@ -6,6 +6,8 @@ A package is a zip:
|
||||
items/NNN.flac the same audio, decoded (used to re-encode when the target differs, e.g. another
|
||||
language whose original has a different size)
|
||||
Each item records archive, target, name, group and the SHA-1 of the original it replaced.
|
||||
"texts": [{"key", "text", "original"}] edited subtitles / menu texts (keys: ackit/texts.py); optional,
|
||||
so packages without texts stay valid
|
||||
"""
|
||||
import io
|
||||
import json
|
||||
@@ -22,9 +24,17 @@ from .originals import sha1_16
|
||||
FORMAT = 1
|
||||
|
||||
|
||||
def export_mod(ws, path, keys, name="", author="", description=""):
|
||||
"""Write the applied versions of `keys` (from the originals store) into a package."""
|
||||
def export_mod(ws, path, keys, name="", author="", description="", texts=()):
|
||||
"""Write the applied versions of sound `keys` and text keys `texts` (from the originals store) into a
|
||||
package. Returns the number of entries."""
|
||||
items = []
|
||||
recs = ws.texts.records()
|
||||
text_items = []
|
||||
for k in texts:
|
||||
r = recs.get(k)
|
||||
if not r or r.get("applied") is None:
|
||||
raise ValueError("%s is not modified" % k)
|
||||
text_items.append({"key": k, "text": r["applied"], "original": r["orig"]})
|
||||
with zipfile.ZipFile(path + ".tmp", "w", zipfile.ZIP_DEFLATED) as z:
|
||||
for i, key in enumerate(keys):
|
||||
e = ws.store.get(key)
|
||||
@@ -43,9 +53,10 @@ def export_mod(ws, path, keys, name="", author="", description=""):
|
||||
"wem": base + ".wem", "audio": base + ".flac", "meta": e.get("meta", {})})
|
||||
z.writestr("manifest.json", json.dumps({"format": FORMAT, "name": name, "author": author,
|
||||
"description": description, "created": time.time(),
|
||||
"game": "Batman: Arkham City (PC)", "items": items}, indent=1))
|
||||
"game": "Batman: Arkham City (PC)", "items": items,
|
||||
"texts": text_items}, indent=1, ensure_ascii=False))
|
||||
os.replace(path + ".tmp", path)
|
||||
return len(items)
|
||||
return len(items) + len(text_items)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -89,6 +100,20 @@ def match_mod(ws, path):
|
||||
return man, out
|
||||
|
||||
|
||||
def match_texts(ws, man):
|
||||
"""[(key, text, problem or '')] for the package's texts in this game (problem: why it cannot be applied)."""
|
||||
out = []
|
||||
for it in man.get("texts", []):
|
||||
k, text = it["key"], it["text"]
|
||||
try:
|
||||
ws.texts.current(k)
|
||||
ws.texts.check(k, text)
|
||||
out.append((k, text, ""))
|
||||
except Exception as e: # missing key, too long, characters the file cannot hold
|
||||
out.append((k, text, str(e)))
|
||||
return out
|
||||
|
||||
|
||||
def prepare_import(ws, path, items, progress=None):
|
||||
"""Fill `wem` for matched items: reuse the packaged WEM when it targets the very same original,
|
||||
otherwise re-encode the packaged audio for the local sound."""
|
||||
@@ -110,11 +135,15 @@ def prepare_import(ws, path, items, progress=None):
|
||||
|
||||
|
||||
def import_mod(ws, path, force=False, progress=None):
|
||||
"""Match, prepare and apply a package. Returns the list of ImportItem (see .how/.note)."""
|
||||
_, items = match_mod(ws, path)
|
||||
"""Match, prepare and apply a package. Returns ([ImportItem] (see .how/.note), [(key, text, problem)])."""
|
||||
man, items = match_mod(ws, path)
|
||||
prepare_import(ws, path, items, progress)
|
||||
changes = [(im.sound, im.wem, {"source": "mod:" + path.replace("\\", "/").split("/")[-1]})
|
||||
for im in items if im.wem]
|
||||
source = {"source": "mod:" + path.replace("\\", "/").split("/")[-1]}
|
||||
changes = [(im.sound, im.wem, source) for im in items if im.wem]
|
||||
if changes:
|
||||
ws.apply(changes, force=force, progress=progress)
|
||||
return items
|
||||
texts = match_texts(ws, man)
|
||||
ok = {k: v for k, v, problem in texts if not problem}
|
||||
if ok:
|
||||
ws.texts.apply(ok, force=force, progress=progress, meta=source)
|
||||
return items, texts
|
||||
|
||||
255
ackit/subtitles.py
Normal file
255
ackit/subtitles.py
Normal file
@@ -0,0 +1,255 @@
|
||||
"""Subtitles: the RDialogueEvent objects inside the game's packages (docs/formats/subtitles.md).
|
||||
|
||||
Each dialogue object references its speech event (an export named like the speech line, e.g.
|
||||
'Speech_CHURCH_B1_Ch2_POST_Explosion_BM_Cash_04') and holds the subtitle text of every language:
|
||||
LocalizedSubtitles: array of {Language: FString, Subtitles: array of {Text: FString, (float @0x0c)}}
|
||||
in Rocksteady's compact tagged-property format (u16 type, u16 field offset, value; structs and arrays with
|
||||
a byte size; struct elements end with a u16 0).
|
||||
|
||||
A dialogue object can be stored in several packages; a subtitle edit is written into every copy, in place:
|
||||
the new text replaces the bytes of the old FString and is padded with NUL characters (the game stops at the
|
||||
first NUL), so no size field, export or package layout changes. The text must therefore fit the original
|
||||
string's bytes (Latin-1/cp1252 strings: one byte per character; UTF-16 strings: two).
|
||||
"""
|
||||
import csv
|
||||
import gzip
|
||||
import io
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
from collections import defaultdict, namedtuple
|
||||
|
||||
from .upk import Package
|
||||
|
||||
CLASS = "RDialogueEvent"
|
||||
T_STRUCT, T_STRING, T_OBJECT = 0x09, 0x0D, 0x10
|
||||
|
||||
# one subtitle string inside a package: pos = absolute uncompressed offset of the FString length field,
|
||||
# size = bytes of the whole FString (length field included), enc 'a' (cp1252) or 'w' (UTF-16)
|
||||
Field = namedtuple("Field", "lang index text pos size enc")
|
||||
Dialogue = namedtuple("Dialogue", "speech export fields") # speech: name of the referenced speech event
|
||||
|
||||
|
||||
class SubtitleError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def read_fstring(b, o):
|
||||
"""-> (text, end offset, enc)"""
|
||||
n = struct.unpack_from("<i", b, o)[0]
|
||||
if n >= 0:
|
||||
raw = bytes(b[o + 4:o + 4 + n])
|
||||
return raw.split(b"\0", 1)[0].decode("cp1252", "replace"), o + 4 + n, "a"
|
||||
raw = bytes(b[o + 4:o + 4 - 2 * n])
|
||||
return raw.decode("utf-16-le", "replace").split("\0", 1)[0], o + 4 - 2 * n, "w"
|
||||
|
||||
|
||||
def encode_fstring(text, size, enc):
|
||||
"""The FString bytes (length field included) for `text`, exactly `size` bytes long (NUL padded).
|
||||
Raises SubtitleError when the text does not fit."""
|
||||
if "\0" in text:
|
||||
raise SubtitleError("the text must not contain NUL characters")
|
||||
if enc == "a":
|
||||
try:
|
||||
raw = text.encode("cp1252")
|
||||
except UnicodeEncodeError:
|
||||
raise SubtitleError("this subtitle is stored as Latin-1 text; '%s' cannot be written"
|
||||
% next(c for c in text if not _cp1252(c))) from None
|
||||
n = size - 4
|
||||
if len(raw) + 1 > n:
|
||||
raise SubtitleError("the text is %d characters long but this subtitle has room for %d"
|
||||
% (len(raw), n - 1))
|
||||
return struct.pack("<i", n) + raw + bytes(n - len(raw))
|
||||
raw = text.encode("utf-16-le")
|
||||
n = (size - 4) // 2
|
||||
if len(raw) // 2 + 1 > n:
|
||||
raise SubtitleError("the text is %d characters long but this subtitle has room for %d" % (len(raw) // 2, n - 1))
|
||||
return struct.pack("<i", -n) + raw + bytes(2 * n - len(raw))
|
||||
|
||||
|
||||
def _cp1252(c):
|
||||
try:
|
||||
c.encode("cp1252")
|
||||
return True
|
||||
except UnicodeEncodeError:
|
||||
return False
|
||||
|
||||
|
||||
def room(field):
|
||||
"""Maximum number of characters for a subtitle string."""
|
||||
return (field.size - 4 - 1) if field.enc == "a" else (field.size - 4) // 2 - 1
|
||||
|
||||
|
||||
def _parse_lines(b, o, end, base, lang):
|
||||
"""Subtitles array content [o, end) -> [Field]."""
|
||||
count = struct.unpack_from("<I", b, o)[0]
|
||||
o += 4
|
||||
out = []
|
||||
for i in range(count):
|
||||
# element: props until u16 0; the text is the string at field offset 0
|
||||
while True:
|
||||
t, fo = struct.unpack_from("<HH", b, o)
|
||||
if t == 0:
|
||||
o += 2
|
||||
break
|
||||
if t == T_STRING:
|
||||
text, e, enc = read_fstring(b, o + 4)
|
||||
if fo == 0:
|
||||
out.append(Field(lang, i, text, base + o + 4, e - (o + 4), enc))
|
||||
o = e
|
||||
elif t == 0x04: # 4-byte value (the line's time)
|
||||
o += 8
|
||||
else:
|
||||
raise SubtitleError("unexpected property type %#x in a subtitle line" % t)
|
||||
if o > end:
|
||||
raise SubtitleError("subtitle line overruns its array")
|
||||
if o != end:
|
||||
raise SubtitleError("subtitle array size mismatch")
|
||||
return out
|
||||
|
||||
|
||||
def parse_dialogue(b, base, names_idx):
|
||||
"""Fields of one RDialogueEvent export (bytes `b` starting at uncompressed offset `base`)."""
|
||||
ls, sub = names_idx
|
||||
i = b.find(struct.pack("<II", ls, 0))
|
||||
if i < 4 or struct.unpack_from("<H", b, i - 4)[0] != T_STRUCT:
|
||||
return []
|
||||
size = struct.unpack_from("<i", b, i + 8)[0]
|
||||
o, end = i + 16, i + 16 + size
|
||||
count = struct.unpack_from("<I", b, o)[0]
|
||||
o += 4
|
||||
fields = []
|
||||
for _ in range(count):
|
||||
lang = None
|
||||
while True:
|
||||
t, fo = struct.unpack_from("<HH", b, o)
|
||||
if t == 0:
|
||||
o += 2
|
||||
break
|
||||
if t == T_STRING:
|
||||
lang, o, _ = read_fstring(b, o + 4)
|
||||
elif t == T_STRUCT:
|
||||
nm, _nn, ssize, _ai = struct.unpack_from("<iiii", b, o + 4)
|
||||
c0 = o + 20
|
||||
if nm == sub:
|
||||
fields += _parse_lines(b, c0, c0 + ssize, base, lang)
|
||||
o = c0 + ssize
|
||||
else:
|
||||
raise SubtitleError("unexpected property type %#x in a localized subtitle" % t)
|
||||
if o != end:
|
||||
raise SubtitleError("localized subtitle size mismatch")
|
||||
return fields
|
||||
|
||||
|
||||
def _speech_ref(b, exports, imports):
|
||||
"""Name of the object referenced by the dialogue's first object property (its speech event)."""
|
||||
m = re.search(rb"\x10\x00\x2c\x00", b)
|
||||
if not m:
|
||||
return ""
|
||||
ref = struct.unpack_from("<i", b, m.start() + 4)[0]
|
||||
if ref > 0 and ref <= len(exports):
|
||||
return exports[ref - 1].name
|
||||
if ref < 0 and -ref <= len(imports):
|
||||
return imports[-ref - 1][1]
|
||||
return ""
|
||||
|
||||
|
||||
class Collector:
|
||||
"""`visit` hook for banks.scan: collects the dialogue objects of every package."""
|
||||
|
||||
def __init__(self):
|
||||
self.dialogues = defaultdict(list) # speech name -> [(file, Dialogue)]
|
||||
self.issues = []
|
||||
|
||||
def __call__(self, rel, pkg, start, data):
|
||||
try:
|
||||
for d in scan_package(pkg, data, start):
|
||||
self.dialogues[d.speech].append((rel, d))
|
||||
except (SubtitleError, struct.error, ValueError, IndexError) as e:
|
||||
self.issues.append(("subtitles", "%s: %s" % (rel, e)))
|
||||
|
||||
|
||||
# --- catalog files: SUB_<LANG>.csv (texts) + SUB_locations.csv.gz (every copy) ------------------
|
||||
LOCATIONS = "SUB_locations.csv.gz"
|
||||
TEXT_FIELDS = ["speech", "line", "text", "room", "enc", "sound"]
|
||||
SubLine = namedtuple("SubLine", "lang speech line text room enc sound") # sound: linked WAD sound key
|
||||
SubLoc = namedtuple("SubLoc", "lang speech line file pos size enc")
|
||||
|
||||
|
||||
def key(lang, speech, line):
|
||||
return "SUB:%s:%s:%d" % (lang, speech, line)
|
||||
|
||||
|
||||
def rows(collector, links):
|
||||
"""({lang: [SubLine]}, [SubLoc]) from a Collector; links: {speech: {lang: sound key}}."""
|
||||
texts = defaultdict(list)
|
||||
locs = []
|
||||
for speech in sorted(collector.dialogues):
|
||||
copies = collector.dialogues[speech]
|
||||
rel0, d0 = copies[0]
|
||||
for f in d0.fields:
|
||||
ln = links.get(speech, {})
|
||||
snd = ln.get(f.lang) or ln.get("INT", "")
|
||||
texts[f.lang].append(SubLine(f.lang, speech, f.index, f.text, room(f), f.enc, snd))
|
||||
for rel, d in copies:
|
||||
for f in d.fields:
|
||||
locs.append(SubLoc(f.lang, speech, f.index, rel, f.pos, f.size, f.enc))
|
||||
return dict(texts), locs
|
||||
|
||||
|
||||
def write_texts(path, lines):
|
||||
with open(path + ".tmp", "w", encoding="utf-8", newline="") as fh:
|
||||
w = csv.writer(fh, lineterminator="\n")
|
||||
w.writerow(TEXT_FIELDS)
|
||||
for s in lines:
|
||||
w.writerow([s.speech, s.line, s.text, s.room, s.enc, s.sound])
|
||||
os.replace(path + ".tmp", path)
|
||||
|
||||
|
||||
def read_texts(path, lang):
|
||||
with open(path, encoding="utf-8", newline="") as fh:
|
||||
return [SubLine(lang, r["speech"], int(r["line"]), r["text"], int(r["room"]), r["enc"], r["sound"])
|
||||
for r in csv.DictReader(fh)]
|
||||
|
||||
|
||||
def write_locations(path, locs):
|
||||
raw = io.StringIO()
|
||||
w = csv.writer(raw, lineterminator="\n")
|
||||
w.writerow(["lang", "speech", "line", "file", "pos", "size", "enc"])
|
||||
for l in sorted(locs, key=lambda l: (l.lang, l.speech, l.line, l.file, l.pos)):
|
||||
w.writerow([l.lang, l.speech, l.line, l.file, "%X" % l.pos, l.size, l.enc])
|
||||
with open(path + ".tmp", "wb") as fh:
|
||||
with gzip.GzipFile(fileobj=fh, mode="wb", mtime=0) as gz:
|
||||
gz.write(raw.getvalue().encode("utf-8"))
|
||||
os.replace(path + ".tmp", path)
|
||||
|
||||
|
||||
def read_locations(path, lang=None):
|
||||
"""{SUB key: [SubLoc]} (only `lang` if given)."""
|
||||
out = defaultdict(list)
|
||||
with gzip.open(path, "rt", encoding="utf-8", newline="") as fh:
|
||||
for r in csv.DictReader(fh):
|
||||
if lang and r["lang"] != lang:
|
||||
continue
|
||||
l = SubLoc(r["lang"], r["speech"], int(r["line"]), r["file"], int(r["pos"], 16), int(r["size"]), r["enc"])
|
||||
out[key(l.lang, l.speech, l.line)].append(l)
|
||||
return dict(out)
|
||||
|
||||
|
||||
def scan_package(pkg, data=None, start=None, tables=None):
|
||||
"""[Dialogue] of one package (empty if it has none)."""
|
||||
if data is None:
|
||||
start, data = pkg.read_all()
|
||||
names, imports, exports = tables or pkg.tables(data, start)
|
||||
cls = {-(i + 1) for i, (c, o, _) in enumerate(imports) if c == "Class" and o == CLASS}
|
||||
if not cls or "LocalizedSubtitles" not in names or "Subtitles" not in names:
|
||||
return []
|
||||
idx = (names.index("LocalizedSubtitles"), names.index("Subtitles"))
|
||||
out = []
|
||||
for e in exports:
|
||||
if e.cls in cls and e.size > 0:
|
||||
b = data[e.offset - start:e.offset - start + e.size]
|
||||
fields = parse_dialogue(b, e.offset, idx)
|
||||
if fields:
|
||||
out.append(Dialogue(_speech_ref(b, exports, imports), e.name, fields))
|
||||
return out
|
||||
222
ackit/texts.py
Normal file
222
ackit/texts.py
Normal file
@@ -0,0 +1,222 @@
|
||||
"""Subtitles and menu text: reading, states, apply and revert (shared by the CLI and the editor).
|
||||
|
||||
Keys (also used in mod packages):
|
||||
SUB:<LANG>:<speech event>:<line> a subtitle string (docs/formats/subtitles.md)
|
||||
MENU:<LANG>:<File>:<Section>:<Key>[#n] a menu/UI text (docs/formats/text.md)
|
||||
Records live in the originals store (data["texts"]): {key: {"orig", "applied", "files", ...}}. The files
|
||||
holding the texts are derived files of the Workspace: each is rebuilt from its original copy plus every edit
|
||||
applied now (sounds and texts), so reverting the last edit restores it byte-for-byte.
|
||||
"""
|
||||
import os
|
||||
import time
|
||||
|
||||
from . import loctext, subtitles
|
||||
from .catalog import load_subtitles
|
||||
from .game import GameError
|
||||
|
||||
ORIGINAL, MODIFIED, LOST, CHANGED, FOREIGN = "original", "modified", "lost", "changed", "foreign"
|
||||
|
||||
|
||||
class TextConflictError(GameError):
|
||||
def __init__(self, keys):
|
||||
self.keys = keys
|
||||
super().__init__("These texts were changed outside this tool (Steam update or 'verify', or another mod "
|
||||
"tool): %s.\nRevert them or confirm to overwrite." % ", ".join(keys[:5])
|
||||
+ (" ..." if len(keys) > 5 else ""))
|
||||
|
||||
|
||||
class Texts:
|
||||
def __init__(self, ws):
|
||||
self.ws = ws
|
||||
self._subs = {} # lang -> ([SubLine], {key: [SubLoc]})
|
||||
self._menu = {} # lang -> [Entry]
|
||||
self.loc_root = _find_dir(ws.game.root, "BmGame", "Localization")
|
||||
|
||||
# --- languages and lists ---------------------------------------------------------------------
|
||||
def menu_languages(self):
|
||||
return loctext.languages(self.loc_root) if self.loc_root else []
|
||||
|
||||
def subtitle_languages(self):
|
||||
from .catalog import _shipped_index
|
||||
langs = sorted(_shipped_index().get("SUB", {}).get("lines", {}))
|
||||
return langs or self.menu_languages()
|
||||
|
||||
def subtitles(self, lang, progress=None):
|
||||
"""[SubLine] of a language (original texts from the catalog; use `current` for the game's text)."""
|
||||
if lang not in self._subs:
|
||||
self._subs[lang] = load_subtitles(self.ws.game, lang, progress)
|
||||
return self._subs[lang][0]
|
||||
|
||||
def subtitle_locations(self, key):
|
||||
lang = key.split(":")[1]
|
||||
self.subtitles(lang)
|
||||
return self._subs[lang][1].get(key, [])
|
||||
|
||||
def for_sound(self, sound):
|
||||
"""Subtitle lines of a speech sound in its own language (or the INT line for other languages)."""
|
||||
lang = sound.archive if sound.archive in self.subtitle_languages() else None
|
||||
if not lang:
|
||||
return []
|
||||
return [s for s in self.subtitles(lang) if s.sound == sound.key]
|
||||
|
||||
def menu(self, lang):
|
||||
"""[loctext.Entry] of the game's current files (the values the game shows now)."""
|
||||
if lang not in self._menu:
|
||||
self._menu[lang] = loctext.load(self.loc_root, lang) if self.loc_root else []
|
||||
return self._menu[lang]
|
||||
|
||||
def is_menu_file(self, path):
|
||||
return bool(self.loc_root) and os.path.normcase(os.path.abspath(path)).startswith(
|
||||
os.path.normcase(os.path.abspath(self.loc_root)) + os.sep)
|
||||
|
||||
# --- values and states ----------------------------------------------------------------------
|
||||
def records(self):
|
||||
return self.ws.store.data.setdefault("texts", {})
|
||||
|
||||
def modified(self):
|
||||
return [k for k, r in self.records().items() if r.get("applied") is not None]
|
||||
|
||||
def _menu_entry(self, key):
|
||||
lang = key.split(":")[1]
|
||||
raw = key[len("MENU:"):]
|
||||
for e in loctext.load(self.loc_root, lang):
|
||||
if e.key == raw:
|
||||
return e
|
||||
raise GameError("no menu text %s" % raw)
|
||||
|
||||
def files(self, key):
|
||||
"""Absolute paths of the files holding a text."""
|
||||
if key.startswith("SUB:"):
|
||||
return sorted({self.ws.bank_path(l.file) for l in self.subtitle_locations(key)})
|
||||
e = self._menu_entry(key)
|
||||
return [loctext.files(self.loc_root, e.lang)[e.file]]
|
||||
|
||||
def current(self, key):
|
||||
"""The text in the game files now."""
|
||||
if key.startswith("SUB:"):
|
||||
locs = self.subtitle_locations(key)
|
||||
if not locs:
|
||||
raise GameError("no subtitle %s" % key)
|
||||
from .banks import open_file
|
||||
l = locs[0]
|
||||
b = open_file(self.ws.bank_path(l.file)).read(l.pos, l.size)
|
||||
return subtitles.read_fstring(b, 0)[0]
|
||||
return self._menu_entry(key).value
|
||||
|
||||
def catalog_text(self, key):
|
||||
"""Original text according to the shipped catalog (subtitles only), or None."""
|
||||
if not key.startswith("SUB:"):
|
||||
return None
|
||||
_, lang, rest = key.split(":", 2)
|
||||
speech, line = rest.rsplit(":", 1)
|
||||
for s in self.subtitles(lang):
|
||||
if s.speech == speech and s.line == int(line):
|
||||
return s.text
|
||||
return None
|
||||
|
||||
def original(self, key):
|
||||
r = self.records().get(key)
|
||||
return r["orig"] if r else self.current(key)
|
||||
|
||||
def state(self, key, current=None):
|
||||
cur = self.current(key) if current is None else current
|
||||
r = self.records().get(key)
|
||||
if r:
|
||||
if r.get("applied") is not None and cur == r["applied"]:
|
||||
return MODIFIED
|
||||
if cur == r["orig"]:
|
||||
return LOST if r.get("applied") is not None else ORIGINAL
|
||||
return CHANGED
|
||||
cat = self.catalog_text(key)
|
||||
if cat is not None and cur != cat:
|
||||
return FOREIGN
|
||||
return ORIGINAL
|
||||
|
||||
def limit(self, key):
|
||||
"""Maximum characters for a subtitle (None = no limit: menu text)."""
|
||||
if not key.startswith("SUB:"):
|
||||
return None
|
||||
return min((subtitles.room(subtitles.Field(l.lang, l.line, "", l.pos, l.size, l.enc))
|
||||
for l in self.subtitle_locations(key)), default=0)
|
||||
|
||||
def check(self, key, text):
|
||||
"""Raise SubtitleError / TextError if `text` cannot be written for `key`."""
|
||||
if key.startswith("SUB:"):
|
||||
for l in self.subtitle_locations(key):
|
||||
subtitles.encode_fstring(text, l.size, l.enc)
|
||||
else:
|
||||
e = self._menu_entry(key)
|
||||
path = loctext.files(self.loc_root, e.lang)[e.file]
|
||||
loctext.write(open(path, "rb").read(), e.lang, e.file, {e.key: text})
|
||||
|
||||
# --- writing --------------------------------------------------------------------------------
|
||||
def apply(self, changes, force=False, progress=None, meta=None):
|
||||
"""changes: {key: new text}. Every file holding one of them is rebuilt once."""
|
||||
for k, v in changes.items():
|
||||
self.check(k, v)
|
||||
conflicts = [k for k in changes if self.state(k) in (CHANGED, FOREIGN)]
|
||||
paths = {p for k in changes for p in self.files(k)}
|
||||
changed = set(self.ws.changed_files(paths))
|
||||
conflicts += [k for k in changes if k not in conflicts and set(self.files(k)) & changed]
|
||||
if conflicts and not force:
|
||||
raise TextConflictError(conflicts)
|
||||
for p in paths:
|
||||
self.ws._keep_original(p)
|
||||
recs = self.records()
|
||||
now = time.time()
|
||||
for k, v in changes.items():
|
||||
if k not in recs:
|
||||
recs[k] = {"orig": self.current(k), "files": [self.ws.game.rel(p) for p in self.files(k)],
|
||||
"first_modified": now, "applied": None}
|
||||
recs[k].update(applied=v, applied_at=now, meta=meta or {})
|
||||
self.ws.store.save()
|
||||
self._sync(paths, progress)
|
||||
|
||||
def revert(self, keys, progress=None):
|
||||
recs = self.records()
|
||||
keys = [k for k in keys if k in recs]
|
||||
paths = {p for k in keys for p in self.files(k)}
|
||||
for k in keys:
|
||||
recs.pop(k)
|
||||
self.ws.store.save()
|
||||
self._sync(paths, progress)
|
||||
return keys
|
||||
|
||||
def revert_all(self, progress=None):
|
||||
return self.revert(list(self.records()), progress)
|
||||
|
||||
def _sync(self, paths, progress):
|
||||
self._menu.clear()
|
||||
self.ws.sync_files(paths, progress)
|
||||
|
||||
# --- derived-file hooks (called by Workspace.sync_files) ------------------------------------
|
||||
def subtitle_edits(self, rel):
|
||||
out = []
|
||||
for k, r in self.records().items():
|
||||
if r.get("applied") is None or not k.startswith("SUB:"):
|
||||
continue
|
||||
for l in self.subtitle_locations(k):
|
||||
if l.file == rel:
|
||||
out.append((l.pos, subtitles.encode_fstring(r["applied"], l.size, l.enc)))
|
||||
return out
|
||||
|
||||
def menu_content(self, path, orig_raw):
|
||||
fname = os.path.basename(path)
|
||||
lang = os.path.basename(os.path.dirname(path)).upper()
|
||||
values = {}
|
||||
for k, r in self.records().items():
|
||||
if r.get("applied") is None or not k.startswith("MENU:"):
|
||||
continue
|
||||
_, lg, f, _rest = k.split(":", 3)
|
||||
if lg == lang and f == fname:
|
||||
values[k[len("MENU:"):]] = r["applied"]
|
||||
return loctext.write(orig_raw, lang, fname, values) if values else orig_raw
|
||||
|
||||
|
||||
def _find_dir(root, *parts):
|
||||
d = root
|
||||
for p in parts:
|
||||
if not d or not os.path.isdir(d):
|
||||
return None
|
||||
d = next((os.path.join(d, x) for x in os.listdir(d) if x.lower() == p.lower()), None)
|
||||
return d if d and os.path.isdir(d) else None
|
||||
42
ackit/upk.py
42
ackit/upk.py
@@ -8,7 +8,7 @@ Patching rewrites only the blocks that contain changed bytes (stored as one LZO
|
||||
lzo.compress_stored) and moves the chunks after them; every other byte of the file stays identical.
|
||||
"""
|
||||
import struct
|
||||
from collections import OrderedDict
|
||||
from collections import OrderedDict, namedtuple
|
||||
|
||||
from . import lzo
|
||||
|
||||
@@ -20,6 +20,9 @@ class PackageError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
Export = namedtuple("Export", "index cls outer name size offset entry") # entry: table position (uncompressed)
|
||||
|
||||
|
||||
class Block:
|
||||
__slots__ = ("chunk", "uoff", "usize", "coff", "csize")
|
||||
|
||||
@@ -52,6 +55,7 @@ class Package:
|
||||
f12 = struct.unpack_from("<12i", b, o)
|
||||
self.name_count, self.name_offset = f12[1], f12[2]
|
||||
self.export_count, self.export_offset = f12[3], f12[4]
|
||||
self.import_count, self.import_offset = f12[5], f12[6]
|
||||
o += 48
|
||||
self.guid = b[o:o + 16]
|
||||
o += 16
|
||||
@@ -106,6 +110,42 @@ class Package:
|
||||
return self.file_size
|
||||
return self.chunks[-1][0] + self.chunks[-1][1]
|
||||
|
||||
# tables (need the uncompressed data: `read_all`) ---------------------------------------------
|
||||
def tables(self, data=None, start=None):
|
||||
"""(names, imports, exports) of the package. imports: [(class name, object name, outer)],
|
||||
exports: [Export]. Layouts: docs/formats/upk.md (licensee 101 adds one u32 to each export)."""
|
||||
if data is None:
|
||||
start, data = self.read_all()
|
||||
buf = memoryview(data)
|
||||
names = []
|
||||
o = self.name_offset - start
|
||||
for _ in range(self.name_count):
|
||||
n = struct.unpack_from("<i", buf, o)[0]
|
||||
o += 4
|
||||
if n >= 0:
|
||||
names.append(bytes(buf[o:o + n - 1]).decode("latin-1"))
|
||||
o += n
|
||||
else:
|
||||
names.append(bytes(buf[o:o - 2 * n - 2]).decode("utf-16-le", "replace"))
|
||||
o += -2 * n
|
||||
o += 8 # name flags
|
||||
imports = []
|
||||
o = self.import_offset - start
|
||||
for _ in range(self.import_count):
|
||||
_pkg, _pn, cls, _cn, outer, nm, nn = struct.unpack_from("<7i", buf, o)
|
||||
imports.append((names[cls], names[nm], outer))
|
||||
o += 28
|
||||
exports = []
|
||||
o = self.export_offset - start
|
||||
for i in range(self.export_count):
|
||||
cls, sup, outer, nm, nn, arch, _x = struct.unpack_from("<7i", buf, o)
|
||||
size, off = struct.unpack_from("<ii", buf, o + 36)
|
||||
ngen = struct.unpack_from("<i", buf, o + 48)[0]
|
||||
exports.append(Export(i + 1, cls, outer, names[nm] + ("_%d" % (nn - 1) if nn else ""), size, off,
|
||||
o + start))
|
||||
o += 72 + 4 * ngen
|
||||
return names, imports, exports
|
||||
|
||||
def fingerprint(self):
|
||||
"""Cheap identity of the package contents: guid + chunk table (changes with any re-cook/patch)."""
|
||||
import hashlib
|
||||
|
||||
@@ -16,7 +16,7 @@ import hashlib
|
||||
import os
|
||||
import random
|
||||
|
||||
from .. import banks, classify
|
||||
from .. import banks, classify, subtitles
|
||||
from ..catalog import (Catalog, available_archives, bank_sounds, hash_names, head_owners, is_shipped, load_characters,
|
||||
scan_archive)
|
||||
from ..inject import read_target
|
||||
@@ -102,7 +102,9 @@ def check_banks(rep, game, rows, locations, characters, quick=False, sample=40,
|
||||
pk = [f for f in files if f.lower().endswith(".upk")]
|
||||
rnd = random.Random(1)
|
||||
files = [f for f in files if f.startswith("SFX/")] + sorted(rnd.sample(pk, min(sample, len(pk))))
|
||||
sc = banks.scan(game.cooked, progress, files=files)
|
||||
col = subtitles.Collector()
|
||||
sc = banks.scan(game.cooked, progress, files=files, visit=col)
|
||||
check_subtitles(rep, game, col, set(files), quick)
|
||||
owners = head_owners(game, sc)
|
||||
index = collections.defaultdict(set)
|
||||
for m, ls in locations.items():
|
||||
@@ -135,6 +137,44 @@ def check_banks(rep, game, rows, locations, characters, quick=False, sample=40,
|
||||
rep.add(banks.ARCHIVE, "copies", "%08X" % m, "prefetch head without a speech line")
|
||||
|
||||
|
||||
def check_subtitles(rep, game, col, files, quick):
|
||||
"""Subtitle copies found in the scanned packages vs SUB_locations + the SUB_<LANG>.csv texts."""
|
||||
from ..catalog import _shipped_index, catalog_dir
|
||||
loc_path = os.path.join(catalog_dir(), subtitles.LOCATIONS)
|
||||
if not os.path.exists(loc_path) or not is_shipped(game, banks.ARCHIVE):
|
||||
return
|
||||
index = collections.defaultdict(set)
|
||||
for k, ls in subtitles.read_locations(loc_path).items():
|
||||
for l in ls:
|
||||
if l.file in files:
|
||||
index[k].add((l.file, l.pos, l.size))
|
||||
found, texts = collections.defaultdict(set), {}
|
||||
for speech, copies in col.dialogues.items():
|
||||
for rel, d in copies:
|
||||
for f in d.fields:
|
||||
k = subtitles.key(f.lang, speech, f.index)
|
||||
found[k].add((rel, f.pos, f.size))
|
||||
texts[k] = f.text
|
||||
for k in sorted(set(found) | set(index)):
|
||||
if found.get(k, set()) != index.get(k, set()):
|
||||
rep.add("SUB", "copies", k, "index %d, game %d" % (len(index.get(k, ())), len(found.get(k, ()))))
|
||||
langs = sorted(_shipped_index().get("SUB", {}).get("lines", {}))
|
||||
for lang in langs:
|
||||
p = os.path.join(catalog_dir(), "SUB_%s.csv" % lang)
|
||||
rows = subtitles.read_texts(p, lang)
|
||||
rep.checked["SUB_" + lang] += len(rows)
|
||||
seen = set()
|
||||
for s in rows:
|
||||
k = subtitles.key(lang, s.speech, s.line)
|
||||
if k in seen:
|
||||
rep.add("SUB", "duplicate", k)
|
||||
seen.add(k)
|
||||
if k in texts and texts[k] != s.text:
|
||||
rep.add("SUB", "text", k, "catalog %r, game %r" % (s.text[:40], texts[k][:40]))
|
||||
if s.sound and not s.sound.startswith(lang + ":") and not s.sound.startswith("INT:"):
|
||||
rep.add("SUB", "language", k, "linked to %s" % s.sound)
|
||||
|
||||
|
||||
def run(game, archives=None, quick=False, log=print, progress=None):
|
||||
"""Check the given archives (default: all). Returns a Report."""
|
||||
rep = Report()
|
||||
|
||||
@@ -22,6 +22,7 @@ from .game import Game, GameError
|
||||
from .inject import inject, read_target, parse_target, pack_bnk_path, SizeLimitError
|
||||
from .originals import OriginalsStore, sha256, sha1_16
|
||||
from .pack import Pack
|
||||
from .texts import Texts
|
||||
from .paths import user_dir
|
||||
from .wad import Wad
|
||||
|
||||
@@ -51,6 +52,7 @@ class Workspace:
|
||||
self.store = OriginalsStore(store_root or default_store_root(self.game), self.game.root)
|
||||
self.catalog = Catalog(self.game, archives, progress)
|
||||
self.full_backup = full_backup
|
||||
self.texts = Texts(self)
|
||||
|
||||
# reading -----------------------------------------------------------------------------------
|
||||
def sound(self, name_or_key):
|
||||
@@ -153,9 +155,9 @@ class Workspace:
|
||||
"""items: [(sound, new_bytes, meta, is_revert)] all in one archive."""
|
||||
if archive == banks.ARCHIVE:
|
||||
return self._write_banks(items, progress)
|
||||
heads = sorted({l.file for s, _, _, _ in items for l in self.locations(s)})
|
||||
for rel in heads:
|
||||
self._keep_original_file(rel)
|
||||
heads = sorted({self.bank_path(l.file) for s, _, _, _ in items for l in self.locations(s)})
|
||||
for path in heads:
|
||||
self._keep_original(path)
|
||||
wad = self.game.wad(archive)
|
||||
self.game.check_writable(wad)
|
||||
packs = {parse_target(s.target)[0] for s, _, _, _ in items if s.is_pack_media}
|
||||
@@ -181,43 +183,44 @@ class Workspace:
|
||||
else:
|
||||
self.store.record_applied(s.key, data, meta)
|
||||
self.store.save()
|
||||
self._sync_bank_files(heads, progress)
|
||||
self.sync_files(heads, progress)
|
||||
|
||||
# bank files (in-memory sounds, prefetch heads) ----------------------------------------------
|
||||
def _bank_files(self):
|
||||
return self.store.data.setdefault("bank_files", {})
|
||||
# derived files: bank files and packages (in-memory sounds, prefetch heads, subtitles) and the
|
||||
# localization files (menu text). Each is rebuilt from its original copy + every edit applied now.
|
||||
def _derived(self):
|
||||
return self.store.data.setdefault("derived", {})
|
||||
|
||||
def _fingerprint(self, path):
|
||||
"""Content hash of a bank file (whole file: any change by another tool is detected)."""
|
||||
"""Content hash of a file (whole file: any change by another tool is detected)."""
|
||||
h = hashlib.sha1()
|
||||
with open(path, "rb") as f:
|
||||
for c in iter(lambda: f.read(1 << 22), b""):
|
||||
h.update(c)
|
||||
return h.hexdigest()
|
||||
|
||||
def _keep_original_file(self, rel):
|
||||
"""Before the first change to a bank file: verified full copy + its fingerprint."""
|
||||
rec = self._bank_files().get(rel)
|
||||
def _keep_original(self, path):
|
||||
"""Before the first change to a derived file: verified full copy + its fingerprint."""
|
||||
rel = self.game.rel(path)
|
||||
rec = self._derived().get(rel)
|
||||
if rec:
|
||||
return rec
|
||||
path = self.bank_path(rel)
|
||||
self.game.check_writable(path)
|
||||
self.store.full_backup(path, self.game.rel(path))
|
||||
rec = self._bank_files()[rel] = {"orig_fp": self._fingerprint(path), "written_fp": None}
|
||||
self.store.full_backup(path, rel)
|
||||
rec = self._derived()[rel] = {"orig_fp": self._fingerprint(path), "written_fp": None}
|
||||
self.store.save()
|
||||
return rec
|
||||
|
||||
def changed_files(self, sounds):
|
||||
"""Bank files of `sounds` changed since we last wrote them (Steam update/verify, other tool)."""
|
||||
def changed_files(self, paths):
|
||||
"""Derived files changed since we last wrote them (Steam update/verify, another tool)."""
|
||||
out = []
|
||||
for rel in sorted({l.file for s in sounds for l in self.locations(s)}):
|
||||
rec = self._bank_files().get(rel)
|
||||
if rec and self._fingerprint(self.bank_path(rel)) not in (rec["orig_fp"], rec["written_fp"]):
|
||||
out.append(rel)
|
||||
for path in sorted(set(paths)):
|
||||
rec = self._derived().get(self.game.rel(path))
|
||||
if rec and self._fingerprint(path) not in (rec["orig_fp"], rec["written_fp"]):
|
||||
out.append(path)
|
||||
return out
|
||||
|
||||
def _edits(self, rel):
|
||||
"""[(offset, bytes)] that the replacements applied now need in bank file `rel`."""
|
||||
def _audio_edits(self, rel):
|
||||
"""[(offset, bytes)] that the sound replacements applied now need in bank file `rel` (cooked-relative)."""
|
||||
out = []
|
||||
for key, e in self.store.entries.items():
|
||||
if not e.get("applied"):
|
||||
@@ -235,24 +238,31 @@ class Workspace:
|
||||
out.append((l.offset, payload))
|
||||
return out
|
||||
|
||||
def _sync_bank_files(self, rels, progress=None):
|
||||
"""Rebuild each bank file from its original copy and the replacements applied now."""
|
||||
for rel in rels:
|
||||
path = self.bank_path(rel)
|
||||
rec = self._keep_original_file(rel)
|
||||
def _build(self, path, orig):
|
||||
"""Content of derived file `path` from its original copy `orig`."""
|
||||
if self.texts.is_menu_file(path):
|
||||
return self.texts.menu_content(path, open(orig, "rb").read())
|
||||
rel = os.path.relpath(path, self.game.cooked).replace(os.sep, "/")
|
||||
edits = self._audio_edits(rel) + self.texts.subtitle_edits(rel)
|
||||
return banks.open_file(orig).patched(edits) if edits else open(orig, "rb").read()
|
||||
|
||||
def sync_files(self, paths, progress=None):
|
||||
"""Rebuild each derived file from its original copy and the edits applied now (atomic writes)."""
|
||||
for path in sorted(set(paths)):
|
||||
rec = self._keep_original(path)
|
||||
orig = self.store.full_copy(self.game.rel(path))
|
||||
if orig is None:
|
||||
raise GameError("the original copy of %s is missing from the store" % rel)
|
||||
raise GameError("the original copy of %s is missing from the store" % self.game.rel(path))
|
||||
if progress:
|
||||
progress("writing %s" % rel)
|
||||
edits = self._edits(rel)
|
||||
data = banks.open_file(orig).patched(edits) if edits else open(orig, "rb").read()
|
||||
progress("writing %s" % os.path.basename(path))
|
||||
data = self._build(path, orig)
|
||||
self.game.check_writable(path)
|
||||
tmp = path + ".ackit-tmp"
|
||||
with open(tmp, "wb") as f:
|
||||
f.write(data)
|
||||
os.replace(tmp, path)
|
||||
rec["written_fp"] = self._fingerprint(path) if edits else None
|
||||
same = data == open(orig, "rb").read()
|
||||
rec["written_fp"] = None if same else self._fingerprint(path)
|
||||
self.store.save()
|
||||
|
||||
def _write_banks(self, items, progress=None):
|
||||
@@ -262,9 +272,9 @@ class Workspace:
|
||||
raise GameError("no copy of %s was found in the game's banks" % s.name)
|
||||
if not is_revert and len(data) > min(l.size for l in locs):
|
||||
raise SizeLimitError(s.target, min(l.size for l in locs), len(data))
|
||||
files = sorted({l.file for s, _, _, _ in items for l in self.locations(s)})
|
||||
for rel in files:
|
||||
self._keep_original_file(rel)
|
||||
files = sorted({self.bank_path(l.file) for s, _, _, _ in items for l in self.locations(s)})
|
||||
for path in files:
|
||||
self._keep_original(path)
|
||||
for s, data, meta, is_revert in items:
|
||||
if is_revert:
|
||||
self.store.forget(s.key)
|
||||
@@ -273,7 +283,7 @@ class Workspace:
|
||||
self.current_bytes(s))
|
||||
self.store.record_applied(s.key, data, meta)
|
||||
self.store.save() # the store says what the files must contain before they change
|
||||
self._sync_bank_files(files, progress)
|
||||
self.sync_files(files, progress)
|
||||
|
||||
def apply(self, changes, force=False, progress=None):
|
||||
"""changes: [(sound, wem_bytes, meta dict)]. One atomic rewrite per WAD / bank file."""
|
||||
@@ -281,9 +291,9 @@ class Workspace:
|
||||
for s, _, _ in changes:
|
||||
if self.state(s) in (CHANGED, FOREIGN):
|
||||
conflicts.append(s)
|
||||
changed = set(self.changed_files([s for s, _, _ in changes]))
|
||||
changed = set(self.changed_files([self.bank_path(l.file) for s, _, _ in changes for l in self.locations(s)]))
|
||||
conflicts += [s for s, _, _ in changes
|
||||
if s not in conflicts and {l.file for l in self.locations(s)} & changed]
|
||||
if s not in conflicts and {self.bank_path(l.file) for l in self.locations(s)} & changed]
|
||||
if conflicts and not force:
|
||||
raise ConflictError(conflicts)
|
||||
by_arch = defaultdict(list)
|
||||
@@ -328,8 +338,9 @@ class Workspace:
|
||||
shutil.copy2(src, tmp)
|
||||
os.replace(tmp, dst)
|
||||
out.append(rel)
|
||||
for rec in self._bank_files().values():
|
||||
for rec in self._derived().values():
|
||||
rec["written_fp"] = None
|
||||
self.store.data["texts"] = {}
|
||||
self.store.data["entries"] = {}
|
||||
self.store.save()
|
||||
self.store.gc()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -4,6 +4,8 @@
|
||||
|------|---------|
|
||||
| `<ARCHIVE>.csv` | every sound of one audio archive: `ITA`, `INT`, `FRA`, `DEU`, `ESN` (speech WADs, one per language), `SFX` (music, ambience and effects streamed from the SFX WAD) and `BANK` (in-memory sounds stored inside soundbanks: footsteps, foley, combat, weapons, UI, efforts; `docs/formats/banks.md`) |
|
||||
| `BANK_locations.csv.gz` | every stored copy of the `BANK` sounds, and the prefetch heads of speech lines (below) |
|
||||
| `SUB_<LANG>.csv` | subtitles of the speech lines, one file per language (11 languages; `docs/formats/subtitles.md`) |
|
||||
| `SUB_locations.csv.gz` | every stored copy of each subtitle string (dialogue objects in the packages) |
|
||||
| `characters.csv` | `name,character`: verified speaker per sound name (language independent) |
|
||||
| `index.json` | per archive: fingerprint of the game files the CSV was built from (`ackit/game.py: wad_fingerprint`, `ackit/banks.py: game_fingerprint`) and counts |
|
||||
| `source/<lang>/*.txt` | **the curated original lists** (one file per character, one wemscan-style file name per line, e.g. `Speech_TAPE_JOKER_1_4.wem`). `Files.txt` = all files, `Stats.txt` = counts, `Checkpoint.txt` = progress marker. Never edited by tools. |
|
||||
@@ -40,6 +42,19 @@ All CSVs are UTF-8 with a header row. They contain only names and metadata (no g
|
||||
A replacement is written into every `m` copy of a BANK sound; replacing a speech line also rewrites its `p`
|
||||
heads (`docs/formats/banks.md`).
|
||||
|
||||
## `SUB_<LANG>.csv` columns
|
||||
| column | meaning |
|
||||
|--------|---------|
|
||||
| `speech` | the speech event of the line (e.g. `Speech_CHURCH_B1_Ch2_POST_Explosion_BM_Cash_04`) |
|
||||
| `line` | 0-based line of a multi-line subtitle |
|
||||
| `text` | original subtitle text |
|
||||
| `room` | maximum length of a replacement in characters (it is written in place) |
|
||||
| `enc` | `a` = cp1252 string, `w` = UTF-16 string |
|
||||
| `sound` | key of the linked speech sound (this language's WAD, or `INT:` for languages without speech WADs); empty when no event reaches a sound |
|
||||
|
||||
`SUB_locations.csv.gz`: `lang, speech, line, file, pos` (hex, uncompressed offset of the FString), `size`, `enc`.
|
||||
Menu text has no catalog: the localization files are read directly (`docs/formats/text.md`).
|
||||
|
||||
The editor uses these files directly when the installed game matches `index.json`; for another game build it
|
||||
scans the game itself and applies the same rules (no checksums then; the bank scan takes a few minutes).
|
||||
`ackit verify --catalog` cross-checks the catalog in use against the game files (`--quick` samples packages).
|
||||
|
||||
9639
data/catalog/SUB_DEU.csv
Normal file
9639
data/catalog/SUB_DEU.csv
Normal file
File diff suppressed because it is too large
Load Diff
9477
data/catalog/SUB_ESM.csv
Normal file
9477
data/catalog/SUB_ESM.csv
Normal file
File diff suppressed because it is too large
Load Diff
9469
data/catalog/SUB_ESN.csv
Normal file
9469
data/catalog/SUB_ESN.csv
Normal file
File diff suppressed because it is too large
Load Diff
9541
data/catalog/SUB_FRA.csv
Normal file
9541
data/catalog/SUB_FRA.csv
Normal file
File diff suppressed because it is too large
Load Diff
9380
data/catalog/SUB_INT.csv
Normal file
9380
data/catalog/SUB_INT.csv
Normal file
File diff suppressed because it is too large
Load Diff
9353
data/catalog/SUB_ITA.csv
Normal file
9353
data/catalog/SUB_ITA.csv
Normal file
File diff suppressed because it is too large
Load Diff
9017
data/catalog/SUB_JPN.csv
Normal file
9017
data/catalog/SUB_JPN.csv
Normal file
File diff suppressed because it is too large
Load Diff
9230
data/catalog/SUB_KOR.csv
Normal file
9230
data/catalog/SUB_KOR.csv
Normal file
File diff suppressed because it is too large
Load Diff
9353
data/catalog/SUB_POL.csv
Normal file
9353
data/catalog/SUB_POL.csv
Normal file
File diff suppressed because it is too large
Load Diff
9451
data/catalog/SUB_POR.csv
Normal file
9451
data/catalog/SUB_POR.csv
Normal file
File diff suppressed because it is too large
Load Diff
9470
data/catalog/SUB_RUS.csv
Normal file
9470
data/catalog/SUB_RUS.csv
Normal file
File diff suppressed because it is too large
Load Diff
BIN
data/catalog/SUB_locations.csv.gz
Normal file
BIN
data/catalog/SUB_locations.csv.gz
Normal file
Binary file not shown.
@@ -73,7 +73,7 @@
|
||||
},
|
||||
"copies": 93910,
|
||||
"files": 1499,
|
||||
"named_by_event": 6416,
|
||||
"named_by_event": 6456,
|
||||
"prefetch_heads": {
|
||||
"lines": 396,
|
||||
"copies": 2101,
|
||||
@@ -81,5 +81,37 @@
|
||||
},
|
||||
"issues": []
|
||||
},
|
||||
"SUB": {
|
||||
"fingerprint": "3f9766af425b8eef",
|
||||
"lines": {
|
||||
"DEU": 9638,
|
||||
"ESM": 9476,
|
||||
"ESN": 9468,
|
||||
"FRA": 9540,
|
||||
"INT": 9379,
|
||||
"ITA": 9352,
|
||||
"JPN": 9016,
|
||||
"KOR": 9229,
|
||||
"POL": 9352,
|
||||
"POR": 9450,
|
||||
"RUS": 9469
|
||||
},
|
||||
"speech_lines": 9016,
|
||||
"linked": {
|
||||
"DEU": 8884,
|
||||
"ESM": 8712,
|
||||
"ESN": 8741,
|
||||
"FRA": 8790,
|
||||
"INT": 8637,
|
||||
"ITA": 8623,
|
||||
"JPN": 8318,
|
||||
"KOR": 8510,
|
||||
"POL": 8613,
|
||||
"POR": 8700,
|
||||
"RUS": 8724
|
||||
},
|
||||
"copies": 168122,
|
||||
"issues": []
|
||||
},
|
||||
"_info": "unmodified game files"
|
||||
}
|
||||
@@ -8,6 +8,8 @@
|
||||
| [building.md](building.md) | building the editor and its installer (Windows, Linux) |
|
||||
| [formats/wad.md](formats/wad.md) | WWAD archives, grouped entries, BKPK stream packs |
|
||||
| [formats/banks.md](formats/banks.md) | soundbanks and the in-memory sounds not covered yet (footsteps, foley, effects) |
|
||||
| [formats/subtitles.md](formats/subtitles.md) | subtitles: dialogue objects in the packages, link to the speech sounds |
|
||||
| [formats/text.md](formats/text.md) | menu and UI text: the localization files |
|
||||
| [formats/upk.md](formats/upk.md) | UE3 packages: summary, LZO chunks, how ackit reads and patches them |
|
||||
| [formats/wem.md](formats/wem.md) | WEM layout the game accepts, setup stripping, hash and decode-alloc formulas |
|
||||
| [reverse-engineering/decoder.md](reverse-engineering/decoder.md) | BatmanAC.exe decoder functions and addresses |
|
||||
|
||||
@@ -79,6 +79,19 @@ tables): high risk for little gain, not done.
|
||||
packages' name tables (FNV-1 of the name = event ID), following event -> action -> containers -> sounds
|
||||
(6,416 sounds); the rest are numbered per bank. Categories use the bank name.
|
||||
|
||||
## 4c. Subtitles and menu text
|
||||
- **Subtitles** are written in place into every copy of the dialogue object, NUL-padded inside the original
|
||||
string (the game stops at the first NUL), so nothing in the package moves; the limit is the original
|
||||
string's length ("room"). Moving the object to the end of the package (new export offset/size and an extra
|
||||
chunk) would lift the limit, but no dialogue is loaded on the title screen, so the result could not be
|
||||
checked automatically in the game: not done.
|
||||
- **Links** subtitle -> speech sound come from the soundbanks (event -> ... -> per-language media -> WAD), not
|
||||
from names: the `_NN` suffixes do not follow the WEM order inside WAD entries.
|
||||
- **Menu text** files are plain UTF-16 text: no size limit; only the edited values change.
|
||||
- Both use the originals store like sounds: texts are recorded as strings (`texts` in store.json) and every
|
||||
file is rebuilt from its original copy + the applied edits (sound and text edits combine in one package).
|
||||
- Keys (`SUB:<LANG>:<speech>:<line>`, `MENU:<LANG>:<File>:<Section>:<Key>`) are stable and used in mod packages.
|
||||
|
||||
## 5. Catalog
|
||||
- Shipped CSVs per archive (names, categories, characters, original size + checksum) make loading instant and
|
||||
enable change detection; any other build is scanned at runtime (header reads only, cached).
|
||||
|
||||
46
docs/formats/subtitles.md
Normal file
46
docs/formats/subtitles.md
Normal file
@@ -0,0 +1,46 @@
|
||||
# Subtitles (`RDialogueEvent` objects in the packages)
|
||||
|
||||
There are no subtitle files: every speech line has a dialogue object (class `RDialogueEvent`, Rocksteady) inside
|
||||
the UE3 packages (`upk.md`), holding the subtitle text of all 11 languages. ackit reads them with
|
||||
`ackit/subtitles.py` and writes them through `Workspace.texts` (`ackit/texts.py`).
|
||||
|
||||
## Inventory (buildid 3718300)
|
||||
- 14,743 dialogue objects in the packages for 9,016 speech lines (a line's object is copied into every package
|
||||
that uses it; all copies are identical).
|
||||
- 11 languages: INT, FRA, ITA, DEU, ESN, ESM, POL, POR, RUS, JPN, KOR (the speech WADs exist only for INT, FRA,
|
||||
ITA, DEU, ESN; the other languages play the INT voices with their own subtitles).
|
||||
- ~9,400 subtitle strings per language: 94 % of the subtitles have one line, the rest up to 5 lines.
|
||||
- Strings are stored as cp1252 (positive FString length) or UTF-16 (negative length): RUS, POL, JPN, KOR
|
||||
and some INT/ESN/FRA/DEU strings are UTF-16.
|
||||
|
||||
## Object layout (compact tagged properties)
|
||||
The export data is a list of properties `u16 type, u16 field offset, value`, ended by `u16 0`. Types seen:
|
||||
`0x10` object reference (i32 export/import index), `0x0D` FString, `0x09` struct/array (name i32+i32 = struct
|
||||
or property name from the name table, i32 byte size, i32 array index, then `size` bytes), `0x04` 4-byte value,
|
||||
`0x03` bool, `0x01` enum. Relevant properties:
|
||||
|
||||
| field | content |
|
||||
|-------|---------|
|
||||
| `0x10` @ `0x2C` | the speech event this line plays, e.g. export `Speech_CHURCH_B1_Ch2_POST_Explosion_BM_Cash_04` |
|
||||
| `0x0D` @ `0x34` | speaker, e.g. `Batman` |
|
||||
| `0x09 LocalizedSubtitles` | u32 count, then per language: `0D 00 00 00` FString language code, `09 00 0C 00 Subtitles` struct, `00 00` |
|
||||
| `Subtitles` | u32 line count, then per line: `0D 00 00 00` FString text, `04 00 0C 00` 4-byte value, `00 00` |
|
||||
|
||||
The sizes add up exactly in all 14,743 objects (checked by the parser: `SubtitleError` otherwise).
|
||||
|
||||
## Link to the speech sound
|
||||
The speech event name is hashed (FNV-1) to the Wwise event ID, followed through the soundbanks (event ->
|
||||
action -> containers -> Sound) to the streamed media, and that media ID is looked up in the per-language
|
||||
BKPK tables (`banks.md`) to find the WAD entry and variation of each language. This is exact; the `_NN`
|
||||
suffix of the event name does **not** match the variation order inside the WAD entry (WEMs are stored in
|
||||
media-ID order). Linked: ~92 % of the strings (e.g. INT 8,637 / 9,379); the rest have no event that reaches a
|
||||
streamed media (unused or placeholder lines such as `..._TEMP_29`) and stay editable without a sound link.
|
||||
Event names with a number (`Speech_X_04`) exist only as export names, not in the name table.
|
||||
|
||||
## How ackit writes them
|
||||
- In place, into **every copy** (`data/catalog/SUB_locations.csv.gz`): the new text replaces the FString's
|
||||
bytes and is padded with NUL characters, keeping the length field. The game stops at the first NUL, so no
|
||||
size field, export or package layout changes. Limit: the new text must fit the original string's bytes
|
||||
(cp1252: one byte per character; UTF-16: two) - shown as "room" in the CLI and the editor.
|
||||
- A cp1252 string can only hold cp1252 characters; UTF-16 strings can hold any character.
|
||||
- Packages are rebuilt from the store's original copy + all applied sound and text edits (`docs/decisions.md`).
|
||||
19
docs/formats/text.md
Normal file
19
docs/formats/text.md
Normal file
@@ -0,0 +1,19 @@
|
||||
# Menu and UI text (`BmGame\Localization\<LANG>\*.<lang>`)
|
||||
|
||||
The menus, HUD, character bios, riddles, credits and other UI strings are UE3 localization files, one folder
|
||||
per language (INT, ITA, FRA, DEU, ESN, ESM, POL, POR, RUS, JPN, KOR), ~37 files and ~7,000 strings each
|
||||
(`FRONTEND`, `GFxUI`, `Bios`, `Credits`, `PC`, `RIDDLER`, ...). ackit reads and writes them with
|
||||
`ackit/loctext.py` through `Workspace.texts`.
|
||||
|
||||
## Format
|
||||
- UTF-16LE with a BOM (`FF FE`), CRLF line ends; the first line is empty. (`INT\Version.int` and
|
||||
`VersionData.int` are plain ASCII build IDs, not UI text: ackit ignores them.)
|
||||
- `[Section]` lines, `Key=Value` lines, `;` comments. A few values are in double quotes (`Quit="Quit game"`).
|
||||
- File extension = language code (`FRONTEND.int`, `FRONTEND.ITA`); case varies, lookups are case-insensitive.
|
||||
- Markers inside values (e.g. `\n`, button tags) are the game's own; ackit keeps them as plain text.
|
||||
|
||||
## Keys and writing
|
||||
- Key: `MENU:<LANG>:<File>:<Section>:<Key>` (`#n` for a repeated key; none in the shipped files).
|
||||
- Writing replaces only the value part of the edited lines (quotes kept); BOM, encoding, other lines and line
|
||||
ends stay byte-identical. Values cannot contain line breaks; characters must fit the file's encoding.
|
||||
- The files are rebuilt from the store's original copy + the applied edits, so reverting restores them exactly.
|
||||
120
tests/synth.py
120
tests/synth.py
@@ -89,10 +89,11 @@ def pack(media):
|
||||
return bytes(hdr) + bank, bytes(payload)
|
||||
|
||||
|
||||
def bank(bank_id, media, name=None, extra_sources=(), prefetch=()):
|
||||
def bank(bank_id, media, name=None, extra_sources=(), prefetch=(), events=()):
|
||||
"""A v56 soundbank holding `media` {id: wem} in memory (DIDX/DATA, 16-byte aligned) with one HIRC
|
||||
Sound object per media; extra_sources: [(media id, bank id, off, size)] for sources of other banks;
|
||||
media ids in `prefetch` get a prefetch source (their DIDX data is the head of a streamed WEM)."""
|
||||
media ids in `prefetch` get a prefetch source (their DIDX data is the head of a streamed WEM);
|
||||
events: [(event name, streamed media id)] -> event -> action -> streamed Sound object."""
|
||||
data = bytearray()
|
||||
didx = b""
|
||||
for m in sorted(media):
|
||||
@@ -108,10 +109,18 @@ def bank(bank_id, media, name=None, extra_sources=(), prefetch=()):
|
||||
objs.append((m, bank_id if m not in prefetch else 0x1234, data_pos + off, len(media[m]),
|
||||
2 if m in prefetch else 0))
|
||||
objs += [x + (0,) for x in extra_sources]
|
||||
hirc = struct.pack("<I", len(objs))
|
||||
from ackit.bnk import fnv1
|
||||
hirc_objs = []
|
||||
for i, (m, b, off, size, stream) in enumerate(objs):
|
||||
body = struct.pack("<7I", 0x100 + i, 0x00040001, stream, m, b, off, size) + bytes(6)
|
||||
hirc += bytes([2]) + struct.pack("<I", len(body)) + body
|
||||
hirc_objs.append((2, struct.pack("<7I", 0x100 + i, 0x00040001, stream, m, b, off, size) + bytes(6)))
|
||||
for i, (ev, m) in enumerate(events):
|
||||
sid, aid = 0x300 + i, 0x400 + i
|
||||
hirc_objs.append((2, struct.pack("<7I", sid, 0x00040001, 1, m, 0, 0, 0) + bytes(6)))
|
||||
hirc_objs.append((3, struct.pack("<III", aid, 0x4011, sid) + bytes(8)))
|
||||
hirc_objs.append((4, struct.pack("<III", fnv1(ev), 1, aid)))
|
||||
hirc = struct.pack("<I", len(hirc_objs))
|
||||
for t, body in hirc_objs:
|
||||
hirc += bytes([t]) + struct.pack("<I", len(body)) + body
|
||||
out = head + b"DATA" + struct.pack("<I", len(data)) + bytes(data) + b"HIRC" + struct.pack("<I", len(hirc)) + hirc
|
||||
if name:
|
||||
nm = name.encode()
|
||||
@@ -120,9 +129,10 @@ def bank(bank_id, media, name=None, extra_sources=(), prefetch=()):
|
||||
return out
|
||||
|
||||
|
||||
def package(path, body, block_size=0x1000, chunk_blocks=3):
|
||||
def package(path, body, block_size=0x1000, chunk_blocks=3, tables=(0, 0, 0, 0, 0, 0)):
|
||||
"""A fully LZO-compressed UE3 package (docs/formats/upk.md) whose uncompressed data area is `body`.
|
||||
Small blocks so that tests cross block and chunk boundaries."""
|
||||
Small blocks so that tests cross block and chunk boundaries. `tables`: (NameCount, NameOffset,
|
||||
ExportCount, ExportOffset, ImportCount, ImportOffset) as absolute uncompressed offsets."""
|
||||
from ackit import lzo
|
||||
folder = b"None\x00"
|
||||
nblocks = max(1, _a(len(body), block_size) // block_size)
|
||||
@@ -130,7 +140,7 @@ def package(path, body, block_size=0x1000, chunk_blocks=3):
|
||||
|
||||
def summary(chunks):
|
||||
s = struct.pack("<IHHi", 0x9E2A83C1, 805, 101, 0) + struct.pack("<i", len(folder)) + folder
|
||||
s += struct.pack("<12i", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + b"GUID-synthetic!!"
|
||||
s += struct.pack("<12i", 0, *tables, 0, 0, 0, 0, 0) + b"GUID-synthetic!!"
|
||||
s += struct.pack("<i3i", 1, 0, 0, 0) + struct.pack("<3I", 7748, 0, 2)
|
||||
s += struct.pack("<i", len(chunks)) + b"".join(struct.pack("<4i", *c) for c in chunks)
|
||||
return s + struct.pack("<Ii", 0x12345678, 0)
|
||||
@@ -157,6 +167,69 @@ def package(path, body, block_size=0x1000, chunk_blocks=3):
|
||||
return header_u
|
||||
|
||||
|
||||
HEADER_U = None # uncompressed summary size of synthetic packages (fixed; computed on first use)
|
||||
|
||||
|
||||
def _header_u():
|
||||
global HEADER_U
|
||||
if HEADER_U is None:
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
HEADER_U = package(os.path.join(d, "x.upk"), b"\0" * 16)
|
||||
return HEADER_U
|
||||
|
||||
|
||||
def ue_package(path, objects, prefix=b""):
|
||||
"""A package with name/import/export tables. objects: [(class name, object name, bytes or callable)]:
|
||||
a callable gets {name: index} and the export index map and returns the object's bytes.
|
||||
Returns {object name: (absolute offset, size)}."""
|
||||
classes = sorted({c for c, _, _ in objects})
|
||||
names = ["None", "Core", "Class", "LocalizedSubtitles", "Subtitles"] + classes + [n for _, n, _ in objects]
|
||||
idx = {n: i for i, n in enumerate(names)}
|
||||
exp_index = {n: i + 1 for i, (_, n, _) in enumerate(objects)}
|
||||
blobs = [o(idx, exp_index) if callable(o) else o for _, _, o in objects]
|
||||
nt = b"".join(struct.pack("<i", len(n) + 1) + n.encode() + b"\0" + bytes(8) for n in names)
|
||||
it = b"".join(struct.pack("<7i", idx["Core"], 0, idx["Class"], 0, 0, idx[c], 0) for c in classes)
|
||||
base = _header_u() + len(prefix)
|
||||
name_off, imp_off = base, base + len(nt)
|
||||
exp_off = imp_off + len(it)
|
||||
data_off = exp_off + 72 * len(objects)
|
||||
et, pos, offs = b"", data_off, {}
|
||||
for (c, n, _), b in zip(objects, blobs):
|
||||
et += struct.pack("<7i", -(classes.index(c) + 1), 0, 0, idx[n], 0, 0, 0) + bytes(8)
|
||||
et += struct.pack("<ii", len(b), pos) + bytes(4 + 4 + 16 + 4)
|
||||
offs[n] = (pos, len(b))
|
||||
pos += len(b)
|
||||
body = prefix + nt + it + et + b"".join(blobs) + bytes(64)
|
||||
package(path, body, tables=(len(names), name_off, len(objects), exp_off, len(classes), imp_off))
|
||||
return offs
|
||||
|
||||
|
||||
def _fstr(text, wide=False):
|
||||
if wide:
|
||||
return struct.pack("<i", -(len(text) + 1)) + text.encode("utf-16-le") + b"\0\0"
|
||||
return struct.pack("<i", len(text) + 1) + text.encode("cp1252") + b"\0"
|
||||
|
||||
|
||||
def dialogue(speech_object, texts, wide=()):
|
||||
"""An RDialogueEvent export (compact tagged properties, docs/formats/subtitles.md) referencing the
|
||||
export `speech_object`; texts: {lang: [line, ...]}; languages in `wide` are stored as UTF-16."""
|
||||
def build(idx, exp_index):
|
||||
content = struct.pack("<I", len(texts))
|
||||
for lang, lines in texts.items():
|
||||
sub = struct.pack("<I", len(lines))
|
||||
for ln in lines:
|
||||
sub += struct.pack("<HH", 0x0D, 0) + _fstr(ln, lang in wide) + struct.pack("<HHf", 4, 0x0C, 1.5)
|
||||
sub += struct.pack("<H", 0)
|
||||
content += struct.pack("<HH", 0x0D, 0) + _fstr(lang)
|
||||
content += struct.pack("<HHiiii", 9, 0x0C, idx["Subtitles"], 0, len(sub), 0) + sub + struct.pack("<H", 0)
|
||||
b = bytes(4) + struct.pack("<HHi", 0x10, 0x2C, exp_index[speech_object])
|
||||
b += struct.pack("<HH", 0x0D, 0x34) + _fstr("Batman")
|
||||
b += struct.pack("<HHiiii", 9, 0x9C, idx["LocalizedSubtitles"], 0, len(content), 0) + content
|
||||
return b + struct.pack("<H", 0)
|
||||
return build
|
||||
|
||||
|
||||
def make_game(root):
|
||||
"""Create the folder; returns the dict of original WEMs by target."""
|
||||
cooked = os.path.join(root, "BmGame", "CookedPCConsole")
|
||||
@@ -175,6 +248,7 @@ def make_game(root):
|
||||
originals["%08X" % SINGLE_ENTRY] = single
|
||||
originals.update({"%08X/%08X" % (PACK_ID, m): w for m, w in music.items()})
|
||||
originals.update(make_banks(cooked, v))
|
||||
make_localization(root)
|
||||
return originals
|
||||
|
||||
|
||||
@@ -213,5 +287,35 @@ def make_banks(cooked, speech_wems):
|
||||
synth_b = filler[:1000] + bank(fnv1(names[3]), gui) + filler + bank(fnv1(names[2]), foley) + filler
|
||||
package(os.path.join(cooked, "Synth_A.upk"), synth_a)
|
||||
package(os.path.join(cooked, "Synth_B.upk"), synth_b)
|
||||
# dialogue (subtitles) in two packages; the first one also holds a bank with an in-memory sound and the
|
||||
# event that links the dialogue's speech event to INT variation 1 of SPEECH_ENTRY
|
||||
dlg = dialogue(SPEECH_EVENT, SUBTITLES, wide=("RUS",))
|
||||
dlg_wem = wem(0.3, 1400)
|
||||
ev_bank = bank(fnv1("NPC_SynthDialogue"), {DLG_MEDIA: dlg_wem}, name="NPC_SynthDialogue",
|
||||
events=[(SPEECH_EVENT, LINE_MEDIA[1])])
|
||||
objs = [("AkEvent", SPEECH_EVENT, bytes(8)), ("RDialogueEvent", "Radio_02", dlg)]
|
||||
ue_package(os.path.join(cooked, "Synth_Dialogue.upk"), objs + [("AkBank", "NPC_SynthDialogue", ev_bank)])
|
||||
ue_package(os.path.join(cooked, "Synth_Dialogue2.upk"), objs, prefix=bytes(3000))
|
||||
with open(os.path.join(cooked, "CookedBanks.txt"), "a", newline="\r\n") as f:
|
||||
f.write("NPC_SynthDialogue\n")
|
||||
out = {"%08X" % m: w for m, w in list(foot.items()) + list(foley.items()) + list(gui.items())}
|
||||
out["%08X" % DLG_MEDIA] = dlg_wem
|
||||
return out
|
||||
|
||||
|
||||
SPEECH_EVENT = "Speech_Synth_Radio_02"
|
||||
SUBTITLES = {"INT": ["Scanner line two.", "Second part."], "ITA": ["Riga due dello scanner."],
|
||||
"RUS": ["Строка два"]}
|
||||
DLG_MEDIA = 0x0E000001
|
||||
MENU = {"INT": ("FRONTEND.int", "\r\n[Menu]\r\nStart=Press any key\r\nQuit=\"Quit game\"\r\n; comment\r\n"
|
||||
"[Tips]\r\nTip=Use the grapnel\r\nTip=Glide over the city\r\n"),
|
||||
"ITA": ("FRONTEND.ITA", "\r\n[Menu]\r\nStart=Premi un tasto\r\nQuit=\"Esci\"\r\n"
|
||||
"[Tips]\r\nTip=Usa il rampino\r\nTip=Plana sulla citta\r\n")}
|
||||
|
||||
|
||||
def make_localization(root):
|
||||
for lang, (fname, text) in MENU.items():
|
||||
d = os.path.join(root, "BmGame", "Localization", lang)
|
||||
os.makedirs(d, exist_ok=True)
|
||||
open(os.path.join(d, fname), "wb").write(b"\xff\xfe" + text.encode("utf-16-le"))
|
||||
open(os.path.join(root, "BmGame", "Localization", "INT", "Version.int"), "wb").write(b"[ID]\r\nUnreal=SYNTH\r\n")
|
||||
|
||||
@@ -97,7 +97,7 @@ def test_bank_sound_apply_revert_in_editor(win, tmp_path):
|
||||
w, root, originals = win
|
||||
before = files(root)
|
||||
w.cmb_arch.setCurrentText("BANK")
|
||||
assert w.proxy.rowCount() == 5
|
||||
assert w.proxy.rowCount() == 6
|
||||
assert w.select_name("CHR_SynthFoley_1")
|
||||
ed = w.editor
|
||||
assert "bank CHR_SynthFoley (2 copies)" in ed.meta.text()
|
||||
|
||||
@@ -32,7 +32,7 @@ def test_bank_catalog(synth_game, tmp_path):
|
||||
root, originals = synth_game
|
||||
ws = _ws(root, tmp_path)
|
||||
bank = {s.target: s for s in ws.catalog.sounds if s.archive == "BANK"}
|
||||
assert set(bank) == {"%08X" % m for m in synth.FOOT + synth.FOLEY + synth.GUI}
|
||||
assert set(bank) == {"%08X" % m for m in synth.FOOT + synth.FOLEY + synth.GUI + (synth.DLG_MEDIA,)}
|
||||
foot = bank["%08X" % synth.FOOT[0]]
|
||||
assert foot.bank == "CHR_SynthFootsteps_Media" and foot.name == "CHR_SynthFootsteps_Media_1"
|
||||
assert (foot.group, foot.category) == ("SFX", "Footsteps")
|
||||
|
||||
@@ -57,7 +57,13 @@ def test_characters_csv_covers_user_lists():
|
||||
|
||||
|
||||
def test_shipped_csvs_are_consistent():
|
||||
from ackit import subtitles
|
||||
for f in os.listdir(catalog_dir()):
|
||||
if f.startswith("SUB_") and f.endswith(".csv"):
|
||||
lines = subtitles.read_texts(os.path.join(catalog_dir(), f), f[4:7])
|
||||
assert len({(s.speech, s.line) for s in lines}) == len(lines), f
|
||||
assert all(len(s.text) <= s.room for s in lines), f
|
||||
continue
|
||||
if f.endswith(".csv") and f != "characters.csv":
|
||||
rows = read_csv(os.path.join(catalog_dir(), f))
|
||||
keys = {r.key for r in rows}
|
||||
|
||||
@@ -12,7 +12,7 @@ def test_clean_synthetic_catalog_passes(synth_game):
|
||||
root, _ = synth_game
|
||||
rep = catalogcheck.run(Game(root), log=lambda *a: None)
|
||||
assert rep.ok, rep.issues
|
||||
assert rep.checked["BANK"] == 5 and rep.checked["INT"] == 4
|
||||
assert rep.checked["BANK"] == 6 and rep.checked["INT"] == 4
|
||||
|
||||
|
||||
def test_corrupted_catalog_is_reported(synth_game):
|
||||
|
||||
140
tests/test_texts.py
Normal file
140
tests/test_texts.py
Normal file
@@ -0,0 +1,140 @@
|
||||
"""Subtitles (dialogue objects in packages) and menu text (localization files): parse, apply, revert."""
|
||||
import hashlib
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
import synth
|
||||
from ackit import loctext, subtitles
|
||||
from ackit.game import Game
|
||||
from ackit.texts import MODIFIED, ORIGINAL, TextConflictError
|
||||
from ackit.workspace import Workspace
|
||||
|
||||
SUB_KEY = "SUB:INT:%s:0" % synth.SPEECH_EVENT
|
||||
|
||||
|
||||
def _sha(p):
|
||||
return hashlib.sha1(open(p, "rb").read()).hexdigest()
|
||||
|
||||
|
||||
def _ws(root, tmp_path):
|
||||
return Workspace(root, store_root=str(tmp_path / "store"), archives=["INT", "SFX", "BANK"])
|
||||
|
||||
|
||||
def _files(root):
|
||||
d = os.path.join(root, "BmGame")
|
||||
return {os.path.join(a, f): _sha(os.path.join(a, f)) for a, _, fs in os.walk(d) for f in fs}
|
||||
|
||||
|
||||
# --- formats ------------------------------------------------------------------------------------
|
||||
def test_fstring_encoding():
|
||||
b = subtitles.encode_fstring("Hi", 10, "a")
|
||||
assert b == b"\x06\x00\x00\x00Hi\0\0\0\0" and subtitles.read_fstring(b, 0)[0] == "Hi"
|
||||
w = subtitles.encode_fstring("Привет", 4 + 2 * 8, "w")
|
||||
assert subtitles.read_fstring(w, 0) == ("Привет", 20, "w")
|
||||
with pytest.raises(subtitles.SubtitleError):
|
||||
subtitles.encode_fstring("too long", 10, "a")
|
||||
with pytest.raises(subtitles.SubtitleError):
|
||||
subtitles.encode_fstring("→", 10, "a") # not in cp1252
|
||||
|
||||
|
||||
def test_menu_parse_and_write():
|
||||
fname, text = synth.MENU["INT"]
|
||||
raw = b"\xff\xfe" + text.encode("utf-16-le")
|
||||
es = {e.key: e for e in loctext.parse(raw, "INT", fname)}
|
||||
assert es["INT:FRONTEND.int:Menu:Start"].value == "Press any key"
|
||||
assert es["INT:FRONTEND.int:Menu:Quit"].quoted and es["INT:FRONTEND.int:Tips:Tip#2"].value == "Glide over the city"
|
||||
assert loctext.write(raw, "INT", fname, {}) == raw
|
||||
new = loctext.write(raw, "INT", fname, {"INT:FRONTEND.int:Menu:Quit": "Leave", "INT:FRONTEND.int:Tips:Tip#2": "Fly"})
|
||||
assert new.decode("utf-16") == text.replace('"Quit game"', '"Leave"').replace("Glide over the city", "Fly")
|
||||
with pytest.raises(loctext.TextError):
|
||||
loctext.write(raw, "INT", fname, {"INT:FRONTEND.int:Menu:Start": "two\nlines"})
|
||||
|
||||
|
||||
# --- subtitles through the workspace ------------------------------------------------------------
|
||||
def test_subtitle_catalog_and_link(synth_game, tmp_path):
|
||||
root, _ = synth_game
|
||||
ws = _ws(root, tmp_path)
|
||||
lines = ws.texts.subtitles("INT")
|
||||
assert [(s.speech, s.line, s.text) for s in lines] == [(synth.SPEECH_EVENT, 0, "Scanner line two."),
|
||||
(synth.SPEECH_EVENT, 1, "Second part.")]
|
||||
line = ws.catalog.by_key["INT:%08X:1" % synth.SPEECH_ENTRY]
|
||||
assert [s.text for s in ws.texts.for_sound(line)] == ["Scanner line two.", "Second part."]
|
||||
assert ws.texts.subtitles("ITA")[0].sound == line.key # no ITA speech WAD here: INT line
|
||||
assert ws.texts.subtitles("RUS")[0].enc == "w"
|
||||
assert sorted(os.path.basename(p) for p in ws.texts.files(SUB_KEY)) == ["Synth_Dialogue.upk", "Synth_Dialogue2.upk"]
|
||||
assert ws.texts.limit(SUB_KEY) == len("Scanner line two.")
|
||||
|
||||
|
||||
def test_subtitle_apply_revert_every_copy(synth_game, tmp_path):
|
||||
root, _ = synth_game
|
||||
ws = _ws(root, tmp_path)
|
||||
before = _files(root)
|
||||
ws.texts.apply({SUB_KEY: "Line two.", "SUB:RUS:%s:0" % synth.SPEECH_EVENT: "Две"})
|
||||
assert ws.texts.current(SUB_KEY) == "Line two." and ws.texts.state(SUB_KEY) == MODIFIED
|
||||
for p in ws.texts.files(SUB_KEY): # every copy
|
||||
ws2 = _ws(root, tmp_path / "other")
|
||||
assert ws2.texts.current(SUB_KEY) == "Line two."
|
||||
with pytest.raises(subtitles.SubtitleError):
|
||||
ws.texts.apply({SUB_KEY: "This line is far too long for the room"})
|
||||
ws.texts.revert([SUB_KEY])
|
||||
assert ws.texts.current(SUB_KEY) == "Scanner line two." and ws.texts.current("SUB:RUS:%s:0" % synth.SPEECH_EVENT) == "Две"
|
||||
ws.texts.revert_all()
|
||||
assert _files(root) == before and ws.texts.state(SUB_KEY) == ORIGINAL
|
||||
|
||||
|
||||
def test_subtitle_and_sound_in_one_package(synth_game, tmp_path):
|
||||
root, _ = synth_game
|
||||
ws = _ws(root, tmp_path)
|
||||
before = _files(root)
|
||||
s = ws.catalog.by_key["BANK:%08X" % synth.DLG_MEDIA]
|
||||
ws.apply([(s, synth.wem(0.1, 1900), {})])
|
||||
ws.texts.apply({SUB_KEY: "Both."})
|
||||
assert ws.texts.current(SUB_KEY) == "Both." and ws.state(s) == "modified"
|
||||
ws.revert([s])
|
||||
assert ws.texts.current(SUB_KEY) == "Both." and ws.state(s) == "original"
|
||||
ws.texts.revert_all()
|
||||
assert _files(root) == before
|
||||
|
||||
|
||||
# --- menu text through the workspace ------------------------------------------------------------
|
||||
def test_menu_apply_revert(synth_game, tmp_path):
|
||||
root, _ = synth_game
|
||||
ws = _ws(root, tmp_path)
|
||||
before = _files(root)
|
||||
assert "INT" in ws.texts.menu_languages()
|
||||
keys = {e.key: e for e in ws.texts.menu("INT")}
|
||||
assert "INT:Version.int:ID:Unreal" not in keys
|
||||
k = "MENU:INT:FRONTEND.int:Menu:Start"
|
||||
ws.texts.apply({k: "Press Enter"})
|
||||
assert ws.texts.current(k) == "Press Enter" and ws.texts.state(k) == MODIFIED
|
||||
assert {e.key: e.value for e in ws.texts.menu("INT")}["INT:FRONTEND.int:Menu:Start"] == "Press Enter"
|
||||
ws.texts.revert([k])
|
||||
assert _files(root) == before
|
||||
|
||||
|
||||
def test_menu_changed_underneath(synth_game, tmp_path):
|
||||
root, _ = synth_game
|
||||
ws = _ws(root, tmp_path)
|
||||
k = "MENU:INT:FRONTEND.int:Tips:Tip#2"
|
||||
ws.texts.apply({k: "Glide"})
|
||||
path = ws.texts.files(k)[0]
|
||||
raw = open(path, "rb").read()
|
||||
open(path, "wb").write(loctext.write(raw, "INT", "FRONTEND.int", {"INT:FRONTEND.int:Menu:Start": "Other tool"}))
|
||||
with pytest.raises(TextConflictError):
|
||||
ws.texts.apply({k: "Soar"})
|
||||
ws.texts.apply({k: "Soar"}, force=True)
|
||||
assert ws.texts.current(k) == "Soar"
|
||||
ws.texts.revert_all()
|
||||
assert open(path, "rb").read() == open(ws.store.full_copy(ws.game.rel(path)), "rb").read()
|
||||
|
||||
|
||||
# --- real game (read-only) ----------------------------------------------------------------------
|
||||
def test_real_subtitles_and_menu(real_game_dir, tmp_path):
|
||||
ws = Workspace(Game(real_game_dir), store_root=str(tmp_path / "store"), archives=["ITA"])
|
||||
lines = ws.texts.subtitles("ITA")
|
||||
assert len(lines) > 9000
|
||||
s = next(x for x in lines if x.sound)
|
||||
assert ws.texts.current(subtitles.key("ITA", s.speech, s.line)) == s.text
|
||||
menu = ws.texts.menu("ITA")
|
||||
assert len(menu) > 7000 and all(e.lang == "ITA" for e in menu)
|
||||
@@ -136,7 +136,7 @@ def test_mod_export_import_roundtrip(synth_game, synth_template, tmp_path):
|
||||
ws2 = ws_for(other, store_root=str(tmp_path / "store2"))
|
||||
_, items = match_mod(ws2, pkg)
|
||||
assert [i.how for i in items] == ["exact", "exact"]
|
||||
res = import_mod(ws2, pkg)
|
||||
res, _ = import_mod(ws2, pkg)
|
||||
assert all(i.note == "exact copy" for i in res)
|
||||
assert ws2.current_bytes(ws2.catalog.by_key[a.key]) == wa
|
||||
assert ws2.current_bytes(ws2.catalog.by_key[b.key]) == wb
|
||||
|
||||
@@ -7,6 +7,8 @@ Outputs
|
||||
data/catalog/<ARCHIVE>.csv every sound of one archive (schema: data/catalog/README.md)
|
||||
data/catalog/BANK.csv in-memory sounds (soundbanks in SFX/*.bnk and the .upk packages)
|
||||
data/catalog/BANK_locations.csv.gz every stored copy of them + the prefetch heads of speech lines
|
||||
data/catalog/SUB_<LANG>.csv subtitles of the speech lines (one file per language)
|
||||
data/catalog/SUB_locations.csv.gz every stored copy of each subtitle string
|
||||
data/catalog/index.json fingerprints the CSVs were built from + counts
|
||||
Run it only on clean game files (`ackit status` must say nothing is modified).
|
||||
"""
|
||||
@@ -20,8 +22,8 @@ import sys
|
||||
import time
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from ackit import banks, settings # noqa: E402
|
||||
from ackit.catalog import scan_archive, scan_banks, write_csv, hash_names # noqa: E402
|
||||
from ackit import banks, settings, subtitles # noqa: E402
|
||||
from ackit.catalog import scan_archive, scan_text, write_csv, hash_names # noqa: E402
|
||||
from ackit.classify import USER_LABELS # noqa: E402
|
||||
from ackit.game import Game, default_game_dir, wad_fingerprint # noqa: E402
|
||||
from ackit.inject import read_target # noqa: E402
|
||||
@@ -113,7 +115,8 @@ def main():
|
||||
|
||||
# in-memory sounds: loose SFX\*.bnk + banks inside the .upk packages (a few minutes)
|
||||
t0 = time.time()
|
||||
sounds, locs, sc = scan_banks(game, progress=lambda k, n: print(" banks: %d/%d files" % (k, n), flush=True)
|
||||
sounds, locs, sc, texts, sublocs, sub_issues = scan_text(
|
||||
game, progress=lambda k, n: print(" banks + subtitles: %d/%d files" % (k, n), flush=True)
|
||||
if k % 500 == 0 else None, characters=charmap)
|
||||
write_csv(os.path.join(out_dir, "%s.csv" % banks.ARCHIVE), sounds)
|
||||
banks.write_locations(os.path.join(out_dir, banks.LOCATIONS), locs)
|
||||
@@ -128,6 +131,23 @@ def main():
|
||||
"issues": ["%s: %s" % i for i in sc.issues]}
|
||||
print("%s.csv: %d sounds %s, %d copies in %d files, %d prefetch heads (%.0fs)"
|
||||
% (banks.ARCHIVE, len(sounds), dict(g), len(locs) - len(heads), len(sc.files), len(heads), time.time() - t0))
|
||||
|
||||
# subtitles (dialogue objects in the packages), one CSV per language + every stored copy
|
||||
for f in os.listdir(out_dir):
|
||||
if f.startswith("SUB_") and f.endswith(".csv"):
|
||||
os.remove(os.path.join(out_dir, f))
|
||||
for lang, rows in texts.items():
|
||||
subtitles.write_texts(os.path.join(out_dir, "SUB_%s.csv" % lang), rows)
|
||||
subtitles.write_locations(os.path.join(out_dir, subtitles.LOCATIONS), sublocs)
|
||||
any_lang = next(iter(texts.values()), [])
|
||||
index["archives"]["SUB"] = {
|
||||
"fingerprint": banks.game_fingerprint(game.cooked),
|
||||
"lines": {lang: len(rows) for lang, rows in sorted(texts.items())},
|
||||
"speech_lines": len({s.speech for s in any_lang}),
|
||||
"linked": {lang: sum(1 for s in rows if s.sound) for lang, rows in sorted(texts.items())},
|
||||
"copies": len(sublocs), "issues": ["%s: %s" % i for i in sub_issues]}
|
||||
print("SUB_*.csv: %d languages, %d subtitle strings, %d copies"
|
||||
% (len(texts), sum(len(r) for r in texts.values()), len(sublocs)))
|
||||
# flatten for catalog.load_archive
|
||||
flat = {a: v for a, v in index["archives"].items()}
|
||||
json.dump(dict(flat, _info=index["built_from"]), open(os.path.join(out_dir, "index.json"), "w", newline="\n"), indent=1)
|
||||
|
||||
Reference in New Issue
Block a user