Edit subtitles and menu text

2026-10-09 22:36:46 +02:00
parent 5d6c2447f7
commit ef68edbe45
40 changed files with 105877 additions and 1291 deletions
--- a/tests/synth.py
+++ b/tests/synth.py
@@ -89,10 +89,11 @@ def pack(media):
    return bytes(hdr) + bank, bytes(payload)


-def bank(bank_id, media, name=None, extra_sources=(), prefetch=()):
+def bank(bank_id, media, name=None, extra_sources=(), prefetch=(), events=()):
    """A v56 soundbank holding `media` {id: wem} in memory (DIDX/DATA, 16-byte aligned) with one HIRC
    Sound object per media; extra_sources: [(media id, bank id, off, size)] for sources of other banks;
-    media ids in `prefetch` get a prefetch source (their DIDX data is the head of a streamed WEM)."""
+    media ids in `prefetch` get a prefetch source (their DIDX data is the head of a streamed WEM);
+    events: [(event name, streamed media id)] -> event -> action -> streamed Sound object."""
    data = bytearray()
    didx = b""
    for m in sorted(media):
@@ -108,10 +109,18 @@ def bank(bank_id, media, name=None, extra_sources=(), prefetch=()):
        objs.append((m, bank_id if m not in prefetch else 0x1234, data_pos + off, len(media[m]),
                     2 if m in prefetch else 0))
    objs += [x + (0,) for x in extra_sources]
-    hirc = struct.pack("<I", len(objs))
+    from ackit.bnk import fnv1
+    hirc_objs = []
    for i, (m, b, off, size, stream) in enumerate(objs):
-        body = struct.pack("<7I", 0x100 + i, 0x00040001, stream, m, b, off, size) + bytes(6)
-        hirc += bytes([2]) + struct.pack("<I", len(body)) + body
+        hirc_objs.append((2, struct.pack("<7I", 0x100 + i, 0x00040001, stream, m, b, off, size) + bytes(6)))
+    for i, (ev, m) in enumerate(events):
+        sid, aid = 0x300 + i, 0x400 + i
+        hirc_objs.append((2, struct.pack("<7I", sid, 0x00040001, 1, m, 0, 0, 0) + bytes(6)))
+        hirc_objs.append((3, struct.pack("<III", aid, 0x4011, sid) + bytes(8)))
+        hirc_objs.append((4, struct.pack("<III", fnv1(ev), 1, aid)))
+    hirc = struct.pack("<I", len(hirc_objs))
+    for t, body in hirc_objs:
+        hirc += bytes([t]) + struct.pack("<I", len(body)) + body
    out = head + b"DATA" + struct.pack("<I", len(data)) + bytes(data) + b"HIRC" + struct.pack("<I", len(hirc)) + hirc
    if name:
        nm = name.encode()
@@ -120,9 +129,10 @@ def bank(bank_id, media, name=None, extra_sources=(), prefetch=()):
    return out


-def package(path, body, block_size=0x1000, chunk_blocks=3):
+def package(path, body, block_size=0x1000, chunk_blocks=3, tables=(0, 0, 0, 0, 0, 0)):
    """A fully LZO-compressed UE3 package (docs/formats/upk.md) whose uncompressed data area is `body`.
-    Small blocks so that tests cross block and chunk boundaries."""
+    Small blocks so that tests cross block and chunk boundaries. `tables`: (NameCount, NameOffset,
+    ExportCount, ExportOffset, ImportCount, ImportOffset) as absolute uncompressed offsets."""
    from ackit import lzo
    folder = b"None\x00"
    nblocks = max(1, _a(len(body), block_size) // block_size)
@@ -130,7 +140,7 @@ def package(path, body, block_size=0x1000, chunk_blocks=3):

    def summary(chunks):
        s = struct.pack("<IHHi", 0x9E2A83C1, 805, 101, 0) + struct.pack("<i", len(folder)) + folder
-        s += struct.pack("<12i", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + b"GUID-synthetic!!"
+        s += struct.pack("<12i", 0, *tables, 0, 0, 0, 0, 0) + b"GUID-synthetic!!"
        s += struct.pack("<i3i", 1, 0, 0, 0) + struct.pack("<3I", 7748, 0, 2)
        s += struct.pack("<i", len(chunks)) + b"".join(struct.pack("<4i", *c) for c in chunks)
        return s + struct.pack("<Ii", 0x12345678, 0)
@@ -157,6 +167,69 @@ def package(path, body, block_size=0x1000, chunk_blocks=3):
    return header_u


+HEADER_U = None   # uncompressed summary size of synthetic packages (fixed; computed on first use)
+
+
+def _header_u():
+    global HEADER_U
+    if HEADER_U is None:
+        import tempfile
+        with tempfile.TemporaryDirectory() as d:
+            HEADER_U = package(os.path.join(d, "x.upk"), b"\0" * 16)
+    return HEADER_U
+
+
+def ue_package(path, objects, prefix=b""):
+    """A package with name/import/export tables. objects: [(class name, object name, bytes or callable)]:
+    a callable gets {name: index} and the export index map and returns the object's bytes.
+    Returns {object name: (absolute offset, size)}."""
+    classes = sorted({c for c, _, _ in objects})
+    names = ["None", "Core", "Class", "LocalizedSubtitles", "Subtitles"] + classes + [n for _, n, _ in objects]
+    idx = {n: i for i, n in enumerate(names)}
+    exp_index = {n: i + 1 for i, (_, n, _) in enumerate(objects)}
+    blobs = [o(idx, exp_index) if callable(o) else o for _, _, o in objects]
+    nt = b"".join(struct.pack("<i", len(n) + 1) + n.encode() + b"\0" + bytes(8) for n in names)
+    it = b"".join(struct.pack("<7i", idx["Core"], 0, idx["Class"], 0, 0, idx[c], 0) for c in classes)
+    base = _header_u() + len(prefix)
+    name_off, imp_off = base, base + len(nt)
+    exp_off = imp_off + len(it)
+    data_off = exp_off + 72 * len(objects)
+    et, pos, offs = b"", data_off, {}
+    for (c, n, _), b in zip(objects, blobs):
+        et += struct.pack("<7i", -(classes.index(c) + 1), 0, 0, idx[n], 0, 0, 0) + bytes(8)
+        et += struct.pack("<ii", len(b), pos) + bytes(4 + 4 + 16 + 4)
+        offs[n] = (pos, len(b))
+        pos += len(b)
+    body = prefix + nt + it + et + b"".join(blobs) + bytes(64)
+    package(path, body, tables=(len(names), name_off, len(objects), exp_off, len(classes), imp_off))
+    return offs
+
+
+def _fstr(text, wide=False):
+    if wide:
+        return struct.pack("<i", -(len(text) + 1)) + text.encode("utf-16-le") + b"\0\0"
+    return struct.pack("<i", len(text) + 1) + text.encode("cp1252") + b"\0"
+
+
+def dialogue(speech_object, texts, wide=()):
+    """An RDialogueEvent export (compact tagged properties, docs/formats/subtitles.md) referencing the
+    export `speech_object`; texts: {lang: [line, ...]}; languages in `wide` are stored as UTF-16."""
+    def build(idx, exp_index):
+        content = struct.pack("<I", len(texts))
+        for lang, lines in texts.items():
+            sub = struct.pack("<I", len(lines))
+            for ln in lines:
+                sub += struct.pack("<HH", 0x0D, 0) + _fstr(ln, lang in wide) + struct.pack("<HHf", 4, 0x0C, 1.5)
+                sub += struct.pack("<H", 0)
+            content += struct.pack("<HH", 0x0D, 0) + _fstr(lang)
+            content += struct.pack("<HHiiii", 9, 0x0C, idx["Subtitles"], 0, len(sub), 0) + sub + struct.pack("<H", 0)
+        b = bytes(4) + struct.pack("<HHi", 0x10, 0x2C, exp_index[speech_object])
+        b += struct.pack("<HH", 0x0D, 0x34) + _fstr("Batman")
+        b += struct.pack("<HHiiii", 9, 0x9C, idx["LocalizedSubtitles"], 0, len(content), 0) + content
+        return b + struct.pack("<H", 0)
+    return build
+
+
 def make_game(root):
    """Create the folder; returns the dict of original WEMs by target."""
    cooked = os.path.join(root, "BmGame", "CookedPCConsole")
@@ -175,6 +248,7 @@ def make_game(root):
    originals["%08X" % SINGLE_ENTRY] = single
    originals.update({"%08X/%08X" % (PACK_ID, m): w for m, w in music.items()})
    originals.update(make_banks(cooked, v))
+    make_localization(root)
    return originals


@@ -213,5 +287,35 @@ def make_banks(cooked, speech_wems):
    synth_b = filler[:1000] + bank(fnv1(names[3]), gui) + filler + bank(fnv1(names[2]), foley) + filler
    package(os.path.join(cooked, "Synth_A.upk"), synth_a)
    package(os.path.join(cooked, "Synth_B.upk"), synth_b)
+    # dialogue (subtitles) in two packages; the first one also holds a bank with an in-memory sound and the
+    # event that links the dialogue's speech event to INT variation 1 of SPEECH_ENTRY
+    dlg = dialogue(SPEECH_EVENT, SUBTITLES, wide=("RUS",))
+    dlg_wem = wem(0.3, 1400)
+    ev_bank = bank(fnv1("NPC_SynthDialogue"), {DLG_MEDIA: dlg_wem}, name="NPC_SynthDialogue",
+                   events=[(SPEECH_EVENT, LINE_MEDIA[1])])
+    objs = [("AkEvent", SPEECH_EVENT, bytes(8)), ("RDialogueEvent", "Radio_02", dlg)]
+    ue_package(os.path.join(cooked, "Synth_Dialogue.upk"), objs + [("AkBank", "NPC_SynthDialogue", ev_bank)])
+    ue_package(os.path.join(cooked, "Synth_Dialogue2.upk"), objs, prefix=bytes(3000))
+    with open(os.path.join(cooked, "CookedBanks.txt"), "a", newline="\r\n") as f:
+        f.write("NPC_SynthDialogue\n")
    out = {"%08X" % m: w for m, w in list(foot.items()) + list(foley.items()) + list(gui.items())}
+    out["%08X" % DLG_MEDIA] = dlg_wem
    return out
+
+
+SPEECH_EVENT = "Speech_Synth_Radio_02"
+SUBTITLES = {"INT": ["Scanner line two.", "Second part."], "ITA": ["Riga due dello scanner."],
+             "RUS": ["Строка два"]}
+DLG_MEDIA = 0x0E000001
+MENU = {"INT": ("FRONTEND.int", "\r\n[Menu]\r\nStart=Press any key\r\nQuit=\"Quit game\"\r\n; comment\r\n"
+                                "[Tips]\r\nTip=Use the grapnel\r\nTip=Glide over the city\r\n"),
+        "ITA": ("FRONTEND.ITA", "\r\n[Menu]\r\nStart=Premi un tasto\r\nQuit=\"Esci\"\r\n"
+                                "[Tips]\r\nTip=Usa il rampino\r\nTip=Plana sulla citta\r\n")}
+
+
+def make_localization(root):
+    for lang, (fname, text) in MENU.items():
+        d = os.path.join(root, "BmGame", "Localization", lang)
+        os.makedirs(d, exist_ok=True)
+        open(os.path.join(d, fname), "wb").write(b"\xff\xfe" + text.encode("utf-16-le"))
+    open(os.path.join(root, "BmGame", "Localization", "INT", "Version.int"), "wb").write(b"[ID]\r\nUnreal=SYNTH\r\n")
--- a/tests/test_app.py
+++ b/tests/test_app.py
@@ -97,7 +97,7 @@ def test_bank_sound_apply_revert_in_editor(win, tmp_path):
    w, root, originals = win
    before = files(root)
    w.cmb_arch.setCurrentText("BANK")
-    assert w.proxy.rowCount() == 5
+    assert w.proxy.rowCount() == 6
    assert w.select_name("CHR_SynthFoley_1")
    ed = w.editor
    assert "bank CHR_SynthFoley (2 copies)" in ed.meta.text()
--- a/tests/test_banks.py
+++ b/tests/test_banks.py
@@ -32,7 +32,7 @@ def test_bank_catalog(synth_game, tmp_path):
    root, originals = synth_game
    ws = _ws(root, tmp_path)
    bank = {s.target: s for s in ws.catalog.sounds if s.archive == "BANK"}
-    assert set(bank) == {"%08X" % m for m in synth.FOOT + synth.FOLEY + synth.GUI}
+    assert set(bank) == {"%08X" % m for m in synth.FOOT + synth.FOLEY + synth.GUI + (synth.DLG_MEDIA,)}
    foot = bank["%08X" % synth.FOOT[0]]
    assert foot.bank == "CHR_SynthFootsteps_Media" and foot.name == "CHR_SynthFootsteps_Media_1"
    assert (foot.group, foot.category) == ("SFX", "Footsteps")
--- a/tests/test_catalog.py
+++ b/tests/test_catalog.py
@@ -57,7 +57,13 @@ def test_characters_csv_covers_user_lists():


 def test_shipped_csvs_are_consistent():
+    from ackit import subtitles
    for f in os.listdir(catalog_dir()):
+        if f.startswith("SUB_") and f.endswith(".csv"):
+            lines = subtitles.read_texts(os.path.join(catalog_dir(), f), f[4:7])
+            assert len({(s.speech, s.line) for s in lines}) == len(lines), f
+            assert all(len(s.text) <= s.room for s in lines), f
+            continue
        if f.endswith(".csv") and f != "characters.csv":
            rows = read_csv(os.path.join(catalog_dir(), f))
            keys = {r.key for r in rows}
--- a/tests/test_catalogcheck.py
+++ b/tests/test_catalogcheck.py
@@ -12,7 +12,7 @@ def test_clean_synthetic_catalog_passes(synth_game):
    root, _ = synth_game
    rep = catalogcheck.run(Game(root), log=lambda *a: None)
    assert rep.ok, rep.issues
-    assert rep.checked["BANK"] == 5 and rep.checked["INT"] == 4
+    assert rep.checked["BANK"] == 6 and rep.checked["INT"] == 4


 def test_corrupted_catalog_is_reported(synth_game):
--- a/tests/test_texts.py
+++ b/tests/test_texts.py
@@ -0,0 +1,140 @@
+"""Subtitles (dialogue objects in packages) and menu text (localization files): parse, apply, revert."""
+import hashlib
+import os
+
+import pytest
+
+import synth
+from ackit import loctext, subtitles
+from ackit.game import Game
+from ackit.texts import MODIFIED, ORIGINAL, TextConflictError
+from ackit.workspace import Workspace
+
+SUB_KEY = "SUB:INT:%s:0" % synth.SPEECH_EVENT
+
+
+def _sha(p):
+    return hashlib.sha1(open(p, "rb").read()).hexdigest()
+
+
+def _ws(root, tmp_path):
+    return Workspace(root, store_root=str(tmp_path / "store"), archives=["INT", "SFX", "BANK"])
+
+
+def _files(root):
+    d = os.path.join(root, "BmGame")
+    return {os.path.join(a, f): _sha(os.path.join(a, f)) for a, _, fs in os.walk(d) for f in fs}
+
+
+# --- formats ------------------------------------------------------------------------------------
+def test_fstring_encoding():
+    b = subtitles.encode_fstring("Hi", 10, "a")
+    assert b == b"\x06\x00\x00\x00Hi\0\0\0\0" and subtitles.read_fstring(b, 0)[0] == "Hi"
+    w = subtitles.encode_fstring("Привет", 4 + 2 * 8, "w")
+    assert subtitles.read_fstring(w, 0) == ("Привет", 20, "w")
+    with pytest.raises(subtitles.SubtitleError):
+        subtitles.encode_fstring("too long", 10, "a")
+    with pytest.raises(subtitles.SubtitleError):
+        subtitles.encode_fstring("→", 10, "a")                      # not in cp1252
+
+
+def test_menu_parse_and_write():
+    fname, text = synth.MENU["INT"]
+    raw = b"\xff\xfe" + text.encode("utf-16-le")
+    es = {e.key: e for e in loctext.parse(raw, "INT", fname)}
+    assert es["INT:FRONTEND.int:Menu:Start"].value == "Press any key"
+    assert es["INT:FRONTEND.int:Menu:Quit"].quoted and es["INT:FRONTEND.int:Tips:Tip#2"].value == "Glide over the city"
+    assert loctext.write(raw, "INT", fname, {}) == raw
+    new = loctext.write(raw, "INT", fname, {"INT:FRONTEND.int:Menu:Quit": "Leave", "INT:FRONTEND.int:Tips:Tip#2": "Fly"})
+    assert new.decode("utf-16") == text.replace('"Quit game"', '"Leave"').replace("Glide over the city", "Fly")
+    with pytest.raises(loctext.TextError):
+        loctext.write(raw, "INT", fname, {"INT:FRONTEND.int:Menu:Start": "two\nlines"})
+
+
+# --- subtitles through the workspace ------------------------------------------------------------
+def test_subtitle_catalog_and_link(synth_game, tmp_path):
+    root, _ = synth_game
+    ws = _ws(root, tmp_path)
+    lines = ws.texts.subtitles("INT")
+    assert [(s.speech, s.line, s.text) for s in lines] == [(synth.SPEECH_EVENT, 0, "Scanner line two."),
+                                                          (synth.SPEECH_EVENT, 1, "Second part.")]
+    line = ws.catalog.by_key["INT:%08X:1" % synth.SPEECH_ENTRY]
+    assert [s.text for s in ws.texts.for_sound(line)] == ["Scanner line two.", "Second part."]
+    assert ws.texts.subtitles("ITA")[0].sound == line.key            # no ITA speech WAD here: INT line
+    assert ws.texts.subtitles("RUS")[0].enc == "w"
+    assert sorted(os.path.basename(p) for p in ws.texts.files(SUB_KEY)) == ["Synth_Dialogue.upk", "Synth_Dialogue2.upk"]
+    assert ws.texts.limit(SUB_KEY) == len("Scanner line two.")
+
+
+def test_subtitle_apply_revert_every_copy(synth_game, tmp_path):
+    root, _ = synth_game
+    ws = _ws(root, tmp_path)
+    before = _files(root)
+    ws.texts.apply({SUB_KEY: "Line two.", "SUB:RUS:%s:0" % synth.SPEECH_EVENT: "Две"})
+    assert ws.texts.current(SUB_KEY) == "Line two." and ws.texts.state(SUB_KEY) == MODIFIED
+    for p in ws.texts.files(SUB_KEY):                                # every copy
+        ws2 = _ws(root, tmp_path / "other")
+        assert ws2.texts.current(SUB_KEY) == "Line two."
+    with pytest.raises(subtitles.SubtitleError):
+        ws.texts.apply({SUB_KEY: "This line is far too long for the room"})
+    ws.texts.revert([SUB_KEY])
+    assert ws.texts.current(SUB_KEY) == "Scanner line two." and ws.texts.current("SUB:RUS:%s:0" % synth.SPEECH_EVENT) == "Две"
+    ws.texts.revert_all()
+    assert _files(root) == before and ws.texts.state(SUB_KEY) == ORIGINAL
+
+
+def test_subtitle_and_sound_in_one_package(synth_game, tmp_path):
+    root, _ = synth_game
+    ws = _ws(root, tmp_path)
+    before = _files(root)
+    s = ws.catalog.by_key["BANK:%08X" % synth.DLG_MEDIA]
+    ws.apply([(s, synth.wem(0.1, 1900), {})])
+    ws.texts.apply({SUB_KEY: "Both."})
+    assert ws.texts.current(SUB_KEY) == "Both." and ws.state(s) == "modified"
+    ws.revert([s])
+    assert ws.texts.current(SUB_KEY) == "Both." and ws.state(s) == "original"
+    ws.texts.revert_all()
+    assert _files(root) == before
+
+
+# --- menu text through the workspace ------------------------------------------------------------
+def test_menu_apply_revert(synth_game, tmp_path):
+    root, _ = synth_game
+    ws = _ws(root, tmp_path)
+    before = _files(root)
+    assert "INT" in ws.texts.menu_languages()
+    keys = {e.key: e for e in ws.texts.menu("INT")}
+    assert "INT:Version.int:ID:Unreal" not in keys
+    k = "MENU:INT:FRONTEND.int:Menu:Start"
+    ws.texts.apply({k: "Press Enter"})
+    assert ws.texts.current(k) == "Press Enter" and ws.texts.state(k) == MODIFIED
+    assert {e.key: e.value for e in ws.texts.menu("INT")}["INT:FRONTEND.int:Menu:Start"] == "Press Enter"
+    ws.texts.revert([k])
+    assert _files(root) == before
+
+
+def test_menu_changed_underneath(synth_game, tmp_path):
+    root, _ = synth_game
+    ws = _ws(root, tmp_path)
+    k = "MENU:INT:FRONTEND.int:Tips:Tip#2"
+    ws.texts.apply({k: "Glide"})
+    path = ws.texts.files(k)[0]
+    raw = open(path, "rb").read()
+    open(path, "wb").write(loctext.write(raw, "INT", "FRONTEND.int", {"INT:FRONTEND.int:Menu:Start": "Other tool"}))
+    with pytest.raises(TextConflictError):
+        ws.texts.apply({k: "Soar"})
+    ws.texts.apply({k: "Soar"}, force=True)
+    assert ws.texts.current(k) == "Soar"
+    ws.texts.revert_all()
+    assert open(path, "rb").read() == open(ws.store.full_copy(ws.game.rel(path)), "rb").read()
+
+
+# --- real game (read-only) ----------------------------------------------------------------------
+def test_real_subtitles_and_menu(real_game_dir, tmp_path):
+    ws = Workspace(Game(real_game_dir), store_root=str(tmp_path / "store"), archives=["ITA"])
+    lines = ws.texts.subtitles("ITA")
+    assert len(lines) > 9000
+    s = next(x for x in lines if x.sound)
+    assert ws.texts.current(subtitles.key("ITA", s.speech, s.line)) == s.text
+    menu = ws.texts.menu("ITA")
+    assert len(menu) > 7000 and all(e.lang == "ITA" for e in menu)
--- a/tests/test_workspace.py
+++ b/tests/test_workspace.py
@@ -136,7 +136,7 @@ def test_mod_export_import_roundtrip(synth_game, synth_template, tmp_path):
    ws2 = ws_for(other, store_root=str(tmp_path / "store2"))
    _, items = match_mod(ws2, pkg)
    assert [i.how for i in items] == ["exact", "exact"]
-    res = import_mod(ws2, pkg)
+    res, _ = import_mod(ws2, pkg)
    assert all(i.note == "exact copy" for i in res)
    assert ws2.current_bytes(ws2.catalog.by_key[a.key]) == wa
    assert ws2.current_bytes(ws2.catalog.by_key[b.key]) == wb