From 1ccaa8b398be952959e5d5ee5440fbff5fc60131 Mon Sep 17 00:00:00 2001 From: Cristian Gora Date: Tue, 13 Oct 2026 21:34:13 +0200 Subject: [PATCH] Full correctness check against the docs: safer writes, reverts and texts Safety - Writes are serialized (Workspace.lock): two background jobs rebuilding one package (a sound and its subtitle applied side by side) lost an edit. - A store copies a bank file or package as "original" only if it matches the shipped catalog (Workspace.unoriginal_files: in-memory sounds, bank copies of voice lines, subtitles); the editor refuses to switch stores while changes are applied. A second store could otherwise resurrect a reverted sound. - Mod import in the editor asks before overwriting files changed outside the tool (it forced silently). - Revert all in the editor reverts texts too; the uninstaller counts text edits before offering to delete the store. - ackit restore-backup: the full-backup restore decisions.md described; it keeps the sounds of WADs without a full copy recorded. - build_catalog also stops when a subtitle differs from the catalog. Languages and texts - Game.language() returns languages without voices (JPN, RUS, ...) and Game.voice_archive() their INT voices; ackit text --lang JPN works. - Menu lookups parse one cached file (find & replace: minutes -> ms). - Subtitles decode cp1252 as Windows does: the UTF-8-in-cp1252 line Speech_TAPE_J_Answer_1_00 (KOR/POL/POR/RUS) is written back byte-for-byte; SUB_KOR/SUB_RUS rebuilt from the original game (only those rows changed). - Mods exported while another language is loaded keep name and line. Other fixes - Down-mix folds extra channels in at -3 dB (a 5.1 centre voice was dropped). - e2e_editor no longer overwrites the user's settings.json; kill_game waits for the game to exit before reverts. - Recorder handles Int32/UInt8 input and split frames; package block cache locked; conflict messages, CLI help examples, stale docstrings and doc references fixed; tests no longer read this PC's game language. Docs (decisions, user guide, README, subtitles.md, banks.md, catalog README, REPORT, NOTES, CLAUDE.md) updated; check_docs: 102 claims match. 148 tests; e2e_features 9/9, e2e_editor and verify PASS; game original afterwards. --- CLAUDE.md | 5 +- NOTES.md | 36 +++++++++ README.md | 7 +- REPORT.md | 14 +++- ackit/audio.py | 26 ++++--- ackit/banks.py | 12 +-- ackit/catalog.py | 14 ++-- ackit/cli.py | 49 ++++++++---- ackit/game.py | 20 ++++- ackit/loctext.py | 18 ++++- ackit/modpack.py | 2 +- ackit/subtitles.py | 27 ++++++- ackit/texts.py | 131 +++++++++++++++++++++++-------- ackit/upk.py | 17 ++-- ackit/verify/catalogcheck.py | 7 +- ackit/verify/debugger.py | 7 +- ackit/verify/selfcheck.py | 4 +- ackit/workspace.py | 142 +++++++++++++++++++++++++++++----- app/audioio.py | 29 ++++--- app/editor.py | 4 +- app/mainwindow.py | 84 ++++++++++++-------- data/catalog/README.md | 2 +- data/catalog/SUB_KOR.csv | 2 +- data/catalog/SUB_RUS.csv | 2 +- docs/decisions.md | 28 +++++-- docs/formats/banks.md | 2 +- docs/formats/subtitles.md | 4 + docs/user-guide.md | 21 +++-- tests/conftest.py | 8 +- tests/test_app.py | 69 +++++++++++++++++ tests/test_catalog.py | 16 ++++ tests/test_cli.py | 20 +++++ tests/test_codec.py | 7 ++ tests/test_game.py | 14 ++++ tests/test_realgame.py | 2 +- tests/test_texts.py | 14 ++++ tests/test_workspace.py | 95 +++++++++++++++++++++++ tools/build_catalog.py | 25 ++++-- tools/check_docs.py | 26 +++++++ tools/e2e_editor.py | 8 +- tools/installer/uninstall.ps1 | 12 ++- 41 files changed, 846 insertions(+), 186 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6ca1267..e93cc94 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,8 @@ Batman: Arkham City (PC) audio toolkit + editor. Read `NOTES.md` first (status, - Layout: `ackit/` library + CLI (`python -m ackit -h`), `app/` PySide6 editor, `tests/` pytest, `data/catalog/` CSVs (+ user's original lists in `data/catalog/source/`, never edit them), `docs/`, `tools/` build scripts. - Dev env: `.venv` (Python 3.14, `pip install -e .[app,dev,verify]`). Run tests with `.venv/Scripts/python -m pytest -q`. -- Writing game files goes through `ackit.workspace.Workspace` only (originals store, atomic writes, conflict checks). +- Writing game files goes through `ackit.workspace.Workspace` only (originals store, atomic writes, conflict checks, + one write at a time: `Workspace.lock`, held by every apply/revert/rebuild incl. `Texts`). Speech/ambience: in place, never larger than the original WEM; music packs: any size (`docs/formats/wad.md`). In-memory sounds (archive BANK, loose `.bnk` + `.upk` packages): in place into every copy; a bank file is always rebuilt from the store's original copy + the applied replacements (`docs/formats/banks.md`, `upk.md`). @@ -15,7 +16,7 @@ Batman: Arkham City (PC) audio toolkit + editor. Read `NOTES.md` first (status, `ackit verify --ingame [--banks | --texts]` (Windows; needs Steam running and logged in, WinDbg; ~1.5 min; reverts and checks byte-identity itself). - Rebuild catalogs only from original game files: `python tools/build_catalog.py` (~5 min, scans all packages; - stops if a sound differs from the current catalog). Voice lines are identified by the game's Wwise Sound + stops if a sound or subtitle differs from the current catalog). Voice lines are identified by the game's Wwise Sound objects (`ackit/lines.py`, column `line`; one recording per language), never by WAD position or name: positions differ per language (`docs/decisions.md` 5, `docs/formats/banks.md` *Lines*). - Figures in the docs are checked, not copied: after changing data, code or docs run `python tools/check_docs.py` diff --git a/NOTES.md b/NOTES.md index 32f0e8f..969c978 100644 --- a/NOTES.md +++ b/NOTES.md @@ -126,6 +126,42 @@ action bar level with the others; A/B and Loop are icons next to the volume. Fits a 1000-px window without horizontal scrolling (minimum 984 px). Docs: recordings the game shares between languages (user saw English lines "leaking" into other languages: the game's own data, banks.md *Lines*). +- 2026-10-13 (user: "full check of the source code for correctness, also against the documentation"): figures + all matched (`check_docs --scan`), the formats code matched the docs; fixed: editor **Revert all** reverted + sounds only (docs and uninstaller promised everything; the uninstaller also counted only sound records and + could offer to delete the store while texts were modified) -> sounds + texts, uninstaller counts texts; + concurrent background writes to one package lost an edit (reproduced) -> `Workspace.lock` serializes every + write; editor mod import forced over conflicts silently -> asks; a second originals store (Settings / `--store`) + copied a package modified by the first as its "original" (reproduced: a replaced sound survived both reverts) + -> `Workspace.unoriginal_files` (catalog checksums, heads, subtitles; 1,447 files, no false alarm, ~15 s for + all) makes it a conflict and the editor refuses a store change while changes exist; `ackit text --lang JPN` + (text-only languages) failed -> `Game.voice_archive`; menu lookups reloaded all ~37 files per call (find & + replace of "the" in INT: ~4 min frozen) -> one cached file per lookup (0.04 s for 1,193 keys); + build_catalog's safety check ignored subtitles -> also compares them; `ackit restore-backup` added (decisions + named a restore that nothing exposed; it also forgot WAD sounds without a full copy); recorder handles + Int32/UInt8 input and split frames; stale docstrings, `ackit -h` examples, conflict message, case-insensitive + Localization path in `verify --catalog`. 146 tests. During the checks an e2e_editor game launch failed + ("code never decrypted") and its revert found the WAD still locked by the dying game (left Music_393BC53A + modified; reverted by hand, SFX catalog OK): `debugger.kill_game` now waits until the game has exited. +- 2026-10-13 second full check (user: "do this again to make sure"): codec (wwvorbis, codebooks, allocsize, ogg), + classify vs categories.md, verify harness, tools and widgets re-read; fixed: `tools/e2e_editor.py` saved its own + settings over the user's settings.json (lost store/device/background settings; it now uses the user's + settings and never saves); `Game.language()` ignored languages without voices (a JPN game opened the editor + and `ackit text` in INT; now the game's language, `Game.voice_archive` for its speech archive) and the tests + read this PC's UserEngine.ini (conftest isolates it); a mod exported while another language was loaded lost + name + line of that language's changes (`Workspace.sound_for_key`); down-mix dropped the extra channels (a + 5.1 centre-only voice became silent stereo; now folded in at -3 dB, peak-limited); `Texts.for_sound` default + for shared in-memory lines; package block cache locked (concurrent readers); `catalog._whole_recordings` -> + `whole_recordings` in docs/comments; the user guide no longer sends app users to the CLI-only restore. + 148 tests. +- 2026-10-13 third check (property checks on the real game data): every menu file rewrites byte-identically, + no ambiguous menu keys, every subtitle copy has the catalog's room; 2 of 103,658 subtitle strings could not be + written back as stored: one line (`Speech_TAPE_J_Answer_1_00`) is UTF-8 bytes in a cp1252 string in KOR, POL, + POR, RUS (the game's data; KOR's damaged), and the RUS/KOR ones contain the 5 bytes Python's cp1252 leaves + undefined (decoded as U+FFFD, lossy) -> cp1252 as Windows maps it (lossless; all 103,658 reproduce), catalog + rebuilt from the original game (`--accept-changes`: exactly those 2 rows changed), subtitles.md documents it + (claim derived by check_docs). User guide: Space plays the replacement once one is loaded (not always the + original); tooltip too. App smoke test on the real game PASS. - DONE by the user in game: replaced voice lines, edited subtitles and replaced footsteps (after the fill fix). - OPEN: channel order for >2 channels unverified; 19 speaker labels to check by listening; the Linux build was never built or run (no Linux here). diff --git a/README.md b/README.md index d8519a4..635f5fa 100644 --- a/README.md +++ b/README.md @@ -38,13 +38,14 @@ localization files. Details in `docs/`. ackit find joker tape # search by name, category or character ackit extract Speech_TAPE_JOKER_1_04 orig.wav # listen to the original (a line has one name in every language) ackit replace Speech_TAPE_JOKER_1_04 my_line.wav --match-loudness -ackit revert Speech_TAPE_JOKER_1_04 # or: ackit revert --all -ackit text find --lang INT protocol # subtitles and menu texts (keys SUB:... / MENU:...) +ackit revert Speech_TAPE_JOKER_1_04 # or: ackit revert --all (every sound and text) +ackit text find --lang INT protocol # subtitles and menu texts (keys SUB:... / MENU:...; any of the + # 11 languages, e.g. --lang JPN) ackit text set "MENU:INT:FRONTEND.int:Buttonpress:DoubleTap" "Tap twice" ``` More: `ackit status`, `ackit list --character Joker`, `ackit export-mod my.acmod` / `ackit import-mod`, `ackit verify` (offline self-check), `ackit verify --catalog` (catalog vs game files), `ackit game` (detected -install, language, store). `ackit -h` lists all commands. The game folder is found through Steam (Windows, Linux, Flatpak, Proton); otherwise pass `--game DIR`. +install, language, store), `ackit restore-backup` (last resort if the originals store is damaged). `ackit -h` lists all commands. The game folder is found through Steam (Windows, Linux, Flatpak, Proton); otherwise pass `--game DIR`. Rules worth knowing: speech/effects replacements must fit in the original's size (the tool squeezes or refuses with the maximum length); music can be any length. Subtitles must fit the original text's length (menu text: diff --git a/REPORT.md b/REPORT.md index 84129f8..d53eb33 100644 --- a/REPORT.md +++ b/REPORT.md @@ -41,8 +41,14 @@ editor Unheard City (mission 2). export/import with texts, changed-outside banner for sounds and texts. - **Safety**: every changed file (WAD, bank, package, localization file) is rebuilt from a verified original copy in the store + the edits applied now; reverting the last edit restores it byte-for-byte; files changed by - another tool or Steam are detected (whole-file SHA-1) and not overwritten without confirmation. -- **Tests**: 135 automated tests (unit, synthetic game with packages/banks/dialogues/localization, a + another tool or Steam are detected (whole-file SHA-1) and not overwritten without confirmation (also when + installing a mod in the editor). The original copy is taken only from a file that matches the catalog + (another tool's or another originals store's change is a conflict; checked on all 1,447 files with sounds or + subtitles: no false alarm), and the editor does not switch stores while changes are applied. Writes are + serialized, so a sound and its subtitle applied at the same time both reach the package. **Revert all** + (editor and `ackit revert --all`) reverts sounds and texts; `ackit restore-backup` is the last resort for a + damaged store. +- **Tests**: 148 automated tests (unit, synthetic game with packages/banks/dialogues/localization, a two-language game for the lines (incl. a recording also stored whole in a bank), the shipped catalog's cross-references, real-game read-only (every name of every language's catalog resolves to its own sound), offscreen GUI incl. list order, selection and loading overlays) pass. `ackit verify` PASS (incl. @@ -60,7 +66,9 @@ editor Unheard City (mission 2). - **Package**: `dist/UnheardCity-3.2.3-windows-amd64.zip` (86 MB, catalogs included). With `PATH` reduced to the Windows folders the packaged exe passes its smoke test (in the game's language - Italian on the test PC: 21,015 sounds loaded, encode + inject, 9,377 subtitles and 7,020 menu texts loaded, an in-memory sound decoded from a package) and the GUI starts and - quits cleanly. The installer is unchanged since mission 2 (install/upgrade/uninstall verified then). + quits cleanly. The installer is unchanged since mission 2 (install/upgrade/uninstall verified then); the + uninstaller now also counts text edits before offering to delete the user data (2026-10-13; that count was + tested, the full uninstall cycle was not run again). ## Unresolved / limitations - **Checked by a person in game** (2026-10-10), not automatically (nothing of this plays or shows on the title diff --git a/ackit/audio.py b/ackit/audio.py index 95fabab..d35bcdf 100644 --- a/ackit/audio.py +++ b/ackit/audio.py @@ -53,21 +53,29 @@ def resample(x, rate, new_rate): def to_channels(x, channels): - """Up/down-mix. Mono is copied to every channel; extra channels get the average.""" + """Up/down-mix. Up: mono is copied to every channel, other sources keep their channels and the new ones get + the average. Down: nothing is dropped - every channel goes into the mono sum, or the channels beyond the + kept ones (e.g. the centre and surrounds of 5.1) are added to every kept one at -3 dB (the standard + fold-down); a result louder than the source's peak is scaled back to it, so it does not clip.""" x = _as2d(x) cur = x.shape[1] if cur == channels: return x - if channels == 1: - return x.mean(axis=1, keepdims=True) if cur == 1: return np.repeat(x, channels, axis=1) - out = np.empty((len(x), channels), dtype=np.float32) - keep = min(cur, channels) - out[:, :keep] = x[:, :keep] - if channels > keep: - out[:, keep:] = x.mean(axis=1, keepdims=True) - return out + if channels > cur: + out = np.empty((len(x), channels), dtype=np.float32) + out[:, :cur] = x + out[:, cur:] = x.mean(axis=1, keepdims=True) + return out + if channels == 1: + out = x.sum(axis=1, keepdims=True) + else: + out = x[:, :channels] + 0.7071 * x[:, channels:].sum(axis=1, keepdims=True) + peak_in, peak_out = float(np.max(np.abs(x), initial=0.0)), float(np.max(np.abs(out), initial=0.0)) + if peak_out > peak_in > 0: + out = out * (peak_in / peak_out) + return out.astype(np.float32) def trim(x, rate, start=None, end=None): diff --git a/ackit/banks.py b/ackit/banks.py index a8fe037..d06b387 100644 --- a/ackit/banks.py +++ b/ackit/banks.py @@ -4,10 +4,12 @@ in the .upk packages (docs/formats/banks.md, upk.md). Catalog archive `BANK`, ta A media ID can be stored many times (the same bank copied into several packages, or several banks holding the same media). The location index (data/catalog/BANK_locations.csv.gz) lists every copy: media, file (relative to CookedPCConsole, '/'-separated), offset (in the *uncompressed* file), size, kind -kind 'm' = an in-memory sound, 'p' = the in-memory head of a streamed WAD sound (prefetch); for 'p' rows -`sound` is the WAD sound's key (e.g. ITA:2B14E472:3): speech lines have per-language media IDs and heads. -Replacements are written in place into every copy (<= original size, zero-padded), so bank and package -layouts never change. +kind 'm' = an in-memory sound, 'p' = a bank copy of a streamed WAD recording (the in-memory head of a prefetched +sound, or a whole copy with the same media ID); for 'p' rows `sound` is the WAD sound's key (e.g. ITA:2B14E472:3): +speech lines have per-language media IDs and heads. +Replacements are written in place into every copy, no larger than the smallest; a whole in-memory WEM fills its +slot exactly (wem.fill: the game plays a zero-padded one as silence), a head gets the start of the new +recording. So bank and package layouts never change. """ import csv import gzip @@ -157,7 +159,7 @@ class Scan: self.actions = {} # id -> target id self.sound_media = defaultdict(set) # sound object id -> media ids self.prefetch = set() # media ids whose bank copies belong to a WAD recording (prefetch - # heads; whole recordings: catalog._whole_recordings) + # heads; whole recordings: catalog.whole_recordings) self.lang_media = defaultdict(set) # language -> media ids named by Sound objects of that language's banks # BKPK tables: WAD entry id -> {language: {media id: (offset in entry, size)}}. A table belongs to the # language its localized AkBank export declares ('' = outside such an export). The same media ID can be diff --git a/ackit/catalog.py b/ackit/catalog.py index a6e4efd..eb3e6ec 100644 --- a/ackit/catalog.py +++ b/ackit/catalog.py @@ -1,10 +1,10 @@ -"""Index of every sound in the game's WAD archives, with names, categories and characters -(sounds stored inside soundbanks are not indexed yet: docs/formats/banks.md). +"""Index of every sound of the game, with names, categories and characters: the WAD archives (one per voice +language, SFX) and the in-memory sounds stored inside soundbanks (archive BANK, docs/formats/banks.md). Sources, in order: - 1. data/catalog/.csv -- shipped catalog (instant) when the WAD fingerprint matches; - 2. otherwise a header scan of the WAD (a few seconds, cached in the user cache), classified with - `classify` and labelled with data/catalog/characters.csv (speaker per sound name). + 1. data/catalog/.csv -- shipped catalog (instant) when the game's fingerprint matches; + 2. otherwise a scan of the game (WAD headers: seconds; banks: minutes; cached in the user cache), classified + with `classify` and labelled with data/catalog/characters.csv (speaker per sound name). Names come from the game: a WAD entry ID is the Wwise ID (FNV-1) of a soundbank listed in the game's CookedPCConsole/CookedBanks.txt (`bank_names`); an entry with one WEM is NAME (alias NAME_1), entries with n variations are NAME_1..NAME_n (alias NAME#i); stream-pack media are @@ -517,14 +517,16 @@ class Catalog: def __init__(self, game, archives=None, progress=None): self.game = game avail = available_archives(game) - self.archives = archives or [a for a in (game.language(), "SFX", banks.ARCHIVE) if a in avail] + self.archives = archives or [a for a in (game.voice_archive(game.language()), "SFX", banks.ARCHIVE) if a in avail] self.sounds = [] self.locations = {} # BANK media id -> [Location] self.heads = {} # WAD sound key -> [Location] of its prefetch heads + self.bank_sha1 = {} # BANK media id -> original checksum ('' if unknown), every language bank_sounds_ = [] if banks.ARCHIVE in avail: # always loaded: speech lines may have prefetch heads that must be written with them bank_sounds_, self.locations = load_banks(game, progress) + self.bank_sha1 = {int(s.target, 16): s.sha1 for s in bank_sounds_} for ls in self.locations.values(): for l in ls: if l.kind == banks.PREFETCH_HEAD and l.sound: diff --git a/ackit/cli.py b/ackit/cli.py index 8f8820c..1d3407b 100644 --- a/ackit/cli.py +++ b/ackit/cli.py @@ -1,15 +1,17 @@ """ackit - Batman: Arkham City audio toolkit (command line). Everyday use, by sound name (see `ackit find`): - ackit find joker tape search names, categories, characters - ackit info Speech_TAPE_JOKER_1_4 details, size limit, modified state - ackit extract Speech_TAPE_JOKER_1_4 orig.wav save the original (or --current) - ackit replace Speech_TAPE_JOKER_1_4 line.wav encode + install a replacement - ackit revert Speech_TAPE_JOKER_1_4 put the original back (--all for everything) - ackit status what you have changed + ackit find joker tape search names, categories, characters + ackit info Speech_TAPE_JOKER_1_04 details, size limit, modified state + ackit extract Speech_TAPE_JOKER_1_04 orig.wav save the original (or --current) + ackit replace Speech_TAPE_JOKER_1_04 line.wav encode + install a replacement + ackit revert Speech_TAPE_JOKER_1_04 put the original back (--all: every sound and text) + ackit status what you have changed + ackit text find --lang JPN batman subtitles and menu text (any of the 11 languages) ackit export-mod my.acmod / import-mod my.acmod share your changes -The game folder is detected from Steam (Windows, Linux, Proton); override with --game DIR. +The game folder is detected from Steam (Windows, Linux, Proton); override with --game DIR. --lang picks the +language (default: the game's); languages without their own voices use the INT voice lines, like the game. """ import argparse import os @@ -36,7 +38,7 @@ def _workspace(a, archives=None): st = settings.load() game = _game(a) if archives is None and getattr(a, "lang", None): - archives = [a.lang, "SFX", "BANK"] + archives = [game.voice_archive(a.lang), "SFX", "BANK"] return Workspace(game, a.store or st.get("store_root"), archives, full_backup=not getattr(a, "no_full_backup", False) and st.get("full_backup", True), progress=lambda n, t: None) @@ -73,7 +75,8 @@ def cmd_list(a): c = a.character.lower() rows = [s for s in rows if c in s.character.lower() or (a.inferred and c in s.characters_inferred.lower())] if a.modified: - rows = [s for s in rows if s.key in ws.modified_keys()] + mod = set(ws.modified_keys()) + rows = [s for s in rows if s.key in mod] if a.csv: from .catalog import write_csv write_csv(a.csv, rows) @@ -178,6 +181,23 @@ def cmd_revert(a): print("reverted %d sound(s)" % len(done)) +def cmd_restore_backup(a): + ws = _workspace(a) + full = sorted(ws.store.data.get("full", {})) + if not full: + print("no full copies in this store (%s)" % ws.store.root) + return + if not a.yes: + print("would copy back (or rebuild) %d file(s) from the full copies in %s:" % (len(full), ws.store.root)) + for rel in full: + print(" " + rel) + print("Sounds and texts in these files are forgotten; sounds of WADs without a full copy stay recorded.\n" + "Use this only if the originals store is damaged (normally: ackit revert --all). Run again with --yes.") + return + done = ws.restore_from_full_backup(progress=print) + print("restored %d file(s); %d sound(s) still recorded" % (len(done), len(ws.store.entries))) + + def cmd_status(a): ws = _workspace(a) keys = ws.modified_keys() @@ -286,7 +306,7 @@ def cmd_game(a): from .workspace import default_store_root print("game folder: ", g.root) print("archives: ", ", ".join("%s=%s" % (k, g.rel(v)) for k, v in g.archives.items())) - print("speech language: ", g.language()) + print("language: ", "%s (voices: %s)" % (g.language(), g.voice_archive(g.language()))) print("originals store: ", a.store or st.get("store_root") or default_store_root(g)) print("settings file: ", settings.path()) @@ -311,7 +331,7 @@ def cmd_verify(a): ok = selfcheck.run(ws, log=print) sys.exit(0 if ok else 1) from .verify import ingame - if a.lang and a.lang != ws.game.language(): + if a.lang and a.lang != ws.game.language(): # voices and texts follow the game's setting raise GameError("the game plays its own language (%s); set it in Steam (game properties > Language) to " "test %s in the game" % (ws.game.language(), a.lang)) if a.banks: @@ -371,9 +391,12 @@ def build_parser(): s.add_argument("name"); s.add_argument("audio") s = add("apply", cmd_apply, "replace every sound named by the files in a folder", (enc,)) s.add_argument("folder") - s = add("revert", cmd_revert, "put original sounds back", aliases=["restore"]) + s = add("revert", cmd_revert, "put original sounds back (--all: every sound and text)", aliases=["restore"]) s.add_argument("names", nargs="*"); s.add_argument("--all", action="store_true") - add("status", cmd_status, "list the sounds you modified") + add("status", cmd_status, "list the sounds and texts you modified") + s = add("restore-backup", cmd_restore_backup, "last resort if the originals store is damaged: put back the " + "full file copies it keeps (lists them; --yes to do it)") + s.add_argument("--yes", action="store_true") s = add("text", cmd_text, "subtitles and menu text: find WORDS | show KEY | set KEY TEXT | revert KEY/--all") s.add_argument("action", choices=["find", "show", "set", "revert"]) s.add_argument("args", nargs="*") diff --git a/ackit/game.py b/ackit/game.py index cd4f394..7e76caf 100644 --- a/ackit/game.py +++ b/ackit/game.py @@ -123,6 +123,14 @@ class Game: def languages(self): return [a for a in self.archives if a != "SFX"] + def voice_archive(self, lang): + """The speech archive used for a language: its own, or INT for the languages the game has no voices for + (the game plays the INT voices with their own subtitles and menu text).""" + lang = (lang or "INT").upper() + if lang in self.languages: + return lang + return "INT" if "INT" in self.languages else self.languages[0] + def wad(self, archive): try: return self.archives[archive.upper()] @@ -153,8 +161,16 @@ class Game: dirs.append(os.path.join(pfx, docs, "WB Games", "Batman Arkham City GOTY", "BmGame", "Config")) return [d for d in dirs if os.path.isdir(d)] + def all_languages(self): + """Every language of the game: the voice languages (speech WADs) and the text languages (folders of + BmGame/Localization; the game plays the INT voices for those without their own).""" + loc = _iglob_one(self.root, "BmGame", "Localization") + texts = [d.upper() for d in os.listdir(loc) if os.path.isdir(os.path.join(loc, d))] if loc else [] + return sorted(set(self.languages) | set(texts)) + def language(self): - """Speech language the game is set to (falls back to INT, then the first available).""" + """The language the game is set to (voices or texts only, e.g. JPN; `voice_archive` gives its speech + archive); falls back to INT, then the first voice language.""" for d in self.config_dirs(): for f in ("UserEngine.ini", "BmEngine.ini"): p = os.path.join(d, f) @@ -163,7 +179,7 @@ class Game: raw = open(p, "rb").read() txt = raw.decode("utf-16") if raw[:2] in (b"\xff\xfe", b"\xfe\xff") else raw.decode("latin-1") m = re.search(r"^\s*Language\s*=\s*(\w+)", txt, re.M | re.I) - if m and m.group(1).upper() in self.archives: + if m and m.group(1).upper() in self.all_languages(): return m.group(1).upper() if "INT" in self.archives: return "INT" diff --git a/ackit/loctext.py b/ackit/loctext.py index a812890..3c8411d 100644 --- a/ackit/loctext.py +++ b/ackit/loctext.py @@ -64,6 +64,21 @@ def parse(raw, lang, fname): return out +def encoding(raw): + """Encoding of a file's bytes: 'utf-16', 'utf-8' or 'cp1252'.""" + return "utf-16" if raw[:2] == b"\xff\xfe" else "utf-8" if raw[:3] == b"\xef\xbb\xbf" else "cp1252" + + +def check_value(value, enc, fname): + """Raise TextError if `value` cannot be written into a file of encoding `enc` (what `write` refuses).""" + if "\r" in value or "\n" in value: + raise TextError("a menu text cannot contain line breaks (use the game's own markers, e.g. \\n)") + try: + _encode(value, enc) + except UnicodeEncodeError: + raise TextError("%s is stored as %s and cannot hold these characters" % (fname, enc)) from None + + def write(raw, lang, fname, values): """New file bytes with {entry key: new value} applied (keys not in the file raise TextError).""" text, enc = _decode(raw) @@ -73,8 +88,7 @@ def write(raw, lang, fname, values): e = by_key.get(key) if e is None: raise TextError("no text %s in %s" % (key, fname)) - if "\r" in value or "\n" in value: - raise TextError("a menu text cannot contain line breaks (use the game's own markers, e.g. \\n)") + check_value(value, enc, fname) line = lines[e.line] end = line[len(line.rstrip("\r\n")):] head = line[:line.index("=") + 1] diff --git a/ackit/modpack.py b/ackit/modpack.py index 0c9008e..e7f191e 100644 --- a/ackit/modpack.py +++ b/ackit/modpack.py @@ -41,7 +41,7 @@ def export_mod(ws, path, keys, name="", author="", description="", texts=()): e = ws.store.get(key) if not e or not e.get("applied"): raise ValueError("%s is not modified" % key) - s = ws.catalog.by_key.get(key) + s = ws.sound_for_key(key) # also a change made in another language (its name and line) wem = ws.store.get_blob(e["applied"]) x, sr = audio.decode_wem(wem) buf = io.BytesIO() diff --git a/ackit/subtitles.py b/ackit/subtitles.py index 2199711..501fbae 100644 --- a/ackit/subtitles.py +++ b/ackit/subtitles.py @@ -36,12 +36,33 @@ class SubtitleError(ValueError): pass +# Windows-1252 as Windows itself maps it: the 5 bytes Python's cp1252 leaves undefined stand for the same code +# points (U+0081 ...). One line is UTF-8 bytes in a cp1252 FString in 4 languages (Speech_TAPE_J_Answer_1_00, +# docs/formats/subtitles.md); the RUS and KOR ones contain such bytes. Read and written back losslessly, as stored. +_CP1252_UNDEFINED = frozenset((0x81, 0x8D, 0x8F, 0x90, 0x9D)) + + +def _decode_cp1252(raw): + try: + return raw.decode("cp1252") + except UnicodeDecodeError: + return "".join(chr(x) if x in _CP1252_UNDEFINED else bytes([x]).decode("cp1252") for x in raw) + + +def _encode_cp1252(text): + """UnicodeEncodeError for a character that is not in Windows-1252.""" + try: + return text.encode("cp1252") + except UnicodeEncodeError: + return b"".join(bytes([ord(c)]) if ord(c) in _CP1252_UNDEFINED else c.encode("cp1252") for c in text) + + def read_fstring(b, o): """-> (text, end offset, enc)""" n = struct.unpack_from("= 0: raw = bytes(b[o + 4:o + 4 + n]) - return raw.split(b"\0", 1)[0].decode("cp1252", "replace"), o + 4 + n, "a" + return _decode_cp1252(raw.split(b"\0", 1)[0]), o + 4 + n, "a" raw = bytes(b[o + 4:o + 4 - 2 * n]) return raw.decode("utf-16-le", "replace").split("\0", 1)[0], o + 4 - 2 * n, "w" @@ -53,7 +74,7 @@ def encode_fstring(text, size, enc): raise SubtitleError("the text must not contain NUL characters") if enc == "a": try: - raw = text.encode("cp1252") + raw = _encode_cp1252(text) except UnicodeEncodeError: raise SubtitleError("this subtitle is stored as Western European (Windows-1252) text; '%s' cannot be " "written" @@ -72,7 +93,7 @@ def encode_fstring(text, size, enc): def _cp1252(c): try: - c.encode("cp1252") + _encode_cp1252(c) return True except UnicodeEncodeError: return False diff --git a/ackit/texts.py b/ackit/texts.py index 87c5811..fb54cd9 100644 --- a/ackit/texts.py +++ b/ackit/texts.py @@ -5,31 +5,45 @@ Keys (also used in mod packages): MENU:::
:[#n] a menu/UI text (docs/formats/text.md) Records live in the originals store (data["texts"]): {key: {"orig", "applied", "files", ...}}. The files holding the texts are derived files of the Workspace: each is rebuilt from its original copy plus every edit -applied now (sounds and texts), so reverting the last edit restores it byte-for-byte. +applied now (sounds and texts), so reverting the last edit restores it byte-for-byte. Writes hold the +workspace's lock (Workspace.lock). """ import os import time -from . import loctext, subtitles -from .catalog import load_subtitles +from . import banks, loctext, subtitles +from .catalog import _shipped_index, catalog_dir, load_subtitles from .game import GameError ORIGINAL, MODIFIED, LOST, CHANGED, FOREIGN = "original", "modified", "lost", "changed", "foreign" +def _locked(f): + """Run a Texts method holding the workspace's write lock.""" + def run(self, *a, **kw): + with self.ws.lock: + return f(self, *a, **kw) + run.__name__, run.__doc__ = f.__name__, f.__doc__ + return run + + class TextConflictError(GameError): def __init__(self, keys): self.keys = keys super().__init__("These texts were changed outside this tool (Steam update or 'verify', or another mod " - "tool): %s.\nRevert them or confirm to overwrite." % ", ".join(keys[:5]) - + (" ..." if len(keys) > 5 else "")) + "tool): %s.\nRevert them or confirm to overwrite." + % (", ".join(keys[:5]) + (" ..." if len(keys) > 5 else ""))) class Texts: def __init__(self, ws): self.ws = ws self._subs = {} # lang -> ([SubLine], {key: [SubLoc]}) + self._sub_text = {} # lang -> {(speech, line): original text} (catalog) self._menu = {} # lang -> [Entry] + self._menu_paths = {} # lang -> {file name: path} + self._menu_file = {} # path -> ((size, mtime), encoding, {entry key: Entry}) + self._shipped_subs = None # (by file: [SubLoc], {(lang, speech, line): text}) of the shipped catalog self.loc_root = _find_dir(ws.game.root, "BmGame", "Localization") # --- languages and lists --------------------------------------------------------------------- @@ -37,7 +51,6 @@ class Texts: return loctext.languages(self.loc_root) if self.loc_root else [] def subtitle_languages(self): - from .catalog import _shipped_index langs = sorted(_shipped_index().get("SUB", {}).get("lines", {})) return langs or self.menu_languages() @@ -54,8 +67,11 @@ class Texts: def for_sound(self, sound, lang=None): """Subtitle lines of a speech sound, in `lang` (default: the sound's own language). Languages without - their own voices link their subtitles to the INT sounds, so e.g. JPN subtitles are found for INT lines.""" - lang = lang or (sound.languages[0] if sound.languages else sound.archive) + their own voices link their subtitles to the INT sounds, so e.g. JPN subtitles are found for INT lines. + A sound shared by every language (an in-memory line stored once) defaults to the game's language.""" + game = self.ws.game + lang = lang or (sound.languages[0] if sound.languages else + sound.archive if sound.archive in game.languages else game.language()) if lang not in self.subtitle_languages(): return [] return [s for s in self.subtitles(lang) if sound.key in s.sounds] @@ -77,20 +93,44 @@ class Texts: def modified(self): return [k for k, r in self.records().items() if r.get("applied") is not None] + def _menu_path(self, key): + """Path of the file holding menu text `key` (MENU:::...).""" + parts = key.split(":", 3) + if len(parts) < 4 or not self.loc_root: + raise GameError("no menu text %s" % key[len("MENU:"):]) + lang = parts[1] + if lang not in self._menu_paths: + self._menu_paths[lang] = loctext.files(self.loc_root, lang) + path = self._menu_paths[lang].get(parts[2]) + if not path: + raise GameError("no menu text %s" % key[len("MENU:"):]) + return path + + def _menu_entries(self, path, lang): + """(encoding, {entry key: Entry}) of one menu file as it is now (parsed again only when it changed).""" + st = os.stat(path) + stamp = (st.st_size, st.st_mtime_ns) + hit = self._menu_file.get(path) + if not hit or hit[0] != stamp: + raw = open(path, "rb").read() + hit = (stamp, loctext.encoding(raw), + {e.key: e for e in loctext.parse(raw, lang.upper(), os.path.basename(path))}) + self._menu_file[path] = hit + return hit[1], hit[2] + def _menu_entry(self, key): - lang = key.split(":")[1] - raw = key[len("MENU:"):] - for e in loctext.load(self.loc_root, lang): - if e.key == raw: - return e - raise GameError("no menu text %s" % raw) + path = self._menu_path(key) + e = self._menu_entries(path, key.split(":")[1])[1].get(key[len("MENU:"):]) + if e is None: + raise GameError("no menu text %s" % key[len("MENU:"):]) + return e def files(self, key): """Absolute paths of the files holding a text.""" if key.startswith("SUB:"): return sorted({self.ws.bank_path(l.file) for l in self.subtitle_locations(key)}) - e = self._menu_entry(key) - return [loctext.files(self.loc_root, e.lang)[e.file]] + self._menu_entry(key) + return [self._menu_path(key)] def current(self, key): """The text in the game files now.""" @@ -98,22 +138,20 @@ class Texts: locs = self.subtitle_locations(key) if not locs: raise GameError("no subtitle %s" % key) - from .banks import open_file l = locs[0] - b = open_file(self.ws.bank_path(l.file)).read(l.pos, l.size) + b = banks.open_file(self.ws.bank_path(l.file)).read(l.pos, l.size) return subtitles.read_fstring(b, 0)[0] return self._menu_entry(key).value def catalog_text(self, key): - """Original text according to the shipped catalog (subtitles only), or None.""" + """Original text according to the catalog (subtitles only), or None.""" if not key.startswith("SUB:"): return None _, lang, rest = key.split(":", 2) speech, line = rest.rsplit(":", 1) - for s in self.subtitles(lang): - if s.speech == speech and s.line == int(line): - return s.text - return None + if lang not in self._sub_text: + self._sub_text[lang] = {(s.speech, s.line): s.text for s in self.subtitles(lang)} + return self._sub_text[lang].get((speech, int(line))) def original(self, key): r = self.records().get(key) @@ -141,24 +179,54 @@ class Texts: for l in self.subtitle_locations(key)), default=0) def check(self, key, text): - """Raise SubtitleError / TextError if `text` cannot be written for `key`.""" + """Raise SubtitleError / TextError (ValueError) if `text` cannot be written for `key`, GameError if + there is no such text.""" if key.startswith("SUB:"): for l in self.subtitle_locations(key): subtitles.encode_fstring(text, l.size, l.enc) else: - e = self._menu_entry(key) - path = loctext.files(self.loc_root, e.lang)[e.file] - loctext.write(open(path, "rb").read(), e.lang, e.file, {e.key: text}) + path = self._menu_path(key) + self._menu_entry(key) + loctext.check_value(text, self._menu_entries(path, key.split(":")[1])[0], os.path.basename(path)) + + def unoriginal_subtitles(self, rel): + """Subtitle strings in package `rel` (cooked-relative) that differ from the shipped catalog: [SUB key] + (none if the catalog is not the game's).""" + if self._shipped_subs is None: + self._shipped_subs = ({}, {}) + shipped = _shipped_index().get("SUB", {}) + loc = os.path.join(catalog_dir(), subtitles.LOCATIONS) + if shipped.get("fingerprint") == banks.game_fingerprint(self.ws.game.cooked) and os.path.exists(loc): + by_file, texts = self._shipped_subs + for ls in subtitles.read_locations(loc).values(): + for l in ls: + by_file.setdefault(l.file, []).append(l) + for lang in shipped.get("lines", {}): + p = os.path.join(catalog_dir(), "SUB_%s.csv" % lang) + if os.path.exists(p): + texts.update({(lang, s.speech, s.line): s.text for s in subtitles.read_texts(p, lang)}) + by_file, texts = self._shipped_subs + out = [] + locs = sorted(by_file.get(rel, []), key=lambda l: l.pos) + if locs: + r = banks.open_file(self.ws.bank_path(rel)) + for l in locs: + want = texts.get((l.lang, l.speech, l.line)) + if want is not None and subtitles.read_fstring(r.read(l.pos, l.size), 0)[0] != want: + out.append(subtitles.key(l.lang, l.speech, l.line)) + return out # --- writing -------------------------------------------------------------------------------- + @_locked def apply(self, changes, force=False, progress=None, meta=None): """changes: {key: new text}. Every file holding one of them is rebuilt once.""" for k, v in changes.items(): self.check(k, v) conflicts = [k for k in changes if self.state(k) in (CHANGED, FOREIGN)] - paths = {p for k in changes for p in self.files(k)} - changed = set(self.ws.changed_files(paths)) - conflicts += [k for k in changes if k not in conflicts and set(self.files(k)) & changed] + files = {k: self.files(k) for k in changes} + paths = {p for ps in files.values() for p in ps} + changed = set(self.ws.changed_files(paths)) | set(self.ws.unoriginal_files(paths)) + conflicts += [k for k in changes if k not in conflicts and set(files[k]) & changed] if conflicts and not force: raise TextConflictError(conflicts) for p in paths: @@ -167,12 +235,13 @@ class Texts: now = time.time() for k, v in changes.items(): if k not in recs: - recs[k] = {"orig": self.current(k), "files": [self.ws.game.rel(p) for p in self.files(k)], + recs[k] = {"orig": self.current(k), "files": [self.ws.game.rel(p) for p in files[k]], "first_modified": now, "applied": None} recs[k].update(applied=v, applied_at=now, meta=meta or {}) self.ws.store.save() self._sync(paths, progress) + @_locked def revert(self, keys, progress=None): recs = self.records() keys = [k for k in keys if k in recs] diff --git a/ackit/upk.py b/ackit/upk.py index d0b0237..b495ea0 100644 --- a/ackit/upk.py +++ b/ackit/upk.py @@ -8,6 +8,7 @@ Patching rewrites only the blocks that contain changed bytes (stored as one LZO lzo.compress_stored) and moves the chunks after them; every other byte of the file stays identical. """ import struct +import threading from collections import OrderedDict, namedtuple from . import lzo @@ -41,6 +42,7 @@ class Package: self.file_size = f.seek(0, 2) self._cache = OrderedDict() self._cache_n = cache_blocks + self._lock = threading.Lock() # summary ----------------------------------------------------------------------------------- def _parse_summary(self, b, f): @@ -157,9 +159,11 @@ class Package: # reading ----------------------------------------------------------------------------------- def _block(self, i, f=None): - if i in self._cache: - self._cache.move_to_end(i) - return self._cache[i] + with self._lock: # the editor reads one package from several background jobs + data = self._cache.get(i) + if data is not None: + self._cache.move_to_end(i) + return data bl = self.blocks[i] own = f is None f = f or open(self.path, "rb") @@ -169,9 +173,10 @@ class Package: finally: if own: f.close() - self._cache[i] = data - if len(self._cache) > self._cache_n: - self._cache.popitem(last=False) + with self._lock: + self._cache[i] = data + if len(self._cache) > self._cache_n: + self._cache.popitem(last=False) return data def _block_index(self, off): diff --git a/ackit/verify/catalogcheck.py b/ackit/verify/catalogcheck.py index 1907f26..0c3953f 100644 --- a/ackit/verify/catalogcheck.py +++ b/ackit/verify/catalogcheck.py @@ -399,8 +399,9 @@ def check_speakers(rep, rows, col): def check_menu(rep, game): """Menu texts: the same keys in every language (facts; differences are the game's own).""" from .. import loctext - root = os.path.join(game.root, "BmGame", "Localization") - if not os.path.isdir(root): + from ..texts import _find_dir + root = _find_dir(game.root, "BmGame", "Localization") # case-insensitive (Linux) + if not root: return keys = {} for lang in loctext.languages(root): @@ -443,7 +444,7 @@ def run(game, archives=None, quick=False, log=print, progress=None): check_subtitle_lines(rep, rows_all) check_speakers(rep, rows_all, col) check_menu(rep, game) - characters = load_characters(lang=game.language()) + characters = load_characters(lang=game.voice_archive(game.language())) known = {n.lower() for s in cat.sounds for n in s.names()} # speaker names are the same in every language; only meaningful for the build the lists were made for if any(a in game.languages and is_shipped(game, a) for a in archives): diff --git a/ackit/verify/debugger.py b/ackit/verify/debugger.py index 50bfcd1..bd9719e 100644 --- a/ackit/verify/debugger.py +++ b/ackit/verify/debugger.py @@ -104,9 +104,14 @@ def running_game_pids(): return [int(m.group(1)) for m in re.finditer(r'"BatmanAC\.exe","(\d+)"', out)] -def kill_game(): +def kill_game(timeout=20): + """Kill the game and wait until it has exited: it keeps the WADs and packages open until then, so a revert + right after taskkill would find them locked.""" for pid in running_game_pids(): subprocess.run(["taskkill", "/F", "/PID", str(pid)], capture_output=True) + t0 = time.time() + while running_game_pids() and time.time() - t0 < timeout: + time.sleep(0.25) def launch_game(exe, extra_args=()): diff --git a/ackit/verify/selfcheck.py b/ackit/verify/selfcheck.py index 0c81513..f470064 100644 --- a/ackit/verify/selfcheck.py +++ b/ackit/verify/selfcheck.py @@ -65,7 +65,9 @@ def run(ws, sample=300, log=print): if n >= sample // max(1, len(ws.game.archives)): break log("%-4s layout %s, rebuilt %d/%d sample WEMs byte-identical" % (arch, "OK" if not probs else probs[:2], good, n)) - ok &= not probs and good >= n - 2 # 4 looping music tracks are known exceptions + # known exceptions (docs/formats/wem.md): a few SFX WEMs (music seek tables and sample counts, 4-channel + # ambience with a loop chunk) and one INT speech WEM (avg-bytes/s field) are not reproduced byte-for-byte + ok &= not probs and good >= n - 2 bank = [s for s in ws.catalog.sounds if s.archive == "BANK"] if bank: n = good = 0 diff --git a/ackit/workspace.py b/ackit/workspace.py index cbb252d..4039817 100644 --- a/ackit/workspace.py +++ b/ackit/workspace.py @@ -10,14 +10,19 @@ In-memory sounds (archive BANK) and the prefetch heads of speech lines live in l packages (ackit/banks.py); a replacement fills each in-memory slot exactly (wem.fill). Before the first change to such a file a verified full copy is kept, and the file is always rebuilt as "original copy + every replacement currently applied", so reverting the last -change restores it byte-for-byte and any later operation repairs an interrupted one. +change restores it byte-for-byte and any later operation repairs an interrupted one. That copy is taken only +from a file that is original according to the catalog (`unoriginal_files`), else the write is a conflict. +Writes are serialized (`Workspace.lock`): the editor runs them as background jobs, and two rebuilds of one +file at the same time would lose one of the edits. """ +import functools import hashlib import os +import threading from collections import defaultdict from . import audio, banks -from .catalog import Catalog +from .catalog import Catalog, available_archives, is_shipped, load_archive from .encoder import EncodeError, encode_fit from .game import Game, GameError from .inject import inject, read_target, parse_target, pack_bnk_path, SizeLimitError @@ -41,25 +46,50 @@ class ConflictError(GameError): self.sounds = sounds super().__init__("These sounds were changed outside this tool (Steam update or 'verify', or " "another mod tool): %s.\nRevert them or confirm to overwrite." - % ", ".join(s.name for s in sounds[:5]) + (" ..." if len(sounds) > 5 else "")) + % (", ".join(s.name for s in sounds[:5]) + (" ..." if len(sounds) > 5 else ""))) def default_store_root(game): return os.path.join(user_dir("data"), "stores", game.store_id) +def locked(f): + """Run a Workspace method holding its write lock (Texts has the same wrapper: texts._locked).""" + @functools.wraps(f) + def run(self, *a, **kw): + with self.lock: + return f(self, *a, **kw) + return run + + class Workspace: def __init__(self, game_dir, store_root=None, archives=None, full_backup=True, progress=None): self.game = game_dir if isinstance(game_dir, Game) else Game(game_dir) self.store = OriginalsStore(store_root or default_store_root(self.game), self.game.root) self.catalog = Catalog(self.game, archives, progress) self.full_backup = full_backup + self.lock = threading.RLock() # one write (apply, revert, file rebuild) at a time + self._by_file = None # cooked-relative file -> [Location] (unoriginal_files) + self._shipped = None # the bank catalog is the shipped one (has checksums) + self._wad_rows = {} # voice archive -> {key: Sound} (heads' recordings) + self._other = {} # archive not loaded here -> Catalog (sound_for_key) self.texts = Texts(self) # reading ----------------------------------------------------------------------------------- def sound(self, name_or_key): return self.catalog.resolve(name_or_key) + def sound_for_key(self, key): + """The Sound of a key, also from an archive not loaded here (a change made in another language), or + None.""" + if key in self.catalog.by_key: + return self.catalog.by_key[key] + archive = key.split(":")[0] + if archive not in self._other: + self._other[archive] = Catalog(self.game, [archive]) if archive in available_archives(self.game) \ + else None + return self._other[archive].by_key.get(key) if self._other[archive] else None + def wad_path(self, s): return self.game.wad(s.archive) @@ -241,6 +271,55 @@ class Workspace: out.append(path) return out + def unoriginal_files(self, paths): + """Bank files and packages this store has never written that are not the game's original according to + the shipped catalog: an in-memory sound, the bank copy of a voice recording or a subtitle in them differs + (another originals store or another tool changed them). Their current content must not become this + store's "original copy". Menu files and other game builds have no catalog checksums: not checked.""" + out = [] + for path in sorted(set(paths)): + if self._derived().get(self.game.rel(path)) or self.texts.is_menu_file(path): + continue + rel = os.path.relpath(path, self.game.cooked).replace(os.sep, "/") + if not self._bank_copies_original(rel) or self.texts.unoriginal_subtitles(rel): + out.append(path) + return out + + def _bank_copies_original(self, rel): + if self._by_file is None: + self._by_file = defaultdict(list) + for ls in self.catalog.locations.values(): + for l in ls: + self._by_file[l.file].append(l) + locs = sorted(self._by_file.get(rel, []), key=lambda l: l.offset) + if self._shipped is None: + self._shipped = is_shipped(self.game, banks.ARCHIVE) + if not locs or not self._shipped: + return True + r = banks.open_file(self.bank_path(rel)) + for l in locs: + cur = r.read(l.offset, l.size) + if l.kind == banks.MEMORY: + want = self.catalog.bank_sha1.get(l.media) + if want and sha1_16(cur) != want: + return False + elif l.sound: # a bank copy of a WAD recording: the start of the original + orig = self._wad_original(l.sound) + if orig is None or cur != (orig + bytes(l.size))[:l.size]: + return False + return True + + def _wad_original(self, key): + """Original bytes of a WAD recording (this store's record, or the game's when it matches the catalog).""" + if self.store.get(key): + return self.store.original(key) + archive, _, target = key.partition(":") + if archive not in self._wad_rows: + self._wad_rows[archive] = {s.key: s for s in load_archive(self.game, archive)} + s = self._wad_rows[archive].get(key) + cur = read_target(self.game.wad(archive), target) + return None if s and s.sha1 and sha1_16(cur) != s.sha1 else cur + def _audio_edits(self, rel): """[(offset, bytes)] that the sound replacements applied now need in bank file `rel` (cooked-relative).""" out = [] @@ -271,6 +350,7 @@ class Workspace: edits = self._audio_edits(rel) + self.texts.subtitle_edits(rel) return banks.open_file(orig).patched(edits) if edits else open(orig, "rb").read() + @locked def sync_files(self, paths, progress=None): """Rebuild each derived file from its original copy and the edits applied now (atomic writes).""" for path in sorted(set(paths)): @@ -310,6 +390,7 @@ class Workspace: self.store.save() # the store says what the files must contain before they change self.sync_files(files, progress) + @locked def apply(self, changes, force=False, progress=None): """changes: [(sound, wem_bytes, meta dict)]. One atomic rewrite per WAD / bank file. A recording stored in two places of a WAD gets the same bytes in both (Workspace.copies).""" @@ -326,7 +407,8 @@ class Workspace: for s, _, _ in changes: if self.state(s) in (CHANGED, FOREIGN): conflicts.append(s) - changed = set(self.changed_files([self.bank_path(l.file) for s, _, _ in changes for l in self.locations(s)])) + paths = [self.bank_path(l.file) for s, _, _ in changes for l in self.locations(s)] + changed = set(self.changed_files(paths)) | set(self.unoriginal_files(paths)) conflicts += [s for s, _, _ in changes if s not in conflicts and {self.bank_path(l.file) for l in self.locations(s)} & changed] if conflicts and not force: @@ -337,6 +419,7 @@ class Workspace: for a, items in by_arch.items(): self._write(a, items, progress) + @locked def revert(self, sounds, progress=None): """Put the stored originals back (sounds without a record are skipped). Returns reverted.""" by_arch = defaultdict(list) @@ -354,31 +437,50 @@ class Workspace: self.store.gc() return done + @locked def revert_all(self, progress=None): - sounds = [self.catalog.by_key[k] for k in list(self.store.entries) if k in self.catalog.by_key] - missing = [k for k in self.store.entries if k not in self.catalog.by_key] - if missing: - # records of archives not loaded in this catalog (e.g. another language) - other = Catalog(self.game, sorted({k.split(":")[0] for k in missing})) - sounds += [other.by_key[k] for k in missing if k in other.by_key] + """Revert every sound this store has changed (texts: Texts.revert_all).""" + # records of archives not loaded in this catalog (e.g. another language) included + sounds = [s for s in map(self.sound_for_key, list(self.store.entries)) if s] return self.revert(sounds, progress) - def restore_from_full_backup(self): - """Last-resort fallback: copy back every full backup taken by this store.""" + @locked + def forget(self, keys=(), text_keys=()): + """Stop tracking changes (the game files are kept as they are).""" + for k in keys: + self.store.forget(k) + for k in text_keys: + self.texts.records().pop(k, None) + self.store.save() + self.store.gc() + + @locked + def restore_from_full_backup(self, progress=None): + """Last resort when the originals store is damaged (`ackit restore-backup`): put back the full copies. + A WAD (and its pack indexes) is copied back and its sounds are forgotten; bank files, packages and menu + files (always copied) are rebuilt from their copies with the edits of the sounds still recorded, i.e. + those of WADs without a full copy (full backups off), which stay modified and revertible. Returns the + restored files (game-relative).""" + import shutil + full = self.store.data.get("full", {}) + derived = self._derived() out = [] - for rel in self.store.data.get("full", {}): - src = self.store.full_copy(rel) + for rel in full: + if rel in derived: + continue dst = os.path.join(self.game.root, rel) self.game.check_writable(dst) tmp = dst + ".ackit-tmp" - import shutil - shutil.copy2(src, tmp) + shutil.copy2(self.store.full_copy(rel), tmp) os.replace(tmp, dst) out.append(rel) - for rec in self._derived().values(): - rec["written_fp"] = None + entries = self.store.data["entries"] + for k, e in list(entries.items()): + if e["archive"] == banks.ARCHIVE or e.get("wad") in full: + del entries[k] self.store.data["texts"] = {} - self.store.data["entries"] = {} self.store.save() + paths = [os.path.join(self.game.root, rel) for rel in derived if rel in full] + self.sync_files(paths, progress) self.store.gc() - return out + return sorted(out + [rel for rel in derived if rel in full]) diff --git a/app/audioio.py b/app/audioio.py index b351679..bccfa0e 100644 --- a/app/audioio.py +++ b/app/audioio.py @@ -163,7 +163,8 @@ class Recorder(QObject): self.chunks = [] self.rate = 48000 self.channels = 1 - self.fmt_float = False + self.sample = "/UnheardCity/stores//` (Windows `%APPDATA%`, Linux `~/.local/share/unheardcity`), id = hash of the game path; configurable (`--store`, editor settings). Before a sound is first changed, its original WEM bytes (SHA-256-named blob) and metadata are saved; - the applied replacement is saved too. **Revert** re-injects the stored original; **Revert all** does this for - every record. Stream-pack padding bytes are stored so pack reverts are byte-identical. + the applied replacement is saved too. **Revert** re-injects the stored original; **Revert all** (editor toolbar, + `ackit revert --all`) does this for every record of the store, sounds and texts, in every language. Stream-pack + padding bytes are stored so pack reverts are byte-identical. The uninstaller counts sound and text records and + does not offer to delete the store while any is left. +- **One store per game at a time:** the changes are recorded only in the store that made them. The editor + refuses to switch to another store folder while changes are applied (Revert all first). A store takes the + full copy of a soundbank or package only from a file that matches the shipped catalog (every in-memory + sound's checksum, every bank copy of a voice recording, every subtitle; `Workspace.unoriginal_files`), so a + second store (`--store`) or another tool's change is a conflict, never a new "original". (Menu files and other + game builds have no catalog checksums to check.) - **Full-file fallback:** before the first change to a WAD (and its pack index), one verified full copy is kept - in the store (`full/`), default on, can be disabled. Used by "Restore from full backup" if the store is damaged. + in the store (`full/`), default on, can be disabled. If the store is damaged, `ackit restore-backup --yes` + copies the WADs back (their sounds are forgotten) and rebuilds the soundbanks, packages and localization files + from their copies, keeping the sounds of WADs without a full copy recorded and revertible. Soundbank files, packages and localization files always get this copy (they are rebuilt from it, 4b/4c). - **Why the user data folder:** it survives app updates and reinstalls, needs no write access next to the game, and works the same on Linux/Proton. Steam never touches it. - **Safe writes:** every WAD is rebuilt into a temp file and atomically renamed over the original, pack indexes first. Several sounds of one WAD are written in one pass. A locked WAD (game running) gives a clear message. + Writes are serialized (`Workspace.lock`): the editor runs them as background jobs, and two rebuilds of one + package at once (e.g. a sound and its subtitle applied side by side) would otherwise lose one edit. - **Changed underneath:** each sound's state is computed from its bytes: original / modified (ours) / lost (ours was replaced by the original, e.g. Steam "verify") / changed (neither: update or another tool) / foreign (never touched by us, but not the catalog's original checksum). Writing over changed/foreign sounds - requires explicit confirmation; the catalog is used only when the WAD fingerprint matches. + or texts requires explicit confirmation (editor: a question, also when installing a mod; CLI: `--force`); the + catalog is used only when the game's fingerprint matches. ## 4. Size rule Speech/ambience variations are addressed by offset+size from BKPK tables inside compressed `.upk` packages, @@ -77,7 +90,7 @@ tables): high risk for little gain, not done. optional like the WAD copies: the file is rebuilt from it). The file content is always "original copy + replacements applied now", so reverting the last one restores the original bytes exactly, and an interrupted operation is repaired by the next one. Conflicts: the whole-file SHA-1 must be the original's or the last one - ackit wrote. + ackit wrote, and before the copy is taken the file must match the catalog (section 3). - **Prefetch heads and whole copies:** 396 voice recordings (81 lines, 78-80 recordings per language) have their first bytes copied into banks, and 49 more are stored whole in banks (same media ID, identical bytes); a replacement of such a recording rewrites those copies too, otherwise the game could play the old line. @@ -130,8 +143,9 @@ copy per voice language) is filtered the same way, so the sound list never mixes WAD voice lines), numbered `NAME#2` in the event's own order when it plays two versions; the other 637 keep the name of their English (INT) recording's position. The position in each language stays an alias (`NAME#i`). - A recording stored twice in a WAD (5 per language) is written and reverted in both places. -- The catalog records the ORIGINAL bytes: `tools/build_catalog.py` stops before writing anything if a sound - differs from the current catalog (a modified game), unless `--accept-changes` (a game update). +- The catalog records the ORIGINAL bytes and texts: `tools/build_catalog.py` stops before writing anything if a + sound or a subtitle differs from the current catalog (a modified game), unless `--accept-changes` (a game + update). - Category scheme: `docs/categories.md`. ## 6. Repository cleanup choices diff --git a/docs/formats/banks.md b/docs/formats/banks.md index cd22e80..c743c73 100644 --- a/docs/formats/banks.md +++ b/docs/formats/banks.md @@ -57,7 +57,7 @@ WAD sounds have such heads (2,101 copies); replacing one of them must rewrite th story lines) are also stored **whole** in banks: an in-memory source with the WAD recording's own media ID, byte-identical to the WAD WEM (84 copies). They are handled like heads: part of the WAD line, written and reverted with it, not -sounds of their own (`catalog._whole_recordings`). Heads and whole copies together: 445 recordings, 2,185 +sounds of their own (`catalog.whole_recordings`). Heads and whole copies together: 445 recordings, 2,185 copies (`BANK_locations.csv.gz` kind `p`). ## Localized banks diff --git a/docs/formats/subtitles.md b/docs/formats/subtitles.md index 1035d84..efeff71 100644 --- a/docs/formats/subtitles.md +++ b/docs/formats/subtitles.md @@ -14,6 +14,10 @@ the UE3 packages (`upk.md`), holding the subtitle text of all 11 languages. acki to 5 lines. - Strings are stored as cp1252 (positive FString length) or UTF-16 (negative length): almost all RUS, JPN and KOR strings and 85 % of POL are UTF-16; the other languages have 17-56 UTF-16 strings each. + 4 cp1252 strings hold UTF-8 bytes instead (the same line in KOR, POL, POR, RUS: `Speech_TAPE_J_Answer_1_00`; + the KOR one with a damaged byte; the game's own data). ackit reads cp1252 as Windows maps it (every byte, also + the 5 that Python's cp1252 leaves undefined, which the RUS and KOR ones contain), so they are shown as stored + and every string is written back byte-for-byte. ## Scripted subtitles (`RSubtitle`) 41 more subtitles (radio and encryption-card lines) are `RSubtitle` objects in `Startup.upk`: the same layout, diff --git a/docs/user-guide.md b/docs/user-guide.md index a903e9d..ecb3909 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -58,8 +58,9 @@ the same line and are changed and reverted together. ## Listen The editor under the list shows the selected sound in three columns side by side (drag the dividers to -resize them): **Listen**, **Replace** and **Subtitle**. Select a row and press **Space** (or double-click) to -play the original. Each column ends with its own buttons on one line. In **Listen**: +resize them): **Listen**, **Replace** and **Subtitle**. Select a row and press **Space** to play the original +(once a replacement is loaded, Space plays the replacement; Space again stops), or double-click it. Each column +ends with its own buttons on one line. In **Listen**: - **Original** / **Installed** play the game's original and what is currently in the game; **Stop** (Esc). Click on a waveform to play from that point. - Above them: **A/B** (switch between the original and your replacement at the same position, to compare), @@ -82,14 +83,22 @@ than the original, since the game may cut it off. ## Undo - **Revert** in the editor puts that sound back exactly. -- Right-click rows → **Revert selected**; toolbar **Revert all** puts every original back. +- Right-click rows → **Revert selected**; toolbar **Revert all** puts every original back: all sounds, + subtitles and menu texts you changed, in every language. - Originals are kept in your user data folder (`%APPDATA%\UnheardCity` on Windows, `~/.local/share/unheardcity` on Linux), so reverting works after updating or reinstalling the app too. Uninstalling keeps this folder; use **Revert all** before uninstalling if you want the game back to original. Before the first change to a game file, a full copy of it is kept there too: optional for the WAD archives - (Settings), always for soundbanks, packages and text files (they are rebuilt from that copy). + (Settings), always for soundbanks, packages and text files (they are rebuilt from that copy). Only files the + game ships unchanged are copied; a file another tool (or another originals store) has changed is reported as + *changed outside this tool* first. +- The originals of your changes are only in the store that made them, so the store folder (Settings) cannot be + changed while changes are applied: use **Revert all** first. - If Steam "Verify integrity of game files" or an update undoes your changes, a bar offers **Re-apply my changes** or **Forget them**. Do not use "verify" to undo mods; use Revert all. +- If the store folder itself is damaged, the full copies can still be put back with the command-line toolkit + that Unheard City is built on (`ackit restore-backup`, see the project README; not part of the editor + download). ## Subtitles The **Subtitles** tab lists the subtitle of every speech line in the selected language (see *Language*). @@ -123,6 +132,7 @@ Click the **★** column to mark favorites. These are stored in your user data f - **Export mod…**: choose which changed sounds and texts to include and save a `.acmod` file (the exact game data plus the audio, so it also works for another language whose original has a different size). - **Import mod…**: shows which sounds match this game, then installs them. You can revert them like any change. + If a sound or text the mod replaces was changed outside the editor, you are asked before it is overwritten. - Changed subtitles and menu texts are exported and imported with the sounds (listed as *Text*); a text stays in its language (a French subtitle edit changes the French subtitle). - In another language, voice lines are matched by the game's own identity of the line, so the same line is @@ -131,7 +141,8 @@ Click the **★** column to mark favorites. These are stored in your user data f ## Settings ![Settings](img/settings.png) -Game folder, where originals are kept, the full-backup option and the playback device. The recording device +Game folder, where originals are kept (changeable only while nothing is modified), the full-backup option and +the playback device. The recording device is chosen next to the Record button. **Background**: show or hide the Arkham City wallpaper behind the filter panel; **Background strength**: how visible it is (10 % by default). The editor panel opens tall enough to show all its controls; drag the divider above it to change that. diff --git a/tests/conftest.py b/tests/conftest.py index 010cfd3..2a27331 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,8 +11,14 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) @pytest.fixture(autouse=True) def isolated_home(tmp_path, monkeypatch): - """User data/config/cache go to a temp folder (never touch the real settings or stores).""" + """User data/config/cache go to a temp folder (never touch the real settings or stores), and the language + the game is set to on this machine (Documents\\WB Games\\...\\UserEngine.ini) does not leak into tests.""" + from ackit import game as G monkeypatch.setenv("ACKIT_HOME", str(tmp_path / "home")) + real = G.Game.config_dirs + mine = os.path.normcase(os.path.join(os.path.expanduser("~"), "Documents", "WB Games")) + monkeypatch.setattr(G.Game, "config_dirs", + lambda self: [d for d in real(self) if not os.path.normcase(d).startswith(mine)]) yield diff --git a/tests/test_app.py b/tests/test_app.py index 69274ea..59f499d 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -223,6 +223,60 @@ def test_mod_with_texts(win, tmp_path): tx.revert_all() +def test_revert_all_reverts_sounds_and_texts(win): + w, root, _ = win + before = files(root) + tx = w.ws.texts + tx.apply({"SUB:INT:%s:0" % synth.SPEECH_EVENT: "Undo me.", "MENU:INT:FRONTEND.int:Menu:Quit": "Leave"}) + s = w.ws.sound("CHR_RadioScanner_1") + w.ws.apply([(s, synth.wem(0.5, 1200), {})]) + w.revert_all() + assert files(root) == before and not w.ws.store.entries and not tx.records() + + +def test_mod_import_asks_before_overwriting(win, tmp_path): + """Files changed outside the editor are overwritten by a mod only when the user confirms.""" + w, root, originals = win + s = w.ws.sound("CHR_RadioScanner_1") + w.ws.apply([(s, synth.wem(0.5, 1200), {})]) + pkg = str(tmp_path / "m.acmod") + w.export_mod_dialog(path=pkg, keys=[s.key], title="m") + other = synth.wem(0.4, 150) + inject(w.ws.wad_path(s), {s.target: other}, {s.target: w.ws.limit(s)}) # another tool writes the slot + asked = [] + w.confirm = lambda *a: asked.append(a) or False + w.import_mod_dialog(path=pkg, confirm=False) + assert asked and w.ws.current_bytes(s) == other # declined: left alone + w.confirm = lambda *a: True + w.import_mod_dialog(path=pkg, confirm=False) + assert w.ws.state(s) == W.MODIFIED + w.revert_all() + + +def test_store_cannot_change_while_changes_are_applied(win, tmp_path, monkeypatch): + from app import mainwindow + w, root, _ = win + s = w.ws.sound("CHR_RadioScanner_1") + w.ws.apply([(s, synth.wem(0.5, 1200), {})]) + store = w.ws.store.root + + class Dlg: + def __init__(self, settings, parent): + pass + + def exec(self): + return mainwindow.QDialog.DialogCode.Accepted + + def values(self): + return dict(w.settings, store_root=str(tmp_path / "elsewhere")) + monkeypatch.setattr(mainwindow, "SettingsDialog", Dlg) + told = [] + w.inform = lambda *a: told.append(a) + w.settings_dialog() + assert told and w.settings.get("store_root") != str(tmp_path / "elsewhere") and w.ws.store.root == store + w.revert_all() + + def test_text_undone_outside_shows_banner(win): w, root, _ = win tx = w.ws.texts @@ -387,3 +441,18 @@ def test_editor_panel_fits_its_contents(win): need = w.editor.sizeHint().height() assert w.scroll.viewport().height() >= need - 2, (w.scroll.viewport().height(), need) assert w.vsplit.sizes()[0] >= 160 + + +def test_recorder_sample_formats(qapp): + """Input devices may deliver 16/32-bit integer, 8-bit or float samples, and reads can split a frame.""" + import numpy as np + from app.audioio import Recorder + r = Recorder() + x = np.array([[0.5, -0.25], [0.125, -1.0]], np.float32) + for sample, raw in ((" Originals store changed, `--store`) must not copy a package that the + first store modified as its "original": the catalog's checksums and texts show it is not original.""" + from ackit.originals import sha1_16 + from ackit.texts import TextConflictError + root, _ = synth_game + A = W.Workspace(root, store_root=str(tmp_path / "A"), archives=["INT", "SFX", "BANK"]) + s = A.catalog.by_key["BANK:%08X" % synth.DLG_MEDIA] + orig_sha = sha1_16(A.current_bytes(s)) + sub = "SUB:INT:%s:0" % synth.SPEECH_EVENT + rel = A.locations(s)[0].file + A.apply([(s, synth.fitting_wem(A, s, 0.1, 1900), {})]) + A.texts.apply({sub: "Edited."}) + + def store_b(): + # the synthetic game has no shipped catalog: give B the checksums and texts a shipped one would have + B = W.Workspace(root, store_root=str(tmp_path / "B"), archives=["INT", "SFX", "BANK"]) + B._shipped = True + B.catalog.bank_sha1 = {synth.DLG_MEDIA: orig_sha} + B.texts._shipped_subs = ({}, {}) + return B + B = store_b() + assert B.unoriginal_files([B.bank_path(rel)]) == [B.bank_path(rel)] + with pytest.raises(TextConflictError): + B.texts.apply({"SUB:INT:%s:1" % synth.SPEECH_EVENT: "Other."}) + A.revert([s]) + B = store_b() + by_file = {} + for l in B.texts.subtitle_locations(sub): + by_file.setdefault(l.file, []).append(l) + B.texts._shipped_subs = (by_file, {("INT", synth.SPEECH_EVENT, 0): "Scanner line two."}) + assert B.texts.unoriginal_subtitles(rel) == [sub] # A's subtitle is still applied + A.texts.revert_all() + assert B.unoriginal_files([B.bank_path(rel)]) == [] + + +def test_mod_export_of_a_change_made_in_another_language(synth_game, tmp_path): + """A change made while another language was loaded keeps its name and line identity in a mod package.""" + import json + import zipfile + root, _ = synth_game + store = str(tmp_path / "store") + every = W.Workspace(root, store_root=store, archives=["SFX", "BANK"]) # every language's lines + s = every.catalog.by_key["BANK:%08X" % synth.GRUNT_MEDIA["ITA"]] + every.apply([(s, synth.fitting_wem(every, s, 0.1, 900), {})]) + ws = W.Workspace(root, store_root=store, archives=["INT", "SFX", "BANK"]) # now INT is loaded + assert s.key not in ws.catalog.by_key + pkg = str(tmp_path / "m.acmod") + export_mod(ws, pkg, ws.modified_keys(), "m") + it = json.loads(zipfile.ZipFile(pkg).read("manifest.json"))["items"][0] + assert (it["name"], it["line"]) == (s.name, s.line) and s.line + assert ws.revert_all() and not ws.store.entries + + def test_mod_from_another_language_matched_by_line(synth_game, tmp_path): """A voice line from another language is found through the game's line identity (its WAD position there can differ); a voice line from a package without line identities is never guessed by name.""" diff --git a/tools/build_catalog.py b/tools/build_catalog.py index 5fdbe7e..ee14b9c 100644 --- a/tools/build_catalog.py +++ b/tools/build_catalog.py @@ -12,7 +12,8 @@ Outputs data/catalog/SUB_locations.csv.gz every stored copy of each subtitle string data/catalog/index.json fingerprints the CSVs were built from + counts Run it only on clean game files (`ackit status` must say nothing is modified). It stops before writing -anything if a sound differs from the current catalog (a modified game); after a game update use --accept-changes. +anything if a sound or a subtitle differs from the current catalog (a modified game); after a game update use +--accept-changes. (Menu text has no catalog.) """ import argparse import collections @@ -88,9 +89,16 @@ def build_characters(game, src_root, final): return rows, unmatched, ambiguous, skipped -def changed_since_catalog(game, out_dir, bank_snd): - """Keys whose original checksum in the existing catalog differs from the game files now.""" +def changed_since_catalog(game, out_dir, bank_snd, texts=None): + """Keys whose original in the existing catalog differs from the game files now: sounds (checksums) and + subtitles (texts: {lang: [SubLine]} scanned now).""" out = [] + for lang, rows in (texts or {}).items(): + p = os.path.join(out_dir, "SUB_%s.csv" % lang) + if os.path.exists(p): + old = {(s.speech, s.line): s.text for s in subtitles.read_texts(p, lang)} + out += [subtitles.key(lang, s.speech, s.line) for s in rows + if (s.speech, s.line) in old and old[(s.speech, s.line)] != s.text] for arch, path in game.archives.items(): p = os.path.join(out_dir, "%s.csv" % arch) if not os.path.exists(p): @@ -133,11 +141,12 @@ def main(): for k, d in lines.issues[:10]: print(" issue:", k, d) - # safety: the catalog records ORIGINAL bytes. A sound that differs from the current catalog means a modified - # game (an applied change, another tool) unless the game itself was updated -> stop before writing anything. - changed = changed_since_catalog(game, out_dir, bank_snd) + # safety: the catalog records ORIGINAL bytes and texts. A sound or subtitle that differs from the current + # catalog means a modified game (an applied change, another tool) unless the game itself was updated -> stop + # before writing anything. + changed = changed_since_catalog(game, out_dir, bank_snd, texts) if changed and not a.accept_changes: - print("STOP: %d sound(s) differ from the current catalog, e.g. %s." % (len(changed), changed[:5])) + print("STOP: %d sound(s) or subtitle(s) differ from the current catalog, e.g. %s." % (len(changed), changed[:5])) print("Revert all changes (ackit status / ackit revert --all) and run again; after a game update use " "--accept-changes.") sys.exit(2) @@ -193,7 +202,7 @@ def main(): "fingerprint": banks.game_fingerprint(game.cooked), "sounds": len(sounds), "groups": dict(g), "named_by_event": by_event, "named_by_bank": len(sounds) - by_event, "copies": sum(1 for l in locs if l.kind == banks.MEMORY), "files": len(sc.files), - # bank copies of WAD recordings: prefetch heads and whole in-memory copies (catalog._whole_recordings) + # bank copies of WAD recordings: prefetch heads and whole in-memory copies (catalog.whole_recordings) "prefetch_heads": {"recordings": len({l.media for l in heads}), "copies": len(heads), "unmapped": len({l.media for l in heads if not l.sound})}, "issues": ["%s: %s" % i for i in sc.issues]} diff --git a/tools/check_docs.py b/tools/check_docs.py index a043cfe..9eb0acb 100644 --- a/tools/check_docs.py +++ b/tools/check_docs.py @@ -240,6 +240,16 @@ def catalog_claims(cat): west = [enc[L]["w"] for L in langs if L not in ("RUS", "JPN", "KOR", "POL")] claim("docs/formats/subtitles.md", "%d %% of POL are UTF-16; the other languages have %s UTF-16 strings each" % (pol, rng(west))) + utf8 = [] # cp1252 strings whose bytes are UTF-8 text (the game's data) + for L, t in sorted(texts.items()): + for s in t: + if s.enc == "a": + u = subtitles._encode_cp1252(s.text).decode("utf-8", "replace") + multi = sum(1 for c in u if ord(c) > 0x7F and c != "\ufffd") # valid multi-byte characters + if multi and u.count("\ufffd") <= multi // 4: # (one corrupt byte tolerated) + utf8.append((L, s.speech)) + claim("docs/formats/subtitles.md", "%d cp1252 strings hold UTF-8 bytes instead (the same line in %s: `%s`" + % (len(utf8), ", ".join(L for L, _ in utf8), "`, `".join(sorted({sp for _, sp in utf8})))) with gzip.open(os.path.join(cat, subtitles.LOCATIONS), "rt", newline="") as f: sublocs = sum(1 for _ in csv.DictReader(f)) claim("REPORT.md", "Edited in place in every copy (%s copies)" % n(sublocs)) @@ -505,6 +515,21 @@ def wem_claims(): % (n(total - len(bad)), n(total), exceptions)) +def originality_claims(game): + """Workspace.unoriginal_files on every file with sounds or subtitles of the (original) game: no false alarm.""" + import tempfile + from ackit import subtitles + from ackit.workspace import Workspace + with tempfile.TemporaryDirectory() as td: + ws = Workspace(game, store_root=td, archives=[game.voice_archive("INT"), "SFX", "BANK"]) + rels = {l.file for ls in ws.catalog.locations.values() for l in ls} + rels |= {l.file for ls in subtitles.read_locations(os.path.join(catalog_dir(), subtitles.LOCATIONS)).values() + for l in ls} + bad = ws.unoriginal_files([ws.bank_path(r) for r in rels]) + claim("REPORT.md", "checked on all %s files with sounds or subtitles: %s" % ( + n(len(rels)), "no false alarm" if not bad else "%d false alarms" % len(bad)), "Workspace.unoriginal_files") + + def test_claims(): import subprocess out = subprocess.run([sys.executable, "-m", "pytest", "--collect-only", "-q", "-p", "no:cacheprovider"], @@ -535,6 +560,7 @@ def main(): if not game: raise SystemExit("--scan needs the game") scan_claims(game) + originality_claims(game) if a.wems: if not game: raise SystemExit("--wems needs the game") diff --git a/tools/e2e_editor.py b/tools/e2e_editor.py index a3da859..57525d0 100644 --- a/tools/e2e_editor.py +++ b/tools/e2e_editor.py @@ -5,6 +5,8 @@ 3. start the game under cdb + speaker loopback: the probe must be heard, no crash, setup parsed; 4. press Revert (EditorPanel.revert) and check the WAD + pack index are byte-identical to before; 5. also apply/revert a speech line in place and check byte identity. +Uses the user's settings and originals store (an interrupted run shows up in `ackit status`); the settings file +is never written. python tools/e2e_editor.py """ @@ -29,11 +31,15 @@ def sha(p): def main(): from PySide6.QtWidgets import QApplication + from ackit import settings as settings_mod from app.mainwindow import MainWindow from app.workers import Runner app = QApplication([]) - w = MainWindow(settings={"full_backup": True}, runner=Runner(sync=True)) + # the user's settings (same game folder and originals store as `ackit status`), never saved: the window + # stores its settings on load and on close, which would overwrite the user's file with this test's ones + w = MainWindow(settings=dict(settings_mod.load(), full_backup=True), runner=Runner(sync=True)) + w.save_settings = lambda: None w.confirm = lambda *a: True w.editor.confirm = lambda *a: True errors = [] diff --git a/tools/installer/uninstall.ps1 b/tools/installer/uninstall.ps1 index 38cdb52..36f0c27 100644 --- a/tools/installer/uninstall.ps1 +++ b/tools/installer/uninstall.ps1 @@ -14,10 +14,14 @@ $Data = @((Join-Path $env:APPDATA "UnheardCity"), (Join-Path $env:LOCALAPPDATA " function Say($msg) { if (-not $Quiet) { Write-Host $msg } } function Modified-Count { - # sounds still changed in the game, according to the originals stores + # sounds and texts (subtitles, menu text) still changed in the game, according to the originals stores $n = 0 Get-ChildItem -Path (Join-Path $Data[0] "stores\*\store.json") -ErrorAction SilentlyContinue | ForEach-Object { - try { $n += @((Get-Content -Raw -LiteralPath $_.FullName | ConvertFrom-Json).entries.PSObject.Properties).Count } catch {} + try { + $s = Get-Content -Raw -LiteralPath $_.FullName | ConvertFrom-Json + $n += @($s.entries.PSObject.Properties).Count + if ($s.texts) { $n += @($s.texts.PSObject.Properties).Count } + } catch {} } return $n } @@ -26,8 +30,8 @@ try { if (Get-Process -Name "UnheardCity" -ErrorAction SilentlyContinue) { throw "$Name is running. Close it first." } $mods = Modified-Count if ($mods -gt 0 -and -not $Quiet) { - Write-Host "$mods sound(s) are still modified in the game. To restore the originals, open $Name and use" - Write-Host "'Revert all' BEFORE uninstalling (your originals store is kept either way)." + Write-Host "$mods sound(s) or text(s) are still modified in the game. To restore the originals, open $Name and" + Write-Host "use 'Revert all' BEFORE uninstalling (your originals store is kept either way)." if ((Read-Host "Uninstall anyway? [y/N]") -notmatch '^(y|yes)$') { exit 1 } } if (-not $Quiet -and -not $RemoveUserData -and $mods -eq 0 -and (Test-Path $Data[0])) {