Full correctness check against the docs: safer writes, reverts and texts
Safety - Writes are serialized (Workspace.lock): two background jobs rebuilding one package (a sound and its subtitle applied side by side) lost an edit. - A store copies a bank file or package as "original" only if it matches the shipped catalog (Workspace.unoriginal_files: in-memory sounds, bank copies of voice lines, subtitles); the editor refuses to switch stores while changes are applied. A second store could otherwise resurrect a reverted sound. - Mod import in the editor asks before overwriting files changed outside the tool (it forced silently). - Revert all in the editor reverts texts too; the uninstaller counts text edits before offering to delete the store. - ackit restore-backup: the full-backup restore decisions.md described; it keeps the sounds of WADs without a full copy recorded. - build_catalog also stops when a subtitle differs from the catalog. Languages and texts - Game.language() returns languages without voices (JPN, RUS, ...) and Game.voice_archive() their INT voices; ackit text --lang JPN works. - Menu lookups parse one cached file (find & replace: minutes -> ms). - Subtitles decode cp1252 as Windows does: the UTF-8-in-cp1252 line Speech_TAPE_J_Answer_1_00 (KOR/POL/POR/RUS) is written back byte-for-byte; SUB_KOR/SUB_RUS rebuilt from the original game (only those rows changed). - Mods exported while another language is loaded keep name and line. Other fixes - Down-mix folds extra channels in at -3 dB (a 5.1 centre voice was dropped). - e2e_editor no longer overwrites the user's settings.json; kill_game waits for the game to exit before reverts. - Recorder handles Int32/UInt8 input and split frames; package block cache locked; conflict messages, CLI help examples, stale docstrings and doc references fixed; tests no longer read this PC's game language. Docs (decisions, user guide, README, subtitles.md, banks.md, catalog README, REPORT, NOTES, CLAUDE.md) updated; check_docs: 102 claims match. 148 tests; e2e_features 9/9, e2e_editor and verify PASS; game original afterwards.
This commit is contained in:
@@ -10,14 +10,19 @@ In-memory sounds (archive BANK) and the prefetch heads of speech lines live in l
|
||||
packages (ackit/banks.py); a replacement fills each in-memory slot exactly (wem.fill). Before the first
|
||||
change to such a file a verified full copy is kept, and the
|
||||
file is always rebuilt as "original copy + every replacement currently applied", so reverting the last
|
||||
change restores it byte-for-byte and any later operation repairs an interrupted one.
|
||||
change restores it byte-for-byte and any later operation repairs an interrupted one. That copy is taken only
|
||||
from a file that is original according to the catalog (`unoriginal_files`), else the write is a conflict.
|
||||
Writes are serialized (`Workspace.lock`): the editor runs them as background jobs, and two rebuilds of one
|
||||
file at the same time would lose one of the edits.
|
||||
"""
|
||||
import functools
|
||||
import hashlib
|
||||
import os
|
||||
import threading
|
||||
from collections import defaultdict
|
||||
|
||||
from . import audio, banks
|
||||
from .catalog import Catalog
|
||||
from .catalog import Catalog, available_archives, is_shipped, load_archive
|
||||
from .encoder import EncodeError, encode_fit
|
||||
from .game import Game, GameError
|
||||
from .inject import inject, read_target, parse_target, pack_bnk_path, SizeLimitError
|
||||
@@ -41,25 +46,50 @@ class ConflictError(GameError):
|
||||
self.sounds = sounds
|
||||
super().__init__("These sounds were changed outside this tool (Steam update or 'verify', or "
|
||||
"another mod tool): %s.\nRevert them or confirm to overwrite."
|
||||
% ", ".join(s.name for s in sounds[:5]) + (" ..." if len(sounds) > 5 else ""))
|
||||
% (", ".join(s.name for s in sounds[:5]) + (" ..." if len(sounds) > 5 else "")))
|
||||
|
||||
|
||||
def default_store_root(game):
|
||||
return os.path.join(user_dir("data"), "stores", game.store_id)
|
||||
|
||||
|
||||
def locked(f):
|
||||
"""Run a Workspace method holding its write lock (Texts has the same wrapper: texts._locked)."""
|
||||
@functools.wraps(f)
|
||||
def run(self, *a, **kw):
|
||||
with self.lock:
|
||||
return f(self, *a, **kw)
|
||||
return run
|
||||
|
||||
|
||||
class Workspace:
|
||||
def __init__(self, game_dir, store_root=None, archives=None, full_backup=True, progress=None):
|
||||
self.game = game_dir if isinstance(game_dir, Game) else Game(game_dir)
|
||||
self.store = OriginalsStore(store_root or default_store_root(self.game), self.game.root)
|
||||
self.catalog = Catalog(self.game, archives, progress)
|
||||
self.full_backup = full_backup
|
||||
self.lock = threading.RLock() # one write (apply, revert, file rebuild) at a time
|
||||
self._by_file = None # cooked-relative file -> [Location] (unoriginal_files)
|
||||
self._shipped = None # the bank catalog is the shipped one (has checksums)
|
||||
self._wad_rows = {} # voice archive -> {key: Sound} (heads' recordings)
|
||||
self._other = {} # archive not loaded here -> Catalog (sound_for_key)
|
||||
self.texts = Texts(self)
|
||||
|
||||
# reading -----------------------------------------------------------------------------------
|
||||
def sound(self, name_or_key):
|
||||
return self.catalog.resolve(name_or_key)
|
||||
|
||||
def sound_for_key(self, key):
|
||||
"""The Sound of a key, also from an archive not loaded here (a change made in another language), or
|
||||
None."""
|
||||
if key in self.catalog.by_key:
|
||||
return self.catalog.by_key[key]
|
||||
archive = key.split(":")[0]
|
||||
if archive not in self._other:
|
||||
self._other[archive] = Catalog(self.game, [archive]) if archive in available_archives(self.game) \
|
||||
else None
|
||||
return self._other[archive].by_key.get(key) if self._other[archive] else None
|
||||
|
||||
def wad_path(self, s):
|
||||
return self.game.wad(s.archive)
|
||||
|
||||
@@ -241,6 +271,55 @@ class Workspace:
|
||||
out.append(path)
|
||||
return out
|
||||
|
||||
def unoriginal_files(self, paths):
|
||||
"""Bank files and packages this store has never written that are not the game's original according to
|
||||
the shipped catalog: an in-memory sound, the bank copy of a voice recording or a subtitle in them differs
|
||||
(another originals store or another tool changed them). Their current content must not become this
|
||||
store's "original copy". Menu files and other game builds have no catalog checksums: not checked."""
|
||||
out = []
|
||||
for path in sorted(set(paths)):
|
||||
if self._derived().get(self.game.rel(path)) or self.texts.is_menu_file(path):
|
||||
continue
|
||||
rel = os.path.relpath(path, self.game.cooked).replace(os.sep, "/")
|
||||
if not self._bank_copies_original(rel) or self.texts.unoriginal_subtitles(rel):
|
||||
out.append(path)
|
||||
return out
|
||||
|
||||
def _bank_copies_original(self, rel):
|
||||
if self._by_file is None:
|
||||
self._by_file = defaultdict(list)
|
||||
for ls in self.catalog.locations.values():
|
||||
for l in ls:
|
||||
self._by_file[l.file].append(l)
|
||||
locs = sorted(self._by_file.get(rel, []), key=lambda l: l.offset)
|
||||
if self._shipped is None:
|
||||
self._shipped = is_shipped(self.game, banks.ARCHIVE)
|
||||
if not locs or not self._shipped:
|
||||
return True
|
||||
r = banks.open_file(self.bank_path(rel))
|
||||
for l in locs:
|
||||
cur = r.read(l.offset, l.size)
|
||||
if l.kind == banks.MEMORY:
|
||||
want = self.catalog.bank_sha1.get(l.media)
|
||||
if want and sha1_16(cur) != want:
|
||||
return False
|
||||
elif l.sound: # a bank copy of a WAD recording: the start of the original
|
||||
orig = self._wad_original(l.sound)
|
||||
if orig is None or cur != (orig + bytes(l.size))[:l.size]:
|
||||
return False
|
||||
return True
|
||||
|
||||
def _wad_original(self, key):
|
||||
"""Original bytes of a WAD recording (this store's record, or the game's when it matches the catalog)."""
|
||||
if self.store.get(key):
|
||||
return self.store.original(key)
|
||||
archive, _, target = key.partition(":")
|
||||
if archive not in self._wad_rows:
|
||||
self._wad_rows[archive] = {s.key: s for s in load_archive(self.game, archive)}
|
||||
s = self._wad_rows[archive].get(key)
|
||||
cur = read_target(self.game.wad(archive), target)
|
||||
return None if s and s.sha1 and sha1_16(cur) != s.sha1 else cur
|
||||
|
||||
def _audio_edits(self, rel):
|
||||
"""[(offset, bytes)] that the sound replacements applied now need in bank file `rel` (cooked-relative)."""
|
||||
out = []
|
||||
@@ -271,6 +350,7 @@ class Workspace:
|
||||
edits = self._audio_edits(rel) + self.texts.subtitle_edits(rel)
|
||||
return banks.open_file(orig).patched(edits) if edits else open(orig, "rb").read()
|
||||
|
||||
@locked
|
||||
def sync_files(self, paths, progress=None):
|
||||
"""Rebuild each derived file from its original copy and the edits applied now (atomic writes)."""
|
||||
for path in sorted(set(paths)):
|
||||
@@ -310,6 +390,7 @@ class Workspace:
|
||||
self.store.save() # the store says what the files must contain before they change
|
||||
self.sync_files(files, progress)
|
||||
|
||||
@locked
|
||||
def apply(self, changes, force=False, progress=None):
|
||||
"""changes: [(sound, wem_bytes, meta dict)]. One atomic rewrite per WAD / bank file. A recording stored
|
||||
in two places of a WAD gets the same bytes in both (Workspace.copies)."""
|
||||
@@ -326,7 +407,8 @@ class Workspace:
|
||||
for s, _, _ in changes:
|
||||
if self.state(s) in (CHANGED, FOREIGN):
|
||||
conflicts.append(s)
|
||||
changed = set(self.changed_files([self.bank_path(l.file) for s, _, _ in changes for l in self.locations(s)]))
|
||||
paths = [self.bank_path(l.file) for s, _, _ in changes for l in self.locations(s)]
|
||||
changed = set(self.changed_files(paths)) | set(self.unoriginal_files(paths))
|
||||
conflicts += [s for s, _, _ in changes
|
||||
if s not in conflicts and {self.bank_path(l.file) for l in self.locations(s)} & changed]
|
||||
if conflicts and not force:
|
||||
@@ -337,6 +419,7 @@ class Workspace:
|
||||
for a, items in by_arch.items():
|
||||
self._write(a, items, progress)
|
||||
|
||||
@locked
|
||||
def revert(self, sounds, progress=None):
|
||||
"""Put the stored originals back (sounds without a record are skipped). Returns reverted."""
|
||||
by_arch = defaultdict(list)
|
||||
@@ -354,31 +437,50 @@ class Workspace:
|
||||
self.store.gc()
|
||||
return done
|
||||
|
||||
@locked
|
||||
def revert_all(self, progress=None):
|
||||
sounds = [self.catalog.by_key[k] for k in list(self.store.entries) if k in self.catalog.by_key]
|
||||
missing = [k for k in self.store.entries if k not in self.catalog.by_key]
|
||||
if missing:
|
||||
# records of archives not loaded in this catalog (e.g. another language)
|
||||
other = Catalog(self.game, sorted({k.split(":")[0] for k in missing}))
|
||||
sounds += [other.by_key[k] for k in missing if k in other.by_key]
|
||||
"""Revert every sound this store has changed (texts: Texts.revert_all)."""
|
||||
# records of archives not loaded in this catalog (e.g. another language) included
|
||||
sounds = [s for s in map(self.sound_for_key, list(self.store.entries)) if s]
|
||||
return self.revert(sounds, progress)
|
||||
|
||||
def restore_from_full_backup(self):
|
||||
"""Last-resort fallback: copy back every full backup taken by this store."""
|
||||
@locked
|
||||
def forget(self, keys=(), text_keys=()):
|
||||
"""Stop tracking changes (the game files are kept as they are)."""
|
||||
for k in keys:
|
||||
self.store.forget(k)
|
||||
for k in text_keys:
|
||||
self.texts.records().pop(k, None)
|
||||
self.store.save()
|
||||
self.store.gc()
|
||||
|
||||
@locked
|
||||
def restore_from_full_backup(self, progress=None):
|
||||
"""Last resort when the originals store is damaged (`ackit restore-backup`): put back the full copies.
|
||||
A WAD (and its pack indexes) is copied back and its sounds are forgotten; bank files, packages and menu
|
||||
files (always copied) are rebuilt from their copies with the edits of the sounds still recorded, i.e.
|
||||
those of WADs without a full copy (full backups off), which stay modified and revertible. Returns the
|
||||
restored files (game-relative)."""
|
||||
import shutil
|
||||
full = self.store.data.get("full", {})
|
||||
derived = self._derived()
|
||||
out = []
|
||||
for rel in self.store.data.get("full", {}):
|
||||
src = self.store.full_copy(rel)
|
||||
for rel in full:
|
||||
if rel in derived:
|
||||
continue
|
||||
dst = os.path.join(self.game.root, rel)
|
||||
self.game.check_writable(dst)
|
||||
tmp = dst + ".ackit-tmp"
|
||||
import shutil
|
||||
shutil.copy2(src, tmp)
|
||||
shutil.copy2(self.store.full_copy(rel), tmp)
|
||||
os.replace(tmp, dst)
|
||||
out.append(rel)
|
||||
for rec in self._derived().values():
|
||||
rec["written_fp"] = None
|
||||
entries = self.store.data["entries"]
|
||||
for k, e in list(entries.items()):
|
||||
if e["archive"] == banks.ARCHIVE or e.get("wad") in full:
|
||||
del entries[k]
|
||||
self.store.data["texts"] = {}
|
||||
self.store.data["entries"] = {}
|
||||
self.store.save()
|
||||
paths = [os.path.join(self.game.root, rel) for rel in derived if rel in full]
|
||||
self.sync_files(paths, progress)
|
||||
self.store.gc()
|
||||
return out
|
||||
return sorted(out + [rel for rel in derived if rel in full])
|
||||
|
||||
Reference in New Issue
Block a user