Full correctness check against the docs: safer writes, reverts and texts
Safety - Writes are serialized (Workspace.lock): two background jobs rebuilding one package (a sound and its subtitle applied side by side) lost an edit. - A store copies a bank file or package as "original" only if it matches the shipped catalog (Workspace.unoriginal_files: in-memory sounds, bank copies of voice lines, subtitles); the editor refuses to switch stores while changes are applied. A second store could otherwise resurrect a reverted sound. - Mod import in the editor asks before overwriting files changed outside the tool (it forced silently). - Revert all in the editor reverts texts too; the uninstaller counts text edits before offering to delete the store. - ackit restore-backup: the full-backup restore decisions.md described; it keeps the sounds of WADs without a full copy recorded. - build_catalog also stops when a subtitle differs from the catalog. Languages and texts - Game.language() returns languages without voices (JPN, RUS, ...) and Game.voice_archive() their INT voices; ackit text --lang JPN works. - Menu lookups parse one cached file (find & replace: minutes -> ms). - Subtitles decode cp1252 as Windows does: the UTF-8-in-cp1252 line Speech_TAPE_J_Answer_1_00 (KOR/POL/POR/RUS) is written back byte-for-byte; SUB_KOR/SUB_RUS rebuilt from the original game (only those rows changed). - Mods exported while another language is loaded keep name and line. Other fixes - Down-mix folds extra channels in at -3 dB (a 5.1 centre voice was dropped). - e2e_editor no longer overwrites the user's settings.json; kill_game waits for the game to exit before reverts. - Recorder handles Int32/UInt8 input and split frames; package block cache locked; conflict messages, CLI help examples, stale docstrings and doc references fixed; tests no longer read this PC's game language. Docs (decisions, user guide, README, subtitles.md, banks.md, catalog README, REPORT, NOTES, CLAUDE.md) updated; check_docs: 102 claims match. 148 tests; e2e_features 9/9, e2e_editor and verify PASS; game original afterwards.
This commit is contained in:
131
ackit/texts.py
131
ackit/texts.py
@@ -5,31 +5,45 @@ Keys (also used in mod packages):
|
||||
MENU:<LANG>:<File>:<Section>:<Key>[#n] a menu/UI text (docs/formats/text.md)
|
||||
Records live in the originals store (data["texts"]): {key: {"orig", "applied", "files", ...}}. The files
|
||||
holding the texts are derived files of the Workspace: each is rebuilt from its original copy plus every edit
|
||||
applied now (sounds and texts), so reverting the last edit restores it byte-for-byte.
|
||||
applied now (sounds and texts), so reverting the last edit restores it byte-for-byte. Writes hold the
|
||||
workspace's lock (Workspace.lock).
|
||||
"""
|
||||
import os
|
||||
import time
|
||||
|
||||
from . import loctext, subtitles
|
||||
from .catalog import load_subtitles
|
||||
from . import banks, loctext, subtitles
|
||||
from .catalog import _shipped_index, catalog_dir, load_subtitles
|
||||
from .game import GameError
|
||||
|
||||
ORIGINAL, MODIFIED, LOST, CHANGED, FOREIGN = "original", "modified", "lost", "changed", "foreign"
|
||||
|
||||
|
||||
def _locked(f):
|
||||
"""Run a Texts method holding the workspace's write lock."""
|
||||
def run(self, *a, **kw):
|
||||
with self.ws.lock:
|
||||
return f(self, *a, **kw)
|
||||
run.__name__, run.__doc__ = f.__name__, f.__doc__
|
||||
return run
|
||||
|
||||
|
||||
class TextConflictError(GameError):
|
||||
def __init__(self, keys):
|
||||
self.keys = keys
|
||||
super().__init__("These texts were changed outside this tool (Steam update or 'verify', or another mod "
|
||||
"tool): %s.\nRevert them or confirm to overwrite." % ", ".join(keys[:5])
|
||||
+ (" ..." if len(keys) > 5 else ""))
|
||||
"tool): %s.\nRevert them or confirm to overwrite."
|
||||
% (", ".join(keys[:5]) + (" ..." if len(keys) > 5 else "")))
|
||||
|
||||
|
||||
class Texts:
|
||||
def __init__(self, ws):
|
||||
self.ws = ws
|
||||
self._subs = {} # lang -> ([SubLine], {key: [SubLoc]})
|
||||
self._sub_text = {} # lang -> {(speech, line): original text} (catalog)
|
||||
self._menu = {} # lang -> [Entry]
|
||||
self._menu_paths = {} # lang -> {file name: path}
|
||||
self._menu_file = {} # path -> ((size, mtime), encoding, {entry key: Entry})
|
||||
self._shipped_subs = None # (by file: [SubLoc], {(lang, speech, line): text}) of the shipped catalog
|
||||
self.loc_root = _find_dir(ws.game.root, "BmGame", "Localization")
|
||||
|
||||
# --- languages and lists ---------------------------------------------------------------------
|
||||
@@ -37,7 +51,6 @@ class Texts:
|
||||
return loctext.languages(self.loc_root) if self.loc_root else []
|
||||
|
||||
def subtitle_languages(self):
|
||||
from .catalog import _shipped_index
|
||||
langs = sorted(_shipped_index().get("SUB", {}).get("lines", {}))
|
||||
return langs or self.menu_languages()
|
||||
|
||||
@@ -54,8 +67,11 @@ class Texts:
|
||||
|
||||
def for_sound(self, sound, lang=None):
|
||||
"""Subtitle lines of a speech sound, in `lang` (default: the sound's own language). Languages without
|
||||
their own voices link their subtitles to the INT sounds, so e.g. JPN subtitles are found for INT lines."""
|
||||
lang = lang or (sound.languages[0] if sound.languages else sound.archive)
|
||||
their own voices link their subtitles to the INT sounds, so e.g. JPN subtitles are found for INT lines.
|
||||
A sound shared by every language (an in-memory line stored once) defaults to the game's language."""
|
||||
game = self.ws.game
|
||||
lang = lang or (sound.languages[0] if sound.languages else
|
||||
sound.archive if sound.archive in game.languages else game.language())
|
||||
if lang not in self.subtitle_languages():
|
||||
return []
|
||||
return [s for s in self.subtitles(lang) if sound.key in s.sounds]
|
||||
@@ -77,20 +93,44 @@ class Texts:
|
||||
def modified(self):
|
||||
return [k for k, r in self.records().items() if r.get("applied") is not None]
|
||||
|
||||
def _menu_path(self, key):
|
||||
"""Path of the file holding menu text `key` (MENU:<LANG>:<File>:...)."""
|
||||
parts = key.split(":", 3)
|
||||
if len(parts) < 4 or not self.loc_root:
|
||||
raise GameError("no menu text %s" % key[len("MENU:"):])
|
||||
lang = parts[1]
|
||||
if lang not in self._menu_paths:
|
||||
self._menu_paths[lang] = loctext.files(self.loc_root, lang)
|
||||
path = self._menu_paths[lang].get(parts[2])
|
||||
if not path:
|
||||
raise GameError("no menu text %s" % key[len("MENU:"):])
|
||||
return path
|
||||
|
||||
def _menu_entries(self, path, lang):
|
||||
"""(encoding, {entry key: Entry}) of one menu file as it is now (parsed again only when it changed)."""
|
||||
st = os.stat(path)
|
||||
stamp = (st.st_size, st.st_mtime_ns)
|
||||
hit = self._menu_file.get(path)
|
||||
if not hit or hit[0] != stamp:
|
||||
raw = open(path, "rb").read()
|
||||
hit = (stamp, loctext.encoding(raw),
|
||||
{e.key: e for e in loctext.parse(raw, lang.upper(), os.path.basename(path))})
|
||||
self._menu_file[path] = hit
|
||||
return hit[1], hit[2]
|
||||
|
||||
def _menu_entry(self, key):
|
||||
lang = key.split(":")[1]
|
||||
raw = key[len("MENU:"):]
|
||||
for e in loctext.load(self.loc_root, lang):
|
||||
if e.key == raw:
|
||||
return e
|
||||
raise GameError("no menu text %s" % raw)
|
||||
path = self._menu_path(key)
|
||||
e = self._menu_entries(path, key.split(":")[1])[1].get(key[len("MENU:"):])
|
||||
if e is None:
|
||||
raise GameError("no menu text %s" % key[len("MENU:"):])
|
||||
return e
|
||||
|
||||
def files(self, key):
|
||||
"""Absolute paths of the files holding a text."""
|
||||
if key.startswith("SUB:"):
|
||||
return sorted({self.ws.bank_path(l.file) for l in self.subtitle_locations(key)})
|
||||
e = self._menu_entry(key)
|
||||
return [loctext.files(self.loc_root, e.lang)[e.file]]
|
||||
self._menu_entry(key)
|
||||
return [self._menu_path(key)]
|
||||
|
||||
def current(self, key):
|
||||
"""The text in the game files now."""
|
||||
@@ -98,22 +138,20 @@ class Texts:
|
||||
locs = self.subtitle_locations(key)
|
||||
if not locs:
|
||||
raise GameError("no subtitle %s" % key)
|
||||
from .banks import open_file
|
||||
l = locs[0]
|
||||
b = open_file(self.ws.bank_path(l.file)).read(l.pos, l.size)
|
||||
b = banks.open_file(self.ws.bank_path(l.file)).read(l.pos, l.size)
|
||||
return subtitles.read_fstring(b, 0)[0]
|
||||
return self._menu_entry(key).value
|
||||
|
||||
def catalog_text(self, key):
|
||||
"""Original text according to the shipped catalog (subtitles only), or None."""
|
||||
"""Original text according to the catalog (subtitles only), or None."""
|
||||
if not key.startswith("SUB:"):
|
||||
return None
|
||||
_, lang, rest = key.split(":", 2)
|
||||
speech, line = rest.rsplit(":", 1)
|
||||
for s in self.subtitles(lang):
|
||||
if s.speech == speech and s.line == int(line):
|
||||
return s.text
|
||||
return None
|
||||
if lang not in self._sub_text:
|
||||
self._sub_text[lang] = {(s.speech, s.line): s.text for s in self.subtitles(lang)}
|
||||
return self._sub_text[lang].get((speech, int(line)))
|
||||
|
||||
def original(self, key):
|
||||
r = self.records().get(key)
|
||||
@@ -141,24 +179,54 @@ class Texts:
|
||||
for l in self.subtitle_locations(key)), default=0)
|
||||
|
||||
def check(self, key, text):
|
||||
"""Raise SubtitleError / TextError if `text` cannot be written for `key`."""
|
||||
"""Raise SubtitleError / TextError (ValueError) if `text` cannot be written for `key`, GameError if
|
||||
there is no such text."""
|
||||
if key.startswith("SUB:"):
|
||||
for l in self.subtitle_locations(key):
|
||||
subtitles.encode_fstring(text, l.size, l.enc)
|
||||
else:
|
||||
e = self._menu_entry(key)
|
||||
path = loctext.files(self.loc_root, e.lang)[e.file]
|
||||
loctext.write(open(path, "rb").read(), e.lang, e.file, {e.key: text})
|
||||
path = self._menu_path(key)
|
||||
self._menu_entry(key)
|
||||
loctext.check_value(text, self._menu_entries(path, key.split(":")[1])[0], os.path.basename(path))
|
||||
|
||||
def unoriginal_subtitles(self, rel):
|
||||
"""Subtitle strings in package `rel` (cooked-relative) that differ from the shipped catalog: [SUB key]
|
||||
(none if the catalog is not the game's)."""
|
||||
if self._shipped_subs is None:
|
||||
self._shipped_subs = ({}, {})
|
||||
shipped = _shipped_index().get("SUB", {})
|
||||
loc = os.path.join(catalog_dir(), subtitles.LOCATIONS)
|
||||
if shipped.get("fingerprint") == banks.game_fingerprint(self.ws.game.cooked) and os.path.exists(loc):
|
||||
by_file, texts = self._shipped_subs
|
||||
for ls in subtitles.read_locations(loc).values():
|
||||
for l in ls:
|
||||
by_file.setdefault(l.file, []).append(l)
|
||||
for lang in shipped.get("lines", {}):
|
||||
p = os.path.join(catalog_dir(), "SUB_%s.csv" % lang)
|
||||
if os.path.exists(p):
|
||||
texts.update({(lang, s.speech, s.line): s.text for s in subtitles.read_texts(p, lang)})
|
||||
by_file, texts = self._shipped_subs
|
||||
out = []
|
||||
locs = sorted(by_file.get(rel, []), key=lambda l: l.pos)
|
||||
if locs:
|
||||
r = banks.open_file(self.ws.bank_path(rel))
|
||||
for l in locs:
|
||||
want = texts.get((l.lang, l.speech, l.line))
|
||||
if want is not None and subtitles.read_fstring(r.read(l.pos, l.size), 0)[0] != want:
|
||||
out.append(subtitles.key(l.lang, l.speech, l.line))
|
||||
return out
|
||||
|
||||
# --- writing --------------------------------------------------------------------------------
|
||||
@_locked
|
||||
def apply(self, changes, force=False, progress=None, meta=None):
|
||||
"""changes: {key: new text}. Every file holding one of them is rebuilt once."""
|
||||
for k, v in changes.items():
|
||||
self.check(k, v)
|
||||
conflicts = [k for k in changes if self.state(k) in (CHANGED, FOREIGN)]
|
||||
paths = {p for k in changes for p in self.files(k)}
|
||||
changed = set(self.ws.changed_files(paths))
|
||||
conflicts += [k for k in changes if k not in conflicts and set(self.files(k)) & changed]
|
||||
files = {k: self.files(k) for k in changes}
|
||||
paths = {p for ps in files.values() for p in ps}
|
||||
changed = set(self.ws.changed_files(paths)) | set(self.ws.unoriginal_files(paths))
|
||||
conflicts += [k for k in changes if k not in conflicts and set(files[k]) & changed]
|
||||
if conflicts and not force:
|
||||
raise TextConflictError(conflicts)
|
||||
for p in paths:
|
||||
@@ -167,12 +235,13 @@ class Texts:
|
||||
now = time.time()
|
||||
for k, v in changes.items():
|
||||
if k not in recs:
|
||||
recs[k] = {"orig": self.current(k), "files": [self.ws.game.rel(p) for p in self.files(k)],
|
||||
recs[k] = {"orig": self.current(k), "files": [self.ws.game.rel(p) for p in files[k]],
|
||||
"first_modified": now, "applied": None}
|
||||
recs[k].update(applied=v, applied_at=now, meta=meta or {})
|
||||
self.ws.store.save()
|
||||
self._sync(paths, progress)
|
||||
|
||||
@_locked
|
||||
def revert(self, keys, progress=None):
|
||||
recs = self.records()
|
||||
keys = [k for k in keys if k in recs]
|
||||
|
||||
Reference in New Issue
Block a user