Version 3.2.1: whole bank copies of voice lines, unique names, docs figures checked by tools/check_docs.py

ackit/__init__.py

@@ -8,4 +8,4 @@ Library layers (low -> high):
             inject (WAD writing), originals (store), workspace (apply/revert), modpack (share)
   tools     cli (`ackit` command), verify/ (offline self-check, in-game test on Windows)
 """
-__version__ = "3.2.0"
+__version__ = "3.2.1"

ackit/banks.py

@@ -156,12 +156,14 @@ class Scan:
         self.events = {}                     # id -> [action ids]
         self.actions = {}                    # id -> target id
         self.sound_media = defaultdict(set)  # sound object id -> media ids
-        self.prefetch = set()                # media ids whose in-memory copy is a prefetch head
+        self.prefetch = set()                # media ids whose bank copies belong to a WAD recording (prefetch
+                                             # heads; whole recordings: catalog._whole_recordings)
         self.lang_media = defaultdict(set)   # language -> media ids named by Sound objects of that language's banks
         # BKPK tables: WAD entry id -> {language: {media id: (offset in entry, size)}}. A table belongs to the
         # language its localized AkBank export declares ('' = outside such an export). The same media ID can be
         # used by several languages (at a different position in each language's WAD), so tables never merge.
         self.packs = defaultdict(_by_language)
+        self.subtitle_objects = {}           # counts of dialogue / RSubtitle objects (catalog.scan_text)
         self.issues = []                     # (kind, detail)
 
 
@@ -368,11 +370,12 @@ def media_language(sc, m):
     return next(iter(langs)) if len(langs) == 1 else ""
 
 
-def media_names(sc):
+def media_names(sc, reserved=()):
     """{media id: (display name, primary bank name)}: named after the most specific 'Play' event that
     reaches the media, otherwise after its bank (<Bank>_<n>). Localized speech has one media per language
     for the same line: they share the name, variations are numbered within a language. Names are unique
-    among the language-neutral media plus any one language (what a catalog shows at a time)."""
+    among the language-neutral media plus any one language (what a catalog shows at a time) and avoid the
+    lower-case names in `reserved` (other archives' sounds)."""
     ev = event_media(sc)
     best = {}
     for name, ms in ev.items():
@@ -386,7 +389,7 @@ def media_names(sc):
                 if m not in best or rank < best[m][0]:
                     best[m] = (rank, name, i, len(g))
     out = {}
-    taken = set()                       # (language, lower-case name)
+    taken = {("", n) for n in reserved}  # (language, lower-case name)
 
     def free(lang, nm):
         k = nm.lower()

ackit/catalog.py

@@ -234,10 +234,11 @@ def scan_archive(archive, wad_path, characters=None, progress=None):
 _CAMEL = re.compile(r"(?<=[a-z])(?=[A-Z])")
 
 
-def bank_sounds(sc, characters=None):
-    """[Sound] of the in-memory sounds found by `banks.scan` (prefetch heads belong to WAD sounds)."""
+def bank_sounds(sc, characters=None, reserved=()):
+    """[Sound] of the in-memory sounds found by `banks.scan` (prefetch heads belong to WAD sounds). `reserved`:
+    lower-case names taken by other archives (sfx_names)."""
     characters = characters if characters is not None else load_characters()
-    names = banks.media_names(sc)
+    names = banks.media_names(sc, reserved)
     every = set().union(*(e.get("langs", set()) for e in sc.media.values())) - {""}
     out = []
     for m, e in sc.media.items():
@@ -275,13 +276,45 @@ def derive_banks(game, sc, characters=None):
     """(bank sounds, locations, Lines) from a Scan: the lines, the prefetch heads' lines, the in-memory sounds
     with their line identities and names."""
     lines = build_lines(game, sc)
+    mark_heads(sc, whole_recordings(sc, lines))
     locs = [l._replace(sound=lines.head_key(l)) if l.kind == banks.PREFETCH_HEAD else l for l in sc.locations]
-    snd = bank_sounds(sc, characters)
+    snd = bank_sounds(sc, characters, sfx_names(game))
     lines.names = line_names(game, sc, lines, {s.key: s.name for s in snd})
     apply_lines(snd, lines, lines.names)
     return snd, locs, lines
 
 
+def whole_recordings(sc, lines):
+    """Media stored in memory that are WAD recordings of their bank's language (the same media ID, identical
+    bytes: 49 in the game, e.g. game-over taunts). Like prefetch heads, these copies belong to the WAD line
+    and are written with it; they are not sounds of their own."""
+    keys = collections.defaultdict(list)
+    for l in sc.locations:
+        if l.kind == banks.MEMORY:
+            k = lines.head_key(l)
+            keys[l.media].append(bool(k) and not k.startswith(banks.ARCHIVE + ":"))
+    return {m for m, wad in keys.items() if all(wad)}
+
+
+def mark_heads(sc, media):
+    """Treat the bank copies of `media` as heads of WAD recordings (see whole_recordings)."""
+    if not media:
+        return
+    for m in media:
+        if m in sc.media:
+            sc.media[m]["kinds"] = {banks.PREFETCH_HEAD}
+            sc.prefetch.add(m)
+    sc.locations = [l._replace(kind=banks.PREFETCH_HEAD) if l.media in media else l for l in sc.locations]
+
+
+def sfx_names(game):
+    """Lower-case names and aliases of the SFX WAD's sounds: in-memory sounds of the same bank must not take
+    them (`<Bank>_<n>` is the position name of the bank's streamed sounds too)."""
+    if "SFX" not in game.archives:
+        return set()
+    return {n.lower() for s in scan_archive("SFX", game.wad("SFX"), characters={}) for n in s.names()}
+
+
 def line_names(game, sc, lines, bank_names=None):
     """{line id: name} (lines.line_names). Lines no event names keep their English recording's name: the
     position-based name in the INT WAD (`NAME_i`, or `NAME#i` where that is taken) or the bank-numbered name."""
@@ -378,6 +411,10 @@ def scan_text(game, progress=None, characters=None):
     sc = banks.scan(game.cooked, progress, visit=col)
     names = {n for n in sc.names if n.lower().startswith("speech_")}
     names |= {n for n, _ in banks.media_names(sc).values() if n.lower().startswith("speech_")}
+    lookup = [n for n in col.dialogues if n.startswith("#lookup:")]
+    sc.subtitle_objects = {"dialogue_objects": sum(len(v) for n, v in col.dialogues.items() if n not in lookup),
+                           "dialogue_lines": len(col.dialogues) - len(lookup),
+                           "scripted_objects": sum(len(col.dialogues[n]) for n in lookup)}
     col.resolve(names)
     # the subtitle objects name their speech events too: an event ID is the FNV-1 hash of its name, so these
     # names find events no package name table lists (e.g. Vicki Vale's radio reports)
@@ -519,6 +556,11 @@ class Catalog:
         key = os.path.splitext(os.path.basename(name))[0].lower() if re.search(r"\.\w{2,4}$", name) else name.lower()
         hits = self._by_name.get(key, [])
         uniq = list({id(s): s for s in hits}.values())
+        if len(uniq) > 1:
+            # a name wins over another sound's alias: a line named after its INT position (NAME#i) can equal
+            # the position alias of another recording in another language's WAD
+            own = [s for s in uniq if s.name.lower() == key]
+            uniq = own if len(own) == 1 else uniq
         if len(uniq) == 1:
             return uniq[0]
         if len(uniq) > 1:

ackit/cli.py

@@ -294,7 +294,8 @@ def cmd_game(a):
 def cmd_verify(a):
     if a.catalog:
         from .verify import catalogcheck
-        rep = catalogcheck.run(_game(a), [x.upper() for x in a.archive] or None, quick=a.quick)
+        archives = [x.upper() for x in a.archive] or ([a.lang] if a.lang else None)
+        rep = catalogcheck.run(_game(a), archives, quick=a.quick)
         for (arch, kind), n in sorted(rep.counts().items()):
             print("%-14s %-10s %d" % (arch, kind, n))
         for arch, kind, key, detail in rep.issues[:a.show]:
@@ -310,6 +311,9 @@ def cmd_verify(a):
         ok = selfcheck.run(ws, log=print)
         sys.exit(0 if ok else 1)
     from .verify import ingame
+    if a.lang and a.lang != ws.game.language():
+        raise GameError("the game plays its own language (%s); set it in Steam (game properties > Language) to "
+                        "test %s in the game" % (ws.game.language(), a.lang))
     if a.banks:
         rep = ingame.run_banks(ws, a.sound or None)
     elif a.texts:
@@ -385,7 +389,7 @@ def build_parser():
     s = add("verify", cmd_verify, "self-check (offline), catalog cross-check (--catalog), or test in the real "
             "game with --ingame (Windows)")
     s.add_argument("--catalog", action="store_true", help="cross-check the catalog against the game files")
-    s.add_argument("--archive", action="append", default=[], help="with --catalog: only this archive (repeatable)")
+    s.add_argument("--archive", action="append", default=[], help="with --catalog: only this archive (repeatable; --lang X: only X)")
     s.add_argument("--quick", action="store_true", help="with --catalog: loose banks + a sample of packages")
     s.add_argument("--show", type=int, default=30, help="with --catalog: issues to list (default 30)")
     s.add_argument("--ingame", action="store_true"); s.add_argument("--target", default="SFX:EDF036D6/393BC53A")

ackit/lines.py

@@ -27,6 +27,7 @@ class Lines:
     objects: dict = field(default_factory=dict)        # line id -> sorted Sound object ids
     obj_keys: dict = field(default_factory=dict)       # Sound object id -> {lang: sound key}
     names: dict = field(default_factory=dict)          # line id -> name (catalog.line_names)
+    unnamed: set = field(default_factory=set)          # line ids no named event plays (named after a recording)
     copy_of: dict = field(default_factory=dict)        # WAD key -> key of the same media stored in another entry
     issues: list = field(default_factory=list)         # (kind, detail): broken one-to-one relations
     stats: dict = field(default_factory=dict)
@@ -161,10 +162,10 @@ def build(game, sc, walk):
     for k, primary in copy_of.items():                 # a second place of the same recording: the same line
         if primary in lines.line_of:
             lines.line_of[k] = lines.line_of[primary]
-    all_keys = set().union(*lines._keys.values()) if lines._keys else set()
-    all_keys |= set(copy_of)
+    recordings = set().union(*lines._keys.values()) if lines._keys else set()
+    all_keys = recordings | set(copy_of)               # + the second places of recordings stored twice
     full = sum(1 for per in lines.members.values() if len(per) == len(langs))
-    lines.stats = {"languages": langs, "wad_wems": wems, "recordings": len(all_keys),
+    lines.stats = {"languages": langs, "wad_wems": wems, "recordings": len(recordings),
                    "in_a_line": len(lines.line_of), "lines": len(lines.members), "lines_in_every_language": full,
                    "copies": sorted(copy_of),
                    "lines_missing_a_language": {lid: sorted(set(langs) - set(per))
@@ -215,8 +216,9 @@ def line_names(sc, lines, fallback):
                 out[lid] = name if i == 0 else "%s#%d" % (name, i + 1)
             else:
                 out[lid] = name if len(lids) == 1 else "%s_%d" % (name, i + 1)
-    for lid, per in lines.members.items():
-        if lid not in out:
-            k = per.get("INT") or per[next(L for L in lines.langs if L in per)]
-            out[lid] = fallback(k)
+    lines.unnamed = set(lines.members) - set(out)
+    for lid in lines.unnamed:
+        per = lines.members[lid]
+        k = per.get("INT") or per[next(L for L in lines.langs if L in per)]
+        out[lid] = fallback(k)
     return out

ackit/subtitles.py

@@ -9,7 +9,7 @@ a byte size; struct elements end with a u16 0).
 A dialogue object can be stored in several packages; a subtitle edit is written into every copy, in place:
 the new text replaces the bytes of the old FString and is padded with NUL characters (the game stops at the
 first NUL), so no size field, export or package layout changes. The text must therefore fit the original
-string's bytes (Latin-1/cp1252 strings: one byte per character; UTF-16 strings: two).
+string's bytes (cp1252 strings: one byte per character; UTF-16 strings: two).
 """
 import csv
 import gzip
@@ -55,7 +55,8 @@ def encode_fstring(text, size, enc):
         try:
             raw = text.encode("cp1252")
         except UnicodeEncodeError:
-            raise SubtitleError("this subtitle is stored as Latin-1 text; '%s' cannot be written"
+            raise SubtitleError("this subtitle is stored as Western European (Windows-1252) text; '%s' cannot be "
+                                "written"
                                 % next(c for c in text if not _cp1252(c))) from None
         n = size - 4
         if len(raw) + 1 > n:
@@ -175,6 +176,7 @@ class Collector:
 
     def __init__(self):
         self.dialogues = defaultdict(list)       # speech name -> [(file, Dialogue)]
+        self.scripted = set()                    # speech names of RSubtitle objects (resolve)
         self.issues = []
 
     def __call__(self, rel, pkg, start, data):
@@ -201,6 +203,7 @@ class Collector:
                 self.issues.append(("subtitles", "no speech line for subtitle %s" % copies[0][1].export))
                 name = copies[0][1].export
             self.dialogues[name].extend((rel, d._replace(speech=name)) for rel, d in copies)
+            self.scripted.add(name)
 
 
 # --- catalog files: SUB_<LANG>.csv (texts) + SUB_locations.csv.gz (every copy) ------------------

ackit/verify/catalogcheck.py

@@ -18,7 +18,8 @@ import random
 
 from .. import banks, classify, subtitles
 from ..catalog import (Catalog, apply_lines, available_archives, bank_names, bank_sounds, build_lines, character_of,
-                       is_shipped, line_names, load_archive, load_banks, load_characters, read_csv, scan_archive)
+                       is_shipped, line_names, load_archive, load_banks, load_characters, mark_heads, read_csv,
+                       scan_archive, sfx_names, whole_recordings)
 from ..inject import read_target
 from ..paths import catalog_dir
 from ..wad import Wad
@@ -133,14 +134,17 @@ def check_banks(rep, game, rows, locations, characters, quick=False, sample=40,
     col.resolve({n for n in sc.names | known if n.lower().startswith("speech_")})
     check_subtitles(rep, game, col, set(files), quick)
     lines = None
-    if not quick:
-        sc.names.update(col.dialogues)        # as the build does: subtitle objects name speech events too
-        lines = build_lines(game, sc)
     index = collections.defaultdict(set)
     for m, ls in locations.items():
         for l in ls:
             if l.file in set(files):
                 index[(l.media, l.kind)].add((l.file, l.offset, l.size, l.sound if l.kind == banks.PREFETCH_HEAD else ""))
+    if not quick:
+        sc.names.update(col.dialogues)        # as the build does: subtitle objects name speech events too
+        lines = build_lines(game, sc)
+        mark_heads(sc, whole_recordings(sc, lines))       # WAD recordings also stored whole in banks
+    else:                                     # a sample cannot derive them: the index says which they are
+        mark_heads(sc, {m for m, k in index if k == banks.PREFETCH_HEAD} - sc.prefetch)
     found = collections.defaultdict(set)
     for l in sc.locations:
         head = l.kind == banks.PREFETCH_HEAD
@@ -152,7 +156,7 @@ def check_banks(rep, game, rows, locations, characters, quick=False, sample=40,
             rep.add(banks.ARCHIVE, "copies", "%08X" % k[0], "%s: index %d, game %d"
                     % ("head" if k[1] == banks.PREFETCH_HEAD else "copies", len(index.get(k, ())), len(found.get(k, ()))))
     cat = {s.target: s for s in rows}
-    fresh_list = bank_sounds(sc, characters)
+    fresh_list = bank_sounds(sc, characters, sfx_names(game))
     if lines is not None:
         lines.names = line_names(game, sc, lines, {s.key: s.name for s in fresh_list})
         apply_lines(fresh_list, lines, lines.names)
@@ -253,11 +257,11 @@ def check_lines(rep, game, sc, lines, locations):
                     rep.add("LINES", "links", k, "media %08X is not a source in any %s bank" % (m, L))
     rep.facts.append("WAD recordings named as a source by a Sound object of their own language's bank: %d of %d"
                      % (named, total))
-    rep.facts.append("voice languages %s: %d recordings (WAD %s + localized in-memory), %d lines, %d of them "
-                     "with exactly one recording in every language"
-                     % ("/".join(lines.langs), st["recordings"],
-                        "/".join(str(st["wad_wems"].get(L, 0)) for L in lines.langs), st["lines"],
-                        st["lines_in_every_language"]))
+    rep.facts.append("voice languages %s: %d recordings (WAD WEMs %s, %d of them second places of a recording "
+                     "stored twice, + localized in-memory), %d lines, %d of them with exactly one recording in every "
+                     "language" % ("/".join(lines.langs), st["recordings"],
+                                   "/".join(str(st["wad_wems"].get(L, 0)) for L in lines.langs), len(st["copies"]),
+                                   st["lines"], st["lines_in_every_language"]))
     # catalog rows vs the game's lines
     rows = {}
     for L in lines.langs + [banks.ARCHIVE]:
@@ -305,7 +309,8 @@ def check_lines(rep, game, sc, lines, locations):
             rep.add("LINES", "copies", k, "differs from %s (same media ID)" % primary)
     rep.facts.append("recordings stored twice in a WAD (same media ID, written together): %d, byte-identical: %d"
                      % (len(lines.copy_of), same_copies))
-    rep.facts.append("prefetch heads: %d of %d stored copies are byte-identical to the start of their WAD recording"
+    rep.facts.append("prefetch heads and whole bank copies: %d of %d are byte-identical to the start of their WAD "
+                     "recording"
                      % (same, len(heads)))
     # independent evidence: the recordings of a line have similar lengths in every language
     ratios = []