Cristian Gora
1ccaa8b398
Safety - Writes are serialized (Workspace.lock): two background jobs rebuilding one package (a sound and its subtitle applied side by side) lost an edit. - A store copies a bank file or package as "original" only if it matches the shipped catalog (Workspace.unoriginal_files: in-memory sounds, bank copies of voice lines, subtitles); the editor refuses to switch stores while changes are applied. A second store could otherwise resurrect a reverted sound. - Mod import in the editor asks before overwriting files changed outside the tool (it forced silently). - Revert all in the editor reverts texts too; the uninstaller counts text edits before offering to delete the store. - ackit restore-backup: the full-backup restore decisions.md described; it keeps the sounds of WADs without a full copy recorded. - build_catalog also stops when a subtitle differs from the catalog. Languages and texts - Game.language() returns languages without voices (JPN, RUS, ...) and Game.voice_archive() their INT voices; ackit text --lang JPN works. - Menu lookups parse one cached file (find & replace: minutes -> ms). - Subtitles decode cp1252 as Windows does: the UTF-8-in-cp1252 line Speech_TAPE_J_Answer_1_00 (KOR/POL/POR/RUS) is written back byte-for-byte; SUB_KOR/SUB_RUS rebuilt from the original game (only those rows changed). - Mods exported while another language is loaded keep name and line. Other fixes - Down-mix folds extra channels in at -3 dB (a 5.1 centre voice was dropped). - e2e_editor no longer overwrites the user's settings.json; kill_game waits for the game to exit before reverts. - Recorder handles Int32/UInt8 input and split frames; package block cache locked; conflict messages, CLI help examples, stale docstrings and doc references fixed; tests no longer read this PC's game language. Docs (decisions, user guide, README, subtitles.md, banks.md, catalog README, REPORT, NOTES, CLAUDE.md) updated; check_docs: 102 claims match. 148 tests; e2e_features 9/9, e2e_editor and verify PASS; game original afterwards.
584 lines
34 KiB
Python
584 lines
34 KiB
Python
"""Check every figure the documentation states against the data it describes (read-only).
|
|
|
|
python tools/check_docs.py [--game DIR] [--scan] [--wems]
|
|
|
|
The figures are recomputed, never copied: from the shipped catalog (`data/catalog/`, built from the original
|
|
game files) and, when the game is installed, its menu files; with --scan also from a fresh scan of every
|
|
package (bank survey: blobs, source kinds, bank names; ~3 minutes) and from `ackit verify --catalog`'s facts
|
|
(relations re-derived from the game; ~6 minutes more); with --wems from rebuilding every WEM of the speech and
|
|
SFX WADs from the format rules (~10 minutes). Each claim is the exact text a document must contain
|
|
(line breaks and repeated spaces ignored). Exit code 1 when a document states anything else.
|
|
History logs (NOTES.md, findings.md, docs/history/) record what was true on their date and are not checked.
|
|
"""
|
|
import argparse
|
|
import collections
|
|
import csv
|
|
import glob
|
|
import gzip
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
sys.path.insert(0, ROOT)
|
|
|
|
from ackit import banks, settings, subtitles # noqa: E402
|
|
from ackit.catalog import read_csv # noqa: E402
|
|
from ackit.game import Game, default_game_dir # noqa: E402
|
|
from ackit.paths import catalog_dir # noqa: E402
|
|
|
|
VOICES = ["INT", "FRA", "ITA", "DEU", "ESN"]
|
|
CLAIMS = [] # (document, text, what it is checked against)
|
|
|
|
|
|
def n(x):
|
|
return "{:,}".format(x)
|
|
|
|
|
|
def rng(values, fmt=n):
|
|
lo, hi = min(values), max(values)
|
|
return fmt(lo) if lo == hi else "%s-%s" % (fmt(lo), fmt(hi))
|
|
|
|
|
|
def pct(a, b):
|
|
return int(round(100.0 * a / b))
|
|
|
|
|
|
def claim(doc, text, why=""):
|
|
CLAIMS.append((doc, text, why))
|
|
|
|
|
|
# --- figures from the shipped catalog --------------------------------------------------------------
|
|
def catalog_claims(cat):
|
|
rows = {a: read_csv(os.path.join(cat, a + ".csv")) for a in VOICES + ["SFX", "BANK"]}
|
|
bank, sfx = rows["BANK"], rows["SFX"]
|
|
wad_n = {len(rows[L]) for L in VOICES}
|
|
assert len(wad_n) == 1, "WADs differ in size: %s" % wad_n
|
|
W = wad_n.pop()
|
|
claim("REPORT.md", "(5 languages x %s), the SFX WAD (%s)" % (n(W), n(len(sfx))), "catalog rows")
|
|
claim("REPORT.md", "the **%s in-memory sounds**" % n(len(bank)), "BANK.csv rows")
|
|
claim("docs/formats/banks.md", "so there are **%s in-memory sounds**" % n(len(bank)), "BANK.csv rows")
|
|
|
|
# languages of the in-memory sounds
|
|
neutral = [s for s in bank if not s.language]
|
|
single = collections.Counter(s.language for s in bank if s.language and ";" not in s.language)
|
|
multi = [s for s in bank if ";" in s.language]
|
|
multi_c = collections.Counter(s.language for s in multi)
|
|
claim("REPORT.md", "%s shared by all languages, ~%s localized speech lines per voice language, %d shared by "
|
|
"several" % (n(len(neutral)), n(int(round(sum(single.values()) / len(single), -1))), len(multi)))
|
|
claim("docs/formats/banks.md", "%s sounds are language-neutral, %s belong to one voice language, %d to several"
|
|
% (n(len(neutral)), rng(single.values()), len(multi)))
|
|
a4 = multi_c.get("ESN;FRA;INT;ITA", 0)
|
|
a3 = multi_c.get("FRA;INT;ITA", 0)
|
|
claim("docs/formats/banks.md", "(%d shared by INT/FRA/ITA/ESN while DEU has its own, %d by INT/FRA/ITA)" % (a4, a3))
|
|
claim("docs/formats/banks.md", "everywhere; %d of these are voice lines" % sum(1 for s in neutral if s.line))
|
|
claim("docs/formats/banks.md", "stored once for every language (%d sounds)"
|
|
% sum(1 for s in neutral if s.bank.startswith("Speech")))
|
|
claim("docs/formats/banks.md", "lines** (about %s per language" % n(int(round(sum(single.values()) / len(single), -1))))
|
|
ch = collections.Counter(s.channels for s in bank)
|
|
claim("docs/formats/banks.md", "of all %s in-memory sounds %s are mono, %s stereo, %d have 3-6 channels"
|
|
% (n(len(bank)), n(ch[1]), n(ch[2]), sum(v for k, v in ch.items() if k >= 3)))
|
|
|
|
# copies and heads
|
|
with gzip.open(os.path.join(cat, "BANK_locations.csv.gz"), "rt", newline="") as f:
|
|
loc = list(csv.DictReader(f))
|
|
m_rows = [r for r in loc if r["kind"] == "m"]
|
|
p_rows = [r for r in loc if r["kind"] == "p"]
|
|
keys = {s.key: s for L in VOICES for s in rows[L]}
|
|
whole = [r for r in p_rows if int(r["size"]) >= keys[r["sound"]].size]
|
|
pref = [r for r in p_rows if r not in whole]
|
|
whole_rec = {r["sound"] for r in whole}
|
|
pref_rec = {r["sound"] for r in pref}
|
|
pref_lines = {keys[k].line for k in pref_rec}
|
|
all_lines = {keys[r["sound"]].line for r in p_rows}
|
|
per_lang_pref = collections.Counter(k.split(":")[0] for k in pref_rec)
|
|
per_lang_whole = collections.Counter(k.split(":")[0] for k in whole_rec)
|
|
claim("REPORT.md", "copy (%s copies). %s WAD voice recordings (%d lines) also have bank copies (%d prefetch heads, %d "
|
|
"whole copies; %s copies)" % (n(len(m_rows)), n(len(pref_rec | whole_rec)), len(all_lines), len(pref_rec),
|
|
len(whole_rec), n(len(p_rows))))
|
|
claim("docs/formats/banks.md", "%d of them are **prefetch heads** of streamed WAD sounds and %d are **whole copies**"
|
|
% (len({r["media"] for r in pref}), len({r["media"] for r in whole})))
|
|
claim("docs/formats/banks.md", "%d WAD sounds have such heads (%s copies)" % (len(pref_rec), n(len(pref))))
|
|
claim("docs/formats/banks.md", "%d more WAD recordings (%s per language" % (len(whole_rec), rng(per_lang_whole.values())))
|
|
claim("docs/formats/banks.md", "byte-identical to the WAD WEM (%d copies)" % len(whole))
|
|
claim("docs/formats/banks.md", "Heads and whole copies together: %d recordings, %s copies"
|
|
% (len(pref_rec | whole_rec), n(len(p_rows))))
|
|
claim("docs/formats/banks.md", "the %d lines with a prefetch head (%d recordings over the 5 languages"
|
|
% (len(pref_lines), len(pref_rec)))
|
|
claim("docs/formats/banks.md", "A WAD recording that a bank also stores whole (%d)" % len(whole_rec))
|
|
claim("docs/decisions.md", "%d voice recordings (%d lines, %s recordings per language) have their first bytes"
|
|
% (len(pref_rec), len(pref_lines), rng(per_lang_pref.values(), str)))
|
|
claim("docs/decisions.md", "and %d more are stored whole in banks" % len(whole_rec))
|
|
places = len(loc)
|
|
media_all = {r["media"] for r in loc}
|
|
loose = {r["media"] for r in loc if r["file"].startswith("SFX/")}
|
|
pk = {r["media"] for r in loc if not r["file"].startswith("SFX/")}
|
|
claim("docs/formats/banks.md", "The two sets share %d media IDs: %s unique media IDs in %s places"
|
|
% (len(loose & pk), n(len(media_all)), n(places)))
|
|
claim("docs/formats/banks.md", "+ `TST_Prep_Media`) | %d | %s |"
|
|
% (len({r["file"] for r in loc if r["file"].startswith("SFX/")}), n(len(loose))))
|
|
claim("docs/formats/banks.md", "packages) | 3,340 | %s |" % n(len(pk)))
|
|
claim("REPORT.md", "in %d loose `.bnk` files and %d `.upk` packages"
|
|
% (len({r["file"] for r in loc if r["file"].startswith("SFX/")}), len({r["file"] for r in loc if r["file"].endswith(".upk")})))
|
|
assert len(media_all) == len(bank) + len(pref_rec | whole_rec)
|
|
|
|
# lines
|
|
lines_all = {s.line for L in VOICES for s in rows[L] if s.line} | {s.line for s in bank if s.line}
|
|
wad_lines = {L: {s.line for s in rows[L] if s.line} for L in VOICES}
|
|
assert len({len(v) for v in wad_lines.values()}) == 1
|
|
WL = len(wad_lines["INT"])
|
|
copies = {L: sum(1 for s in rows[L] if s.line) - len(wad_lines[L]) for L in VOICES}
|
|
recs = sum(len(wad_lines[L]) for L in VOICES) + len({(s.target) for s in bank if s.line})
|
|
for doc, text in [("REPORT.md", "%s lines (WAD + in-memory) with exactly one recording in each of the 5 voice languages"),
|
|
("docs/decisions.md", "recording per language (%s lines, all 5 languages)"),
|
|
("docs/formats/banks.md", "recordings that are sources of the same Sound objects: %s lines")]:
|
|
claim(doc, text % n(len(lines_all)), "distinct line ids")
|
|
claim("docs/formats/banks.md", "in each of the 5 voice languages (%s recordings)" % n(recs))
|
|
claim("REPORT.md", "recordings stored twice (%s per language) are written together" % rng(copies.values(), str))
|
|
claim("docs/formats/banks.md", "- %s recordings per language are stored twice" % rng(copies.values(), str).capitalize())
|
|
claim("docs/categories.md", "Coverage of the %s WAD voice lines" % n(WL))
|
|
# recordings the game shares between languages (the same bytes as the INT recording of the line)
|
|
intl = {s.line: s.sha1 for s in rows["INT"] if s.line}
|
|
same = {L: [s for s in {x.line: x for x in rows[L] if x.line}.values() if intl.get(s.line) == s.sha1]
|
|
for L in ("FRA", "ITA", "DEU", "ESN")}
|
|
ita = collections.Counter(s.category for s in same["ITA"])
|
|
shared = {s.target: s.language for s in bank if ";" in s.language}
|
|
claim("docs/formats/banks.md", "in each voice WAD %s lines are byte-identical to the INT recording (ITA %d: efforts "
|
|
"and emotes %d, cinematics %d, boss fights %d, story %d, radio/TV %d, character lines %d), %d in-memory voice "
|
|
"lines are stored once for every language (efforts, radio chatter, crowds) and %d DLC intro lines"
|
|
% (rng([len(v) for v in same.values()]), len(same["ITA"]), ita["Efforts & emotes"], ita["Cinematics"],
|
|
ita["Boss fights"], ita["Story missions"], ita["Radio & TV"], ita["Character lines"],
|
|
sum(1 for s in bank if s.line and not s.language), len(shared)))
|
|
claim("docs/decisions.md", "of the %s WAD voice lines" % n(WL))
|
|
|
|
# positions differ between languages
|
|
it = {s.target: s.line for s in rows["INT"] if ":" in s.target}
|
|
diff = []
|
|
for L in ["FRA", "ITA", "DEU", "ESN"]:
|
|
other = {s.target: s.line for s in rows[L] if ":" in s.target}
|
|
diff.append(100.0 * sum(1 for t in it if other.get(t) != it[t]) / len(it))
|
|
span = "%.1f-%.1f %%" % (min(diff), max(diff))
|
|
claim("docs/formats/wad.md", "(%s of the WEMs in multi-variation entries, compared with INT" % span)
|
|
claim("docs/decisions.md", "%s of the WEMs in multi-variation entries are at another position than in INT" % span)
|
|
pos = {}
|
|
for L in VOICES:
|
|
s = next(s for s in rows[L] if s.name == "Speech_TAPE_JOKER_1_04")
|
|
pos[L] = int(s.target.split(":")[1])
|
|
claim("docs/formats/wad.md", "`Speech_TAPE_JOKER_1_04` is variation %d in INT and ITA, %d in FRA, %d in DEU, %d in ESN"
|
|
% (pos["INT"], pos["FRA"], pos["DEU"], pos["ESN"]) if pos["INT"] == pos["ITA"] else "(INT != ITA)")
|
|
ents = collections.Counter(s.target.split(":")[0] for s in rows["INT"])
|
|
claim("docs/formats/wad.md", "(every language: %s WEMs in %s entries" % (n(W), n(len(ents))))
|
|
radio = [s for s in rows["INT"] if s.aliases and any(a.startswith("CHR_RadioScanner#") for a in s.names())
|
|
or s.name.startswith("CHR_RadioScanner#")]
|
|
claim("docs/formats/wad.md", "e.g. `CHR_RadioScanner` holds %d" % len({s.target for s in radio}))
|
|
|
|
# SFX WAD
|
|
packs = collections.Counter(s.target.split("/")[0] for s in sfx if "/" in s.target)
|
|
grouped = [s for s in sfx if "/" not in s.target]
|
|
claim("docs/formats/wad.md", "**SFX WAD**: %d grouped ambience/effect entries (%d WEMs) plus %d **stream packs**"
|
|
% (len({s.target.split(":")[0] for s in grouped}), len(grouped), len(packs)))
|
|
claim("docs/formats/wad.md", "`EDF036D6` (Music, %d WEMs" % packs["EDF036D6"])
|
|
claim("docs/formats/wad.md", "`E8F980BE` (Music_PDLC, %d), `57572DAC` (Global, %d), `FCA4F778` (%d)"
|
|
% (packs["E8F980BE"], packs["57572DAC"], packs["FCA4F778"]))
|
|
|
|
# categories
|
|
cats = [collections.Counter(s.category for s in rows[L]) for L in VOICES]
|
|
assert all(c == cats[0] for c in cats), "category counts differ between languages"
|
|
order = ["Story missions", "Predator chatter", "Character lines", "Open-world chatter", "Side missions",
|
|
"Combat barks", "Efforts & emotes", "Interview tapes", "Riddler", "Radio & TV", "Gameplay & hints", "DLC",
|
|
"Boss fights", "Cinematics", "Reactions", "Interrogations"]
|
|
g = collections.Counter(s.group for s in sfx)
|
|
bc = collections.Counter(s.category for s in bank).most_common(8)
|
|
claim("docs/categories.md", "Counts per voice language (identical in all 5): " + ", ".join(
|
|
"%s %d" % (c, cats[0][c]) for c in order) + ". SFX archive: " + ", ".join(
|
|
"%s %d" % (k, g[k]) for k in ["Music", "Ambience", "SFX", "Unknown"]) + ". BANK (all languages): " + ", ".join(
|
|
"%s %d" % kv for kv in bc) + ", ...")
|
|
|
|
# characters (by line)
|
|
first = {}
|
|
for s in rows["INT"]:
|
|
first.setdefault(s.line, s)
|
|
wc = sum(1 for s in first.values() if s.character)
|
|
inf = sum(1 for s in first.values() if not s.character and s.characters_inferred)
|
|
nei = [s for s in first.values() if not s.character and not s.characters_inferred]
|
|
top = collections.Counter(s.category for s in nei).most_common(4)
|
|
assert all(sum(1 for s in rows[L] if s.character) == sum(1 for s in rows["INT"] if s.character) for L in VOICES)
|
|
claim("docs/categories.md", "(every voice language, the same lines): %s with a verified speaker, %s more with inferred "
|
|
"characters, %s with neither (%s, ...)" % (n(wc), n(inf), n(len(nei)), ", ".join(
|
|
"%s %d" % (c.lower().replace("radio & tv", "radio & TV"), k) for c, k in top)))
|
|
claim("docs/categories.md", "A hint for the ~%s lines without a verified speaker" % n(int(round(WL - wc, -2))))
|
|
claim("REPORT.md", "speaker labels (%s in every language)" % n(wc))
|
|
|
|
# subtitles
|
|
langs = sorted(f[4:7] for f in os.listdir(cat) if f.startswith("SUB_") and f.endswith(".csv"))
|
|
texts = {L: subtitles.read_texts(os.path.join(cat, "SUB_%s.csv" % L), L) for L in langs}
|
|
total = sum(len(t) for t in texts.values())
|
|
linked = sum(1 for t in texts.values() for s in t if s.sound)
|
|
i_tot, i_l = len(texts["INT"]), sum(1 for s in texts["INT"] if s.sound)
|
|
for doc in ("REPORT.md", "docs/formats/subtitles.md"):
|
|
claim(doc, "%s of %s strings in %d languages (INT %s" % (n(linked), n(total), len(langs), n(i_l))
|
|
if doc == "REPORT.md" else "Linked: %s of %s strings in the %d languages (INT %s / %s)"
|
|
% (n(linked), n(total), len(langs), n(i_l), n(i_tot)))
|
|
claim("REPORT.md", "(INT %s of %s)" % (n(i_l), n(i_tot)))
|
|
unl = {L: {s.speech for s in t if not s.sound} for L, t in texts.items()}
|
|
unl_n = {len(v) for v in unl.values()}
|
|
assert len(unl_n) == 1, unl_n
|
|
un_strings = [sum(1 for s in t if not s.sound) for t in texts.values()]
|
|
claim("REPORT.md", "%d speech names (%s strings per language, INT %d" % (unl_n.pop(), rng(un_strings),
|
|
i_tot - i_l))
|
|
claim("docs/formats/subtitles.md", "(%d speech names, %s strings per language" % (len(unl["INT"]), rng(un_strings)))
|
|
two = {s.speech for s in texts["INT"] if ";" in s.sound}
|
|
claim("docs/formats/subtitles.md", "(%d subtitled events" % len(two))
|
|
one = []
|
|
for L, t in texts.items():
|
|
c = collections.Counter(s.speech for s in t)
|
|
one.append(100.0 * sum(1 for v in c.values() if v == 1) / len(c))
|
|
claim("docs/formats/subtitles.md", "%s subtitle strings per language: %d-%d %% of the subtitles have one line (JPN: all)"
|
|
% (rng([len(t) for t in texts.values()]), int(min(one)), int(round(max(x for x in one if x < 100)))))
|
|
enc = {L: collections.Counter(s.enc for s in t) for L, t in texts.items()}
|
|
pol = pct(enc["POL"]["w"], len(texts["POL"]))
|
|
west = [enc[L]["w"] for L in langs if L not in ("RUS", "JPN", "KOR", "POL")]
|
|
claim("docs/formats/subtitles.md", "%d %% of POL are UTF-16; the other languages have %s UTF-16 strings each"
|
|
% (pol, rng(west)))
|
|
utf8 = [] # cp1252 strings whose bytes are UTF-8 text (the game's data)
|
|
for L, t in sorted(texts.items()):
|
|
for s in t:
|
|
if s.enc == "a":
|
|
u = subtitles._encode_cp1252(s.text).decode("utf-8", "replace")
|
|
multi = sum(1 for c in u if ord(c) > 0x7F and c != "\ufffd") # valid multi-byte characters
|
|
if multi and u.count("\ufffd") <= multi // 4: # (one corrupt byte tolerated)
|
|
utf8.append((L, s.speech))
|
|
claim("docs/formats/subtitles.md", "%d cp1252 strings hold UTF-8 bytes instead (the same line in %s: `%s`"
|
|
% (len(utf8), ", ".join(L for L, _ in utf8), "`, `".join(sorted({sp for _, sp in utf8}))))
|
|
with gzip.open(os.path.join(cat, subtitles.LOCATIONS), "rt", newline="") as f:
|
|
sublocs = sum(1 for _ in csv.DictReader(f))
|
|
claim("REPORT.md", "Edited in place in every copy (%s copies)" % n(sublocs))
|
|
claim("docs/formats/subtitles.md", "%s stored copies" % n(sublocs))
|
|
claim("docs/user-guide.md", "About %d subtitles per language have no sound" % int(round(sum(un_strings) / len(un_strings), -1)))
|
|
|
|
# subtitle coverage of the INT voice lines
|
|
lk = {k for s in texts["INT"] for k in s.sound.split(";") if k}
|
|
cov, tot = collections.Counter(), collections.Counter()
|
|
for s in rows["INT"] + [b for b in bank if b.line and "INT" in b.language.split(";")]:
|
|
if s.line:
|
|
tot[s.category] += 1
|
|
cov[s.category] += s.key in lk
|
|
p = {c: pct(cov[c], tot[c]) for c in tot}
|
|
claim("REPORT.md", "few for combat barks (%d %%), cinematics and boss fights (%d %%;"
|
|
% (p["Combat barks"], p["Cinematics"]) if p["Cinematics"] == p["Boss fights"] else "(cinematics != boss)")
|
|
hi = [p[c] for c in ("Story missions", "Predator chatter", "Character lines", "Open-world chatter")]
|
|
claim("REPORT.md", "%d-%d %% for story, predator, character and open-world lines" % (min(hi), max(hi)))
|
|
claim("REPORT.md", "(INT voice lines with a subtitle): none for efforts/grunts" if p["Efforts & emotes"] == 0 else "!")
|
|
|
|
# what the editor loads (Italian on the test PC: the smoke test)
|
|
ita = len(rows["ITA"]) + len(sfx) + sum(1 for s in bank if s.in_language("ITA"))
|
|
claim("REPORT.md", "%s sounds loaded, encode + inject, %s subtitles" % (n(ita), n(len(texts["ITA"]))))
|
|
claim("README.md", "~%s sounds of one language" % n(int(round(ita, -3))))
|
|
return rows
|
|
|
|
|
|
# --- figures recorded by the catalog build (index.json: they need the full scan) ---------------------
|
|
def index_claims(cat):
|
|
import json
|
|
ix = json.load(open(os.path.join(cat, "index.json"), encoding="utf-8"))
|
|
b, ln, sub = ix["BANK"], ix["LINES"], ix["SUB"]
|
|
claim("REPORT.md", "Names: %s BANK sounds are named after the Wwise event that plays them (`Play_...`, `Speech_...`), "
|
|
"the other %s after their bank" % (n(b["named_by_event"]), n(b["named_by_bank"])))
|
|
claim("REPORT.md", "**Unnamed sounds**: %s BANK sounds are reached by no event" % n(b["named_by_bank"]))
|
|
claim("docs/decisions.md", "sounds (%s of %s, voice lines included); the other %s are numbered per bank"
|
|
% (n(b["named_by_event"]), n(b["sounds"]), n(b["named_by_bank"])))
|
|
claim("docs/decisions.md", "the most specific speech event that plays it (%s of the %s WAD voice lines)"
|
|
% (n(ln["wad_lines_named_by_event"]), n(ln["wad_lines"])))
|
|
claim("docs/decisions.md", "the other %s keep the name of their English (INT) recording's position"
|
|
% n(ln["wad_lines_named_after_int_recording"]))
|
|
claim("REPORT.md", "- %s voice lines are played by no event whose name the game stores"
|
|
% n(ln["wad_lines_named_after_int_recording"]))
|
|
claim("docs/user-guide.md", "About %d %% of the lines are named by no game event"
|
|
% pct(ln["wad_lines_named_after_int_recording"], ln["wad_lines"]))
|
|
claim("docs/formats/subtitles.md", "%s dialogue objects in the packages for %s speech lines"
|
|
% (n(sub["dialogue_objects"]), n(sub["dialogue_lines"])))
|
|
claim("docs/formats/subtitles.md", "The sizes add up exactly in all %s objects" % n(sub["dialogue_objects"]))
|
|
claim("docs/formats/subtitles.md", "plus %d scripted subtitles" % sub["scripted_objects"])
|
|
claim("docs/formats/subtitles.md", "%d more subtitles (radio and encryption-card lines) are `RSubtitle` objects"
|
|
% sub["scripted_objects"])
|
|
claim("REPORT.md", "(%s speech lines x %d languages) and" % (n(sub["dialogue_lines"]), len(sub["lines"])))
|
|
claim("REPORT.md", "in %d scripted `RSubtitle` objects" % sub["scripted_objects"])
|
|
claim("docs/formats/banks.md", "(in %s of the 2,650 packages)" % n(b["files"] - 67)) # 67 loose .bnk with banks
|
|
|
|
|
|
# --- the game's own files ----------------------------------------------------------------------------
|
|
def menu_claims(game):
|
|
from ackit import loctext
|
|
root = os.path.join(game.root, "BmGame", "Localization")
|
|
counts = {L: len(loctext.load(root, L)) for L in loctext.languages(root)}
|
|
files = {L: len(loctext.files(root, L)) for L in loctext.languages(root)}
|
|
approx = n(int(round(sum(counts.values()) / len(counts), -3)))
|
|
claim("docs/formats/text.md", "~%d files and ~%s strings each" % (collections.Counter(files.values()).most_common(1)[0][0], approx))
|
|
claim("REPORT.md", "~%s strings per language" % approx)
|
|
claim("docs/user-guide.md", "of the selected language (about\n%s)" % approx)
|
|
claim("REPORT.md", "%s menu texts loaded" % n(counts["ITA"]))
|
|
|
|
|
|
def scan_claims(game):
|
|
"""Bank survey from a fresh scan of every package, and the relation facts of `ackit verify --catalog`."""
|
|
from ackit.bnk import find_banks, Bank, BankError, IN_MEMORY, STREAMED, PREFETCH
|
|
blobs = collections.Counter()
|
|
media_sets = collections.defaultdict(set)
|
|
kinds = collections.Counter()
|
|
in_pkg = set()
|
|
|
|
def visit(rel, pkg, start, data):
|
|
for bk in find_banks(data):
|
|
blobs[bk.id] += 1
|
|
media_sets[bk.id].add(frozenset(m.id for m in bk.media()))
|
|
in_pkg.add(bk.id)
|
|
own = {m.id for m in bk.media()}
|
|
for s in bk.sources():
|
|
if s.stream == IN_MEMORY:
|
|
kinds["memory own" if s.media in own else "memory other"] += 1
|
|
elif s.stream == STREAMED:
|
|
kinds["streamed"] += 1
|
|
elif s.stream == PREFETCH:
|
|
kinds["prefetch"] += 1
|
|
|
|
sc = banks.scan(game.cooked, visit=visit)
|
|
for path in glob.glob(os.path.join(game.cooked, "SFX", "*.bnk")): # loose banks: source kinds too
|
|
try:
|
|
bk = Bank(open(path, "rb").read(), 0)
|
|
except BankError:
|
|
continue
|
|
own = {m.id for m in bk.media()}
|
|
for s in bk.sources():
|
|
if s.stream == IN_MEMORY:
|
|
kinds["memory own" if s.media in own else "memory other"] += 1
|
|
elif s.stream == STREAMED:
|
|
kinds["streamed"] += 1
|
|
elif s.stream == PREFETCH:
|
|
kinds["prefetch"] += 1
|
|
names = {}
|
|
cooked_list = [l.strip() for l in open(os.path.join(game.cooked, "CookedBanks.txt"), encoding="latin-1") if l.strip()]
|
|
from ackit.bnk import fnv1
|
|
for nm in cooked_list:
|
|
names[fnv1(nm)] = nm
|
|
loose_ids = {int(os.path.splitext(os.path.basename(p))[0], 16) for p in glob.glob(os.path.join(game.cooked, "SFX", "*.bnk"))
|
|
if re.fullmatch(r"[0-9A-Fa-f]{8}", os.path.splitext(os.path.basename(p))[0])}
|
|
embedded = [nm for h, nm in names.items() if h in in_pkg]
|
|
loose = sorted(nm for h, nm in names.items() if h in loose_ids and h not in in_pkg)
|
|
nowhere = sorted(nm for h, nm in names.items() if h not in in_pkg and h not in loose_ids)
|
|
claim("docs/formats/banks.md", "lists all %s cooked bank names" % n(len(cooked_list)))
|
|
both = sorted(nm for h, nm in names.items() if h in loose_ids and h in in_pkg)
|
|
claim("docs/formats/banks.md", "%s of them are embedded in packages (%s also as loose `.bnk` files), %d are only loose "
|
|
"`.bnk` files (%s); %s were found nowhere"
|
|
% (n(len(embedded)), ", ".join("`%s`" % x for x in both), len(loose), ", ".join("`%s`" % x for x in loose),
|
|
", ".join("`%s`" % x for x in nowhere[:-1]) + " and `%s`" % nowhere[-1]))
|
|
diff_sets = sum(1 for b, v in media_sets.items() if len(v) > 1)
|
|
claim("docs/formats/banks.md", "Bank blobs in packages: %s for %s bank IDs" % (n(sum(blobs.values())), n(len(blobs))))
|
|
by_name = {names.get(b, ""): c for b, c in blobs.items()}
|
|
claim("docs/formats/banks.md", "`CHR_CatwomanFoleySuit` %d times, `CHR_BatmanFoleyBelt` %d). %d bank IDs"
|
|
% (by_name.get("CHR_CatwomanFoleySuit", 0), by_name.get("CHR_BatmanFoleyBelt", 0), diff_sets))
|
|
claim("docs/formats/banks.md", "| banks embedded in `.upk` packages (in %s of the %s packages) | %s |"
|
|
% (n(sum(1 for f in sc.files if f.lower().endswith(".upk"))), n(len(glob.glob(os.path.join(game.cooked, "*.upk")))),
|
|
n(len(blobs))))
|
|
claim("docs/formats/banks.md", "Source kinds over all bank copies: %s in-memory sources of the bank's own media; %s "
|
|
"in-memory sources with bank id 0" % (n(kinds["memory own"]), n(kinds["memory other"])))
|
|
claim("docs/formats/banks.md", "(loaded by another bank); %s streamed sources; %s **prefetch** sources"
|
|
% (n(kinds["streamed"]), n(kinds["prefetch"])))
|
|
refs = {h for h, t in sc.packs.items() if t}
|
|
claim("docs/formats/banks.md", "the BKPK tables in the packages reference %s WAD entries" % n(len(refs)))
|
|
per_bank = collections.defaultdict(set)
|
|
for m, e in sc.media.items():
|
|
for b in e["banks"]:
|
|
per_bank[b].add(m)
|
|
big = sorted(per_bank.items(), key=lambda kv: -len(kv[1]))[:7]
|
|
nm = [(sc.bank_names.get(b, "%08X" % b), len(v)) for b, v in big]
|
|
claim("docs/formats/banks.md", "Largest: `%s` %d WEMs, " % nm[0] + ", ".join("`%s` %d" % x for x in nm[1:]))
|
|
pkm = {l.media for l in sc.locations if not l.file.startswith("SFX/")}
|
|
pre = collections.Counter()
|
|
for m in pkm:
|
|
for p_ in {sc.bank_names.get(b, "%08X" % b).split("_")[0] for b in sc.media[m]["banks"]}:
|
|
pre[p_] += 1
|
|
top = [k for k, v in pre.most_common() if v >= 30]
|
|
text = ", ".join("`%s` %s" % (k, n(pre[k])) if k != "CHR" else "`CHR`\n(character foley, combat, bosses) %s" % n(pre[k])
|
|
for k in top)
|
|
claim("docs/formats/banks.md", "(a media held by banks of two prefixes counts for both): " + text + ", others < 30")
|
|
# media IDs used by several languages' WAD tables (each at its own position)
|
|
langs_of = collections.defaultdict(set)
|
|
for h, tables in sc.packs.items():
|
|
for L, t in tables.items():
|
|
if L:
|
|
for m in t:
|
|
langs_of[m].add(L)
|
|
claim("docs/formats/banks.md", "%d media IDs are used by several\n languages" % sum(1 for v in langs_of.values() if len(v) > 1))
|
|
|
|
# relations, re-derived from the game by the catalog check
|
|
from ackit.verify import catalogcheck
|
|
rep = catalogcheck.run(game, log=lambda *a: None)
|
|
facts = "\n".join(rep.facts)
|
|
print("verify --catalog:", "catalog OK" if rep.ok else "%d issue(s)" % len(rep.issues))
|
|
|
|
def fact(pattern):
|
|
m = re.search(pattern, facts)
|
|
if not m:
|
|
raise SystemExit("fact not found: " + pattern)
|
|
return [int(x) for x in m.groups()]
|
|
src = fact(r"own language's bank: (\d+) of (\d+)")
|
|
claim("REPORT.md", "own language (%s / %s)" % (n(src[0]), n(src[1])))
|
|
claim("docs/formats/banks.md", "of its own language (%s of %s)" % (n(src[0]), n(src[1])))
|
|
hd = fact(r"whole bank copies: (\d+) of (\d+)")
|
|
claim("REPORT.md", "is the start of its recording (%s / %s)" % (n(hd[0]), n(hd[1])))
|
|
claim("docs/formats/banks.md", "line gives it (%s of %s)" % (n(hd[0]), n(hd[1])) if True else "")
|
|
rec, lns, full = fact(r"(\d+) recordings \(WAD .*?\), (\d+) lines, (\d+) of them")
|
|
claim("docs/formats/banks.md", "(%s recordings)" % n(rec))
|
|
ln15 = fact(r"(\d+) of (\d+) lines within a factor 1.5")
|
|
claim("docs/formats/banks.md", "(%s of %s\n within a factor 1.5" % (n(ln15[0]), n(ln15[1])))
|
|
sp = fact(r"speakers: (\d+) labelled voice lines agree .*?, (\d+) listed as Unknown .*?, (\d+) differ")
|
|
claim("REPORT.md", "%s agree, %d differ" % (n(sp[0]), sp[2]))
|
|
claim("docs/categories.md", "%s labelled\n lines agree" % n(sp[0]))
|
|
claim("docs/categories.md", "%d listed\n as Unknown are named there, %d differ" % (sp[1], sp[2]))
|
|
if not rep.ok:
|
|
claim("(game)", "catalog OK", "ackit verify --catalog")
|
|
|
|
|
|
def _rebuild_language(lang):
|
|
"""Keys of the WEMs of one speech WAD that the builder does not reproduce byte for byte."""
|
|
from ackit.inject import read_target
|
|
from ackit.verify.selfcheck import rebuild_identical
|
|
from ackit.wad import Wad
|
|
game = Game(default_game_dir(settings.load()))
|
|
path = game.wad(lang)
|
|
w = Wad(path)
|
|
from ackit.wem import Wem
|
|
bad, count, extra = [], 0, collections.Counter()
|
|
for s in read_csv(os.path.join(catalog_dir(), lang + ".csv")):
|
|
count += 1
|
|
r = read_target(path, s.target, w)
|
|
tags = {c.tag for c in Wem.parse(r).chunks} - {b"fmt ", b"data"}
|
|
if tags:
|
|
extra["/".join(sorted(t.decode() for t in tags))] += 1
|
|
try:
|
|
ok = rebuild_identical(r)
|
|
except Exception: # noqa: BLE001 - a WEM the builder cannot parse is a difference too
|
|
ok = False
|
|
if not ok:
|
|
bad.append(s.key)
|
|
return count, bad, extra
|
|
|
|
|
|
def wem_claims():
|
|
"""Every WEM of the 5 speech WADs and of the SFX WAD rebuilt from the format rules (docs/formats/wem.md;
|
|
~10 minutes)."""
|
|
import multiprocessing
|
|
from ackit.inject import read_target
|
|
from ackit.wem import Wem
|
|
with multiprocessing.Pool(len(VOICES) + 1) as p:
|
|
res = p.map(_rebuild_language, VOICES + ["SFX"])
|
|
extra = collections.Counter()
|
|
for _, _, e in res:
|
|
extra.update(e)
|
|
sfx_count, sfx_bad, _ = res.pop()
|
|
game = Game(default_game_dir(settings.load()))
|
|
sfx = {s.key: s for s in read_csv(os.path.join(catalog_dir(), "SFX.csv"))}
|
|
kinds = collections.Counter()
|
|
per = [] # samples per seek-table entry of the music tracks
|
|
from ackit.verify.selfcheck import rebuild_identical
|
|
for k in sfx_bad:
|
|
s = sfx[k]
|
|
r = read_target(game.wad("SFX"), s.target)
|
|
m = Wem.parse(r)
|
|
try:
|
|
rebuild_identical(r)
|
|
err = False
|
|
except ValueError: # "inconsistent total_frames": more samples than the packets hold
|
|
err = True
|
|
if any(c.tag == b"smpl" for c in m.chunks):
|
|
kinds["smpl"] += 1
|
|
elif err:
|
|
kinds["frames"] += 1
|
|
elif s.group == "Music" and m.vorb["seek_table_size"]:
|
|
kinds["seek"] += 1
|
|
per.append(m.vorb["total_frames"] / (m.vorb["seek_table_size"] // 4))
|
|
else:
|
|
kinds["other"] += 1
|
|
claim("REPORT.md", "%d SFX originals (music seek tables and sample counts, looping ambience)" % len(sfx_bad))
|
|
claim("docs/formats/wem.md", "In SFX %s of %s are identical; the others are %d music tracks whose seek table follows "
|
|
"another placement (one entry per %s samples), %d music tracks whose sample count exceeds what their "
|
|
"packets hold, and %d 4-channel ambience WEMs with a `smpl` (loop) chunk%s"
|
|
% (n(sfx_count - len(sfx_bad)), n(sfx_count), kinds["seek"], rng([int(x) for x in per]), kinds["frames"],
|
|
kinds["smpl"], "" if not kinds["other"] else " (+%d other)" % kinds["other"]))
|
|
total = sum(c for c, _, _ in res)
|
|
bad = [k for _, b, _ in res for k in b]
|
|
claim("docs/formats/wem.md", "`RIFF` size `WAVE`, then `fmt ` (0x42 bytes), optional %s, `data`" % ", ".join(
|
|
"%s (%s %s)" % (" + ".join("`%s`" % t for t in k.split("/")), n(v), "WEM" if v == 1 else "WEMs")
|
|
for k, v in sorted(extra.items())))
|
|
exceptions = ", ".join("%s `%s`" % tuple(k.split(":", 1)) for k in bad) # e.g. INT `FE9963E8:74`
|
|
claim("docs/formats/wem.md", "Checked on every WEM of the 5 speech WADs: %s of %s are identical; the exception (%s)"
|
|
% (n(total - len(bad)), n(total), exceptions))
|
|
|
|
|
|
def originality_claims(game):
|
|
"""Workspace.unoriginal_files on every file with sounds or subtitles of the (original) game: no false alarm."""
|
|
import tempfile
|
|
from ackit import subtitles
|
|
from ackit.workspace import Workspace
|
|
with tempfile.TemporaryDirectory() as td:
|
|
ws = Workspace(game, store_root=td, archives=[game.voice_archive("INT"), "SFX", "BANK"])
|
|
rels = {l.file for ls in ws.catalog.locations.values() for l in ls}
|
|
rels |= {l.file for ls in subtitles.read_locations(os.path.join(catalog_dir(), subtitles.LOCATIONS)).values()
|
|
for l in ls}
|
|
bad = ws.unoriginal_files([ws.bank_path(r) for r in rels])
|
|
claim("REPORT.md", "checked on all %s files with sounds or subtitles: %s" % (
|
|
n(len(rels)), "no false alarm" if not bad else "%d false alarms" % len(bad)), "Workspace.unoriginal_files")
|
|
|
|
|
|
def test_claims():
|
|
import subprocess
|
|
out = subprocess.run([sys.executable, "-m", "pytest", "--collect-only", "-q", "-p", "no:cacheprovider"],
|
|
cwd=ROOT, capture_output=True, text=True).stdout
|
|
m = re.search(r"(\d+) tests? collected", out)
|
|
claim("REPORT.md", "**Tests**: %s automated tests" % (m.group(1) if m else "?"), "pytest --collect-only")
|
|
|
|
|
|
def norm(t):
|
|
return re.sub(r"\s+", " ", t).strip()
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--game")
|
|
ap.add_argument("--scan", action="store_true", help="also scan every package and run the catalog check")
|
|
ap.add_argument("--wems", action="store_true", help="also rebuild every speech and SFX WEM from the rules")
|
|
a = ap.parse_args()
|
|
cat = catalog_dir()
|
|
catalog_claims(cat)
|
|
index_claims(cat)
|
|
test_claims()
|
|
gd = a.game or default_game_dir(settings.load())
|
|
game = Game(gd) if gd else None
|
|
if game:
|
|
menu_claims(game)
|
|
if a.scan:
|
|
if not game:
|
|
raise SystemExit("--scan needs the game")
|
|
scan_claims(game)
|
|
originality_claims(game)
|
|
if a.wems:
|
|
if not game:
|
|
raise SystemExit("--wems needs the game")
|
|
wem_claims()
|
|
docs = {}
|
|
bad = 0
|
|
for doc, text, why in CLAIMS:
|
|
if doc not in docs:
|
|
path = os.path.join(ROOT, doc)
|
|
docs[doc] = norm(open(path, encoding="utf-8").read()) if os.path.exists(path) else ""
|
|
if norm(text) not in docs[doc]:
|
|
bad += 1
|
|
print("MISMATCH %s: expected \"%s\"%s" % (doc, norm(text), (" (" + why + ")") if why else ""))
|
|
print("%d claims checked in %d documents%s: %s" % (len(CLAIMS), len(docs), "" if a.scan else " (without --scan)",
|
|
"all match" if not bad else "%d mismatch(es)" % bad))
|
|
return 1 if bad else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|