"""Check every figure the documentation states against the data it describes (read-only). python tools/check_docs.py [--game DIR] [--scan] [--wems] The figures are recomputed, never copied: from the shipped catalog (`data/catalog/`, built from the original game files) and, when the game is installed, its menu files; with --scan also from a fresh scan of every package (bank survey: blobs, source kinds, bank names; ~3 minutes) and from `ackit verify --catalog`'s facts (relations re-derived from the game; ~6 minutes more); with --wems from rebuilding every WEM of the speech and SFX WADs from the format rules (~10 minutes). Each claim is the exact text a document must contain (line breaks and repeated spaces ignored). Exit code 1 when a document states anything else. History logs (NOTES.md, findings.md, docs/history/) record what was true on their date and are not checked. """ import argparse import collections import csv import glob import gzip import os import re import sys ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, ROOT) from ackit import banks, settings, subtitles # noqa: E402 from ackit.catalog import read_csv # noqa: E402 from ackit.game import Game, default_game_dir # noqa: E402 from ackit.paths import catalog_dir # noqa: E402 VOICES = ["INT", "FRA", "ITA", "DEU", "ESN"] CLAIMS = [] # (document, text, what it is checked against) def n(x): return "{:,}".format(x) def rng(values, fmt=n): lo, hi = min(values), max(values) return fmt(lo) if lo == hi else "%s-%s" % (fmt(lo), fmt(hi)) def pct(a, b): return int(round(100.0 * a / b)) def claim(doc, text, why=""): CLAIMS.append((doc, text, why)) # --- figures from the shipped catalog -------------------------------------------------------------- def catalog_claims(cat): rows = {a: read_csv(os.path.join(cat, a + ".csv")) for a in VOICES + ["SFX", "BANK"]} bank, sfx = rows["BANK"], rows["SFX"] wad_n = {len(rows[L]) for L in VOICES} assert len(wad_n) == 1, "WADs differ in size: %s" % wad_n W = wad_n.pop() claim("REPORT.md", "(5 languages x %s), the SFX WAD (%s)" % (n(W), n(len(sfx))), "catalog rows") claim("REPORT.md", "the **%s in-memory sounds**" % n(len(bank)), "BANK.csv rows") claim("docs/formats/banks.md", "so there are **%s in-memory sounds**" % n(len(bank)), "BANK.csv rows") # languages of the in-memory sounds neutral = [s for s in bank if not s.language] single = collections.Counter(s.language for s in bank if s.language and ";" not in s.language) multi = [s for s in bank if ";" in s.language] multi_c = collections.Counter(s.language for s in multi) claim("REPORT.md", "%s shared by all languages, ~%s localized speech lines per voice language, %d shared by " "several" % (n(len(neutral)), n(int(round(sum(single.values()) / len(single), -1))), len(multi))) claim("docs/formats/banks.md", "%s sounds are language-neutral, %s belong to one voice language, %d to several" % (n(len(neutral)), rng(single.values()), len(multi))) a4 = multi_c.get("ESN;FRA;INT;ITA", 0) a3 = multi_c.get("FRA;INT;ITA", 0) claim("docs/formats/banks.md", "(%d shared by INT/FRA/ITA/ESN while DEU has its own, %d by INT/FRA/ITA)" % (a4, a3)) claim("docs/formats/banks.md", "everywhere; %d of these are voice lines" % sum(1 for s in neutral if s.line)) claim("docs/formats/banks.md", "stored once for every language (%d sounds)" % sum(1 for s in neutral if s.bank.startswith("Speech"))) claim("docs/formats/banks.md", "lines** (about %s per language" % n(int(round(sum(single.values()) / len(single), -1)))) ch = collections.Counter(s.channels for s in bank) claim("docs/formats/banks.md", "of all %s in-memory sounds %s are mono, %s stereo, %d have 3-6 channels" % (n(len(bank)), n(ch[1]), n(ch[2]), sum(v for k, v in ch.items() if k >= 3))) # copies and heads with gzip.open(os.path.join(cat, "BANK_locations.csv.gz"), "rt", newline="") as f: loc = list(csv.DictReader(f)) m_rows = [r for r in loc if r["kind"] == "m"] p_rows = [r for r in loc if r["kind"] == "p"] keys = {s.key: s for L in VOICES for s in rows[L]} whole = [r for r in p_rows if int(r["size"]) >= keys[r["sound"]].size] pref = [r for r in p_rows if r not in whole] whole_rec = {r["sound"] for r in whole} pref_rec = {r["sound"] for r in pref} pref_lines = {keys[k].line for k in pref_rec} all_lines = {keys[r["sound"]].line for r in p_rows} per_lang_pref = collections.Counter(k.split(":")[0] for k in pref_rec) per_lang_whole = collections.Counter(k.split(":")[0] for k in whole_rec) claim("REPORT.md", "copy (%s copies). %s WAD voice recordings (%d lines) also have bank copies (%d prefetch heads, %d " "whole copies; %s copies)" % (n(len(m_rows)), n(len(pref_rec | whole_rec)), len(all_lines), len(pref_rec), len(whole_rec), n(len(p_rows)))) claim("docs/formats/banks.md", "%d of them are **prefetch heads** of streamed WAD sounds and %d are **whole copies**" % (len({r["media"] for r in pref}), len({r["media"] for r in whole}))) claim("docs/formats/banks.md", "%d WAD sounds have such heads (%s copies)" % (len(pref_rec), n(len(pref)))) claim("docs/formats/banks.md", "%d more WAD recordings (%s per language" % (len(whole_rec), rng(per_lang_whole.values()))) claim("docs/formats/banks.md", "byte-identical to the WAD WEM (%d copies)" % len(whole)) claim("docs/formats/banks.md", "Heads and whole copies together: %d recordings, %s copies" % (len(pref_rec | whole_rec), n(len(p_rows)))) claim("docs/formats/banks.md", "the %d lines with a prefetch head (%d recordings over the 5 languages" % (len(pref_lines), len(pref_rec))) claim("docs/formats/banks.md", "A WAD recording that a bank also stores whole (%d)" % len(whole_rec)) claim("docs/decisions.md", "%d voice recordings (%d lines, %s recordings per language) have their first bytes" % (len(pref_rec), len(pref_lines), rng(per_lang_pref.values(), str))) claim("docs/decisions.md", "and %d more are stored whole in banks" % len(whole_rec)) places = len(loc) media_all = {r["media"] for r in loc} loose = {r["media"] for r in loc if r["file"].startswith("SFX/")} pk = {r["media"] for r in loc if not r["file"].startswith("SFX/")} claim("docs/formats/banks.md", "The two sets share %d media IDs: %s unique media IDs in %s places" % (len(loose & pk), n(len(media_all)), n(places))) claim("docs/formats/banks.md", "+ `TST_Prep_Media`) | %d | %s |" % (len({r["file"] for r in loc if r["file"].startswith("SFX/")}), n(len(loose)))) claim("docs/formats/banks.md", "packages) | 3,340 | %s |" % n(len(pk))) claim("REPORT.md", "in %d loose `.bnk` files and %d `.upk` packages" % (len({r["file"] for r in loc if r["file"].startswith("SFX/")}), len({r["file"] for r in loc if r["file"].endswith(".upk")}))) assert len(media_all) == len(bank) + len(pref_rec | whole_rec) # lines lines_all = {s.line for L in VOICES for s in rows[L] if s.line} | {s.line for s in bank if s.line} wad_lines = {L: {s.line for s in rows[L] if s.line} for L in VOICES} assert len({len(v) for v in wad_lines.values()}) == 1 WL = len(wad_lines["INT"]) copies = {L: sum(1 for s in rows[L] if s.line) - len(wad_lines[L]) for L in VOICES} recs = sum(len(wad_lines[L]) for L in VOICES) + len({(s.target) for s in bank if s.line}) for doc, text in [("REPORT.md", "%s lines (WAD + in-memory) with exactly one recording in each of the 5 voice languages"), ("docs/decisions.md", "recording per language (%s lines, all 5 languages)"), ("docs/formats/banks.md", "recordings that are sources of the same Sound objects: %s lines")]: claim(doc, text % n(len(lines_all)), "distinct line ids") claim("docs/formats/banks.md", "in each of the 5 voice languages (%s recordings)" % n(recs)) claim("REPORT.md", "recordings stored twice (%s per language) are written together" % rng(copies.values(), str)) claim("docs/formats/banks.md", "- %s recordings per language are stored twice" % rng(copies.values(), str).capitalize()) claim("docs/categories.md", "Coverage of the %s WAD voice lines" % n(WL)) # recordings the game shares between languages (the same bytes as the INT recording of the line) intl = {s.line: s.sha1 for s in rows["INT"] if s.line} same = {L: [s for s in {x.line: x for x in rows[L] if x.line}.values() if intl.get(s.line) == s.sha1] for L in ("FRA", "ITA", "DEU", "ESN")} ita = collections.Counter(s.category for s in same["ITA"]) shared = {s.target: s.language for s in bank if ";" in s.language} claim("docs/formats/banks.md", "in each voice WAD %s lines are byte-identical to the INT recording (ITA %d: efforts " "and emotes %d, cinematics %d, boss fights %d, story %d, radio/TV %d, character lines %d), %d in-memory voice " "lines are stored once for every language (efforts, radio chatter, crowds) and %d DLC intro lines" % (rng([len(v) for v in same.values()]), len(same["ITA"]), ita["Efforts & emotes"], ita["Cinematics"], ita["Boss fights"], ita["Story missions"], ita["Radio & TV"], ita["Character lines"], sum(1 for s in bank if s.line and not s.language), len(shared))) claim("docs/decisions.md", "of the %s WAD voice lines" % n(WL)) # positions differ between languages it = {s.target: s.line for s in rows["INT"] if ":" in s.target} diff = [] for L in ["FRA", "ITA", "DEU", "ESN"]: other = {s.target: s.line for s in rows[L] if ":" in s.target} diff.append(100.0 * sum(1 for t in it if other.get(t) != it[t]) / len(it)) span = "%.1f-%.1f %%" % (min(diff), max(diff)) claim("docs/formats/wad.md", "(%s of the WEMs in multi-variation entries, compared with INT" % span) claim("docs/decisions.md", "%s of the WEMs in multi-variation entries are at another position than in INT" % span) pos = {} for L in VOICES: s = next(s for s in rows[L] if s.name == "Speech_TAPE_JOKER_1_04") pos[L] = int(s.target.split(":")[1]) claim("docs/formats/wad.md", "`Speech_TAPE_JOKER_1_04` is variation %d in INT and ITA, %d in FRA, %d in DEU, %d in ESN" % (pos["INT"], pos["FRA"], pos["DEU"], pos["ESN"]) if pos["INT"] == pos["ITA"] else "(INT != ITA)") ents = collections.Counter(s.target.split(":")[0] for s in rows["INT"]) claim("docs/formats/wad.md", "(every language: %s WEMs in %s entries" % (n(W), n(len(ents)))) radio = [s for s in rows["INT"] if s.aliases and any(a.startswith("CHR_RadioScanner#") for a in s.names()) or s.name.startswith("CHR_RadioScanner#")] claim("docs/formats/wad.md", "e.g. `CHR_RadioScanner` holds %d" % len({s.target for s in radio})) # SFX WAD packs = collections.Counter(s.target.split("/")[0] for s in sfx if "/" in s.target) grouped = [s for s in sfx if "/" not in s.target] claim("docs/formats/wad.md", "**SFX WAD**: %d grouped ambience/effect entries (%d WEMs) plus %d **stream packs**" % (len({s.target.split(":")[0] for s in grouped}), len(grouped), len(packs))) claim("docs/formats/wad.md", "`EDF036D6` (Music, %d WEMs" % packs["EDF036D6"]) claim("docs/formats/wad.md", "`E8F980BE` (Music_PDLC, %d), `57572DAC` (Global, %d), `FCA4F778` (%d)" % (packs["E8F980BE"], packs["57572DAC"], packs["FCA4F778"])) # categories cats = [collections.Counter(s.category for s in rows[L]) for L in VOICES] assert all(c == cats[0] for c in cats), "category counts differ between languages" order = ["Story missions", "Predator chatter", "Character lines", "Open-world chatter", "Side missions", "Combat barks", "Efforts & emotes", "Interview tapes", "Riddler", "Radio & TV", "Gameplay & hints", "DLC", "Boss fights", "Cinematics", "Reactions", "Interrogations"] g = collections.Counter(s.group for s in sfx) bc = collections.Counter(s.category for s in bank).most_common(8) claim("docs/categories.md", "Counts per voice language (identical in all 5): " + ", ".join( "%s %d" % (c, cats[0][c]) for c in order) + ". SFX archive: " + ", ".join( "%s %d" % (k, g[k]) for k in ["Music", "Ambience", "SFX", "Unknown"]) + ". BANK (all languages): " + ", ".join( "%s %d" % kv for kv in bc) + ", ...") # characters (by line) first = {} for s in rows["INT"]: first.setdefault(s.line, s) wc = sum(1 for s in first.values() if s.character) inf = sum(1 for s in first.values() if not s.character and s.characters_inferred) nei = [s for s in first.values() if not s.character and not s.characters_inferred] top = collections.Counter(s.category for s in nei).most_common(4) assert all(sum(1 for s in rows[L] if s.character) == sum(1 for s in rows["INT"] if s.character) for L in VOICES) claim("docs/categories.md", "(every voice language, the same lines): %s with a verified speaker, %s more with inferred " "characters, %s with neither (%s, ...)" % (n(wc), n(inf), n(len(nei)), ", ".join( "%s %d" % (c.lower().replace("radio & tv", "radio & TV"), k) for c, k in top))) claim("docs/categories.md", "A hint for the ~%s lines without a verified speaker" % n(int(round(WL - wc, -2)))) claim("REPORT.md", "speaker labels (%s in every language)" % n(wc)) # subtitles langs = sorted(f[4:7] for f in os.listdir(cat) if f.startswith("SUB_") and f.endswith(".csv")) texts = {L: subtitles.read_texts(os.path.join(cat, "SUB_%s.csv" % L), L) for L in langs} total = sum(len(t) for t in texts.values()) linked = sum(1 for t in texts.values() for s in t if s.sound) i_tot, i_l = len(texts["INT"]), sum(1 for s in texts["INT"] if s.sound) for doc in ("REPORT.md", "docs/formats/subtitles.md"): claim(doc, "%s of %s strings in %d languages (INT %s" % (n(linked), n(total), len(langs), n(i_l)) if doc == "REPORT.md" else "Linked: %s of %s strings in the %d languages (INT %s / %s)" % (n(linked), n(total), len(langs), n(i_l), n(i_tot))) claim("REPORT.md", "(INT %s of %s)" % (n(i_l), n(i_tot))) unl = {L: {s.speech for s in t if not s.sound} for L, t in texts.items()} unl_n = {len(v) for v in unl.values()} assert len(unl_n) == 1, unl_n un_strings = [sum(1 for s in t if not s.sound) for t in texts.values()] claim("REPORT.md", "%d speech names (%s strings per language, INT %d" % (unl_n.pop(), rng(un_strings), i_tot - i_l)) claim("docs/formats/subtitles.md", "(%d speech names, %s strings per language" % (len(unl["INT"]), rng(un_strings))) two = {s.speech for s in texts["INT"] if ";" in s.sound} claim("docs/formats/subtitles.md", "(%d subtitled events" % len(two)) one = [] for L, t in texts.items(): c = collections.Counter(s.speech for s in t) one.append(100.0 * sum(1 for v in c.values() if v == 1) / len(c)) claim("docs/formats/subtitles.md", "%s subtitle strings per language: %d-%d %% of the subtitles have one line (JPN: all)" % (rng([len(t) for t in texts.values()]), int(min(one)), int(round(max(x for x in one if x < 100))))) enc = {L: collections.Counter(s.enc for s in t) for L, t in texts.items()} pol = pct(enc["POL"]["w"], len(texts["POL"])) west = [enc[L]["w"] for L in langs if L not in ("RUS", "JPN", "KOR", "POL")] claim("docs/formats/subtitles.md", "%d %% of POL are UTF-16; the other languages have %s UTF-16 strings each" % (pol, rng(west))) utf8 = [] # cp1252 strings whose bytes are UTF-8 text (the game's data) for L, t in sorted(texts.items()): for s in t: if s.enc == "a": u = subtitles._encode_cp1252(s.text).decode("utf-8", "replace") multi = sum(1 for c in u if ord(c) > 0x7F and c != "\ufffd") # valid multi-byte characters if multi and u.count("\ufffd") <= multi // 4: # (one corrupt byte tolerated) utf8.append((L, s.speech)) claim("docs/formats/subtitles.md", "%d cp1252 strings hold UTF-8 bytes instead (the same line in %s: `%s`" % (len(utf8), ", ".join(L for L, _ in utf8), "`, `".join(sorted({sp for _, sp in utf8})))) with gzip.open(os.path.join(cat, subtitles.LOCATIONS), "rt", newline="") as f: sublocs = sum(1 for _ in csv.DictReader(f)) claim("REPORT.md", "Edited in place in every copy (%s copies)" % n(sublocs)) claim("docs/formats/subtitles.md", "%s stored copies" % n(sublocs)) claim("docs/user-guide.md", "About %d subtitles per language have no sound" % int(round(sum(un_strings) / len(un_strings), -1))) # subtitle coverage of the INT voice lines lk = {k for s in texts["INT"] for k in s.sound.split(";") if k} cov, tot = collections.Counter(), collections.Counter() for s in rows["INT"] + [b for b in bank if b.line and "INT" in b.language.split(";")]: if s.line: tot[s.category] += 1 cov[s.category] += s.key in lk p = {c: pct(cov[c], tot[c]) for c in tot} claim("REPORT.md", "few for combat barks (%d %%), cinematics and boss fights (%d %%;" % (p["Combat barks"], p["Cinematics"]) if p["Cinematics"] == p["Boss fights"] else "(cinematics != boss)") hi = [p[c] for c in ("Story missions", "Predator chatter", "Character lines", "Open-world chatter")] claim("REPORT.md", "%d-%d %% for story, predator, character and open-world lines" % (min(hi), max(hi))) claim("REPORT.md", "(INT voice lines with a subtitle): none for efforts/grunts" if p["Efforts & emotes"] == 0 else "!") # what the editor loads (Italian on the test PC: the smoke test) ita = len(rows["ITA"]) + len(sfx) + sum(1 for s in bank if s.in_language("ITA")) claim("REPORT.md", "%s sounds loaded, encode + inject, %s subtitles" % (n(ita), n(len(texts["ITA"])))) claim("README.md", "~%s sounds of one language" % n(int(round(ita, -3)))) return rows # --- figures recorded by the catalog build (index.json: they need the full scan) --------------------- def index_claims(cat): import json ix = json.load(open(os.path.join(cat, "index.json"), encoding="utf-8")) b, ln, sub = ix["BANK"], ix["LINES"], ix["SUB"] claim("REPORT.md", "Names: %s BANK sounds are named after the Wwise event that plays them (`Play_...`, `Speech_...`), " "the other %s after their bank" % (n(b["named_by_event"]), n(b["named_by_bank"]))) claim("REPORT.md", "**Unnamed sounds**: %s BANK sounds are reached by no event" % n(b["named_by_bank"])) claim("docs/decisions.md", "sounds (%s of %s, voice lines included); the other %s are numbered per bank" % (n(b["named_by_event"]), n(b["sounds"]), n(b["named_by_bank"]))) claim("docs/decisions.md", "the most specific speech event that plays it (%s of the %s WAD voice lines)" % (n(ln["wad_lines_named_by_event"]), n(ln["wad_lines"]))) claim("docs/decisions.md", "the other %s keep the name of their English (INT) recording's position" % n(ln["wad_lines_named_after_int_recording"])) claim("REPORT.md", "- %s voice lines are played by no event whose name the game stores" % n(ln["wad_lines_named_after_int_recording"])) claim("docs/user-guide.md", "About %d %% of the lines are named by no game event" % pct(ln["wad_lines_named_after_int_recording"], ln["wad_lines"])) claim("docs/formats/subtitles.md", "%s dialogue objects in the packages for %s speech lines" % (n(sub["dialogue_objects"]), n(sub["dialogue_lines"]))) claim("docs/formats/subtitles.md", "The sizes add up exactly in all %s objects" % n(sub["dialogue_objects"])) claim("docs/formats/subtitles.md", "plus %d scripted subtitles" % sub["scripted_objects"]) claim("docs/formats/subtitles.md", "%d more subtitles (radio and encryption-card lines) are `RSubtitle` objects" % sub["scripted_objects"]) claim("REPORT.md", "(%s speech lines x %d languages) and" % (n(sub["dialogue_lines"]), len(sub["lines"]))) claim("REPORT.md", "in %d scripted `RSubtitle` objects" % sub["scripted_objects"]) claim("docs/formats/banks.md", "(in %s of the 2,650 packages)" % n(b["files"] - 67)) # 67 loose .bnk with banks # --- the game's own files ---------------------------------------------------------------------------- def menu_claims(game): from ackit import loctext root = os.path.join(game.root, "BmGame", "Localization") counts = {L: len(loctext.load(root, L)) for L in loctext.languages(root)} files = {L: len(loctext.files(root, L)) for L in loctext.languages(root)} approx = n(int(round(sum(counts.values()) / len(counts), -3))) claim("docs/formats/text.md", "~%d files and ~%s strings each" % (collections.Counter(files.values()).most_common(1)[0][0], approx)) claim("REPORT.md", "~%s strings per language" % approx) claim("docs/user-guide.md", "of the selected language (about\n%s)" % approx) claim("REPORT.md", "%s menu texts loaded" % n(counts["ITA"])) def scan_claims(game): """Bank survey from a fresh scan of every package, and the relation facts of `ackit verify --catalog`.""" from ackit.bnk import find_banks, Bank, BankError, IN_MEMORY, STREAMED, PREFETCH blobs = collections.Counter() media_sets = collections.defaultdict(set) kinds = collections.Counter() in_pkg = set() def visit(rel, pkg, start, data): for bk in find_banks(data): blobs[bk.id] += 1 media_sets[bk.id].add(frozenset(m.id for m in bk.media())) in_pkg.add(bk.id) own = {m.id for m in bk.media()} for s in bk.sources(): if s.stream == IN_MEMORY: kinds["memory own" if s.media in own else "memory other"] += 1 elif s.stream == STREAMED: kinds["streamed"] += 1 elif s.stream == PREFETCH: kinds["prefetch"] += 1 sc = banks.scan(game.cooked, visit=visit) for path in glob.glob(os.path.join(game.cooked, "SFX", "*.bnk")): # loose banks: source kinds too try: bk = Bank(open(path, "rb").read(), 0) except BankError: continue own = {m.id for m in bk.media()} for s in bk.sources(): if s.stream == IN_MEMORY: kinds["memory own" if s.media in own else "memory other"] += 1 elif s.stream == STREAMED: kinds["streamed"] += 1 elif s.stream == PREFETCH: kinds["prefetch"] += 1 names = {} cooked_list = [l.strip() for l in open(os.path.join(game.cooked, "CookedBanks.txt"), encoding="latin-1") if l.strip()] from ackit.bnk import fnv1 for nm in cooked_list: names[fnv1(nm)] = nm loose_ids = {int(os.path.splitext(os.path.basename(p))[0], 16) for p in glob.glob(os.path.join(game.cooked, "SFX", "*.bnk")) if re.fullmatch(r"[0-9A-Fa-f]{8}", os.path.splitext(os.path.basename(p))[0])} embedded = [nm for h, nm in names.items() if h in in_pkg] loose = sorted(nm for h, nm in names.items() if h in loose_ids and h not in in_pkg) nowhere = sorted(nm for h, nm in names.items() if h not in in_pkg and h not in loose_ids) claim("docs/formats/banks.md", "lists all %s cooked bank names" % n(len(cooked_list))) both = sorted(nm for h, nm in names.items() if h in loose_ids and h in in_pkg) claim("docs/formats/banks.md", "%s of them are embedded in packages (%s also as loose `.bnk` files), %d are only loose " "`.bnk` files (%s); %s were found nowhere" % (n(len(embedded)), ", ".join("`%s`" % x for x in both), len(loose), ", ".join("`%s`" % x for x in loose), ", ".join("`%s`" % x for x in nowhere[:-1]) + " and `%s`" % nowhere[-1])) diff_sets = sum(1 for b, v in media_sets.items() if len(v) > 1) claim("docs/formats/banks.md", "Bank blobs in packages: %s for %s bank IDs" % (n(sum(blobs.values())), n(len(blobs)))) by_name = {names.get(b, ""): c for b, c in blobs.items()} claim("docs/formats/banks.md", "`CHR_CatwomanFoleySuit` %d times, `CHR_BatmanFoleyBelt` %d). %d bank IDs" % (by_name.get("CHR_CatwomanFoleySuit", 0), by_name.get("CHR_BatmanFoleyBelt", 0), diff_sets)) claim("docs/formats/banks.md", "| banks embedded in `.upk` packages (in %s of the %s packages) | %s |" % (n(sum(1 for f in sc.files if f.lower().endswith(".upk"))), n(len(glob.glob(os.path.join(game.cooked, "*.upk")))), n(len(blobs)))) claim("docs/formats/banks.md", "Source kinds over all bank copies: %s in-memory sources of the bank's own media; %s " "in-memory sources with bank id 0" % (n(kinds["memory own"]), n(kinds["memory other"]))) claim("docs/formats/banks.md", "(loaded by another bank); %s streamed sources; %s **prefetch** sources" % (n(kinds["streamed"]), n(kinds["prefetch"]))) refs = {h for h, t in sc.packs.items() if t} claim("docs/formats/banks.md", "the BKPK tables in the packages reference %s WAD entries" % n(len(refs))) per_bank = collections.defaultdict(set) for m, e in sc.media.items(): for b in e["banks"]: per_bank[b].add(m) big = sorted(per_bank.items(), key=lambda kv: -len(kv[1]))[:7] nm = [(sc.bank_names.get(b, "%08X" % b), len(v)) for b, v in big] claim("docs/formats/banks.md", "Largest: `%s` %d WEMs, " % nm[0] + ", ".join("`%s` %d" % x for x in nm[1:])) pkm = {l.media for l in sc.locations if not l.file.startswith("SFX/")} pre = collections.Counter() for m in pkm: for p_ in {sc.bank_names.get(b, "%08X" % b).split("_")[0] for b in sc.media[m]["banks"]}: pre[p_] += 1 top = [k for k, v in pre.most_common() if v >= 30] text = ", ".join("`%s` %s" % (k, n(pre[k])) if k != "CHR" else "`CHR`\n(character foley, combat, bosses) %s" % n(pre[k]) for k in top) claim("docs/formats/banks.md", "(a media held by banks of two prefixes counts for both): " + text + ", others < 30") # media IDs used by several languages' WAD tables (each at its own position) langs_of = collections.defaultdict(set) for h, tables in sc.packs.items(): for L, t in tables.items(): if L: for m in t: langs_of[m].add(L) claim("docs/formats/banks.md", "%d media IDs are used by several\n languages" % sum(1 for v in langs_of.values() if len(v) > 1)) # relations, re-derived from the game by the catalog check from ackit.verify import catalogcheck rep = catalogcheck.run(game, log=lambda *a: None) facts = "\n".join(rep.facts) print("verify --catalog:", "catalog OK" if rep.ok else "%d issue(s)" % len(rep.issues)) def fact(pattern): m = re.search(pattern, facts) if not m: raise SystemExit("fact not found: " + pattern) return [int(x) for x in m.groups()] src = fact(r"own language's bank: (\d+) of (\d+)") claim("REPORT.md", "own language (%s / %s)" % (n(src[0]), n(src[1]))) claim("docs/formats/banks.md", "of its own language (%s of %s)" % (n(src[0]), n(src[1]))) hd = fact(r"whole bank copies: (\d+) of (\d+)") claim("REPORT.md", "is the start of its recording (%s / %s)" % (n(hd[0]), n(hd[1]))) claim("docs/formats/banks.md", "line gives it (%s of %s)" % (n(hd[0]), n(hd[1])) if True else "") rec, lns, full = fact(r"(\d+) recordings \(WAD .*?\), (\d+) lines, (\d+) of them") claim("docs/formats/banks.md", "(%s recordings)" % n(rec)) ln15 = fact(r"(\d+) of (\d+) lines within a factor 1.5") claim("docs/formats/banks.md", "(%s of %s\n within a factor 1.5" % (n(ln15[0]), n(ln15[1]))) sp = fact(r"speakers: (\d+) labelled voice lines agree .*?, (\d+) listed as Unknown .*?, (\d+) differ") claim("REPORT.md", "%s agree, %d differ" % (n(sp[0]), sp[2])) claim("docs/categories.md", "%s labelled\n lines agree" % n(sp[0])) claim("docs/categories.md", "%d listed\n as Unknown are named there, %d differ" % (sp[1], sp[2])) if not rep.ok: claim("(game)", "catalog OK", "ackit verify --catalog") def _rebuild_language(lang): """Keys of the WEMs of one speech WAD that the builder does not reproduce byte for byte.""" from ackit.inject import read_target from ackit.verify.selfcheck import rebuild_identical from ackit.wad import Wad game = Game(default_game_dir(settings.load())) path = game.wad(lang) w = Wad(path) from ackit.wem import Wem bad, count, extra = [], 0, collections.Counter() for s in read_csv(os.path.join(catalog_dir(), lang + ".csv")): count += 1 r = read_target(path, s.target, w) tags = {c.tag for c in Wem.parse(r).chunks} - {b"fmt ", b"data"} if tags: extra["/".join(sorted(t.decode() for t in tags))] += 1 try: ok = rebuild_identical(r) except Exception: # noqa: BLE001 - a WEM the builder cannot parse is a difference too ok = False if not ok: bad.append(s.key) return count, bad, extra def wem_claims(): """Every WEM of the 5 speech WADs and of the SFX WAD rebuilt from the format rules (docs/formats/wem.md; ~10 minutes).""" import multiprocessing from ackit.inject import read_target from ackit.wem import Wem with multiprocessing.Pool(len(VOICES) + 1) as p: res = p.map(_rebuild_language, VOICES + ["SFX"]) extra = collections.Counter() for _, _, e in res: extra.update(e) sfx_count, sfx_bad, _ = res.pop() game = Game(default_game_dir(settings.load())) sfx = {s.key: s for s in read_csv(os.path.join(catalog_dir(), "SFX.csv"))} kinds = collections.Counter() per = [] # samples per seek-table entry of the music tracks from ackit.verify.selfcheck import rebuild_identical for k in sfx_bad: s = sfx[k] r = read_target(game.wad("SFX"), s.target) m = Wem.parse(r) try: rebuild_identical(r) err = False except ValueError: # "inconsistent total_frames": more samples than the packets hold err = True if any(c.tag == b"smpl" for c in m.chunks): kinds["smpl"] += 1 elif err: kinds["frames"] += 1 elif s.group == "Music" and m.vorb["seek_table_size"]: kinds["seek"] += 1 per.append(m.vorb["total_frames"] / (m.vorb["seek_table_size"] // 4)) else: kinds["other"] += 1 claim("REPORT.md", "%d SFX originals (music seek tables and sample counts, looping ambience)" % len(sfx_bad)) claim("docs/formats/wem.md", "In SFX %s of %s are identical; the others are %d music tracks whose seek table follows " "another placement (one entry per %s samples), %d music tracks whose sample count exceeds what their " "packets hold, and %d 4-channel ambience WEMs with a `smpl` (loop) chunk%s" % (n(sfx_count - len(sfx_bad)), n(sfx_count), kinds["seek"], rng([int(x) for x in per]), kinds["frames"], kinds["smpl"], "" if not kinds["other"] else " (+%d other)" % kinds["other"])) total = sum(c for c, _, _ in res) bad = [k for _, b, _ in res for k in b] claim("docs/formats/wem.md", "`RIFF` size `WAVE`, then `fmt ` (0x42 bytes), optional %s, `data`" % ", ".join( "%s (%s %s)" % (" + ".join("`%s`" % t for t in k.split("/")), n(v), "WEM" if v == 1 else "WEMs") for k, v in sorted(extra.items()))) exceptions = ", ".join("%s `%s`" % tuple(k.split(":", 1)) for k in bad) # e.g. INT `FE9963E8:74` claim("docs/formats/wem.md", "Checked on every WEM of the 5 speech WADs: %s of %s are identical; the exception (%s)" % (n(total - len(bad)), n(total), exceptions)) def originality_claims(game): """Workspace.unoriginal_files on every file with sounds or subtitles of the (original) game: no false alarm.""" import tempfile from ackit import subtitles from ackit.workspace import Workspace with tempfile.TemporaryDirectory() as td: ws = Workspace(game, store_root=td, archives=[game.voice_archive("INT"), "SFX", "BANK"]) rels = {l.file for ls in ws.catalog.locations.values() for l in ls} rels |= {l.file for ls in subtitles.read_locations(os.path.join(catalog_dir(), subtitles.LOCATIONS)).values() for l in ls} bad = ws.unoriginal_files([ws.bank_path(r) for r in rels]) claim("REPORT.md", "checked on all %s files with sounds or subtitles: %s" % ( n(len(rels)), "no false alarm" if not bad else "%d false alarms" % len(bad)), "Workspace.unoriginal_files") def test_claims(): import subprocess out = subprocess.run([sys.executable, "-m", "pytest", "--collect-only", "-q", "-p", "no:cacheprovider"], cwd=ROOT, capture_output=True, text=True).stdout m = re.search(r"(\d+) tests? collected", out) claim("REPORT.md", "**Tests**: %s automated tests" % (m.group(1) if m else "?"), "pytest --collect-only") def norm(t): return re.sub(r"\s+", " ", t).strip() def main(): ap = argparse.ArgumentParser() ap.add_argument("--game") ap.add_argument("--scan", action="store_true", help="also scan every package and run the catalog check") ap.add_argument("--wems", action="store_true", help="also rebuild every speech and SFX WEM from the rules") a = ap.parse_args() cat = catalog_dir() catalog_claims(cat) index_claims(cat) test_claims() gd = a.game or default_game_dir(settings.load()) game = Game(gd) if gd else None if game: menu_claims(game) if a.scan: if not game: raise SystemExit("--scan needs the game") scan_claims(game) originality_claims(game) if a.wems: if not game: raise SystemExit("--wems needs the game") wem_claims() docs = {} bad = 0 for doc, text, why in CLAIMS: if doc not in docs: path = os.path.join(ROOT, doc) docs[doc] = norm(open(path, encoding="utf-8").read()) if os.path.exists(path) else "" if norm(text) not in docs[doc]: bad += 1 print("MISMATCH %s: expected \"%s\"%s" % (doc, norm(text), (" (" + why + ")") if why else "")) print("%d claims checked in %d documents%s: %s" % (len(CLAIMS), len(docs), "" if a.scan else " (without --scan)", "all match" if not bad else "%d mismatch(es)" % bad)) return 1 if bad else 0 if __name__ == "__main__": sys.exit(main())