Version 3.2.1: whole bank copies of voice lines, unique names, docs figures checked by tools/check_docs.py
This commit is contained in:
@@ -128,7 +128,7 @@ def main():
|
||||
st = lines.stats
|
||||
print("packages scanned (%.0fs); %d recordings in %d lines, %d of them in every language (%s); "
|
||||
"%d recordings without a line; %d relation issues"
|
||||
% (time.time() - t0, st["in_a_line"], st["lines"], st["lines_in_every_language"], "/".join(lines.langs),
|
||||
% (time.time() - t0, st["recordings"], st["lines"], st["lines_in_every_language"], "/".join(lines.langs),
|
||||
len(st["without_line"]), len(lines.issues)))
|
||||
for k, d in lines.issues[:10]:
|
||||
print(" issue:", k, d)
|
||||
@@ -187,14 +187,17 @@ def main():
|
||||
banks.write_locations(os.path.join(out_dir, banks.LOCATIONS), locs)
|
||||
g = collections.Counter(s.group for s in sounds)
|
||||
heads = [l for l in locs if l.kind == banks.PREFETCH_HEAD]
|
||||
played = {m for ms in banks.event_media(sc).values() for m in ms} # reached by a named event
|
||||
by_event = sum(1 for s in sounds if (s.line not in lines.unnamed if s.line else int(s.target, 16) in played))
|
||||
index["archives"][banks.ARCHIVE] = {
|
||||
"fingerprint": banks.game_fingerprint(game.cooked), "sounds": len(sounds), "groups": dict(g),
|
||||
"named_by_event": by_event, "named_by_bank": len(sounds) - by_event,
|
||||
"copies": sum(1 for l in locs if l.kind == banks.MEMORY), "files": len(sc.files),
|
||||
"named_by_event": sum(1 for s in sounds if not s.name.lower().startswith(s.bank.lower() + "_")),
|
||||
"prefetch_heads": {"lines": len({l.media for l in heads}), "copies": len(heads),
|
||||
# bank copies of WAD recordings: prefetch heads and whole in-memory copies (catalog._whole_recordings)
|
||||
"prefetch_heads": {"recordings": len({l.media for l in heads}), "copies": len(heads),
|
||||
"unmapped": len({l.media for l in heads if not l.sound})},
|
||||
"issues": ["%s: %s" % i for i in sc.issues]}
|
||||
print("%s.csv: %d sounds %s, %d copies in %d files, %d prefetch heads"
|
||||
print("%s.csv: %d sounds %s, %d copies in %d files, %d heads of WAD recordings"
|
||||
% (banks.ARCHIVE, len(sounds), dict(g), len(locs) - len(heads), len(sc.files), len(heads)))
|
||||
|
||||
# subtitles (dialogue objects in the packages), one CSV per language + every stored copy
|
||||
@@ -210,9 +213,12 @@ def main():
|
||||
"lines": {lang: len(rows) for lang, rows in sorted(texts.items())},
|
||||
"speech_lines": len({s.speech for s in any_lang}),
|
||||
"linked": {lang: sum(1 for s in rows if s.sound) for lang, rows in sorted(texts.items())},
|
||||
"copies": len(sublocs), "issues": ["%s: %s" % i for i in sub_issues]}
|
||||
"copies": len(sublocs), **sc.subtitle_objects, "issues": ["%s: %s" % i for i in sub_issues]}
|
||||
wad_lines = {lid for lid, per in lines.members.items() if any(not k.startswith("BANK:") for k in per.values())}
|
||||
index["archives"]["LINES"] = {
|
||||
"languages": lines.langs, "recordings": st["recordings"], "lines": st["lines"],
|
||||
"wad_lines": len(wad_lines), "wad_lines_named_by_event": len(wad_lines - lines.unnamed),
|
||||
"wad_lines_named_after_int_recording": len(wad_lines & lines.unnamed),
|
||||
"lines_in_every_language": st["lines_in_every_language"],
|
||||
"lines_missing_a_language": st["lines_missing_a_language"], "without_line": st["without_line"],
|
||||
"issues": ["%s: %s" % i for i in lines.issues]}
|
||||
|
||||
545
tools/check_docs.py
Normal file
545
tools/check_docs.py
Normal file
@@ -0,0 +1,545 @@
|
||||
"""Check every figure the documentation states against the data it describes (read-only).
|
||||
|
||||
python tools/check_docs.py [--game DIR] [--scan] [--wems]
|
||||
|
||||
The figures are recomputed, never copied: from the shipped catalog (`data/catalog/`, built from the original
|
||||
game files) and, when the game is installed, its menu files; with --scan also from a fresh scan of every
|
||||
package (bank survey: blobs, source kinds, bank names; ~3 minutes) and from `ackit verify --catalog`'s facts
|
||||
(relations re-derived from the game; ~6 minutes more); with --wems from rebuilding every WEM of the speech and
|
||||
SFX WADs from the format rules (~10 minutes). Each claim is the exact text a document must contain
|
||||
(line breaks and repeated spaces ignored). Exit code 1 when a document states anything else.
|
||||
History logs (NOTES.md, findings.md, docs/history/) record what was true on their date and are not checked.
|
||||
"""
|
||||
import argparse
|
||||
import collections
|
||||
import csv
|
||||
import glob
|
||||
import gzip
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, ROOT)
|
||||
|
||||
from ackit import banks, settings, subtitles # noqa: E402
|
||||
from ackit.catalog import read_csv # noqa: E402
|
||||
from ackit.game import Game, default_game_dir # noqa: E402
|
||||
from ackit.paths import catalog_dir # noqa: E402
|
||||
|
||||
VOICES = ["INT", "FRA", "ITA", "DEU", "ESN"]
|
||||
CLAIMS = [] # (document, text, what it is checked against)
|
||||
|
||||
|
||||
def n(x):
|
||||
return "{:,}".format(x)
|
||||
|
||||
|
||||
def rng(values, fmt=n):
|
||||
lo, hi = min(values), max(values)
|
||||
return fmt(lo) if lo == hi else "%s-%s" % (fmt(lo), fmt(hi))
|
||||
|
||||
|
||||
def pct(a, b):
|
||||
return int(round(100.0 * a / b))
|
||||
|
||||
|
||||
def claim(doc, text, why=""):
|
||||
CLAIMS.append((doc, text, why))
|
||||
|
||||
|
||||
# --- figures from the shipped catalog --------------------------------------------------------------
|
||||
def catalog_claims(cat):
|
||||
rows = {a: read_csv(os.path.join(cat, a + ".csv")) for a in VOICES + ["SFX", "BANK"]}
|
||||
bank, sfx = rows["BANK"], rows["SFX"]
|
||||
wad_n = {len(rows[L]) for L in VOICES}
|
||||
assert len(wad_n) == 1, "WADs differ in size: %s" % wad_n
|
||||
W = wad_n.pop()
|
||||
claim("REPORT.md", "(5 languages x %s), the SFX WAD (%s)" % (n(W), n(len(sfx))), "catalog rows")
|
||||
claim("REPORT.md", "the **%s in-memory sounds**" % n(len(bank)), "BANK.csv rows")
|
||||
claim("docs/formats/banks.md", "so there are **%s in-memory sounds**" % n(len(bank)), "BANK.csv rows")
|
||||
|
||||
# languages of the in-memory sounds
|
||||
neutral = [s for s in bank if not s.language]
|
||||
single = collections.Counter(s.language for s in bank if s.language and ";" not in s.language)
|
||||
multi = [s for s in bank if ";" in s.language]
|
||||
multi_c = collections.Counter(s.language for s in multi)
|
||||
claim("REPORT.md", "%s shared by all languages, ~%s localized speech lines per voice language, %d shared by "
|
||||
"several" % (n(len(neutral)), n(int(round(sum(single.values()) / len(single), -1))), len(multi)))
|
||||
claim("docs/formats/banks.md", "%s sounds are language-neutral, %s belong to one voice language, %d to several"
|
||||
% (n(len(neutral)), rng(single.values()), len(multi)))
|
||||
a4 = multi_c.get("ESN;FRA;INT;ITA", 0)
|
||||
a3 = multi_c.get("FRA;INT;ITA", 0)
|
||||
claim("docs/formats/banks.md", "(%d shared by INT/FRA/ITA/ESN while DEU has its own, %d by INT/FRA/ITA)" % (a4, a3))
|
||||
claim("docs/formats/banks.md", "everywhere; %d of these are voice lines" % sum(1 for s in neutral if s.line))
|
||||
claim("docs/formats/banks.md", "stored once for every language (%d sounds)"
|
||||
% sum(1 for s in neutral if s.bank.startswith("Speech")))
|
||||
claim("docs/formats/banks.md", "lines** (about %s per language" % n(int(round(sum(single.values()) / len(single), -1))))
|
||||
ch = collections.Counter(s.channels for s in bank)
|
||||
claim("docs/formats/banks.md", "of all %s in-memory sounds %s are mono, %s stereo, %d have 3-6 channels"
|
||||
% (n(len(bank)), n(ch[1]), n(ch[2]), sum(v for k, v in ch.items() if k >= 3)))
|
||||
|
||||
# copies and heads
|
||||
with gzip.open(os.path.join(cat, "BANK_locations.csv.gz"), "rt", newline="") as f:
|
||||
loc = list(csv.DictReader(f))
|
||||
m_rows = [r for r in loc if r["kind"] == "m"]
|
||||
p_rows = [r for r in loc if r["kind"] == "p"]
|
||||
keys = {s.key: s for L in VOICES for s in rows[L]}
|
||||
whole = [r for r in p_rows if int(r["size"]) >= keys[r["sound"]].size]
|
||||
pref = [r for r in p_rows if r not in whole]
|
||||
whole_rec = {r["sound"] for r in whole}
|
||||
pref_rec = {r["sound"] for r in pref}
|
||||
pref_lines = {keys[k].line for k in pref_rec}
|
||||
all_lines = {keys[r["sound"]].line for r in p_rows}
|
||||
per_lang_pref = collections.Counter(k.split(":")[0] for k in pref_rec)
|
||||
per_lang_whole = collections.Counter(k.split(":")[0] for k in whole_rec)
|
||||
claim("REPORT.md", "copy (%s copies). %s WAD voice recordings (%d lines) also have bank copies (%d prefetch heads, %d "
|
||||
"whole copies; %s copies)" % (n(len(m_rows)), n(len(pref_rec | whole_rec)), len(all_lines), len(pref_rec),
|
||||
len(whole_rec), n(len(p_rows))))
|
||||
claim("docs/formats/banks.md", "%d of them are **prefetch heads** of streamed WAD sounds and %d are **whole copies**"
|
||||
% (len({r["media"] for r in pref}), len({r["media"] for r in whole})))
|
||||
claim("docs/formats/banks.md", "%d WAD sounds have such heads (%s copies)" % (len(pref_rec), n(len(pref))))
|
||||
claim("docs/formats/banks.md", "%d more WAD recordings (%s per language" % (len(whole_rec), rng(per_lang_whole.values())))
|
||||
claim("docs/formats/banks.md", "byte-identical to the WAD WEM (%d copies)" % len(whole))
|
||||
claim("docs/formats/banks.md", "Heads and whole copies together: %d recordings, %s copies"
|
||||
% (len(pref_rec | whole_rec), n(len(p_rows))))
|
||||
claim("docs/formats/banks.md", "the %d lines with a prefetch head (%d recordings over the 5 languages"
|
||||
% (len(pref_lines), len(pref_rec)))
|
||||
claim("docs/formats/banks.md", "A WAD recording that a bank also stores whole (%d)" % len(whole_rec))
|
||||
claim("docs/decisions.md", "%d voice recordings (%d lines, %s recordings per language) have their first bytes"
|
||||
% (len(pref_rec), len(pref_lines), rng(per_lang_pref.values(), str)))
|
||||
claim("docs/decisions.md", "and %d more are stored whole in banks" % len(whole_rec))
|
||||
places = len(loc)
|
||||
media_all = {r["media"] for r in loc}
|
||||
loose = {r["media"] for r in loc if r["file"].startswith("SFX/")}
|
||||
pk = {r["media"] for r in loc if not r["file"].startswith("SFX/")}
|
||||
claim("docs/formats/banks.md", "The two sets share %d media IDs: %s unique media IDs in %s places"
|
||||
% (len(loose & pk), n(len(media_all)), n(places)))
|
||||
claim("docs/formats/banks.md", "+ `TST_Prep_Media`) | %d | %s |"
|
||||
% (len({r["file"] for r in loc if r["file"].startswith("SFX/")}), n(len(loose))))
|
||||
claim("docs/formats/banks.md", "packages) | 3,340 | %s |" % n(len(pk)))
|
||||
claim("REPORT.md", "in %d loose `.bnk` files and %d `.upk` packages"
|
||||
% (len({r["file"] for r in loc if r["file"].startswith("SFX/")}), len({r["file"] for r in loc if r["file"].endswith(".upk")})))
|
||||
assert len(media_all) == len(bank) + len(pref_rec | whole_rec)
|
||||
|
||||
# lines
|
||||
lines_all = {s.line for L in VOICES for s in rows[L] if s.line} | {s.line for s in bank if s.line}
|
||||
wad_lines = {L: {s.line for s in rows[L] if s.line} for L in VOICES}
|
||||
assert len({len(v) for v in wad_lines.values()}) == 1
|
||||
WL = len(wad_lines["INT"])
|
||||
copies = {L: sum(1 for s in rows[L] if s.line) - len(wad_lines[L]) for L in VOICES}
|
||||
recs = sum(len(wad_lines[L]) for L in VOICES) + len({(s.target) for s in bank if s.line})
|
||||
for doc, text in [("REPORT.md", "%s lines (WAD + in-memory) with exactly one recording in each of the 5 voice languages"),
|
||||
("docs/decisions.md", "recording per language (%s lines, all 5 languages)"),
|
||||
("docs/formats/banks.md", "recordings that are sources of the same Sound objects: %s lines")]:
|
||||
claim(doc, text % n(len(lines_all)), "distinct line ids")
|
||||
claim("docs/formats/banks.md", "in each of the 5 voice languages (%s recordings)" % n(recs))
|
||||
claim("REPORT.md", "recordings stored twice (%s per language) are written together" % rng(copies.values(), str))
|
||||
claim("docs/formats/banks.md", "- %s recordings per language are stored twice" % rng(copies.values(), str).capitalize())
|
||||
claim("docs/categories.md", "Coverage of the %s WAD voice lines" % n(WL))
|
||||
claim("docs/decisions.md", "of the %s WAD voice lines" % n(WL))
|
||||
|
||||
# positions differ between languages
|
||||
it = {s.target: s.line for s in rows["INT"] if ":" in s.target}
|
||||
diff = []
|
||||
for L in ["FRA", "ITA", "DEU", "ESN"]:
|
||||
other = {s.target: s.line for s in rows[L] if ":" in s.target}
|
||||
diff.append(100.0 * sum(1 for t in it if other.get(t) != it[t]) / len(it))
|
||||
span = "%.1f-%.1f %%" % (min(diff), max(diff))
|
||||
claim("docs/formats/wad.md", "(%s of the WEMs in multi-variation entries, compared with INT" % span)
|
||||
claim("docs/decisions.md", "%s of the WEMs in multi-variation entries are at another position than in INT" % span)
|
||||
pos = {}
|
||||
for L in VOICES:
|
||||
s = next(s for s in rows[L] if s.name == "Speech_TAPE_JOKER_1_04")
|
||||
pos[L] = int(s.target.split(":")[1])
|
||||
claim("docs/formats/wad.md", "`Speech_TAPE_JOKER_1_04` is variation %d in INT and ITA, %d in FRA, %d in DEU, %d in ESN"
|
||||
% (pos["INT"], pos["FRA"], pos["DEU"], pos["ESN"]) if pos["INT"] == pos["ITA"] else "(INT != ITA)")
|
||||
ents = collections.Counter(s.target.split(":")[0] for s in rows["INT"])
|
||||
claim("docs/formats/wad.md", "(every language: %s WEMs in %s entries" % (n(W), n(len(ents))))
|
||||
radio = [s for s in rows["INT"] if s.aliases and any(a.startswith("CHR_RadioScanner#") for a in s.names())
|
||||
or s.name.startswith("CHR_RadioScanner#")]
|
||||
claim("docs/formats/wad.md", "e.g. `CHR_RadioScanner` holds %d" % len({s.target for s in radio}))
|
||||
|
||||
# SFX WAD
|
||||
packs = collections.Counter(s.target.split("/")[0] for s in sfx if "/" in s.target)
|
||||
grouped = [s for s in sfx if "/" not in s.target]
|
||||
claim("docs/formats/wad.md", "**SFX WAD**: %d grouped ambience/effect entries (%d WEMs) plus %d **stream packs**"
|
||||
% (len({s.target.split(":")[0] for s in grouped}), len(grouped), len(packs)))
|
||||
claim("docs/formats/wad.md", "`EDF036D6` (Music, %d WEMs" % packs["EDF036D6"])
|
||||
claim("docs/formats/wad.md", "`E8F980BE` (Music_PDLC, %d), `57572DAC` (Global, %d), `FCA4F778` (%d)"
|
||||
% (packs["E8F980BE"], packs["57572DAC"], packs["FCA4F778"]))
|
||||
|
||||
# categories
|
||||
cats = [collections.Counter(s.category for s in rows[L]) for L in VOICES]
|
||||
assert all(c == cats[0] for c in cats), "category counts differ between languages"
|
||||
order = ["Story missions", "Predator chatter", "Character lines", "Open-world chatter", "Side missions",
|
||||
"Combat barks", "Efforts & emotes", "Interview tapes", "Riddler", "Radio & TV", "Gameplay & hints", "DLC",
|
||||
"Boss fights", "Cinematics", "Reactions", "Interrogations"]
|
||||
g = collections.Counter(s.group for s in sfx)
|
||||
bc = collections.Counter(s.category for s in bank).most_common(8)
|
||||
claim("docs/categories.md", "Counts per voice language (identical in all 5): " + ", ".join(
|
||||
"%s %d" % (c, cats[0][c]) for c in order) + ". SFX archive: " + ", ".join(
|
||||
"%s %d" % (k, g[k]) for k in ["Music", "Ambience", "SFX", "Unknown"]) + ". BANK (all languages): " + ", ".join(
|
||||
"%s %d" % kv for kv in bc) + ", ...")
|
||||
|
||||
# characters (by line)
|
||||
first = {}
|
||||
for s in rows["INT"]:
|
||||
first.setdefault(s.line, s)
|
||||
wc = sum(1 for s in first.values() if s.character)
|
||||
inf = sum(1 for s in first.values() if not s.character and s.characters_inferred)
|
||||
nei = [s for s in first.values() if not s.character and not s.characters_inferred]
|
||||
top = collections.Counter(s.category for s in nei).most_common(4)
|
||||
assert all(sum(1 for s in rows[L] if s.character) == sum(1 for s in rows["INT"] if s.character) for L in VOICES)
|
||||
claim("docs/categories.md", "(every voice language, the same lines): %s with a verified speaker, %s more with inferred "
|
||||
"characters, %s with neither (%s, ...)" % (n(wc), n(inf), n(len(nei)), ", ".join(
|
||||
"%s %d" % (c.lower().replace("radio & tv", "radio & TV"), k) for c, k in top)))
|
||||
claim("docs/categories.md", "A hint for the ~%s lines without a verified speaker" % n(int(round(WL - wc, -2))))
|
||||
claim("REPORT.md", "speaker labels (%s in every language)" % n(wc))
|
||||
|
||||
# subtitles
|
||||
langs = sorted(f[4:7] for f in os.listdir(cat) if f.startswith("SUB_") and f.endswith(".csv"))
|
||||
texts = {L: subtitles.read_texts(os.path.join(cat, "SUB_%s.csv" % L), L) for L in langs}
|
||||
total = sum(len(t) for t in texts.values())
|
||||
linked = sum(1 for t in texts.values() for s in t if s.sound)
|
||||
i_tot, i_l = len(texts["INT"]), sum(1 for s in texts["INT"] if s.sound)
|
||||
for doc in ("REPORT.md", "docs/formats/subtitles.md"):
|
||||
claim(doc, "%s of %s strings in %d languages (INT %s" % (n(linked), n(total), len(langs), n(i_l))
|
||||
if doc == "REPORT.md" else "Linked: %s of %s strings in the %d languages (INT %s / %s)"
|
||||
% (n(linked), n(total), len(langs), n(i_l), n(i_tot)))
|
||||
claim("REPORT.md", "(INT %s of %s)" % (n(i_l), n(i_tot)))
|
||||
unl = {L: {s.speech for s in t if not s.sound} for L, t in texts.items()}
|
||||
unl_n = {len(v) for v in unl.values()}
|
||||
assert len(unl_n) == 1, unl_n
|
||||
un_strings = [sum(1 for s in t if not s.sound) for t in texts.values()]
|
||||
claim("REPORT.md", "%d speech names (%s strings per language, INT %d" % (unl_n.pop(), rng(un_strings),
|
||||
i_tot - i_l))
|
||||
claim("docs/formats/subtitles.md", "(%d speech names, %s strings per language" % (len(unl["INT"]), rng(un_strings)))
|
||||
two = {s.speech for s in texts["INT"] if ";" in s.sound}
|
||||
claim("docs/formats/subtitles.md", "(%d subtitled events" % len(two))
|
||||
one = []
|
||||
for L, t in texts.items():
|
||||
c = collections.Counter(s.speech for s in t)
|
||||
one.append(100.0 * sum(1 for v in c.values() if v == 1) / len(c))
|
||||
claim("docs/formats/subtitles.md", "%s subtitle strings per language: %d-%d %% of the subtitles have one line (JPN: all)"
|
||||
% (rng([len(t) for t in texts.values()]), int(min(one)), int(round(max(x for x in one if x < 100)))))
|
||||
enc = {L: collections.Counter(s.enc for s in t) for L, t in texts.items()}
|
||||
pol = pct(enc["POL"]["w"], len(texts["POL"]))
|
||||
west = [enc[L]["w"] for L in langs if L not in ("RUS", "JPN", "KOR", "POL")]
|
||||
claim("docs/formats/subtitles.md", "%d %% of POL are UTF-16; the other languages have %s UTF-16 strings each"
|
||||
% (pol, rng(west)))
|
||||
with gzip.open(os.path.join(cat, subtitles.LOCATIONS), "rt", newline="") as f:
|
||||
sublocs = sum(1 for _ in csv.DictReader(f))
|
||||
claim("REPORT.md", "Edited in place in every copy (%s copies)" % n(sublocs))
|
||||
claim("docs/formats/subtitles.md", "%s stored copies" % n(sublocs))
|
||||
claim("docs/user-guide.md", "About %d subtitles per language have no sound" % int(round(sum(un_strings) / len(un_strings), -1)))
|
||||
|
||||
# subtitle coverage of the INT voice lines
|
||||
lk = {k for s in texts["INT"] for k in s.sound.split(";") if k}
|
||||
cov, tot = collections.Counter(), collections.Counter()
|
||||
for s in rows["INT"] + [b for b in bank if b.line and "INT" in b.language.split(";")]:
|
||||
if s.line:
|
||||
tot[s.category] += 1
|
||||
cov[s.category] += s.key in lk
|
||||
p = {c: pct(cov[c], tot[c]) for c in tot}
|
||||
claim("REPORT.md", "few for combat barks (%d %%), cinematics and boss fights (%d %%;"
|
||||
% (p["Combat barks"], p["Cinematics"]) if p["Cinematics"] == p["Boss fights"] else "(cinematics != boss)")
|
||||
hi = [p[c] for c in ("Story missions", "Predator chatter", "Character lines", "Open-world chatter")]
|
||||
claim("REPORT.md", "%d-%d %% for story, predator, character and open-world lines" % (min(hi), max(hi)))
|
||||
claim("REPORT.md", "(INT voice lines with a subtitle): none for efforts/grunts" if p["Efforts & emotes"] == 0 else "!")
|
||||
|
||||
# what the editor loads (Italian on the test PC: the smoke test)
|
||||
ita = len(rows["ITA"]) + len(sfx) + sum(1 for s in bank if s.in_language("ITA"))
|
||||
claim("REPORT.md", "%s sounds loaded, encode + inject, %s subtitles" % (n(ita), n(len(texts["ITA"]))))
|
||||
claim("README.md", "~%s sounds of one language" % n(int(round(ita, -3))))
|
||||
return rows
|
||||
|
||||
|
||||
# --- figures recorded by the catalog build (index.json: they need the full scan) ---------------------
|
||||
def index_claims(cat):
|
||||
import json
|
||||
ix = json.load(open(os.path.join(cat, "index.json"), encoding="utf-8"))
|
||||
b, ln, sub = ix["BANK"], ix["LINES"], ix["SUB"]
|
||||
claim("REPORT.md", "Names: %s BANK sounds are named after the Wwise event that plays them (`Play_...`, `Speech_...`), "
|
||||
"the other %s after their bank" % (n(b["named_by_event"]), n(b["named_by_bank"])))
|
||||
claim("REPORT.md", "**Unnamed sounds**: %s BANK sounds are reached by no event" % n(b["named_by_bank"]))
|
||||
claim("docs/decisions.md", "sounds (%s of %s, voice lines included); the other %s are numbered per bank"
|
||||
% (n(b["named_by_event"]), n(b["sounds"]), n(b["named_by_bank"])))
|
||||
claim("docs/decisions.md", "the most specific speech event that plays it (%s of the %s WAD voice lines)"
|
||||
% (n(ln["wad_lines_named_by_event"]), n(ln["wad_lines"])))
|
||||
claim("docs/decisions.md", "the other %s keep the name of their English (INT) recording's position"
|
||||
% n(ln["wad_lines_named_after_int_recording"]))
|
||||
claim("REPORT.md", "- %s voice lines are played by no event whose name the game stores"
|
||||
% n(ln["wad_lines_named_after_int_recording"]))
|
||||
claim("docs/user-guide.md", "About %d %% of the lines are named by no game event"
|
||||
% pct(ln["wad_lines_named_after_int_recording"], ln["wad_lines"]))
|
||||
claim("docs/formats/subtitles.md", "%s dialogue objects in the packages for %s speech lines"
|
||||
% (n(sub["dialogue_objects"]), n(sub["dialogue_lines"])))
|
||||
claim("docs/formats/subtitles.md", "The sizes add up exactly in all %s objects" % n(sub["dialogue_objects"]))
|
||||
claim("docs/formats/subtitles.md", "plus %d scripted subtitles" % sub["scripted_objects"])
|
||||
claim("docs/formats/subtitles.md", "%d more subtitles (radio and encryption-card lines) are `RSubtitle` objects"
|
||||
% sub["scripted_objects"])
|
||||
claim("REPORT.md", "(%s speech lines x %d languages) and" % (n(sub["dialogue_lines"]), len(sub["lines"])))
|
||||
claim("REPORT.md", "in %d scripted `RSubtitle` objects" % sub["scripted_objects"])
|
||||
claim("docs/formats/banks.md", "(in %s of the 2,650 packages)" % n(b["files"] - 67)) # 67 loose .bnk with banks
|
||||
|
||||
|
||||
# --- the game's own files ----------------------------------------------------------------------------
|
||||
def menu_claims(game):
|
||||
from ackit import loctext
|
||||
root = os.path.join(game.root, "BmGame", "Localization")
|
||||
counts = {L: len(loctext.load(root, L)) for L in loctext.languages(root)}
|
||||
files = {L: len(loctext.files(root, L)) for L in loctext.languages(root)}
|
||||
approx = n(int(round(sum(counts.values()) / len(counts), -3)))
|
||||
claim("docs/formats/text.md", "~%d files and ~%s strings each" % (collections.Counter(files.values()).most_common(1)[0][0], approx))
|
||||
claim("REPORT.md", "~%s strings per language" % approx)
|
||||
claim("docs/user-guide.md", "of the selected language (about\n%s)" % approx)
|
||||
claim("REPORT.md", "%s menu texts loaded" % n(counts["ITA"]))
|
||||
|
||||
|
||||
def scan_claims(game):
|
||||
"""Bank survey from a fresh scan of every package, and the relation facts of `ackit verify --catalog`."""
|
||||
from ackit.bnk import find_banks, Bank, BankError, IN_MEMORY, STREAMED, PREFETCH
|
||||
blobs = collections.Counter()
|
||||
media_sets = collections.defaultdict(set)
|
||||
kinds = collections.Counter()
|
||||
in_pkg = set()
|
||||
|
||||
def visit(rel, pkg, start, data):
|
||||
for bk in find_banks(data):
|
||||
blobs[bk.id] += 1
|
||||
media_sets[bk.id].add(frozenset(m.id for m in bk.media()))
|
||||
in_pkg.add(bk.id)
|
||||
own = {m.id for m in bk.media()}
|
||||
for s in bk.sources():
|
||||
if s.stream == IN_MEMORY:
|
||||
kinds["memory own" if s.media in own else "memory other"] += 1
|
||||
elif s.stream == STREAMED:
|
||||
kinds["streamed"] += 1
|
||||
elif s.stream == PREFETCH:
|
||||
kinds["prefetch"] += 1
|
||||
|
||||
sc = banks.scan(game.cooked, visit=visit)
|
||||
for path in glob.glob(os.path.join(game.cooked, "SFX", "*.bnk")): # loose banks: source kinds too
|
||||
try:
|
||||
bk = Bank(open(path, "rb").read(), 0)
|
||||
except BankError:
|
||||
continue
|
||||
own = {m.id for m in bk.media()}
|
||||
for s in bk.sources():
|
||||
if s.stream == IN_MEMORY:
|
||||
kinds["memory own" if s.media in own else "memory other"] += 1
|
||||
elif s.stream == STREAMED:
|
||||
kinds["streamed"] += 1
|
||||
elif s.stream == PREFETCH:
|
||||
kinds["prefetch"] += 1
|
||||
names = {}
|
||||
cooked_list = [l.strip() for l in open(os.path.join(game.cooked, "CookedBanks.txt"), encoding="latin-1") if l.strip()]
|
||||
from ackit.bnk import fnv1
|
||||
for nm in cooked_list:
|
||||
names[fnv1(nm)] = nm
|
||||
loose_ids = {int(os.path.splitext(os.path.basename(p))[0], 16) for p in glob.glob(os.path.join(game.cooked, "SFX", "*.bnk"))
|
||||
if re.fullmatch(r"[0-9A-Fa-f]{8}", os.path.splitext(os.path.basename(p))[0])}
|
||||
embedded = [nm for h, nm in names.items() if h in in_pkg]
|
||||
loose = sorted(nm for h, nm in names.items() if h in loose_ids and h not in in_pkg)
|
||||
nowhere = sorted(nm for h, nm in names.items() if h not in in_pkg and h not in loose_ids)
|
||||
claim("docs/formats/banks.md", "lists all %s cooked bank names" % n(len(cooked_list)))
|
||||
both = sorted(nm for h, nm in names.items() if h in loose_ids and h in in_pkg)
|
||||
claim("docs/formats/banks.md", "%s of them are embedded in packages (%s also as loose `.bnk` files), %d are only loose "
|
||||
"`.bnk` files (%s); %s were found nowhere"
|
||||
% (n(len(embedded)), ", ".join("`%s`" % x for x in both), len(loose), ", ".join("`%s`" % x for x in loose),
|
||||
", ".join("`%s`" % x for x in nowhere[:-1]) + " and `%s`" % nowhere[-1]))
|
||||
diff_sets = sum(1 for b, v in media_sets.items() if len(v) > 1)
|
||||
claim("docs/formats/banks.md", "Bank blobs in packages: %s for %s bank IDs" % (n(sum(blobs.values())), n(len(blobs))))
|
||||
by_name = {names.get(b, ""): c for b, c in blobs.items()}
|
||||
claim("docs/formats/banks.md", "`CHR_CatwomanFoleySuit` %d times, `CHR_BatmanFoleyBelt` %d). %d bank IDs"
|
||||
% (by_name.get("CHR_CatwomanFoleySuit", 0), by_name.get("CHR_BatmanFoleyBelt", 0), diff_sets))
|
||||
claim("docs/formats/banks.md", "| banks embedded in `.upk` packages (in %s of the %s packages) | %s |"
|
||||
% (n(sum(1 for f in sc.files if f.lower().endswith(".upk"))), n(len(glob.glob(os.path.join(game.cooked, "*.upk")))),
|
||||
n(len(blobs))))
|
||||
claim("docs/formats/banks.md", "Source kinds over all bank copies: %s in-memory sources of the bank's own media; %s "
|
||||
"in-memory sources with bank id 0" % (n(kinds["memory own"]), n(kinds["memory other"])))
|
||||
claim("docs/formats/banks.md", "(loaded by another bank); %s streamed sources; %s **prefetch** sources"
|
||||
% (n(kinds["streamed"]), n(kinds["prefetch"])))
|
||||
refs = {h for h, t in sc.packs.items() if t}
|
||||
claim("docs/formats/banks.md", "the BKPK tables in the packages reference %s WAD entries" % n(len(refs)))
|
||||
per_bank = collections.defaultdict(set)
|
||||
for m, e in sc.media.items():
|
||||
for b in e["banks"]:
|
||||
per_bank[b].add(m)
|
||||
big = sorted(per_bank.items(), key=lambda kv: -len(kv[1]))[:7]
|
||||
nm = [(sc.bank_names.get(b, "%08X" % b), len(v)) for b, v in big]
|
||||
claim("docs/formats/banks.md", "Largest: `%s` %d WEMs, " % nm[0] + ", ".join("`%s` %d" % x for x in nm[1:]))
|
||||
pkm = {l.media for l in sc.locations if not l.file.startswith("SFX/")}
|
||||
pre = collections.Counter()
|
||||
for m in pkm:
|
||||
for p_ in {sc.bank_names.get(b, "%08X" % b).split("_")[0] for b in sc.media[m]["banks"]}:
|
||||
pre[p_] += 1
|
||||
top = [k for k, v in pre.most_common() if v >= 30]
|
||||
text = ", ".join("`%s` %s" % (k, n(pre[k])) if k != "CHR" else "`CHR`\n(character foley, combat, bosses) %s" % n(pre[k])
|
||||
for k in top)
|
||||
claim("docs/formats/banks.md", "(a media held by banks of two prefixes counts for both): " + text + ", others < 30")
|
||||
# media IDs used by several languages' WAD tables (each at its own position)
|
||||
langs_of = collections.defaultdict(set)
|
||||
for h, tables in sc.packs.items():
|
||||
for L, t in tables.items():
|
||||
if L:
|
||||
for m in t:
|
||||
langs_of[m].add(L)
|
||||
claim("docs/formats/banks.md", "%d media IDs are used by several\n languages" % sum(1 for v in langs_of.values() if len(v) > 1))
|
||||
|
||||
# relations, re-derived from the game by the catalog check
|
||||
from ackit.verify import catalogcheck
|
||||
rep = catalogcheck.run(game, log=lambda *a: None)
|
||||
facts = "\n".join(rep.facts)
|
||||
print("verify --catalog:", "catalog OK" if rep.ok else "%d issue(s)" % len(rep.issues))
|
||||
|
||||
def fact(pattern):
|
||||
m = re.search(pattern, facts)
|
||||
if not m:
|
||||
raise SystemExit("fact not found: " + pattern)
|
||||
return [int(x) for x in m.groups()]
|
||||
src = fact(r"own language's bank: (\d+) of (\d+)")
|
||||
claim("REPORT.md", "own language (%s / %s)" % (n(src[0]), n(src[1])))
|
||||
claim("docs/formats/banks.md", "of its own language (%s of %s)" % (n(src[0]), n(src[1])))
|
||||
hd = fact(r"whole bank copies: (\d+) of (\d+)")
|
||||
claim("REPORT.md", "is the start of its recording (%s / %s)" % (n(hd[0]), n(hd[1])))
|
||||
claim("docs/formats/banks.md", "line gives it (%s of %s)" % (n(hd[0]), n(hd[1])) if True else "")
|
||||
rec, lns, full = fact(r"(\d+) recordings \(WAD .*?\), (\d+) lines, (\d+) of them")
|
||||
claim("docs/formats/banks.md", "(%s recordings)" % n(rec))
|
||||
ln15 = fact(r"(\d+) of (\d+) lines within a factor 1.5")
|
||||
claim("docs/formats/banks.md", "(%s of %s\n within a factor 1.5" % (n(ln15[0]), n(ln15[1])))
|
||||
sp = fact(r"speakers: (\d+) labelled voice lines agree .*?, (\d+) listed as Unknown .*?, (\d+) differ")
|
||||
claim("REPORT.md", "%s agree, %d differ" % (n(sp[0]), sp[2]))
|
||||
claim("docs/categories.md", "%s labelled\n lines agree" % n(sp[0]))
|
||||
claim("docs/categories.md", "%d listed\n as Unknown are named there, %d differ" % (sp[1], sp[2]))
|
||||
if not rep.ok:
|
||||
claim("(game)", "catalog OK", "ackit verify --catalog")
|
||||
|
||||
|
||||
def _rebuild_language(lang):
|
||||
"""Keys of the WEMs of one speech WAD that the builder does not reproduce byte for byte."""
|
||||
from ackit.inject import read_target
|
||||
from ackit.verify.selfcheck import rebuild_identical
|
||||
from ackit.wad import Wad
|
||||
game = Game(default_game_dir(settings.load()))
|
||||
path = game.wad(lang)
|
||||
w = Wad(path)
|
||||
from ackit.wem import Wem
|
||||
bad, count, extra = [], 0, collections.Counter()
|
||||
for s in read_csv(os.path.join(catalog_dir(), lang + ".csv")):
|
||||
count += 1
|
||||
r = read_target(path, s.target, w)
|
||||
tags = {c.tag for c in Wem.parse(r).chunks} - {b"fmt ", b"data"}
|
||||
if tags:
|
||||
extra["/".join(sorted(t.decode() for t in tags))] += 1
|
||||
try:
|
||||
ok = rebuild_identical(r)
|
||||
except Exception: # noqa: BLE001 - a WEM the builder cannot parse is a difference too
|
||||
ok = False
|
||||
if not ok:
|
||||
bad.append(s.key)
|
||||
return count, bad, extra
|
||||
|
||||
|
||||
def wem_claims():
|
||||
"""Every WEM of the 5 speech WADs and of the SFX WAD rebuilt from the format rules (docs/formats/wem.md;
|
||||
~10 minutes)."""
|
||||
import multiprocessing
|
||||
from ackit.inject import read_target
|
||||
from ackit.wem import Wem
|
||||
with multiprocessing.Pool(len(VOICES) + 1) as p:
|
||||
res = p.map(_rebuild_language, VOICES + ["SFX"])
|
||||
extra = collections.Counter()
|
||||
for _, _, e in res:
|
||||
extra.update(e)
|
||||
sfx_count, sfx_bad, _ = res.pop()
|
||||
game = Game(default_game_dir(settings.load()))
|
||||
sfx = {s.key: s for s in read_csv(os.path.join(catalog_dir(), "SFX.csv"))}
|
||||
kinds = collections.Counter()
|
||||
per = [] # samples per seek-table entry of the music tracks
|
||||
from ackit.verify.selfcheck import rebuild_identical
|
||||
for k in sfx_bad:
|
||||
s = sfx[k]
|
||||
r = read_target(game.wad("SFX"), s.target)
|
||||
m = Wem.parse(r)
|
||||
try:
|
||||
rebuild_identical(r)
|
||||
err = False
|
||||
except ValueError: # "inconsistent total_frames": more samples than the packets hold
|
||||
err = True
|
||||
if any(c.tag == b"smpl" for c in m.chunks):
|
||||
kinds["smpl"] += 1
|
||||
elif err:
|
||||
kinds["frames"] += 1
|
||||
elif s.group == "Music" and m.vorb["seek_table_size"]:
|
||||
kinds["seek"] += 1
|
||||
per.append(m.vorb["total_frames"] / (m.vorb["seek_table_size"] // 4))
|
||||
else:
|
||||
kinds["other"] += 1
|
||||
claim("REPORT.md", "%d SFX originals (music seek tables and sample counts, looping ambience)" % len(sfx_bad))
|
||||
claim("docs/formats/wem.md", "In SFX %s of %s are identical; the others are %d music tracks whose seek table follows "
|
||||
"another placement (one entry per %s samples), %d music tracks whose sample count exceeds what their "
|
||||
"packets hold, and %d 4-channel ambience WEMs with a `smpl` (loop) chunk%s"
|
||||
% (n(sfx_count - len(sfx_bad)), n(sfx_count), kinds["seek"], rng([int(x) for x in per]), kinds["frames"],
|
||||
kinds["smpl"], "" if not kinds["other"] else " (+%d other)" % kinds["other"]))
|
||||
total = sum(c for c, _, _ in res)
|
||||
bad = [k for _, b, _ in res for k in b]
|
||||
claim("docs/formats/wem.md", "`RIFF` size `WAVE`, then `fmt ` (0x42 bytes), optional %s, `data`" % ", ".join(
|
||||
"%s (%s %s)" % (" + ".join("`%s`" % t for t in k.split("/")), n(v), "WEM" if v == 1 else "WEMs")
|
||||
for k, v in sorted(extra.items())))
|
||||
exceptions = ", ".join("%s `%s`" % tuple(k.split(":", 1)) for k in bad) # e.g. INT `FE9963E8:74`
|
||||
claim("docs/formats/wem.md", "Checked on every WEM of the 5 speech WADs: %s of %s are identical; the exception (%s)"
|
||||
% (n(total - len(bad)), n(total), exceptions))
|
||||
|
||||
|
||||
def test_claims():
|
||||
import subprocess
|
||||
out = subprocess.run([sys.executable, "-m", "pytest", "--collect-only", "-q", "-p", "no:cacheprovider"],
|
||||
cwd=ROOT, capture_output=True, text=True).stdout
|
||||
m = re.search(r"(\d+) tests? collected", out)
|
||||
claim("REPORT.md", "**Tests**: %s automated tests" % (m.group(1) if m else "?"), "pytest --collect-only")
|
||||
|
||||
|
||||
def norm(t):
|
||||
return re.sub(r"\s+", " ", t).strip()
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--game")
|
||||
ap.add_argument("--scan", action="store_true", help="also scan every package and run the catalog check")
|
||||
ap.add_argument("--wems", action="store_true", help="also rebuild every speech and SFX WEM from the rules")
|
||||
a = ap.parse_args()
|
||||
cat = catalog_dir()
|
||||
catalog_claims(cat)
|
||||
index_claims(cat)
|
||||
test_claims()
|
||||
gd = a.game or default_game_dir(settings.load())
|
||||
game = Game(gd) if gd else None
|
||||
if game:
|
||||
menu_claims(game)
|
||||
if a.scan:
|
||||
if not game:
|
||||
raise SystemExit("--scan needs the game")
|
||||
scan_claims(game)
|
||||
if a.wems:
|
||||
if not game:
|
||||
raise SystemExit("--wems needs the game")
|
||||
wem_claims()
|
||||
docs = {}
|
||||
bad = 0
|
||||
for doc, text, why in CLAIMS:
|
||||
if doc not in docs:
|
||||
path = os.path.join(ROOT, doc)
|
||||
docs[doc] = norm(open(path, encoding="utf-8").read()) if os.path.exists(path) else ""
|
||||
if norm(text) not in docs[doc]:
|
||||
bad += 1
|
||||
print("MISMATCH %s: expected \"%s\"%s" % (doc, norm(text), (" (" + why + ")") if why else ""))
|
||||
print("%d claims checked in %d documents%s: %s" % (len(CLAIMS), len(docs), "" if a.scan else " (without --scan)",
|
||||
"all match" if not bad else "%d mismatch(es)" % bad))
|
||||
return 1 if bad else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -8,7 +8,8 @@ For every check: the SHA-1 of each touched file is taken, the change is applied
|
||||
of the editor and the CLI), read back from the game files, reverted, and the files must be byte-identical again.
|
||||
1. in-memory sound with the most copies (every copy written; once, it is shared by all languages)
|
||||
2. localized in-memory speech line of the language's voices
|
||||
3. speech line of the language's voices with prefetch heads (WAD + every head copy)
|
||||
3. speech line of the language's voices with prefetch heads (WAD + every head copy), and one that a bank
|
||||
also stores whole (WAD + every whole copy)
|
||||
4. subtitle with several copies (RDialogueEvent), and a scripted subtitle (RSubtitle, Startup.upk)
|
||||
5. menu text
|
||||
6. sound + subtitle in the same package, reverted one at a time
|
||||
@@ -82,7 +83,7 @@ def sound_round_trip(ws, s, label):
|
||||
for l, b in copies_of(ws, s):
|
||||
if l.kind == banks.MEMORY:
|
||||
ok &= b == wem + bytes(l.size - len(wem))
|
||||
else: # prefetch head of a WAD line (stored zero-padded to its size)
|
||||
else: # head or whole copy of a WAD line (zero-padded to its size)
|
||||
full = wem + bytes(max(0, ws.store.get(s.key)["orig_size"] - len(wem)))
|
||||
ok &= b == (full + bytes(l.size))[:l.size]
|
||||
ws.revert([s])
|
||||
@@ -140,7 +141,13 @@ def run_checks(game, lang, voices, td, ws, first):
|
||||
heads = [ws.catalog.by_key[k] for k in ws.catalog.heads if k in ws.catalog.by_key
|
||||
and ws.catalog.by_key[k].archive == voices]
|
||||
sound_round_trip(ws, max(heads, key=lambda s: len(ws.locations(s))),
|
||||
"3 %s speech line with prefetch heads" % lang)
|
||||
"3a %s speech line with prefetch heads" % lang)
|
||||
whole = [s for s in heads if any(l.size >= s.size for l in ws.locations(s))]
|
||||
if whole:
|
||||
sound_round_trip(ws, max(whole, key=lambda s: len(ws.locations(s))),
|
||||
"3b %s speech line also stored whole in a bank" % lang)
|
||||
else:
|
||||
check("3b %s speech line also stored whole in a bank" % lang, False, "none found")
|
||||
|
||||
tx = ws.texts
|
||||
subs = tx.subtitles(lang)
|
||||
|
||||
Reference in New Issue
Block a user