220 lines
11 KiB
Python
220 lines
11 KiB
Python
"""Cross-check the catalog against the installed game (`ackit verify --catalog`).
|
|
|
|
For every archive the catalog in use (shipped CSV or the user-cache scan) is compared with a fresh scan:
|
|
missing a sound of the game that is not in the catalog
|
|
extra a catalog row that is not in the game
|
|
duplicate a target or a name used twice
|
|
size/sha1 the catalog's original size or checksum differs from the game (game modified or other build)
|
|
category the row's group/category is not what the classification rules give for its name
|
|
character the verified speaker differs from data/catalog/characters.csv
|
|
language the archive code does not match the WAD's language folder
|
|
copies (BANK) stored copies or prefetch heads differ from the location index
|
|
Run it on unmodified game files (`ackit status`), otherwise modified sounds show up as size/sha1 issues.
|
|
"""
|
|
import collections
|
|
import hashlib
|
|
import os
|
|
import random
|
|
|
|
from .. import banks, classify, subtitles
|
|
from ..catalog import (Catalog, available_archives, bank_sounds, character_of, hash_names, head_owners, is_shipped,
|
|
load_banks, load_characters, read_csv, scan_archive)
|
|
from ..inject import read_target
|
|
from ..paths import catalog_dir
|
|
from ..wad import Wad
|
|
|
|
FOLDER_LANG = {"ENGLISH(US)": "INT", "ITALIAN": "ITA", "FRENCH(FRANCE)": "FRA", "GERMAN": "DEU",
|
|
"SPANISH(SPAIN)": "ESN"}
|
|
|
|
|
|
class Report:
|
|
def __init__(self):
|
|
self.checked = collections.Counter()
|
|
self.issues = [] # (archive, kind, key, detail)
|
|
|
|
def add(self, archive, kind, key, detail=""):
|
|
self.issues.append((archive, kind, key, detail))
|
|
|
|
def counts(self):
|
|
return collections.Counter((a, k) for a, k, _, _ in self.issues)
|
|
|
|
@property
|
|
def ok(self):
|
|
return not self.issues
|
|
|
|
|
|
def _expected_category(s):
|
|
if s.archive == banks.ARCHIVE:
|
|
return classify.classify(s.name, bank=s.bank)
|
|
pack = None
|
|
if s.is_pack_media:
|
|
pack = hash_names().get(int(s.target.split("/")[0], 16), s.target.split("/")[0])
|
|
return classify.classify(s.name, pack=pack)
|
|
|
|
|
|
def _check_rows(rep, archive, rows, characters):
|
|
seen_t, seen_n = set(), {}
|
|
for s in rows:
|
|
rep.checked[archive] += 1
|
|
if s.archive != archive:
|
|
rep.add(archive, "language", s.key, "row says archive %s" % s.archive)
|
|
if s.target in seen_t:
|
|
rep.add(archive, "duplicate", s.key, "target")
|
|
seen_t.add(s.target)
|
|
n = (s.language, s.name.lower()) # BANK: names are unique per language
|
|
if n in seen_n or (s.language and ("", n[1]) in seen_n):
|
|
rep.add(archive, "duplicate", s.key, "name %s (also %s)" % (s.name, seen_n.get(n) or seen_n[("", n[1])]))
|
|
seen_n[n] = s.key
|
|
g, c = _expected_category(s)
|
|
if (s.group, s.category) != (g, c):
|
|
rep.add(archive, "category", s.key, "%s/%s, rules give %s/%s" % (s.group, s.category, g, c))
|
|
want = character_of(s, characters)
|
|
if s.character != want:
|
|
rep.add(archive, "character", s.key, "%r, curated list says %r" % (s.character, want))
|
|
|
|
|
|
def check_wad(rep, game, archive, rows, characters):
|
|
path = game.wad(archive)
|
|
if archive != "SFX":
|
|
folder = os.path.basename(os.path.dirname(path)).upper()
|
|
if FOLDER_LANG.get(folder, archive) != archive:
|
|
rep.add(archive, "language", archive, "WAD in folder %s" % folder)
|
|
_check_rows(rep, archive, rows, characters)
|
|
fresh = {s.target: s for s in scan_archive(archive, path, characters=characters)}
|
|
if is_shipped(game, archive): # shipped names are the game's line names (built from the banks)
|
|
byk = {s.target: s for s in rows}
|
|
for t, s in fresh.items():
|
|
if t in byk:
|
|
s.name, s.aliases = byk[t].name, byk[t].aliases
|
|
cat = {s.target: s for s in rows}
|
|
for t in sorted(set(fresh) - set(cat)):
|
|
rep.add(archive, "missing", "%s:%s" % (archive, t), fresh[t].name)
|
|
for t in sorted(set(cat) - set(fresh)):
|
|
rep.add(archive, "extra", "%s:%s" % (archive, t), cat[t].name)
|
|
wad = Wad(path)
|
|
for t in sorted(set(cat) & set(fresh)):
|
|
s = cat[t]
|
|
if s.size and s.size != fresh[t].size and not s.is_pack_media:
|
|
rep.add(archive, "size", s.key, "catalog %d, game %d" % (s.size, fresh[t].size))
|
|
if s.sha1:
|
|
h = hashlib.sha1(read_target(path, t, wad)).hexdigest()[:16]
|
|
if h != s.sha1:
|
|
rep.add(archive, "sha1", s.key, "game file differs from the catalog's original")
|
|
|
|
|
|
def check_banks(rep, game, rows, locations, characters, quick=False, sample=40, progress=None):
|
|
_check_rows(rep, banks.ARCHIVE, rows, characters)
|
|
files = banks.bank_files(game.cooked)
|
|
if quick:
|
|
pk = [f for f in files if f.lower().endswith(".upk")]
|
|
rnd = random.Random(1)
|
|
files = [f for f in files if f.startswith("SFX/")] + sorted(rnd.sample(pk, min(sample, len(pk))))
|
|
col = subtitles.Collector()
|
|
sc = banks.scan(game.cooked, progress, files=files, visit=col)
|
|
known = {n.split("#")[0] for s in rows for n in s.names()} # BANK names
|
|
for a in game.languages: # + speech line names of the catalogs
|
|
p = os.path.join(catalog_dir(), "%s.csv" % a)
|
|
if os.path.exists(p):
|
|
known.update(n.split("#")[0] for s in read_csv(p) for n in s.names())
|
|
col.resolve({n for n in sc.names | known if n.lower().startswith("speech_")})
|
|
check_subtitles(rep, game, col, set(files), quick)
|
|
owners = head_owners(game, sc)
|
|
index = collections.defaultdict(set)
|
|
for m, ls in locations.items():
|
|
for l in ls:
|
|
if l.file in set(files):
|
|
index[(l.media, l.kind)].add((l.file, l.offset, l.size, l.sound if l.kind == banks.PREFETCH_HEAD else ""))
|
|
found = collections.defaultdict(set)
|
|
for l in sc.locations:
|
|
found[(l.media, l.kind)].add((l.file, l.offset, l.size, owners.get(l.media, "") if l.kind == banks.PREFETCH_HEAD else ""))
|
|
for k in sorted(set(found) | set(index)):
|
|
if found.get(k, set()) != index.get(k, set()):
|
|
rep.add(banks.ARCHIVE, "copies", "%08X" % k[0], "%s: index %d, game %d"
|
|
% ("head" if k[1] == banks.PREFETCH_HEAD else "copies", len(index.get(k, ())), len(found.get(k, ()))))
|
|
cat = {s.target: s for s in rows}
|
|
fresh = {s.target: s for s in bank_sounds(sc, characters)}
|
|
for t in sorted(set(fresh) - set(cat)):
|
|
rep.add(banks.ARCHIVE, "missing", "BANK:" + t, fresh[t].bank)
|
|
if not quick:
|
|
for t in sorted(set(cat) - set(fresh)):
|
|
rep.add(banks.ARCHIVE, "extra", "BANK:" + t, cat[t].name)
|
|
for t in sorted(set(cat) & set(fresh)):
|
|
s, f = cat[t], fresh[t]
|
|
if s.sha1 and f.sha1 and s.sha1 != f.sha1:
|
|
rep.add(banks.ARCHIVE, "sha1", s.key, "game file differs from the catalog's original")
|
|
if f.size < s.size:
|
|
rep.add(banks.ARCHIVE, "size", s.key, "catalog %d, game %d" % (s.size, f.size))
|
|
for m, ls in locations.items():
|
|
for l in ls:
|
|
if l.kind == banks.PREFETCH_HEAD and not l.sound:
|
|
rep.add(banks.ARCHIVE, "copies", "%08X" % m, "prefetch head without a speech line")
|
|
|
|
|
|
def check_subtitles(rep, game, col, files, quick):
|
|
"""Subtitle copies found in the scanned packages vs SUB_locations + the SUB_<LANG>.csv texts."""
|
|
from ..catalog import _shipped_index, catalog_dir
|
|
loc_path = os.path.join(catalog_dir(), subtitles.LOCATIONS)
|
|
if not os.path.exists(loc_path) or not is_shipped(game, banks.ARCHIVE):
|
|
return
|
|
index = collections.defaultdict(set)
|
|
for k, ls in subtitles.read_locations(loc_path).items():
|
|
for l in ls:
|
|
if l.file in files:
|
|
index[k].add((l.file, l.pos, l.size))
|
|
found, texts = collections.defaultdict(set), {}
|
|
for speech, copies in col.dialogues.items():
|
|
for rel, d in copies:
|
|
for f in d.fields:
|
|
k = subtitles.key(f.lang, speech, f.index)
|
|
found[k].add((rel, f.pos, f.size))
|
|
texts[k] = f.text
|
|
for k in sorted(set(found) | set(index)):
|
|
if found.get(k, set()) != index.get(k, set()):
|
|
rep.add("SUB", "copies", k, "index %d, game %d" % (len(index.get(k, ())), len(found.get(k, ()))))
|
|
langs = sorted(_shipped_index().get("SUB", {}).get("lines", {}))
|
|
bank_lang = {s.key: s.language for s in read_csv(os.path.join(catalog_dir(), "BANK.csv"))}
|
|
for lang in langs:
|
|
p = os.path.join(catalog_dir(), "SUB_%s.csv" % lang)
|
|
rows = subtitles.read_texts(p, lang)
|
|
rep.checked["SUB_" + lang] += len(rows)
|
|
seen = set()
|
|
for s in rows:
|
|
k = subtitles.key(lang, s.speech, s.line)
|
|
if k in seen:
|
|
rep.add("SUB", "duplicate", k)
|
|
seen.add(k)
|
|
if k in texts and texts[k] != s.text:
|
|
rep.add("SUB", "text", k, "catalog %r, game %r" % (s.text[:40], texts[k][:40]))
|
|
if s.sound.startswith(banks.ARCHIVE + ":"):
|
|
b = bank_lang.get(s.sound)
|
|
if b is not None and b not in (lang, "INT"): # INT: languages without own voices
|
|
rep.add("SUB", "language", k, "linked to %s (%s)" % (s.sound, b or "language-neutral"))
|
|
elif s.sound and not s.sound.startswith(lang + ":") and not s.sound.startswith("INT:"):
|
|
rep.add("SUB", "language", k, "linked to %s" % s.sound)
|
|
|
|
|
|
def run(game, archives=None, quick=False, log=print, progress=None):
|
|
"""Check the given archives (default: all). Returns a Report."""
|
|
rep = Report()
|
|
archives = archives or available_archives(game)
|
|
cat = Catalog(game, archives, progress)
|
|
by_arch = collections.defaultdict(list)
|
|
for s in cat.sounds:
|
|
by_arch[s.archive].append(s)
|
|
for a in archives:
|
|
log("checking %s ..." % a)
|
|
characters = load_characters(lang=None if a == banks.ARCHIVE else a)
|
|
if a == banks.ARCHIVE:
|
|
rows = load_banks(game, progress)[0] # every language, not only the loaded one
|
|
check_banks(rep, game, rows, cat.locations, characters, quick=quick, progress=progress)
|
|
else:
|
|
check_wad(rep, game, a, by_arch[a], characters)
|
|
characters = load_characters(lang=game.language())
|
|
known = {n.lower() for s in cat.sounds for n in s.names()}
|
|
# speaker names are the same in every language; only meaningful for the build the lists were made for
|
|
if any(a in game.languages and is_shipped(game, a) for a in archives):
|
|
for n in sorted(characters):
|
|
if n not in known:
|
|
rep.add("characters.csv", "extra", n, "no sound with this name")
|
|
return rep
|