List and edit in-memory sounds in soundbanks
This commit is contained in:
159
ackit/verify/catalogcheck.py
Normal file
159
ackit/verify/catalogcheck.py
Normal file
@@ -0,0 +1,159 @@
|
||||
"""Cross-check the catalog against the installed game (`ackit verify --catalog`).
|
||||
|
||||
For every archive the catalog in use (shipped CSV or the user-cache scan) is compared with a fresh scan:
|
||||
missing a sound of the game that is not in the catalog
|
||||
extra a catalog row that is not in the game
|
||||
duplicate a target or a name used twice
|
||||
size/sha1 the catalog's original size or checksum differs from the game (game modified or other build)
|
||||
category the row's group/category is not what the classification rules give for its name
|
||||
character the verified speaker differs from data/catalog/characters.csv
|
||||
language the archive code does not match the WAD's language folder
|
||||
copies (BANK) stored copies or prefetch heads differ from the location index
|
||||
Run it on unmodified game files (`ackit status`), otherwise modified sounds show up as size/sha1 issues.
|
||||
"""
|
||||
import collections
|
||||
import hashlib
|
||||
import os
|
||||
import random
|
||||
|
||||
from .. import banks, classify
|
||||
from ..catalog import (Catalog, available_archives, bank_sounds, hash_names, head_owners, is_shipped, load_characters,
|
||||
scan_archive)
|
||||
from ..inject import read_target
|
||||
from ..wad import Wad
|
||||
|
||||
FOLDER_LANG = {"ENGLISH(US)": "INT", "ITALIAN": "ITA", "FRENCH(FRANCE)": "FRA", "GERMAN": "DEU",
|
||||
"SPANISH(SPAIN)": "ESN"}
|
||||
|
||||
|
||||
class Report:
|
||||
def __init__(self):
|
||||
self.checked = collections.Counter()
|
||||
self.issues = [] # (archive, kind, key, detail)
|
||||
|
||||
def add(self, archive, kind, key, detail=""):
|
||||
self.issues.append((archive, kind, key, detail))
|
||||
|
||||
def counts(self):
|
||||
return collections.Counter((a, k) for a, k, _, _ in self.issues)
|
||||
|
||||
@property
|
||||
def ok(self):
|
||||
return not self.issues
|
||||
|
||||
|
||||
def _expected_category(s):
|
||||
if s.archive == banks.ARCHIVE:
|
||||
return classify.classify(s.name, bank=s.bank)
|
||||
pack = None
|
||||
if s.is_pack_media:
|
||||
pack = hash_names().get(int(s.target.split("/")[0], 16), s.target.split("/")[0])
|
||||
return classify.classify(s.name, pack=pack)
|
||||
|
||||
|
||||
def _check_rows(rep, archive, rows, characters):
|
||||
seen_t, seen_n = set(), {}
|
||||
for s in rows:
|
||||
rep.checked[archive] += 1
|
||||
if s.archive != archive:
|
||||
rep.add(archive, "language", s.key, "row says archive %s" % s.archive)
|
||||
if s.target in seen_t:
|
||||
rep.add(archive, "duplicate", s.key, "target")
|
||||
seen_t.add(s.target)
|
||||
if s.name.lower() in seen_n:
|
||||
rep.add(archive, "duplicate", s.key, "name %s (also %s)" % (s.name, seen_n[s.name.lower()]))
|
||||
seen_n[s.name.lower()] = s.key
|
||||
g, c = _expected_category(s)
|
||||
if (s.group, s.category) != (g, c):
|
||||
rep.add(archive, "category", s.key, "%s/%s, rules give %s/%s" % (s.group, s.category, g, c))
|
||||
want = next((characters[a.lower()] for a in s.names() if a.lower() in characters), "")
|
||||
if s.character != want:
|
||||
rep.add(archive, "character", s.key, "%r, curated list says %r" % (s.character, want))
|
||||
|
||||
|
||||
def check_wad(rep, game, archive, rows, characters):
|
||||
path = game.wad(archive)
|
||||
if archive != "SFX":
|
||||
folder = os.path.basename(os.path.dirname(path)).upper()
|
||||
if FOLDER_LANG.get(folder, archive) != archive:
|
||||
rep.add(archive, "language", archive, "WAD in folder %s" % folder)
|
||||
_check_rows(rep, archive, rows, characters)
|
||||
fresh = {s.target: s for s in scan_archive(archive, path, characters=characters)}
|
||||
cat = {s.target: s for s in rows}
|
||||
for t in sorted(set(fresh) - set(cat)):
|
||||
rep.add(archive, "missing", "%s:%s" % (archive, t), fresh[t].name)
|
||||
for t in sorted(set(cat) - set(fresh)):
|
||||
rep.add(archive, "extra", "%s:%s" % (archive, t), cat[t].name)
|
||||
wad = Wad(path)
|
||||
for t in sorted(set(cat) & set(fresh)):
|
||||
s = cat[t]
|
||||
if s.size and s.size != fresh[t].size and not s.is_pack_media:
|
||||
rep.add(archive, "size", s.key, "catalog %d, game %d" % (s.size, fresh[t].size))
|
||||
if s.sha1:
|
||||
h = hashlib.sha1(read_target(path, t, wad)).hexdigest()[:16]
|
||||
if h != s.sha1:
|
||||
rep.add(archive, "sha1", s.key, "game file differs from the catalog's original")
|
||||
|
||||
|
||||
def check_banks(rep, game, rows, locations, characters, quick=False, sample=40, progress=None):
|
||||
_check_rows(rep, banks.ARCHIVE, rows, characters)
|
||||
files = banks.bank_files(game.cooked)
|
||||
if quick:
|
||||
pk = [f for f in files if f.lower().endswith(".upk")]
|
||||
rnd = random.Random(1)
|
||||
files = [f for f in files if f.startswith("SFX/")] + sorted(rnd.sample(pk, min(sample, len(pk))))
|
||||
sc = banks.scan(game.cooked, progress, files=files)
|
||||
owners = head_owners(game, sc)
|
||||
index = collections.defaultdict(set)
|
||||
for m, ls in locations.items():
|
||||
for l in ls:
|
||||
if l.file in set(files):
|
||||
index[(l.media, l.kind)].add((l.file, l.offset, l.size, l.sound if l.kind == banks.PREFETCH_HEAD else ""))
|
||||
found = collections.defaultdict(set)
|
||||
for l in sc.locations:
|
||||
found[(l.media, l.kind)].add((l.file, l.offset, l.size, owners.get(l.media, "") if l.kind == banks.PREFETCH_HEAD else ""))
|
||||
for k in sorted(set(found) | set(index)):
|
||||
if found.get(k, set()) != index.get(k, set()):
|
||||
rep.add(banks.ARCHIVE, "copies", "%08X" % k[0], "%s: index %d, game %d"
|
||||
% ("head" if k[1] == banks.PREFETCH_HEAD else "copies", len(index.get(k, ())), len(found.get(k, ()))))
|
||||
cat = {s.target: s for s in rows}
|
||||
fresh = {s.target: s for s in bank_sounds(sc, characters)}
|
||||
for t in sorted(set(fresh) - set(cat)):
|
||||
rep.add(banks.ARCHIVE, "missing", "BANK:" + t, fresh[t].bank)
|
||||
if not quick:
|
||||
for t in sorted(set(cat) - set(fresh)):
|
||||
rep.add(banks.ARCHIVE, "extra", "BANK:" + t, cat[t].name)
|
||||
for t in sorted(set(cat) & set(fresh)):
|
||||
s, f = cat[t], fresh[t]
|
||||
if s.sha1 and f.sha1 and s.sha1 != f.sha1:
|
||||
rep.add(banks.ARCHIVE, "sha1", s.key, "game file differs from the catalog's original")
|
||||
if f.size < s.size:
|
||||
rep.add(banks.ARCHIVE, "size", s.key, "catalog %d, game %d" % (s.size, f.size))
|
||||
for m, ls in locations.items():
|
||||
for l in ls:
|
||||
if l.kind == banks.PREFETCH_HEAD and not l.sound:
|
||||
rep.add(banks.ARCHIVE, "copies", "%08X" % m, "prefetch head without a speech line")
|
||||
|
||||
|
||||
def run(game, archives=None, quick=False, log=print, progress=None):
|
||||
"""Check the given archives (default: all). Returns a Report."""
|
||||
rep = Report()
|
||||
characters = load_characters()
|
||||
archives = archives or available_archives(game)
|
||||
cat = Catalog(game, archives, progress)
|
||||
by_arch = collections.defaultdict(list)
|
||||
for s in cat.sounds:
|
||||
by_arch[s.archive].append(s)
|
||||
for a in archives:
|
||||
log("checking %s ..." % a)
|
||||
if a == banks.ARCHIVE:
|
||||
check_banks(rep, game, by_arch[a], cat.locations, characters, quick=quick, progress=progress)
|
||||
else:
|
||||
check_wad(rep, game, a, by_arch[a], characters)
|
||||
known = {n.lower() for s in cat.sounds for n in s.names()}
|
||||
# speaker names are the same in every language; only meaningful for the build the lists were made for
|
||||
if any(a in game.languages and is_shipped(game, a) for a in archives):
|
||||
for n in sorted(characters):
|
||||
if n not in known:
|
||||
rep.add("characters.csv", "extra", n, "no sound with this name")
|
||||
return rep
|
||||
Reference in New Issue
Block a user