"""Build data/catalog/ from the installed (unmodified!) game and the curated character lists. python tools/build_catalog.py [--game DIR] Outputs data/catalog/characters.csv name -> verified speaker, from data/catalog/source//*.txt data/catalog/.csv every sound of one archive (schema: data/catalog/README.md) data/catalog/BANK.csv in-memory sounds (soundbanks in SFX/*.bnk and the .upk packages) data/catalog/BANK_locations.csv.gz every stored copy of them + the prefetch heads of speech lines data/catalog/index.json fingerprints the CSVs were built from + counts Run it only on clean game files (`ackit status` must say nothing is modified). """ import argparse import collections import csv import hashlib import json import os import sys import time sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from ackit import banks, settings # noqa: E402 from ackit.catalog import scan_archive, scan_banks, write_csv, hash_names # noqa: E402 from ackit.classify import USER_LABELS # noqa: E402 from ackit.game import Game, default_game_dir, wad_fingerprint # noqa: E402 from ackit.inject import read_target # noqa: E402 from ackit.paths import catalog_dir # noqa: E402 from ackit.wad import Wad # noqa: E402 SKIP = {"files.txt", "stats.txt", "checkpoint.txt"} def read_user_lists(src_dir): """{user file name without extension (lower): set(display character)}""" out = collections.defaultdict(set) for f in sorted(os.listdir(src_dir)): if not f.lower().endswith(".txt") or f.lower() in SKIP: continue label = os.path.splitext(f)[0] who = USER_LABELS.get(label, label) for line in open(os.path.join(src_dir, f), encoding="utf-8", errors="replace"): n = line.strip() while os.path.splitext(n)[1].lower() in (".wem", ".wav"): n = os.path.splitext(n)[0] if n: out[n.lower()].add(who) return out def build_characters(sounds, src_root): """Map the user's wemscan-style names onto catalog names (name or alias).""" by_alias = collections.defaultdict(list) for s in sounds: for a in s.names(): by_alias[a.lower()].append(s) rows, unmatched, ambiguous = {}, [], [] for lang in sorted(os.listdir(src_root)): d = os.path.join(src_root, lang) if not os.path.isdir(d): continue for uname, who in read_user_lists(d).items(): hits = list({id(s): s for s in by_alias.get(uname, [])}.values()) if not hits: unmatched.append(uname) continue if len(hits) > 1: ambiguous.append(uname) # wemscan name collision: both candidates get the label for s in hits: cur = rows.setdefault(s.name, set()) cur |= who return rows, unmatched, ambiguous def file_sha1(path, target, wad): return hashlib.sha1(read_target(path, target, wad)).hexdigest()[:16] def main(): ap = argparse.ArgumentParser() ap.add_argument("--game") a = ap.parse_args() game = Game(a.game or default_game_dir(settings.load())) out_dir = catalog_dir() src_root = os.path.join(out_dir, "source") hash_names() # characters from the curated lists (names come from the language they were made for: ITA) ref_lang = "ITA" if "ITA" in game.archives else game.languages[0] ref = scan_archive(ref_lang, game.wad(ref_lang), characters={}) chars, unmatched, ambiguous = build_characters(ref, src_root) with open(os.path.join(out_dir, "characters.csv"), "w", encoding="utf-8", newline="") as f: w = csv.writer(f, lineterminator="\n") w.writerow(["name", "character"]) for n in sorted(chars, key=str.lower): w.writerow([n, "; ".join(sorted(chars[n]))]) print("characters.csv: %d sounds labelled; %d unmatched names %s; %d ambiguous %s" % (len(chars), len(unmatched), unmatched[:3], len(ambiguous), ambiguous[:3])) charmap = {n.lower(): v for n, v in ((n, "; ".join(sorted(c))) for n, c in chars.items())} index = {"built_from": "unmodified game files", "archives": {}} for arch, path in game.archives.items(): sounds = scan_archive(arch, path, characters=charmap) wad = Wad(path) for s in sounds: s.sha1 = file_sha1(path, s.target, wad) write_csv(os.path.join(out_dir, "%s.csv" % arch), sounds) g = collections.Counter(s.group for s in sounds) index["archives"][arch] = {"wad": game.rel(path), "fingerprint": wad_fingerprint(path), "sounds": len(sounds), "groups": dict(g), "with_character": sum(1 for s in sounds if s.character)} print("%s.csv: %d sounds %s" % (arch, len(sounds), dict(g))) # in-memory sounds: loose SFX\*.bnk + banks inside the .upk packages (a few minutes) t0 = time.time() sounds, locs, sc = scan_banks(game, progress=lambda k, n: print(" banks: %d/%d files" % (k, n), flush=True) if k % 500 == 0 else None, characters=charmap) write_csv(os.path.join(out_dir, "%s.csv" % banks.ARCHIVE), sounds) banks.write_locations(os.path.join(out_dir, banks.LOCATIONS), locs) g = collections.Counter(s.group for s in sounds) heads = [l for l in locs if l.kind == banks.PREFETCH_HEAD] index["archives"][banks.ARCHIVE] = { "fingerprint": banks.game_fingerprint(game.cooked), "sounds": len(sounds), "groups": dict(g), "copies": sum(1 for l in locs if l.kind == banks.MEMORY), "files": len(sc.files), "named_by_event": sum(1 for s in sounds if not s.name.lower().startswith(s.bank.lower() + "_")), "prefetch_heads": {"lines": len({l.media for l in heads}), "copies": len(heads), "unmapped": len({l.media for l in heads if not l.sound})}, "issues": ["%s: %s" % i for i in sc.issues]} print("%s.csv: %d sounds %s, %d copies in %d files, %d prefetch heads (%.0fs)" % (banks.ARCHIVE, len(sounds), dict(g), len(locs) - len(heads), len(sc.files), len(heads), time.time() - t0)) # flatten for catalog.load_archive flat = {a: v for a, v in index["archives"].items()} json.dump(dict(flat, _info=index["built_from"]), open(os.path.join(out_dir, "index.json"), "w", newline="\n"), indent=1) if __name__ == "__main__": main()