138 lines
6.3 KiB
Python
138 lines
6.3 KiB
Python
"""Build data/catalog/ from the installed (unmodified!) game and the curated character lists.
|
|
|
|
python tools/build_catalog.py [--game DIR]
|
|
|
|
Outputs
|
|
data/catalog/characters.csv name -> verified speaker, from data/catalog/source/<lang>/*.txt
|
|
data/catalog/<ARCHIVE>.csv every sound of one archive (schema: data/catalog/README.md)
|
|
data/catalog/BANK.csv in-memory sounds (soundbanks in SFX/*.bnk and the .upk packages)
|
|
data/catalog/BANK_locations.csv.gz every stored copy of them + the prefetch heads of speech lines
|
|
data/catalog/index.json fingerprints the CSVs were built from + counts
|
|
Run it only on clean game files (`ackit status` must say nothing is modified).
|
|
"""
|
|
import argparse
|
|
import collections
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
from ackit import banks, settings # noqa: E402
|
|
from ackit.catalog import scan_archive, scan_banks, write_csv, hash_names # noqa: E402
|
|
from ackit.classify import USER_LABELS # noqa: E402
|
|
from ackit.game import Game, default_game_dir, wad_fingerprint # noqa: E402
|
|
from ackit.inject import read_target # noqa: E402
|
|
from ackit.paths import catalog_dir # noqa: E402
|
|
from ackit.wad import Wad # noqa: E402
|
|
|
|
SKIP = {"files.txt", "stats.txt", "checkpoint.txt"}
|
|
|
|
|
|
def read_user_lists(src_dir):
|
|
"""{user file name without extension (lower): set(display character)}"""
|
|
out = collections.defaultdict(set)
|
|
for f in sorted(os.listdir(src_dir)):
|
|
if not f.lower().endswith(".txt") or f.lower() in SKIP:
|
|
continue
|
|
label = os.path.splitext(f)[0]
|
|
who = USER_LABELS.get(label, label)
|
|
for line in open(os.path.join(src_dir, f), encoding="utf-8", errors="replace"):
|
|
n = line.strip()
|
|
while os.path.splitext(n)[1].lower() in (".wem", ".wav"):
|
|
n = os.path.splitext(n)[0]
|
|
if n:
|
|
out[n.lower()].add(who)
|
|
return out
|
|
|
|
|
|
def build_characters(sounds, src_root):
|
|
"""Map the user's wemscan-style names onto catalog names (name or alias)."""
|
|
by_alias = collections.defaultdict(list)
|
|
for s in sounds:
|
|
for a in s.names():
|
|
by_alias[a.lower()].append(s)
|
|
rows, unmatched, ambiguous = {}, [], []
|
|
for lang in sorted(os.listdir(src_root)):
|
|
d = os.path.join(src_root, lang)
|
|
if not os.path.isdir(d):
|
|
continue
|
|
for uname, who in read_user_lists(d).items():
|
|
hits = list({id(s): s for s in by_alias.get(uname, [])}.values())
|
|
if not hits:
|
|
unmatched.append(uname)
|
|
continue
|
|
if len(hits) > 1:
|
|
ambiguous.append(uname) # wemscan name collision: both candidates get the label
|
|
for s in hits:
|
|
cur = rows.setdefault(s.name, set())
|
|
cur |= who
|
|
return rows, unmatched, ambiguous
|
|
|
|
|
|
def file_sha1(path, target, wad):
|
|
return hashlib.sha1(read_target(path, target, wad)).hexdigest()[:16]
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--game")
|
|
a = ap.parse_args()
|
|
game = Game(a.game or default_game_dir(settings.load()))
|
|
out_dir = catalog_dir()
|
|
src_root = os.path.join(out_dir, "source")
|
|
hash_names()
|
|
|
|
# characters from the curated lists (names come from the language they were made for: ITA)
|
|
ref_lang = "ITA" if "ITA" in game.archives else game.languages[0]
|
|
ref = scan_archive(ref_lang, game.wad(ref_lang), characters={})
|
|
chars, unmatched, ambiguous = build_characters(ref, src_root)
|
|
with open(os.path.join(out_dir, "characters.csv"), "w", encoding="utf-8", newline="") as f:
|
|
w = csv.writer(f, lineterminator="\n")
|
|
w.writerow(["name", "character"])
|
|
for n in sorted(chars, key=str.lower):
|
|
w.writerow([n, "; ".join(sorted(chars[n]))])
|
|
print("characters.csv: %d sounds labelled; %d unmatched names %s; %d ambiguous %s"
|
|
% (len(chars), len(unmatched), unmatched[:3], len(ambiguous), ambiguous[:3]))
|
|
charmap = {n.lower(): v for n, v in ((n, "; ".join(sorted(c))) for n, c in chars.items())}
|
|
|
|
index = {"built_from": "unmodified game files", "archives": {}}
|
|
for arch, path in game.archives.items():
|
|
sounds = scan_archive(arch, path, characters=charmap)
|
|
wad = Wad(path)
|
|
for s in sounds:
|
|
s.sha1 = file_sha1(path, s.target, wad)
|
|
write_csv(os.path.join(out_dir, "%s.csv" % arch), sounds)
|
|
g = collections.Counter(s.group for s in sounds)
|
|
index["archives"][arch] = {"wad": game.rel(path), "fingerprint": wad_fingerprint(path),
|
|
"sounds": len(sounds), "groups": dict(g),
|
|
"with_character": sum(1 for s in sounds if s.character)}
|
|
print("%s.csv: %d sounds %s" % (arch, len(sounds), dict(g)))
|
|
|
|
# in-memory sounds: loose SFX\*.bnk + banks inside the .upk packages (a few minutes)
|
|
t0 = time.time()
|
|
sounds, locs, sc = scan_banks(game, progress=lambda k, n: print(" banks: %d/%d files" % (k, n), flush=True)
|
|
if k % 500 == 0 else None, characters=charmap)
|
|
write_csv(os.path.join(out_dir, "%s.csv" % banks.ARCHIVE), sounds)
|
|
banks.write_locations(os.path.join(out_dir, banks.LOCATIONS), locs)
|
|
g = collections.Counter(s.group for s in sounds)
|
|
heads = [l for l in locs if l.kind == banks.PREFETCH_HEAD]
|
|
index["archives"][banks.ARCHIVE] = {
|
|
"fingerprint": banks.game_fingerprint(game.cooked), "sounds": len(sounds), "groups": dict(g),
|
|
"copies": sum(1 for l in locs if l.kind == banks.MEMORY), "files": len(sc.files),
|
|
"named_by_event": sum(1 for s in sounds if not s.name.lower().startswith(s.bank.lower() + "_")),
|
|
"prefetch_heads": {"lines": len({l.media for l in heads}), "copies": len(heads),
|
|
"unmapped": len({l.media for l in heads if not l.sound})},
|
|
"issues": ["%s: %s" % i for i in sc.issues]}
|
|
print("%s.csv: %d sounds %s, %d copies in %d files, %d prefetch heads (%.0fs)"
|
|
% (banks.ARCHIVE, len(sounds), dict(g), len(locs) - len(heads), len(sc.files), len(heads), time.time() - t0))
|
|
# flatten for catalog.load_archive
|
|
flat = {a: v for a, v in index["archives"].items()}
|
|
json.dump(dict(flat, _info=index["built_from"]), open(os.path.join(out_dir, "index.json"), "w", newline="\n"), indent=1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|