Initial commit

tools/build_catalog.py

@@ -0,0 +1,116 @@
+"""Build data/catalog/ from the installed (unmodified!) game and the curated character lists.
+
+    python tools/build_catalog.py [--game DIR]
+
+Outputs
+  data/catalog/characters.csv   name -> verified speaker, from data/catalog/source/<lang>/*.txt
+  data/catalog/<ARCHIVE>.csv    every sound of one archive (schema: data/catalog/README.md)
+  data/catalog/index.json       WAD fingerprints the CSVs were built from + counts
+Run it only on clean game files (`ackit status` must say nothing is modified).
+"""
+import argparse
+import collections
+import csv
+import hashlib
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from ackit import settings  # noqa: E402
+from ackit.catalog import scan_archive, write_csv, hash_names  # noqa: E402
+from ackit.classify import USER_LABELS  # noqa: E402
+from ackit.game import Game, default_game_dir, wad_fingerprint  # noqa: E402
+from ackit.inject import read_target  # noqa: E402
+from ackit.paths import catalog_dir  # noqa: E402
+from ackit.wad import Wad  # noqa: E402
+
+SKIP = {"files.txt", "stats.txt", "checkpoint.txt"}
+
+
+def read_user_lists(src_dir):
+    """{user file name without extension (lower): set(display character)}"""
+    out = collections.defaultdict(set)
+    for f in sorted(os.listdir(src_dir)):
+        if not f.lower().endswith(".txt") or f.lower() in SKIP:
+            continue
+        label = os.path.splitext(f)[0]
+        who = USER_LABELS.get(label, label)
+        for line in open(os.path.join(src_dir, f), encoding="utf-8", errors="replace"):
+            n = line.strip()
+            while os.path.splitext(n)[1].lower() in (".wem", ".wav"):
+                n = os.path.splitext(n)[0]
+            if n:
+                out[n.lower()].add(who)
+    return out
+
+
+def build_characters(sounds, src_root):
+    """Map the user's wemscan-style names onto catalog names (name or alias)."""
+    by_alias = collections.defaultdict(list)
+    for s in sounds:
+        for a in s.names():
+            by_alias[a.lower()].append(s)
+    rows, unmatched, ambiguous = {}, [], []
+    for lang in sorted(os.listdir(src_root)):
+        d = os.path.join(src_root, lang)
+        if not os.path.isdir(d):
+            continue
+        for uname, who in read_user_lists(d).items():
+            hits = list({id(s): s for s in by_alias.get(uname, [])}.values())
+            if not hits:
+                unmatched.append(uname)
+                continue
+            if len(hits) > 1:
+                ambiguous.append(uname)  # wemscan name collision: both candidates get the label
+            for s in hits:
+                cur = rows.setdefault(s.name, set())
+                cur |= who
+    return rows, unmatched, ambiguous
+
+
+def file_sha1(path, target, wad):
+    return hashlib.sha1(read_target(path, target, wad)).hexdigest()[:16]
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--game")
+    a = ap.parse_args()
+    game = Game(a.game or default_game_dir(settings.load()))
+    out_dir = catalog_dir()
+    src_root = os.path.join(out_dir, "source")
+    hash_names()
+
+    # characters from the curated lists (names come from the language they were made for: ITA)
+    ref_lang = "ITA" if "ITA" in game.archives else game.languages[0]
+    ref = scan_archive(ref_lang, game.wad(ref_lang), characters={})
+    chars, unmatched, ambiguous = build_characters(ref, src_root)
+    with open(os.path.join(out_dir, "characters.csv"), "w", encoding="utf-8", newline="") as f:
+        w = csv.writer(f, lineterminator="\n")
+        w.writerow(["name", "character"])
+        for n in sorted(chars, key=str.lower):
+            w.writerow([n, "; ".join(sorted(chars[n]))])
+    print("characters.csv: %d sounds labelled; %d unmatched names %s; %d ambiguous %s"
+          % (len(chars), len(unmatched), unmatched[:3], len(ambiguous), ambiguous[:3]))
+    charmap = {n.lower(): v for n, v in ((n, "; ".join(sorted(c))) for n, c in chars.items())}
+
+    index = {"built_from": "unmodified game files", "archives": {}}
+    for arch, path in game.archives.items():
+        sounds = scan_archive(arch, path, characters=charmap)
+        wad = Wad(path)
+        for s in sounds:
+            s.sha1 = file_sha1(path, s.target, wad)
+        write_csv(os.path.join(out_dir, "%s.csv" % arch), sounds)
+        g = collections.Counter(s.group for s in sounds)
+        index["archives"][arch] = {"wad": game.rel(path), "fingerprint": wad_fingerprint(path),
+                                   "sounds": len(sounds), "groups": dict(g),
+                                   "with_character": sum(1 for s in sounds if s.character)}
+        print("%s.csv: %d sounds %s" % (arch, len(sounds), dict(g)))
+    # flatten for catalog.load_archive
+    flat = {a: v for a, v in index["archives"].items()}
+    json.dump(dict(flat, _info=index["built_from"]), open(os.path.join(out_dir, "index.json"), "w", newline="\n"), indent=1)
+
+
+if __name__ == "__main__":
+    main()