gora/UnheardCity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

List and edit in-memory sounds in soundbanks

Browse Source
This commit is contained in:
Cristian Gora
2026-10-09 20:12:10 +02:00
parent 61bbf26f3b
commit 5d6c2447f7
36 changed files with 71568 additions and 54823 deletions
Download Patch File Download Diff File Expand all files Collapse all files

124
ackit/catalog.py
View File

@@ -9,6 +9,7 @@ Names follow the community hash list (ackit/data/hashlist.txt): an entry with on
(alias NAME_1), entries with n variations are NAME_1..NAME_n (alias NAME#i); stream-pack media are
named by their ID or <Pack>_<ID>.
"""
import collections
import csv
import difflib
import json
@@ -17,16 +18,16 @@ import re
import struct
from dataclasses import dataclass, asdict
from . import classify
from . import banks, classify
from .game import wad_fingerprint
from .inject import pack_bnk_path
from .pack import Pack
from .paths import catalog_dir, package_data, user_dir
from .wad import Wad
FIELDS = ["archive", "target", "name", "aliases", "group", "category", "character",
FIELDS = ["archive", "target", "name", "aliases", "group", "category", "bank", "character",
"characters_inferred", "seconds", "rate", "channels", "size", "sha1", "notes"]
SCAN_VERSION = 1
SCAN_VERSION = 2
@dataclass
@@ -37,6 +38,7 @@ class Sound:
aliases: tuple = ()
group: str = "Unknown"
category: str = "Unknown"
bank: str = "" # BANK sounds: the soundbank holding it (name); WAD sounds: ''
character: str = "" # verified speaker(s), '; '-separated (curated catalog)
characters_inferred: str = "" # characters/roles mentioned by the name (hint)
seconds: float = 0.0
@@ -96,7 +98,7 @@ def read_csv(path):
for r in csv.DictReader(open(path, encoding="utf-8", newline="")):
out.append(Sound(archive=r["archive"], target=r["target"], name=r["name"],
aliases=tuple(a for a in r["aliases"].split("|") if a), group=r["group"],
category=r["category"], character=r["character"],
category=r["category"], bank=r.get("bank") or "", character=r["character"],
characters_inferred=r["characters_inferred"], seconds=float(r["seconds"] or 0),
rate=int(r["rate"] or 0), channels=int(r["channels"] or 0), size=int(r["size"] or 0),
sha1=r["sha1"], notes=r["notes"]))
@@ -185,12 +187,96 @@ def scan_archive(archive, wad_path, characters=None, progress=None):
s.seconds, s.rate, s.channels, s.size = secs, rate, ch, ln
s.group, s.category = classify.classify(s.name)
out.append(s)
# a variation named NAME_i can collide with an entry whose own name is NAME_i: the variation is
# shown as NAME#i and keeps NAME_i as alias (both stay findable, display names stay unique)
taken = collections.Counter(s.name.lower() for s in out)
for s in out:
if taken[s.name.lower()] > 1 and ":" in s.target:
s.name, s.aliases = s.aliases[0], (s.name,)
for s in out:
s.characters_inferred = "; ".join(classify.infer_characters(s.name))
s.character = next((characters[a.lower()] for a in s.names() if a.lower() in characters), "")
return out
_CAMEL = re.compile(r"(?<=[a-z])(?=[A-Z])")
def bank_sounds(sc, characters=None):
"""[Sound] of the in-memory sounds found by `banks.scan` (prefetch heads belong to WAD sounds)."""
characters = characters if characters is not None else load_characters()
names = banks.media_names(sc)
first = {}
for loc in sorted(sc.locations, key=lambda l: (l.media, l.file, l.offset)):
first.setdefault(loc.media, loc)
out = []
for m, e in sc.media.items():
if banks.PREFETCH_HEAD in e["kinds"]:
continue
name, bank = names[m]
rate, ch, secs = _hdr(e["head"])
s = Sound(banks.ARCHIVE, "%08X" % m, name, ("%s/%08X" % (bank, m),), bank=bank,
seconds=secs, rate=rate, channels=ch,
size=min(l.size for l in sc.locations if l.media == m) if len(e["sha1"]) > 1 else e["size"],
sha1=next(iter(e["sha1"])) if len(e["sha1"]) == 1 else "")
s.group, s.category = classify.classify(name, bank=bank)
s.characters_inferred = "; ".join(classify.infer_characters(_CAMEL.sub("_", name + "_" + bank)))
s.character = next((characters[a.lower()] for a in s.names() if a.lower() in characters), "")
out.append(s)
out.sort(key=lambda s: (s.bank.lower(), s.name.lower()))
return out
def head_owners(game, sc, archives=None):
"""{prefetch media id: WAD sound key} -- the speech line (one language) each prefetch head belongs to,
found through the per-language BKPK tables of the packages (docs/formats/banks.md)."""
want = {m for m, e in sc.media.items() if banks.PREFETCH_HEAD in e["kinds"]}
out = {}
for a in archives or [x for x in game.archives if x != "SFX"]:
wad = Wad(game.wad(a))
by_hash = {e.hash: e for e in wad.entries}
with open(game.wad(a), "rb") as f:
for ent, table in sc.packs.items():
mine = [m for m in table if m in want and m not in out]
e = by_hash.get(ent)
if not mine or not e:
continue
wems = _walk_entry(f, e.offset, e.size)
for m in mine:
off, size = table[m]
for i, (o, ln, _) in enumerate(wems):
if (o, ln) == (off, size):
out[m] = "%s:%08X" % (a, ent) + (":%d" % i if len(wems) > 1 else "")
return out
def scan_banks(game, progress=None, characters=None):
"""Full scan of the game's banks -> ([Sound], [Location] with WAD keys on prefetch heads, Scan)."""
sc = banks.scan(game.cooked, progress)
owners = head_owners(game, sc)
locs = [l._replace(sound=owners.get(l.media, "")) if l.kind == banks.PREFETCH_HEAD else l for l in sc.locations]
return bank_sounds(sc, characters), locs, sc
def load_banks(game, progress=None):
"""([Sound], {media: [Location]}) of the in-memory sounds: shipped catalog when the game's bank set
matches it, else a cached or fresh scan (several minutes)."""
fp = banks.game_fingerprint(game.cooked)
shipped = _shipped_index().get(banks.ARCHIVE, {})
csv_path = os.path.join(catalog_dir(), "%s.csv" % banks.ARCHIVE)
loc_path = os.path.join(catalog_dir(), banks.LOCATIONS)
if shipped.get("fingerprint") == fp and os.path.exists(csv_path) and os.path.exists(loc_path):
return read_csv(csv_path), banks.read_locations(loc_path)
cache = os.path.join(user_dir("cache"), "scan_%s_%s_v%d" % (banks.ARCHIVE, fp, SCAN_VERSION))
if not (os.path.exists(cache + ".csv") and os.path.exists(cache + ".locations.csv.gz")):
sounds, locs, _ = scan_banks(game, progress)
for s in sounds:
s.sha1 = "" # unknown build: no trusted original checksums
banks.write_locations(cache + ".locations.csv.gz", locs)
write_csv(cache + ".csv", sounds)
return read_csv(cache + ".csv"), banks.read_locations(cache + ".locations.csv.gz")
# --- catalog -------------------------------------------------------------------------------
def _shipped_index():
p = os.path.join(catalog_dir(), "index.json")
@@ -200,6 +286,14 @@ def _shipped_index():
return {}
def is_shipped(game, archive):
"""True if the game's archive matches the shipped catalog (not a scan of another build)."""
shipped = _shipped_index().get(archive, {})
if archive == banks.ARCHIVE:
return shipped.get("fingerprint") == banks.game_fingerprint(game.cooked)
return shipped.get("fingerprint") == wad_fingerprint(game.wad(archive))
def load_archive(game, archive, progress=None):
"""[Sound] for one archive of a game (shipped CSV, user-cache scan, or fresh scan)."""
path = game.wad(archive)
@@ -217,15 +311,31 @@ def load_archive(game, archive, progress=None):
return sounds
def available_archives(game):
"""WAD archives of the game plus BANK when it has soundbanks."""
return list(game.archives) + ([banks.ARCHIVE] if banks.has_banks(game.cooked) else [])
class Catalog:
"""All sounds of the selected archives (default: the game's speech language + SFX)."""
"""All sounds of the selected archives (default: the game's speech language, SFX and BANK)."""
def __init__(self, game, archives=None, progress=None):
self.game = game
self.archives = archives or [a for a in (game.language(), "SFX") if a in game.archives]
avail = available_archives(game)
self.archives = archives or [a for a in (game.language(), "SFX", banks.ARCHIVE) if a in avail]
self.sounds = []
self.locations = {} # BANK media id -> [Location]
self.heads = {} # WAD sound key -> [Location] of its prefetch heads
bank_sounds_ = []
if banks.ARCHIVE in avail:
# always loaded: speech lines may have prefetch heads that must be written with them
bank_sounds_, self.locations = load_banks(game, progress)
for ls in self.locations.values():
for l in ls:
if l.kind == banks.PREFETCH_HEAD and l.sound:
self.heads.setdefault(l.sound, []).append(l)
for a in self.archives:
self.sounds += load_archive(game, a, progress)
self.sounds += bank_sounds_ if a == banks.ARCHIVE else load_archive(game, a, progress)
self.by_key = {s.key: s for s in self.sounds}
self._by_name = {}
for s in self.sounds: