583 lines
27 KiB
Python
583 lines
27 KiB
Python
"""Index of every sound in the game's WAD archives, with names, categories and characters
|
|
(sounds stored inside soundbanks are not indexed yet: docs/formats/banks.md).
|
|
|
|
Sources, in order:
|
|
1. data/catalog/<ARCHIVE>.csv -- shipped catalog (instant) when the WAD fingerprint matches;
|
|
2. otherwise a header scan of the WAD (a few seconds, cached in the user cache), classified with
|
|
`classify` and labelled with data/catalog/characters.csv (speaker per sound name).
|
|
Names come from the game: a WAD entry ID is the Wwise ID (FNV-1) of a soundbank listed in the game's
|
|
CookedPCConsole/CookedBanks.txt (`bank_names`); an entry with one WEM is NAME
|
|
(alias NAME_1), entries with n variations are NAME_1..NAME_n (alias NAME#i); stream-pack media are
|
|
named by their ID or <Pack>_<ID>.
|
|
"""
|
|
import collections
|
|
import csv
|
|
import difflib
|
|
import json
|
|
import os
|
|
import re
|
|
import struct
|
|
from dataclasses import dataclass, asdict
|
|
|
|
from . import banks, classify, subtitles
|
|
from . import lines as lines_mod
|
|
from .game import wad_fingerprint
|
|
from .bnk import fnv1
|
|
from .lines import bank_langs
|
|
from .inject import pack_bnk_path
|
|
from .pack import Pack
|
|
from .paths import catalog_dir, user_dir
|
|
from .wad import Wad
|
|
|
|
FIELDS = ["archive", "target", "name", "aliases", "group", "category", "bank", "language", "line", "character",
|
|
"characters_inferred", "seconds", "rate", "channels", "size", "sha1", "notes"]
|
|
SCAN_VERSION = 3
|
|
|
|
|
|
@dataclass
|
|
class Sound:
|
|
archive: str
|
|
target: str
|
|
name: str
|
|
aliases: tuple = ()
|
|
group: str = "Unknown"
|
|
category: str = "Unknown"
|
|
bank: str = "" # BANK sounds: the soundbank holding it (name); WAD sounds: ''
|
|
language: str = "" # BANK sounds: language(s) of a localized speech line, ';'-separated ('' = every language)
|
|
line: str = "" # speech: the game's identity of the line in every language (ackit/lines.py)
|
|
character: str = "" # verified speaker(s), '; '-separated (curated catalog)
|
|
characters_inferred: str = "" # characters/roles mentioned by the name (hint)
|
|
seconds: float = 0.0
|
|
rate: int = 0
|
|
channels: int = 0
|
|
size: int = 0 # size of the ORIGINAL WEM (bytes) when known from the catalog
|
|
sha1: str = "" # first 16 hex digits of the original WEM's SHA-1 ('' if unknown)
|
|
notes: str = ""
|
|
|
|
@property
|
|
def key(self):
|
|
return "%s:%s" % (self.archive, self.target)
|
|
|
|
@property
|
|
def is_pack_media(self):
|
|
return "/" in self.target
|
|
|
|
@property
|
|
def limit(self):
|
|
"""Max replacement size in bytes (None = any size)."""
|
|
return None if self.is_pack_media else self.size
|
|
|
|
def names(self):
|
|
return (self.name,) + tuple(self.aliases)
|
|
|
|
@property
|
|
def languages(self):
|
|
"""Voice languages of a localized in-memory sound (() = every language)."""
|
|
return tuple(self.language.split(";")) if self.language else ()
|
|
|
|
def in_language(self, lang):
|
|
return not self.language or lang in self.languages
|
|
|
|
|
|
# --- static data ---------------------------------------------------------------------------
|
|
_names = {} # cooked folder -> {Wwise ID: bank name}
|
|
|
|
|
|
def bank_names(cooked):
|
|
"""{Wwise ID: name} of the game's soundbanks, from its own CookedBanks.txt (ID = FNV-1 of the name). They
|
|
name the WAD entries (each entry holds the streamed media of one bank) and the stream packs."""
|
|
key = os.path.normcase(os.path.abspath(cooked))
|
|
if key not in _names:
|
|
out = {}
|
|
p = os.path.join(cooked, "CookedBanks.txt")
|
|
if os.path.exists(p):
|
|
for ln in open(p, encoding="latin-1"):
|
|
n = ln.strip()
|
|
if n:
|
|
out.setdefault(fnv1(n), n)
|
|
_names[key] = out
|
|
return _names[key]
|
|
|
|
|
|
def cooked_of(wad_path):
|
|
"""CookedPCConsole folder of a WAD (WADs are in CookedPCConsole/<language or SFX>/)."""
|
|
return os.path.dirname(os.path.dirname(os.path.abspath(wad_path)))
|
|
|
|
|
|
def load_characters(path=None, lang=None):
|
|
"""{sound name (lower): 'Speaker' or 'A; B'} from data/catalog/characters.csv: the rows valid in every
|
|
language (keyed by line names, the same line everywhere) plus the rows of `lang` (a sound of that
|
|
language only; none at present)."""
|
|
path = path or os.path.join(catalog_dir(), "characters.csv")
|
|
out = {}
|
|
if os.path.exists(path):
|
|
for row in csv.DictReader(open(path, encoding="utf-8")):
|
|
if not row.get("language") or row["language"] == lang:
|
|
out[row["name"].lower()] = row["character"]
|
|
return out
|
|
|
|
|
|
def character_of(s, characters):
|
|
"""Verified speaker of a sound from a load_characters() map, by its name only: names are unique in a
|
|
language and a line has the same name in every language, while aliases (positions) are local."""
|
|
return characters.get(s.name.lower(), "")
|
|
|
|
|
|
def read_csv(path):
|
|
out = []
|
|
for r in csv.DictReader(open(path, encoding="utf-8", newline="")):
|
|
out.append(Sound(archive=r["archive"], target=r["target"], name=r["name"],
|
|
aliases=tuple(a for a in r["aliases"].split("|") if a), group=r["group"],
|
|
category=r["category"], bank=r.get("bank") or "", language=r.get("language") or "",
|
|
line=r.get("line") or "",
|
|
character=r["character"],
|
|
characters_inferred=r["characters_inferred"], seconds=float(r["seconds"] or 0),
|
|
rate=int(r["rate"] or 0), channels=int(r["channels"] or 0), size=int(r["size"] or 0),
|
|
sha1=r["sha1"], notes=r["notes"]))
|
|
return out
|
|
|
|
|
|
def write_csv(path, sounds):
|
|
with open(path, "w", encoding="utf-8", newline="") as f:
|
|
w = csv.DictWriter(f, FIELDS, lineterminator="\n")
|
|
w.writeheader()
|
|
for s in sounds:
|
|
d = asdict(s)
|
|
d["aliases"] = "|".join(s.aliases)
|
|
d["seconds"] = "%.3f" % s.seconds
|
|
w.writerow({k: d[k] for k in FIELDS})
|
|
|
|
|
|
# --- scanning ------------------------------------------------------------------------------
|
|
def _hdr(b):
|
|
"""(rate, channels, seconds) from the first bytes of a WEM (fmt is always the first chunk)."""
|
|
if len(b) >= 0x30 and b[:4] == b"RIFF" and b[12:16] == b"fmt ":
|
|
ch, rate = struct.unpack_from("<HI", b, 22)
|
|
frames = struct.unpack_from("<I", b, 44)[0]
|
|
return rate, ch, (frames / float(rate) if rate else 0.0)
|
|
return 0, 0, 0.0
|
|
|
|
|
|
def _walk_entry(f, offset, size):
|
|
"""Offsets/lengths of the WEMs inside one WAD entry, reading only headers."""
|
|
out = []
|
|
pos = 0
|
|
while pos + 12 <= size:
|
|
f.seek(offset + pos)
|
|
h = f.read(0x30)
|
|
if h[:4] != b"RIFF" or h[8:12] != b"WAVE":
|
|
break
|
|
ln = 8 + struct.unpack_from("<I", h, 4)[0]
|
|
out.append((pos, ln, h))
|
|
# next WEM starts at the first non-zero byte after this one (zero padding in between)
|
|
cand = pos + ln
|
|
f.seek(offset + cand)
|
|
nxt = None
|
|
while cand < size and nxt is None:
|
|
chunk = f.read(min(0x10000, size - cand))
|
|
if not chunk:
|
|
break
|
|
stripped = chunk.lstrip(b"\0")
|
|
if stripped:
|
|
nxt = cand + len(chunk) - len(stripped)
|
|
cand += len(chunk)
|
|
if nxt is None:
|
|
break
|
|
pos = nxt
|
|
return out
|
|
|
|
|
|
def scan_archive(archive, wad_path, characters=None, progress=None):
|
|
"""Header scan of one WAD -> [Sound]."""
|
|
names = bank_names(cooked_of(wad_path))
|
|
characters = characters if characters is not None else load_characters(lang=archive)
|
|
wad = Wad(wad_path)
|
|
out = []
|
|
with open(wad_path, "rb") as f:
|
|
for n, e in enumerate(wad.entries):
|
|
if progress and n % 200 == 0:
|
|
progress(n, len(wad.entries))
|
|
bnk = pack_bnk_path(wad_path, e.hash)
|
|
if bnk and open(bnk, "rb").read(4) == b"BKPK":
|
|
pk = Pack(open(bnk, "rb").read())
|
|
pname = names.get(e.hash, "%08X" % e.hash)
|
|
for mid, size, off in pk.entries:
|
|
f.seek(e.offset + off)
|
|
rate, ch, secs = _hdr(f.read(0x30))
|
|
out.append(Sound(archive, "%08X/%08X" % (e.hash, mid), names.get(mid, "%s_%08X" % (pname, mid)),
|
|
seconds=secs, rate=rate, channels=ch, size=size))
|
|
out[-1].group, out[-1].category = classify.classify(out[-1].name, pack=pname)
|
|
continue
|
|
wems = _walk_entry(f, e.offset, e.size)
|
|
base = names.get(e.hash, "%08X" % e.hash)
|
|
for i, (off, ln, h) in enumerate(wems):
|
|
rate, ch, secs = _hdr(h)
|
|
if len(wems) == 1:
|
|
s = Sound(archive, "%08X" % e.hash, base, (base + "_1",))
|
|
else:
|
|
s = Sound(archive, "%08X:%d" % (e.hash, i), "%s_%d" % (base, i + 1), ("%s#%d" % (base, i + 1),))
|
|
s.seconds, s.rate, s.channels, s.size = secs, rate, ch, ln
|
|
s.group, s.category = classify.classify(s.name)
|
|
out.append(s)
|
|
# a variation named NAME_i can collide with an entry whose own name is NAME_i: the variation is
|
|
# shown as NAME#i and keeps NAME_i as alias (both stay findable, display names stay unique)
|
|
taken = collections.Counter(s.name.lower() for s in out)
|
|
for s in out:
|
|
if taken[s.name.lower()] > 1 and ":" in s.target:
|
|
s.name, s.aliases = s.aliases[0], (s.name,)
|
|
for s in out:
|
|
s.characters_inferred = "; ".join(classify.infer_characters(s.name))
|
|
s.character = character_of(s, characters)
|
|
return out
|
|
|
|
|
|
_CAMEL = re.compile(r"(?<=[a-z])(?=[A-Z])")
|
|
|
|
|
|
def bank_sounds(sc, characters=None, reserved=()):
|
|
"""[Sound] of the in-memory sounds found by `banks.scan` (prefetch heads belong to WAD sounds). `reserved`:
|
|
lower-case names taken by other archives (sfx_names)."""
|
|
characters = characters if characters is not None else load_characters()
|
|
names = banks.media_names(sc, reserved)
|
|
every = set().union(*(e.get("langs", set()) for e in sc.media.values())) - {""}
|
|
out = []
|
|
for m, e in sc.media.items():
|
|
if banks.PREFETCH_HEAD in e["kinds"]:
|
|
continue
|
|
name, bank = names[m]
|
|
rate, ch, secs = _hdr(e["head"])
|
|
ls = bank_langs(sc, m) # localized: the languages whose banks hold it
|
|
s = Sound(banks.ARCHIVE, "%08X" % m, name, ("%s/%08X" % (bank, m),), bank=bank,
|
|
language="" if set(ls) == every else ";".join(ls), seconds=secs, rate=rate, channels=ch,
|
|
size=min(l.size for l in sc.locations if l.media == m) if len(e["sha1"]) > 1 else e["size"],
|
|
sha1=next(iter(e["sha1"])) if len(e["sha1"]) == 1 else "")
|
|
s.group, s.category = classify.classify(name, bank=bank)
|
|
s.characters_inferred = "; ".join(classify.infer_characters(_CAMEL.sub("_", name + "_" + bank)))
|
|
s.character = character_of(s, characters)
|
|
out.append(s)
|
|
out.sort(key=lambda s: (s.bank.lower(), s.name.lower()))
|
|
return out
|
|
|
|
|
|
def build_lines(game, sc):
|
|
"""The game's speech lines across languages (ackit/lines.py)."""
|
|
return lines_mod.build(game, sc, _walk_entry)
|
|
|
|
|
|
def scan_banks(game, progress=None, characters=None, visit=None):
|
|
"""Full scan of the game's banks -> ([Sound], [Location] with WAD keys on prefetch heads, Scan, Lines).
|
|
Localized in-memory speech lines carry their line identity and the name shared by every language."""
|
|
sc = banks.scan(game.cooked, progress, visit=visit)
|
|
snd, locs, lines = derive_banks(game, sc, characters)
|
|
return snd, locs, sc, lines
|
|
|
|
|
|
def derive_banks(game, sc, characters=None):
|
|
"""(bank sounds, locations, Lines) from a Scan: the lines, the prefetch heads' lines, the in-memory sounds
|
|
with their line identities and names."""
|
|
lines = build_lines(game, sc)
|
|
mark_heads(sc, whole_recordings(sc, lines))
|
|
locs = [l._replace(sound=lines.head_key(l)) if l.kind == banks.PREFETCH_HEAD else l for l in sc.locations]
|
|
snd = bank_sounds(sc, characters, sfx_names(game))
|
|
lines.names = line_names(game, sc, lines, {s.key: s.name for s in snd})
|
|
apply_lines(snd, lines, lines.names)
|
|
return snd, locs, lines
|
|
|
|
|
|
def whole_recordings(sc, lines):
|
|
"""Media stored in memory that are WAD recordings of their bank's language (the same media ID, identical
|
|
bytes: 49 in the game, e.g. game-over taunts). Like prefetch heads, these copies belong to the WAD line
|
|
and are written with it; they are not sounds of their own."""
|
|
keys = collections.defaultdict(list)
|
|
for l in sc.locations:
|
|
if l.kind == banks.MEMORY:
|
|
k = lines.head_key(l)
|
|
keys[l.media].append(bool(k) and not k.startswith(banks.ARCHIVE + ":"))
|
|
return {m for m, wad in keys.items() if all(wad)}
|
|
|
|
|
|
def mark_heads(sc, media):
|
|
"""Treat the bank copies of `media` as heads of WAD recordings (see whole_recordings)."""
|
|
if not media:
|
|
return
|
|
for m in media:
|
|
if m in sc.media:
|
|
sc.media[m]["kinds"] = {banks.PREFETCH_HEAD}
|
|
sc.prefetch.add(m)
|
|
sc.locations = [l._replace(kind=banks.PREFETCH_HEAD) if l.media in media else l for l in sc.locations]
|
|
|
|
|
|
def sfx_names(game):
|
|
"""Lower-case names and aliases of the SFX WAD's sounds: in-memory sounds of the same bank must not take
|
|
them (`<Bank>_<n>` is the position name of the bank's streamed sounds too)."""
|
|
if "SFX" not in game.archives:
|
|
return set()
|
|
return {n.lower() for s in scan_archive("SFX", game.wad("SFX"), characters={}) for n in s.names()}
|
|
|
|
|
|
def line_names(game, sc, lines, bank_names=None):
|
|
"""{line id: name} (lines.line_names). Lines no event names keep their English recording's name: the
|
|
position-based name in the INT WAD (`NAME_i`, or `NAME#i` where that is taken) or the bank-numbered name."""
|
|
pos = {}
|
|
for L in {k.split(":")[0] for per in lines.members.values() for k in per.values()} - {banks.ARCHIVE}:
|
|
for s in scan_archive(L, game.wad(L), characters={}):
|
|
pos[s.key] = s
|
|
bank_names = bank_names if bank_names is not None else {}
|
|
|
|
def fallback(k):
|
|
return pos[k].name if k in pos else bank_names.get(k, k)
|
|
|
|
out = lines_mod.line_names(sc, lines, fallback)
|
|
taken = collections.Counter(n.lower() for n in out.values())
|
|
for lid, n in out.items(): # a fallback name equal to an event-based one -> NAME#i of the INT WEM
|
|
if taken[n.lower()] > 1:
|
|
k = lines.members[lid].get("INT") or next(iter(lines.members[lid].values()))
|
|
alt = [a for a in (pos[k].names() if k in pos else ()) if "#" in a]
|
|
if alt and alt[0] != n:
|
|
out[lid] = alt[0]
|
|
return out
|
|
|
|
|
|
def apply_lines(sounds, lines, names):
|
|
"""Give every speech sound of one archive (a WAD language or BANK) its line identity and the line's name,
|
|
the same in every language. A WAD sound keeps its position-based names as aliases (NAME#i: position in
|
|
this language's WAD; single-WEM entries: the entry name). Names stay unique in the archive (and, for
|
|
BANK, per language together with the language-neutral sounds): a clash is resolved by the sound that
|
|
is not the line's own name falling back to its position name / media-ID name."""
|
|
for s in sounds:
|
|
lid = lines.line_of.get(s.key)
|
|
if not lid:
|
|
continue
|
|
s.line = lid
|
|
if s.key in lines.copy_of: # a second place of a recording: keeps its position name, written with it
|
|
continue
|
|
name = names[lid]
|
|
if s.archive != banks.ARCHIVE:
|
|
if ":" in s.target:
|
|
s.aliases = tuple(a for a in s.names() if "#" in a)
|
|
else:
|
|
s.aliases = (s.name,) + tuple(a for a in s.aliases if a != s.name)
|
|
if name in s.aliases:
|
|
s.aliases = tuple(a for a in s.aliases if a != name)
|
|
s.name = name
|
|
neutral, by_lang = set(), collections.defaultdict(set) # lower-case names taken
|
|
|
|
def clash(s, n):
|
|
if s.archive != banks.ARCHIVE or not s.languages:
|
|
return n in neutral or any(n in v for v in by_lang.values())
|
|
return n in neutral or any(n in by_lang[L] for L in s.languages)
|
|
|
|
# a line's own recordings first: they keep the line's name in a clash (copies and others give way)
|
|
for s in sorted(sounds, key=lambda s: (not s.line or s.key in lines.copy_of, s.key)):
|
|
if clash(s, s.name.lower()):
|
|
if s.archive == banks.ARCHIVE:
|
|
s.name = "%s_%s" % (s.name, s.target)
|
|
else:
|
|
alt = [a for a in s.aliases if "#" in a and a != s.name]
|
|
s.aliases, s.name = (s.name,), alt[0] if alt else "%s@%s" % (s.name, s.target)
|
|
if s.archive == banks.ARCHIVE and s.languages:
|
|
for L in s.languages:
|
|
by_lang[L].add(s.name.lower())
|
|
else:
|
|
neutral.add(s.name.lower())
|
|
for s in sounds:
|
|
if s.archive != banks.ARCHIVE:
|
|
s.group, s.category = classify.classify(s.name)
|
|
s.characters_inferred = "; ".join(classify.infer_characters(s.name))
|
|
return sounds
|
|
|
|
|
|
def subtitle_links(sc, lines, speech_names):
|
|
"""{speech event name: {language: sound keys}}: event -> actions -> containers -> Sound objects (the same
|
|
in every language) -> that language's recording of each line (a WAD line or a localized in-memory one).
|
|
An event that plays several lines (two versions) links its subtitle to all of them: 'k1;k2', in order."""
|
|
ev = banks.event_objects(sc)
|
|
out = {}
|
|
for n in speech_names:
|
|
keys = collections.defaultdict(list)
|
|
for o in ev.get(n, ()):
|
|
for L, k in lines.obj_keys.get(o, {}).items():
|
|
if k not in keys[L]:
|
|
keys[L].append(k)
|
|
if keys:
|
|
out[n] = {L: ";".join(ks) for L, ks in keys.items()}
|
|
return out
|
|
|
|
|
|
def scan_text(game, progress=None, characters=None):
|
|
"""One pass over every package: (bank sounds, bank locations, Scan, {lang: [SubLine]}, [SubLoc], issues,
|
|
Lines)."""
|
|
col = subtitles.Collector()
|
|
sc = banks.scan(game.cooked, progress, visit=col)
|
|
names = {n for n in sc.names if n.lower().startswith("speech_")}
|
|
names |= {n for n, _ in banks.media_names(sc).values() if n.lower().startswith("speech_")}
|
|
lookup = [n for n in col.dialogues if n.startswith("#lookup:")]
|
|
sc.subtitle_objects = {"dialogue_objects": sum(len(v) for n, v in col.dialogues.items() if n not in lookup),
|
|
"dialogue_lines": len(col.dialogues) - len(lookup),
|
|
"scripted_objects": sum(len(col.dialogues[n]) for n in lookup)}
|
|
col.resolve(names)
|
|
# the subtitle objects name their speech events too: an event ID is the FNV-1 hash of its name, so these
|
|
# names find events no package name table lists (e.g. Vicki Vale's radio reports)
|
|
sc.names.update(col.dialogues)
|
|
snd, locs, lines = derive_banks(game, sc, characters)
|
|
links = subtitle_links(sc, lines, list(col.dialogues))
|
|
texts, sublocs = subtitles.rows(col, links)
|
|
return snd, locs, sc, texts, sublocs, col.issues, lines
|
|
|
|
|
|
def load_subtitles(game, lang, progress=None):
|
|
"""([SubLine], {SUB key: [SubLoc]}) of one language: shipped catalog when the game matches it, else a
|
|
cached or fresh scan of the packages (several minutes)."""
|
|
fp = banks.game_fingerprint(game.cooked)
|
|
shipped = _shipped_index().get("SUB", {})
|
|
t = os.path.join(catalog_dir(), "SUB_%s.csv" % lang)
|
|
loc = os.path.join(catalog_dir(), subtitles.LOCATIONS)
|
|
if shipped.get("fingerprint") == fp and os.path.exists(t) and os.path.exists(loc):
|
|
return subtitles.read_texts(t, lang), subtitles.read_locations(loc, lang)
|
|
cache = os.path.join(user_dir("cache"), "scan_SUB_%s_v%d" % (fp, SCAN_VERSION))
|
|
if not os.path.exists(cache + ".locations.csv.gz"):
|
|
_, _, _, texts, sublocs, _, _ = scan_text(game, progress)
|
|
for lg, rows in texts.items():
|
|
subtitles.write_texts(cache + "_%s.csv" % lg, rows)
|
|
subtitles.write_locations(cache + ".locations.csv.gz", sublocs)
|
|
t = cache + "_%s.csv" % lang
|
|
return (subtitles.read_texts(t, lang) if os.path.exists(t) else []), \
|
|
subtitles.read_locations(cache + ".locations.csv.gz", lang)
|
|
|
|
|
|
def load_banks(game, progress=None):
|
|
"""([Sound], {media: [Location]}) of the in-memory sounds: shipped catalog when the game's bank set
|
|
matches it, else a cached or fresh scan (several minutes)."""
|
|
fp = banks.game_fingerprint(game.cooked)
|
|
shipped = _shipped_index().get(banks.ARCHIVE, {})
|
|
csv_path = os.path.join(catalog_dir(), "%s.csv" % banks.ARCHIVE)
|
|
loc_path = os.path.join(catalog_dir(), banks.LOCATIONS)
|
|
if shipped.get("fingerprint") == fp and os.path.exists(csv_path) and os.path.exists(loc_path):
|
|
return read_csv(csv_path), banks.read_locations(loc_path)
|
|
cache = os.path.join(user_dir("cache"), "scan_%s_%s_v%d" % (banks.ARCHIVE, fp, SCAN_VERSION))
|
|
if not (os.path.exists(cache + ".csv") and os.path.exists(cache + ".locations.csv.gz")):
|
|
sounds, locs, _, lines = scan_banks(game, progress)
|
|
for s in sounds:
|
|
s.sha1 = "" # unknown build: no trusted original checksums
|
|
# the speech WADs of this build with the lines found now (load_archive reads these caches)
|
|
for a in game.languages:
|
|
snd = apply_lines(scan_archive(a, game.wad(a), progress=progress), lines, lines.names)
|
|
write_csv(_wad_cache(game, a), snd)
|
|
banks.write_locations(cache + ".locations.csv.gz", locs)
|
|
write_csv(cache + ".csv", sounds)
|
|
return read_csv(cache + ".csv"), banks.read_locations(cache + ".locations.csv.gz")
|
|
|
|
|
|
# --- catalog -------------------------------------------------------------------------------
|
|
def _shipped_index():
|
|
p = os.path.join(catalog_dir(), "index.json")
|
|
try:
|
|
return json.load(open(p))
|
|
except (OSError, ValueError):
|
|
return {}
|
|
|
|
|
|
def is_shipped(game, archive):
|
|
"""True if the game's archive matches the shipped catalog (not a scan of another build)."""
|
|
shipped = _shipped_index().get(archive, {})
|
|
if archive == banks.ARCHIVE:
|
|
return shipped.get("fingerprint") == banks.game_fingerprint(game.cooked)
|
|
return shipped.get("fingerprint") == wad_fingerprint(game.wad(archive))
|
|
|
|
|
|
def _wad_cache(game, archive):
|
|
return os.path.join(user_dir("cache"), "scan_%s_%s_v%d.csv" % (archive, wad_fingerprint(game.wad(archive)), SCAN_VERSION))
|
|
|
|
|
|
def load_archive(game, archive, progress=None):
|
|
"""[Sound] for one archive of a game (shipped CSV, user-cache scan, or fresh scan)."""
|
|
path = game.wad(archive)
|
|
fp = wad_fingerprint(path)
|
|
shipped = _shipped_index().get(archive, {})
|
|
csv_path = os.path.join(catalog_dir(), "%s.csv" % archive)
|
|
if shipped.get("fingerprint") == fp and os.path.exists(csv_path):
|
|
return read_csv(csv_path)
|
|
cache = _wad_cache(game, archive)
|
|
if os.path.exists(cache):
|
|
return read_csv(cache)
|
|
sounds = scan_archive(archive, path, progress=progress)
|
|
# unknown build: no original checksums (states then rely on the originals store only)
|
|
write_csv(cache, sounds)
|
|
return sounds
|
|
|
|
|
|
def available_archives(game):
|
|
"""WAD archives of the game plus BANK when it has soundbanks."""
|
|
return list(game.archives) + ([banks.ARCHIVE] if banks.has_banks(game.cooked) else [])
|
|
|
|
|
|
class Catalog:
|
|
"""All sounds of the selected archives (default: the game's speech language, SFX and BANK)."""
|
|
|
|
def __init__(self, game, archives=None, progress=None):
|
|
self.game = game
|
|
avail = available_archives(game)
|
|
self.archives = archives or [a for a in (game.language(), "SFX", banks.ARCHIVE) if a in avail]
|
|
self.sounds = []
|
|
self.locations = {} # BANK media id -> [Location]
|
|
self.heads = {} # WAD sound key -> [Location] of its prefetch heads
|
|
bank_sounds_ = []
|
|
if banks.ARCHIVE in avail:
|
|
# always loaded: speech lines may have prefetch heads that must be written with them
|
|
bank_sounds_, self.locations = load_banks(game, progress)
|
|
for ls in self.locations.values():
|
|
for l in ls:
|
|
if l.kind == banks.PREFETCH_HEAD and l.sound:
|
|
self.heads.setdefault(l.sound, []).append(l)
|
|
# localized in-memory speech: only the languages loaded here (all of them if none is)
|
|
langs = [a for a in self.archives if a in game.languages] or None
|
|
bank_sounds_ = [s for s in bank_sounds_ if not langs or any(s.in_language(L) for L in langs)]
|
|
for a in self.archives:
|
|
self.sounds += bank_sounds_ if a == banks.ARCHIVE else load_archive(game, a, progress)
|
|
self.by_key = {s.key: s for s in self.sounds}
|
|
# copies: one recording stored in two places of a WAD (same line, same archive); written together
|
|
same = collections.defaultdict(list)
|
|
for s in self.sounds:
|
|
if s.line and s.archive != banks.ARCHIVE:
|
|
same[(s.archive, s.line)].append(s)
|
|
self.copies = {s.key: [c for c in ss if c is not s] for ss in same.values() if len(ss) > 1 for s in ss}
|
|
self._by_name = {}
|
|
for s in self.sounds:
|
|
for n in s.names():
|
|
self._by_name.setdefault(n.lower(), []).append(s)
|
|
|
|
def get(self, key):
|
|
return self.by_key[key]
|
|
|
|
def resolve(self, name):
|
|
"""One sound by name, alias, key ('ITA:B49CDF48:11') or file name ('X_3.wav')."""
|
|
if name in self.by_key:
|
|
return self.by_key[name]
|
|
key = os.path.splitext(os.path.basename(name))[0].lower() if re.search(r"\.\w{2,4}$", name) else name.lower()
|
|
hits = self._by_name.get(key, [])
|
|
uniq = list({id(s): s for s in hits}.values())
|
|
if len(uniq) > 1:
|
|
# a name wins over another sound's alias: a line named after its INT position (NAME#i) can equal
|
|
# the position alias of another recording in another language's WAD
|
|
own = [s for s in uniq if s.name.lower() == key]
|
|
uniq = own if len(own) == 1 else uniq
|
|
if len(uniq) == 1:
|
|
return uniq[0]
|
|
if len(uniq) > 1:
|
|
raise LookupError("'%s' is ambiguous; use one of: %s" % (name, ", ".join(
|
|
next((a for a in s.names() if a.lower() != key), s.key) for s in uniq)))
|
|
near = [s.name for s in self.search(key, limit=8)] or self.suggest(key)
|
|
raise LookupError("no sound named '%s'.%s" % (name, ("\n did you mean: " + ", ".join(near)) if near else ""))
|
|
|
|
def search(self, text, limit=None):
|
|
"""All words must appear in the name, category or character (case-insensitive)."""
|
|
words = [w for w in re.split(r"[\s_]+", text.lower()) if w]
|
|
res = [s for s in self.sounds
|
|
if all(w in s.name.lower() or w in s.category.lower() or w in s.character.lower() for w in words)]
|
|
res.sort(key=lambda s: (len(s.name), s.name))
|
|
return res[:limit] if limit else res
|
|
|
|
def suggest(self, text, n=8):
|
|
names = {s.name.lower(): s.name for s in self.sounds}
|
|
return [names[k] for k in difflib.get_close_matches(text.lower(), list(names), n=n, cutoff=0.6)]
|