Initial commit

ackit/verify/debugger.py

@@ -0,0 +1,183 @@
+"""Windows only. Unattended game run under cdb (WinDbg console debugger) with logging breakpoints.
+
+Launch method (tested, see NOTES.md): with Steam running, BatmanAC.exe can be started directly
+with SteamAppId=200260 in the environment (no relaunch through Steam). SteamStub decrypts .text
+at runtime, so we wait until a known code signature is readable, then attach cdb non-intrusively
+except for int3 breakpoints placed on already-decrypted code.
+"""
+import ctypes
+import ctypes.wintypes as wt
+import os
+import re
+import subprocess
+import time
+
+APPID = 200260
+
+
+def find_cdb():
+    """x86 cdb.exe from the WinDbg Store package (its path changes on every auto-update)."""
+    if os.environ.get("ACKIT_CDB"):
+        return os.environ["ACKIT_CDB"]
+    out = subprocess.run(["powershell", "-NoProfile", "-Command",
+                          "(Get-AppxPackage Microsoft.WinDbg).InstallLocation"],
+                         capture_output=True, text=True).stdout.strip()
+    for base in out.splitlines()[::-1]:
+        p = os.path.join(base.strip(), "x86", "cdb.exe")
+        if os.path.exists(p):
+            return p
+    for p in (r"C:\Program Files (x86)\Windows Kits\10\Debuggers\x86\cdb.exe",):
+        if os.path.exists(p):
+            return p
+    raise FileNotFoundError("x86 cdb.exe not found (install WinDbg or set ACKIT_CDB)")
+
+# Runtime addresses (image base 0x400000; ASLR off for this exe). See docs/decoder_re.md.
+ADDR = dict(
+    get_or_create_setup=0x010B3400,   # __thiscall(cache, AkSrcInfo*, ...) ; info+0x9c hash
+    parse_setup=0x010B4D10,           # WwiseVorbisParseSetupHeader(info, channels, reader, pool) -> 0 ok
+    parse_setup_ret=0x010B351A,       # in get_or_create_setup, right after the call (eax = result)
+    sig_site=0x010B347B,              # mov ebx,[ebp+0x94]  (decryption probe)
+)
+SIG = bytes.fromhex("8B9D94000000")
+
+k32 = ctypes.WinDLL("kernel32", use_last_error=True)
+PROCESS_VM_READ = 0x0010
+PROCESS_QUERY_INFORMATION = 0x0400
+k32.OpenProcess.restype = wt.HANDLE
+k32.ReadProcessMemory.argtypes = [wt.HANDLE, wt.LPCVOID, wt.LPVOID, ctypes.c_size_t, ctypes.POINTER(ctypes.c_size_t)]
+
+
+def read_mem(pid, addr, n):
+    h = k32.OpenProcess(PROCESS_VM_READ | PROCESS_QUERY_INFORMATION, False, pid)
+    if not h:
+        return None
+    try:
+        buf = ctypes.create_string_buffer(n)
+        got = ctypes.c_size_t()
+        if not k32.ReadProcessMemory(h, ctypes.c_void_p(addr), buf, n, ctypes.byref(got)):
+            return None
+        return buf.raw[:got.value]
+    finally:
+        k32.CloseHandle(h)
+
+
+u32 = ctypes.WinDLL("user32", use_last_error=True)
+WM_KEYDOWN, WM_KEYUP = 0x0100, 0x0101
+VK = dict(enter=0x0D, space=0x20, esc=0x1B, up=0x26, down=0x28, left=0x25, right=0x27, e=0x45)
+
+
+def game_windows(pid):
+    """Top-level windows of the game process."""
+    found = []
+    cb_t = ctypes.WINFUNCTYPE(wt.BOOL, wt.HWND, wt.LPARAM)
+
+    def cb(hwnd, _):
+        p = wt.DWORD()
+        u32.GetWindowThreadProcessId(hwnd, ctypes.byref(p))
+        if p.value == pid and u32.IsWindowVisible(hwnd):
+            found.append(hwnd)
+        return True
+    u32.EnumWindows(cb_t(cb), 0)
+    return found
+
+
+def post_key(pid, key, hold=0.08):
+    """Post a key press to the game's window only (never global SendInput)."""
+    vk = VK[key] if isinstance(key, str) else key
+    sc = u32.MapVirtualKeyW(vk, 0)
+    n = 0
+    for hwnd in game_windows(pid):
+        u32.PostMessageW(hwnd, WM_KEYDOWN, vk, 1 | (sc << 16))
+        time.sleep(hold)
+        u32.PostMessageW(hwnd, WM_KEYUP, vk, 1 | (sc << 16) | (3 << 30))
+        n += 1
+    return n
+
+
+def running_game_pids():
+    out = subprocess.run(["tasklist", "/FI", "IMAGENAME eq BatmanAC.exe", "/FO", "CSV", "/NH"],
+                         capture_output=True, text=True).stdout
+    return [int(m.group(1)) for m in re.finditer(r'"BatmanAC\.exe","(\d+)"', out)]
+
+
+def kill_game():
+    for pid in running_game_pids():
+        subprocess.run(["taskkill", "/F", "/PID", str(pid)], capture_output=True)
+
+
+def launch_game(exe, extra_args=()):
+    """Start BatmanAC.exe directly (Steam must be running; SteamAppId makes steam_api accept it)."""
+    env = dict(os.environ, SteamAppId=str(APPID), SteamGameId=str(APPID))
+    return subprocess.Popen([exe] + list(extra_args), cwd=os.path.dirname(exe), env=env)
+
+
+def wait_decrypted(pid, timeout=60):
+    t0 = time.time()
+    while time.time() - t0 < timeout:
+        if read_mem(pid, ADDR["sig_site"], len(SIG)) == SIG:
+            return time.time() - t0
+        time.sleep(0.05)
+    raise TimeoutError("code never decrypted (signature not found)")
+
+
+def cdb_script(extra_bps="", dump_info=False):
+    """Debugger command file: log codec setup creation + parse result; log 2nd-chance crashes."""
+    a = ADDR
+    dump = "dd poi(@esp+4) L0x30; " if dump_info else ""
+    return "\n".join([
+        ".echotime",
+        "sxd av",
+        'sxd -c2 ".echo ACKIT_CRASH; .echotime; r; kv 30; .ecxr; q" av',
+        "sxn ch", "sxn sse", "sxn eh", "sxn clr", "sxn ld", "sxn ud",
+        "sxi ibp",
+        # stream codec creation: log the AkVorbisInfo fields of the stream being opened
+        'bp %#x ".printf \\"ACKIT_SETUP hash=%%08x frames=%%u alloc=%%u ch=%%u rate=%%u\\\\n\\", '
+        'poi(poi(@esp+4)+0x9c), poi(poi(@esp+4)+0x78), poi(poi(@esp+4)+0x94), '
+        'wo(poi(@esp+4)+0x5a), poi(poi(@esp+4)+0x5c); %sgc"' % (a["get_or_create_setup"], dump),
+        # setup parse result (0 = OK)
+        'bp %#x ".printf \\"ACKIT_PARSE ret=%%x\\\\n\\", @eax; gc"' % a["parse_setup_ret"],
+        extra_bps,
+        ".echo ACKIT_ATTACHED",
+        "g",
+    ])
+
+
+class Run:
+    def __init__(self, exe, log_path, script_extra="", dump_info=False):
+        self.exe = exe
+        self.log_path = log_path
+        self.script_extra = script_extra
+        self.dump_info = dump_info
+        self.game = None
+        self.dbg = None
+
+    def start(self):
+        kill_game()
+        self.t0 = time.time()
+        self.game = launch_game(self.exe)
+        pid = self.game.pid
+        dt = wait_decrypted(pid)
+        script = self.log_path + ".cdbcmds"
+        open(script, "w").write(cdb_script(self.script_extra, self.dump_info))
+        self.dbg = subprocess.Popen([find_cdb(), "-p", str(pid), "-pd", "-logo", self.log_path,
+                                     "-y", "", "-cf", script],
+                                    stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
+                                    stderr=subprocess.DEVNULL)
+        return dict(pid=pid, decrypt_s=dt)
+
+    def alive(self):
+        return self.game.poll() is None
+
+    def stop(self):
+        kill_game()
+        if self.dbg:
+            try:
+                self.dbg.wait(15)
+            except subprocess.TimeoutExpired:
+                self.dbg.kill()
+
+    def log(self):
+        try:
+            return open(self.log_path, encoding="utf-8", errors="replace").read()
+        except FileNotFoundError:
+            return ""

ackit/verify/ingame.py

@@ -0,0 +1,121 @@
+"""Automated in-game test (Windows): does a replaced sound actually play?
+
+Flow: apply the replacement through `Workspace.apply` (the same code path as the editor) ->
+start the game under cdb with logging breakpoints on the decoder + record the speakers (WASAPI
+loopback) -> check: no crash, the codec setup of the injected WEM was created and parsed OK, and the
+injected audio is heard -> kill the game -> revert and check the files are byte-identical again.
+
+Default target: SFX music EDF036D6/393BC53A (intro/title music), which the game starts ~5 s after
+launch without input. That slot ignores sources shorter than ~10 s, so probes are 30 s long.
+Only correlation scores of the recording are kept (the loopback may contain other apps' audio).
+"""
+import hashlib
+import json
+import os
+import re
+import time
+
+import numpy as np
+
+from .. import audio
+from ..originals import sha256
+
+DEFAULT_TARGET = "SFX:EDF036D6/393BC53A"
+RATE = 48000
+
+
+def probe_signal(seconds=30.0, rate=RATE, seed=1234):
+    """Pink-ish noise + vibrato tone: easy to recognise in a recording."""
+    rng = np.random.default_rng(seed)
+    n = int(seconds * rate)
+    white = rng.standard_normal(n)
+    pink = np.cumsum(white) * 0.002
+    pink -= np.convolve(pink, np.ones(2048) / 2048, mode="same")
+    t = np.arange(n) / rate
+    tone = 0.25 * np.sin(2 * np.pi * (440 + 20 * np.sin(2 * np.pi * 0.5 * t)) * t)
+    x = (pink / (np.max(np.abs(pink)) + 1e-9) * 0.35 + tone).astype(np.float32)
+    return x[:, None], rate
+
+
+def _mono48(wem):
+    x, sr = audio.decode_wem(wem)
+    return audio.to_channels(audio.resample(x, sr, RATE), 1)[:, 0]
+
+
+def _sha1_file(p):
+    h = hashlib.sha1()
+    with open(p, "rb") as f:
+        for c in iter(lambda: f.read(1 << 22), b""):
+            h.update(c)
+    return h.hexdigest()
+
+
+def run(ws, key=DEFAULT_TARGET, wem=None, mode="modded", seconds=40.0, threshold=0.5,
+        logdir=None, revert_after=True, log=print):
+    """mode: 'modded' (apply `wem`, test, revert), 'baseline' (nothing), 'installed' (test the
+    current state). Returns a report dict with report['result'] in ('PASS', 'FAIL')."""
+    from .debugger import Run, kill_game
+    from .loopback import LoopbackRecorder, detect_chunks
+    from ..inject import pack_bnk_path, parse_target
+
+    s = ws.catalog.resolve(key)
+    wad = ws.wad_path(s)
+    files = [wad] + ([pack_bnk_path(wad, parse_target(s.target)[0])] if s.is_pack_media else [])
+    logdir = logdir or os.path.join(os.getcwd(), "logs")
+    os.makedirs(logdir, exist_ok=True)
+    stamp = time.strftime("%Y%m%d_%H%M%S")
+    logp = os.path.join(logdir, "ingame_%s.log" % stamp)
+    kill_game()
+    report = dict(target=s.key, name=s.name, mode=mode)
+    before = {f: _sha1_file(f) for f in files} if mode == "modded" else {}
+
+    if mode == "modded":
+        ws.apply([(s, wem, {"source": "ingame-test"})], progress=log)
+    expect = ws.current_bytes(s)
+    orig = ws.original_bytes(s)
+    from ..wem import Wem
+    report["expect_hash"] = "%08x" % Wem.parse(expect).vorb["codebook_hash"]
+    ref_expect, ref_orig = _mono48(expect), _mono48(orig)
+
+    rec = LoopbackRecorder(seconds + 10)
+    rec.start()
+    runner = Run(ws.game.exe, logp)
+    try:
+        report.update(runner.start())
+        while time.time() - runner.t0 < seconds and runner.alive():
+            time.sleep(0.5)
+            if re.search(r"^ACKIT_CRASH", runner.log(), re.M):
+                break
+        report["alive_at_end"] = runner.alive()
+    finally:
+        runner.stop()
+        rec.stop_flag = True
+        rec.join(5)
+        if mode == "modded" and revert_after:
+            ws.revert([s], progress=log)
+            report["reverted_byte_identical"] = all(_sha1_file(f) == h for f, h in before.items())
+
+    text = runner.log()
+    report["crash"] = bool(re.search(r"^ACKIT_CRASH", text, re.M))
+    ev = []
+    for ln in text.splitlines():
+        m = re.match(r"ACKIT_SETUP hash=([0-9a-f]+)", ln)
+        if m:
+            ev.append([m.group(1), "cached"])
+        m = re.match(r"ACKIT_PARSE ret=([0-9a-f]+)", ln)
+        if m and ev and ev[-1][1] == "cached":
+            ev[-1][1] = m.group(1)
+    report["setups"] = ["%s:%s" % tuple(e) for e in ev]
+    mine = [r for h, r in ev if h == report["expect_hash"]]
+    report["setup_ok"] = bool(mine) and all(r in ("0", "cached") for r in mine)
+    cap = rec.audio()
+    report["capture_error"] = rec.error
+    report["ncc_expected"] = round(detect_chunks(cap, ref_expect)[0], 3)
+    report["ncc_original"] = round(detect_chunks(cap, ref_orig)[0], 3)
+    ok = (not report["crash"] and report["alive_at_end"] and report["setup_ok"]
+          and report["ncc_expected"] >= threshold
+          and (mode == "baseline" or sha256(expect) == sha256(orig) or report["ncc_expected"] > report["ncc_original"])
+          and report.get("reverted_byte_identical", True))
+    report["result"] = "PASS" if ok else "FAIL"
+    json.dump(report, open(logp.replace(".log", ".json"), "w"), indent=1)
+    return report

ackit/verify/loopback.py

@@ -0,0 +1,87 @@
+"""WASAPI loopback capture of the default speaker + detection of a reference sound in it."""
+import threading
+import warnings
+
+import numpy as np
+
+RATE = 48000
+
+
+class LoopbackRecorder(threading.Thread):
+    def __init__(self, seconds, rate=RATE):
+        super().__init__(daemon=True)
+        self.seconds = seconds
+        self.rate = rate
+        self.chunks = []
+        self.stop_flag = False
+        self.error = None
+
+    def run(self):
+        try:
+            import soundcard as sc
+            sp = sc.default_speaker()
+            mic = sc.get_microphone(id=str(sp.name), include_loopback=True)
+            block = self.rate // 10
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")  # "data discontinuity" warnings
+                with mic.recorder(samplerate=self.rate, blocksize=block) as rec:
+                    n = int(self.seconds * 10)
+                    for _ in range(n):
+                        if self.stop_flag:
+                            break
+                        self.chunks.append(rec.record(numframes=block))
+        except Exception as e:  # noqa
+            self.error = repr(e)
+
+    def audio(self):
+        if not self.chunks:
+            return np.zeros(0, dtype=np.float32)
+        a = np.concatenate(self.chunks, axis=0)
+        return a.mean(axis=1).astype(np.float32) if a.ndim == 2 else a.astype(np.float32)
+
+
+def resample(x, src_rate, dst_rate):
+    if src_rate == dst_rate:
+        return x.astype(np.float32)
+    n = int(round(len(x) * dst_rate / src_rate))
+    t = np.linspace(0, len(x) - 1, n)
+    return np.interp(t, np.arange(len(x)), x).astype(np.float32)
+
+
+def detect(capture, ref, rate=RATE):
+    """Max normalized cross-correlation of ref against every same-length window of capture.
+    Returns (score 0..1, time_offset_seconds, ref_rms_in_window)."""
+    if len(capture) < len(ref) or len(ref) == 0:
+        return 0.0, None, 0.0
+    ref = ref - ref.mean()
+    rn = np.linalg.norm(ref)
+    if rn == 0:
+        return 0.0, None, 0.0
+    n = 1 << int(np.ceil(np.log2(len(capture) + len(ref))))
+    xc = np.fft.irfft(np.fft.rfft(capture, n) * np.conj(np.fft.rfft(ref, n)), n)[:len(capture) - len(ref) + 1]
+    c2 = np.concatenate([[0.0], np.cumsum(capture.astype(np.float64) ** 2)])
+    win = np.sqrt(np.maximum(c2[len(ref):] - c2[:-len(ref)], 1e-12))
+    ncc = xc / (rn * win)
+    # ignore near-silent windows
+    ncc[win < 1e-3 * np.sqrt(len(ref))] = 0
+    i = int(np.argmax(np.abs(ncc)))
+    return float(abs(ncc[i])), i / rate, float(win[i] / np.sqrt(len(ref)))
+
+
+def detect_chunks(capture, ref, rate=RATE, chunk_s=1.0, step_s=0.5):
+    """Best NCC over 1 s chunks of the reference (robust to partial playback and resampler drift).
+    Returns (best score, chunk start in ref [s], position in capture [s])."""
+    c = int(chunk_s * rate)
+    s = int(step_s * rate)
+    best = (0.0, None, None)
+    if len(ref) <= c:
+        sc, t, _ = detect(capture, ref, rate)
+        return sc, 0.0, t
+    for i in range(0, len(ref) - c + 1, s):
+        seg = ref[i:i + c]
+        if np.sqrt(np.mean(seg ** 2)) < 1e-3:
+            continue
+        sc, t, _ = detect(capture, seg, rate)
+        if sc > best[0]:
+            best = (sc, i / rate, t)
+    return best

ackit/verify/selfcheck.py

@@ -0,0 +1,93 @@
+"""Offline self-check against the installed game (no game launch, nothing is modified).
+
+1. every audio archive parses and follows the WAD layout rules;
+2. a sample of game WEMs is rebuilt byte-for-byte from the derived format rules;
+3. a test signal is encoded for a speech and a music sound and decoded again (correlation check);
+4. the originals store is consistent (blobs intact, recorded states).
+"""
+import random
+
+import numpy as np
+
+from .. import audio
+from ..builder import build_wem, setup_header_values
+from ..convert import library
+from ..inject import is_pack
+from ..wad import Wad
+from ..wem import Wem, split_concatenated
+from ..wwvorbis import transcode_setup, W2V
+
+
+def rebuild_identical(r):
+    """True if a game WEM is reproduced byte-for-byte by the builder."""
+    m = Wem.parse(r)
+    v = m.vorb
+    s = m.setup_packet()
+    _, info = transcode_setup(s, W2V, m.channels, library())
+    pk, _ = m.audio_packets()
+    mb = info.mode_bits
+    if v["bs0_exp"] != v["bs1_exp"]:
+        flags = [info.mode_blockflags[p[0] & ((1 << mb) - 1)] if mb else info.mode_blockflags[0] for _, p in pk]
+    else:
+        flags = [info.mode_blockflags[(p[0] >> 1) & ((1 << mb) - 1)] if mb else info.mode_blockflags[0] for _, p in pk]
+    hv = setup_header_values(s, m.channels)
+    extra = [c for c in m.chunks if c.tag not in (b"fmt ", b"data")]
+    for interval in (8192, 1024, 2048, 4096, 16384, 32768, 512, 256):
+        out = build_wem(m.channels, m.rate, v["bs0_exp"], v["bs1_exp"], s, [p for _, p in pk], flags,
+                        v["total_frames"], hv["hash"], hv["alloc"], hv["alloc64"], m.channel_mask,
+                        seek=v["seek_table_size"] > 0, seek_interval=interval, valid_bits=m.valid_bits,
+                        extra_chunks=extra)
+        if out == r:
+            return True
+        if v["seek_table_size"] == 0:
+            break
+    return False
+
+
+def run(ws, sample=300, log=print):
+    ok = True
+    # 1 + 2
+    for arch, path in ws.game.archives.items():
+        w = Wad(path)
+        probs = w.check()
+        rnd = random.Random(1)
+        ents = rnd.sample(w.entries, min(len(w.entries), 60))
+        n = good = 0
+        for e in ents:
+            if is_pack(path, e.hash):  # stream packs are checked by the tests (they are large)
+                continue
+            for _, r in split_concatenated(w.read(e.hash))[0][:5]:
+                n += 1
+                try:
+                    good += rebuild_identical(r)
+                except Exception:
+                    pass
+                if n >= sample // max(1, len(ws.game.archives)):
+                    break
+        log("%-4s layout %s, rebuilt %d/%d sample WEMs byte-identical" % (arch, "OK" if not probs else probs[:2], good, n))
+        ok &= not probs and good >= n - 2  # 4 looping music tracks are known exceptions
+    # 3
+    t = np.arange(48000 * 2) / 48000.0
+    sig = (0.3 * np.sin(2 * np.pi * 330 * t) * np.sin(2 * np.pi * 1.5 * t)).astype(np.float32)[:, None]
+    for s in [x for x in ws.catalog.sounds if x.group == "Speech"][:1] + \
+             [x for x in ws.catalog.sounds if x.group == "Music"][:1]:
+        wem, info = ws.prepare(s, sig, 48000)
+        y, sr = audio.decode_wem(wem)
+        ref = audio.to_channels(audio.resample(sig, 48000, sr), y.shape[1])
+        n = min(len(ref), len(y))
+        c = float(np.corrcoef(ref[:n, 0], y[:n, 0])[0, 1])
+        log("encode/decode %-30s %d Hz q%.1f corr %.3f" % (s.name[:30], info["rate"], info["quality"], c))
+        ok &= c > 0.95
+    # 4
+    bad = 0
+    for k, e in ws.store.entries.items():
+        try:
+            ws.store.get_blob(e["orig"])
+            if e.get("applied"):
+                ws.store.get_blob(e["applied"])
+        except (OSError, IOError):
+            bad += 1
+    log("originals store: %d record(s), %d damaged" % (len(ws.store.entries), bad))
+    ok &= bad == 0
+    log("SELF-CHECK %s" % ("PASS" if ok else "FAIL"))
+    return ok