Initial commit

ackit/verify/selfcheck.py

@@ -0,0 +1,93 @@
+"""Offline self-check against the installed game (no game launch, nothing is modified).
+
+1. every audio archive parses and follows the WAD layout rules;
+2. a sample of game WEMs is rebuilt byte-for-byte from the derived format rules;
+3. a test signal is encoded for a speech and a music sound and decoded again (correlation check);
+4. the originals store is consistent (blobs intact, recorded states).
+"""
+import random
+
+import numpy as np
+
+from .. import audio
+from ..builder import build_wem, setup_header_values
+from ..convert import library
+from ..inject import is_pack
+from ..wad import Wad
+from ..wem import Wem, split_concatenated
+from ..wwvorbis import transcode_setup, W2V
+
+
+def rebuild_identical(r):
+    """True if a game WEM is reproduced byte-for-byte by the builder."""
+    m = Wem.parse(r)
+    v = m.vorb
+    s = m.setup_packet()
+    _, info = transcode_setup(s, W2V, m.channels, library())
+    pk, _ = m.audio_packets()
+    mb = info.mode_bits
+    if v["bs0_exp"] != v["bs1_exp"]:
+        flags = [info.mode_blockflags[p[0] & ((1 << mb) - 1)] if mb else info.mode_blockflags[0] for _, p in pk]
+    else:
+        flags = [info.mode_blockflags[(p[0] >> 1) & ((1 << mb) - 1)] if mb else info.mode_blockflags[0] for _, p in pk]
+    hv = setup_header_values(s, m.channels)
+    extra = [c for c in m.chunks if c.tag not in (b"fmt ", b"data")]
+    for interval in (8192, 1024, 2048, 4096, 16384, 32768, 512, 256):
+        out = build_wem(m.channels, m.rate, v["bs0_exp"], v["bs1_exp"], s, [p for _, p in pk], flags,
+                        v["total_frames"], hv["hash"], hv["alloc"], hv["alloc64"], m.channel_mask,
+                        seek=v["seek_table_size"] > 0, seek_interval=interval, valid_bits=m.valid_bits,
+                        extra_chunks=extra)
+        if out == r:
+            return True
+        if v["seek_table_size"] == 0:
+            break
+    return False
+
+
+def run(ws, sample=300, log=print):
+    ok = True
+    # 1 + 2
+    for arch, path in ws.game.archives.items():
+        w = Wad(path)
+        probs = w.check()
+        rnd = random.Random(1)
+        ents = rnd.sample(w.entries, min(len(w.entries), 60))
+        n = good = 0
+        for e in ents:
+            if is_pack(path, e.hash):  # stream packs are checked by the tests (they are large)
+                continue
+            for _, r in split_concatenated(w.read(e.hash))[0][:5]:
+                n += 1
+                try:
+                    good += rebuild_identical(r)
+                except Exception:
+                    pass
+                if n >= sample // max(1, len(ws.game.archives)):
+                    break
+        log("%-4s layout %s, rebuilt %d/%d sample WEMs byte-identical" % (arch, "OK" if not probs else probs[:2], good, n))
+        ok &= not probs and good >= n - 2  # 4 looping music tracks are known exceptions
+    # 3
+    t = np.arange(48000 * 2) / 48000.0
+    sig = (0.3 * np.sin(2 * np.pi * 330 * t) * np.sin(2 * np.pi * 1.5 * t)).astype(np.float32)[:, None]
+    for s in [x for x in ws.catalog.sounds if x.group == "Speech"][:1] + \
+             [x for x in ws.catalog.sounds if x.group == "Music"][:1]:
+        wem, info = ws.prepare(s, sig, 48000)
+        y, sr = audio.decode_wem(wem)
+        ref = audio.to_channels(audio.resample(sig, 48000, sr), y.shape[1])
+        n = min(len(ref), len(y))
+        c = float(np.corrcoef(ref[:n, 0], y[:n, 0])[0, 1])
+        log("encode/decode %-30s %d Hz q%.1f corr %.3f" % (s.name[:30], info["rate"], info["quality"], c))
+        ok &= c > 0.95
+    # 4
+    bad = 0
+    for k, e in ws.store.entries.items():
+        try:
+            ws.store.get_blob(e["orig"])
+            if e.get("applied"):
+                ws.store.get_blob(e["applied"])
+        except (OSError, IOError):
+            bad += 1
+    log("originals store: %d record(s), %d damaged" % (len(ws.store.entries), bad))
+    ok &= bad == 0
+    log("SELF-CHECK %s" % ("PASS" if ok else "FAIL"))
+    return ok