144 lines
5.2 KiB
Python
144 lines
5.2 KiB
Python
"""Originals store: keeps the game's original bytes of every sound you modify, so any change can be
|
|
reverted exactly, without keeping full copies of the 0.4-0.5 GB WADs.
|
|
|
|
Layout (one store per game folder, in the user data folder by default; see docs/decisions.md):
|
|
<root>/store.json records: {key: {...}} + per-WAD info
|
|
<root>/blobs/<sha256>.wem original WEMs and the replacements you applied
|
|
<root>/gaps/<PACK>.json original non-zero padding of stream packs (byte-exact pack reverts)
|
|
<root>/full/<rel path> optional full copies of WADs / pack indexes (fallback, see `full_backup`)
|
|
Writes to store.json are atomic (temp file + rename).
|
|
"""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
import time
|
|
|
|
STORE_VERSION = 1
|
|
|
|
|
|
def sha256(b):
|
|
return hashlib.sha256(b).hexdigest()
|
|
|
|
|
|
def sha1_16(b):
|
|
return hashlib.sha1(b).hexdigest()[:16]
|
|
|
|
|
|
class OriginalsStore:
|
|
def __init__(self, root, game_root=None):
|
|
self.root = root
|
|
os.makedirs(os.path.join(root, "blobs"), exist_ok=True)
|
|
self.path = os.path.join(root, "store.json")
|
|
if os.path.exists(self.path):
|
|
self.data = json.load(open(self.path, encoding="utf-8"))
|
|
else:
|
|
self.data = {"version": STORE_VERSION, "game_root": game_root, "created": time.time(),
|
|
"entries": {}, "wads": {}}
|
|
self.save()
|
|
|
|
# persistence -------------------------------------------------------------------------------
|
|
def save(self):
|
|
tmp = self.path + ".tmp"
|
|
with open(tmp, "w", encoding="utf-8") as f:
|
|
json.dump(self.data, f, indent=1)
|
|
os.replace(tmp, self.path)
|
|
|
|
def put_blob(self, b):
|
|
h = sha256(b)
|
|
p = os.path.join(self.root, "blobs", h + ".wem")
|
|
if not os.path.exists(p):
|
|
tmp = p + ".tmp"
|
|
with open(tmp, "wb") as f:
|
|
f.write(b)
|
|
os.replace(tmp, p)
|
|
return h
|
|
|
|
def get_blob(self, h):
|
|
p = os.path.join(self.root, "blobs", h + ".wem")
|
|
b = open(p, "rb").read()
|
|
if sha256(b) != h:
|
|
raise IOError("originals store is damaged: %s" % p)
|
|
return b
|
|
|
|
# records ---------------------------------------------------------------------------------
|
|
@property
|
|
def entries(self):
|
|
return self.data["entries"]
|
|
|
|
def get(self, key):
|
|
return self.entries.get(key)
|
|
|
|
def original(self, key):
|
|
e = self.get(key)
|
|
return self.get_blob(e["orig"]) if e else None
|
|
|
|
def remember_original(self, key, archive, target, wad_rel, data):
|
|
"""Store the original bytes the first time a sound is modified (no-op afterwards)."""
|
|
if key not in self.entries:
|
|
self.entries[key] = {"archive": archive, "target": target, "wad": wad_rel,
|
|
"orig": self.put_blob(data), "orig_size": len(data),
|
|
"orig_sha1": sha1_16(data), "first_modified": time.time(),
|
|
"applied": None}
|
|
return self.entries[key]
|
|
|
|
def record_applied(self, key, data, meta=None):
|
|
e = self.entries[key]
|
|
e["applied"] = self.put_blob(data)
|
|
e["applied_at"] = time.time()
|
|
e["meta"] = meta or {}
|
|
|
|
def forget(self, key):
|
|
self.entries.pop(key, None)
|
|
|
|
def gc(self):
|
|
"""Delete blobs no record refers to."""
|
|
used = {e[k] for e in self.entries.values() for k in ("orig", "applied") if e.get(k)}
|
|
d = os.path.join(self.root, "blobs")
|
|
for f in os.listdir(d):
|
|
if f.endswith(".wem") and f[:-4] not in used:
|
|
os.remove(os.path.join(d, f))
|
|
|
|
# pack padding ------------------------------------------------------------------------------
|
|
def pack_gaps(self, pack_id):
|
|
p = os.path.join(self.root, "gaps", "%08X.json" % pack_id)
|
|
if not os.path.exists(p):
|
|
return None
|
|
return {int(k): bytes.fromhex(v) for k, v in json.load(open(p)).items()}
|
|
|
|
def remember_pack_gaps(self, pack_id, gaps):
|
|
p = os.path.join(self.root, "gaps", "%08X.json" % pack_id)
|
|
if os.path.exists(p):
|
|
return
|
|
os.makedirs(os.path.dirname(p), exist_ok=True)
|
|
json.dump({str(k): v.hex() for k, v in gaps.items()}, open(p, "w"))
|
|
|
|
# full-file fallback backups ----------------------------------------------------------------
|
|
def full_backup(self, src, rel):
|
|
"""Copy a game file once (before its first modification), verified by SHA-1."""
|
|
dst = os.path.join(self.root, "full", rel)
|
|
if os.path.exists(dst):
|
|
return dst
|
|
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
|
tmp = dst + ".tmp"
|
|
shutil.copy2(src, tmp)
|
|
if _file_sha1(tmp) != _file_sha1(src):
|
|
os.remove(tmp)
|
|
raise IOError("backup copy of %s does not match the original" % rel)
|
|
os.replace(tmp, dst)
|
|
self.data.setdefault("full", {})[rel] = {"size": os.path.getsize(dst), "time": time.time()}
|
|
self.save()
|
|
return dst
|
|
|
|
def full_copy(self, rel):
|
|
p = os.path.join(self.root, "full", rel)
|
|
return p if os.path.exists(p) else None
|
|
|
|
|
|
def _file_sha1(p):
|
|
h = hashlib.sha1()
|
|
with open(p, "rb") as f:
|
|
for c in iter(lambda: f.read(1 << 22), b""):
|
|
h.update(c)
|
|
return h.hexdigest()
|