#!/usr/bin/env python3 """PARALLAX - verify.py Proves that the dataset in this folder is exactly the dataset PARALLAX sealed, and that it was sealed before you asked for it. Nothing here contacts PARALLAX. If we vanished tomorrow, this check would still work. python verify.py The core check needs NOTHING beyond a standard Python install. If you also have duckdb (pip install duckdb) it goes further: recomputing every column digest, re-deriving every published figure, and re-running our controls against the data you actually hold. What each step defends against files someone altered a byte of the data after we sealed it root someone altered the manifest itself and updated the root to match figures we published a headline number the data does not support controls our pipeline silently broke in a way that still produces output timestamp we backdated any of the above Exit code 0 means every check passed. """ import argparse import hashlib import json import os import sys CANON = dict(sort_keys=True, separators=(",", ":"), ensure_ascii=False) # Detached OpenTimestamps file header, followed by: version byte, op byte, digest. OTS_MAGIC = b"\x00OpenTimestamps\x00\x00Proof\x00\xbf\x89\xe2\xe8\x84\xe8\x92\x94" OTS_OP_SHA256 = 0x08 def ots_digest(raw): """The digest a detached OTS receipt commits to, or None if unparseable.""" n = len(OTS_MAGIC) if raw[:n] != OTS_MAGIC or len(raw) < n + 34: return None if raw[n + 1] != OTS_OP_SHA256: # raw[n] is the format version return None return raw[n + 2:n + 34] PASS, FAIL, INFO = "PASS", "FAIL", " . " _fails = [] _preview = [False] def report(ok, label, detail=""): tag = PASS if ok else FAIL print(" [%s] %-46s %s" % (tag, label, detail)) if not ok: _fails.append(label) return ok def sha256_file(path): h = hashlib.sha256() with open(path, "rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) return h.hexdigest() def compute_root(manifest): body = {k: v for k, v in manifest.items() if k != "root"} return hashlib.sha256(json.dumps(body, **CANON).encode("utf-8")).hexdigest() def head(title): print("\n" + title) print("-" * 74) def main(): ap = argparse.ArgumentParser() ap.add_argument("--dir", default=os.path.dirname(os.path.abspath(__file__)), help="product folder (default: folder containing this script)") args = ap.parse_args() d = args.dir mpath = os.path.join(d, "manifest.json") if not os.path.exists(mpath): sys.exit("No manifest.json in %s" % d) with open(mpath, encoding="utf-8") as f: m = json.load(f) print("=" * 74) print("PARALLAX - %s (v%s)" % (m.get("title"), m.get("version"))) print("sealed %s" % m.get("generated_utc")) print("=" * 74) # Preview kit: manifest + root + receipt, but no data. That is a legitimate # state, not a broken product - you are meant to be able to verify our proof # BEFORE you buy the data. Detect it and check everything that is checkable. present = [t for t in m["tables"] if os.path.exists(os.path.join(d, t["files"]["parquet"]["path"]))] preview = not present _preview[0] = preview # ---------------------------------------------------------------- 1. files if preview: head("1. Files (preview kit - data not included)") print(INFO + "This folder is the free preview: the manifest, the root and the") print(INFO + "Bitcoin receipt, without the data itself. Steps 2 and 3 below still") print(INFO + "prove what we sealed and when. Buy the dataset and re-run this script") print(INFO + "in that folder to check the data too.") else: head("1. Files are byte-for-byte what was sealed") for t in m["tables"]: for kind, info in t["files"].items(): p = os.path.join(d, info["path"]) if not os.path.exists(p): report(False, "%s (%s)" % (t["name"], kind), "MISSING") continue got = sha256_file(p) report(got == info["sha256"], "%s (%s)" % (t["name"], kind), got[:16] + "...") # ----------------------------------------------------------------- 2. root head("2. The root is recomputed, not taken on trust") recomputed = compute_root(m) report(recomputed == m["root"], "root recomputed from manifest contents", recomputed[:16] + "...") rp = os.path.join(d, "root.hash") if os.path.exists(rp): with open(rp, encoding="utf-8") as f: on_disk = f.read().strip() report(on_disk == m["root"], "root.hash on disk agrees with manifest") else: report(False, "root.hash present", "MISSING") # control: a mutated manifest MUST break the root mutated = json.loads(json.dumps(m)) mutated["tables"][0]["rows"] = int(mutated["tables"][0]["rows"]) + 1 report(compute_root(mutated) != m["root"], "CONTROL: a tampered manifest fails this check") # ------------------------------------------------------------ 3. timestamp head("3. Committed to the Bitcoin blockchain") op = os.path.join(d, "root.hash.ots") if not os.path.exists(op): report(False, "OpenTimestamps receipt present", "MISSING") else: raw = open(op, "rb").read() report(raw[:len(OTS_MAGIC)] == OTS_MAGIC, "OpenTimestamps receipt present", "%d bytes" % len(raw)) # Presence is not proof. A receipt for some OTHER root would sit here # just as happily, so bind the receipt to THIS root explicitly. The # detached format carries the digest it commits to, right after the # magic: one version byte, one op byte (0x08 = sha256), 32-byte digest. stamped = ots_digest(raw) ours = hashlib.sha256(open(rp, "rb").read()).digest() if os.path.exists(rp) else b"" report(stamped is not None and stamped == ours, "receipt commits to THIS root, not another", (stamped or b"").hex()[:16] + "...") # control: a receipt for a different root must be rejected forged = bytearray(raw) forged[33] ^= 0xFF report(ots_digest(bytes(forged)) != ours, "CONTROL: a receipt for a different root fails") print(INFO + "The above proves WHICH root was stamped, offline. To confirm the") print(INFO + "Bitcoin block height it landed in, run:") print(INFO + " ots verify root.hash.ots") print(INFO + "(pip install opentimestamps-client - it can check against your own node)") # ----------------------------------------------------- 4. deep check (duckdb) if preview: head("4. Data checks (not applicable to the preview kit)") print(INFO + "The manifest above already tells you exactly what the full dataset") print(INFO + "contains: %d tables, %s rows, every column digest, every published" % (len(m["tables"]), format(sum(t["rows"] for t in m["tables"]), ","))) print(INFO + "figure with the SQL behind it, and the controls the build had to pass.") return finish() try: import duckdb # noqa: F401 except ImportError: head("4. Deeper checks (skipped)") print(INFO + "Install duckdb to also recompute every column digest,") print(INFO + "re-derive every published figure, and re-run our controls:") print(INFO + " pip install duckdb && python verify.py") return finish() import duckdb con = duckdb.connect(":memory:") for t in m["tables"]: p = os.path.join(d, t["files"]["parquet"]["path"]).replace("\\", "/") con.execute('CREATE VIEW "%s" AS SELECT * FROM read_parquet(\'%s\')' % (t["name"], p)) head("4. Column digests recomputed from your copy of the data") for t in m["tables"]: bad = [] for col, dig in t["columns"].items(): row = con.execute( 'SELECT count(*), bit_xor(hash("%s")), sum(hash("%s"))::VARCHAR FROM "%s"' % (col, col, t["name"])).fetchone() if {"count": row[0], "xor": str(row[1]), "sum": str(row[2])} != dig: bad.append(col) rows = con.execute('SELECT count(*) FROM "%s"' % t["name"]).fetchone()[0] report(not bad and rows == t["rows"], t["name"], "%d rows, %d columns" % (rows, len(t["columns"])) if not bad else "mismatch: " + ", ".join(bad)) head("5. Published figures re-derived from your copy") for name, fig in m["figures"].items(): got = con.execute(fig["sql"]).fetchone()[0] report(str(got) == fig["value"], name, str(got)) head("6. Controls - cases where the answer is already known") for name, c in m["controls"].items(): got = con.execute(c["sql"]).fetchone()[0] report(got == c["expected"], name, str(got)) print(INFO + c["why"]) return finish() def finish(): print("\n" + "=" * 74) if _fails: print("FAILED - %d check(s) did not pass:" % len(_fails)) for f in _fails: print(" - %s" % f) print("\nIf you believe the data is correct and this script is wrong, we want to") print("know. A verifier that cannot fail is worthless to you.") print("=" * 74) return 1 print("ALL CHECKS PASSED") if _preview[0]: # Do not overclaim: in preview mode we have verified the SEAL, not data # we were never given. print("The seal checks out: this manifest is intact and was committed to") print("Bitcoin before you downloaded it. The data itself is not in this") print("folder - buy it and re-run this script there to check it too.") else: print("This data is exactly what PARALLAX sealed, and it was sealed before you") print("received it. You did not have to trust us for any part of that.") print("=" * 74) return 0 if __name__ == "__main__": sys.exit(main())