diff --git a/DURABLE_STATE.md b/DURABLE_STATE.md index d0fc5d18de7ca5ece53fdfd60d4465d0ed921e43..d46f67303ae995ac16bf72f7d520f5cda41b1d91 100644 --- a/DURABLE_STATE.md +++ b/DURABLE_STATE.md @@ -113,21 +113,31 @@ blob values. ## Rebuilding a flawed projection The first-boot importer intentionally skips a nonempty database, so it cannot -repair an older projection containing embedded base64. During maintenance, -with the application fully stopped and `DISABLE_EPOCH_LOOP=1`: +repair an older projection containing embedded base64. For this one migration, +start the corrected image with both maintenance flags: ```sh -mv /data/constitution.rocks /data/constitution.rocks.pre-blob-refs -python scripts/import-ledger.py \ - /data/ledger.jsonl \ - /data/constitution.rocks \ - --batch-size 100 +DISABLE_EPOCH_LOOP=1 +REBUILD_ROCKS_FROM_JSONL=1 ``` -Do not use `--force` here: retaining the closed old directory makes rollback -immediate. Start the application still paused, verify health count/chain and -representative epoch, comparison, commit, patch, prompt, and attempt routes, -then unpause. If import or verification fails, stop, remove the incomplete new -directory if present, move `.pre-blob-refs` back to -`/data/constitution.rocks`, and restart the prior image while remaining -paused. +Use a non-overlapping deployment strategy so the prior process releases +RocksDB first. Before opening its live handle, startup requires the tape, +verifies and closes a probe of the nonempty projection, refuses to proceed if +`/data/constitution.rocks.pre-rebuild` already exists, then atomically renames +the old directory to that exact backup path. It imports and verifies a fresh +`/data/constitution.rocks`. On failure it destroys the partial fresh directory +and renames the backup back automatically. On success it serves from the new +projection and deliberately retains the backup. + +While still paused, verify the health count and representative epoch, +comparison, commit, patch, prompt, and attempt routes. Then remove +`REBUILD_ROCKS_FROM_JSONL` before any restart or second deployment; leaving it +enabled will correctly fail startup because the deterministic backup exists. +Restart once with only `DISABLE_EPOCH_LOOP=1`, verify again, and only then +unpause. Delete `.pre-rebuild` manually only after the verification window. + +For rollback after a successful rebuild, stop the process, move the fresh +directory aside, rename `.pre-rebuild` back to `/data/constitution.rocks`, and +start the prior image while still paused. The rebuild flag is exact and +inactive for every value except `1`. diff --git a/constitution.py b/constitution.py index 05d7048cbaf42bf4d57345a445e2501ae88f05b4..d4ace9591f035d67721178a7c17e6053039800d1 100644 --- a/constitution.py +++ b/constitution.py @@ -334,6 +334,82 @@ def prepare_state_sync() -> dict: ``IMPORT_JSONL_ON_EMPTY=1``. """ global state_db + should_rebuild = os.environ.get("REBUILD_ROCKS_FROM_JSONL") == "1" + if should_rebuild: + if state_db is not None: + raise RuntimeError( + "cannot rebuild RocksDB after the live state handle is open" + ) + rocks_path = pathlib.Path(ROCKS_PATH) + backup_path = pathlib.Path(str(rocks_path) + ".pre-rebuild") + if not JSONL_PATH.is_file(): + message = f"archival tape not found: {JSONL_PATH}" + STATE_STATUS.update( + ready=False, importing=False, error=message, event_count=0 + ) + raise RuntimeError(message) + if backup_path.exists(): + message = f"rebuild backup already exists: {backup_path}" + STATE_STATUS.update( + ready=False, importing=False, error=message, event_count=0 + ) + raise RuntimeError(message) + if not rocks_path.exists(): + message = f"Rocks projection does not exist: {rocks_path}" + STATE_STATUS.update( + ready=False, importing=False, error=message, event_count=0 + ) + raise RuntimeError(message) + + probe = RocksDb.open(rocks_path) + try: + existing_count = ROOT.events.len(probe) + finally: + probe.close() + if existing_count == 0: + message = f"Rocks projection is empty; refusing rebuild: {rocks_path}" + STATE_STATUS.update( + ready=False, importing=False, error=message, event_count=0 + ) + raise RuntimeError(message) + + rocks_path.rename(backup_path) + STATE_STATUS.update( + ready=False, importing=True, error=None, event_count=0 + ) + try: + import_jsonl_tape(JSONL_PATH, rocks_path) + except BaseException as exc: + try: + if rocks_path.exists(): + RocksDb.open(rocks_path).destroy() + if rocks_path.exists(): + raise RuntimeError( + f"incomplete projection could not be removed: {rocks_path}" + ) + backup_path.rename(rocks_path) + except BaseException as rollback_exc: + message = ( + f"rebuild failed ({exc}); automatic rollback failed " + f"({rollback_exc}); backup retained at {backup_path}" + ) + STATE_STATUS.update( + ready=False, + importing=False, + error=message, + event_count=0, + ) + raise RuntimeError(message) from rollback_exc + STATE_STATUS.update( + ready=False, + importing=False, + error=str(exc), + event_count=existing_count, + ) + raise + finally: + STATE_STATUS["importing"] = False + if state_db is not None: count = ROOT.events.len(state_db) STATE_STATUS.update( diff --git a/tests/test_state_migration.py b/tests/test_state_migration.py index c2c1e22654d0da4546a0b32374aebc6174c29b91..613ee684a186abed5c6186434b2346ab01bd26fe 100644 --- a/tests/test_state_migration.py +++ b/tests/test_state_migration.py @@ -349,6 +349,7 @@ def _configure_first_boot(monkeypatch, tape, rocks): monkeypatch.setattr(c, "ROCKS_PATH", rocks) monkeypatch.setattr(c, "state_db", None) monkeypatch.setenv("IMPORT_JSONL_ON_EMPTY", "1") + monkeypatch.delenv("REBUILD_ROCKS_FROM_JSONL", raising=False) c.STATE_STATUS.update( ready=False, importing=False, error=None, event_count=0 ) @@ -471,3 +472,124 @@ def test_import_rejects_duplicate_unique_events(tmp_path, duplicate_type): assert c.ROOT.events.len(db) == 0 finally: db.destroy() + + +def _seed_flawed_projection(path): + db = c.RocksDb.open(path) + db.apply([ + c.ROOT.events.push({"type": "redemption", "timestamp_ms": 9}), + ]) + db.close() + + +def _configure_rebuild(monkeypatch, tape, rocks): + monkeypatch.setattr(c, "JSONL_PATH", tape) + monkeypatch.setattr(c, "ROCKS_PATH", rocks) + monkeypatch.setattr(c, "state_db", None) + monkeypatch.setenv("REBUILD_ROCKS_FROM_JSONL", "1") + monkeypatch.delenv("IMPORT_JSONL_ON_EMPTY", raising=False) + c.STATE_STATUS.update( + ready=False, importing=False, error=None, event_count=0 + ) + + +def test_startup_rebuild_retains_backup_and_opens_verified_projection( + tmp_path, monkeypatch +): + tape = tmp_path / "ledger.jsonl" + rocks = tmp_path / "constitution.rocks" + backup = tmp_path / "constitution.rocks.pre-rebuild" + rows, _ = _sample_tape(tape) + _seed_flawed_projection(rocks) + _configure_rebuild(monkeypatch, tape, rocks) + + status = c.prepare_state_sync() + try: + assert status["ready"] is True + assert status["event_count"] == len(rows) + assert backup.is_dir() + old = c.RocksDb.open(backup) + try: + assert c.ROOT.events.len(old) == 1 + assert c.ROOT.events.get(old, 0)["timestamp_ms"] == 9 + finally: + old.close() + finally: + c.state_db.close() + c.state_db = None + + +def test_startup_rebuild_refuses_existing_backup_without_mutation( + tmp_path, monkeypatch +): + tape = tmp_path / "ledger.jsonl" + rocks = tmp_path / "constitution.rocks" + backup = tmp_path / "constitution.rocks.pre-rebuild" + _sample_tape(tape) + _seed_flawed_projection(rocks) + backup.mkdir() + _configure_rebuild(monkeypatch, tape, rocks) + + with pytest.raises(RuntimeError, match="backup already exists"): + c.prepare_state_sync() + original = c.RocksDb.open(rocks) + try: + assert c.ROOT.events.len(original) == 1 + assert c.ROOT.events.get(original, 0)["timestamp_ms"] == 9 + finally: + original.close() + + +def test_startup_rebuild_failure_removes_partial_and_restores_backup( + tmp_path, monkeypatch +): + tape = tmp_path / "ledger.jsonl" + rocks = tmp_path / "constitution.rocks" + backup = tmp_path / "constitution.rocks.pre-rebuild" + _sample_tape(tape) + _seed_flawed_projection(rocks) + _configure_rebuild(monkeypatch, tape, rocks) + + def fail_after_partial_write(_tape, target): + partial = c.RocksDb.open(target) + partial.apply([ + c.ROOT.events.push({"type": "redemption", "timestamp_ms": 99}), + ]) + partial.close() + raise RuntimeError("injected import failure") + + monkeypatch.setattr(c, "import_jsonl_tape", fail_after_partial_write) + with pytest.raises(RuntimeError, match="injected import failure"): + c.prepare_state_sync() + + assert not backup.exists() + restored = c.RocksDb.open(rocks) + try: + assert c.ROOT.events.len(restored) == 1 + assert c.ROOT.events.get(restored, 0)["timestamp_ms"] == 9 + finally: + restored.close() + assert c.state_db is None + assert c.STATE_STATUS["ready"] is False + assert c.STATE_STATUS["importing"] is False + + +def test_startup_rebuild_requires_exact_flag_value(tmp_path, monkeypatch): + tape = tmp_path / "ledger.jsonl" + rocks = tmp_path / "constitution.rocks" + backup = tmp_path / "constitution.rocks.pre-rebuild" + _sample_tape(tape) + _seed_flawed_projection(rocks) + monkeypatch.setattr(c, "JSONL_PATH", tape) + monkeypatch.setattr(c, "ROCKS_PATH", rocks) + monkeypatch.setattr(c, "state_db", None) + monkeypatch.setenv("REBUILD_ROCKS_FROM_JSONL", "true") + monkeypatch.delenv("IMPORT_JSONL_ON_EMPTY", raising=False) + + status = c.prepare_state_sync() + try: + assert status["event_count"] == 1 + assert not backup.exists() + finally: + c.state_db.close() + c.state_db = None