Source code for energy_data_retrieval.hydro_catalogue.replay
"""Validate a complete reviewed enrichment batch before writing any JSONL shard."""
import json
from collections.abc import Callable
from pathlib import Path
from typing import Any
from energy_data_retrieval.pv_catalogue.catalogue import canonical_json
[docs]
def replay_reviewed(
root: Path,
accepted: list[dict[str, Any]],
transform: Callable[[dict[str, Any], dict[str, Any]], dict[str, Any]],
) -> int:
"""Reject duplicate/missing identities and transformation conflicts before writes.
The callback receives an existing record and its explicitly reviewed entry.
Unreviewed records are preserved. This is validation-before-write, not a
filesystem transaction; an I/O failure still requires deterministic replay.
"""
lookup = {entry["record_id"]: entry for entry in accepted}
if len(lookup) != len(accepted):
raise ValueError("Duplicate reviewed record identity")
seen: set[str] = set()
staged: dict[Path, str] = {}
for path in sorted((root / "records").rglob("*.jsonl")):
original = path.read_text()
rows = [json.loads(line) for line in original.splitlines()]
for i, row in enumerate(rows):
identity = row["record_id"]
if identity in seen:
raise ValueError("Duplicate canonical record identity")
seen.add(identity)
if identity in lookup:
created_at = row["created_at"]
updated = transform(row, lookup[identity])
if updated["record_id"] != identity or updated["created_at"] != created_at:
raise ValueError("Replay changed immutable identity or creation date")
rows[i] = updated
text = "".join(canonical_json(row) + "\n" for row in rows)
if text != original:
staged[path] = text
if missing := lookup.keys() - seen:
raise ValueError(f"Reviewed identities absent from catalogue: {sorted(missing)}")
for path, text in staged.items():
path.write_text(text)
return len(lookup)