Source code for energy_data_retrieval.storage

"""Partition-sized Parquet outputs with verifiable, restart-safe provenance."""

from __future__ import annotations

import contextlib
import hashlib
import json
import os
import tempfile
from collections.abc import Callable, Iterable, Iterator, Mapping
from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
from dataclasses import asdict, dataclass
from pathlib import Path

import pandas as pd

from energy_data_retrieval.transport import JSONValue, atomic_bytes


[docs] @dataclass(frozen=True, slots=True) class Artifact: filename: str sha256: str bytes: int rows: int request_fingerprint: str
[docs] def sha256_file(path: Path) -> str: with path.open("rb") as stream: return hashlib.file_digest(stream, "sha256").hexdigest()
[docs] def fingerprint(metadata: Mapping[str, JSONValue]) -> str: """Fingerprint only non-secret request semantics and source/time conventions.""" return hashlib.sha256( json.dumps(metadata, sort_keys=True, allow_nan=False).encode() ).hexdigest()
[docs] def existing_artifact(path: Path, request_fingerprint: str) -> Artifact | None: manifest_path = path.with_suffix(path.suffix + ".json") if not path.exists() and not manifest_path.exists(): return None if not path.exists() or not manifest_path.exists(): raise ValueError( "Incomplete prior output; inspect and remove or choose a new output directory" ) saved = json.loads(manifest_path.read_text()) record = Artifact(**saved["artifact"]) if ( record.filename != path.name or record.request_fingerprint != request_fingerprint or record.bytes != path.stat().st_size or record.sha256 != sha256_file(path) ): raise ValueError("Existing output does not match the requested snapshot or its checksum") return record
[docs] def write_parquet( frame: pd.DataFrame, path: Path, *, request: Mapping[str, JSONValue], provenance: Mapping[str, JSONValue], ) -> Artifact: """Publish a Parquet partition, then its manifest; refuse mismatched overwrites. A crash between the two atomic publications is detected on restart. The user must inspect that incomplete pair; an unaudited file is never silently reused. """ identity = fingerprint(request) previous = existing_artifact(path, identity) if previous is not None: return previous path.parent.mkdir(parents=True, exist_ok=True) descriptor, name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) os.close(descriptor) temporary = Path(name) try: frame.to_parquet(temporary, engine="pyarrow", compression="zstd", index=False) with temporary.open("rb") as stream: os.fsync(stream.fileno()) record = Artifact( path.name, sha256_file(temporary), temporary.stat().st_size, len(frame), identity ) temporary.replace(path) atomic_bytes( path.with_suffix(path.suffix + ".json"), json.dumps( { "artifact": asdict(record), "request": dict(request), "provenance": dict(provenance), }, ensure_ascii=False, indent=2, allow_nan=False, ).encode(), ) return record finally: temporary.unlink(missing_ok=True)
[docs] def bounded_map[Input, Output]( function: Callable[[Input], Output], items: Iterable[Input], *, workers: int = 2, ) -> Iterator[Output]: """Yield completion-order results with at most ``workers`` tasks in flight.""" if not 1 <= workers <= 8: raise ValueError("workers must be between 1 and 8") source = iter(items) pending: set[Future[Output]] = set() with ThreadPoolExecutor(max_workers=workers) as executor: for _ in range(workers): try: pending.add(executor.submit(function, next(source))) except StopIteration: break while pending: completed, pending = wait(pending, return_when=FIRST_COMPLETED) for future in completed: yield future.result() with contextlib.suppress(StopIteration): pending.add(executor.submit(function, next(source)))