Source code for energy_data_retrieval.gas_catalogue.catalogue

"""Gas-fired plant identities, electrical boundaries and deterministic snapshots."""

from __future__ import annotations

import json
import re
import unicodedata
from datetime import date
from importlib.resources import files
from pathlib import Path
from typing import Any
from uuid import NAMESPACE_URL, uuid5

from energy_data_retrieval.catalogue_common import validate_common, write_snapshot
from energy_data_retrieval.data_center_catalogue.catalogue import upsert_records as _upsert_records
from energy_data_retrieval.pv_catalogue.catalogue import canonical_json, load_records

SCHEMA: dict[str, Any] = json.loads(files(__package__).joinpath("schema.json").read_text())
CAPACITIES = (
    "installed_capacity_mw",
    "operational_capacity_mw",
    "planned_capacity_mw",
    "historical_capacity_mw",
)


[docs] def stable_id(entity_type: str, *identity: str) -> str: """Use immutable registry identity, never mutable plant capacity or ownership.""" parts = [" ".join(unicodedata.normalize("NFKC", s).split()) for s in identity] if entity_type not in SCHEMA["entity_types"] or not parts or not all(parts): raise ValueError("Require registered entity type and nonempty identity") return str( uuid5(NAMESPACE_URL, canonical_json(["energy-data-insights/gas", entity_type, *parts])) )
[docs] def validate_record(row: dict[str, Any], sources: dict[str, Any]) -> None: """Validate provenance, coordinates, status dates and electrical/thermal boundaries.""" validate_common(row, sources, SCHEMA) if not re.fullmatch(r"[A-Z]{3}", row.get("country_code", "")): raise ValueError("Require ISO alpha-3 country") for key in ("status", "status_as_of", "status_date_basis", "technology", "aggregation_role"): if not row.get(key): raise ValueError(f"Require {key}") if date.fromisoformat(row["status_as_of"]) > date.fromisoformat(row["last_verified_at"][:10]): raise ValueError("Future status observation") if row["record_scope"] == "complex" and row["aggregation_role"] != "aggregate": raise ValueError("Complex must be aggregate") if row["aggregation_role"] == "child" and not row.get("parent_record_id"): raise ValueError("Child requires parent") if row.get("historical_capacity_mw") is not None and not row.get("dataset_vintage_year"): raise ValueError("Historical capacity requires dataset vintage") if (row.get("latitude") is None) != (row.get("longitude") is None): raise ValueError("Coordinates must be paired") if row.get("latitude") is not None and not row.get("location_accuracy"): raise ValueError("Coordinates require precision") if not row.get("fuel_type"): raise ValueError("Require explicit gas fuel classification") for field in CAPACITIES: prefix = field.removesuffix("_mw") if row.get(field) is None: continue if not row.get(prefix + "_basis"): raise ValueError("Electrical capacity requires gross/net/unspecified basis") if field != "historical_capacity_mw": observed = row.get(prefix + "_as_of") if not observed or observed > row["last_verified_at"][:10]: raise ValueError("Capacity needs a nonfuture evidence date") for obs in json.loads(row.get("performance_observations_json") or "[]"): required = { "metric", "value", "unit", "fuel_basis", "electrical_basis", "conditions", "period", "boundary", "source_id", "locator", } if not isinstance(obs, dict) or not required <= obs.keys(): raise ValueError("Performance needs conditions, units, period and evidence") if ( obs["fuel_basis"] not in {"HHV", "LHV", "unspecified"} or obs["electrical_basis"] not in {"gross", "net", "unspecified"} or obs["source_id"] not in row["sources"] ): raise ValueError("Invalid performance basis or source") for obs in json.loads(row.get("observations_json") or "[]"): if ( not {"field", "value", "unit", "kind", "period", "boundary", "source_id", "locator"} <= obs.keys() ): raise ValueError("Observation needs period, boundary and evidence") if obs["source_id"] not in row["sources"] or obs["kind"] not in { "reported", "measured", "modeled", "design", }: raise ValueError("Invalid observation provenance")
[docs] def validate_catalogue(rows: list[dict[str, Any]], sources: dict[str, Any]) -> None: """Reject duplicate IDs, external identities and invalid/cyclic hierarchy.""" lookup = {r["record_id"]: r for r in rows} if len(lookup) != len(rows): raise ValueError("Duplicate record ID") seen: set[tuple[str, str]] = set() for row in rows: validate_record(row, sources) for registry, value in json.loads(row.get("external_ids_json") or "{}").items(): key = (registry, str(value)) if key in seen: raise ValueError("Duplicate registry identity") seen.add(key) visited = {row["record_id"]} child = row while parent := child.get("parent_record_id"): if parent not in lookup or parent in visited: raise ValueError("Invalid or cyclic parent") visited.add(parent) child = lookup[parent] if child["aggregation_role"] != "aggregate": raise ValueError("Parent capacity must be excluded as aggregate")
[docs] def capacity_summary(rows: list[dict[str, Any]], field: str) -> dict[str, Any]: """Separate lifecycle, technology and historical vintage; exclude ambiguous scopes.""" if field not in CAPACITIES: raise ValueError("Choose an explicit gas capacity category") prefix = field.removesuffix("_mw") selected = [ r for r in rows if r["aggregation_role"] in {"independent", "child"} and not r.get("overlap_group") ] result: dict[str, Any] = { "excluded": len(rows) - len(selected), "unknown": sum(r.get(field) is None for r in selected), "unqualified": 0, "by_status": {}, } for row in selected: if row.get(field) is None: continue basis = row.get(prefix + "_basis") period = ( row.get("capacity_reference_year") if field == "historical_capacity_mw" else row.get(prefix + "_as_of") ) if basis not in {"gross", "net"} or not period: result["unqualified"] += 1 continue key = f"{row['status']}/{row['technology']}/{basis}@{period}" group = result["by_status"].setdefault(key, {"reported_mw": 0.0, "records": 0}) group["reported_mw"] += row[field] group["records"] += 1 return result
[docs] def build_parquet(root: Path, output: Path) -> dict[str, Any]: """Validate JSONL and publish an immutable, typed Zstandard snapshot.""" rows = load_records(root) sources = json.loads((root / "sources.json").read_text()) validate_catalogue(rows, sources) return write_snapshot( output, rows, sources, SCHEMA, "gas", {k: capacity_summary(rows, k) for k in CAPACITIES} )
[docs] def upsert_records( existing: list[dict[str, Any]], incoming: list[dict[str, Any]], *, now: str ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: """Preserve creation dates and quarantine conflicts using the shared upsert.""" return _upsert_records(existing, incoming, now=now)
__all__ = [ "SCHEMA", "build_parquet", "capacity_summary", "stable_id", "upsert_records", "validate_catalogue", "validate_record", ]