"""Gas-fired plant identities, electrical boundaries and deterministic snapshots."""
from __future__ import annotations
import json
import re
import unicodedata
from datetime import date
from importlib.resources import files
from pathlib import Path
from typing import Any
from uuid import NAMESPACE_URL, uuid5
from energy_data_retrieval.catalogue_common import validate_common, write_snapshot
from energy_data_retrieval.data_center_catalogue.catalogue import upsert_records as _upsert_records
from energy_data_retrieval.pv_catalogue.catalogue import canonical_json, load_records
SCHEMA: dict[str, Any] = json.loads(files(__package__).joinpath("schema.json").read_text())
CAPACITIES = (
"installed_capacity_mw",
"operational_capacity_mw",
"planned_capacity_mw",
"historical_capacity_mw",
)
[docs]
def stable_id(entity_type: str, *identity: str) -> str:
"""Use immutable registry identity, never mutable plant capacity or ownership."""
parts = [" ".join(unicodedata.normalize("NFKC", s).split()) for s in identity]
if entity_type not in SCHEMA["entity_types"] or not parts or not all(parts):
raise ValueError("Require registered entity type and nonempty identity")
return str(
uuid5(NAMESPACE_URL, canonical_json(["energy-data-insights/gas", entity_type, *parts]))
)
[docs]
def validate_record(row: dict[str, Any], sources: dict[str, Any]) -> None:
"""Validate provenance, coordinates, status dates and electrical/thermal boundaries."""
validate_common(row, sources, SCHEMA)
if not re.fullmatch(r"[A-Z]{3}", row.get("country_code", "")):
raise ValueError("Require ISO alpha-3 country")
for key in ("status", "status_as_of", "status_date_basis", "technology", "aggregation_role"):
if not row.get(key):
raise ValueError(f"Require {key}")
if date.fromisoformat(row["status_as_of"]) > date.fromisoformat(row["last_verified_at"][:10]):
raise ValueError("Future status observation")
if row["record_scope"] == "complex" and row["aggregation_role"] != "aggregate":
raise ValueError("Complex must be aggregate")
if row["aggregation_role"] == "child" and not row.get("parent_record_id"):
raise ValueError("Child requires parent")
if row.get("historical_capacity_mw") is not None and not row.get("dataset_vintage_year"):
raise ValueError("Historical capacity requires dataset vintage")
if (row.get("latitude") is None) != (row.get("longitude") is None):
raise ValueError("Coordinates must be paired")
if row.get("latitude") is not None and not row.get("location_accuracy"):
raise ValueError("Coordinates require precision")
if not row.get("fuel_type"):
raise ValueError("Require explicit gas fuel classification")
for field in CAPACITIES:
prefix = field.removesuffix("_mw")
if row.get(field) is None:
continue
if not row.get(prefix + "_basis"):
raise ValueError("Electrical capacity requires gross/net/unspecified basis")
if field != "historical_capacity_mw":
observed = row.get(prefix + "_as_of")
if not observed or observed > row["last_verified_at"][:10]:
raise ValueError("Capacity needs a nonfuture evidence date")
for obs in json.loads(row.get("performance_observations_json") or "[]"):
required = {
"metric",
"value",
"unit",
"fuel_basis",
"electrical_basis",
"conditions",
"period",
"boundary",
"source_id",
"locator",
}
if not isinstance(obs, dict) or not required <= obs.keys():
raise ValueError("Performance needs conditions, units, period and evidence")
if (
obs["fuel_basis"] not in {"HHV", "LHV", "unspecified"}
or obs["electrical_basis"] not in {"gross", "net", "unspecified"}
or obs["source_id"] not in row["sources"]
):
raise ValueError("Invalid performance basis or source")
for obs in json.loads(row.get("observations_json") or "[]"):
if (
not {"field", "value", "unit", "kind", "period", "boundary", "source_id", "locator"}
<= obs.keys()
):
raise ValueError("Observation needs period, boundary and evidence")
if obs["source_id"] not in row["sources"] or obs["kind"] not in {
"reported",
"measured",
"modeled",
"design",
}:
raise ValueError("Invalid observation provenance")
[docs]
def validate_catalogue(rows: list[dict[str, Any]], sources: dict[str, Any]) -> None:
"""Reject duplicate IDs, external identities and invalid/cyclic hierarchy."""
lookup = {r["record_id"]: r for r in rows}
if len(lookup) != len(rows):
raise ValueError("Duplicate record ID")
seen: set[tuple[str, str]] = set()
for row in rows:
validate_record(row, sources)
for registry, value in json.loads(row.get("external_ids_json") or "{}").items():
key = (registry, str(value))
if key in seen:
raise ValueError("Duplicate registry identity")
seen.add(key)
visited = {row["record_id"]}
child = row
while parent := child.get("parent_record_id"):
if parent not in lookup or parent in visited:
raise ValueError("Invalid or cyclic parent")
visited.add(parent)
child = lookup[parent]
if child["aggregation_role"] != "aggregate":
raise ValueError("Parent capacity must be excluded as aggregate")
[docs]
def capacity_summary(rows: list[dict[str, Any]], field: str) -> dict[str, Any]:
"""Separate lifecycle, technology and historical vintage; exclude ambiguous scopes."""
if field not in CAPACITIES:
raise ValueError("Choose an explicit gas capacity category")
prefix = field.removesuffix("_mw")
selected = [
r
for r in rows
if r["aggregation_role"] in {"independent", "child"} and not r.get("overlap_group")
]
result: dict[str, Any] = {
"excluded": len(rows) - len(selected),
"unknown": sum(r.get(field) is None for r in selected),
"unqualified": 0,
"by_status": {},
}
for row in selected:
if row.get(field) is None:
continue
basis = row.get(prefix + "_basis")
period = (
row.get("capacity_reference_year")
if field == "historical_capacity_mw"
else row.get(prefix + "_as_of")
)
if basis not in {"gross", "net"} or not period:
result["unqualified"] += 1
continue
key = f"{row['status']}/{row['technology']}/{basis}@{period}"
group = result["by_status"].setdefault(key, {"reported_mw": 0.0, "records": 0})
group["reported_mw"] += row[field]
group["records"] += 1
return result
[docs]
def build_parquet(root: Path, output: Path) -> dict[str, Any]:
"""Validate JSONL and publish an immutable, typed Zstandard snapshot."""
rows = load_records(root)
sources = json.loads((root / "sources.json").read_text())
validate_catalogue(rows, sources)
return write_snapshot(
output, rows, sources, SCHEMA, "gas", {k: capacity_summary(rows, k) for k in CAPACITIES}
)
[docs]
def upsert_records(
existing: list[dict[str, Any]], incoming: list[dict[str, Any]], *, now: str
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
"""Preserve creation dates and quarantine conflicts using the shared upsert."""
return _upsert_records(existing, incoming, now=now)
__all__ = [
"SCHEMA",
"build_parquet",
"capacity_summary",
"stable_id",
"upsert_records",
"validate_catalogue",
"validate_record",
]