Source code for energy_data_retrieval.hydro_catalogue.eia

"""Bounded EIA-860 hydro enrichment with exact registry identity and dated unit evidence."""

from __future__ import annotations

import copy
import json
import math
from collections import defaultdict
from typing import Any

from energy_data_retrieval.pv_catalogue.catalogue import canonical_json
from energy_data_retrieval.wind_catalogue.eia import _rows

UNIT_FIELDS = (
    "Generator ID",
    "Prime Mover",
    "Technology",
    "Status",
    "Nameplate Capacity (MW)",
    "Summer Capacity (MW)",
    "Winter Capacity (MW)",
    "Operating Month",
    "Operating Year",
    "Utility ID",
    "Utility Name",
    "Energy Source 1",
    "Turbines or Hydrokinetic Buoys",
)
PLANT_FIELDS = (
    "Plant Code",
    "Plant Name",
    "Latitude",
    "Longitude",
    "State",
    "County",
    "Balancing Authority Code",
    "Balancing Authority Name",
    "Grid Voltage (kV)",
)


[docs] def parse_partition( plant_bytes: bytes, generator_bytes: bytes, *, year: int, first: int, last: int ) -> list[dict[str, Any]]: """Read every operable generator at each hydro-containing plant in an ID range. Operable includes standby/out-of-service units: eligibility is checked later. No generator ID is numerically normalized; leading zeroes are identity text. """ if first < 1 or last < first: raise ValueError("Invalid plant-code partition") plants = _rows(plant_bytes, "Plant", PLANT_FIELDS, year) units = _rows(generator_bytes, "Operable", ("Plant Code", *UNIT_FIELDS), year) grouped: dict[int, list[dict[str, Any]]] = defaultdict(list) seen = set() for unit in units: code = unit["Plant Code"] if code is None and str(unit.get("Utility ID", "")).startswith("NOTE:"): if any(v is not None for k, v in unit.items() if k not in {"Utility ID", "_row"}): raise ValueError("Malformed EIA footer") continue if type(code) is not int or code < 1: raise ValueError("Missing or invalid generator plant code") if not first <= code <= last: continue key = (code, str(unit["Generator ID"])) if key in seen or not unit["Generator ID"]: raise ValueError("Missing or duplicate generator identity") seen.add(key) grouped[code].append({k: unit[k] for k in (*UNIT_FIELDS, "_row")}) results = [] plant_ids = set() for plant in plants: code = plant["Plant Code"] if code in plant_ids: raise ValueError("Duplicate plant code") plant_ids.add(code) selected = grouped.get(code, []) if not any(u["Prime Mover"] in {"HY", "PS"} for u in selected): continue results.append( { "year": year, "plant_code": code, "plant": {k: plant[k] for k in (*PLANT_FIELDS, "_row")}, "units": sorted(selected, key=lambda u: str(u["Generator ID"])), } ) orphaned = [ code for code, selected in grouped.items() if code not in plant_ids and any(u["Prime Mover"] in {"HY", "PS"} for u in selected) ] if orphaned: raise ValueError(f"Hydro generator plants absent from Plant sheet: {sorted(orphaned)}") return sorted(results, key=lambda r: r["plant_code"])
[docs] def eligibility(existing: dict[str, Any] | None, item: dict[str, Any]) -> str: """No fuzzy matching; different source scopes and ambiguous identities stay on hold.""" if existing is None: return "missing_existing_id" if existing.get("status") != "unknown" and existing.get("status_as_of", "") > ( f"{item['year']}-12-31" ): return "newer_status_review" ids = json.loads(existing["external_ids_json"]) code = item["plant_code"] if existing["country_code"] != "USA" or ids.get("wri_gppd") != f"USA{code:07d}": return "registry_id_review" if existing["name"] != item["plant"]["Plant Name"]: return "name_review" for old, new in (("latitude", "Latitude"), ("longitude", "Longitude")): a, b = existing.get(old), item["plant"].get(new) if a is None or b is None or not math.isfinite(b) or abs(a - b) > 0.0001: return "coordinate_review" if not item["units"] or any( u["Prime Mover"] != "HY" or u["Status"] != "OP" for u in item["units"] ): return "mixed_technology_or_state" seen = set() for unit in item["units"]: identity = str(unit["Generator ID"]) if not identity or identity in seen: return "duplicate_unit_review" seen.add(identity) value = unit["Nameplate Capacity (MW)"] if isinstance(value, bool) or not isinstance(value, (int, float)): return "capacity_review" if not math.isfinite(value) or value < 0: return "capacity_review" return "accepted_enrichment"
[docs] def enrich( existing: dict[str, Any], item: dict[str, Any], *, source_id: str, identity_source_id: str, now: str, ) -> dict[str, Any]: """Retain plant UUID/history and all unit evidence; no promotion to 2026 operation. Aggregation stays unresolved, including split reporting facilities. Pumping power, storage energy, turbine count and hydraulic subtype are not inferred. """ decision = eligibility(existing, item) if decision != "accepted_enrichment": raise ValueError(decision) row = copy.deepcopy(existing) ids = json.loads(row["external_ids_json"]) code = str(item["plant_code"]) if ids.get("eia_plant_code", code) != code: raise ValueError("Conflicting EIA plant identity") ext = json.loads(row.get("extensions_json") or "{}") observation = {**item, "source_id": source_id} history = ext.setdefault("eia_hydro_annual_inventory", []) prior = [x for x in history if x["source_id"] == source_id] if prior: if prior != [observation]: raise ValueError("Immutable EIA observation changed") return row history.append(observation) ids["eia_plant_code"] = code mw = math.fsum(u["Nameplate Capacity (MW)"] for u in item["units"]) for key in ("installed_capacity_mw", "operational_capacity_mw"): if row.get(key) is not None and row[key] != mw: raise ValueError("Existing capacity requires conflict review") facts = { "external_ids_json": canonical_json(ids), "extensions_json": canonical_json(ext), "status": "operational", "status_as_of": f"{item['year']}-12-31", "status_date_basis": f"EIA final annual inventory for year {item['year']}; not live status", "installed_capacity_mw": mw, "operational_capacity_mw": mw, "capacity_scope": ( f"EIA plant {code}; all Operable-tab units HY and OP at year-end {item['year']}; " "nameplate generating MW, not net seasonal capability; cross-plant scope unresolved" ), "operator_names": sorted({u["Utility Name"] for u in item["units"]}), } fs = json.loads(row["field_sources_json"]) ev = json.loads(row["evidence_json"]) for key, value in facts.items(): row[key] = value refs = [source_id, identity_source_id] if key in {"external_ids_json", "extensions_json"}: refs = sorted(set(fs.get(key, []) + refs)) fs[key] = refs ev[key] = { "locator": f"2___Plant_Y{item['year']}.xlsx Plant row {item['plant']['_row']}; " "3_1_Generator Operable row locators in extensions_json", "confidence": "high", "unit": "MW" if key.endswith("_mw") else None, "normalization": "Registry ID/name/coordinate checks; sum distinct OP/HY units", "original_value": value, } row.update( field_sources_json=canonical_json(fs), evidence_json=canonical_json(ev), sources=sorted(set(row["sources"] + [source_id, identity_source_id])), updated_at=now, last_verified_at=now, notes=f"EIA year-end {item['year']} generating inventory enriched; " "historical WRI facts retained. " "Aggregation unresolved; excludes pumping power and energy-storage inference.", ) return row
[docs] def review_partition( existing: list[dict[str, Any]], items: list[dict[str, Any]], *, source_id: str, identity_source_id: str, now: str, ) -> dict[str, Any]: """Prepare reviewable decisions without mutating canonical records. Preserve full evidence for held identities, including mixed/pumped units. Existing-value conflicts are held rather than aborting a whole partition; duplicate registry identities abort because selecting either would be unsafe. """ lookup: dict[str, dict[str, Any]] = {} for row in existing: key = json.loads(row["external_ids_json"]).get("wri_gppd") if not key: continue if key in lookup: raise ValueError("Duplicate existing WRI identity") lookup[key] = row accepted, decisions = [], [] seen: set[int] = set() for item in sorted(items, key=lambda item: item["plant_code"]): code = item["plant_code"] if code in seen: raise ValueError("Duplicate inventory plant code") seen.add(code) row = lookup.get(f"USA{code:07d}") decision = eligibility(row, item) detail = None if decision == "accepted_enrichment" and row is not None: try: enrich( row, item, source_id=source_id, identity_source_id=identity_source_id, now=now ) except ValueError as error: decision, detail = "existing_value_conflict", str(error) else: accepted.append({"record_id": row["record_id"], "inventory": item}) entry = {"plant_code": code, "name": item["plant"]["Plant Name"], "decision": decision} if decision != "accepted_enrichment": entry["inventory"] = item if detail: entry["detail"] = detail decisions.append(entry) return {"accepted": accepted, "decisions": decisions}