Source code for energy_data_retrieval.hydro_catalogue.ons_new

"""Explicitly reviewed new ONS identities; no automatic name matching or MW sums."""

from __future__ import annotations

import json
import unicodedata
from pathlib import Path
from typing import Any

from energy_data_retrieval.pv_catalogue.catalogue import canonical_json, load_records

from .catalogue import SCHEMA, stable_id, upsert_records, validate_catalogue
from .gppd import evidence_record
from .ons import CEG, enrich_dispatch_metadata, wri_id


[docs] def name_key(value: str) -> str: """Conservative collision screen only; this key never authorizes a merge.""" value = unicodedata.normalize("NFKD", value).casefold() return "".join(c for c in value if c.isalnum())
[docs] def new_record(entry: dict[str, Any], *, now: str) -> dict[str, Any]: """Build a CEG-keyed record from reviewed unit evidence and attributed facts.""" item, review = entry["inventory"], entry["review"] ceg, units = item["ceg"], item["units"] if not CEG.fullmatch(ceg) or not units: raise ValueError("Require full hydro CEG and units") names = {u["nom_usina"] for u in units} if ( review.get("decision") != "reviewed_new_identity" or review.get("ceg") != ceg or names != {review.get("source_name")} or not review.get("identity_basis") ): raise ValueError("Explicit new-identity review does not match inputs") equipment = [u["cod_equipamento"] for u in units] if len(set(equipment)) != len(equipment) or any(u["ceg"] != ceg for u in units): raise ValueError("Duplicate equipment or mixed registry identity") sid = entry["source_id"] ext = { "ons_new_identity_review": review, "ons_hydro_unit_inventory": [ {**item, "source_id": sid, "retrieved_at": entry["source_retrieved_at"]} ], } row = evidence_record( { "record_id": stable_id("projects", "ANEEL-CEG", ceg), "entity_type": "projects", "name": review["source_name"], "country_code": "BRA", "record_scope": "plant", "capacity_scope": "CEG facility; unit observations are not a complete plant rating", "aggregation_role": "unresolved", "status": "unknown", "status_as_of": now[:10], "status_date_basis": "Review date; inventory does not establish current lifecycle", "technology": "unknown", "external_ids_json": canonical_json({"aneel_ceg": ceg}), "owner_names": sorted( {u["nom_agenteproprietario"] for u in units if u["nom_agenteproprietario"]} ), "operator_names": sorted( {u["nom_agenteoperador"] for u in units if u["nom_agenteoperador"]} ), "extensions_json": canonical_json(ext), }, source_id=sid, now=now, locator="CAPACIDADE_GERACAO.csv rows " + ",".join(str(u["_row"]) for u in units), ) row = enrich_dispatch_metadata(row, now=now) fs, ev = json.loads(row["field_sources_json"]), json.loads(row["evidence_json"]) allowed = { "installed_capacity_mw", "river", "basin", "technology", "turbine_type", "turbine_count", "reservoir_volume_m3", "commissioning_date", "environmental_constraints", } for field, fact in entry.get("facts", {}).items(): if field not in allowed or not fact.get("locator") or not fact.get("basis"): raise ValueError("Unsupported or unattributed reviewed fact") row[field] = fact["value"] fs[field] = [fact["source_id"]] ev[field] = { "locator": fact["locator"], "confidence": "high", "original_value": fact["original_value"], "unit": SCHEMA["fields"][field].get("unit"), "normalization": fact["basis"], } row["sources"] = sorted(set(row["sources"] + [fact["source_id"]])) row.update(field_sources_json=canonical_json(fs), evidence_json=canonical_json(ev)) return row
[docs] def accept_new(root: Path, batch: dict[str, Any], output: Path) -> int: """Preflight the whole batch, then append only new records to one reviewable shard. Exact CEGs, historical CEG-derived WRI IDs and normalized names/aliases all block new acceptance. A replay of an identical record is a no-op. Changed evidence or another record with the same identity needs a separate review. """ existing = load_records(root) lookup = {r["record_id"]: r for r in existing} incoming = [new_record(e, now=batch["reviewed_at"]) for e in batch["accepted"]] if len({r["record_id"] for r in incoming}) != len(incoming): raise ValueError("Duplicate incoming identity") new = [] for row in incoming: if row["record_id"] in lookup: if lookup[row["record_id"]] != row: raise ValueError("Existing record differs; conflict review required") continue ceg = json.loads(row["external_ids_json"])["aneel_ceg"] for prior in existing + new: ids = json.loads(prior.get("external_ids_json") or "{}") if ids.get("aneel_ceg") == ceg or ids.get("wri_gppd") == wri_id(ceg): raise ValueError("Existing registry identity; enrich rather than insert") if prior["country_code"] == "BRA" and name_key(row["name"]) in { name_key(n) for n in [prior["name"], *prior.get("aliases", [])] }: raise ValueError("Name collision requires explicit identity review") new.append(row) combined, conflicts = upsert_records(existing, new, now=batch["reviewed_at"]) if conflicts: raise ValueError("Unexpected insertion conflict") validate_catalogue(combined, json.loads((root / "sources.json").read_text())) if new: if output.exists(): raise ValueError("Refuse to overwrite an existing shard") output.parent.mkdir(parents=True, exist_ok=True) output.write_text( "".join(canonical_json(r) + "\n" for r in sorted(new, key=lambda r: r["record_id"])) ) return len(new)