Source code for energy_data_retrieval.wind_catalogue.epe

"""Checksum-pinned EPE-hosted SIGA observations, without lifecycle promotion."""

from __future__ import annotations

import copy
import hashlib
import json
import math
import re
from collections import defaultdict
from datetime import date
from pathlib import Path
from typing import Any

from energy_data_retrieval.pv_catalogue.catalogue import canonical_json

REFERENCE_IDS = (
    "epe_siga_bahia_20260113",
    "epe_siga_ceara_20260113",
    "epe_siga_rio_grande_do_norte_20260113",
    "wri_brazil_identity",
)
POWER_FIELDS = {
    "mdapotenciaoutorgadakw": "authorized_mw",
    "mdapotenciafiscalizadakw": "inspected_mw",
    "mdagarantiafisicakw": "physical_guarantee_mw",
}
DATE_FIELDS = (
    "datgeracaoconjuntodados",
    "datentradaoperacao",
    "datiniciovigencia",
    "datfimvigencia",
)


[docs] def verify_capture(batch: dict[str, Any], directory: Path) -> None: """Reparse checksum-pinned original pages and both ID queries without writes. The reviewed ledger remains the acceptance boundary. This check additionally proves its candidate transcription agrees with the retained original bytes. It does not establish the accuracy of the registry or reconcile farm scope. """ originals = {} for name, digest in batch["checksums"].items(): if Path(name).name != name: raise ValueError("Capture filenames must be basenames") data = (directory / name).read_bytes() if hashlib.sha256(data).hexdigest() != digest: raise ValueError(f"Capture checksum mismatch: {name}") originals[name] = data page_names = [entry["name"] + ".json" for entry in batch["downloads"]] id_names = [name for name in originals if name.endswith("-ids.json")] if len(id_names) != 1 or not page_names: raise ValueError("Capture requires pages and one before/after ID pair") first = id_names[0] second = first.removesuffix("-ids.json") + "-ids-after.json" candidates = batch["all_reviewed_candidates"] parsed = parse_partition( [originals[name] for name in page_names], sha256=[batch["checksums"][name] for name in page_names], expected_ids=json.loads(originals[first])["objectIds"], confirmed_ids=json.loads(originals[second])["objectIds"], state=candidates[0]["original"]["sigufprincipal"], retrieved_at=batch["retrieved_at"], ) if parsed != candidates or len(parsed) != batch["reviewed"]: raise ValueError("Reviewed candidates differ from original capture")
[docs] def ceg_key(value: str) -> tuple[str, int, int]: """Compare CEG suffix zero padding only; never collapse revisions or nuclei.""" match = re.fullmatch(r"(EOL\.[A-Z]{2}\.[A-Z]{2}\.\d{6})-(\d)\.(\d{1,2})", value) if not match or int(match[1].rsplit(".", 1)[1]) == 0: raise ValueError("Invalid EPE wind CEG") return match[1], int(match[2]), int(match[3])
[docs] def parse_partition( pages: list[bytes], *, sha256: list[str], expected_ids: list[int], confirmed_ids: list[int], state: str, retrieved_at: str, ) -> list[dict[str, Any]]: """Validate complete ID-addressed pages and retain original SIGA quantities. The two independent ID queries must agree. The layer's dataset date is not retrieval time or an assertion of present operation. kW values are divided by 1000 separately; physical guarantee is not installed capacity. Coordinates returned by the service in EPSG:4326 are observations of unknown accuracy. """ if ( not pages or len(pages) != len(sha256) or not re.fullmatch(r"[A-Z]{2}", state) or not expected_ids or any(type(i) is not int for i in expected_ids + confirmed_ids) or len(set(expected_ids)) != len(expected_ids) or len(set(confirmed_ids)) != len(confirmed_ids) or set(expected_ids) != set(confirmed_ids) ): raise ValueError("Invalid or drifting EPE partition IDs") selected: list[dict[str, Any]] = [] seen: set[int] = set() for data, digest in zip(pages, sha256, strict=True): if hashlib.sha256(data).hexdigest() != digest: raise ValueError("EPE original checksum mismatch") doc = json.loads(data) if ( "error" in doc or doc.get("exceededTransferLimit") is not False or doc.get("spatialReference", {}).get("wkid") != 4326 ): raise ValueError("Incomplete EPE response or unexpected coordinate system") for feature in doc["features"]: raw = feature["attributes"] oid = raw["objectid"] if type(oid) is not int or oid in seen: raise ValueError("Duplicate or invalid EPE object ID") seen.add(oid) if raw["sigtipogeracao"] != "EOL" or raw["sigufprincipal"] != state: raise ValueError("EPE row outside selected partition") ceg = raw["codceg"] key = ceg_key(ceg) if key[0].split(".")[2] != state or not raw["nomempreendimento"]: raise ValueError("EPE identity/state disagreement") for field in DATE_FIELDS: value = raw[field] if value is not None: if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", value): raise ValueError("Invalid EPE date precision") date.fromisoformat(value) if raw["datgeracaoconjuntodados"] is None: raise ValueError("Missing EPE dataset date") normalized = {} for field, label in POWER_FIELDS.items(): value = raw[field] if value is not None and ( type(value) not in (int, float) or not math.isfinite(value) or value < 0 ): raise ValueError("Invalid EPE kW quantity") normalized[label] = None if value is None else value / 1000 geometry = feature.get("geometry") if geometry is not None: for field, bound in (("x", 180), ("y", 90)): value = geometry[field] if ( type(value) not in (int, float) or not math.isfinite(value) or abs(value) > bound ): raise ValueError("Invalid EPE coordinate") selected.append( dict( ceg=ceg, wri_gppd=f"BRA{int(key[0].rsplit('.', 1)[1]):07d}", name=raw["nomempreendimento"], object_id=oid, original=copy.deepcopy(raw), normalized_power=normalized, normalization=( "Original kW / 1000 to MW; separate reported categories; " "gross/net unspecified" ), geometry=copy.deepcopy(geometry), coordinate_reference="EPSG:4326 returned by service; native layer EPSG:4674", positional_accuracy_m=None, coordinate_semantics=( "Registry point, not boundary or turbine location; " "decimal precision is not accuracy" ), observation_date=raw["datgeracaoconjuntodados"], retrieved_at=retrieved_at, source_sha256=digest, locator=f"Layer0 objectid={oid}; codceg={ceg}; original field names retained", scope=( "Dated SIGA registry observation; no current operating-census, " "turbine completeness or aggregation eligibility inference" ), ) ) if seen != set(expected_ids): raise ValueError("Missing or unexpected EPE partition rows") if len({c["observation_date"] for c in selected}) != 1: raise ValueError("Mixed EPE dataset dates") if len({c["ceg"] for c in selected}) != len(selected): raise ValueError("Duplicate EPE CEG") return sorted(selected, key=lambda c: c["ceg"])
[docs] def reconcile( candidates: list[dict[str, Any]], existing: list[dict[str, Any]], *, source_id: str, identity_source_id: str, now: str, ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: """Attach unique WRI numeric-ID plus exact-name matches as dated observations. Neither name-only matches nor multiple CEG revisions are accepted. Existing CEGs are checked without changing their spelling. Replays are idempotent; changed observations under the same source ID are rejected. """ if len({r["record_id"] for r in existing}) != len(existing): raise ValueError("Duplicate catalogue identity") if len({c["ceg"] for c in candidates}) != len(candidates): raise ValueError("Duplicate EPE candidate") by_wri: dict[str, list[dict[str, Any]]] = defaultdict(list) revisions: dict[str, list[str]] = defaultdict(list) for row in existing: if row["entity_type"] == "projects" and row.get("country_code") == "BRA": ids = json.loads(row.get("external_ids_json") or "{}") by_wri[ids.get("wri_gppd")].append(row) for item in candidates: revisions[item["wri_gppd"]].append(item["ceg"]) enriched, ledger = [], [] for item in candidates: matches = by_wri.get(item["wri_gppd"], []) reason = None if len(revisions[item["wri_gppd"]]) != 1: reason = "Multiple CEG versions; phase/repowering identity unresolved" elif len(matches) != 1: reason = "Missing or nonunique WRI identity; no new farm inferred" elif matches[0]["name"].strip().casefold() != item["name"].strip().casefold(): reason = "Registry ID/name disagreement; no fuzzy match" elif ( prior_ceg := json.loads(matches[0].get("external_ids_json") or "{}").get("aneel_ceg") ) and ceg_key(prior_ceg) != ceg_key(item["ceg"]): reason = "Existing CEG disagreement; no registry overwrite" elif any( ceg_key(observation["ceg"]) != ceg_key(item["ceg"]) for observation in json.loads(matches[0].get("extensions_json") or "{}").get( "epe_siga_snapshots", [] ) ): reason = "Prior EPE CEG revision disagreement; phase/repowering crosswalk required" decision = dict( ceg=item["ceg"], wri_gppd=item["wri_gppd"], object_id=item["object_id"], name=item["name"], decision="deferred" if reason else "enriched", reason=reason, ) ledger.append(decision) if reason: continue row = copy.deepcopy(matches[0]) decision["record_id"] = row["record_id"] ext = json.loads(row.get("extensions_json") or "{}") history = ext.setdefault("epe_siga_snapshots", []) observation = {**copy.deepcopy(item), "source_id": source_id} previous = [o for o in history if o["source_id"] == source_id] if previous: if previous != [observation]: raise ValueError("Immutable EPE observation changed") enriched.append(row) continue if any(o["source_sha256"] == item["source_sha256"] for o in history): raise ValueError("Duplicate EPE snapshot under another source") history.append(observation) fs, ev = json.loads(row["field_sources_json"]), json.loads(row["evidence_json"]) fs["extensions_json"] = sorted( set([*fs.get("extensions_json", []), source_id, identity_source_id]) ) ev["extensions_json"] = dict( prior_observation=ev.get("extensions_json"), source_id=source_id, locator=item["locator"], confidence="high", normalization=( "WRI numeric CEG identity plus exact name; " "original CEG suffix zero padding checked; scalars unchanged" ), ) row.update( extensions_json=canonical_json(ext), field_sources_json=canonical_json(fs), evidence_json=canonical_json(ev), sources=sorted(set([*row["sources"], source_id, identity_source_id])), updated_at=now, last_verified_at=now, ) enriched.append(row) return sorted(enriched, key=lambda r: r["record_id"]), ledger
BRAZIL_STATES = frozenset( [ "AC", "AL", "AM", "AP", "BA", "CE", "DF", "ES", "GO", "MA", "MG", "MS", "MT", "PA", "PB", "PE", "PI", "PR", "RJ", "RN", "RO", "RR", "RS", "SC", "SE", "SP", "TO", ] )
[docs] def audit_coverage( partitions: dict[str, list[int]], *, expected_ids: list[int], confirmed_ids: list[int], unknown_state_ids: list[int], ) -> dict[str, Any]: """Check saved partition identities against a fresh complete service roster. An empty partition is an explicit empty ID list, never an absent query or null response. This proves ID coverage at capture time only, not accuracy, project acceptance, lifecycle completeness, or a current operating census. """ if set(partitions) != BRAZIL_STATES: raise ValueError("Coverage requires all 27 explicit state partitions") lists = [*partitions.values(), expected_ids, confirmed_ids, unknown_state_ids] if any( not isinstance(ids, list) or any(type(i) is not int or i < 0 for i in ids) or len(ids) != len(set(ids)) for ids in lists ): raise ValueError("Invalid coverage IDs; missing is not empty") if set(expected_ids) != set(confirmed_ids): raise ValueError("National EPE roster drift") if unknown_state_ids: raise ValueError("Unassigned state identities require separate review") combined = [i for ids in partitions.values() for i in ids] if len(combined) != len(set(combined)): raise ValueError("Overlapping EPE state partitions") if set(combined) != set(expected_ids): raise ValueError("Saved EPE partitions differ from current roster") return dict( reviewed_ids=len(combined), populated_states=sorted(s for s, ids in partitions.items() if ids), empty_states=sorted(s for s, ids in partitions.items() if not ids), unassigned_ids=0, scope="EPE-hosted SIGA EOL roster at capture; not a current operating census", )
[docs] def parse_id_response(data: bytes, sha256: str) -> list[int]: """Read a checksum-pinned ArcGIS ID response; errors/null are not zero.""" if hashlib.sha256(data).hexdigest() != sha256: raise ValueError("EPE ID response checksum mismatch") doc = json.loads(data) ids = doc.get("objectIds") if ( "error" in doc or doc.get("objectIdFieldName") != "objectid" or doc.get("exceededTransferLimit") is True or not isinstance(ids, list) or any(type(i) is not int or i < 0 for i in ids) or len(set(ids)) != len(ids) ): raise ValueError("Invalid EPE ID response; unknown is not zero") return sorted(ids)