"""Offline REPD reconciliation through WRI's explicit historical registry crosswalk.
Planning applications are not automatically new farms. This adapter enriches only
existing, singly linked historical identities with matching names; every other
selected candidate is retained in a decision ledger. Scope remains unresolved.
"""
from __future__ import annotations
import csv
import io
import json
import math
import re
from collections import Counter
from datetime import date, datetime
from typing import Any
from energy_data_retrieval.pv_catalogue.catalogue import canonical_json
from .gppd import evidence_record
STATUS = {
"Operational": "operational",
"Under Construction": "under_construction",
"Planning Permission Granted": "permitted",
"Appeal Granted": "permitted",
"Secretary of State - Granted": "permitted",
"Planning Application Submitted": "announced",
"Abandoned": "cancelled",
"Decommissioned": "decommissioned",
}
REQUIRED = {
"Ref ID",
"Site Name",
"Technology Type",
"Development Status",
"Installed Capacity (MWelec)",
"Country",
"No. of Turbines",
}
def _number(value: str) -> float | None:
if not value.strip():
return None
number = float(value)
if not math.isfinite(number) or number < 0:
raise ValueError("Invalid nonnegative REPD quantity")
return number
def _name(value: str) -> str:
return re.sub(r"[^a-z0-9]", "", value.casefold())
[docs]
def resubmission_label_match(current: str, historical: str) -> bool:
"""Compare names after removing only administrative resubmission labels.
This is a corroboration check after an explicit registry crosswalk, never an
identity matcher. Phase, extension, revised and repowering words are retained.
"""
def clean(value: str) -> str:
return _name(re.sub(r"\bresubmitted\b|\bre-?submission\b", "", value, flags=re.IGNORECASE))
return bool(clean(current)) and clean(current) == clean(historical)
[docs]
def reconcile_repd(
text: str,
crosswalk: str,
existing: list[dict[str, Any]],
*,
source_id: str,
crosswalk_source_id: str,
now: str,
as_of: str,
allow_resubmission_labels: bool = False,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
"""Return reviewed enrichments and decisions for all Wind CSV rows.
``text`` is decoded by the caller (Q2 2026 uses Windows-1252). ``crosswalk``
is WRI resources/GBR/matches_GBR.csv from a pinned commit. No ID-prefix
inference, fuzzy merge, coordinate conversion, model inference, capacity
substitution or aggregation approval occurs here. Dates are UTC ISO strings.
"""
if date.fromisoformat(as_of) > date.fromisoformat(now[:10]):
raise ValueError("Future REPD snapshot")
reader = csv.DictReader(io.StringIO(text))
if not set(reader.fieldnames or []) >= REQUIRED:
raise ValueError("Missing REPD headers")
raw_rows = [
(line, dict(r))
for line, r in enumerate(reader, 2)
if r["Technology Type"] in {"Wind Onshore", "Wind Offshore"}
]
ids = [r["Ref ID"] for _, r in raw_rows]
if any(not x or not x.isdigit() for x in ids) or len(ids) != len(set(ids)):
raise ValueError("Missing/duplicate/invalid REPD ID")
cr = csv.DictReader(io.StringIO(crosswalk))
if not {"gppd_id", "repd_id", "repd_id2", "repd_name", "fuel"} <= set(cr.fieldnames or []):
raise ValueError("Missing WRI crosswalk headers")
links: dict[str, list[dict[str, str]]] = {}
for entry in cr:
if entry["fuel"] == "Wind" and entry["repd_id"]:
links.setdefault(entry["repd_id"], []).append(entry)
baseline = {
json.loads(r.get("external_ids_json") or "{}").get("wri_gppd"): r
for r in existing
if r.get("country_code") == "GBR"
}
names = Counter(_name(r["Site Name"]) for _, r in raw_rows)
incoming, decisions = [], []
for line, raw in raw_rows:
ref = raw["Ref ID"]
locator = f"CSV record {line}; Ref ID={ref}; Q2 2026"
decision: dict[str, Any] = {
"repd_id": ref,
"name": raw["Site Name"],
"locator": locator,
"source_id": source_id,
"decision": "deferred_identity",
}
candidates = links.get(ref, [])
reason = None
if len(candidates) != 1:
reason = (
"No unique explicit WRI crosswalk; review as a separate application/phase candidate"
)
else:
link = candidates[0]
old = baseline.get(link["gppd_id"])
if old is None:
reason = "Crosswalk identity absent from baseline"
elif link["repd_id2"]:
reason = "Crosswalk has a second REPD application; scope review required"
elif _name(raw["Site Name"]) != _name(link["repd_name"]) and not (
allow_resubmission_labels
and resubmission_label_match(raw["Site Name"], link["repd_name"])
and raw["Installed Capacity (MWelec)"].strip()
and link.get("capacity", "").strip()
and float(raw["Installed Capacity (MWelec)"]) == float(link["capacity"])
):
reason = "Registry name differs from explicit historical crosswalk"
elif names[_name(raw["Site Name"])] != 1 and not (
allow_resubmission_labels
and _name(raw["Site Name"]) != _name(link["repd_name"])
and old.get("aggregation_role") == "unresolved"
):
reason = "Repeated registry name; application/phase identity review required"
elif old.get("status") != "unknown":
reason = (
"Already has primary lifecycle evidence; reconcile dated observations manually"
)
elif (
raw.get("Are they re-applying (New REPD Ref)", "").strip()
or raw.get("Are they re-applying (Old REPD Ref) ", "").strip()
or raw["Development Status"] == "Revised"
):
reason = (
"Revised/replacement planning application; not a new generation by inference"
)
if reason:
decision["reason"] = reason
decisions.append(decision)
continue
assert old is not None
try:
capacity = _number(raw["Installed Capacity (MWelec)"])
count = _number(raw["No. of Turbines"])
unit = _number(raw.get("Turbine Capacity (MW)", ""))
if count is not None and not count.is_integer():
raise ValueError("Nonintegral turbine count")
status = STATUS.get(raw["Development Status"], "unknown")
milestones = []
for column in (
"Planning Application Submitted",
"Planning Permission Granted",
"Under Construction",
"Operational",
"Planning Permission Expired",
):
if raw.get(column):
value = datetime.strptime(raw[column], "%d/%m/%Y").date().isoformat()
milestones.append(
{
"event": column.strip(),
"date": value,
"basis": "REPD application milestone; future dates are not achieved",
"source_id": source_id,
"locator": locator,
}
)
milestones.extend(procedural_milestones(raw, source_id=source_id, locator=locator))
observation = {
"field": "registry_application_capacity",
"value": capacity,
"unit": "MW",
"kind": "reported",
"period": as_of,
"boundary": (
"REPD application; source column does not itself establish commissioned MW"
),
"source_id": source_id,
"locator": locator + "; Installed Capacity (MWelec)",
}
observations = json.loads(old.get("observations_json") or "[]")
if capacity is not None:
observations.append(observation)
if unit is not None:
observations.append(
{
**observation,
"field": "registry_unit_rating",
"value": unit,
"boundary": (
"Registry unit rating; exact model and mixed-fleet composition unknown"
),
"locator": locator + "; Turbine Capacity (MW)",
}
)
extension = json.loads(old.get("extensions_json") or "{}")
extension["repd"] = {
"record_updated_original": raw.get("Record Last Updated (dd/mm/yyyy)"),
"development_status_original": raw["Development Status"],
"operator_or_applicant": raw.get("Operator (or Applicant)") or None,
"address": raw.get("Address") or None,
"county": raw.get("County") or None,
"country_within_uk": raw["Country"],
"x_coordinate_original": raw.get("X-coordinate") or None,
"y_coordinate_original": raw.get("Y-coordinate") or None,
"coordinate_note": (
"Original grid values retained; CRS not reconciled; no lat/lon conversion"
),
"height_of_turbines_m_original": raw.get("Height of Turbines (m)") or None,
"height_note": "Hub/tip definition not established; neither scalar populated",
}
values = {
"record_id": old["record_id"],
"external_ids_json": canonical_json(
{**json.loads(old["external_ids_json"]), "uk_repd": ref}
),
"status": status,
"status_as_of": as_of,
"status_date_basis": (
"REPD Q2 2026 snapshot published 2026-08-03; not retrieval-date operation"
),
"technology": "onshore"
if raw["Technology Type"] == "Wind Onshore"
else "offshore_unspecified",
"turbine_count": int(count) if count is not None else None,
"region": raw.get("Region") or raw["Country"],
"milestones_json": canonical_json(milestones),
"observations_json": canonical_json(observations),
"extensions_json": canonical_json(extension),
"permits_json": canonical_json(
{
k: raw.get(k) or None
for k in (
"Planning Authority",
"Planning Application Reference",
"Appeal Reference",
"Secretary of State Reference",
)
}
),
"offtake_json": canonical_json(
{
k: raw.get(k) or None
for k in (
"CfD Allocation Round",
"CfD Capacity (MW)",
"RO Banding (ROC/MWh)",
"FiT Tariff (p/kWh)",
)
}
),
}
if status == "operational":
values["installed_capacity_mw"] = capacity
elif status in {"announced", "permitted", "under_construction"}:
values["planned_capacity_mw"] = capacity
update = evidence_record(values, source_id=source_id, now=now, locator=locator)
merged = dict(old)
for key, value in values.items():
if value is not None:
merged[key] = value
merged.update(
updated_at=now,
last_verified_at=now,
sources=sorted(set(old["sources"]) | {source_id, crosswalk_source_id}),
)
for key in ("field_sources_json", "evidence_json"):
evidence: dict[str, Any] = {**json.loads(old[key]), **json.loads(update[key])}
if key == "field_sources_json":
evidence["external_ids_json"] = sorted(
set(old["sources"]) | {source_id, crosswalk_source_id}
)
for compound in ("observations_json", "extensions_json"):
evidence[compound] = sorted(set(old["sources"]) | {source_id})
else:
evidence["external_ids_json"]["locator"] += (
f"; WRI crosswalk gppd_id={link['gppd_id']}"
)
for detail in json.loads(update[key]):
evidence[detail]["source_revision"] = "Q2 2026"
evidence[detail]["source_publication_date"] = "2026-08-03"
if detail.endswith("_json"):
evidence[detail].pop("original_value", None)
merged[key] = canonical_json(evidence)
if old.get("source_urls") is not None:
merged.pop("source_urls") # Optional; source IDs remain authoritative.
incoming.append(merged)
decision.update(
identity_method=(
"explicit crosswalk; name corroboration; "
"capacity equality for resubmission-label changes"
),
repeated_name_scope_unresolved=names[_name(raw["Site Name"])] > 1,
decision="enriched",
record_id=old["record_id"],
wri_gppd=link["gppd_id"],
aggregation_role=old["aggregation_role"],
deferred_fields=[
"phase_scope",
"repowering",
"exact_fleet",
"coordinate_crs",
"turbine_height_basis",
],
)
except ValueError as exc:
decision.update(decision="deferred_invalid_value", reason=str(exc))
decisions.append(decision)
return incoming, decisions
PROCEDURAL_DATES = (
"Planning Application Withdrawn",
"Planning Permission Refused",
"Appeal Lodged",
"Appeal Withdrawn",
"Appeal Refused",
"Appeal Granted",
"Secretary of State - Intervened",
"Secretary of State - Refusal",
"Secretary of State - Granted",
)
[docs]
def procedural_milestones(
raw: dict[str, str], *, source_id: str, locator: str
) -> list[dict[str, str]]:
"""Read dated application history without treating a refusal as current status.
Dates are strictly day/month/year. Empty source cells produce no event; a
malformed nonempty date fails the record rather than becoming a null date.
Appeal success is not evidence of construction, operation or approved scope.
"""
events = []
for column in PROCEDURAL_DATES:
value = raw.get(column, "").strip()
if value:
events.append(
{
"event": column,
"date": datetime.strptime(value, "%d/%m/%Y").date().isoformat(),
"basis": "REPD application milestone; future dates are not achieved",
"source_id": source_id,
"locator": locator + "; " + column,
}
)
return events