Source code for energy_data_retrieval.wind_catalogue.boe

"""Offline BOE discovery and reviewed notice staging with technology boundaries.

The index adapter selects every Series A notice mentioning wind; it never reads
MW or a lifecycle from the title. Reviewed facts require checksummed originals.
"""

from __future__ import annotations

import hashlib
import json
import re
import unicodedata
from html.parser import HTMLParser
from typing import Any
from urllib.parse import parse_qs, urljoin, urlsplit

from energy_data_retrieval.pv_catalogue.catalogue import canonical_json

from .catalogue import stable_id
from .gppd import evidence_record

REFERENCE_IDS = ("boe_a_2026_22987",)


def _registry_keys(row: dict[str, Any]) -> set[tuple[str, str]]:
    """Only project/procedure registries, never a notice shared by several farms.

    A shared procedure triggers review; it does not prove two farms are identical.
    Namespace and identifier are exact: no punctuation or case-based aliasing.
    """
    identifiers = json.loads(row.get("external_ids_json") or "{}")
    return {
        (namespace, value)
        for namespace, value in identifiers.items()
        if namespace in {"gencat_expediente", "miteco_expediente"}
        and isinstance(value, str)
        and value.strip()
    }


def _normalized(text: str) -> str:
    return " ".join(
        "".join(c for c in unicodedata.normalize("NFKD", text) if not unicodedata.combining(c))
        .casefold()
        .split()
    )


def _names(row: dict[str, Any]) -> set[str]:
    """Screen documented aliases as well as the current name; never merge by name."""
    return {_normalized(name) for name in [row["name"], *(row.get("aliases") or [])]}


class _Index(HTMLParser):
    def __init__(self, series: str = "A") -> None:
        super().__init__()
        self.series = series
        self.depth = 0
        self.paragraph = False
        self.title: list[str] = []
        self.ids: set[str] = set()
        self.notices: list[dict[str, str]] = []

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        values = dict(attrs)
        if tag == "li":
            if self.depth:
                self.depth += 1
            elif "dispo" in (values.get("class") or "").split():
                self.depth = 1
                self.title, self.ids = [], set()
        if self.depth and tag == "p":
            self.paragraph = True
        if self.depth and tag == "a":
            url = urlsplit(values.get("href") or "")
            if url.path == "/diario_boe/txt.php":
                self.ids.update(parse_qs(url.query).get("id", []))

    def handle_data(self, data: str) -> None:
        if self.paragraph:
            self.title.append(data)

    def handle_endtag(self, tag: str) -> None:
        if tag == "p":
            self.paragraph = False
        if tag == "li" and self.depth:
            self.depth -= 1
            if not self.depth:
                title = " ".join("".join(self.title).split())
                if "eolic" in _normalized(title):
                    if len(self.ids) != 1:
                        raise ValueError("Wind notice requires one document identifier")
                    ident = next(iter(self.ids))
                    if re.fullmatch(rf"BOE-{self.series}-\d{{4}}-\d+", ident):
                        self.notices.append({"notice_id": ident, "title": title})


[docs] def select_wind_notices(html: str, *, series: str = "A") -> list[dict[str, str]]: """Select Series A wind mentions, including solar hybrids and auxiliaries. Series B announcements require explicit ``series="B"`` selection. A selected title is a review candidate, never an automatic project record. """ if "sumario" not in html.casefold() or "dispo" not in html: raise ValueError("Expected a complete BOE daily index, not an error/login page") if series not in {"A", "B"}: raise ValueError("Select exactly one BOE series: A or B") parser = _Index(series) parser.feed(html) parser.close() if parser.depth: raise ValueError("Truncated BOE index") ids = [n["notice_id"] for n in parser.notices] if len(ids) != len(set(ids)): raise ValueError("Duplicate BOE notice") return parser.notices
[docs] def issue_urls(html: str, daily_url: str) -> list[str]: """Discover same-day extra issues; callers must fetch every returned URL. A default daily index may show only the extraordinary issue. Discovery cannot establish completeness of content that has not yet been fetched. """ base = urlsplit(daily_url) if ( base.scheme != "https" or base.netloc != "www.boe.es" or not re.fullmatch(r"/boe/dias/\d{4}/\d{2}/\d{2}/", base.path) or base.query ): raise ValueError("Expected an official BOE daily directory URL") found = {daily_url} class Links(HTMLParser): def handle_starttag(self, tag, attrs): if tag != "a": return href = dict(attrs).get("href", "") url = urlsplit(urljoin(daily_url, href)) query = parse_qs(url.query) if ( url.scheme == base.scheme and url.netloc == base.netloc and url.path == base.path + "index.php" and set(query) == {"d"} and len(query["d"]) == 1 and query["d"][0].isdigit() ): found.add(urljoin(daily_url, href).split("#")[0]) Links().feed(html) return sorted(found)
[docs] def review_daily_indexes(daily_url: str, originals: dict[str, bytes]) -> dict[str, Any]: """Replay all discovered issues before declaring a daily title scan complete. Missing extra issues fail closed, including when the default issue contains no wind titles. Aliased issues may repeat a notice; conflicting titles fail. This is title-discovery completeness, not an inventory of every wind project. """ if daily_url not in originals: raise ValueError("Missing default daily index") expected = set(issue_urls(originals[daily_url].decode("utf-8"), daily_url)) for raw in originals.values(): expected.update(issue_urls(raw.decode("utf-8"), daily_url)) if set(originals) != expected: raise ValueError("Missing or unexpected daily issue URLs") notices: dict[str, dict[str, str]] = {} captures = [] for url in sorted(originals): raw = originals[url] text = raw.decode("utf-8") for series in ("A", "B"): for notice in select_wind_notices(text, series=series): ident = notice["notice_id"] if ident in notices and notices[ident] != notice: raise ValueError("Conflicting notice titles across issues") notices[ident] = notice captures.append({"url": url, "sha256": hashlib.sha256(raw).hexdigest()}) return { "daily_url": daily_url, "scope": "Complete discovered Series A/B wind-title scan", "captures": captures, "notices": [notices[key] for key in sorted(notices)], }
[docs] def stage_boe_notices( reviews: list[dict[str, Any]], originals: dict[str, bytes], existing: list[dict[str, Any]], *, expected_ids: list[str], now: str, ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: """Stage a complete reviewed partition; keep procedural and plant scopes apart. All new records remain excluded from capacity sums until farm boundaries are independently reconciled. Existing names require explicit reconciliation. """ ids = [r["notice_id"] for r in reviews] if len(ids) != len(set(ids)) or len(expected_ids) != len(set(expected_ids)): raise ValueError("Duplicate review/selection identifier") if set(ids) != set(expected_ids): raise ValueError("Incomplete notice review") names = { name for r in existing if r["entity_type"] == "projects" and r.get("country_code") == "ESP" for name in _names(r) } record_ids = {r["record_id"] for r in existing} registry_keys = set().union( *( _registry_keys(r) for r in existing if r["entity_type"] == "projects" and r.get("country_code") == "ESP" ) ) rows, decisions = [], [] for review in reviews: ident = review["notice_id"] if hashlib.sha256(originals[ident]).hexdigest() != review["sha256"]: raise ValueError("Original BOE checksum mismatch") if "projects" in review: if not review["projects"]: raise ValueError("Empty multi-project notice") for project in review["projects"]: child = {k: v for k, v in review.items() if k != "projects"} child.update(project) if child["notice_id"] != ident or child["sha256"] != review["sha256"]: raise ValueError("Project cannot replace its notice provenance") staged, ledger = stage_boe_notices( [child], originals, existing + rows, expected_ids=[ident], now=now ) rows.extend(staged) names.update(name for r in staged for name in _names(r)) record_ids.update(r["record_id"] for r in staged) registry_keys.update(key for r in staged for key in _registry_keys(r)) decisions.extend({**d, "identity": child["identity"]} for d in ledger) continue if not review.get("values"): decisions.append( {"notice_id": ident, "decision": "deferred", "reason": review["reason"]} ) continue values = dict(review["values"]) if values.get("country_code") != "ESP" or not values.get("technology"): raise ValueError("Reviewed country and turbine technology required") target, effect = review["decision_target"], review["decision_effect"] if review.get("required_turbine_removals") and ( any( values.get(field) is not None for field in ("planned_capacity_mw", "installed_capacity_mw", "turbine_count") ) or json.loads(values.get("fleet_json") or "[]") ): raise ValueError( "Required turbine removals need a separately reviewed revised design; " "retain original proposal and arithmetic in observations" ) if target not in {"wind", "solar", "auxiliary"}: raise ValueError("Explicit decision technology boundary required") if target != "wind" and values["status"] == "cancelled": raise ValueError("Non-wind decision cannot cancel the wind farm") if effect in {"application_refusal", "withdrawn"} and values["status"] not in { "cancelled", "unknown", }: raise ValueError( "Application refusal or withdrawal cannot establish operation or retirement" ) if effect == "public_consultation" and values["status"] not in {"announced", "unknown"}: raise ValueError("Public consultation does not establish permission or operation") if effect in { "repowering_authorization", "construction_authorization", "preliminary_authorization", } and values["status"] not in { "permitted", "unknown", }: raise ValueError( "Repowering authorization or construction authorization " "does not establish replacement operation" ) if effect == "environmental_conditions" and values["status"] not in { "announced", "unknown", }: raise ValueError( "Environmental decision alone is not construction authorization or operation" ) if ( target == "wind" and ( values["status"] in {"announced", "permitted", "cancelled"} or effect in { "withdrawn", "application_refusal", "public_consultation", "construction_authorization", "preliminary_authorization", "repowering_authorization", "environmental_conditions", } ) and values.get("installed_capacity_mw") is not None ): raise ValueError("Proposed authorization MW is not installed capacity") if set(values) != set(review["locators"]): raise ValueError("Every reviewed field requires a locator") rid = stable_id("projects", "BOE", review["identity"]) shared_keys = _registry_keys(values) & registry_keys if shared_keys: decisions.append( { "notice_id": ident, "decision": "existing_registry_review", "shared_identifiers": [list(key) for key in sorted(shared_keys)], "reason": ( "Shared procedure requires farm/phase reconciliation, not automatic merge" ), } ) continue if rid in record_ids or _names(values) & names: decisions.append({"notice_id": ident, "decision": "existing_identity_review"}) continue names.update(_names(values)) record_ids.add(rid) registry_keys.update(_registry_keys(values)) values.update( record_id=rid, entity_type="projects", record_scope=values.get("record_scope", "farm"), aggregation_role="unresolved", ) row = evidence_record(values, source_id=ident, now=now, locator=review["locator"]) evidence = json.loads(row["evidence_json"]) for field, locator in review["locators"].items(): if field in evidence: evidence[field].update(locator=locator, publication_date=review["published_at"]) field_sources = json.loads(row["field_sources_json"]) for field, observations in review.get("supporting_evidence", {}).items(): if field not in evidence: raise ValueError("Supporting evidence requires an existing factual field") for observation in observations: sid = observation["source_id"] if sid not in ids or not observation.get("locator"): raise ValueError("Supporting notice must be reviewed with a locator") row["sources"] = sorted(set(row["sources"] + [sid])) field_sources[field] = sorted(set(field_sources[field] + [sid])) evidence[field]["supporting_evidence"] = observations row["field_sources_json"] = canonical_json(field_sources) row["evidence_json"] = canonical_json(evidence) rows.append(row) decisions.append({"notice_id": ident, "decision": "added", "record_id": rid}) return sorted(rows, key=lambda r: r["record_id"]), decisions