Source code for energy_data_retrieval.bess_catalogue.mastr

"""Offline, review-only staging of MaStR revision 26.2 storage XML partitions.

Units and storage installations have different IDs and rating boundaries. No
site aggregation, fuzzy identity matching, operator-name lookup or COD inference
is performed. Downloaded XML and ZIP members remain outside the repository.
"""

from __future__ import annotations

import hashlib
import math
import re
import xml.etree.ElementTree as ET
from collections import Counter
from datetime import date
from pathlib import Path
from typing import Any


def _rows(path: Path, root_name: str, row_name: str):
    """Yield one flat source row at a time without retaining the XML tree."""
    with path.open("rb") as stream:
        context = ET.iterparse(stream, events=("start", "end"))
        _, root = next(context)
        if root.tag != root_name:
            raise ValueError(f"Unexpected XML root: {root.tag}")
        for event, element in context:
            if event == "end" and element.tag == row_name:
                keys = [child.tag for child in element]
                if len(set(keys)) != len(keys):
                    raise ValueError("Duplicate XML field")
                yield {child.tag: child.text for child in element}
                root.clear()


def _number(value: str | None) -> float | None:
    if value is None or value == "":
        return None
    result = float(value)
    if not math.isfinite(result) or result < 0:
        raise ValueError("Expected finite nonnegative registry rating")
    return result


def _identity(value: str | None, prefix: str) -> str:
    if not value or not re.fullmatch(prefix + r"\d{12}", value):
        raise ValueError(f"Invalid {prefix} identity")
    return value


[docs] def stage_mastr_storage( unit_paths: list[Path], storage_paths: list[Path], values_path: Path, *, snapshot_date: str, minimum_kw: float = 1000, ) -> list[dict[str, Any]]: """Stage ratings >= threshold, including flagged non-battery exclusions. Duplicate IDs fail across partitions. Shared locations are flagged using all supplied battery units, including those below the power threshold. This is not a guarantee of completeness across omitted export partitions. Energy joins require a reciprocal, single-unit storage link; otherwise energy is withheld. Raw source dates remain observations, never commercial COD. """ as_of = date.fromisoformat(snapshot_date) if not math.isfinite(minimum_kw) or minimum_kw < 0: raise ValueError("Invalid minimum power") values = {r["Id"]: r["Wert"] for r in _rows(values_path, "Katalogwerte", "Katalogwert")} if values.get("524") != "Batterie" or values.get("35") != "In Betrieb": raise ValueError("Unreviewed MaStR classification mapping") seen: set[str] = set() locations: Counter[str] = Counter() staged: list[dict[str, Any]] = [] for path in unit_paths: with path.open("rb") as stream: digest = hashlib.file_digest(stream, "sha256").hexdigest() for number, r in enumerate( _rows(path, "EinheitenStromSpeicher", "EinheitStromSpeicher"), 1 ): uid = _identity(r.get("EinheitMastrNummer"), "SEE") if uid in seen: raise ValueError("Duplicate unit identity across supplied partitions") seen.add(uid) location = r.get("LokationMaStRNummer") if location and r.get("Technologie") == "524": locations[location] += 1 power = _number(r.get("Nettonennleistung")) if power is None or power < minimum_kw: continue flags = [] if r.get("Technologie") != "524": flags.append("not_battery") if r.get("Land") != "84": flags.append("not_germany") if not location: flags.append("missing_location_identity") if r.get("EinheitBetriebsstatus") not in {"31", "35", "37", "38"}: flags.append("unmapped_status") if not r.get("NameStromerzeugungseinheit"): flags.append("missing_name") commissioning = r.get("Inbetriebnahmedatum") if commissioning: parsed = date.fromisoformat(commissioning) if parsed.year < 1990 or parsed > as_of: flags.append("commissioning_date_requires_review") staged.append( { "unit_id": uid, "storage_id": r.get("SpeMastrNummer"), "location_id": location, "operator_id": r.get("AnlagenbetreiberMastrNummer"), "name": r.get("NameStromerzeugungseinheit"), "locality": r.get("Ort"), "municipality": r.get("Gemeinde"), "region": values.get(r.get("Bundesland", "")), "net_power_mw": power / 1000, "gross_power_kw": _number(r.get("Bruttoleistung")), "inverter_active_power_kw": _number( r.get("ZugeordnenteWirkleistungWechselrichter") ), "chemistry_label": values.get(r.get("Batterietechnologie", "")), "technology_label": values.get(r.get("Technologie", "")), "status_code": r.get("EinheitBetriebsstatus"), "status_label": values.get(r.get("EinheitBetriebsstatus", "")), "commissioning_date_reported": commissioning, "retirement_date_reported": r.get("DatumEndgueltigeStilllegung"), "last_source_update": r.get("DatumLetzteAktualisierung"), "network_verification_label": values.get( r.get("NetzbetreiberpruefungStatus", "") ), "network_verification_date": r.get("NetzbetreiberpruefungDatum"), "coupling_label": values.get(r.get("AcDcKoppelung", "")), "latitude_reported": r.get("Breitengrad"), "longitude_reported": r.get("Laengengrad"), "source_member": path.name, "source_row": number, "source_sha256": digest, "snapshot_date": snapshot_date, "review_flags": flags, } ) wanted = {r["storage_id"] for r in staged} installations: dict[str, dict[str, Any]] = {} storage_seen: set[str] = set() for path in storage_paths: with path.open("rb") as stream: digest = hashlib.file_digest(stream, "sha256").hexdigest() for number, r in enumerate(_rows(path, "AnlagenStromSpeicher", "AnlageStromSpeicher"), 1): sid = _identity(r.get("MaStRNummer"), "SSE") if sid in storage_seen: raise ValueError("Duplicate storage identity across supplied partitions") storage_seen.add(sid) if sid in wanted: installations[sid] = { "usable_energy_kwh": _number(r.get("NutzbareSpeicherkapazitaet")), "linked_units": r.get("VerknuepfteEinheitenMaStRNummern"), "source_member": path.name, "source_row": number, "source_sha256": digest, } for row in staged: flags = row["review_flags"] if locations[row["location_id"]] > 1: flags.append("shared_battery_location_in_supplied_partitions") row["storage_observation"] = installations.get(row["storage_id"]) storage = row["storage_observation"] row["usable_energy_mwh"] = None if not storage or storage["linked_units"] != row["unit_id"]: flags.append("missing_or_nonexclusive_reciprocal_storage_link") elif storage["usable_energy_kwh"] is not None: row["usable_energy_mwh"] = storage["usable_energy_kwh"] / 1000 return staged