Source code for energy_data_retrieval.sources.cen.daily_report

"""Public CEN daily PDF reports: reported PV generation and availability notices.

This is a source-specific daily product, not the authenticated hourly API.
No schedule shortfall is interpreted as a technical loss or fault attribution.
"""

from __future__ import annotations

import argparse
import hashlib
import json
import os
import re
from dataclasses import asdict, dataclass
from datetime import date, timedelta
from html.parser import HTMLParser
from pathlib import Path
from urllib.parse import urlsplit

from energy_data_retrieval.transport import HTTPClient, HTTPConfig

INDEX_URL = "https://www.coordinador.cl/operacion/documentos/novedades-cdc/"
HOSTS = frozenset({"www.coordinador.cl"})
_NUMBER = re.compile(r"(?:\d{1,3}(?:,\d{3})+|\d+)\.\d{2}")
_PERCENT = re.compile(r"(-?\d+(?:\.\d+)?)\s*%")
_MONTHS = {
    name: i
    for i, name in enumerate(
        (
            "enero",
            "febrero",
            "marzo",
            "abril",
            "mayo",
            "junio",
            "julio",
            "agosto",
            "septiembre",
            "octubre",
            "noviembre",
            "diciembre",
        ),
        1,
    )
}


[docs] @dataclass(frozen=True) class GenerationRow: plant_name: str scheduled_mwh: float actual_mwh: float reported_deviation_percent: float | None status_codes: str source_page: int
[docs] @dataclass(frozen=True) class AvailabilityNotice: plant_name: str category: str reported_availability_percent: float notice_ids: tuple[str, ...] source_page: int
class _Links(HTMLParser): def __init__(self) -> None: super().__init__() self.links: list[str] = [] def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: if tag == "a": self.links.extend(value for key, value in attrs if key == "href" and value)
[docs] def discover_reports(html: str) -> dict[date, str]: """Use observed public links, including revised filenames; never invent URLs.""" parser = _Links() parser.feed(html) reports: dict[date, str] = {} for url in parser.links: parts = urlsplit(url) match = re.fullmatch( r"/wp-content/uploads/\d{4}/\d{2}/Informe-Diario-(\d{2})-(\d{2})-(\d{4})(?:-\d+)?\.pdf", parts.path, ) if parts.scheme != "https" or parts.hostname not in HOSTS or not match: continue if parts.query or parts.fragment or parts.username or parts.port not in (None, 443): continue day = date(int(match[3]), int(match[2]), int(match[1])) if day in reports and reports[day] != url: raise ValueError(f"Multiple report revisions for {day}; select one explicitly") reports[day] = url return reports
[docs] def parse_report_pages( pages: list[str], expected_day: date, ) -> tuple[tuple[GenerationRow, ...], tuple[AvailabilityNotice, ...]]: """Parse the tested CEN column-text layout, failing on malformed PFV rows. Selection is exact source labels starting with ``PFV`` followed by a space in section 1.1, excluding PMGD, PMG-prefixed rows, BESS and CSP. Names stay source identities, not a verified mapping to the separate InfoTécnica registry. """ if not pages or len(pages) > 100: raise ValueError("Expected 1..100 report pages") cover = " ".join(pages[0].split()).lower() stamp = re.search(r"(\d{1,2}) de (\w+) del (\d{4})", cover) if not stamp or stamp[2] not in _MONTHS: raise ValueError("Missing Spanish report cover date") if date(int(stamp[3]), _MONTHS[stamp[2]], int(stamp[1])) != expected_day: raise ValueError("Report cover date differs from selected publication") rows: dict[str, GenerationRow] = {} notices: list[AvailabilityNotice] = [] generation_section = False category = "unspecified" for page_number, text in enumerate(pages, 1): lines = [line.strip() for line in text.splitlines() if line.strip()] if "1.1. Centrales" in text: generation_section = True for i, line in enumerate(lines): if line.startswith(("1.2.", "JUSTIFICACIÓN DE", "ESTADO DE LAS CENTRALES")): generation_section = False if re.match(r"3\.\d\s", line): category = line if not line.startswith("PFV "): continue if generation_section: name_parts = [line] j = i + 1 while j < len(lines) and not _NUMBER.fullmatch(lines[j]): if j - i > 3 or lines[j].startswith(("PFV ", "Central ")): raise ValueError(f"Malformed generation row on page {page_number}: {line}") name_parts.append(lines[j]) j += 1 if j + 2 >= len(lines) or not _NUMBER.fullmatch(lines[j + 1]): raise ValueError(f"Missing numeric generation on page {page_number}") name = " ".join(name_parts) scheduled, actual = (float(lines[k].replace(",", "")) for k in (j, j + 1)) percentage = _PERCENT.fullmatch(lines[j + 2]) if not percentage and lines[j + 2] != "-": raise ValueError("Unrecognized deviation cell") deviation = float(percentage[1]) if percentage else None if scheduled and deviation is not None: # Both published energies and the reported percentage are rounded. tolerance = 0.02 + 0.5 * (1 + actual / scheduled) / scheduled if abs(100 * (actual / scheduled - 1) - deviation) > tolerance: raise ValueError("Reported deviation does not reconcile with energy cells") status = lines[j + 3] if j + 3 < len(lines) else "" if not re.fullmatch(r"-|[A-Z]+(?:,\s*[A-Z]+)*", status): raise ValueError("Unrecognized status cell; source layout may have changed") if name in rows: raise ValueError(f"Duplicate source plant: {name}") rows[name] = GenerationRow(name, scheduled, actual, deviation, status, page_number) elif i + 1 < len(lines) and (availability := _PERCENT.fullmatch(lines[i + 1])): value = float(availability[1]) if not 0 <= value <= 100: raise ValueError("Invalid reported availability") end = i + 2 while end < len(lines): if re.match(r"3\.\d\s", lines[end]): break if end + 1 < len(lines) and _PERCENT.fullmatch(lines[end + 1]): break end += 1 ids = tuple( sorted( set( re.findall( r"(?:SDCF|SICF|IL|IF)\s+\d{8,}", " ".join(lines[i + 2 : end]), re.I, ) ) ) ) notices.append(AvailabilityNotice(line, category, value, ids, page_number)) if not rows: raise ValueError("No PFV generation rows found; source layout may have changed") return tuple(rows[k] for k in sorted(rows)), tuple(notices)
[docs] def parse_report(path: Path, expected_day: date) -> dict: """Retain source hash and page references; pypdf is an optional dependency.""" from pypdf import PdfReader body = path.read_bytes() if not body.startswith(b"%PDF-") or len(body) > 10 * 1024 * 1024: raise ValueError("Expected a PDF no larger than 10 MiB") reader = PdfReader(path) rows, notices = parse_report_pages([p.extract_text() for p in reader.pages], expected_day) return { "report_date": expected_day.isoformat(), "source_sha256": hashlib.sha256(body).hexdigest(), "pages": len(reader.pages), "generation": [asdict(r) for r in rows], "availability_notices": [asdict(n) for n in notices], }
[docs] def retrieve_reports(start: date, end: date, output: Path, http: HTTPClient) -> dict: """Retrieve a bounded public window, preserving exact PDFs and HTTP manifests.""" if not 0 <= (end - start).days <= 30: raise ValueError("Select 1..31 calendar days") index = http.get_text(INDEX_URL) urls = discover_reports(index.value) days = [start + timedelta(days=i) for i in range((end - start).days + 1)] missing = [day.isoformat() for day in days if day not in urls] if missing: raise ValueError(f"Reports absent from public index: {missing}") raw = output / "raw" raw.mkdir(parents=True, exist_ok=True) reports = [] for day in days: path = raw / f"{day}.pdf" record = http.download_file(urls[day], path) reports.append( {**parse_report(path, day), "source_url": urls[day], "retrieval": asdict(record)} ) result = { "schema_version": "1.0", "review_type": "cen_dispatch_schedule", "source": "Coordinador Electrico Nacional public daily reports", "index_url": INDEX_URL, "index_retrieval": asdict(index.record), "timezone": "America/Santiago", "start_date": start.isoformat(), "end_date": end.isoformat(), "energy_unit": "MWh", "reports": reports, "scope": "Exact PFV-prefixed rows in section 1.1; not all Chilean solar assets", "baseline": "CEN dispatch schedule, not unconstrained technical production", "price_per_mwh": None, "recoverable_revenue": None, } (output / "snapshot.json").write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n") return result
[docs] def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--start", type=date.fromisoformat, required=True) parser.add_argument("--end", type=date.fromisoformat, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--use-env-proxy", action="store_true", help="Use configured HTTPS_PROXY") args = parser.parse_args() transport = None if args.use_env_proxy: import httpx if not os.environ.get("HTTPS_PROXY"): raise ValueError("HTTPS_PROXY is not configured") transport = httpx.HTTPTransport(proxy=os.environ["HTTPS_PROXY"]) with HTTPClient( HOSTS, config=HTTPConfig(maximum_response_bytes=10 * 1024 * 1024), transport=transport ) as http: result = retrieve_reports(args.start, args.end, args.output, http) print(f"Retrieved {len(result['reports'])} CEN reports to {args.output}")
if __name__ == "__main__": main()