"""Public CEN daily PDF reports: reported PV generation and availability notices.
This is a source-specific daily product, not the authenticated hourly API.
No schedule shortfall is interpreted as a technical loss or fault attribution.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
from dataclasses import asdict, dataclass
from datetime import date, timedelta
from html.parser import HTMLParser
from pathlib import Path
from urllib.parse import urlsplit
from energy_data_retrieval.transport import HTTPClient, HTTPConfig
INDEX_URL = "https://www.coordinador.cl/operacion/documentos/novedades-cdc/"
HOSTS = frozenset({"www.coordinador.cl"})
_NUMBER = re.compile(r"(?:\d{1,3}(?:,\d{3})+|\d+)\.\d{2}")
_PERCENT = re.compile(r"(-?\d+(?:\.\d+)?)\s*%")
_MONTHS = {
name: i
for i, name in enumerate(
(
"enero",
"febrero",
"marzo",
"abril",
"mayo",
"junio",
"julio",
"agosto",
"septiembre",
"octubre",
"noviembre",
"diciembre",
),
1,
)
}
[docs]
@dataclass(frozen=True)
class GenerationRow:
plant_name: str
scheduled_mwh: float
actual_mwh: float
reported_deviation_percent: float | None
status_codes: str
source_page: int
[docs]
@dataclass(frozen=True)
class AvailabilityNotice:
plant_name: str
category: str
reported_availability_percent: float
notice_ids: tuple[str, ...]
source_page: int
class _Links(HTMLParser):
def __init__(self) -> None:
super().__init__()
self.links: list[str] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
if tag == "a":
self.links.extend(value for key, value in attrs if key == "href" and value)
[docs]
def discover_reports(html: str) -> dict[date, str]:
"""Use observed public links, including revised filenames; never invent URLs."""
parser = _Links()
parser.feed(html)
reports: dict[date, str] = {}
for url in parser.links:
parts = urlsplit(url)
match = re.fullmatch(
r"/wp-content/uploads/\d{4}/\d{2}/Informe-Diario-(\d{2})-(\d{2})-(\d{4})(?:-\d+)?\.pdf",
parts.path,
)
if parts.scheme != "https" or parts.hostname not in HOSTS or not match:
continue
if parts.query or parts.fragment or parts.username or parts.port not in (None, 443):
continue
day = date(int(match[3]), int(match[2]), int(match[1]))
if day in reports and reports[day] != url:
raise ValueError(f"Multiple report revisions for {day}; select one explicitly")
reports[day] = url
return reports
[docs]
def parse_report_pages(
pages: list[str],
expected_day: date,
) -> tuple[tuple[GenerationRow, ...], tuple[AvailabilityNotice, ...]]:
"""Parse the tested CEN column-text layout, failing on malformed PFV rows.
Selection is exact source labels starting with ``PFV`` followed by a space
in section 1.1, excluding PMGD, PMG-prefixed rows, BESS and CSP. Names stay
source identities, not a verified mapping to the separate InfoTécnica registry.
"""
if not pages or len(pages) > 100:
raise ValueError("Expected 1..100 report pages")
cover = " ".join(pages[0].split()).lower()
stamp = re.search(r"(\d{1,2}) de (\w+) del (\d{4})", cover)
if not stamp or stamp[2] not in _MONTHS:
raise ValueError("Missing Spanish report cover date")
if date(int(stamp[3]), _MONTHS[stamp[2]], int(stamp[1])) != expected_day:
raise ValueError("Report cover date differs from selected publication")
rows: dict[str, GenerationRow] = {}
notices: list[AvailabilityNotice] = []
generation_section = False
category = "unspecified"
for page_number, text in enumerate(pages, 1):
lines = [line.strip() for line in text.splitlines() if line.strip()]
if "1.1. Centrales" in text:
generation_section = True
for i, line in enumerate(lines):
if line.startswith(("1.2.", "JUSTIFICACIÓN DE", "ESTADO DE LAS CENTRALES")):
generation_section = False
if re.match(r"3\.\d\s", line):
category = line
if not line.startswith("PFV "):
continue
if generation_section:
name_parts = [line]
j = i + 1
while j < len(lines) and not _NUMBER.fullmatch(lines[j]):
if j - i > 3 or lines[j].startswith(("PFV ", "Central ")):
raise ValueError(f"Malformed generation row on page {page_number}: {line}")
name_parts.append(lines[j])
j += 1
if j + 2 >= len(lines) or not _NUMBER.fullmatch(lines[j + 1]):
raise ValueError(f"Missing numeric generation on page {page_number}")
name = " ".join(name_parts)
scheduled, actual = (float(lines[k].replace(",", "")) for k in (j, j + 1))
percentage = _PERCENT.fullmatch(lines[j + 2])
if not percentage and lines[j + 2] != "-":
raise ValueError("Unrecognized deviation cell")
deviation = float(percentage[1]) if percentage else None
if scheduled and deviation is not None:
# Both published energies and the reported percentage are rounded.
tolerance = 0.02 + 0.5 * (1 + actual / scheduled) / scheduled
if abs(100 * (actual / scheduled - 1) - deviation) > tolerance:
raise ValueError("Reported deviation does not reconcile with energy cells")
status = lines[j + 3] if j + 3 < len(lines) else ""
if not re.fullmatch(r"-|[A-Z]+(?:,\s*[A-Z]+)*", status):
raise ValueError("Unrecognized status cell; source layout may have changed")
if name in rows:
raise ValueError(f"Duplicate source plant: {name}")
rows[name] = GenerationRow(name, scheduled, actual, deviation, status, page_number)
elif i + 1 < len(lines) and (availability := _PERCENT.fullmatch(lines[i + 1])):
value = float(availability[1])
if not 0 <= value <= 100:
raise ValueError("Invalid reported availability")
end = i + 2
while end < len(lines):
if re.match(r"3\.\d\s", lines[end]):
break
if end + 1 < len(lines) and _PERCENT.fullmatch(lines[end + 1]):
break
end += 1
ids = tuple(
sorted(
set(
re.findall(
r"(?:SDCF|SICF|IL|IF)\s+\d{8,}",
" ".join(lines[i + 2 : end]),
re.I,
)
)
)
)
notices.append(AvailabilityNotice(line, category, value, ids, page_number))
if not rows:
raise ValueError("No PFV generation rows found; source layout may have changed")
return tuple(rows[k] for k in sorted(rows)), tuple(notices)
[docs]
def parse_report(path: Path, expected_day: date) -> dict:
"""Retain source hash and page references; pypdf is an optional dependency."""
from pypdf import PdfReader
body = path.read_bytes()
if not body.startswith(b"%PDF-") or len(body) > 10 * 1024 * 1024:
raise ValueError("Expected a PDF no larger than 10 MiB")
reader = PdfReader(path)
rows, notices = parse_report_pages([p.extract_text() for p in reader.pages], expected_day)
return {
"report_date": expected_day.isoformat(),
"source_sha256": hashlib.sha256(body).hexdigest(),
"pages": len(reader.pages),
"generation": [asdict(r) for r in rows],
"availability_notices": [asdict(n) for n in notices],
}
[docs]
def retrieve_reports(start: date, end: date, output: Path, http: HTTPClient) -> dict:
"""Retrieve a bounded public window, preserving exact PDFs and HTTP manifests."""
if not 0 <= (end - start).days <= 30:
raise ValueError("Select 1..31 calendar days")
index = http.get_text(INDEX_URL)
urls = discover_reports(index.value)
days = [start + timedelta(days=i) for i in range((end - start).days + 1)]
missing = [day.isoformat() for day in days if day not in urls]
if missing:
raise ValueError(f"Reports absent from public index: {missing}")
raw = output / "raw"
raw.mkdir(parents=True, exist_ok=True)
reports = []
for day in days:
path = raw / f"{day}.pdf"
record = http.download_file(urls[day], path)
reports.append(
{**parse_report(path, day), "source_url": urls[day], "retrieval": asdict(record)}
)
result = {
"schema_version": "1.0",
"review_type": "cen_dispatch_schedule",
"source": "Coordinador Electrico Nacional public daily reports",
"index_url": INDEX_URL,
"index_retrieval": asdict(index.record),
"timezone": "America/Santiago",
"start_date": start.isoformat(),
"end_date": end.isoformat(),
"energy_unit": "MWh",
"reports": reports,
"scope": "Exact PFV-prefixed rows in section 1.1; not all Chilean solar assets",
"baseline": "CEN dispatch schedule, not unconstrained technical production",
"price_per_mwh": None,
"recoverable_revenue": None,
}
(output / "snapshot.json").write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n")
return result
[docs]
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--start", type=date.fromisoformat, required=True)
parser.add_argument("--end", type=date.fromisoformat, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--use-env-proxy", action="store_true", help="Use configured HTTPS_PROXY")
args = parser.parse_args()
transport = None
if args.use_env_proxy:
import httpx
if not os.environ.get("HTTPS_PROXY"):
raise ValueError("HTTPS_PROXY is not configured")
transport = httpx.HTTPTransport(proxy=os.environ["HTTPS_PROXY"])
with HTTPClient(
HOSTS, config=HTTPConfig(maximum_response_bytes=10 * 1024 * 1024), transport=transport
) as http:
result = retrieve_reports(args.start, args.end, args.output, http)
print(f"Retrieved {len(result['reports'])} CEN reports to {args.output}")
if __name__ == "__main__":
main()