Source code for energy_data_retrieval.pv_catalogue.philippines_doe

"""Stage the Philippines DOE's grid-connected awarded-solar PDF table.

Generate the XHTML with Poppler's `pdftotext -bbox-layout`. The August 2026
layout uses fixed rows and columns. A changed layout requires review. Each
result describes a service-contract entry with native MW labels. A source
row can cover part of a plant or multiple phases.
"""

from __future__ import annotations

import re
from collections import defaultdict
from datetime import date
from math import isfinite
from xml.etree import ElementTree

from .sam import verify_sha256

_COLUMNS = (
    "grid",
    "region",
    "province",
    "municipality",
    "name",
    "company",
    "stage",
    "potential_mw",
    "installed_mw",
)
_COLUMN_STARTS = (50, 105, 168, 229, 324, 430, 552, 625, 687)
_STAGES = {"Pre-Development", "Development", "Commercial Operation", "Development Development"}
_GRIDS = {"Luzon", "Visayas", "Mindanao"}
_TOTALS = {"Luzon Total", "Visayas Total", "Mindanao Total", "Grand Total"}


def _number(value: str) -> float | None:
    if not value:
        return None
    if not re.fullmatch(r"(?:[0-9]+|[0-9]{1,3}(?:,[0-9]{3})+)(?:\.[0-9]+)?", value):
        raise ValueError("Unexpected DOE capacity cell")
    parsed = float(value.replace(",", ""))
    if not isfinite(parsed):
        raise ValueError("Unexpected DOE capacity magnitude")
    return parsed


def _page_rows(page: ElementTree.Element, first_page: bool) -> list[tuple[int, list[str]]]:
    if (float(page.attrib["width"]), float(page.attrib["height"])) != (792, 612):
        raise ValueError("Unexpected DOE page dimensions")
    cells: dict[tuple[int, int], list[tuple[float, float, str]]] = defaultdict(list)
    for word in page.findall(".//{*}word"):
        x = float(word.attrib["xMin"])
        y = (float(word.attrib["yMin"]) + float(word.attrib["yMax"])) / 2
        if not 50 <= x < 737 or not 54.72 <= y < 552.48:
            continue
        row = int((y - 54.72) / 29.28)
        column = sum(start <= x for start in _COLUMN_STARTS) - 1
        cells[row, column].append((round(y, 1), x, word.text or ""))
    first_row = 2 if first_page else 0
    return [
        (
            row - first_row + 1,
            [
                " ".join(text for _, _, text in sorted(cells.get((row, column), [])))
                for column in range(len(_COLUMNS))
            ],
        )
        for row in range(first_row, 17)
    ]


[docs] def read_philippines_awarded_solar( content: bytes, *, sha256: str, snapshot: str ) -> list[dict[str, str | float | int | None]]: """Check a pinned layout capture and stage source rows with PDF page locators.""" verify_sha256(content, sha256) if len(content) > 12_000_000 or b"<!ENTITY" in content: raise ValueError("Unexpected DOE layout capture") try: document = ElementTree.fromstring(content) except ElementTree.ParseError as exc: raise ValueError("Invalid DOE layout XML") from exc pages = document.findall(".//{*}page") if not pages: raise ValueError("Missing DOE pages") release = date.fromisoformat(snapshot) title = f"AWARDED SOLAR PROJECTS as of {release.day} {release:%B %Y}".upper() header = " ".join(w.text or "" for w in pages[0].findall(".//{*}word")) if not header.upper().startswith(title + " "): raise ValueError("DOE layout title or release date changed") staged: list[dict[str, str | float | int | None]] = [] for page_no, page in enumerate(pages, 1): location = [""] * 4 for row_no, row in _page_rows(page, page_no == 1): if not any(row[4:]): continue if row[0] in _TOTALS and not any(row[1:7]): continue if row[6] not in _STAGES or not row[4]: raise ValueError(f"Unexpected DOE contract row on page {page_no}, row {row_no}") if row[6] == "Development Development": # Two compressed lines share a regular slot in this layout. staged.append({"page": page_no, "row": row_no, "stage": row[6]}) continue for level, label in enumerate(row[:4]): if label: location[level] = label location[level + 1 :] = [""] * (3 - level) if location[0] not in _GRIDS: raise ValueError(f"Unverified grid label on page {page_no}, row {row_no}") record: dict[str, str | float | int | None] = dict( zip(_COLUMNS[:7], [*location, *row[4:7]], strict=True) ) record.update( page=page_no, row=row_no, potential_mw=_number(row[7]), installed_mw=_number(row[8]), ) staged.append(record) return staged