Source code for energy_data_retrieval.bess_catalogue.innovation_fund

"""Stage the European Commission Innovation Fund project inventory offline.

Grant status is not plant lifecycle status. Category alone is not technology:
manufacturing, thermal, hydrogen, carbon storage and transport must be reviewed.
"""

from __future__ import annotations

import hashlib
from html.parser import HTMLParser
from typing import Any


class _Inventory(HTMLParser):
    def __init__(self) -> None:
        super().__init__()
        self.rows: list[tuple[list[str], list[str]]] = []
        self.cells: list[str] | None = None
        self.cell: list[str] | None = None
        self.links: list[str] = []

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        if tag == "tr":
            self.cells, self.links = [], []
        elif self.cells is not None:
            if tag in {"td", "th"}:
                self.cell = []
            elif tag == "a":
                href = dict(attrs).get("href")
                if href:
                    self.links.append(href)
            elif tag in {"br", "p"} and self.cell is not None:
                self.cell.append(" ")

    def handle_data(self, data: str) -> None:
        if self.cell is not None:
            self.cell.append(data)

    def handle_endtag(self, tag: str) -> None:
        if tag in {"td", "th"} and self.cell is not None and self.cells is not None:
            self.cells.append(" ".join("".join(self.cell).split()))
            self.cell = None
        elif tag == "tr" and self.cells is not None:
            self.rows.append((self.cells, list(dict.fromkeys(self.links))))
            self.cells = None
        elif tag == "p" and self.cell is not None:
            self.cell.append(" ")


[docs] def parse_innovation_fund_html(content: bytes) -> list[dict[str, Any]]: """Return attributed candidate rows, never canonical records or inferred status. Retain all rows so screening cannot silently omit a battery project filed under another category. Conflicting repeated identities fail closed. Reused factsheet URLs are flagged because distinct projects can have a bad link. """ parser = _Inventory() parser.feed(content.decode("utf-8-sig")) header = ["Project", "Type", "Category", "Sector", "Location", "Technology pathways", "Status"] if not parser.rows or parser.rows[0][0] != header: raise ValueError("Changed Innovation Fund header") out: list[dict[str, Any]] = [] identities: set[tuple[str, str]] = set() url_counts: dict[str, int] = {} for i, (cells, urls) in enumerate(parser.rows[1:], 1): if len(cells) != 7 or not all(cells): raise ValueError("Incomplete or changed inventory row") identity = (cells[0], cells[1]) if identity in identities: raise ValueError("Duplicate project/call identity") identities.add(identity) for url in urls: url_counts[url] = url_counts.get(url, 0) + 1 out.append( dict( source_row=i, project_label=cells[0], call=cells[1], category=cells[2], sector=cells[3], location=cells[4], technology=cells[5], grant_status=cells[6], factsheet_urls=urls, review_required=True, source_sha256=hashlib.sha256(content).hexdigest(), ) ) if not out: raise ValueError("Empty project inventory") for row in out: row["shared_factsheet_url"] = any(url_counts[u] > 1 for u in row["factsheet_urls"]) return out