Source code for energy_data_retrieval.pv_catalogue.gem_batches
"""Select a reviewed GEM partition only after verifying the complete accepted set."""
from __future__ import annotations
import hashlib
import re
from collections import Counter
from typing import Any
[docs]
def select_reviewed_gem_batch(
rows: list[dict[str, Any]],
*,
review: dict[str, Any],
summary: dict[str, Any],
batch: int,
) -> list[dict[str, Any]]:
"""Validate a disjoint, exhaustive ledger and return one unchanged partition.
This is publication selection, not a new acceptance or deduplication decision.
A partial input, altered ID set, duplicate row or inconsistent batch ledger
fails before any output. Country counts and every partition count are checked.
"""
ids = [row["record_id"] for row in rows]
if len(set(ids)) != len(ids):
raise ValueError("Duplicate accepted record ID")
digest = hashlib.sha256("\n".join(sorted(ids)).encode()).hexdigest()
if digest != review["accepted_ids_sha256"] or len(rows) != review["accepted"]:
raise ValueError("Complete reviewed GEM identity set required")
if len(rows) != summary["new_projects"]:
raise ValueError("Summary/review count mismatch")
if any(
row["entity_type"] != "projects" or review["source_id"] not in row["sources"]
for row in rows
):
raise ValueError("Unexpected entity or source in GEM batch")
prefixes: set[str] = set()
numbers: set[int] = set()
selected = None
for partition in summary["batches"]:
number = partition["batch"]
start, end = partition["prefix_start"], partition["prefix_end"]
if (
number in numbers
or not re.fullmatch(r"[0-9a-f]{2}", start)
or not re.fullmatch(r"[0-9a-f]{2}", end)
or start > end
):
raise ValueError("Invalid batch number or prefix range")
numbers.add(number)
covered = {f"{i:02x}" for i in range(int(start, 16), int(end, 16) + 1)}
if prefixes & covered:
raise ValueError("Overlapping batch prefix ranges")
prefixes.update(covered)
subset = [row for row in rows if row["record_id"][:2] in covered]
countries = dict(Counter(row["country_code"] for row in subset))
if (
len(subset) != partition["new_projects"]
or countries != partition["new_country_counts"]
or len(countries) != partition["countries"]
):
raise ValueError("Batch counts or country distribution changed")
if number == batch:
selected = subset
if prefixes != {f"{i:02x}" for i in range(256)}:
raise ValueError("Batch ledger does not cover all UUID prefixes")
if selected is None:
raise ValueError("Unknown batch number")
return sorted(selected, key=lambda row: row["record_id"])