"""PV device reliability evidence, with incompatible metrics kept separate.
Published extracts are not the underlying confidential maintenance databases.
The two PDF connectors reproduce specific public tables; other records are
reviewed, attributed transcriptions. No raw fleet access is implied.
"""
from __future__ import annotations
import hashlib
import json
import math
import re
from importlib.resources import files
from typing import Any
import httpx
import pandas as pd
from energy_data_retrieval.failure_database import _pdf
REFERENCE_IDS = ("baschel_2018_reliability", "lillo_2018_failures")
DATA = files("energy_data_retrieval").joinpath("pv_reliability_data")
[docs]
def pv_reliability_sources() -> pd.DataFrame:
"""Research inventory including provenance, access gaps and metric limits."""
return pd.DataFrame(json.loads(DATA.joinpath("sources.json").read_text()))
[docs]
def load_pv_reliability_evidence(
*, source_id: str | None = None, metric: str | None = None
) -> pd.DataFrame:
"""Load source observations, including flagged records for transparent review.
Filter ``metric`` before comparing numbers. Counts, rates, availability,
survival, prevalence, individual ages and drift are different quantities.
Aggregate rows overlap their constituent rows and must not be summed.
"""
records = json.loads(DATA.joinpath("records.json").read_text())
sources = pv_reliability_sources().set_index("source_id")
for record in records:
source = sources.loc[record["source_id"]]
record.update(source_url=source.url, retrieved_at=source.retrieved_at)
result = pd.DataFrame(records)
if result.record_id.duplicated().any():
raise ValueError("Duplicate PV evidence IDs")
if source_id is not None:
if source_id not in sources.index:
raise ValueError(f"Unknown source: {source_id}")
result = result[result.source_id == source_id]
if metric is not None:
if metric not in {r["metric"] for r in records}:
raise ValueError(f"Unknown metric: {metric}")
result = result[result.metric == metric]
return result.reset_index(drop=True)
def _record(source: str, key: str, device: str, metric: str, value: float, unit: str) -> dict:
return {
"record_id": f"{source}:{key}",
"source_id": source,
"device": device,
"metric": metric,
"value": value,
"unit": unit,
"quality_flags": [],
}
[docs]
def parse_baschel_2018(content: bytes) -> list[dict[str, Any]]:
"""All 52 Table 1 rate alternatives, pp. 5-6; retain source arithmetic errors.
The fixed PDF contains an overlaid draft on page 6. Its ABCDEE-font layer
is excluded, retaining the final URWPalladio table. Fixed coordinates and
row counts deliberately fail closed on a different document layout.
"""
groups = [
("PV module", 5),
("Thin-film PV module", 1),
("PV connector", 2),
("PV string cable", 1),
("Terminal lug", 1),
("Terminal metal sleeve", 1),
("Terminal screw", 1),
("Terminal stud", 1),
("Terminal block", 2),
("Terminal strip", 1),
("Fuse", 1),
("String fuse", 3),
("String monitoring unit", 2),
("DC switch", 1),
("DC main cable", 1),
("AC cable", 1),
("Disconnector", 1),
("String inverter", 3),
("Central inverter", 3),
("DC capacitor", 3),
("DC main breaker", 2),
("IGBT module", 2),
("AC filter capacitor", 1),
("AC circuit breaker", 4),
("Central control board", 3),
("Cooling fan", 1),
("Transformer", 2),
("Power switchgear", 2),
]
devices = [name for name, count in groups for _ in range(count)]
result: list[dict[str, Any]] = []
with _pdf(content) as pdf:
for index, low, high, expected in [(4, 265, 750, 44), (5, 125, 220, 8)]:
page = pdf.pages[index].filter(
lambda obj: not obj.get("fontname", "").startswith("ABCDEE")
)
words = page.extract_words(extra_attrs=["fontname"])
values = [w for w in words if 245 < w["x0"] < 280 and low < w["top"] < high]
if len(values) != expected:
raise ValueError("Baschel Table 1 layout changed")
for word in values:
tail = [
w["text"] for w in words if w["x0"] > 300 and abs(w["top"] - word["top"]) < 2
]
if len(tail) not in (2, 3) or not tail[1].endswith("%"):
raise ValueError("Baschel Table 1 columns changed")
rate = float(word["text"]) / 1e6
row = _record(
"baschel_2018",
f"t1-{len(result) + 1:02}",
devices[len(result)],
"failure_rate",
rate,
"1/operating_hour",
)
source_ref = tail[2] if len(tail) == 3 else None
mtbf = 1e6 if tail[0] == "1Mio" else float(tail[0].replace(",", ""))
survival = float(tail[1][:-1]) / 100
row.update(
source_locator=f"Table 1, PDF p. {index + 1}",
population="Historical PV components; mixed juwi field and literature sources",
evidence_origin="juwi_field" if source_ref == "-" else "cited_literature",
original_reference=source_ref,
selected_in_source="Bold" in word["fontname"],
source_mtbf_years=mtbf,
source_survival_20y=survival,
operating_hours_per_year=4015,
)
if source_ref is None:
row["quality_flags"].append("missing_original_reference")
if abs(mtbf * rate * 4015 - 1) > 0.1:
row["quality_flags"].append("source_mtbf_rate_mismatch")
if abs(survival - math.exp(-rate * 4015 * 20)) > 0.005:
row["quality_flags"].append("source_survival_rate_mismatch")
result.append(row)
if len(result) != len(devices):
raise ValueError("Incomplete Baschel table")
return result
[docs]
def parse_lillo_2018(content: bytes) -> list[dict[str, Any]]:
"""Tables 2-3, p. 10: all mode counts and reported normalized event rates.
Counts include resets, communication incidents and weather events. A box
count has no box inventory denominator; FRSF is ALL solar-field events per
installed module-year, not a module hardware failure rate. Published totals
are retained and overlapping aggregates are explicitly marked.
"""
with _pdf(content) as pdf:
page = pdf.pages[9].extract_text()
counts_text, rates_text = page.split("Table3.", 1)
count_rows = [
line.split()
for line in counts_text.splitlines()
if re.fullmatch(r"(?:[A-Z][A-Z0-9]?|Total)(?: \d+){16}", line)
]
rate_rows = [
line.split()
for line in rates_text.splitlines()
if re.fullmatch(r"(?:[A-Z][A-Z0-9]?|Total)(?: [\d.]+){7}", line)
]
if len(count_rows) != 16 or len(rate_rows) != 16:
raise ValueError("Lillo tables changed")
modes = [
(0, "PV module", "module event"),
(1, "DC wiring", "wiring event"),
(2, "Junction / combiner box", "junction box event"),
(4, "Inverter", "operation"),
(5, "Inverter", "start / stop"),
(6, "Inverter", "monitoring"),
(8, "Transformer station", "operation"),
(9, "Transformer station", "weather"),
(11, "Grid connection", "operation"),
(12, "Grid connection", "weather"),
(14, "Monitoring system", "all events"),
]
rate_devices = [
("Solar field", "events/module_year", "All field events normalized by modules"),
("Inverter", "events/inverter_year", "Includes operation, start/stop and monitoring"),
("Transformer station", "events/station_year", "Includes weather events"),
("Grid connection", "events/plant_year", "External grid events"),
("Monitoring system", "events/plant_year", "Communication incidents, often auto-reset"),
("Whole plant", "events/plant_year", "All events; overlaps component rows"),
("Whole plant", "events/kWac_year", "All events normalized by nominal AC power"),
]
result = []
for raw in count_rows:
park, values = raw[0], list(map(int, raw[1:]))
inconsistent = sum(values[i] for i in [3, 7, 10, 13, 14]) != values[15]
for index, device, mode in modes:
row = _record(
"lillo_2018",
f"t2-{park}-{index:02}",
device,
"event_count",
values[index],
"events/15_months",
)
row.update(park=park, failure_mode=mode, aggregate=park == "Total")
if inconsistent:
row["quality_flags"].append("source_park_total_differs_from_component_sum")
result.append(row)
for raw in rate_rows:
for index, (device, unit, meaning) in enumerate(rate_devices):
row = _record(
"lillo_2018",
f"t3-{raw[0]}-{index}",
device,
"normalized_event_rate",
float(raw[index + 1]),
unit,
)
row.update(
park=raw[0], meaning=meaning, aggregate=raw[0] == "Total" or device == "Whole plant"
)
if raw[0] == "Total" and index == 4:
row["quality_flags"].append("reported_10.4_differs_from_193_div_18.75")
result.append(row)
for row in result:
row.update(
source_locator="Tables 2-3, PDF p. 10",
population="15 fixed-tilt plants in Spain/Italy; Jan 2014-Mar 2015",
)
return result
[docs]
def refresh_pv_reliability_source(
source_id: str, *, client: httpx.Client | None = None
) -> list[dict]:
"""Bounded, SHA-256-pinned refresh for the two machine-extracted PDF tables.
No arbitrary URL, automatic overwrite, or silent source-version update.
Manual evidence extracts are deliberately not advertised as live APIs.
"""
parsers = {"baschel_2018": parse_baschel_2018, "lillo_2018": parse_lillo_2018}
if source_id not in parsers:
raise ValueError("This source has no automatic table connector")
source = pv_reliability_sources().set_index("source_id").loc[source_id]
if client is None:
with httpx.Client(follow_redirects=True, timeout=60) as owned:
return refresh_pv_reliability_source(source_id, client=owned)
chunks, size = [], 0
with client.stream("GET", str(source["download_url"])) as response:
response.raise_for_status()
for chunk in response.iter_bytes():
size += len(chunk)
if size > 20_000_000:
raise ValueError("PDF exceeds 20 MB download limit")
chunks.append(chunk)
content = b"".join(chunks)
if hashlib.sha256(content).hexdigest() != source.sha256:
raise ValueError("Source changed; manual review required")
return parsers[source_id](content)