"""Public failure databases and attributed, versioned offline extracts.
No credentials or live calls are needed to load the bundled snapshots. Live
refreshes are bounded and explicit. NRC raw IRIS records, PVROM work orders and
commercial databases are not public APIs; these connectors use published data.
"""
from __future__ import annotations
import hashlib
import io
import json
import re
from collections import Counter
from importlib.resources import files
from typing import Any
from zipfile import ZipFile
import httpx
import numpy as np
import pandas as pd
Record = dict[str, Any]
REFERENCE_IDS = (
"inl_2021_component_reliability",
"klise_2018_pv_failures",
"carroll_2016_offshore_failures",
"backblaze_drive_stats",
)
DATA = files("energy_data_retrieval").joinpath("failure_data")
[docs]
def source_catalogue() -> pd.DataFrame:
"""Sources, access constraints, population limits, retrieval URLs and hashes."""
return pd.DataFrame(json.loads(DATA.joinpath("sources.json").read_text()))
def _record(source: str, identifier: str, device: str, population: str) -> Record:
return {
"record_id": f"{source}:{identifier}",
"source_id": source,
"device": device,
"population": population,
"failure_mode": identifier,
"quality_flags": [],
"source_page": None,
"data_period": None,
}
[docs]
def load_failure_database(
source_id: str | None = None, *, include_flagged: bool = False
) -> pd.DataFrame:
"""Load all accessible snapshot records, preserving source-specific units.
Flagged rows are retained on disk but excluded by default. Different
populations/modes overlap and must NOT be summed as a system failure rate.
The source catalogue explains the scope and excludes license-gated sources.
"""
sources = source_catalogue()
available = sources.loc[sources["snapshot_file"].notna()]
if source_id is not None:
available = available[available.source_id == source_id]
if available.empty:
raise ValueError(
f"No public numeric snapshot for {source_id!r}; inspect source_catalogue()"
)
rows = []
for source in available.to_dict("records"):
records = json.loads(DATA.joinpath(source["snapshot_file"]).read_text())
for row in records:
row["source_url"] = source["url"]
row["retrieved_at"] = source["retrieved_at"]
if include_flagged or not row["quality_flags"]:
rows.append(row)
result = pd.DataFrame(rows)
if result.record_id.duplicated().any():
raise ValueError("Duplicate source record IDs")
return result
def _pdf(content: bytes) -> Any:
try:
import pdfplumber
except ImportError as exc:
raise ImportError("Install energy-data-retrieval[reliability] for PDF refreshes") from exc
return pdfplumber.open(io.BytesIO(content))
def _number(value: str | None) -> float | None:
if value is None:
return None
try:
number = float(value.replace(",", ""))
except ValueError:
return None
return number if np.isfinite(number) else None
[docs]
def parse_nrc_2020(content: bytes) -> list[Record]:
"""Extract ALL 300 rows of Table 1, PDF pages 30-38 (printed 6-14).
Preserve inconsistent published units and quarantine their rows. Missing
counts remain missing. Gamma beta is a RATE, not a lifetime scale. Source
5th/95th percentiles describe a selected industry distribution, not a new
frequentist confidence interval. This is intentionally version-specific.
"""
records = []
device = ""
with _pdf(content) as pdf:
for index in range(29, 38):
table: Any = max(pdf.pages[index].extract_tables(), key=len)
header = table[0]
device_col = header.index("Component Type")
code_col = header.index("Component Failure\nMode")
for raw in table[3:]:
if raw[device_col]:
device = raw[device_col].replace("\n", " ")
tail_start = next((j for j, x in enumerate(raw) if x in {"Beta", "Gamma"}), None)
if tail_start is None:
continue
pre = raw[code_col:tail_start]
tail = [x.replace("\n", " ") for x in raw[tail_start:] if x]
if len(pre) != 7 or len(tail) != 10:
raise ValueError("NRC table layout changed; manual source review required")
code = pre[0].replace("\n", "").replace(" ", "")
row = _record("nrc_2020", code, device, "US commercial nuclear plants")
basis = {"h": "hour", "d": "demand", "h-ft": "foot_hour"}[pre[5]]
row.update(
description=pre[1].replace("\n", " "),
original_data_source=pre[2],
failures=_number(pre[3]),
exposure=_number(pre[4]),
basis=basis,
reported_unit=pre[5],
components=_number(pre[6]),
distribution="gamma_rate" if tail[0] == "Gamma" else "beta_probability",
analysis_type=tail[1],
p05=float(tail[2]),
median=float(tail[3]),
mean=float(tail[4]),
p95=float(tail[5]),
parameter1=float(tail[6]),
parameter2=float(tail[7]),
data_period=tail[9],
source_page=index + 1,
evidence="industry_distribution",
count_note=pre[3],
)
if (tail[0] == "Beta" and basis != "demand") or (
tail[0] == "Gamma" and basis == "demand"
):
row["quality_flags"].append("published_unit_distribution_conflict")
derived = row["parameter1"] / (
row["parameter2"] + (row["parameter1"] if tail[0] == "Beta" else 0)
)
if not np.isclose(derived, row["mean"], rtol=0.03, atol=0):
row["quality_flags"].append("published_parameter_mean_conflict")
if not 0 <= row["p05"] <= row["median"] <= row["p95"]:
row["quality_flags"].append("invalid_published_quantiles")
records.append(row)
if len(records) != 300 or len({r["record_id"] for r in records}) != 300:
raise ValueError("Expected 300 unique NRC Table 1 rows; extraction failed")
return records
[docs]
def parse_pvrom_2018(content: bytes) -> list[Record]:
"""All Appendix A failure fits (not repair fits), PDF pages 25-26.
Prefer day/hour rows over duplicate year fits, whose parameters sometimes
conflict. Lognormal without '-n' has ambiguous parameter semantics and is
quarantined. Normal models require explicit conditioning on positive time.
Preventive maintenance is retained but excluded from failure calculations.
"""
pattern = re.compile(
r"^(.*?)\s+(\d{4}-\d{4})\s+(.*?)\s+"
r"(Weibull-2|Exponential-1|Gamma|Normal|Lognormal-n|Lognormal)\s+"
r"([-+\d.Ee]+)(?:\s+([-+\d.Ee]+))?\s+(day|hour|year)\b(.*)$"
)
kinds = {
"Weibull-2": "weibull_lifetime",
"Exponential-1": "exponential_lifetime",
"Gamma": "gamma_lifetime",
"Normal": "normal_positive_lifetime",
"Lognormal-n": "lognormal_log_lifetime",
"Lognormal": "ambiguous_lognormal",
}
records = []
section = ""
with _pdf(content) as pdf:
for page in (24, 25):
for line in pdf.pages[page].extract_text(x_tolerance=1).splitlines():
if "Type" in line and ("Vintage" in line or "Data Range" in line):
label = re.split(r"\s+(?:Vintage|Data Range)", line)[0]
if label and not label.startswith("Vintage"):
section = label
match = pattern.match(line)
if not match:
continue
label, period, equipment, kind, p, q, unit, notes = match.groups()
row = _record("pvrom_2018", f"a{page + 1}_{len(records) + 1:02d}", section, label)
row.update(
distribution=kinds[kind],
parameter1=float(p),
parameter2=float(q) if q else 1.0,
basis=unit,
data_period=period,
equipment_context=equipment,
source_page=page + 1,
evidence="fitted_fault_interval",
failure_mode=section,
source_row=label,
source_notes=notes.strip(),
)
if unit == "year":
row["quality_flags"].append("duplicate_year_fit_prefer_native_time_row")
if kind == "Lognormal":
row["quality_flags"].append("ambiguous_lognormal_parameter_convention")
if "inverter_PM" in section:
row["quality_flags"].append("preventive_maintenance_not_failure")
records.append(row)
if len(records) != 63:
raise ValueError(f"Expected 63 PVROM Appendix A rows; found {len(records)}")
return records
WIND_DEVICES = (
"Pitch and hydraulics",
"Other components",
"Generator",
"Gearbox",
"Blades",
"Grease, oil and cooling liquid",
"Electrical components",
"Contactor, circuit breaker and relay",
"Controls",
"Safety",
"Sensors",
"Pumps and motors",
"Hub",
"Heaters and coolers",
"Yaw system",
"Tower and foundation",
"Power supply and converter",
"Service items",
"Transformer",
)
[docs]
def parse_wind_2016(content: bytes) -> list[Record]:
"""Figure 3's embedded numerical table, 19 subassemblies x 4 repair classes.
Rates are failures/turbine/year; the primary study does not publish counts
and exposure per cell, so no confidence interval is invented.
"""
records = []
with _pdf(content) as pdf:
for index, page in enumerate(pdf.pages):
page_text = page.extract_text(x_tolerance=1)
if "Figure 3." not in page_text or "No Cost Data" not in page_text:
continue
for mode in ("Major Replacement", "Major Repair", "Minor Repair", "No Cost Data"):
match = re.search(rf"(?m)^{mode}\s+(.+)$", page_text)
if match is None:
raise ValueError(f"Missing wind repair class: {mode}")
values = re.findall(r"\d+\.\d{3}", match.group(1))
if len(values) != 19:
raise ValueError("Expected 19 wind subassembly rates per repair class")
for j, (device, rate) in enumerate(zip(WIND_DEVICES, values, strict=True)):
row = _record(
"wind_2016",
f"{mode.lower().replace(' ', '_')}_{j:02d}",
device,
"~350 offshore turbines, European study fleet",
)
row.update(
failure_mode=mode,
mean=float(rate),
distribution="point_rate",
basis="turbine_year",
source_page=index + 1,
evidence="published_rounded_rate",
data_period="study published 2016",
)
records.append(row)
if len(records) != 76:
raise ValueError("Expected 76 wind Figure 3 values")
return records
[docs]
def aggregate_backblaze(frame: pd.DataFrame) -> pd.DataFrame:
"""Aggregate daily drive snapshots by model into failures and drive-days.
Exactly one observed drive-day per row. Gaps add no exposure. Reject
duplicate keys, missing identifiers and non-binary flags. This estimates an
exposure-weighted rate; the first observed date is NOT a drive's birth date.
"""
columns = {"date", "serial_number", "model", "failure"}
if not columns <= set(frame):
raise ValueError(f"Missing Backblaze columns: {sorted(columns - set(frame))}")
if frame.empty or frame[list(columns)].isna().any().any():
raise ValueError("Backblaze rows must be nonempty with complete identifiers and flags")
data = frame[list(columns)].copy()
data["date"] = pd.to_datetime(data["date"], format="%Y-%m-%d", errors="raise")
if not data.failure.isin([0, 1]).all():
raise ValueError("failure flags must be binary")
if data.duplicated(["date", "serial_number"]).any():
raise ValueError("Duplicate drive-day records")
if (
data[["model", "serial_number"]]
.astype(str)
.apply(lambda s: s.str.strip().eq(""))
.any()
.any()
):
raise ValueError("Blank model or serial number")
return (
data.groupby("model", sort=True)
.agg(failures=("failure", "sum"), exposure=("failure", "size"))
.reset_index()
)
[docs]
def parse_backblaze_zip(
content: bytes, *, max_uncompressed_bytes: int = 1_000_000_000
) -> list[Record]:
"""Read the public 2013 archive without extracting files; bounded memory.
Validate each daily file and count only its observed drive-days. Each day's
count is aggregated before reading the next. Repeated flags and post-failure
reappearances are audited separately.
Each reported flag remains an event, matching the provider's AFR numerator;
these records must not be treated as independent first-failure lifetimes.
"""
totals: dict[str, list[int]] = {}
dates: set[str] = set()
failed_serials: set[str] = set()
reappearances: Counter[str] = Counter()
repeat_flags: Counter[str] = Counter()
with ZipFile(io.BytesIO(content)) as archive:
members = [
m
for m in archive.infolist()
if re.fullmatch(r"(?:.*/)?\d{4}-\d{2}-\d{2}\.csv", m.filename)
and not m.filename.startswith("__MACOSX/")
]
if not members or sum(m.file_size for m in members) > max_uncompressed_bytes:
raise ValueError("Archive empty or exceeds uncompressed byte limit")
for member in sorted(members, key=lambda m: m.filename.rsplit("/", 1)[-1]):
with archive.open(member) as stream:
frame = pd.read_csv(stream, usecols=["date", "serial_number", "model", "failure"])
day = member.filename.rsplit("/", 1)[-1][:-4]
if day in dates or not frame.date.eq(day).all():
raise ValueError("Duplicate date or date/filename mismatch")
dates.add(day)
returned = frame.serial_number.isin(failed_serials)
reappearances.update(frame.loc[returned, "model"].astype(str).tolist())
repeat_flags.update(
frame.loc[returned & frame.failure.eq(1), "model"].astype(str).tolist()
)
daily = aggregate_backblaze(frame)
failed_serials.update(frame.loc[frame.failure == 1, "serial_number"])
for model, failures, exposure in daily.itertuples(index=False, name=None):
item = totals.setdefault(model, [0, 0])
item[0] += int(failures)
item[1] += int(exposure)
records = []
for model, (failures, exposure) in sorted(totals.items()):
row = _record("backblaze_2013", model, "Hard disk drive", model)
row.update(
failure_mode="reported drive failure",
distribution="observed_count",
failures=failures,
exposure=exposure,
basis="drive_day",
evidence="observed_daily_snapshots",
data_period=f"{min(dates)} to {max(dates)}",
observed_days=len(dates),
post_failure_drive_days=reappearances[model],
repeated_failure_flags=repeat_flags[model],
)
records.append(row)
return records
[docs]
def fetch_failure_database(
source_id: str, *, client: httpx.Client | None = None, max_bytes: int = 100_000_000
) -> tuple[list[Record], dict[str, str]]:
"""Fetch a registered public release, verify its hash, then normalize it.
Fixed source URLs, no arbitrary URL fetch or archive extraction. A changed
release fails closed for review. Returns records and provenance; callers
decide where to save, so live retrieval never overwrites the shipped data.
"""
catalogue = source_catalogue().set_index("source_id")
parsers = {
"nrc_2020": parse_nrc_2020,
"pvrom_2018": parse_pvrom_2018,
"wind_2016": parse_wind_2016,
"backblaze_2013": parse_backblaze_zip,
}
if source_id not in parsers:
raise ValueError("No public numeric connector for this source")
if isinstance(max_bytes, bool) or max_bytes <= 0:
raise ValueError("max_bytes must be positive")
source = catalogue.loc[source_id]
url = str(source["download_url"])
own_client = client is None
session = client or httpx.Client(timeout=60, follow_redirects=True)
try:
buffer = bytearray()
with session.stream("GET", url) as response:
response.raise_for_status()
for chunk in response.iter_bytes():
if len(buffer) + len(chunk) > max_bytes:
raise ValueError("Download exceeds max_bytes")
buffer.extend(chunk)
finally:
if own_client:
session.close()
digest = hashlib.sha256(buffer).hexdigest()
if digest != source.sha256:
raise ValueError("Source hash changed; review the new release before updating the manifest")
rows = parsers[source_id](bytes(buffer))
return rows, {"source_id": source_id, "url": url, "sha256": digest}
[docs]
def database_coverage() -> pd.DataFrame:
"""Record, population and device counts by source, including quarantined rows."""
data = load_failure_database(include_flagged=True)
result = []
for source, group in data.groupby("source_id", sort=True):
reasons = Counter(flag for flags in group.quality_flags for flag in flags)
result.append(
{
"source_id": source,
"records": len(group),
"usable_records": int(group.quality_flags.map(len).eq(0).sum()),
"device_groups": group.device.nunique(),
"populations": group.population.nunique(),
"exclusions": dict(reasons),
}
)
return pd.DataFrame(result)