"""NIST native CSV/TOA5 import, after download through the public PV portal."""
from __future__ import annotations
import csv
import io
import re
from dataclasses import dataclass
from itertools import islice
from pathlib import Path
import pandas as pd
from energy_data_retrieval.generation import GenerationResult
from energy_data_retrieval.sources._generation_utils import (
bounded_csv,
local_export,
numeric,
unique_observations,
utc_labels,
)
from energy_data_retrieval.transport import RetrievalError
PORTAL_URL = "https://pvdata.nist.gov/"
REFERENCE_IDS = ("nist_pv_testbeds", "nist_pv_dictionary", "nist_pv_measurements_paper")
[docs]
def parse_generation_csv(
content: str | bytes,
*,
array: str,
power_column: str = "InvPAC_kW_Avg",
timestamp_column: str = "TIMESTAMP",
source_timezone: str = "-05:00",
resolution: str = "one-minute",
maximum_rows: int = 1_000_000,
) -> pd.DataFrame:
"""Read an inverter/meter AC power channel from a NIST native export.
NIST labels TIMESTAMP as local standard time (LST); Gaithersburg is fixed
UTC-05:00. An explicit override supports exports that document another clock.
One-minute averages and one-second samples remain distinct. Timestamp labels
are retained without guessing which edge of the averaging interval they mark.
"""
if array.lower() not in {"ground", "roof", "canopy"}:
raise ValueError("array must be Ground, Roof, or Canopy")
if not re.fullmatch(r"(?:InvPAC_kW|PwrMtrP_kW)(?:_Avg)?(?:\(\d+\))?", power_column):
raise ValueError("Select an AC real-power channel InvPAC_kW or PwrMtrP_kW")
if resolution not in {"one-minute", "one-second"}:
raise ValueError("resolution must be one-minute or one-second")
try:
text = (
content.decode("utf-8-sig") if isinstance(content, bytes) else content.lstrip("\ufeff")
)
except UnicodeDecodeError:
raise RetrievalError("NIST export must be UTF-8 CSV") from None
header_rows = list(islice(csv.reader(io.StringIO(text)), 4))
toa5 = bool(header_rows and header_rows[0] and header_rows[0][0] == "TOA5")
if resolution == "one-minute" and "_Avg" not in power_column and not toa5:
raise ValueError("NIST one-minute exports use the _Avg channel suffix")
if resolution == "one-second" and "_Avg" in power_column:
raise ValueError("Use an instantaneous channel for one-second samples")
if toa5:
if len(header_rows) < 4 or power_column not in header_rows[1]:
raise RetrievalError("Incomplete NIST TOA5 header or missing power channel")
index = header_rows[1].index(power_column)
if len(header_rows[2]) <= index or header_rows[2][index].lower() != "kw":
raise RetrievalError("NIST TOA5 power channel must declare kW units")
aggregation = "Avg" if resolution == "one-minute" else "Smp"
if len(header_rows[3]) <= index or header_rows[3][index].lower() != aggregation.lower():
raise RetrievalError(f"NIST {resolution} power must declare {aggregation} aggregation")
frame = bounded_csv(text, maximum_rows=maximum_rows, skiprows=[0, 2, 3] if toa5 else None)
if not {timestamp_column, power_column}.issubset(frame.columns):
raise RetrievalError("NIST export lacks the selected timestamp or AC power channel")
frame["timestamp"] = utc_labels(
frame[timestamp_column], source_timezone=source_timezone, timestamp_format="ISO8601"
)
measurements = frame[power_column]
frame["value"] = numeric(measurements.where(measurements.ne("NAN")), power_column)
frame["unit"] = "kW"
frame["source"] = "nist"
frame["system_id"] = array.lower()
frame["technology"] = "solar_pv"
frame["channel"] = power_column
frame["measurement_kind"] = (
"interval_average_power" if resolution == "one-minute" else "instantaneous_power"
)
frame["sampling_seconds"] = 60 if resolution == "one-minute" else 1
frame = unique_observations(frame, ["system_id", "timestamp", "channel"])
frame.attrs.update(
source_url=PORTAL_URL,
source_timezone=source_timezone,
resolution=resolution,
timestamp_semantics="Native logger label; NIST dictionary specifies local standard time",
units_basis="NIST InvPAC_kW/PwrMtrP_kW AC real-power dictionary, kW",
reference_ids=list(REFERENCE_IDS),
)
return frame
[docs]
@dataclass(frozen=True, slots=True)
class NISTClient:
"""Import a user-downloaded export; no account, portal token or form submission."""
maximum_rows: int = 1_000_000
[docs]
def read_export(
self,
path: Path,
*,
array: str,
power_column: str = "InvPAC_kW_Avg",
timestamp_column: str = "TIMESTAMP",
source_timezone: str = "-05:00",
resolution: str = "one-minute",
) -> GenerationResult:
"""Normalize a downloaded CSV and record the exact input file's SHA-256."""
content, provenance = local_export(path)
frame = parse_generation_csv(
content,
array=array,
power_column=power_column,
timestamp_column=timestamp_column,
source_timezone=source_timezone,
resolution=resolution,
maximum_rows=self.maximum_rows,
)
frame.attrs.update(provenance)
return GenerationResult(frame, ())