Source code for space_pv_connectors.birds

from __future__ import annotations

import re
from pathlib import Path
from typing import ClassVar

import pandas as pd

from .http_base import DownloadReceipt, HttpConnector
from .tabular import normalize_birds_like


[docs] class BirdsEPSConnector(HttpConnector): """Connector for the public BIRDS on-orbit EPS dataset.""" API = "https://api.github.com/repos/BIRDSOpenSource/EPS_dataset/contents" SATELLITES: ClassVar[frozenset[str]] = frozenset({"NEPALISAT", "RAAVANA", "TSURU", "UGUISU"}) DATASET_DOI = "10.17632/8kp25ycf63.1"
[docs] def download(self, satellite: str, destination: str | Path) -> DownloadReceipt: """Download one satellite workbook from the upstream BIRDS repository.""" name = satellite.upper() if name not in self.SATELLITES: raise ValueError( f"Unknown BIRDS EPS satellite: {satellite}; choose {sorted(self.SATELLITES)}" ) metadata = self._client.get(f"{self.API}/{name}.xlsx") metadata.raise_for_status() download_url = metadata.json()["download_url"] response = self._client.get(download_url) response.raise_for_status() path = Path(destination) path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(response.content) return self.receipt("BIRDS EPS", download_url, response.content, path=path)
[docs] def download_frame( self, satellite: str, destination: str | Path, *, normalized: bool = True, campaign_year: int | None = None, ) -> pd.DataFrame: """Download and load a BIRDS EPS workbook. ``campaign_year`` supplies the year only for campaign sheet names that omit it; see :func:`load_birds_eps_workbook` for the time convention. """ receipt = self.download(satellite, destination) path = Path(receipt.path or destination) return load_birds_eps_workbook(path, normalized=normalized, campaign_year=campaign_year)
_BIRDS_PANEL_CODES = { "py": "+Y", "px": "+X", "mz": "-Z", "mx": "-X", "pz": "+Z", } _BIRDS_TEST_SHEETS = frozenset({"test1 w batt", "test2 wo batt"}) def _column_slug(value: object) -> str: text = str(value).strip().lower() return re.sub(r"[^a-z0-9]+", "_", text).strip("_") def _campaign_anchor( sheet_name: str | None, campaign_year: int | None, ) -> tuple[pd.Timestamp, str]: """Parse a dated campaign without guessing an omitted year or clock time.""" name = " ".join((sheet_name or "").split()) basis = "sheet_date" if not re.search(r"\b\d{4}\b", name): if not re.fullmatch(r"(?:[A-Za-z]+\s+\d{1,2}|\d{1,2}\s+[A-Za-z]+)", name): raise ValueError( f"BIRDS campaign sheet {sheet_name!r} has no recognizable date; " "numeric Time Stamp values are elapsed seconds, not Unix timestamps." ) if campaign_year is None: raise ValueError( f"BIRDS campaign sheet {sheet_name!r} omits its year; " "pass campaign_year explicitly after checking the workbook date context." ) name = f"{name} {campaign_year}" basis = "explicit_campaign_year" try: parsed = pd.to_datetime(name, utc=True, dayfirst=True) if not isinstance(parsed, pd.Timestamp): raise ValueError("Campaign date is missing") anchor = parsed.normalize() except (ValueError, OverflowError) as exc: raise ValueError(f"Cannot parse BIRDS campaign date from sheet {sheet_name!r}") from exc return anchor, basis def _normalize_birds_calibrated( frame: pd.DataFrame, *, campaign_date: str | None = None, campaign_year: int | None = None, ) -> pd.DataFrame | None: columns = {_column_slug(column): column for column in frame.columns} if "time_stamp" not in columns: return None available = [ code for code in _BIRDS_PANEL_CODES if f"v{code}_mv" in columns and f"i{code}_ma" in columns ] if not available: return None raw_timestamp = frame[columns["time_stamp"]] numeric_timestamp = pd.to_numeric(raw_timestamp, errors="coerce") numeric_clock = ( not pd.api.types.is_datetime64_any_dtype(raw_timestamp) and numeric_timestamp.notna().any() ) metadata: dict[str, object] if numeric_clock: campaign_start, date_basis = _campaign_anchor(campaign_date, campaign_year) # Some sheets contain hundreds of calibration/footer rows with no # clock. Their proportion must not turn elapsed seconds into epoch ns. numeric_timestamp = numeric_timestamp.where( numeric_timestamp.ge(0) & numeric_timestamp.lt(float("inf")) ) timestamp = campaign_start + pd.to_timedelta(numeric_timestamp, unit="s") metadata = { "elapsed_seconds": numeric_timestamp, "campaign_date": campaign_start, "timestamp_basis": "campaign_date_midnight_plus_elapsed_seconds", "campaign_date_basis": date_basis, } else: timestamp = pd.to_datetime(raw_timestamp, errors="coerce", utc=True) metadata = {"timestamp_basis": "reported_timestamp"} parts: list[pd.DataFrame] = [] for code in available: part = pd.DataFrame( { "timestamp": timestamp, "panel_id": _BIRDS_PANEL_CODES[code], "voltage_v": pd.to_numeric(frame[columns[f"v{code}_mv"]], errors="coerce") / 1000.0, "current_a": pd.to_numeric(frame[columns[f"i{code}_ma"]], errors="coerce") / 1000.0, **metadata, } ) temp_key = f"t{code}_c" if temp_key in columns: part["temperature_c"] = pd.to_numeric(frame[columns[temp_key]], errors="coerce") parts.append(part) return pd.concat(parts, ignore_index=True).dropna( subset=["timestamp", "voltage_v", "current_a"] ) def _header_score(values: list[object]) -> int: text = " ".join(str(value).lower() for value in values if pd.notna(value)) keywords = ("volt", "current", "curr", "temp", "date", "time", "panel", "solar", "battery") return sum(keyword in text for keyword in keywords) def _discover_header_row(path: Path | pd.ExcelFile, sheet_name: str) -> int: preview = pd.read_excel(path, sheet_name=sheet_name, header=None, nrows=30) scores = [_header_score(row.tolist()) for _, row in preview.iterrows()] if not scores: return 0 best = int(max(range(len(scores)), key=scores.__getitem__)) return best if scores[best] >= 2 else 0
[docs] def load_birds_eps_workbook( path: str | Path, *, normalized: bool = True, campaign_year: int | None = None, ) -> pd.DataFrame: """Load a downloaded BIRDS EPS workbook into the canonical panel schema. Parameters ---------- path : str or pathlib.Path Downloaded upstream XLSX workbook. normalized : bool, default True Return panel-level flight telemetry. The known undated TSURU sheets ``Test1 w batt`` and ``Test2 wo batt`` are excluded from this mode. ``False`` returns raw rows from every nonempty sheet, including tests. campaign_year : int, optional Explicit year for campaign sheet dates that omit it. For example, the TSURU 2021 flight campaigns require ``campaign_year=2021``. A year already stated in a sheet name is always preserved. No current-year assumption is made; an omitted year raises an actionable error. Notes ----- Calibrated numeric ``Time Stamp`` values are elapsed seconds. The returned ``timestamp`` anchors these to midnight on ``campaign_date`` for ordering; this is **not a known absolute UTC acquisition time**. The original clock is retained in ``elapsed_seconds`` and the convention in ``timestamp_basis``. ``campaign_date_basis`` records whether a year was supplied by the caller. Rows without a valid elapsed clock or voltage/current pair are omitted. """ if campaign_year is not None and ( isinstance(campaign_year, bool) or not isinstance(campaign_year, int) or not 1 <= campaign_year <= 9999 ): raise ValueError("campaign_year must be an integer year between 1 and 9999") workbook = Path(path) raw_parts: list[pd.DataFrame] = [] normalized_parts: list[pd.DataFrame] = [] errors: list[str] = [] with pd.ExcelFile(workbook) as excel: for sheet in excel.sheet_names: if normalized and " ".join(sheet.lower().split()) in _BIRDS_TEST_SHEETS: continue header_row = _discover_header_row(excel, sheet) frame = pd.read_excel(excel, sheet_name=sheet, header=header_row) frame = frame.dropna(how="all") if frame.empty: continue frame["source_sheet"] = sheet raw_parts.append(frame) if normalized: clean = frame.drop(columns=["source_sheet"]) part = _normalize_birds_calibrated( clean, campaign_date=sheet, campaign_year=campaign_year ) if part is None: try: part = normalize_birds_like(clean) except ValueError as exc: errors.append(f"{sheet}: {exc}") continue if part.empty: errors.append(f"{sheet}: no valid panel observations") continue part["source_sheet"] = sheet normalized_parts.append(part) if normalized: if not normalized_parts: detail = "; ".join(errors) if errors else "no readable sheets" raise ValueError(f"Could not normalize BIRDS EPS workbook {workbook.name}: {detail}") return ( pd.concat(normalized_parts, ignore_index=True) .sort_values(["panel_id", "timestamp"]) .reset_index(drop=True) ) if not raw_parts: raise ValueError(f"No readable sheets found in {workbook}") return pd.concat(raw_parts, ignore_index=True)
__all__ = ["BirdsEPSConnector", "load_birds_eps_workbook"]