Source code for space_pv_connectors.flying_laptop

"""Calibrated on-orbit solar-array telemetry from the Flying Laptop mission."""

from __future__ import annotations

import json
import re
import tarfile
from dataclasses import asdict
from datetime import datetime
from hashlib import md5, sha256
from pathlib import Path, PurePosixPath

import numpy as np
import pandas as pd

from .http_base import DownloadReceipt, HttpConnector

_DATASET_DOI = "10.18419/darus-451"
_DESCRIPTION_DOI = "10.18419/darus-456"
_PANELS = {"00": "right", "01": "left", "02": "center", "03": "test"}
_PACKET_NAME = re.compile(r"hk4900_\d{4}-\d{2}(?:_\d+)?\.csv")
_VALUE_COLUMNS = {f"PST{quantity}{code}" for code in _PANELS for quantity in ("CUR", "VLT")}


def _validate_month(month: str) -> None:
    if not isinstance(month, str) or not re.fullmatch(r"\d{4}-\d{2}", month):
        raise ValueError("month must have the form YYYY-MM")
    try:
        datetime.strptime(month, "%Y-%m")
    except ValueError as exc:
        raise ValueError("month must be a valid calendar month in YYYY-MM form") from exc


def _provenance_path(path: Path) -> Path:
    return path.with_name(f"{path.name}.provenance.json")


def _utc_bound(value: str | pd.Timestamp | None) -> pd.Timestamp | None:
    if value is None:
        return None
    bound = pd.Timestamp(value)
    if not isinstance(bound, pd.Timestamp):
        raise ValueError("Time bounds must be valid timestamps")
    return bound.tz_localize("UTC") if bound.tzinfo is None else bound.tz_convert("UTC")


[docs] class FlyingLaptopConnector(HttpConnector): """Retrieve CC BY 4.0 Flying Laptop monthly archives from DaRUS. The archive contains many housekeeping packets. The normalizer reads only packet 4900, whose calibrated array currents and voltages have documented units of A and V. Downloads retain a checksum-bearing provenance sidecar. """ BASE_URL = "https://darus.uni-stuttgart.de" DATASET_DOI = _DATASET_DOI
[docs] def download_month(self, month: str, destination: str | Path) -> DownloadReceipt: """Download one YYYY-MM archive after checking its catalog checksum.""" _validate_month(month) response = self._client.get( f"{self.BASE_URL}/api/datasets/:persistentId/", params={"persistentId": f"doi:{self.DATASET_DOI}"}, ) response.raise_for_status() version = response.json()["data"]["latestVersion"] filename = f"FLP_TM_HK_{month}.tar.bz2" matches = [ entry["dataFile"] for entry in version["files"] if entry["dataFile"]["filename"] == filename ] if len(matches) != 1: raise LookupError(f"No unique Flying Laptop archive found for {month}") file_metadata = matches[0] file_id = file_metadata["id"] if isinstance(file_id, bool) or not isinstance(file_id, int) or file_id <= 0: raise ValueError("Flying Laptop catalog contains an invalid datafile ID") url = f"{self.BASE_URL}/api/access/datafile/{file_id}" response = self._client.get(url) response.raise_for_status() content = response.content checksum = file_metadata.get("checksum", {}) algorithm = str(checksum.get("type", "")).upper() if algorithm == "MD5": actual_checksum = md5(content, usedforsecurity=False).hexdigest() elif algorithm == "SHA-256": actual_checksum = sha256(content).hexdigest() else: raise ValueError("Flying Laptop catalog does not supply a supported checksum") if actual_checksum != str(checksum.get("value", "")).lower(): raise ValueError("Flying Laptop download does not match its catalog checksum") if not content.startswith(b"BZh"): raise ValueError("Flying Laptop download is not a bzip2 archive") path = Path(destination) path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(content) receipt = self.receipt("Flying Laptop / DaRUS", url, content, path=path) metadata = { **asdict(receipt), "dataset_doi": self.DATASET_DOI, "description_doi": _DESCRIPTION_DOI, "license": "CC BY 4.0", "month": month, "source_filename": filename, "datafile_id": file_id, "dataset_version": f"{version.get('versionNumber')}.{version.get('versionMinorNumber')}", "catalog_checksum": checksum, } _provenance_path(path).write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") return receipt
[docs] def download_frame( self, month: str, destination: str | Path, *, time_min: str | pd.Timestamp | None = None, time_max: str | pd.Timestamp | None = None, ) -> pd.DataFrame: """Download a month and normalize the requested UTC time interval.""" receipt = self.download_month(month, destination) return load_flying_laptop_power( receipt.path or destination, time_min=time_min, time_max=time_max )
[docs] def load_flying_laptop_power( path: str | Path, *, time_min: str | pd.Timestamp | None = None, time_max: str | pd.Timestamp | None = None, ) -> pd.DataFrame: """Read calibrated solar-array V, I and computed power from a local archive. ``time_min`` is inclusive and ``time_max`` exclusive; naive bounds are UTC. Timestamps are the source's Unix epoch milliseconds in UTC, identifying packet generation. The source documents up to 400 ms of additional sensor acquisition delay. Missing/invalid readings remain missing and are omitted per voltage/current pair; signed sensor values are not clipped. Four panel identifiers are preserved: ``right``, ``left``, ``center`` and ``test``. Here ``test`` is the spacecraft's experimental array channel, not a separate ground-test recording. Power is computed as V times I; no independently transmitted power measurement is implied. The reader streams only matching packet 4900 members, never extracting archive paths to disk. A downloaded provenance sidecar, when present, is verified against the local archive. Source metadata is returned in attrs. """ start, stop = _utc_bound(time_min), _utc_bound(time_max) if start is not None and stop is not None and start >= stop: raise ValueError("time_min must be earlier than time_max") archive_path = Path(path) digest = sha256(archive_path.read_bytes()).hexdigest() provenance: dict[str, object] = { "dataset_doi": _DATASET_DOI, "description_doi": _DESCRIPTION_DOI, "license": "CC BY 4.0", "sha256": digest, "timestamp_basis": "UTC packet generation; Unix epoch milliseconds", "sensor_acquisition_delay_max_ms": 400, } sidecar = _provenance_path(archive_path) if sidecar.exists(): downloaded_metadata = json.loads(sidecar.read_text(encoding="utf-8")) if downloaded_metadata.get("sha256") != digest: raise ValueError("Flying Laptop cache does not match its provenance checksum") provenance.update(downloaded_metadata) parts: list[pd.DataFrame] = [] source_files: list[str] = [] try: with tarfile.open(archive_path, mode="r|bz2") as archive: for member in archive: if not member.isfile() or not _PACKET_NAME.fullmatch( PurePosixPath(member.name).name ): continue source_files.append(member.name) stream = archive.extractfile(member) if stream is None: continue with stream: raw = pd.read_csv( stream, sep="\t", usecols=lambda column: column in _VALUE_COLUMNS | {"timestamp"}, ) missing = (_VALUE_COLUMNS | {"timestamp"}) - set(raw.columns) if missing: raise ValueError( f"Flying Laptop packet {member.name} lacks columns: {sorted(missing)}" ) timestamp = pd.to_datetime( pd.to_numeric(raw["timestamp"], errors="coerce"), unit="ms", utc=True, errors="coerce", ) if len(raw) and timestamp.isna().all(): raise ValueError( f"Flying Laptop packet {member.name} has no valid epoch-ms timestamps" ) selected = timestamp.notna() if start is not None: selected &= timestamp >= start if stop is not None: selected &= timestamp < stop raw = raw.loc[selected] for code, panel_id in _PANELS.items(): panel = pd.DataFrame( { "timestamp": timestamp.loc[selected], "panel_id": panel_id, "voltage_v": pd.to_numeric(raw[f"PSTVLT{code}"], errors="coerce"), "current_a": pd.to_numeric(raw[f"PSTCUR{code}"], errors="coerce"), "satellite": "Flying Laptop", "source_packet": "4900", "source_file": member.name, } ) panel = panel[ np.isfinite(panel["voltage_v"]) & np.isfinite(panel["current_a"]) ].copy() panel["power_w"] = panel["voltage_v"] * panel["current_a"] parts.append(panel) except (tarfile.TarError, EOFError) as exc: raise ValueError("Could not read the Flying Laptop bzip2 tar archive") from exc if not source_files: raise ValueError("Flying Laptop archive has no packet 4900 CSV members") if not parts: raise ValueError("Flying Laptop archive contains no readable packet 4900 data") result = ( pd.concat(parts, ignore_index=True) .sort_values(["panel_id", "timestamp"]) .reset_index(drop=True) ) provenance["source_files"] = source_files result.attrs.update(provenance) return result
__all__ = ["FlyingLaptopConnector", "load_flying_laptop_power"]