Source code for space_pv_connectors.satellitecots

"""Pinned BUPT-1 platform telemetry from the SatelliteCOTS research archive.

The source has no explicit redistribution licence. Download into a local cache;
do not bundle its raw telemetry with OrbitalPV. Naive source clocks stay naive
unless the caller supplies a timezone. MPPT input, controller output and bus
load are distinct measurement boundaries, none an available-MPP measurement.
"""

from __future__ import annotations

import json
import zipfile
from dataclasses import asdict
from hashlib import sha256
from pathlib import Path
from typing import Literal

import numpy as np
import pandas as pd

from .http_base import DownloadReceipt, HttpConnector

SATELLITECOTS_COMMIT = "951b41521351d535b7c2354916d9c4991602e8c7"
_BASE = (
    "https://raw.githubusercontent.com/TiansuanConstellation/"
    f"MobiCom24-SatelliteCOTS/{SATELLITECOTS_COMMIT}"
)
_SOURCES = {
    "common": (
        "CommonData-Telemetries/telemetry_all.csv.zip",
        "5d761d0bb65730cdbdb364f9c8706e9469bbbc6049cc0ed0b22d95ead8d656fa",
        80_884_357,
    ),
    "multiplexed": (
        "Energy-ShortTerm/telemetries/all.csv",
        "817fa3bb188acef2923436c5e72a5e64383598028df97976a4d375c2d2b39dec",
        499_260,
    ),
}
Source = Literal["common", "multiplexed"]


def _digest(path: Path) -> str:
    digest = sha256()
    with path.open("rb") as stream:
        while chunk := stream.read(1024 * 1024):
            digest.update(chunk)
    return digest.hexdigest()


def _source(source: Source) -> tuple[str, str, int]:
    if source not in _SOURCES:
        raise ValueError("source must be 'common' or 'multiplexed'")
    return _SOURCES[source]


[docs] class SatelliteCOTSConnector(HttpConnector): """Download the authors' original files, pinned by commit and SHA-256.""" def download( self, destination: str | Path, *, source: Source = "common", refresh: bool = False ) -> DownloadReceipt: relative, expected, size = _source(source) path = Path(destination) sidecar = path.with_name(path.name + ".provenance.json") url = f"{_BASE}/{relative}" if path.exists() and not refresh: if _digest(path) != expected: raise ValueError("BUPT-1 cache does not match the pinned source checksum") if not sidecar.exists(): raise ValueError("Verified cache lacks provenance; use refresh=True to retrieve it") meta = json.loads(sidecar.read_text(encoding="utf-8")) if meta.get("sha256") != expected or meta.get("url") != url: raise ValueError("BUPT-1 cache provenance does not match the pinned source") return DownloadReceipt( **{key: meta[key] for key in DownloadReceipt.__dataclass_fields__} ) path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(path.name + ".download") digest = sha256() downloaded = 0 try: with self._client.stream("GET", url) as response, temporary.open("wb") as stream: response.raise_for_status() for chunk in response.iter_bytes(): downloaded += len(chunk) if downloaded > size: raise ValueError("BUPT-1 download exceeds the pinned source size") stream.write(chunk) digest.update(chunk) if downloaded != size or digest.hexdigest() != expected: raise ValueError("BUPT-1 download does not match the pinned source checksum") temporary.replace(path) finally: temporary.unlink(missing_ok=True) # Receipt timestamp is retrieval time, never a measurement timestamp. receipt = self.receipt("BUPT-1 / SatelliteCOTS", url, b"", path=path) receipt = DownloadReceipt( receipt.source, url, receipt.retrieved_at, expected, downloaded, str(path) ) sidecar.write_text( json.dumps( { **asdict(receipt), "source_commit": SATELLITECOTS_COMMIT, "source_schema": source, "source_paper": "https://arxiv.org/abs/2401.03435", "license": "Not declared in the source repository", "source_timezone": "Unspecified", }, indent=2, ) + "\n", encoding="utf-8", ) return receipt
def _timestamps(values: pd.Series, source_timezone: str | None) -> pd.Series: times = pd.to_datetime(values, errors="coerce", format="mixed") if len(times) and times.isna().all(): raise ValueError("BUPT-1 data has no readable source timestamps") if getattr(times.dt, "tz", None) is not None: return times.dt.tz_convert("UTC") if source_timezone is not None: return times.dt.tz_localize( source_timezone, ambiguous="raise", nonexistent="raise" ).dt.tz_convert("UTC") return times
[docs] def normalize_bupt1_telemetry( raw: pd.DataFrame, *, source: Source = "common", source_timezone: str | None = None ) -> pd.DataFrame: """Normalize units without interpolating or demultiplexing measurements. The common archive supplies separate MPPT1/2 and battery1/2 columns. The short-term source cycles through hardware groups; its group-specific power must not be mistaken for simultaneous whole-spacecraft PV power. Duplicate source clocks and signed sensor offsets remain visible. Zero temperature is retained because zero is not documented as a universal missing sentinel. """ _source(source) required = {"Time", "Total_U", "Total_I"} if source == "common": required |= {f"MPPT{n}_{q}" for n in (1, 2) for q in ("Uin", "Iin", "Iout")} else: required |= {"MPPT_Group", "MPPT_Uin", "MPPT_Iin", "MPPT_Iout", "BATTERY_Group"} missing = required - set(raw.columns) if missing: raise ValueError(f"BUPT-1 {source} source lacks columns: {sorted(missing)}") out = pd.DataFrame(index=raw.index) out["source_row"] = raw.index out["source_timestamp"] = raw["Time"].astype(str) out["timestamp"] = _timestamps(raw["Time"], source_timezone) def numeric(column: str) -> pd.Series: values = pd.to_numeric(raw[column], errors="coerce") return values.where(np.isfinite(values)) out["bus_voltage_v"] = numeric("Total_U") / 1000 out["bus_current_a"] = numeric("Total_I") / 1000 out["bus_power_w"] = out["bus_voltage_v"] * out["bus_current_a"] if source == "common": for number in (1, 2): prefix = f"mppt_{number}" for source_quantity, quantity in ( ("Uin", "input_voltage_v"), ("Iin", "input_current_a"), ("Iout", "output_current_a"), ): out[f"{prefix}_{quantity}"] = numeric(f"MPPT{number}_{source_quantity}") / 1000 out[f"{prefix}_input_power_w"] = ( out[f"{prefix}_input_voltage_v"] * out[f"{prefix}_input_current_a"] ) out[f"{prefix}_output_power_w"] = ( out["bus_voltage_v"] * out[f"{prefix}_output_current_a"] ) if {f"BATTERY{number}_U", f"BATTERY{number}_I"} <= set(raw.columns): out[f"battery_{number}_voltage_v"] = numeric(f"BATTERY{number}_U") / 1000 out[f"battery_{number}_current_a"] = numeric(f"BATTERY{number}_I") / 1000 out[f"battery_{number}_source_signed_power_w"] = ( out[f"battery_{number}_voltage_v"] * out[f"battery_{number}_current_a"] ) out["pv_input_power_w"] = out[["mppt_1_input_power_w", "mppt_2_input_power_w"]].sum( axis=1, min_count=2 ) out["solar_bus_power_w"] = out[["mppt_1_output_power_w", "mppt_2_output_power_w"]].sum( axis=1, min_count=2 ) battery_columns = [f"battery_{n}_source_signed_power_w" for n in (1, 2)] if set(battery_columns) <= set(out.columns): out["battery_source_signed_power_w"] = out[battery_columns].sum(axis=1, min_count=2) else: out["mppt_group"] = numeric("MPPT_Group") out["battery_group"] = numeric("BATTERY_Group") out["group_pv_input_power_w"] = numeric("MPPT_Uin") * numeric("MPPT_Iin") / 1e6 out["group_mppt_output_power_w"] = numeric("MPPT_Iout") * out["bus_voltage_v"] / 1000 if {"Battery_U", "Battery_I"} <= set(raw.columns): out["group_battery_source_signed_power_w"] = ( numeric("Battery_U") * numeric("Battery_I") / 1e6 ) for source_name, name in ( ("I_Atlas200DK-A", "atlas_a_current_a"), ("I_Atlas200DK-B", "atlas_b_current_a"), ("I_Pi-A", "pi_a_current_a"), ("I_Pi-B", "pi_b_current_a"), ("POBC_I_5V", "pobc_current_a"), ("XMIT_A_12V", "transmitter_a_current_a"), ("XMIT_B_12V", "transmitter_b_current_a"), ): if source_name in raw: out[name] = numeric(source_name) / 1000 for source_name, name in ( ("ATLAS_A_TEMP", "atlas_a_temperature_c"), ("ATLAS_B_TEMP", "atlas_b_temperature_c"), ("PI_A_TEMP", "pi_a_temperature_c"), ): if source_name in raw: out[name] = numeric(source_name) out["valid_timestamp"] = out["timestamp"].notna() out["duplicate_timestamp"] = out["timestamp"].duplicated(keep=False) out.attrs.update( satellite="BUPT-1", schema=source, source_commit=SATELLITECOTS_COMMIT, timestamp_basis=( "Source-local clock; timezone unspecified" if source_timezone is None else f"Caller-supplied timezone {source_timezone}; converted to UTC" ), source_timezone=source_timezone, license="Not declared in the source repository", power_boundary="Observed operating input/output/bus power; available MPP is not observed", battery_sign="Source-signed V*I; source plotting code negates it for discharge", temperature_missing_policy="Zero values retained; no undocumented sentinel replacement", ) return out.reset_index(drop=True)
[docs] def load_bupt1_telemetry( path: str | Path, *, source: Source = "common", source_timezone: str | None = None, time_min: str | pd.Timestamp | None = None, time_max: str | pd.Timestamp | None = None, verify_source: bool = True, chunksize: int = 100_000, ) -> pd.DataFrame: """Stream a local pinned archive, filtering inclusive/exclusive source times. Bounds must match the timestamp basis: naive for an unspecified source timezone, or timezone-aware if ``source_timezone`` is supplied. The full archive is large; supply a time window. No ZIP member is extracted to disk. Set ``verify_source=False`` only for independently provided files, which are then labelled unverified rather than claimed as the pinned archive. """ _, expected, _ = _source(source) if isinstance(chunksize, bool) or not isinstance(chunksize, int) or chunksize < 1: raise ValueError("chunksize must be a positive integer") file = Path(path) digest = _digest(file) if verify_source and digest != expected: raise ValueError("BUPT-1 local file does not match the pinned source checksum") bounds = [None if value is None else pd.Timestamp(value) for value in (time_min, time_max)] for value in bounds: if value is not None and (value.tzinfo is None) != (source_timezone is None): raise ValueError("Time bounds must match the source timezone interpretation") start, stop = bounds if start is not None and stop is not None and start >= stop: raise ValueError("time_min must be earlier than time_max") pieces: list[pd.DataFrame] = [] def read(stream) -> None: for chunk in pd.read_csv(stream, chunksize=chunksize, encoding="utf-8-sig"): times = _timestamps(chunk["Time"], source_timezone) selected = times.notna() if start is not None: selected &= times >= start if stop is not None: selected &= times < stop if selected.any(): pieces.append( normalize_bupt1_telemetry( chunk.loc[selected], source=source, source_timezone=source_timezone ) ) if source == "common": try: with zipfile.ZipFile(file) as archive: matches = [ item for item in archive.infolist() if item.filename == "telemetry_all.csv" ] if len(matches) != 1 or matches[0].file_size > 4_000_000_000: raise ValueError("BUPT-1 archive must contain one bounded telemetry_all.csv") with archive.open(matches[0]) as stream: read(stream) except zipfile.BadZipFile as exc: raise ValueError("BUPT-1 source is not a readable ZIP archive") from exc else: with file.open("rb") as stream: read(stream) if not pieces: raise ValueError("No BUPT-1 measurements in the requested interval") metadata = pieces[0].attrs.copy() result = pd.concat(pieces, ignore_index=True) result["duplicate_timestamp"] = result["timestamp"].duplicated(keep=False) result.attrs.update( {**metadata, "sha256": digest, "verified_source": verify_source, "source_path": str(file)} ) return result
__all__ = [ "SATELLITECOTS_COMMIT", "SatelliteCOTSConnector", "load_bupt1_telemetry", "normalize_bupt1_telemetry", ]