Source code for energy_data_retrieval.sources.tepco.client

"""TEPCO Tokyo-area actual/estimated generation mix, in half-hour average MW."""

from __future__ import annotations

import csv
import io
import math
import tempfile
from datetime import date, datetime
from pathlib import Path

import pandas as pd

from energy_data_retrieval.generation import GenerationResult
from energy_data_retrieval.transport import HTTPClient, RetrievalError

TEPCO_HOSTS = frozenset({"www.tepco.co.jp"})
GENERATION_BASE = "https://www.tepco.co.jp/forecast/html/images"
REFERENCE_IDS = ("tepco_generation", "tepco_generation_definitions")
_SERIES = {
    "原子力": ("nuclear", "generation"),
    "火力(LNG)": ("lng", "generation"),
    "火力(石炭)": ("coal", "generation"),
    "火力(石油)": ("oil", "generation"),
    "火力(その他)": ("other_thermal", "generation"),
    "水力": ("hydro", "generation"),
    "地熱": ("geothermal", "generation"),
    "バイオマス": ("biomass", "generation"),
    "太陽光発電実績": ("solar", "generation"),
    "風力発電実績": ("wind", "generation"),
    "その他": ("other", "generation"),
    "太陽光出力制御量": ("solar", "curtailment"),
    "風力出力制御量": ("wind", "curtailment"),
    "揚水": ("pumped_storage", "storage_net"),
    "蓄電池": ("battery", "storage_net"),
    "連系線": ("interconnector", "net_import"),
}
_COLUMNS = [
    "timestamp",
    "interval_end",
    "source",
    "area",
    "technology",
    "value",
    "unit",
    "source_unit",
    "source_value",
    "source_series",
    "flow",
    "aggregation",
    "source_datetime",
    "may_include_estimates",
]


[docs] def generation_urls(start: date, end: date, *, maximum_requests: int = 24) -> tuple[str, ...]: """Plan the current TEPCO monthly export format (February 2024 onward).""" if type(maximum_requests) is not int or maximum_requests < 1: raise ValueError("maximum_requests must be a positive integer") if type(start) is not date or type(end) is not date or end < start: raise ValueError("start and end must be ordered calendar dates") if start < date(2024, 2, 1): raise ValueError("This TEPCO connector supports the February 2024 onward CSV format") count = (end.year - start.year) * 12 + end.month - start.month + 1 if count > maximum_requests: raise ValueError("TEPCO interval exceeds maximum_requests") first = start.year * 12 + start.month - 1 return tuple( f"{GENERATION_BASE}/eria_jukyu_{(first + n) // 12:04d}{(first + n) % 12 + 1:02d}_03.csv" for n in range(count) )
[docs] def parse_generation_csv( data: bytes | str, *, include_flows: bool = False, maximum_rows: int = 1_000_000 ) -> pd.DataFrame: """Parse the declared CSV units, preserving half-hour average power. CSV starts with a physical-unit row and a Japanese header. Raw bytes may be CP932; no lossy replacement is used. MW averages are not interval MWh. Signed storage, net imports and curtailment are available only with include_flows; they retain explicit flow types. Demand and the supplied total are excluded. """ if type(maximum_rows) is not int or maximum_rows < 1: raise ValueError("maximum_rows must be a positive integer") if isinstance(data, bytes): try: text = data.decode("utf-8-sig") except UnicodeDecodeError: try: text = data.decode("cp932") except UnicodeDecodeError: raise RetrievalError("TEPCO CSV is neither UTF-8 nor CP932") from None else: text = data.lstrip("\ufeff") lines = csv.reader(io.StringIO(text)) units = next(lines, []) scales = {"単位[MW平均]": ("MW", 1.0), "単位[万kW平均]": ("10000kW", 10.0)} if not units or units[0].strip() not in scales: raise RetrievalError("TEPCO CSV omitted a supported average-power unit declaration") source_unit, scale = scales[units[0].strip()] header = next(lines, []) required = {"DATE", "TIME", "エリア需要", "合計", *_SERIES} if set(header) != required or len(header) != len(required): raise RetrievalError("TEPCO CSV has an unexpected generation-mix header") rows: list[dict[str, object]] = [] seen: set[pd.Timestamp] = set() for values in lines: if not values: continue if len(values) != len(header): raise RetrievalError("TEPCO CSV row width differs from its header") item = dict(zip(header, values, strict=True)) label = f"{item['DATE']} {item['TIME']}" try: parsed = datetime.strptime(label, "%Y/%m/%d %H:%M") if parsed.minute not in (0, 30): raise ValueError stamp = pd.Timestamp(parsed, tz="Asia/Tokyo").tz_convert("UTC") except (ValueError, TypeError): raise RetrievalError("TEPCO returned an invalid half-hour start label") from None if stamp in seen: raise RetrievalError("TEPCO returned duplicate half-hour observations") seen.add(stamp) for source_series, (technology, flow) in _SERIES.items(): if flow != "generation" and not include_flows: continue raw = item[source_series].strip() try: value = float(raw) if raw not in ("", "-", "―") else float("nan") except ValueError: raise RetrievalError("TEPCO generation is not numeric") from None if raw not in ("", "-", "―") and not math.isfinite(value): raise RetrievalError("TEPCO generation contains a non-finite value") if len(rows) >= maximum_rows: raise RetrievalError("TEPCO response exceeds maximum_rows") rows.append( { "timestamp": stamp, "interval_end": stamp + pd.Timedelta(minutes=30), "source": "tepco", "area": "JP-Tokyo", "technology": technology, "value": value * scale, "unit": "MW", "source_unit": source_unit, "source_value": value, "source_series": source_series, "flow": flow, "aggregation": "area_technology", "source_datetime": label, "may_include_estimates": True, } ) frame = pd.DataFrame(rows, columns=_COLUMNS) for name in ("timestamp", "interval_end"): frame[name] = pd.to_datetime(frame[name], utc=True) return frame.sort_values(["timestamp", "flow", "technology"]).reset_index(drop=True)
[docs] class TEPCOClient: """Download bounded monthly native-encoding CSV files with byte provenance.""" def __init__( self, http: HTTPClient, *, maximum_requests: int = 24, maximum_rows: int = 1_000_000 ) -> None: if type(maximum_requests) is not int or maximum_requests < 1: raise ValueError("maximum_requests must be a positive integer") if type(maximum_rows) is not int or maximum_rows < 1: raise ValueError("maximum_rows must be a positive integer") self.http, self.maximum_requests, self.maximum_rows = http, maximum_requests, maximum_rows
[docs] def generation( self, start: date, end: date, *, include_flows: bool = False ) -> GenerationResult: """Fetch inclusive Japanese calendar dates; timestamps are interval starts.""" urls = generation_urls(start, end, maximum_requests=self.maximum_requests) frames, records = [], [] total = 0 with tempfile.TemporaryDirectory(prefix="tepco-generation-") as folder: for index, url in enumerate(urls): if total >= self.maximum_rows: raise RetrievalError("TEPCO response exceeds maximum_rows") target = Path(folder) / f"{index}.csv" record = self.http.download_file(url, target) frame = parse_generation_csv( target.read_bytes(), include_flows=include_flows, maximum_rows=self.maximum_rows - total, ) total += len(frame) frames.append(frame) records.append(record) frame = pd.concat(frames, ignore_index=True) lower = pd.Timestamp(start, tz="Asia/Tokyo").tz_convert("UTC") upper = pd.Timestamp(end, tz="Asia/Tokyo").tz_convert("UTC") + pd.Timedelta(days=1) frame = frame.loc[(frame.timestamp >= lower) & (frame.timestamp < upper)] return GenerationResult(frame.reset_index(drop=True), tuple(records))