Source PSCR from MPSC rate book

Author: javaDevJT

custom_components/dte_rates/const.py

@@ -6,9 +6,9 @@
     "residential/Service-Request/pricing/residential-pricing-options/"
     "ResidentialElectricRateCard.pdf"
 )
-RIDER18_CALCULATOR_URL = (
-    "https://www.dteenergy.com/content/dam/dteenergy/deg/website/"
-    "hybris/rooftop-solar/Rider18Calculator.xlsx"
+PSCR_RATE_BOOK_URL = (
+    "https://www.michigan.gov/-/media/Project/Websites/mpsc/consumer/"
+    "rate-books/electric/dte/dtee1cur.pdf"
 )
 
 UPDATE_INTERVAL = timedelta(days=7)

custom_components/dte_rates/coordinator.py

@@ -7,10 +7,10 @@
 from homeassistant.helpers.aiohttp_client import async_get_clientsession
 from homeassistant.helpers.update_coordinator import DataUpdateCoordinator, UpdateFailed
 
-from .const import RATE_CARD_URL, RIDER18_CALCULATOR_URL, UPDATE_INTERVAL
+from .const import PSCR_RATE_BOOK_URL, RATE_CARD_URL, UPDATE_INTERVAL
 from .models import ParsedRateCard
 from .pdf_parser import parse_rate_card_pdf
-from .pscr_parser import parse_pscr_cents_from_xlsx
+from .pscr_parser import parse_pscr_cents_from_pdf
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -36,12 +36,19 @@ async def _async_update_data(self) -> ParsedRateCard:
 
         pscr_bytes: bytes | None = None
         try:
-            async with session.get(RIDER18_CALCULATOR_URL, timeout=60) as resp:
+            async with session.get(
+                PSCR_RATE_BOOK_URL,
+                headers={
+                    "Accept": "application/pdf,*/*",
+                    "User-Agent": "DTE-Rates-for-Home-Assistant/1.0",
+                },
+                timeout=60,
+            ) as resp:
                 resp.raise_for_status()
                 pscr_bytes = await resp.read()
         except Exception as err:
             _LOGGER.warning(
-                "Failed downloading DTE Rider 18 calculator; export rates will omit PSCR: %s",
+                "Failed downloading MPSC DTE rate book; export rates will omit PSCR: %s",
                 err,
             )
 
@@ -59,13 +66,13 @@ async def _async_update_data(self) -> ParsedRateCard:
 
         try:
             parsed.pscr_cents = await self.hass.async_add_executor_job(
-                parse_pscr_cents_from_xlsx,
+                parse_pscr_cents_from_pdf,
                 pscr_bytes,
             )
-            parsed.pscr_source_url = RIDER18_CALCULATOR_URL
+            parsed.pscr_source_url = PSCR_RATE_BOOK_URL
         except Exception as err:
             _LOGGER.warning(
-                "Failed parsing DTE Rider 18 PSCR; export rates will omit PSCR: %s",
+                "Failed parsing MPSC DTE rate book PSCR; export rates will omit PSCR: %s",
                 err,
             )
 

custom_components/dte_rates/pscr_parser.py

@@ -1,124 +1,26 @@
 from __future__ import annotations
 
-from decimal import Decimal, InvalidOperation
-from io import BytesIO
-import posixpath
+from decimal import Decimal
+import io
 import re
-from typing import Any
-from xml.etree import ElementTree as ET
-from zipfile import ZipFile
 
+from pypdf import PdfReader
 
-_CELL_RE = re.compile(r"([A-Z]+)(\d+)")
 
+_D111_PSCR_RE = re.compile(
+    r"\bD1\.11\s+Standard\s+TOU\s+(?P<pscr>\d+\.\d+)",
+    re.IGNORECASE,
+)
 
-def parse_pscr_cents_from_xlsx(xlsx_bytes: bytes) -> Decimal:
-    """Extract the Rider 18 PSCR value as cents/kWh."""
-    with ZipFile(BytesIO(xlsx_bytes)) as workbook:
-        shared_strings = _shared_strings(workbook)
-        sheet_path = _rates_sheet_path(workbook)
-        rows = _worksheet_rows(workbook, sheet_path, shared_strings)
 
-    for row_number in sorted(rows):
-        row = rows[row_number]
-        label = " ".join(row.values()).lower()
-        if "pscr" not in label:
-            continue
+def parse_pscr_cents_from_pdf(pdf_bytes: bytes) -> Decimal:
+    """Extract the current PSCR value from the MPSC DTE electric rate book."""
+    reader = PdfReader(io.BytesIO(pdf_bytes))
+    text = "\n".join(page.extract_text() or "" for page in reader.pages)
+    normalized = re.sub(r"\s+", " ", text)
 
-        for column in ("C", "D", "B", "E"):
-            cents = _dollars_to_cents(row.get(column))
-            if cents is not None:
-                return cents
+    match = _D111_PSCR_RE.search(normalized)
+    if match:
+        return Decimal(match.group("pscr"))
 
-    raise ValueError("Rider 18 workbook does not contain a PSCR value")
-
-
-def _shared_strings(workbook: ZipFile) -> list[str]:
-    try:
-        root = ET.fromstring(workbook.read("xl/sharedStrings.xml"))
-    except KeyError:
-        return []
-
-    return ["".join(text.text or "" for text in item.findall(".//{*}t")) for item in root.findall("{*}si")]
-
-
-def _rates_sheet_path(workbook: ZipFile) -> str:
-    workbook_root = ET.fromstring(workbook.read("xl/workbook.xml"))
-    rels_root = ET.fromstring(workbook.read("xl/_rels/workbook.xml.rels"))
-    rel_targets = {
-        rel.attrib["Id"]: _normalize_target(rel.attrib.get("Target", ""))
-        for rel in rels_root.findall("{*}Relationship")
-        if "Id" in rel.attrib
-    }
-
-    fallback: str | None = None
-    for sheet in workbook_root.findall(".//{*}sheet"):
-        name = sheet.attrib.get("name", "")
-        rel_id = sheet.attrib.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id")
-        target = rel_targets.get(rel_id or "")
-        if target is None:
-            continue
-        if name.strip().lower() == "rates and credits":
-            return target
-        if fallback is None and "rate" in name.lower():
-            fallback = target
-
-    if fallback is None:
-        raise ValueError("Rider 18 workbook does not contain a rates sheet")
-    return fallback
-
-
-def _normalize_target(target: str) -> str:
-    if target.startswith("/"):
-        return target.lstrip("/")
-    return posixpath.normpath(posixpath.join("xl", target))
-
-
-def _worksheet_rows(
-    workbook: ZipFile,
-    sheet_path: str,
-    shared_strings: list[str],
-) -> dict[int, dict[str, str]]:
-    root = ET.fromstring(workbook.read(sheet_path))
-    rows: dict[int, dict[str, str]] = {}
-
-    for cell in root.findall(".//{*}c"):
-        ref = cell.attrib.get("r", "")
-        match = _CELL_RE.match(ref)
-        if not match:
-            continue
-
-        column, row_number_raw = match.groups()
-        value = _cell_value(cell, shared_strings)
-        if value is not None:
-            rows.setdefault(int(row_number_raw), {})[column] = " ".join(value.split())
-
-    return rows
-
-
-def _cell_value(cell: ET.Element, shared_strings: list[str]) -> str | None:
-    cell_type = cell.attrib.get("t")
-    if cell_type == "inlineStr":
-        return "".join(text.text or "" for text in cell.findall(".//{*}t"))
-
-    value = cell.find("{*}v")
-    if value is None or value.text is None:
-        return None
-
-    if cell_type == "s":
-        try:
-            return shared_strings[int(value.text)]
-        except (IndexError, ValueError):
-            return None
-    return value.text
-
-
-def _dollars_to_cents(value: Any) -> Decimal | None:
-    text = str(value or "").replace("$", "").replace(",", "").strip()
-    if not text:
-        return None
-
-    try:
-        return (abs(Decimal(text)) * Decimal("100")).quantize(Decimal("0.001"))
-    except InvalidOperation:
-        return None
+    raise ValueError("MPSC DTE rate book does not contain a D1.11 PSCR value")

docs/research/rider18-export-rates.md

@@ -4,11 +4,11 @@
 
 - Enhancement request: calculate Rider 18 export credits to match DTE's Rider 18 calculator.
 - Residential rate card: source of the active generation components for each rate period.
-- Rider 18 calculator workbook: source of the current PSCR scalar.
+- MPSC DTE electric rate book: source of the current PSCR scalar.
 
 ## Implementation Decision
 
-Do not use the Rider 18 calculator workbook's period-specific outflow credits as the authoritative rates. The workbook is used only to extract the current PSCR value, because the residential rate card PDF does not include that scalar.
+Do not use the Rider 18 calculator workbook's period-specific outflow credits as the authoritative rates. PSCR is parsed dynamically from the MPSC DTE electric rate book PDF at `https://www.michigan.gov/-/media/Project/Websites/mpsc/consumer/rate-books/electric/dte/dtee1cur.pdf`, because the residential rate card PDF does not include that scalar.
 
 Formula:
 

tests/test_pscr_parser.py

@@ -1,56 +1,29 @@
 from __future__ import annotations
 
 from decimal import Decimal
-from html import escape
-from io import BytesIO
-from zipfile import ZipFile
-
-from custom_components.dte_rates.pscr_parser import parse_pscr_cents_from_xlsx
-
-
-def _xlsx_with_pscr() -> bytes:
-    cells = {
-        "B2": "FULL SERVICE - Rider 18 Credits",
-        "B4": "PSCR (November 1, 2024)",
-        "C4": "0.01877",
-    }
-    rows: dict[int, list[str]] = {}
-    for ref, value in cells.items():
-        row = int("".join(ch for ch in ref if ch.isdigit()))
-        if value.replace(".", "", 1).isdigit():
-            cell = f'<c r="{ref}"><v>{value}</v></c>'
-        else:
-            cell = f'<c r="{ref}" t="inlineStr"><is><t>{escape(value)}</t></is></c>'
-        rows.setdefault(row, []).append(cell)
-
-    sheet_data = "".join(f'<row r="{row}">{"".join(cells)}</row>' for row, cells in sorted(rows.items()))
-    workbook_xml = (
-        '<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" '
-        'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">'
-        "<sheets>"
-        '<sheet name="Rates and Credits" sheetId="1" r:id="rId1"/>'
-        "</sheets>"
-        "</workbook>"
-    )
-    rels_xml = (
-        '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
-        '<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" '
-        'Target="worksheets/sheet1.xml"/>'
-        "</Relationships>"
-    )
-    sheet_xml = (
-        '<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">'
-        f"<sheetData>{sheet_data}</sheetData>"
-        "</worksheet>"
-    )
-
-    buffer = BytesIO()
-    with ZipFile(buffer, "w") as workbook:
-        workbook.writestr("xl/workbook.xml", workbook_xml)
-        workbook.writestr("xl/_rels/workbook.xml.rels", rels_xml)
-        workbook.writestr("xl/worksheets/sheet1.xml", sheet_xml)
-    return buffer.getvalue()
-
-
-def test_parse_pscr_cents_from_xlsx():
-    assert parse_pscr_cents_from_xlsx(_xlsx_with_pscr()) == Decimal("1.877")
+
+from custom_components.dte_rates.pscr_parser import parse_pscr_cents_from_pdf
+
+
+SAMPLE_RATE_BOOK_TEXT = """
+C8.5 SURCHARGES AND CREDITS APPLICABLE TO POWER SUPPLY SERVICE
+Rate Schedule Description PSCR Factor Other Charges
+D1.11 Standard TOU 1.877 0.0221
+D1.13 Overnight Savers 1.877 0.0221
+"""
+
+
+class _FakePage:
+    def extract_text(self):
+        return SAMPLE_RATE_BOOK_TEXT
+
+
+class _FakeReader:
+    def __init__(self, _bytes):
+        self.pages = [_FakePage()]
+
+
+def test_parse_pscr_cents_from_mpsc_rate_book_pdf(monkeypatch):
+    monkeypatch.setattr("custom_components.dte_rates.pscr_parser.PdfReader", _FakeReader)
+
+    assert parse_pscr_cents_from_pdf(b"fake") == Decimal("1.877")

tests/test_sensor.py

@@ -43,7 +43,7 @@ def _coordinator_with_rate() -> SimpleNamespace:
             rates={"D1.11": rate},
             raw_text_hash="hash",
             pscr_cents=Decimal("1.877"),
-            pscr_source_url="https://example.test/Rider18Calculator.xlsx",
+            pscr_source_url="https://www.michigan.gov/-/media/Project/Websites/mpsc/consumer/rate-books/electric/dte/dtee1cur.pdf",
         )
     )