"""tabulardata.py — numerical companion to the glossary entry
'tabulardata' (tabular data).

The demo gathers actual weather measurements from the open data server
of the Finnish Meteorological Institute (FMI): daily minimum
temperature, precipitation, and maximum temperature at the Helsinki
Kaisaniemi station (fmisid 100971) for the year 2024.  The
measurements arrive as one value per (day, attribute) pair and are
stored as a table: one row per day, one column per attribute.

The blocks verify the entry's claims: every row has one cell per
column and the cells of one column hold values of the same attribute;
the value range of a cell can include a special value (FMI reports the
precipitation of a dry day as -1.0); and which attributes serve as the
feature and which as the label is a design choice — here the minimum
temperature column is read as the feature and the maximum temperature
column as the label of a learning task.

Deterministic: historical measurements, no randomness.
Self-contained: numpy + matplotlib only (stdlib urllib for the
download).

Blocks
------
[B-fetch]   Download the 366 daily rows for 2024 from the FMI open
            data server and write them to tabulardata_weather.csv;
            check the count, one pinned measurement, and that the
            table has no empty cell — dry days carry the special
            precipitation value -1.0 instead.
[B-table]   Select four consecutive days that contain both genuine
            precipitation values and the special value -1.0; this
            window is the table shown in the entry's figure
            (tabulardata_table.csv).
[B-picture] Read the minimum-temperature column as the feature and
            the maximum-temperature column as the label: scatterplot
            of the 366 days and a fitted curve (degree-3 least
            squares), written to tabulardata_scatter.csv and
            tabulardata_curve.csv for the entry's figure.

Outputs
-------
tabulardata_weather.csv : day, tmin, rrday, tmax for the 366 days
tabulardata_table.csv   : the four rows shown in the entry's figure
tabulardata_scatter.csv : tmin, tmax of the 366 days
tabulardata_curve.csv   : tmin, pred -- the fitted curve on a grid
tabulardata.png         : preview (checking only) -- the scatterplot
                          with the curve, and the four-day table
"""

import re
import urllib.request
from pathlib import Path

import numpy as np
import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt

OUT_DIR = Path(__file__).parent

report = []


def check(name, ok):
    report.append((name, bool(ok)))
    print(f"  [{'ok' if ok else 'FAIL'}] {name}")


# ---- [B-fetch] daily measurements from the FMI open data server
URL = ("https://opendata.fmi.fi/wfs?service=WFS&version=2.0.0"
       "&request=getFeature"
       "&storedquery_id=fmi::observations::weather::daily::simple"
       "&fmisid=100971&starttime=2024-01-01T00:00:00Z"
       "&endtime=2024-12-31T00:00:00Z&parameters=tmin,tmax,rrday")
with urllib.request.urlopen(URL, timeout=120) as resp:
    xml = resp.read().decode()
triples = re.findall(
    r"<BsWfs:Time>(\S+?)T.*?</BsWfs:Time>\s*"
    r"<BsWfs:ParameterName>(\w+)</BsWfs:ParameterName>\s*"
    r"<BsWfs:ParameterValue>(\S+)</BsWfs:ParameterValue>", xml, re.S)
rows = {}
for day, name, value in triples:
    rows.setdefault(day, {})[name] = float(value)
days = sorted(rows)
with open(OUT_DIR / "tabulardata_weather.csv", "w") as f:
    f.write("day,tmin,rrday,tmax\n")
    for d in days:
        r = rows[d]
        f.write(f"{d},{r['tmin']:.1f},{r['rrday']:.1f},{r['tmax']:.1f}\n")
check("[B-fetch] 366 daily rows downloaded for 2024", len(days) == 366)
check("[B-fetch] the record matches the archive "
      "(May 2: 4.5 to 14.5 degrees)",
      rows["2024-05-02"]["tmin"] == 4.5
      and rows["2024-05-02"]["tmax"] == 14.5)
check("[B-fetch] every row has one cell per column (no empty cells)",
      all(len(rows[d]) == 3 for d in days))
n_special = sum(rows[d]["rrday"] == -1.0 for d in days)
print(f"    special precipitation value -1.0 (dry day) on "
      f"{n_special} of {len(days)} days")
check("[B-fetch] the value range of the precipitation column includes "
      "the special value -1.0 for a dry day",
      0 < n_special < len(days))

# ---- [B-table] four consecutive days for the entry's figure
window = None
for k in range(len(days) - 3):
    rr = [rows[d]["rrday"] for d in days[k:k + 4]]
    if sum(v > 0 for v in rr) >= 2 and sum(v == -1.0 for v in rr) >= 1:
        window = days[k:k + 4]
        break
print(f"    four-day window for the figure: {window[0]} .. {window[-1]}")
with open(OUT_DIR / "tabulardata_table.csv", "w") as f:
    f.write("day,tmin,rrday,tmax\n")
    for d in window:
        r = rows[d]
        f.write(f"{d},{r['tmin']:.1f},{r['rrday']:.1f},{r['tmax']:.1f}\n")
check("[B-table] the window mixes genuine precipitation values with "
      "the special value",
      any(rows[d]["rrday"] > 0 for d in window)
      and any(rows[d]["rrday"] == -1.0 for d in window))
check("[B-table] all four rows share the same fixed set of attributes",
      all(sorted(rows[d]) == ["rrday", "tmax", "tmin"] for d in window))

# ---- [B-picture] feature and label are a design choice
tmin = np.array([rows[d]["tmin"] for d in days])
tmax = np.array([rows[d]["tmax"] for d in days])
np.savetxt(OUT_DIR / "tabulardata_scatter.csv",
           np.stack([tmin, tmax], 1),
           delimiter=",", header="tmin,tmax", comments="", fmt="%.1f")
coef = np.polyfit(tmin, tmax, 3)
grid = np.linspace(tmin.min(), tmin.max(), 200)
np.savetxt(OUT_DIR / "tabulardata_curve.csv",
           np.stack([grid, np.polyval(coef, grid)], 1),
           delimiter=",", header="tmin,pred", comments="", fmt="%.2f")
err_fit = float(np.mean((tmax - np.polyval(coef, tmin)) ** 2))
err_const = float(np.var(tmax))
print(f"    average squared error: fitted curve {err_fit:.1f}, "
      f"constant {err_const:.1f}")
check("[B-picture] the fitted curve predicts the label far better "
      "than a constant", err_fit < err_const / 3)
check("[B-picture] warmer mornings go with warmer days "
      "(positive correlation)",
      float(np.corrcoef(tmin, tmax)[0, 1]) > 0.8)

# ---- preview
fig, axes = plt.subplots(1, 2, figsize=(9.6, 3.6),
                         gridspec_kw={"width_ratios": [3, 2]})
ax = axes[0]
ax.plot(tmin, tmax, "o", ms=2.5, color="0.6", label="days of 2024")
ax.plot(grid, np.polyval(coef, grid), "-", lw=2, color="black",
        label="fitted curve")
ax.set_xlabel("minimum temperature of the day (°C)")
ax.set_ylabel("maximum temperature of the day (°C)")
ax.set_title("feature: min. temp., label: max. temp. (Kaisaniemi)")
ax.legend(frameon=False, fontsize=8)
ax = axes[1]
ax.axis("off")
cells = [[d, f"{rows[d]['tmin']:.1f}", f"{rows[d]['rrday']:.1f}",
          f"{rows[d]['tmax']:.1f}"] for d in window]
tab = ax.table(cellText=cells,
               colLabels=["day", "min. temp.", "precip.", "max. temp."],
               loc="center")
tab.auto_set_font_size(False)
tab.set_fontsize(8)
ax.set_title("four rows of the table (-1.0 = dry day)")
fig.tight_layout()
fig.savefig(OUT_DIR / "tabulardata.png", dpi=150)

failed = [name for name, ok in report if not ok]
print(f"{len(report) - len(failed)}/{len(report)} checks passed"
      + (f", FAILED: {failed}" if failed else ""))