gen_c7dreq_htmls.py

import json
import os
import re
import shutil
import sys
from collections import defaultdict
from html import escape, unescape
from pathlib import Path

from data_request_api.content import dreq_content as dc
from data_request_api.content import dump_transformation as dt
from data_request_api.query import data_request as dr
from data_request_api.query import dreq_query as dq

# === Select version & load data request ===
version = "v1.0"
version = "v1.1"
version = "v1.2"
version = "v1.2.1"
version = "v1.2.2"
version = "v1.2.2.1"
version = "v1.2.2.2"
version = "v1.2.2.3"
version = "v1.2.2.4"


if len(sys.argv) > 1:
    version = sys.argv[1]


prev_version = {
    "v1.0": "",
    "v1.1": "v1.0",
    "v1.2": "v1.1",
    "v1.2.1": "v1.2",
    "v1.2.2": "v1.2.1",
    "v1.2.2.1": "v1.2.2",
    "v1.2.2.2": "v1.2.2.1",
    "v1.2.2.3": "v1.2.2.2",
    "v1.2.2.4": "v1.2.2.3",
}
prev_version = prev_version[version]


full_content = dc.load(version=version, export="release", consolidate=True)
content_dic = dt.get_transformed_content(version=version)
DR = dr.DataRequest.from_separated_inputs(**content_dic)

INPUT_FILES = [
    dc._dreq_res + "/" + version + "/VS_release_consolidate_content.json",
    dc._dreq_res + "/" + version + "/DR_release_consolidate_content.json",
]

with open(INPUT_FILES[0], encoding="utf-8") as f:
    version_json = json.load(f)
if version != version_json.get("version"):
    raise Exception(f"Version mismatch: {version} != {version_json.get('version')}")

# Output directory with versioning
BASE_OUTPUT = "docs"
OUTPUT_DIR = os.path.join(BASE_OUTPUT, version)


def path_in_version(subpath):
    return f"/{BASE_OUTPUT}/{version}/{subpath}"


def sanitize(value):
    """Replace illegal characters in URLs or filenames."""
    value = re.sub(r"[^A-Za-z0-9_-]", "_", value)
    value = re.sub(r"_+", "_", value)
    return value.strip("_")


# === STEP 1: LOAD & MERGE DATA ===
issued_warnings = []
main_data = {}
extra_data = {}
uid_map = {}
cat_link_to_uid = {}
cat_map = {
    "lead_theme": "data_request_themes",
    "cf_standard_name": "cf_standard_names",
    "cmip6_tables_identifier": "cmip6_tables_identifiers",
    "physical_parameter": "physical_parameters",
    "references": "docs_for_opportunities",
    "dimensions": "coordinates_and_dimensions",
    "coordinates": "coordinates_and_dimensions",
    "primary_modelling_realm": "modelling_realm",
    "extra_dimensions": "coordinates_and_dimensions",
}

# Load main data (VS data)
with open(INPUT_FILES[0], encoding="utf-8") as f:
    main = json.load(f)
    for category, records in main.items():
        if category == "version":
            continue
        elif category == "glossary":
            continue
        main_data[category] = records
        for record_id, record in records.items():
            # Use record_id for mapping:
            # uid = record_id
            # record["uid"] = uid
            # Use UID for mapping:
            uid = record["uid"]
            if uid != sanitize(uid):
                uid = sanitize(uid)
                print(
                    f"ERROR: UID had to be sanitized - before: '{record['uid']}' after: '{uid}'"
                )
                record["uid"] = uid
            # Remove image as it is nested dict and not required
            if "image" in record:
                del record["image"]
            # Hide cmip7_compound_name if generated by data_transformation
            if "cmip7_compound_name" in record:
                if "default" in record["cmip7_compound_name"]:
                    del record["cmip7_compound_name"]
            # Default "name" is "cmip7_compound_name"
            # -> Use "cmip6_compound_name" as "name" if "cmip7_compound_name" is not defined
            #    and "name" thus is "undef"
            if "cmip6_compound_name" in record:
                if "name" in record and record["name"] == "undef":
                    record["name"] = record["cmip6_compound_name"]
            if uid:
                uid_map[uid] = {
                    "category": category,
                    "record_id": record_id,
                    "main": record.copy(),
                    "extra": {},
                }
            if category in cat_link_to_uid:
                cat_link_to_uid[category][record_id] = uid
            else:
                cat_link_to_uid[category] = {record_id: uid}
# Dump UID map / category-link to UID map
# with open("dump.json", "w") as f:
#    json.dump(uid_map, f, indent=2)
#    json.dump(cat_link_to_uid, f, indent=2)

# Load extra data (DR data)
with open(INPUT_FILES[1], encoding="utf-8") as f:
    extra = json.load(f)
    for category, records in extra.items():
        if category == "version":
            continue
        for record_id, extra_attrs in records.items():
            # Use record_id for mapping:
            # uid = record_id
            # extra_attrs["uid"] = uid
            # Use UID for mapping:
            uid = cat_link_to_uid[cat_map.get(category, category)][record_id]
            extra_attrs["uid"] = uid
            if uid and uid in uid_map:
                uid_map[uid]["extra"] = extra_attrs.copy()

# Build flat UID to record map for link resolution
uid_record_lookup = {}
for uid, obj in uid_map.items():
    full_record = obj["main"].copy()
    full_record.update(obj["extra"])
    uid_record_lookup[uid] = full_record
# with open("uid_record_lookup.json", "w") as f:
#    json.dump(uid_record_lookup, f, indent=2)

# Build combined dict (for reverse lookup table)
version = main.get("version") or extra.get("version")
main.pop("version", None)
extra.pop("version", None)
data_all = {}

# Merge main data
for category, records in main.items():
    data_all[category] = {
        sanitize(r["uid"]): uid_map[sanitize(r["uid"])]["main"]
        for r in records.values()
    }

# Merge in extras (adds or extends attributes)
for category, extra_records in extra.items():
    if category not in data_all:
        continue  # Only enrich existing categories
    for rid, extra_attrs in extra_records.items():
        uid = cat_link_to_uid[cat_map.get(category, category)][rid]
        if uid in data_all[category]:
            data_all[category][uid].update(uid_map[uid]["extra"])
        else:
            print("ERROR: UID", uid, "not found in VS/main data for category", category)

with open("data_all.json", "w") as f:
    json.dump(data_all, f, indent=2)

# Extract description strings - category and fields per category
#   Adapted dump_transformation functions for consistent
#    renaming of categories and fields
category_desc = {}
field_desc = {}
transform_settings = dt.get_transform_settings(version=version)["one_to_transform"]


def transform_category_name(category, settings):
    for patt, repl in settings["tables_to_rename"].items():
        if re.compile(patt).match(category) is not None:
            category = re.sub(patt, repl, category)
    return category


def transform_field_name(field, category, settings):
    patterns_to_rename = settings["keys_to_rename"]
    patterns_to_merge = settings["keys_to_merge"]
    for patt, repl in patterns_to_rename.get(category, {}).items():
        patt = re.compile(patt)
        if patt.match(field) is not None:
            field = repl
    for patt, repl in patterns_to_merge.get(category, {}).items():
        patt = re.compile(patt)
        if patt.match(field) is not None:
            field = repl
    return field


for category, category_content in full_content["Data Request"].items():
    if category == "version":
        continue
    formatted_category = transform_category_name(
        dt.correct_key_string(category), transform_settings
    )
    category_desc[formatted_category] = category_content.get("description", "").strip()
    if category_desc[formatted_category] == "None":
        category_desc[formatted_category] = ""
    field_desc[formatted_category] = {}
    for field, field_content in category_content["fields"].items():
        formatted_field = transform_field_name(
            dt.correct_key_string(field_content["name"]),
            formatted_category,
            transform_settings,
        )
        if field_desc[formatted_category].get(formatted_field, "") != "":
            desc_tmp = str(field_content.get("description", "")).strip()
            if desc_tmp and desc_tmp != "None":
                field_desc[formatted_category][formatted_field] += " " + desc_tmp
        else:
            desc_tmp = str(field_content.get("description", "")).strip()
            if desc_tmp and desc_tmp != "None":
                field_desc[formatted_category][formatted_field] = desc_tmp
            else:
                field_desc[formatted_category][formatted_field] = ""

        field_desc[formatted_category][formatted_field] = field_desc[
            formatted_category
        ][formatted_field].strip()

for i in category_desc:
    print(i, category_desc[i])

for i in field_desc:
    for j in field_desc[i]:
        if field_desc[i][j]:
            print(i, j, field_desc[i][j])

for cat in data_all:
    if not cat in category_desc:
        errmsg = f"ERROR: '{cat}' missing in category descriptions"
        if not errmsg in issued_warnings:
            print(errmsg)
            issued_warnings.append(errmsg)
for cat in data_all:
    for rec in data_all[cat]:
        for f in data_all[cat][rec]:
            if not cat in field_desc:
                errmsg = f"ERROR: '{cat}' missing in field descriptions"
                if not errmsg in issued_warnings:
                    print(errmsg)
                    issued_warnings.append(errmsg)
            elif not f in field_desc[cat]:
                errmsg = f"ERROR: Missing field description for {cat} {f}"
                if not errmsg in issued_warnings:
                    print(errmsg)
                    issued_warnings.append(errmsg)

# === Create output dir ===
Path(f"{OUTPUT_DIR}/u").mkdir(parents=True, exist_ok=True)


# === Helper functions ===
def write_file(path, content, title="", style_location="../"):
    with open(os.path.join(OUTPUT_DIR, path), "w", encoding="utf-8") as f:
        f.write("<!DOCTYPE html>")
        f.write("<html>")
        f.write("<head>")
        f.write('<meta charset="UTF-8">')
        f.write(
            '<meta name="viewport" content="width=device-width, initial-scale=1.0">'
        )
        if title:
            f.write(f"<title>{title}</title>")
        f.write(f'<link rel="stylesheet" href="{style_location}style_c7dreq.css">')
        f.write(f'<link rel="icon" type="image/x-icon" href="{style_location}ress/favico.ico">')
        f.write("</head>")
        f.write("<body>")
        f.write(content)
        f.write("</body>")
        # f.write('<footer id="footer">')
        # f.write('<div class="footer-content">')
        # f.write('<a href="https://www.dkrz.de/about-en/contact/impressum">About</a>')
        # f.write('<a href="https://www.dkrz.de/about-en/contact/en-datenschutzhinweise">DKRZ Privacy Policy</a>')
        # f.write('</div></footer>')
        f.write("</html>")


def link_label(uid, uid_lookup):
    record = uid_lookup.get(uid, {})
    compound = record.get("cmip7_compound_name")
    name = record.get("cmip6_compound_name", record.get("name"))
    title = record.get("title")

    if compound and name and not "default" in compound:
        label = f"{compound} | {name}"
    elif name:
        label = name
    elif compound:
        label = compound
    else:
        label = uid
    return label, title


def make_link(value, uid_lookup, cat, current_dir=""):
    def format_link(uid, single=False):
        label, title = link_label(uid, uid_lookup)
        label = escape(label)

        if current_dir == "index":
            href = f"u/{sanitize(uid)}.html"
        elif current_dir == "category":
            href = f"u/{sanitize(uid)}.html"
        elif current_dir == "record":
            href = f"{sanitize(uid)}.html"
        else:
            href = f"{sanitize(uid)}.html"

        if single and title:
            return f'<span><a href="{href}">{label}</a> ({escape(str(title))})</span>'
        else:
            return f'<span><a href="{href}">{label}</a></span>'

    if isinstance(value, str) and value.startswith("link::"):
        uid = cat_link_to_uid[cat_map.get(cat, cat)][value.split("::")[1]]
        return format_link(uid, single=True)

    elif isinstance(value, str) and (
        value.startswith("http://") or value.startswith("https://")
    ):
        return f'<a href="{value}" target="_blank">{escape(value)}</a>'

    elif isinstance(value, str) and "https://" in value:
        value = linkify_text(value)
        return value

    elif isinstance(value, list):
        links = []
        for v in value:
            if isinstance(v, str) and v.startswith("link::"):
                uid = cat_link_to_uid[cat_map.get(cat, cat)][v.split("::")[1]]
                links.append(format_link(uid))
            elif isinstance(v, str) and (
                v.startswith("http://") or v.startswith("https://")
            ):
                links.append(f'<a href="{v}" target="_blank">{escape(v)}</a>')
            else:
                links.append(escape(str(v)))

        if len(links) <= 10:
            return "<span>" + ", ".join(links) + "</span>"
        else:
            visible = ", ".join(links[:5])
            hidden = ", ".join(links)
            return f"""<details><summary><span class="short-view">{visible} ... and {len(links) - 5} more</span><span class="full-view">{hidden}</span></summary></details>"""

    else:
        return escape(str(value))


def build_uid_to_category_map(data_all):
    uid_to_category = {}
    for category, records in data_all.items():
        if not isinstance(records, dict):
            continue  # skip version or other non-record categories
        for uid in records:
            uid_to_category[uid] = category
    return uid_to_category


def build_reverse_links_map(data_all):
    reverse_links = {}

    for category, records in data_all.items():
        if not isinstance(records, dict):  # skip 'version' category
            continue
        for source_uid, record in records.items():
            for attr, attr_val in record.items():
                links = []

                if isinstance(attr_val, str) and attr_val.startswith("link::"):
                    links = [attr_val]
                elif isinstance(attr_val, list):
                    links = [
                        v
                        for v in attr_val
                        if isinstance(v, str) and v.startswith("link::")
                    ]

                for link in links:
                    target_uid = link.replace("link::", "")
                    target_uid = cat_link_to_uid[cat_map.get(attr, attr)][target_uid]
                    reverse_links.setdefault(target_uid, {}).setdefault(
                        category, []
                    ).append(source_uid)

    return reverse_links


def strip_html_tags(html):
    """Extracts visible text from HTML (e.g., link label)."""
    return unescape(re.sub(r"<.*?>", "", html))


def linkify_text(text):
    # Matches URLs with or without surrounding <>
    url_pattern = re.compile(r"<(https?://[^\s<>]+)>|(https?://[^\s,)\]>]+)")

    def repl(match):
        url = match.group(1) or match.group(2)
        escaped_url = escape(url)
        return f'<a href="{escaped_url}" target="_blank">{escaped_url}</a>'

    return url_pattern.sub(repl, text)


def render_reverse_links_section(uid, reverse_links, data_all, uid_to_category):
    if uid not in reverse_links:
        return ""  # No backlinks to this uid

    html = ["<h2>Links from Other Categories</h2>"]

    for category, sources in reverse_links[uid].items():
        html.append(f"<details><summary><strong>{category}:</strong></summary><ul>")
        link_html = list()
        for source_uid in sources:
            source_cat = uid_to_category.get(source_uid)
            if not source_cat:
                print("ERROR: No category for", source_uid)
                continue
            source_record = data_all.get(source_cat, {}).get(source_uid)
            if not source_record:
                print("ERROR: No record for UID", source_uid, "in category", source_cat)
                continue

            # Build label like before
            label_parts = []
            if "cmip7_compound_name" in source_record:
                label_parts.append(source_record["cmip7_compound_name"])
            if "cmip6_compound_name" in source_record:
                label_parts.append(source_record["cmip6_compound_name"])
            elif "name" in source_record:
                label_parts.append(source_record["name"])
            label = " / ".join(label_parts)
            if "title" in source_record and label_parts:
                label += f" ({source_record['title']})"

            link_html.append(
                f'<li><a href="{sanitize(source_uid)}.html">{label or source_uid}</a></li>'
            )

        link_html.sort(key=lambda x: strip_html_tags(x))
        html.extend(link_html)
        html.append("</ul></details><br><br>")

    return "".join(html)


# Reverse links HTML (after extra data section)
uid_to_category = build_uid_to_category_map(data_all)
reverse_links = build_reverse_links_map(data_all)

# === index.html ===
index_html = f"<h1>DReq {version} All Categories</h1>"
index_html += f"<p><a href='../index.html'>Back to Version Index</a></p><ul>"
index_html += f"<script src='../check_deprecated.js' data-dreq-version=\"{version}\" data-dreq-latest=\"../dreq_latest_version.txt\"></script>"
for category in sorted(main_data.keys()):
    if category_desc.get(category):
        index_html += f'<li><a title="{escape(category_desc.get(category))}" href="{category}.html">{escape(category)}</a></li>'
    else:
        index_html += f'<li><a href="{category}.html">{escape(category)}</a></li>'
index_html += f"</ul><p><a href='../index.html'>Back to Version Index</a></p>"
write_file(
    "index.html", index_html, title=f"DReq {version} Index", style_location="../"
)

# === Category Pages ===
for category, records in main_data.items():
    html = f"<h1>Category: {escape(category)} ({version})</h1>"
    html += f"<p><a href='index.html'>Back to Category Index</a></p>"
    html += f"<script src='../check_deprecated.js' data-dreq-version=\"{version}\" data-dreq-latest=\"../dreq_latest_version.txt\"></script><ul>"
    if category_desc.get(category) not in ["", None]:
        print(f"category {category}: '{category_desc.get(category)}'")
        html += f"<p><strong>Category Description:</strong> {escape(category_desc.get(category))}</p>"
    link_html = list()
    for record_id, record in records.items():
        uid = record.get("uid")
        if uid:
            label, title = link_label(uid, uid_record_lookup)
            if title:
                link_html.append(
                    f'<li><a href="u/{sanitize(uid)}.html">{escape(label)}</a> ({escape(title)})</li>'
                )
            else:
                link_html.append(
                    f'<li><a href="u/{sanitize(uid)}.html">{escape(label)}</a></li>'
                )
    link_html.sort(key=lambda x: strip_html_tags(x))
    html += "".join(link_html)
    html += f"</ul><p><a href='index.html'>Back to Category Index</a></p>"
    write_file(f"{category}.html", html, title=f"{version} {category} Index")

# === Record Pages ===
for uid, obj in uid_map.items():
    category = obj["category"]
    record_id = obj["record_id"]
    main_record = obj["main"]
    extra_record = obj["extra"]

    if "default" in record_id:
        html = f"<h1>{category} record: {escape(uid)} ({version})</h1>"
    else:
        html = f"<h1>{category} record: {escape(record_id)} ({version})</h1>"
    html += f"<p><a href='../{category}.html'>Back to {escape(category)}</a> | <a href='../index.html'>Category Index</a></p>"
    html += f"<script src='../../check_deprecated.js' data-dreq-version=\"{version}\" data-dreq-latest=\"../../dreq_latest_version.txt\"></script>"
    if category_desc.get(category) not in ["", None]:
        html += f"<br><details><summary><strong>Category Description</strong></summary><p>{escape(category_desc.get(category))}</p></details><br><br>"
    else:
        errmsg = f"ERROR: '{category}' has no description"
        if not errmsg in issued_warnings:
            print(errmsg)
            issued_warnings.append(errmsg)
    if category == "variables":
        html += f"<p>Link to <a href='../../variable_changes.html?uid={sanitize(uid)}' target='_blank'>CMIP7-AFT Data Request - Variable Changes</a></p>"
    html += "<table><tr><th>Attribute</th><th>Value</th></tr>"
    for attr, value in main_record.items():
        if extra_record:
            if attr in extra_record and attr != "uid":
                continue
        formatted_value = make_link(
            value, uid_record_lookup, cat=attr, current_dir="record"
        )
        if category not in field_desc:
            errmsg = f"ERROR: category '{category}' not in field description"
            if not errmsg in issued_warnings:
                print(errmsg)
                issued_warnings.append(errmsg)
        elif field_desc[category].get(attr) not in ["", None]:
            html += f"<tr><td class='field_name' title='{escape(field_desc[category].get(attr))}'><strong>{escape(attr)}</strong></td><td>{formatted_value}</td></tr>"
        else:
            wmsg = f"WARNING: field description of category '{category}' lacks attribute '{attr}'"
            if not wmsg in issued_warnings:
                print(wmsg)
                issued_warnings.append(wmsg)
            html += f"<tr><td class='field_name'><strong>{escape(attr)}</strong></td><td>{formatted_value}</td></tr>"
    html += "</table>"
    if not "uid" in main_record:
        errmsg = f"ERROR: '{uid}' has no uid"
        if not errmsg in issued_warnings:
            print(errmsg)
            issued_warnings.append(errmsg)

    if extra_record:
        html += "<h2>Data Request Information</h2><table>"
        for attr, value in extra_record.items():
            if attr == "uid":
                continue
            formatted_value = make_link(
                value, uid_record_lookup, cat=attr, current_dir="record"
            )
            if field_desc[category].get(attr, "") != "":
                html += f"<tr><td class='field_name' title='{escape(field_desc[category].get(attr))}'><strong>{escape(attr)}</strong></td><td>{formatted_value}</td></tr>"
            else:
                html += f"<tr><td class='field_name'><strong>{escape(attr)}</strong></td><td>{formatted_value}</td></tr>"
        html += "</table>"

    reverse_html = render_reverse_links_section(
        uid, reverse_links, data_all, uid_to_category
    )
    html += reverse_html

    html += f"<p><a href='../{category}.html'>Back to {escape(category)}</a> | <a href='../index.html'>Category Index</a></p>"
    write_file(
        f"u/{sanitize(uid)}.html",
        html,
        title=(
            f"{category} record: {escape(record_id)} ({version})"
            if "default" not in record_id
            else f"{category} record: {escape(uid)} ({version})"
        ),
        style_location="../../",
    )

print()
print(f"HTML files generated in '{OUTPUT_DIR}/'")


# === Variable Search Index ===
vi_file = f"{OUTPUT_DIR}/../variable_index.json"
if os.path.exists(vi_file):
    # Backup
    shutil.copy(vi_file, "variable_index.json")
    var_index_old = json.load(open(vi_file))
else:
    var_index_old = {}
var_index = dict()

for var in sorted(
    list(data_all["variables"].keys()),
    key=lambda x: data_all["variables"][x].get("cmip7_compound_name", x),
):
    varatts = data_all["variables"][var]
    php = data_all["physical_parameters"][
        cat_link_to_uid["physical_parameters"][
            varatts["physical_parameter"].split("::")[1]
        ]
    ]
    cfsn = (
        data_all["cf_standard_names"][
            cat_link_to_uid["cf_standard_names"][php["cf_standard_name"].split("::")[1]]
        ]
        if "cf_standard_name" in php
        else {}
    )
    var_index[varatts["uid"]] = {
        "cmip7_name": varatts.get("cmip7_compound_name", "undef"),
        "cmip6_name": varatts.get("cmip6_compound_name", "undef"),
        "long_name": php.get("title", "undef"),
        "root_name": (
            varatts["branded_variable_name"].split("_")[0]
            if "branded_variable_name" in varatts
            and "_" in varatts["branded_variable_name"]
            else "undef"
        ),
        "branding_label": (
            varatts["branded_variable_name"].split("_")[1]
            if "branded_variable_name" in varatts
            and "_" in varatts["branded_variable_name"]
            else "undef"
        ),
        "standard_name": cfsn.get("name", "undef"),
        "description": varatts.get("description", "undef"),
        "url": f"latest/u/{sanitize(varatts['uid'])}.html",
        "history": "",
        "state": True,
    }
    if varatts["uid"] not in var_index_old and version != "v1.0":
        var_index[varatts["uid"]]["history"] = "Added in version " + version
        print(
            f"INFO: Variable {var} ({varatts.get('cmip6_compound_name', 'undef')}) added in version "
            + version
        )
    elif (
        varatts["uid"] in var_index_old
        and "Removed" in var_index_old[varatts["uid"]]["history"]
        and var_index_old[varatts["uid"]]["state"] is False
    ):
        var_index[varatts["uid"]]["history"] += " | Re-added in version " + version
        print(
            f"INFO: Variable {var} ({varatts.get('cmip6_compound_name', 'undef')}) re-added in version "
            + version
        )
for var in var_index_old:
    if var not in var_index:
        var_index[var] = var_index_old[var]
        if var_index_old[var]["history"] != "" and var_index_old[var]["state"] is True:
            var_index[var]["history"] += " | Removed in version " + version
        elif var_index_old[var]["history"] == "":
            var_index[var]["history"] = "Removed in version " + version
        if var_index_old[var]["state"] is True:
            print(
                f"WARNING: Variable {var} ({var_index_old[var]['cmip6_name']}) removed in version "
                + version
            )
            print(var_index[var]["history"])
        else:
            print(
                f"INFO: Encountered previously removed variable {var} ({var_index_old[var]['cmip6_name']})"
            )
            print(var_index[var]["history"])
        var_index[var]["state"] = False
        # Write placeholder html file to avoid broken links:
        html = f"<h1>{category} record: {escape(var)} ({version})</h1>"
        html += f"<p><a href='../variables.html'>Back to variables</a> | <a href='../index.html'>Category Index</a></p>"
        html += f"<script src='../../check_deprecated.js' data-dreq-version=\"{version}\" data-dreq-latest=\"../../dreq_latest_version.txt\"></script>"
        html += (
            "<p><strong>Note:</strong> This record was removed with the current or with a previous release of the CMIP7 Data Request. Link to record of previous version: <a href='../../"
            + var_index[var]["url"].replace("latest/", f"{prev_version}/")
            + "'>LINK</a></p>"
        )
        html += f"<p>Link to <a href='../../variable_changes.html?uid={var}' target='_blank'>CMIP7-AFT Data Request - Variable Changes</a></p>"
        html += "<table><tr><th>Attribute</th><th>Value</th></tr>"
        for attr, value in var_index_old[var].items():
            if attr == "url" or attr == "state":
                continue
            elif attr == "history":
                html += f"<tr><td class='field_name'><strong>{escape(attr)}</strong></td><td>{escape(value).replace(' | ', '<br>')}</td></tr>"
                continue
            html += f"<tr><td class='field_name'><strong>{escape(attr)}</strong></td><td>{escape(value)}</td></tr>"
        html += "</table>"
        html += f"<p><a href='../variables.html'>Back to variables</a> | <a href='../index.html'>Category Index</a></p>"
        write_file(
            f"u/{sanitize(var)}.html",
            html,
            title=f"variables record: {escape(var)} ({version})",
            style_location="../../",
        )


with open(vi_file, "w") as f:
    json.dump(var_index, f, indent=4)
print(f"Variable index written to '{vi_file}'")


# === Data Request Changes - Requests ===
drchfile = f"{OUTPUT_DIR}/../dreq_changes.json"
dreqvfile = f"data_request_req_{version}.json"
dreqpvfile = f"data_request_req_{prev_version}.json"
indexfile = f"{OUTPUT_DIR}/../dreq_index.json"

# Update index of opportunities and experiments
# Load index
if os.path.exists(indexfile):
    with open(indexfile) as f:
        dreqindex = json.load(f)
else:
    dreqindex = {
        "experiments": {},
        "mips": {},
        "opportunities": {},
    }
# Get updated opps / exps
all_opps = DR.get_opportunities()
all_mips = DR.get_mips()
opps = dict()
for opp in all_opps:
    try:
        opps[str(opp.attributes["uid"])] = (
            f'{str(opp.attributes["name"])} (ID: {str(opp.attributes["opportunity_id"])})'
        )
    except KeyError:
        opps[str(opp.attributes["uid"])] = f'{str(opp.attributes["name"])}'
all_exps = DR.get_experiments()
exps = dict()
for exp in all_exps:
    exps[str(exp.attributes["uid"])] = str(exp.attributes["name"])
mips = dict()
for mip in all_mips:
    mips[str(mip.attributes["uid"])] = str(mip.attributes["name"])
# Update index
dreqindex["opportunities"].update(opps)
dreqindex["experiments"].update(exps)
dreqindex["mips"].update(mips)
with open(indexfile, "w") as f:
    json.dump(dreqindex, f, indent=4)
print(f"Data Request index written to '{indexfile}'")

# Current version requests
dreqvoppreq = dict()
oppexps = dict()
oppvars = dict()
oppens = dict()
for opp in all_opps:
    oppexps[str(opp.attributes["uid"])] = [
        str(exp.attributes["uid"])
        for expg in opp.get_experiment_groups()
        for exp in expg.get_experiments()
    ]
    oppvars[str(opp.attributes["uid"])] = [
        str(var.attributes["uid"])
        for varg in opp.get_variable_groups()
        for var in varg.get_variables()
    ]
    oppens[str(opp.attributes["uid"])] = str(opp.attributes["minimum_ensemble_size"])
    dreqvoppreq[str(opp.attributes["uid"])] = {
        "ensemble_size": oppens[str(opp.attributes["uid"])],
        "experiments": oppexps[str(opp.attributes["uid"])],
        "variables": oppvars[str(opp.attributes["uid"])],
        "type": "opportunity",
    }
# for mip in all_mips:
#    mipexps = [str(exp.attributes["uid"]) for exp in mip.get_experiments()]
#    mipvars = [str(var.attributes["uid"]) for varg in mip.get_variable_groups() for var in varg.get_variables()]
#    mipopps = [str(opp.attributes["uid"]) for opp in mip.get_opportunities()]
#    dreqvoppreq[str(mip.attributes["uid"])] = {
#        "experiments": mipexps,
#        "opportunities": mipopps,
#        "variables": mipvars,
#        "type": "mip",
#    }

with open(dreqvfile, "w") as f:
    json.dump(dreqvoppreq, f, indent=4)
print(f"Data Request requests written to '{dreqvfile}'")

# Previous version requests
if prev_version:
    with open(dreqpvfile) as f:
        dreqpvoppreq = json.load(f)
else:
    dreqpvoppreq = dict()

# Previous changes
if os.path.exists(drchfile):
    with open(drchfile) as f:
        changes = json.load(f)
else:
    changes = dict()

# Get changes
changes[version] = defaultdict(dict)
for opp in dreqvoppreq.keys():
    changes[version][opp] = {"state": ""}
    if opp in dreqpvoppreq.keys():
        if dreqvoppreq[opp]["ensemble_size"] != dreqpvoppreq[opp]["ensemble_size"]:
            changes[version][opp][
                "ensemble_size"
            ] = f"{dreqvoppreq[opp]['ensemble_size']} -> {dreqpvoppreq[opp]['ensemble_size']}"
        if set(dreqvoppreq[opp]["experiments"]) != set(
            dreqpvoppreq[opp]["experiments"]
        ):
            added = [
                exp
                for exp in dreqvoppreq[opp]["experiments"]
                if exp not in dreqpvoppreq[opp]["experiments"]
            ]
            removed = [
                exp
                for exp in dreqpvoppreq[opp]["experiments"]
                if exp not in dreqvoppreq[opp]["experiments"]
            ]
            if added:
                changes[version][opp]["experiments_added"] = added
            if removed:
                changes[version][opp]["experiments_removed"] = removed
        if set(dreqvoppreq[opp]["variables"]) != set(dreqpvoppreq[opp]["variables"]):
            added = [
                var
                for var in dreqvoppreq[opp]["variables"]
                if var not in dreqpvoppreq[opp]["variables"]
            ]
            removed = [
                var
                for var in dreqpvoppreq[opp]["variables"]
                if var not in dreqvoppreq[opp]["variables"]
            ]
            if added:
                changes[version][opp]["variables_added"] = added
            if removed:
                changes[version][opp]["variables_removed"] = removed
    else:
        changes[version][opp] = {
            "state": "added",
        }
for opp in dreqpvoppreq.keys():
    if opp not in dreqvoppreq.keys():
        changes[version][opp] = {
            "state": "removed",
        }

if prev_version:
    with open(drchfile, "w") as f:
        json.dump(changes, f, indent=4)
    print(f"Data request changes written to '{drchfile}'")


# === Data Request Changes - Variables ===
from data_request_api.query import dreq_query as dq

vchfile = f"{OUTPUT_DIR}/../variable_changes.json"
vkeys = sorted(
    [
        "frequency",
        "modeling_realm",
        "standard_name",
        "units",
        "cell_methods",
        "cell_measures",
        "long_name",
        "comment",
        "processing_note",
        "dimensions",
        "out_name",
        "type",
        "positive",
        "spatial_shape",
        "temporal_shape",
        "cmip6_table",
        "physical_parameter_name",
        "variableRootDD",
        "branding_label",
        "branded_variable_name",
        "region",
        "cmip6_compound_name",
        "cmip7_compound_name",
        "flag_values",
        "flag_meanings"
    ]
)
vchanges = defaultdict(dict)
vinfo = dict()
vpinfo = dict()
vinfofile = os.path.join(f"data_request_var_{version}.json")
vpinfofile = os.path.join(f"data_request_var_{prev_version}.json")

# Variable changes so far
if os.path.exists(vchfile):
    with open(vchfile) as f:
        vchanges_total = json.load(f)
else:
    vchanges_total = dict()

# Variable Info from previous version
if prev_version:
    with open(vpinfofile) as f:
        vpinfo = json.load(f)
else:
    vpinfo = dict()
# Variable Info from current version
vinfo = dq.get_variables_metadata(dc.load(version), version)
with open(vinfofile, "w") as f:
    json.dump(vinfo, f, indent=4)
    print(f"Data Request variable info written to '{vinfofile}'")


# Compile changes
def prep_val(val):
    if val is None:
        return None
    elif not isinstance(val, str):
        return str(val)
    else:
        return "".join(val.split()).lower()


if prev_version:
    for var in [v for v in vinfo.keys() if v in vpinfo.keys()]:
        for key in vkeys:
            new = prep_val(vinfo[var].get(key, None))
            old = prep_val(vpinfo[var].get(key, None))
            if old is None:
                # Attributes defined for the first time should not be treated as changes
                #  as they would dominate the list of changes (i.e cmip7_compound_name
                #  root_name or branded_variable_name, introduced in v1.2.?)
                pass
            elif new is None:
                vchanges[var][key] = {"old": vpinfo[var][key].strip(), "new": ""}
                vchanges[var]["state"] = ""
            elif new != old:
                vchanges[var][key] = {
                    "old": vpinfo[var][key].strip(),
                    "new": vinfo[var][key].strip(),
                }
                vchanges[var]["state"] = ""
    for var in [v for v in vinfo.keys() if v not in vpinfo.keys()]:
        vchanges[var] = {"state": "added"}
    for var in [v for v in vpinfo.keys() if v not in vinfo.keys()]:
        vchanges[var] = {"state": "removed"}
    vchanges_total[version] = vchanges
    with open(vchfile, "w") as f:
        json.dump(vchanges_total, f, indent=4)
    print(f"Data Request variable changes written to '{vchfile}'")