generate_metadata.py

import concurrent.futures
import itertools
import logging
from collections.abc import Iterator, Sequence
from datetime import datetime, timezone
from json import loads
from pathlib import Path
from sys import argv

import dacite
from git import Repo
from jinja2 import Environment, FileSystemLoader
from urllib3 import request

import build_warnings
import sphinx_lint
from generate import LanguageProjectData
from repositories import Language

generation_time = datetime.now(timezone.utc)


def get_projects_metadata(
    completion_progress: Sequence[LanguageProjectData],
) -> Iterator[tuple[int, int, datetime | None]]:
    with concurrent.futures.ProcessPoolExecutor() as executor:
        return executor.map(
            get_metadata,
            *zip(*map(get_language_repo_and_completion, completion_progress)),
            itertools.repeat(Path('clones')),
        )


def get_metadata(
    language: Language, repo: str | None, completion: float, clones_dir: str
) -> tuple[int, int, datetime | None]:
    if not repo or not (repo_path := Path(clones_dir, 'translations', repo)).exists():
        return 0, 0, None
    latest_commit = Repo(repo_path).head.commit.committed_datetime
    if not completion:
        return 0, 0, latest_commit
    return (
        build_warnings.number(clones_dir, repo, language.code),
        sphinx_lint.store_and_count_failures(clones_dir, repo, language.code),
        latest_commit,
    )


def get_language_repo_and_completion(
    project: LanguageProjectData,
) -> tuple[Language, str | None, float]:
    return project.language, project.repository, project.completion


if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    logging.info(f'starting at {generation_time}')
    env = Environment(loader=FileSystemLoader('templates'))
    if (index_path := Path('build/index.json')).exists():
        index_json = loads(Path('build/index.json').read_text())
    else:
        index_json = request('GET', argv[1]).json()

    completion_progress = [
        dacite.from_dict(LanguageProjectData, project) for project in index_json
    ]

    output = env.get_template('metadata.html.jinja').render(
        metadata=zip(completion_progress, get_projects_metadata(completion_progress)),
        generation_time=generation_time,
        duration=(datetime.now(timezone.utc) - generation_time).seconds,
    )

    Path('build/metadata.html').write_text(output)