data_loading_helpers.py

from __future__ import annotations

import os
from pathlib import Path
from typing import Optional, Union

import requests


def get_url_text(
    url: Union[str, tuple[str, str]],
    data_path: Optional[Path],
    encoding: Optional[str] = None,
) -> str:
    """
    If data_path is not None, returns the file from that path
    (assuming it's stored there with the same filename as in the URL).

    Otherwise, downloads it from the web.

    :param url: If url is a str, assumes that the path in the local case is
        just data_path + the "basename" of the url.
        If url is a tuple of the form (web_prefix, common_path), then the
        path in the local case is assuemd to be `common_path`
    """
    if isinstance(url, tuple):
        web_prefix, common_path = url
    else:
        web_prefix, common_path = os.path.split(url)

    if data_path is not None:
        path = Path(data_path, common_path)
        return path.read_text(encoding=encoding)
    else:
        web_url = os.path.join(web_prefix, common_path)
        return requests.get(web_url).text