diff --git a/cwmscli/__main__.py b/cwmscli/__main__.py index 49f189c..088d892 100644 --- a/cwmscli/__main__.py +++ b/cwmscli/__main__.py @@ -85,6 +85,7 @@ def cli( cli.add_command(commands_cwms.csv2cwms_cmd) cli.add_command(commands_cwms.update_cli_cmd) cli.add_command(commands_cwms.blob_group) +cli.add_command(commands_cwms.clob_group) cli.add_command(commands_cwms.users_group) cli.add_command(load.load_group) add_version_to_help_tree(cli) diff --git a/cwmscli/commands/blob.py b/cwmscli/commands/blob.py index cc26fbd..16fb31a 100644 --- a/cwmscli/commands/blob.py +++ b/cwmscli/commands/blob.py @@ -722,4 +722,12 @@ def list_cmd( else: # Friendly console preview with pd.option_context("display.max_rows", 500, "display.max_columns", None): - logging.info(df.to_string(index=False)) + # Left-align all columns + logging.info( + "\n" + + df.apply( + lambda s: (s := s.astype(str).str.strip()).str.ljust( + s.str.len().max() + ) + ).to_string(index=False, justify="left") + ) diff --git a/cwmscli/commands/clob.py b/cwmscli/commands/clob.py new file mode 100644 index 0000000..a2584b9 --- /dev/null +++ b/cwmscli/commands/clob.py @@ -0,0 +1,340 @@ +import json +import logging +import os +import sys +from typing import Optional, Sequence + +import cwms +import pandas as pd +import requests +from cwms import api as cwms_api + +from cwmscli.utils import get_api_key, has_invalid_chars, log_scoped_read_hint + + +def _join_api_url(api_root: str, path: str) -> str: + return f"{api_root.rstrip('/')}/{path.lstrip('/')}" + + +def _resolve_optional_api_key(api_key: Optional[str], anonymous: bool) -> Optional[str]: + if anonymous or not api_key: + return None + return get_api_key(api_key, None) + + +def _write_clob_content(content: str, dest: str) -> str: + os.makedirs(os.path.dirname(dest) or ".", exist_ok=True) + with open(dest, "w", encoding="utf-8", newline="") as f: + f.write(content) + return dest + + +def _clob_endpoint_id(clob_id: str) -> tuple[str, Optional[str]]: + normalized = clob_id.upper() + if has_invalid_chars(normalized): + return "ignored", normalized + return normalized, None + + +def _get_special_clob_text(*, office: str, clob_id: str) -> str: + with cwms_api.SESSION.get( + "clobs/ignored", + params={"office": office, "clob-id": clob_id}, + headers={"Accept": "text/plain"}, + ) as response: + response.raise_for_status() + return response.text + + +def list_clobs( + office: Optional[str] = None, + clob_id_like: Optional[str] = None, + columns: Optional[Sequence[str]] = None, + sort_by: Optional[Sequence[str]] = None, + ascending: bool = True, + limit: Optional[int] = None, + page_size: Optional[int] = None, +) -> pd.DataFrame: + logging.info(f"Listing clobs for office: {office!r}...") + fetch_page_size = page_size if page_size is not None else limit + result = cwms.get_clobs( + office_id=office, + clob_id_like=clob_id_like, + page_size=fetch_page_size, + ) + + # Accept either a DataFrame or a JSON/dict-like response + if isinstance(result, pd.DataFrame): + df = result.copy() + else: + # Expecting normal clob return structure + data = getattr(result, "json", None) + if callable(data): + data = result.json() + df = pd.DataFrame((data or {}).get("clobs", [])) + + # Allow column filtering + if columns: + keep = [c for c in columns if c in df.columns] + if keep: + df = df[keep] + + # Sort by option + if sort_by: + by = [c for c in sort_by if c in df.columns] + if by: + df = df.sort_values(by=by, ascending=ascending, kind="stable") + + # Optional limit + if limit is not None: + df = df.head(limit) + + logging.info(f"Found {len(df):,} clob(s)") + # List the clobs in the logger + for _, row in df.iterrows(): + logging.info(f"clob ID: {row['id']}, Description: {row.get('description')}") + return df + + +def upload_cmd( + input_file: str, + clob_id: str, + description: str, + overwrite: bool, + dry_run: bool, + office: str, + api_root: str, + api_key: str, +): + cwms.init_session(api_root=api_root, api_key=get_api_key(api_key, None)) + try: + file_size = os.path.getsize(input_file) + with open(input_file, "r", encoding="utf-8") as f: + file_data = f.read() + logging.info(f"Read file: {input_file} ({file_size} bytes)") + except Exception as e: + logging.error(f"Failed to read file: {e}") + sys.exit(1) + + clob_id_up = clob_id.upper() + logging.debug(f"Office={office} clobID={clob_id_up}") + + clob = { + "office-id": office, + "id": clob_id_up, + "description": ( + json.dumps(description) + if isinstance(description, (dict, list)) + else description + ), + "value": file_data, + } + params = {"fail-if-exists": not overwrite} + view_url = _join_api_url(api_root, f"clobs/{clob_id_up}?office={office}") + + if dry_run: + logging.info( + f"DRY RUN: would POST {_join_api_url(api_root, 'clobs')} with params={params}" + ) + logging.info( + json.dumps( + { + "url": _join_api_url(api_root, "clobs"), + "params": params, + "clob": {**clob, "value": f'<{len(clob["value"])} chars>'}, + }, + indent=2, + ) + ) + return + + try: + cwms.store_clobs(clob, fail_if_exists=not overwrite) + logging.info(f"Uploaded clob: {clob_id_up}") + if has_invalid_chars(clob_id_up): + logging.info( + f"View: {_join_api_url(api_root, f'clobs/ignored?clob-id={clob_id_up}&office={office}')}" + ) + else: + logging.info(f"View: {view_url}") + except requests.HTTPError as e: + detail = getattr(e.response, "text", "") or str(e) + logging.error(f"Failed to upload (HTTP): {detail}") + sys.exit(1) + except Exception as e: + logging.error(f"Failed to upload: {e}") + sys.exit(1) + + +def download_cmd( + clob_id: str, + dest: str, + office: str, + api_root: str, + api_key: str, + dry_run: bool, + anonymous: bool = False, +): + if dry_run: + logging.info( + f"DRY RUN: would GET {api_root} clob with clob-id={clob_id} office={office}." + ) + return + resolved_api_key = _resolve_optional_api_key(api_key, anonymous) + cwms.init_session(api_root=api_root, api_key=resolved_api_key) + bid = clob_id.upper() + logging.debug(f"Office={office} clobID={bid}") + + try: + path_id, query_id = _clob_endpoint_id(bid) + if query_id is None: + clob = cwms.get_clob(office_id=office, clob_id=path_id) + payload = getattr(clob, "json", clob) + if callable(payload): + payload = payload() + if isinstance(payload, dict): + content = payload.get("value", "") + else: + content = str(payload) + else: + content = _get_special_clob_text(office=office, clob_id=query_id) + target = dest or bid + _write_clob_content(content, target) + logging.info(f"Downloaded clob to: {target}") + except requests.HTTPError as e: + detail = getattr(e.response, "text", "") or str(e) + logging.error(f"Failed to download (HTTP): {detail}") + log_scoped_read_hint( + api_key=resolved_api_key, + anonymous=anonymous, + office=office, + action="download", + resource="clob content", + ) + sys.exit(1) + except Exception as e: + logging.error(f"Failed to download: {e}") + log_scoped_read_hint( + api_key=resolved_api_key, + anonymous=anonymous, + office=office, + action="download", + resource="clob content", + ) + sys.exit(1) + + +def delete_cmd(clob_id: str, office: str, api_root: str, api_key: str, dry_run: bool): + + if dry_run: + logging.info( + f"DRY RUN: would DELETE {api_root} clob with clob-id={clob_id} office={office}" + ) + return + cwms.init_session(api_root=api_root, api_key=get_api_key(api_key, None)) + cid = clob_id.upper() + path_id, query_id = _clob_endpoint_id(cid) + if query_id is None: + cwms.delete_clob(office_id=office, clob_id=cid) + else: + cwms_api.delete( + f"clobs/{path_id}", params={"office": office, "clob-id": query_id} + ) + logging.info(f"Deleted clob: {clob_id} for office: {office}") + + +def update_cmd( + input_file: str, + clob_id: str, + description: str, + ignore_nulls: bool, + dry_run: bool, + office: str, + api_root: str, + api_key: str, +): + if dry_run: + logging.info( + f"DRY RUN: would PATCH {api_root} clob with clob-id={clob_id} office={office}" + ) + return + file_data = None + if input_file: + try: + file_size = os.path.getsize(input_file) + with open(input_file, "r", encoding="utf-8") as f: + file_data = f.read() + logging.info(f"Read file: {input_file} ({file_size} bytes)") + except Exception as e: + logging.error(f"Failed to read file: {e}") + sys.exit(1) + # Setup minimum required payload + clob = {"office-id": office, "id": clob_id.upper()} + if description: + clob["description"] = description + + if file_data: + clob["value"] = file_data + cwms.init_session(api_root=api_root, api_key=get_api_key(api_key, None)) + cid = clob_id.upper() + path_id, query_id = _clob_endpoint_id(cid) + if query_id is None: + cwms.update_clob(clob, cid, ignore_nulls=ignore_nulls) + else: + cwms_api.patch( + f"clobs/{path_id}", + data=clob, + params={"clob-id": query_id, "ignore-nulls": ignore_nulls}, + ) + logging.info(f"Updated clob: {clob_id} for office: {office}") + + +def list_cmd( + clob_id_like: str, + columns: list[str], + sort_by: list[str], + desc: bool, + limit: int, + page_size: int, + to_csv: str, + office: str, + api_root: str, + api_key: str, + anonymous: bool = False, +): + resolved_api_key = _resolve_optional_api_key(api_key, anonymous) + cwms.init_session(api_root=api_root, api_key=resolved_api_key) + try: + df = list_clobs( + office=office, + clob_id_like=clob_id_like, + columns=columns, + sort_by=sort_by, + ascending=not desc, + limit=limit, + page_size=page_size, + ) + except Exception: + log_scoped_read_hint( + api_key=resolved_api_key, + anonymous=anonymous, + office=office, + action="list", + resource="clob content", + ) + raise + if to_csv: + df.to_csv(to_csv, index=False) + logging.info(f"Wrote {len(df)} rows to {to_csv}") + else: + # Friendly console preview + with pd.option_context("display.max_rows", 500, "display.max_columns", None): + # Left-align all columns + logging.info( + "\n" + + df.apply( + lambda s: (s := s.astype(str).str.strip()).str.ljust( + s.str.len().max() + ) + ).to_string(index=False, justify="left") + ) diff --git a/cwmscli/commands/commands_cwms.py b/cwmscli/commands/commands_cwms.py index ebe6fa4..97af6cc 100644 --- a/cwmscli/commands/commands_cwms.py +++ b/cwmscli/commands/commands_cwms.py @@ -411,6 +411,174 @@ def list_cmd(**kwargs): list_cmd(**kwargs) +# endregion + + +# region Clob +# ================================================================================ +# CLOB +# ================================================================================ +@click.group( + "clob", + help="Manage CWMS Clobs (upload, download, delete, update, list)", + epilog=textwrap.dedent( + """ + Example Usage:\n + - Download a clob by id to your local filesystem\n + - Update a clob's name/description/mime-type\n + - Bulk list clobs for an office +""" + ), +) +@requires(reqs.cwms) +def clob_group(): + pass + + +# ================================================================================ +# Upload +# ================================================================================ +@clob_group.command("upload", help="Upload a file as a clob") +@click.option( + "--input-file", + required=True, + type=click.Path(exists=True, dir_okay=False, readable=True, path_type=str), + help="Path to the file to upload.", +) +@click.option("--clob-id", required=True, type=str, help="Clob ID to create.") +@click.option("--description", default=None, help="Optional description JSON or text.") +@click.option( + "--overwrite/--no-overwrite", + default=False, + show_default=True, + help="If true, replace existing clob.", +) +@click.option("--dry-run", is_flag=True, help="Show request; do not send.") +@common_api_options +def clob_upload(**kwargs): + from cwmscli.commands.clob import upload_cmd + + upload_cmd(**kwargs) + + +# ================================================================================ +# Download +# ================================================================================ +@clob_group.command("download", help="Download a clob by ID") +# TODO: test XML +@click.option("--clob-id", required=True, type=str, help="Clob ID to download.") +@click.option( + "--dest", + default=None, + help="Destination file path. Defaults to clob-id.", +) +@click.option( + "--anonymous", + is_flag=True, + help="Do not send credentials for this read request, even if they are configured.", +) +@click.option("--dry-run", is_flag=True, help="Show request; do not send.") +@common_api_options +def clob_download(**kwargs): + from cwmscli.commands.clob import download_cmd + + download_cmd(**kwargs) + + +# ================================================================================ +# Delete +# ================================================================================ +@clob_group.command("delete", help="Delete a clob by ID") +@click.option("--clob-id", required=True, type=str, help="Clob ID to delete.") +@click.option("--dry-run", is_flag=True, help="Show request; do not send.") +@common_api_options +def delete_cmd(**kwargs): + from cwmscli.commands.clob import delete_cmd + + delete_cmd(**kwargs) + + +# ================================================================================ +# Update +# ================================================================================ +@clob_group.command("update", help="Update/patch a clob by ID") +@click.option("--clob-id", required=True, type=str, help="Clob ID to update.") +@click.option("--dry-run", is_flag=True, help="Show request; do not send.") +@click.option( + "--description", + default=None, + help="New description JSON or text.", +) +@click.option( + "--input-file", + required=False, + type=click.Path(exists=True, dir_okay=False, readable=True, path_type=str), + help="Optional file content to upload with update.", +) +@click.option( + "--ignore-nulls/--no-ignore-nulls", + default=True, + show_default=True, + help="If true, null and empty fields in the provided clob will be ignored and the existing value of those fields left in place.", +) +@common_api_options +def update_cmd(**kwargs): + from cwmscli.commands.clob import update_cmd + + update_cmd(**kwargs) + + +# ================================================================================ +# List +# ================================================================================ +@clob_group.command("list", help="List clobs with optional filters and sorting") +# TODO: Add link to regex docs when new CWMS-DATA site is deployed to PROD +@click.option( + "--clob-id-like", help="LIKE filter for clob ID (e.g., ``*PNG``)." +) # Escape the wildcard/asterisk for RTD generation with double backticks +@click.option( + "--columns", + multiple=True, + callback=csv_to_list, + help="Columns to show (repeat or comma-separate).", +) +@click.option( + "--sort-by", + multiple=True, + callback=csv_to_list, + help="Columns to sort by (repeat or comma-separate).", +) +@click.option( + "--desc/--asc", + default=False, + show_default=True, + help="Sort descending instead of ascending.", +) +@click.option("--limit", type=int, default=None, help="Max rows to show.") +@click.option( + "--page-size", + type=int, + default=None, + help="Max rows to request from the clob endpoint. Defaults to --limit when set.", +) +@click.option( + "--to-csv", + type=click.Path(dir_okay=False, writable=True, path_type=str), + help="If set, write results to this CSV file.", +) +@click.option( + "--anonymous", + is_flag=True, + help="Do not send credentials for this read request, even if they are configured.", +) +@common_api_options +def list_cmd(**kwargs): + from cwmscli.commands.clob import list_cmd + + list_cmd(**kwargs) + + +# endregion # ================================================================================ # USERS # ================================================================================ diff --git a/cwmscli/utils/__init__.py b/cwmscli/utils/__init__.py index 9ba2089..353538a 100644 --- a/cwmscli/utils/__init__.py +++ b/cwmscli/utils/__init__.py @@ -15,6 +15,18 @@ def to_uppercase(ctx, param, value): return value.upper() +def has_invalid_chars(id: str) -> bool: + """ + Checks if ID contains any invalid web path characters. + """ + INVALID_PATH_CHARS = ["/", "\\", "&", "?", "="] + + for char in INVALID_PATH_CHARS: + if char in id: + return True + return False + + def _set_log_level(ctx, param, value): if value is None: return diff --git a/docs/cli.rst b/docs/cli.rst index db31cd8..3926606 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -6,6 +6,8 @@ CLI reference See also -------- +- :doc:`blob ` +- :doc:`clob ` - :doc:`csv2cwms ` - :doc:`CDA Regex Guide ` - :doc:`load location ids-all ` diff --git a/docs/cli/blob.rst b/docs/cli/blob.rst index f5622e3..39fe484 100644 --- a/docs/cli/blob.rst +++ b/docs/cli/blob.rst @@ -5,6 +5,27 @@ Blob commands Use ``cwms-cli blob`` to upload, download, delete, update, and list CWMS blobs. +See also +-------- + +- :doc:`Clob commands ` + +Choose ``blob`` when you are working with binary or media-oriented files such as +PDFs, images, spreadsheets, archives, or other files where media type and +byte-preserving download behavior matter. + +Choose :doc:`clob ` when you are working with text files such as JSON, +XML, notes, templates, or other character-based content that you want the CLI +to handle as text. + +NOTE: + +- Use ``blob`` when you want to set or preserve a media type such as + ``application/json`` so API consumers and JavaScript clients can treat the + payload as JSON (or other media types). +- Use :doc:`clob ` when you mainly want to manage text through + the CLI and don't care about media types. + Quick reference --------------- @@ -15,6 +36,15 @@ Quick reference - Directory upload stops before sending anything if generated blob IDs would collide. - ``blob upload --overwrite``: To replace existing blobs. +Compared with :doc:`clob `, the ``blob`` command group exposes more +file-oriented options: + +- ``--media-type`` for upload and update +- directory upload with ``--input-dir`` +- regex filtering with ``--file-regex`` +- recursive traversal with ``--recursive`` +- generated IDs via ``--blob-id-prefix`` + .. _blob-download-behavior: Download behavior @@ -25,6 +55,7 @@ Download behavior - Text responses are written as text. - Binary responses are written as bytes. - If the destination path has no extension, cwms-cli will try to infer one from the blob media type. +- For text-only content without blob media handling, see :doc:`clob `. Example: @@ -89,6 +120,12 @@ The ``upload`` command supports two modes: 1. Single file upload with explicit blob ID (existing behavior) 2. Directory upload with regex matching (bulk behavior) +If you only need single-file text storage, :doc:`clob ` is usually the +simpler command group. + +If you are uploading JSON and want clients to receive it with a JSON media +type, ``blob upload --media-type application/json`` is usually the better fit. + Single file upload ~~~~~~~~~~~~~~~~~~ diff --git a/docs/cli/clob.rst b/docs/cli/clob.rst new file mode 100644 index 0000000..529144c --- /dev/null +++ b/docs/cli/clob.rst @@ -0,0 +1,198 @@ +Clob commands +============= + +Use ``cwms-cli clob`` to upload, download, delete, update, and list CWMS clobs. + +See also +-------- + +- :doc:`Blob commands ` + +Choose ``clob`` when you are working with text content such as configuration, +JSON, XML, notes, templates, or other character-based files that you want the +CLI to handle as text. + +Choose :doc:`blob ` when you are working with binary or media-oriented +files such as PDFs, images, spreadsheets, or other content where media type and +binary download behavior matter. + +JSON deserves a specific note: + +- Use :doc:`blob ` for JSON when you want the payload treated as a typed + artifact with media type such as ``application/json`` so downstream clients + can recognize and handle it as JSON. +- Use ``clob`` for JSON when you mainly want to manage it as editable text + through the CLI. + +Quick reference +--------------- + +- ``clob upload`` stores text content from a local file. +- ``clob download`` writes the returned clob text to disk. +- ``clob list`` and ``clob download`` send an API key if one is configured, + unless ``--anonymous`` is used. +- ``clob list --limit`` caps displayed rows, and sets the clob endpoint + request page size unless ``--page-size`` is provided to override the fetch size. +- ``clob upload --overwrite`` replaces an existing clob. +- ``clob update`` supports partial updates and ``--ignore-nulls`` behavior. + +Compared with :doc:`blob `, the ``clob`` command group is intentionally +smaller: + +- no ``--media-type`` option +- no directory upload mode +- no generated IDs from file paths +- ``update`` supports ``--ignore-nulls`` instead of blob-style media updates + +.. _clob-text-behavior: + +Text behavior +------------- + +``cwms-cli clob`` is the text-oriented companion to :doc:`blob `. + +- Clob commands treat file content as text. +- ``clob download`` writes UTF-8 text output to the target file. +- Unlike :doc:`blob `, clob commands do not infer file extensions from a + media type and do not perform binary decoding logic. + +Example: + +.. code-block:: bash + + cwms-cli clob download \ + --clob-id FEBRUARY_SUMMARY_JSON \ + --dest ./downloads/february-summary.json \ + --office SWT + +.. _clob-auth-scope: + +Auth and scope +-------------- + +Clob reads follow the same access pattern as :doc:`blob ` reads. + +- If ``--api-key`` is provided, or ``CDA_API_KEY`` is set, cwms-cli sends that key. +- If no key is provided, clob read commands default to anonymous access. +- Use ``--anonymous`` on ``clob download`` or ``clob list`` to force an anonymous read even when a key is configured. +- If a keyed read fails because the key scope is narrower than the content you are trying to view, the CLI logs a scope hint telling you to retry with ``--anonymous`` or remove the configured key. + +Examples: + +.. code-block:: bash + + # Use configured key, if present + cwms-cli clob download --clob-id A.JSON --office SWT --api-root http://localhost:8082/cwms-data + + # Force anonymous read even if CDA_API_KEY is set + cwms-cli clob download --clob-id A.JSON --office SWT --api-root http://localhost:8082/cwms-data --anonymous + + # Anonymous list + cwms-cli clob list --office SWT --api-root http://localhost:8082/cwms-data --anonymous + +List pagination +--------------- + +``cwms-cli clob list`` can cap the local output and also control how many rows +the CDA clob endpoint returns for the request. + +- ``--limit`` caps how many rows cwms-cli prints or writes. +- When ``--limit`` is set, cwms-cli also uses that value as the clob endpoint + request ``page_size``. +- Use ``--page-size`` to override the request size explicitly, especially if + you want to fetch more rows than you plan to display. + +Examples: + +.. code-block:: bash + + # Fetch and show up to 250 rows + cwms-cli clob list --office SWT --limit 250 + + # Fetch 500 rows from CDA but only show the first 50 locally + cwms-cli clob list --office SWT --limit 50 --page-size 500 + +.. _clob-overwrite-flag: + +Overwrite behavior +------------------ + +``clob upload`` uses a normal Click flag pair: + +- ``--overwrite`` replaces an existing clob +- ``--no-overwrite`` keeps the default behavior and fails if the clob already exists + +Example: + +.. code-block:: bash + + cwms-cli clob upload \ + --input-file ./config/ops-template.json \ + --clob-id OPS_TEMPLATE_JSON \ + --overwrite \ + --office SWT + +.. _clob-update-behavior: + +Update behavior +--------------- + +``clob update`` is intended for text metadata and text file changes. + +- Use ``--description`` to replace the description field. +- Use ``--input-file`` to replace the stored clob text. +- Use ``--ignore-nulls`` to leave existing fields in place when the updated payload omits them. + +Example: + +.. code-block:: bash + + cwms-cli clob update \ + --clob-id OPS_TEMPLATE_JSON \ + --input-file ./config/ops-template.json \ + --description "Updated operational template" \ + --ignore-nulls \ + --office SWT + +Special-character IDs +--------------------- + +Clob IDs that contain ``/`` or other characters that are not supported +in the URL path: + +- cwms-cli detects those IDs automatically. +- For path-sensitive operations such as download, update, and delete, the CLI + uses the CDA fallback pattern with an ``ignored`` path segment and the clob ID + in the query string. +- You can still use the normal ``--clob-id`` argument from the CLI. + +Example: + +.. code-block:: bash + + cwms-cli clob download \ + --clob-id "OPS/TEMPLATES/CONFIG.JSON" \ + --dest ./downloads/config.json \ + --office SWT + +Blob vs clob +------------ + +Use :doc:`blob ` when you need: + +- binary-safe upload and download +- media type tracking +- explicit JSON-friendly media type handling such as ``application/json`` +- extension inference on download +- directory upload with regex matching and generated IDs + +Use ``clob`` when you need: + +- plain text upload and download +- human-readable file content handled as text +- simple text updates without blob media handling + + +.. click:: cwmscli.commands.commands_cwms:clob_group + :prog: cwms-cli clob + :nested: full diff --git a/docs/index.rst b/docs/index.rst index f6474fa..1e53bb2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -58,6 +58,7 @@ Contents cli/csv2cwms cli/blob + cli/clob cli/users cli/load_location_ids_all cli/update diff --git a/tests/commands/test_clob.py b/tests/commands/test_clob.py new file mode 100644 index 0000000..d32ef8d --- /dev/null +++ b/tests/commands/test_clob.py @@ -0,0 +1,430 @@ +import sys +import types + +import pandas as pd + +from cwmscli.commands import commands_cwms +from cwmscli.commands.clob import ( + _clob_endpoint_id, + delete_cmd, + download_cmd, + list_cmd, + update_cmd, +) + + +def test_blob_and_clob_upload_keep_no_overwrite_flag(): + blob_overwrite = next( + param + for param in commands_cwms.blob_upload.params + if getattr(param, "name", None) == "overwrite" + ) + clob_overwrite = next( + param + for param in commands_cwms.clob_upload.params + if getattr(param, "name", None) == "overwrite" + ) + + assert "--overwrite" in blob_overwrite.opts + assert "--no-overwrite" in blob_overwrite.secondary_opts + assert "--overwrite" in clob_overwrite.opts + assert "--no-overwrite" in clob_overwrite.secondary_opts + + +def test_clob_endpoint_id_uses_ignored_path_for_special_chars(): + assert _clob_endpoint_id("plain_id") == ("PLAIN_ID", None) + assert _clob_endpoint_id("path/id") == ("ignored", "PATH/ID") + + +def test_download_cmd_uses_default_dest_and_writes_text(tmp_path, monkeypatch): + calls = [] + + class FakeClobResponse: + json = {"value": "retrieved clob text"} + + class FakeCwms: + @staticmethod + def init_session(api_root, api_key): + calls.append(("init_session", api_root, api_key)) + return None + + @staticmethod + def get_clob(office_id, clob_id): + calls.append(("get_clob", office_id, clob_id)) + return FakeClobResponse() + + monkeypatch.setitem(sys.modules, "cwms", FakeCwms) + monkeypatch.setattr("cwmscli.commands.clob.cwms", FakeCwms) + + class FakeHTTPError(Exception): + pass + + monkeypatch.setitem( + sys.modules, "requests", types.SimpleNamespace(HTTPError=FakeHTTPError) + ) + monkeypatch.setattr( + "cwmscli.commands.clob.requests", + types.SimpleNamespace(HTTPError=FakeHTTPError), + ) + + monkeypatch.chdir(tmp_path) + + download_cmd( + clob_id="test_clob", + dest=None, + office="SWT", + api_root="https://example.test/", + api_key="apikey 123", + dry_run=False, + ) + + saved = tmp_path / "TEST_CLOB" + assert saved.exists() + assert saved.read_text(encoding="utf-8") == "retrieved clob text" + assert calls == [ + ("init_session", "https://example.test/", "apikey 123"), + ("get_clob", "SWT", "TEST_CLOB"), + ] + + +def test_download_cmd_uses_query_override_for_special_char_ids(tmp_path, monkeypatch): + calls = [] + + class FakeCwms: + @staticmethod + def init_session(api_root, api_key): + calls.append(("init_session", api_root, api_key)) + return None + + @staticmethod + def get_clob(office_id, clob_id, clob_id_query=None): + calls.append(("get_clob", office_id, clob_id, clob_id_query)) + raise AssertionError("special-char path should use direct SESSION.get") + + class FakeResponse: + text = "retrieved clob text" + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def raise_for_status(self): + return None + + class FakeSession: + @staticmethod + def get(endpoint, params=None, headers=None): + calls.append(("session.get", endpoint, params, headers)) + return FakeResponse() + + monkeypatch.setitem(sys.modules, "cwms", FakeCwms) + monkeypatch.setattr("cwmscli.commands.clob.cwms", FakeCwms) + monkeypatch.setattr("cwmscli.commands.clob.cwms_api.SESSION", FakeSession()) + + class FakeHTTPError(Exception): + pass + + monkeypatch.setitem( + sys.modules, "requests", types.SimpleNamespace(HTTPError=FakeHTTPError) + ) + monkeypatch.setattr( + "cwmscli.commands.clob.requests", + types.SimpleNamespace(HTTPError=FakeHTTPError), + ) + + download_cmd( + clob_id="path/id", + dest=str(tmp_path / "downloaded.txt"), + office="SWT", + api_root="https://example.test/", + api_key="apikey 123", + dry_run=False, + ) + + assert calls == [ + ("init_session", "https://example.test/", "apikey 123"), + ( + "session.get", + "clobs/ignored", + {"office": "SWT", "clob-id": "PATH/ID"}, + {"Accept": "text/plain"}, + ), + ] + + +def test_download_cmd_anonymous_skips_api_key(tmp_path, monkeypatch): + calls = [] + + class FakeClobResponse: + json = {"value": "retrieved clob text"} + + class FakeCwms: + @staticmethod + def init_session(api_root, api_key): + calls.append(("init_session", api_root, api_key)) + return None + + @staticmethod + def get_clob(office_id, clob_id): + return FakeClobResponse() + + monkeypatch.setitem(sys.modules, "cwms", FakeCwms) + monkeypatch.setattr("cwmscli.commands.clob.cwms", FakeCwms) + + class FakeHTTPError(Exception): + pass + + monkeypatch.setitem( + sys.modules, "requests", types.SimpleNamespace(HTTPError=FakeHTTPError) + ) + monkeypatch.setattr( + "cwmscli.commands.clob.requests", + types.SimpleNamespace(HTTPError=FakeHTTPError), + ) + + download_cmd( + clob_id="test_clob", + dest=str(tmp_path / "downloaded.txt"), + office="SWT", + api_root="https://example.test/", + api_key="apikey 123", + dry_run=False, + anonymous=True, + ) + + assert calls == [("init_session", "https://example.test/", None)] + + +def test_list_cmd_initializes_session_with_api_key(monkeypatch): + calls = [] + + class FakeCwms: + @staticmethod + def init_session(api_root, api_key): + calls.append(("init_session", api_root, api_key)) + return None + + @staticmethod + def get_clobs(office_id, clob_id_like, page_size=None): + calls.append(("get_clobs", office_id, clob_id_like, page_size)) + return pd.DataFrame([{"id": "TEST_CLOB", "description": "x"}]) + + monkeypatch.setitem(sys.modules, "cwms", FakeCwms) + monkeypatch.setattr("cwmscli.commands.clob.cwms", FakeCwms) + + list_cmd( + clob_id_like="TEST_.*", + columns=[], + sort_by=[], + desc=False, + limit=None, + page_size=None, + to_csv=None, + office="SWT", + api_root="https://example.test/", + api_key="apikey 123", + ) + + assert calls == [ + ("init_session", "https://example.test/", "apikey 123"), + ("get_clobs", "SWT", "TEST_.*", None), + ] + + +def test_list_cmd_uses_limit_as_fetch_page_size(monkeypatch): + calls = [] + + class FakeCwms: + @staticmethod + def init_session(api_root, api_key): + return None + + @staticmethod + def get_clobs(office_id, clob_id_like, page_size=None): + calls.append((office_id, clob_id_like, page_size)) + return pd.DataFrame( + [ + {"id": "A", "description": "x"}, + {"id": "B", "description": "y"}, + ] + ) + + monkeypatch.setitem(sys.modules, "cwms", FakeCwms) + monkeypatch.setattr("cwmscli.commands.clob.cwms", FakeCwms) + + list_cmd( + clob_id_like="TEST_.*", + columns=[], + sort_by=[], + desc=False, + limit=25, + page_size=None, + to_csv=None, + office="SWT", + api_root="https://example.test/", + api_key="apikey 123", + ) + + assert calls == [("SWT", "TEST_.*", 25)] + + +def test_list_cmd_page_size_overrides_limit_for_fetch(monkeypatch): + calls = [] + + class FakeCwms: + @staticmethod + def init_session(api_root, api_key): + return None + + @staticmethod + def get_clobs(office_id, clob_id_like, page_size=None): + calls.append((office_id, clob_id_like, page_size)) + return pd.DataFrame([{"id": "A", "description": "x"}]) + + monkeypatch.setitem(sys.modules, "cwms", FakeCwms) + monkeypatch.setattr("cwmscli.commands.clob.cwms", FakeCwms) + + list_cmd( + clob_id_like="TEST_.*", + columns=[], + sort_by=[], + desc=False, + limit=25, + page_size=200, + to_csv=None, + office="SWT", + api_root="https://example.test/", + api_key="apikey 123", + ) + + assert calls == [("SWT", "TEST_.*", 200)] + + +def test_list_cmd_anonymous_skips_api_key(monkeypatch): + calls = [] + + class FakeCwms: + @staticmethod + def init_session(api_root, api_key): + calls.append(("init_session", api_root, api_key)) + return None + + @staticmethod + def get_clobs(office_id, clob_id_like, page_size=None): + calls.append(("get_clobs", office_id, clob_id_like, page_size)) + return pd.DataFrame([{"id": "TEST_CLOB", "description": "x"}]) + + monkeypatch.setitem(sys.modules, "cwms", FakeCwms) + monkeypatch.setattr("cwmscli.commands.clob.cwms", FakeCwms) + + list_cmd( + clob_id_like="TEST_.*", + columns=[], + sort_by=[], + desc=False, + limit=None, + page_size=None, + to_csv=None, + office="SWT", + api_root="https://example.test/", + api_key="apikey 123", + anonymous=True, + ) + + assert calls == [ + ("init_session", "https://example.test/", None), + ("get_clobs", "SWT", "TEST_.*", None), + ] + + +def test_delete_cmd_uses_query_override_for_special_char_ids(monkeypatch): + calls = [] + + class FakeCwms: + @staticmethod + def init_session(api_root, api_key): + calls.append(("init_session", api_root, api_key)) + return None + + @staticmethod + def delete_clob(office_id, clob_id): + calls.append(("delete_clob", office_id, clob_id)) + + class FakeApi: + @staticmethod + def delete(endpoint, params=None): + calls.append(("api.delete", endpoint, params)) + + monkeypatch.setitem(sys.modules, "cwms", FakeCwms) + monkeypatch.setattr("cwmscli.commands.clob.cwms", FakeCwms) + monkeypatch.setattr("cwmscli.commands.clob.cwms_api", FakeApi) + + delete_cmd( + clob_id="path/id", + office="SWT", + api_root="https://example.test/", + api_key="apikey 123", + dry_run=False, + ) + + assert calls == [ + ("init_session", "https://example.test/", "apikey 123"), + ( + "api.delete", + "clobs/ignored", + {"office": "SWT", "clob-id": "PATH/ID"}, + ), + ] + + +def test_update_cmd_uses_query_override_for_special_char_ids(tmp_path, monkeypatch): + calls = [] + file_path = tmp_path / "updated.txt" + file_path.write_text("updated clob text", encoding="utf-8") + + class FakeCwms: + @staticmethod + def init_session(api_root, api_key): + calls.append(("init_session", api_root, api_key)) + return None + + @staticmethod + def update_clob(data, clob_id, ignore_nulls=True): + calls.append(("update_clob", data, clob_id, ignore_nulls)) + + class FakeApi: + @staticmethod + def patch(endpoint, data=None, params=None): + calls.append(("api.patch", endpoint, data, params)) + + monkeypatch.setitem(sys.modules, "cwms", FakeCwms) + monkeypatch.setattr("cwmscli.commands.clob.cwms", FakeCwms) + monkeypatch.setattr("cwmscli.commands.clob.cwms_api", FakeApi) + + update_cmd( + input_file=str(file_path), + clob_id="path/id", + description="updated description", + ignore_nulls=False, + dry_run=False, + office="SWT", + api_root="https://example.test/", + api_key="apikey 123", + ) + + assert calls == [ + ("init_session", "https://example.test/", "apikey 123"), + ( + "api.patch", + "clobs/ignored", + { + "office-id": "SWT", + "id": "PATH/ID", + "description": "updated description", + "value": "updated clob text", + }, + {"clob-id": "PATH/ID", "ignore-nulls": False}, + ), + ]