diff --git a/.github/ISSUE_TEMPLATE/new_issue.md b/.github/ISSUE_TEMPLATE/new_issue.md index 75ad524..b6fd8a7 100644 --- a/.github/ISSUE_TEMPLATE/new_issue.md +++ b/.github/ISSUE_TEMPLATE/new_issue.md @@ -7,15 +7,15 @@ assignees: '' --- -## Description +### Summary A clear and concise description of what the problem is. -## Considerations +### Considerations Make sure not to forget about these. -## Sub-tasks +### Sub-tasks List any sub-tasks that might help create a clear flow or sub-issues. -## Definition of Done +### Definition of Done This might help in cases when what is to be delivered is more than just one piece of functional code. diff --git a/cfa/dataops/catalog.py b/cfa/dataops/catalog.py index 2e18e02..29431bf 100644 --- a/cfa/dataops/catalog.py +++ b/cfa/dataops/catalog.py @@ -300,6 +300,44 @@ def _get_version_blobs(self, version: str = "latest") -> list: key=lambda x: x["creation_time"], ) + def download_version_to_local( + self, local_path: str, version: str = "latest", force: bool = False + ) -> bool: + """Download a specific version of the data to a local path + + Args: + local_path (str): the local path to download to + version (str, optional): the version to download. Defaults to "latest". + force (bool, optional): whether to force re-download if local. + Returns: + bool: whether any files were written + """ + written = False + blobs = self._get_version_blobs(version) + for blob in blobs: + blob_data = read_blob_stream( + blob_url=blob["name"], + account_name=self.account, + container_name=self.container, + ) + relative_path = blob["name"].removeprefix(f"{self.prefix}/") + local_file_path = os.path.join(local_path, relative_path) + local_dir = os.path.dirname(local_file_path) + os.makedirs(local_dir, exist_ok=True) + if os.path.exists(local_file_path) and not force: + continue + # Handle both raw bytes and objects with content_as_bytes() method + if isinstance(blob_data, bytes): + file_bytes = blob_data + else: + file_bytes = blob_data.content_as_bytes() + with open(local_file_path, "wb") as f: + f.write(file_bytes) + written = True + if written: + self.ledger_entry(action="read") + return written + def get_dataframe( self, output="pandas", version="latest", pl_lazy: bool = False ) -> pd.DataFrame | pl.DataFrame: diff --git a/cfa/dataops/command.py b/cfa/dataops/command.py new file mode 100644 index 0000000..4b01046 --- /dev/null +++ b/cfa/dataops/command.py @@ -0,0 +1,157 @@ +""" +CLI tools for managing data operations in the CFA project. +""" + +import os +from argparse import ArgumentParser + +from rich.console import Console + +from .catalog import datacat +from .utils import tree + + +def _get_dataset_namespaces() -> list[str]: + """ + Helper function to get a list of dataset namespaces. + """ + return datacat.__namespace_list__ + + +def _get_stages_list(dataset_namespace: str) -> list[str]: + """ + Helper function to get a list of stages for a given dataset namespace. + """ + stages_start_with = ["load", "extract", "stage"] + if dataset_namespace not in _get_dataset_namespaces(): + Console().print( + f"[bold red]Error:[/bold red] Dataset namespace '{dataset_namespace}' not found. Available datasets are:\n" + + "\n".join(f"- {ds}" for ds in _get_dataset_namespaces()) + ) + return + dataset_dict = eval(f"datacat.{dataset_namespace}.__dict__") + stages = [ + key + for key in dataset_dict.keys() + if any(key.startswith(prefix) for prefix in stages_start_with) + ] + return sorted(stages) + + +def _get_versions_list(dataset_namespace: str, stage: str) -> list[str]: + """ + Helper function to get a list of versions for a given dataset namespace and stage. + """ + stages = _get_stages_list(dataset_namespace) + if stage not in stages: + Console().print( + f"[bold red]Error:[/bold red] Stage '{stage}' not found for dataset '{dataset_namespace}'. Available stages are:\n" + + "\n".join(f"- {s}" for s in stages) + ) + return + versions = eval(f"datacat.{dataset_namespace}.{stage}.get_versions()") + return versions + + +def get_available_data(): + """ + Retrieve a list of available datasets for CFA. + """ + parser = ArgumentParser(description="Get list of available datasets") + parser.add_argument( + "-p", "--prefix", help="optional prefix filter", default=None + ) + args = parser.parse_args() + datasets = datacat.__namespace_list__ + if args.prefix: + datasets = [ds for ds in datasets if ds.startswith(args.prefix)] + formatted_list = "\n".join(f"- {dataset}" for dataset in sorted(datasets)) + Console().print(f"[bold]Available Datasets:[/bold]\n{formatted_list}") + + +def get_dataset_stages(): + """ + Retrieve stages of available datasets for CFA. + """ + parser = ArgumentParser(description="Get dataset stages") + parser.add_argument("dataset", help="full dataset namespace") + args = parser.parse_args() + dataset = args.dataset + stages = _get_stages_list(dataset) + formatted_stages = "\n".join( + f"- [red]{stage}[/red]" if stage == stages[-1] else f"- {stage}" + for stage in stages + ) + disclaimer = "[italic yellow]Note: Stages in [red]red[/red] indicate the default stage for loading the dataset.[/italic yellow]" + Console().print( + f"[bold]Stages for {dataset}:[/bold]\n{formatted_stages}\n{disclaimer}" + ) + + +def get_dataset_versions(): + """ + Retrieve versions of available datasets for CFA. + """ + parser = ArgumentParser(description="Get dataset versions") + parser.add_argument("dataset", help="full dataset namespace") + parser.add_argument( + "--stage", "-s", help="specific stage to get version for", default=None + ) + args = parser.parse_args() + dataset = args.dataset + available_stages = _get_stages_list(dataset) + if args.stage is None: + stage = available_stages[-1] + else: + stage = args.stage + versions = _get_versions_list(dataset, stage) + formatted_versions = "\n".join( + f"- [red]{version}[/red]" if version == versions[0] else f"- {version}" + for version in versions + ) + Console().print(f"[bold]{dataset}[/bold]:\n{formatted_versions}") + + +def save_data_locally(): + """ + Save a datasets to the local cache. + """ + parser = ArgumentParser(description="Download a dataset locally") + parser.add_argument("dataset", help="full dataset namespace") + parser.add_argument( + "location", + help="local location (directory) to save dataset to. If it does not exist, it will be created.", + ) + parser.add_argument( + "--stage", "-s", help="specific stage to get version for", default=None + ) + parser.add_argument( + "--version", "-v", help="specific version to get", default=None + ) + parser.add_argument( + "--force", "-f", help="force re-download of data", action="store_true" + ) + args = parser.parse_args() + dataset = args.dataset + stage = args.stage + version = args.version + if stage is None: + stages = _get_stages_list(dataset) + stage = stages[-1] + if version is None: + versions = _get_versions_list(dataset, stage) + version = versions[0] + local_path = os.path.abspath(args.location) + written = eval( + f"datacat.{dataset}.{stage}.download_version_to_local('{local_path}', version='{version}', force={args.force})" + ) + if not written: + Console().print( + f"[bold yellow]Dataset '{dataset}' version '{version}' at stage '{stage}' is already present at location '{local_path}'. Use --force to re-download.[/bold yellow]" + ) + return + else: + tree_output = tree(local_path, show_hidden=False) + Console().print( + f"[bold green]Dataset '{dataset}' version '{version}' at stage '{stage}' has been saved locally.[/bold green]\n\n{local_path}\n{tree_output}" + ) diff --git a/cfa/dataops/utils.py b/cfa/dataops/utils.py index 16e56ff..6a4aa20 100644 --- a/cfa/dataops/utils.py +++ b/cfa/dataops/utils.py @@ -4,6 +4,8 @@ import glob import os from datetime import datetime +from itertools import islice +from pathlib import Path from typing import Optional @@ -124,3 +126,66 @@ def get_user() -> str: return getpass.getuser() except Exception: return "unknown_user" + + +def tree( + dir_path: Path, + level: int = -1, + limit_to_directories: bool = False, + length_limit: int = 1000, + show_hidden: bool = False, +): + """Given a directory Path object print a visual tree structure + + Args: + dir_path (Path): the root directory path + level (int): how many levels deep to traverse, -1 for unlimited + limit_to_directories (bool): whether to only show directories + length_limit (int): maximum number of lines to print + show_hidden (bool): whether to show hidden files and directories + Returns: + str: visual tree structure + """ + space = " " + branch = "│ " + tee = "├── " + last = "└── " + dir_path = Path(dir_path) # accept string coercible to Path + files = 0 + directories = 0 + + def inner(dir_path: Path, prefix: str = "", level=-1): + nonlocal files, directories + if not level: + return # 0, stop iterating + if limit_to_directories: + contents = [d for d in dir_path.iterdir() if d.is_dir()] + else: + contents = list(dir_path.iterdir()) + pointers = [tee] * (len(contents) - 1) + [last] + for pointer, path in zip(pointers, contents): + if not show_hidden and path.name.startswith("."): + continue + if path.is_dir(): + yield prefix + pointer + path.name + directories += 1 + extension = branch if pointer == tee else space + yield from inner( + path, prefix=prefix + extension, level=level - 1 + ) + elif not limit_to_directories: + yield prefix + pointer + path.name + files += 1 + + lines = [] + lines.append(dir_path.name) + iterator = inner(dir_path, level=level) + for line in islice(iterator, length_limit): + lines.append(line) + if next(iterator, None): + lines.append(f"... length_limit, {length_limit}, reached, counted:") + return ( + "\n".join(lines) + + f"\n{directories} directories" + + (f", {files} files" if files else "") + ) diff --git a/changelog.md b/changelog.md index 47f8737..d360438 100644 --- a/changelog.md +++ b/changelog.md @@ -8,6 +8,24 @@ The versioning pattern is `YYYY.MM.DD.micro(a/b/{none if release}) --- +## [2025.10.31.0a] + +### Added + +- **New CLI commands** for dataset management: + - `dataops_datasets`: List available datasets with optional prefix filtering + - `dataops_stages`: View available stages for a specific dataset + - `dataops_versions`: List available versions for a dataset stage + - `dataops_save`: Download and save dataset versions locally +- **Tree utility function**: New `tree()` function in `utils.py` for displaying directory structures +- **Local data download**: `download_version_to_local()` method for BlobEndpoint to download datasets to local filesystem +- **Tests**: Comprehensive test coverage for new CLI commands and utilities +- New docs for CLI-tools + +### Updated + +- Documentation formatting: Changed "Description" to "Summary" with adjusted heading levels + ## [2025.10.14.0a] ### Added diff --git a/docs/cli_tools.md b/docs/cli_tools.md new file mode 100644 index 0000000..0005dba --- /dev/null +++ b/docs/cli_tools.md @@ -0,0 +1,218 @@ +# CLI Tools Reference + +The cfa-dataops package provides several command-line tools for managing and accessing datasets. These tools make it easy to explore available datasets, check versions, and download data locally without writing any Python code. + +## Available Commands + +### `dataops_datasets` - List Available Datasets + +Lists all datasets available across all installed catalogs. + +**Basic Usage:** +```bash +dataops_datasets +``` + +**Output Example:** +``` +Available Datasets: +- catalog1.dataset_a +- catalog1.dataset_b +- catalog2.dataset_x +- catalog2.dataset_y +``` + +**Filter by Prefix:** + +You can filter the dataset list using the `--prefix` or `-p` option: + +```bash +dataops_datasets --prefix catalog1 +``` + +This will show only datasets that start with "catalog1". + +--- + +### `dataops_stages` - View Dataset Stages + +Shows all available stages for a specific dataset. The last stage in red is the default stage used when loading data. + +**Usage:** +```bash +dataops_stages +``` + +**Example:** +```bash +dataops_stages "catalog.my_dataset" +``` + +**Output Example:** +``` +Stages for catalog.my_dataset: +- extract +- load # will be in red + +Note: Stages in red indicate the default stage for loading the dataset. +``` + +--- + +### `dataops_versions` - List Dataset Versions + +Lists all available versions for a dataset stage, with the most recent version highlighted in red (default version). + +**Basic Usage (uses default stage):** +```bash +dataops_versions +``` + +**Example:** +```bash +dataops_versions "catalog.my_dataset" +``` + +**Specify a Stage:** + +Use the `--stage` or `-s` option to view versions for a specific stage: + +```bash +dataops_versions "catalog.my_dataset" --stage "extract" +``` + +**Output Example:** +``` +catalog.my_dataset: +- 2025-10-31 # will be in red +- 2025-10-30 +- 2025-10-29 +``` + +The most recent version (at the top) is displayed in red, indicating it's the default. + +--- + +### `dataops_save` - Download Data Locally + +Downloads a specific dataset version to your local filesystem. This is useful for offline work, creating local caches, or working with data in external tools. + +**Basic Usage:** +```bash +dataops_save +``` + +**Example:** +```bash +dataops_save "catalog.my_dataset" "./data/my_dataset" +``` + +This downloads the latest version of the default stage to `./data/my_dataset`. + +**Specify Stage and Version:** + +```bash +dataops_save "catalog.my_dataset" "./data" --stage "load" --version "2025-10-30" +``` + +**Force Re-download:** + +By default, if data already exists locally, it won't be re-downloaded. Use the `--force` or `-f` flag to force a re-download: + +```bash +dataops_save "catalog.my_dataset" ./data --force +``` + +**Command Options:** +- `dataset`: (required) Full dataset namespace (e.g., `catalog.dataset_name`) +- `location`: (required) Local directory path where data will be saved (will be created if it doesn't exist) +- `--stage` or `-s`: (optional) Specific stage to download (defaults to the last stage) +- `--version` or `-v`: (optional) Specific version to download (defaults to the most recent) +- `--force` or `-f`: (optional) Force re-download even if data already exists locally + +**Output Example:** +``` +Dataset 'catalog.my_dataset' version '2025-10-31' at stage 'load' has been saved locally. + +/home/user/data/my_dataset +├── file1.parquet +├── file2.parquet +└── metadata.json +``` + +--- + +## Common Workflows + +### Exploring a New Catalog + +1. **List all available datasets:** + ```bash + dataops_datasets + ``` + +2. **Check stages for a dataset of interest:** + ```bash + dataops_stages "catalog.interesting_dataset" + ``` + +3. **See what versions are available:** + ```bash + dataops_versions "catalog.interesting_dataset" + ``` + +4. **Download the latest data:** + ```bash + dataops_save "catalog.interesting_dataset" "./local_data" + ``` + +### Working with Multiple Catalogs + +Filter datasets by catalog prefix: +```bash +dataops_datasets --prefix "public" +``` + +This helps when you have multiple catalogs installed and want to see what's available in a specific one. + +### Refreshing Local Data + +Force re-download to get the latest data: +```bash +dataops_save "catalog.dataset" "./data" --force +``` + +--- + +## Tips + +- **Tab Completion**: Depending on your shell configuration, you may be able to use tab completion for dataset names +- **Help**: Add `--help` to any command to see its usage information + ```bash + dataops_datasets --help + dataops_stages --help + dataops_versions --help + dataops_save --help + ``` +- **Directory Creation**: The `dataops_save` command automatically creates the target directory if it doesn't exist +- **Tree Display**: After downloading data, the command shows a tree view of the downloaded files for easy verification + +--- + +## Error Handling + +The CLI tools provide helpful error messages: + +- **Invalid dataset name**: Shows list of available datasets +- **Invalid stage**: Shows list of available stages for that dataset +- **Invalid version**: Shows list of available versions for that stage +- **Permission errors**: Indicates if there are file system permission issues +- **Already downloaded**: Warns when data already exists (use `--force` to override) + +--- + +## See Also + +- [Data User Guide](data_user_guide.md) - For programmatic data access using Python +- [Managing Catalogs](managing_catalogs.md) - For setting up and managing data catalogs +- [Data Developer Guide](data_developer_guide.md) - For creating and managing datasets diff --git a/docs/index.md b/docs/index.md index e0e2ee4..7e901a1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,9 +9,10 @@ Welcome to the CFA DataOps system - a catalog-based approach to data management, ## Using the System -3. **[Data User Guide](data_user_guide.md)** - Access and work with datasets using `datacat` -4. **[Data Developer Guide](data_developer_guide.md)** - Add datasets and ETL processes to catalogs -5. **[Report Generation](report_generation.md)** - Create and generate reports using `reportcat` +3. **[CLI Tools Reference](cli_tools.md)** - Command-line tools for exploring and downloading datasets +4. **[Data User Guide](data_user_guide.md)** - Access and work with datasets using `datacat` +5. **[Data Developer Guide](data_developer_guide.md)** - Add datasets and ETL processes to catalogs +6. **[Report Generation](report_generation.md)** - Create and generate reports using `reportcat` ## Quick Reference diff --git a/pyproject.toml b/pyproject.toml index 6e533e6..25a2383 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "cfa.dataops" -version = "2025.10.14.0a" +version = "2025.10.31.0a" description = "Data cataloging, ETL, modeling, verification, and validation for CFA" authors = [ { name = "Phil Rogers", email = "ap66@cdc.gov" }, @@ -60,6 +60,11 @@ build-backend = "poetry.core.masonry.api" [tool.poetry.scripts] dataops_catalog_init = "cfa.dataops.create_catalog.command:main" +dataops_datasets = "cfa.dataops.command:get_available_data" +dataops_stages = "cfa.dataops.command:get_dataset_stages" +dataops_versions = "cfa.dataops.command:get_dataset_versions" +dataops_save = "cfa.dataops.command:save_data_locally" + [tool.pytest.ini_options] addopts = "tests --doctest-modules -vv --cov=cfa --cov-report html --cov-report term" diff --git a/tests/test_blob_endpoint_download.py b/tests/test_blob_endpoint_download.py new file mode 100644 index 0000000..2032213 --- /dev/null +++ b/tests/test_blob_endpoint_download.py @@ -0,0 +1,506 @@ +"""Tests for BlobEndpoint.download_version_to_local method""" + +import os +import tempfile + +import pytest + +from cfa.dataops.catalog import BlobEndpoint + + +@pytest.fixture +def blob_endpoint(mocker, mock_write_blob_stream): + """Create a BlobEndpoint instance for testing""" + mocker.patch( + "cfa.dataops.catalog.write_blob_stream", + mock_write_blob_stream, + ) + ledger_location = { + "account": "account_test", + "container": "container_test", + "prefix": "_access/test/ledger/", + } + return BlobEndpoint( + account="account_test", + container="container_test", + prefix="test/prefix", + ledger_location=ledger_location, + ns="test.endpoint", + ) + + +class TestDownloadVersionToLocal: + """Tests for the download_version_to_local method""" + + def test_download_version_to_local_single_file( + self, mocker, blob_endpoint + ): + """Test downloading a single file version to local path""" + # Mock the blob reading + test_content = b"test file content" + + def mock_read_blob_stream(blob_url, account_name, container_name): + return test_content + + mocker.patch( + "cfa.dataops.catalog.read_blob_stream", + side_effect=mock_read_blob_stream, + ) + + # Mock the version blobs + mocker.patch.object( + blob_endpoint, + "_get_version_blobs", + return_value=[ + { + "name": "test/prefix/2025-01-01T12-00-00/data.csv", + "creation_time": "2025-01-01T12:00:00", + } + ], + ) + + with tempfile.TemporaryDirectory() as tmpdir: + result = blob_endpoint.download_version_to_local( + local_path=tmpdir, + version="2025-01-01T12-00-00", + force=False, + ) + + assert result is True + + # Verify file was written + expected_file = os.path.join( + tmpdir, "2025-01-01T12-00-00", "data.csv" + ) + assert os.path.exists(expected_file) + + # Verify content + with open(expected_file, "rb") as f: + assert f.read() == test_content + + def test_download_version_to_local_multiple_files( + self, mocker, blob_endpoint + ): + """Test downloading multiple files in a version to local path""" + test_content_1 = b"test file 1" + test_content_2 = b"test file 2" + + def mock_read_blob_stream(blob_url, account_name, container_name): + if "data_0.parquet" in blob_url: + return test_content_1 + elif "data_1.parquet" in blob_url: + return test_content_2 + + mocker.patch( + "cfa.dataops.catalog.read_blob_stream", + side_effect=mock_read_blob_stream, + ) + + mocker.patch.object( + blob_endpoint, + "_get_version_blobs", + return_value=[ + { + "name": "test/prefix/2025-01-01T12-00-00/data_0.parquet", + "creation_time": "2025-01-01T12:00:00", + }, + { + "name": "test/prefix/2025-01-01T12-00-00/data_1.parquet", + "creation_time": "2025-01-01T12:00:01", + }, + ], + ) + + with tempfile.TemporaryDirectory() as tmpdir: + result = blob_endpoint.download_version_to_local( + local_path=tmpdir, + version="2025-01-01T12-00-00", + force=False, + ) + + assert result is True + + # Verify both files were written + file1 = os.path.join( + tmpdir, "2025-01-01T12-00-00", "data_0.parquet" + ) + file2 = os.path.join( + tmpdir, "2025-01-01T12-00-00", "data_1.parquet" + ) + assert os.path.exists(file1) + assert os.path.exists(file2) + + with open(file1, "rb") as f: + assert f.read() == test_content_1 + with open(file2, "rb") as f: + assert f.read() == test_content_2 + + def test_download_version_to_local_nested_structure( + self, mocker, blob_endpoint + ): + """Test downloading files with nested directory structure""" + test_content = b"nested file content" + + def mock_read_blob_stream(blob_url, account_name, container_name): + return test_content + + mocker.patch( + "cfa.dataops.catalog.read_blob_stream", + side_effect=mock_read_blob_stream, + ) + + mocker.patch.object( + blob_endpoint, + "_get_version_blobs", + return_value=[ + { + "name": "test/prefix/2025-01-01T12-00-00/subdir/nested/file.txt", + "creation_time": "2025-01-01T12:00:00", + } + ], + ) + + with tempfile.TemporaryDirectory() as tmpdir: + result = blob_endpoint.download_version_to_local( + local_path=tmpdir, + version="2025-01-01T12-00-00", + force=False, + ) + + assert result is True + + # Verify nested directory was created + expected_file = os.path.join( + tmpdir, "2025-01-01T12-00-00", "subdir", "nested", "file.txt" + ) + assert os.path.exists(expected_file) + + with open(expected_file, "rb") as f: + assert f.read() == test_content + + def test_download_version_to_local_latest(self, mocker, blob_endpoint): + """Test downloading the latest version""" + test_content = b"latest version content" + + def mock_read_blob_stream(blob_url, account_name, container_name): + return test_content + + mocker.patch( + "cfa.dataops.catalog.read_blob_stream", + side_effect=mock_read_blob_stream, + ) + + mocker.patch.object( + blob_endpoint, + "get_versions", + return_value=["2025-01-02T12-00-00", "2025-01-01T12-00-00"], + ) + + mocker.patch.object( + blob_endpoint, + "_get_version_blobs", + return_value=[ + { + "name": "test/prefix/2025-01-02T12-00-00/data.csv", + "creation_time": "2025-01-02T12:00:00", + } + ], + ) + + with tempfile.TemporaryDirectory() as tmpdir: + result = blob_endpoint.download_version_to_local( + local_path=tmpdir, + version="latest", + force=False, + ) + + assert result is True + + # Verify the latest version was downloaded + expected_file = os.path.join( + tmpdir, "2025-01-02T12-00-00", "data.csv" + ) + assert os.path.exists(expected_file) + + def test_download_version_to_local_no_redownload_without_force( + self, mocker, blob_endpoint + ): + """Test that existing files are not re-downloaded without force flag""" + test_content = b"test content" + + def mock_read_blob_stream(blob_url, account_name, container_name): + return test_content + + mocker.patch( + "cfa.dataops.catalog.read_blob_stream", + side_effect=mock_read_blob_stream, + ) + + mocker.patch.object( + blob_endpoint, + "_get_version_blobs", + return_value=[ + { + "name": "test/prefix/2025-01-01T12-00-00/data.csv", + "creation_time": "2025-01-01T12:00:00", + } + ], + ) + + with tempfile.TemporaryDirectory() as tmpdir: + # Create the file beforehand + expected_file = os.path.join( + tmpdir, "2025-01-01T12-00-00", "data.csv" + ) + os.makedirs(os.path.dirname(expected_file), exist_ok=True) + existing_content = b"existing content" + with open(expected_file, "wb") as f: + f.write(existing_content) + + result = blob_endpoint.download_version_to_local( + local_path=tmpdir, + version="2025-01-01T12-00-00", + force=False, + ) + + # Should return False since no files were written + assert result is False + + # Verify file content is unchanged + with open(expected_file, "rb") as f: + assert f.read() == existing_content + + def test_download_version_to_local_force_redownload( + self, mocker, blob_endpoint + ): + """Test that existing files are re-downloaded with force flag""" + test_content = b"new test content" + + def mock_read_blob_stream(blob_url, account_name, container_name): + return test_content + + mocker.patch( + "cfa.dataops.catalog.read_blob_stream", + side_effect=mock_read_blob_stream, + ) + + mocker.patch.object( + blob_endpoint, + "_get_version_blobs", + return_value=[ + { + "name": "test/prefix/2025-01-01T12-00-00/data.csv", + "creation_time": "2025-01-01T12:00:00", + } + ], + ) + + with tempfile.TemporaryDirectory() as tmpdir: + # Create the file beforehand with different content + expected_file = os.path.join( + tmpdir, "2025-01-01T12-00-00", "data.csv" + ) + os.makedirs(os.path.dirname(expected_file), exist_ok=True) + with open(expected_file, "wb") as f: + f.write(b"old content") + + result = blob_endpoint.download_version_to_local( + local_path=tmpdir, + version="2025-01-01T12-00-00", + force=True, + ) + + assert result is True + + # Verify file content was updated + with open(expected_file, "rb") as f: + assert f.read() == test_content + + def test_download_version_to_local_creates_directories( + self, mocker, blob_endpoint + ): + """Test that download creates necessary directories""" + test_content = b"test content" + + def mock_read_blob_stream(blob_url, account_name, container_name): + return test_content + + mocker.patch( + "cfa.dataops.catalog.read_blob_stream", + side_effect=mock_read_blob_stream, + ) + + mocker.patch.object( + blob_endpoint, + "_get_version_blobs", + return_value=[ + { + "name": "test/prefix/2025-01-01T12-00-00/deep/nested/path/data.csv", + "creation_time": "2025-01-01T12:00:00", + } + ], + ) + + with tempfile.TemporaryDirectory() as tmpdir: + result = blob_endpoint.download_version_to_local( + local_path=tmpdir, + version="2025-01-01T12-00-00", + force=False, + ) + + assert result is True + + # Verify nested directories were created + expected_file = os.path.join( + tmpdir, + "2025-01-01T12-00-00", + "deep", + "nested", + "path", + "data.csv", + ) + assert os.path.exists(expected_file) + assert os.path.isfile(expected_file) + + def test_download_version_to_local_ledger_entry( + self, mocker, blob_endpoint + ): + """Test that ledger entry is created when files are written""" + test_content = b"test content" + + def mock_read_blob_stream(blob_url, account_name, container_name): + return test_content + + mocker.patch( + "cfa.dataops.catalog.read_blob_stream", + side_effect=mock_read_blob_stream, + ) + + mocker.patch.object( + blob_endpoint, + "_get_version_blobs", + return_value=[ + { + "name": "test/prefix/2025-01-01T12-00-00/data.csv", + "creation_time": "2025-01-01T12:00:00", + } + ], + ) + + # Mock ledger_entry to track if it was called + mock_ledger = mocker.patch.object(blob_endpoint, "ledger_entry") + + with tempfile.TemporaryDirectory() as tmpdir: + blob_endpoint.download_version_to_local( + local_path=tmpdir, + version="2025-01-01T12-00-00", + force=False, + ) + + # Verify ledger entry was called with 'read' action + mock_ledger.assert_called_once_with(action="read") + + def test_download_version_to_local_no_ledger_entry_when_not_written( + self, mocker, blob_endpoint + ): + """Test that no ledger entry is created when no files are written""" + test_content = b"test content" + + def mock_read_blob_stream(blob_url, account_name, container_name): + return test_content + + mocker.patch( + "cfa.dataops.catalog.read_blob_stream", + side_effect=mock_read_blob_stream, + ) + + mocker.patch.object( + blob_endpoint, + "_get_version_blobs", + return_value=[ + { + "name": "test/prefix/2025-01-01T12-00-00/data.csv", + "creation_time": "2025-01-01T12:00:00", + } + ], + ) + + # Mock ledger_entry to track if it was called + mock_ledger = mocker.patch.object(blob_endpoint, "ledger_entry") + + with tempfile.TemporaryDirectory() as tmpdir: + # Create the file beforehand + expected_file = os.path.join( + tmpdir, "2025-01-01T12-00-00", "data.csv" + ) + os.makedirs(os.path.dirname(expected_file), exist_ok=True) + with open(expected_file, "wb") as f: + f.write(b"existing content") + + blob_endpoint.download_version_to_local( + local_path=tmpdir, + version="2025-01-01T12-00-00", + force=False, + ) + + # Verify ledger entry was NOT called + mock_ledger.assert_not_called() + + def test_download_version_to_local_mixed_new_and_existing_files( + self, mocker, blob_endpoint + ): + """Test downloading when some files exist and some don't""" + test_content_1 = b"content 1" + test_content_2 = b"content 2" + + def mock_read_blob_stream(blob_url, account_name, container_name): + if "file1.txt" in blob_url: + return test_content_1 + elif "file2.txt" in blob_url: + return test_content_2 + + mocker.patch( + "cfa.dataops.catalog.read_blob_stream", + side_effect=mock_read_blob_stream, + ) + + mocker.patch.object( + blob_endpoint, + "_get_version_blobs", + return_value=[ + { + "name": "test/prefix/2025-01-01T12-00-00/file1.txt", + "creation_time": "2025-01-01T12:00:00", + }, + { + "name": "test/prefix/2025-01-01T12-00-00/file2.txt", + "creation_time": "2025-01-01T12:00:01", + }, + ], + ) + + with tempfile.TemporaryDirectory() as tmpdir: + # Create file1 beforehand + file1 = os.path.join(tmpdir, "2025-01-01T12-00-00", "file1.txt") + os.makedirs(os.path.dirname(file1), exist_ok=True) + with open(file1, "wb") as f: + f.write(b"existing") + + result = blob_endpoint.download_version_to_local( + local_path=tmpdir, + version="2025-01-01T12-00-00", + force=False, + ) + + # Should return True because file2 was written + assert result is True + + # Verify file1 wasn't overwritten + with open(file1, "rb") as f: + assert f.read() == b"existing" + + # Verify file2 was created + file2 = os.path.join(tmpdir, "2025-01-01T12-00-00", "file2.txt") + assert os.path.exists(file2) + with open(file2, "rb") as f: + assert f.read() == test_content_2 diff --git a/tests/test_command.py b/tests/test_command.py new file mode 100644 index 0000000..f397d21 --- /dev/null +++ b/tests/test_command.py @@ -0,0 +1,173 @@ +"""Tests for command.py functions""" + +from unittest.mock import MagicMock + +import pytest + +from cfa.dataops.command import ( + _get_dataset_namespaces, + _get_stages_list, + _get_versions_list, +) + + +@pytest.fixture +def mock_datacat(mocker): + """Create a mock datacat object for testing""" + # Create a custom namespace that behaves like the real datacat + from types import SimpleNamespace + + # Create mock stages + mock_extract = MagicMock() + mock_load = MagicMock() + mock_load.get_versions.return_value = [ + "2025-01-02T12-00-00", + "2025-01-01T12-00-00", + ] + mock_load.download_version_to_local.return_value = True + mock_stage_01 = MagicMock() + + # Create dataset with __dict__ for stages + dataset1 = SimpleNamespace( + extract=mock_extract, + load=mock_load, + stage_01=mock_stage_01, + _ledger_endpoint=MagicMock(), + ) + + # Create test namespace + test_ns = SimpleNamespace(dataset1=dataset1, dataset2=MagicMock()) + + # Create prod namespace + prod_ns = SimpleNamespace(dataset1=MagicMock()) + + # Create main catalog + catalog = SimpleNamespace( + test=test_ns, + prod=prod_ns, + __namespace_list__=[ + "test.dataset1", + "test.dataset2", + "prod.dataset1", + ], + ) + + mocker.patch("cfa.dataops.command.datacat", catalog) + return catalog + + +class TestGetDatasetNamespaces: + """Tests for _get_dataset_namespaces helper function""" + + def test_get_dataset_namespaces(self, mock_datacat): + """Test getting dataset namespaces""" + result = _get_dataset_namespaces() + + assert isinstance(result, list) + assert "test.dataset1" in result + assert "test.dataset2" in result + assert "prod.dataset1" in result + + +class TestGetStagesList: + """Tests for _get_stages_list helper function""" + + def test_get_stages_list_valid_dataset(self, mock_datacat, capsys): + """Test getting stages for a valid dataset""" + result = _get_stages_list("test.dataset1") + + assert isinstance(result, list) + assert "extract" in result + assert "load" in result + assert "stage_01" in result + assert "_ledger_endpoint" not in result # Should be filtered out + + # Verify stages are sorted + assert result == sorted(result) + + def test_get_stages_list_invalid_dataset(self, mock_datacat, capsys): + """Test getting stages for invalid dataset prints error""" + result = _get_stages_list("nonexistent.dataset") + + assert result is None + + # Check error message was printed + captured = capsys.readouterr() + assert "Error" in captured.out + assert ( + "Dataset namespace 'nonexistent.dataset' not found" in captured.out + ) + + def test_get_stages_list_filters_correct_prefixes(self, mock_datacat): + """Test that only stages with correct prefixes are returned""" + # Add a mock dataset with various keys + mock_dataset = MagicMock() + mock_dataset.__dict__ = { + "extract": MagicMock(), + "load": MagicMock(), + "stage_01": MagicMock(), + "stage_02": MagicMock(), + "config": {}, # Should be filtered + "metadata": {}, # Should be filtered + } + mock_datacat.test.dataset2 = mock_dataset + + result = _get_stages_list("test.dataset2") + + assert "extract" in result + assert "load" in result + assert "stage_01" in result + assert "stage_02" in result + assert "config" not in result + assert "metadata" not in result + + +class TestGetVersionsList: + """Tests for _get_versions_list helper function""" + + def test_get_versions_list_valid_stage(self, mock_datacat, capsys): + """Test getting versions for a valid stage""" + result = _get_versions_list("test.dataset1", "load") + + assert isinstance(result, list) + assert "2025-01-02T12-00-00" in result + assert "2025-01-01T12-00-00" in result + + def test_get_versions_list_invalid_stage(self, mock_datacat, capsys): + """Test getting versions for invalid stage prints error""" + result = _get_versions_list("test.dataset1", "nonexistent_stage") + + assert result is None + + # Check error message was printed + captured = capsys.readouterr() + assert "Error" in captured.out + assert "Stage 'nonexistent_stage' not found" in captured.out + + +class TestHelperFunctionsIntegration: + """Integration tests for command helper functions""" + + def test_stages_sorted_alphabetically(self, mock_datacat): + """Test that stages are returned in sorted order""" + # Create a dataset with unsorted stages + mock_dataset = MagicMock() + mock_dataset.__dict__ = { + "stage_03": MagicMock(), + "load": MagicMock(), + "stage_01": MagicMock(), + "extract": MagicMock(), + "stage_02": MagicMock(), + } + mock_datacat.test.dataset2 = mock_dataset + + result = _get_stages_list("test.dataset2") + + expected_order = [ + "extract", + "load", + "stage_01", + "stage_02", + "stage_03", + ] + assert result == expected_order diff --git a/tests/test_utils_tree.py b/tests/test_utils_tree.py new file mode 100644 index 0000000..9964f21 --- /dev/null +++ b/tests/test_utils_tree.py @@ -0,0 +1,310 @@ +"""Tests for utils.tree function""" + +import os +import tempfile +from pathlib import Path + +from cfa.dataops.utils import tree + + +class TestTreeFunction: + """Tests for the tree function""" + + def test_tree_simple_directory(self): + """Test tree output for a simple directory structure""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create a simple structure + Path(tmpdir, "file1.txt").touch() + Path(tmpdir, "file2.txt").touch() + + result = tree(tmpdir, level=-1) + + # Should contain the directory name and files + assert os.path.basename(tmpdir) in result + assert "file1.txt" in result + assert "file2.txt" in result + assert "2 files" in result + + def test_tree_nested_directories(self): + """Test tree output for nested directory structure""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create nested structure + subdir = Path(tmpdir, "subdir") + subdir.mkdir() + Path(tmpdir, "file1.txt").touch() + Path(subdir, "file2.txt").touch() + + result = tree(tmpdir, level=-1) + + assert "subdir" in result + assert "file1.txt" in result + assert "file2.txt" in result + assert "1 directories" in result or "1 directory" in result + + def test_tree_level_limit(self): + """Test tree respects level parameter""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create deep nested structure + level1 = Path(tmpdir, "level1") + level2 = Path(level1, "level2") + level3 = Path(level2, "level3") + + level1.mkdir() + level2.mkdir() + level3.mkdir() + + Path(tmpdir, "file0.txt").touch() + Path(level1, "file1.txt").touch() + Path(level2, "file2.txt").touch() + Path(level3, "file3.txt").touch() + + # Level 1 should only show first level (directories and files at root) + result = tree(tmpdir, level=1) + assert "level1" in result + assert "file0.txt" in result + # Should not show deeper levels (level 1 means one level of recursion into level1) + assert "level2" not in result + assert "file2.txt" not in result + assert "file3.txt" not in result + + def test_tree_limit_to_directories(self): + """Test tree with limit_to_directories option""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create structure with both files and directories + subdir = Path(tmpdir, "subdir") + subdir.mkdir() + Path(tmpdir, "file1.txt").touch() + Path(tmpdir, "file2.txt").touch() + Path(subdir, "file3.txt").touch() + + result = tree(tmpdir, limit_to_directories=True) + + # Should show directory + assert "subdir" in result + # Should not show files + assert "file1.txt" not in result + assert "file2.txt" not in result + assert "file3.txt" not in result + # Should only count directories, not files + assert "1 directories" in result or "1 directory" in result + assert "files" not in result or "0 files" in result + + def test_tree_show_hidden_false(self): + """Test tree hides hidden files by default""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create normal and hidden files + Path(tmpdir, "visible.txt").touch() + Path(tmpdir, ".hidden.txt").touch() + hidden_dir = Path(tmpdir, ".hidden_dir") + hidden_dir.mkdir() + Path(hidden_dir, "file.txt").touch() + + result = tree(tmpdir, show_hidden=False) + + # Should show visible file + assert "visible.txt" in result + # Should not show hidden files/directories + assert ".hidden.txt" not in result + assert ".hidden_dir" not in result + + def test_tree_show_hidden_true(self): + """Test tree shows hidden files when requested""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create normal and hidden files + Path(tmpdir, "visible.txt").touch() + Path(tmpdir, ".hidden.txt").touch() + hidden_dir = Path(tmpdir, ".hidden_dir") + hidden_dir.mkdir() + + result = tree(tmpdir, show_hidden=True) + + # Should show all files + assert "visible.txt" in result + assert ".hidden.txt" in result + assert ".hidden_dir" in result + + def test_tree_length_limit(self): + """Test tree respects length_limit parameter""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create many files + for i in range(20): + Path(tmpdir, f"file{i:02d}.txt").touch() + + result = tree(tmpdir, length_limit=5) + + # Should contain the limit message + assert "length_limit" in result + assert "5, reached" in result + + def test_tree_empty_directory(self): + """Test tree output for an empty directory""" + with tempfile.TemporaryDirectory() as tmpdir: + result = tree(tmpdir) + + # Should contain directory name and count + assert os.path.basename(tmpdir) in result + assert "0 directories" in result + + def test_tree_directory_counts(self): + """Test tree correctly counts directories and files""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create known structure + Path(tmpdir, "dir1").mkdir() + Path(tmpdir, "dir2").mkdir() + Path(tmpdir, "dir3").mkdir() + Path(tmpdir, "file1.txt").touch() + Path(tmpdir, "file2.txt").touch() + + result = tree(tmpdir) + + assert "3 directories" in result + assert "2 files" in result + + def test_tree_visual_structure(self): + """Test tree produces correct visual structure with branches""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create specific structure to test visual elements + Path(tmpdir, "file1.txt").touch() + Path(tmpdir, "file2.txt").touch() + + result = tree(tmpdir) + + # Should contain tree drawing characters + assert "├──" in result or "└──" in result + + def test_tree_accepts_string_path(self): + """Test tree accepts string paths in addition to Path objects""" + with tempfile.TemporaryDirectory() as tmpdir: + Path(tmpdir, "test.txt").touch() + + # Should work with string path + result = tree(str(tmpdir)) + + assert "test.txt" in result + + def test_tree_accepts_path_object(self): + """Test tree accepts Path objects""" + with tempfile.TemporaryDirectory() as tmpdir: + Path(tmpdir, "test.txt").touch() + + # Should work with Path object + result = tree(Path(tmpdir)) + + assert "test.txt" in result + + def test_tree_deep_nesting(self): + """Test tree handles deep directory nesting""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create a deeply nested structure + current = Path(tmpdir) + for i in range(5): + current = current / f"level{i}" + current.mkdir(parents=True) + Path(current, f"file{i}.txt").touch() + + result = tree(tmpdir, level=-1) + + # Should show all levels + for i in range(5): + assert f"level{i}" in result + assert f"file{i}.txt" in result + + def test_tree_mixed_files_and_dirs(self): + """Test tree with mixed files and directories at various levels""" + with tempfile.TemporaryDirectory() as tmpdir: + # Root level + Path(tmpdir, "root_file.txt").touch() + + # First level + dir1 = Path(tmpdir, "dir1") + dir1.mkdir() + Path(dir1, "file_in_dir1.txt").touch() + + # Second level + dir2 = Path(dir1, "dir2") + dir2.mkdir() + Path(dir2, "file_in_dir2.txt").touch() + + result = tree(tmpdir, level=-1) + + assert "root_file.txt" in result + assert "dir1" in result + assert "file_in_dir1.txt" in result + assert "dir2" in result + assert "file_in_dir2.txt" in result + + def test_tree_level_zero(self): + """Test tree with level=0 shows nothing beyond root""" + with tempfile.TemporaryDirectory() as tmpdir: + Path(tmpdir, "file.txt").touch() + subdir = Path(tmpdir, "subdir") + subdir.mkdir() + + result = tree(tmpdir, level=0) + + # Should only show the root directory name + assert os.path.basename(tmpdir) in result + # Should not traverse into contents + assert "file.txt" not in result + assert "subdir" not in result + + def test_tree_multiple_subdirectories(self): + """Test tree displays multiple subdirectories correctly""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create multiple subdirectories + for i in range(4): + subdir = Path(tmpdir, f"subdir_{i}") + subdir.mkdir() + Path(subdir, f"file_{i}.txt").touch() + + result = tree(tmpdir) + + # All subdirs should be present + for i in range(4): + assert f"subdir_{i}" in result + assert f"file_{i}.txt" in result + + def test_tree_special_characters_in_names(self): + """Test tree handles files/directories with special characters""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create files with special characters (safe for filesystem) + Path(tmpdir, "file-with-dash.txt").touch() + Path(tmpdir, "file_with_underscore.txt").touch() + Path(tmpdir, "file with spaces.txt").touch() + + result = tree(tmpdir) + + assert "file-with-dash.txt" in result + assert "file_with_underscore.txt" in result + assert "file with spaces.txt" in result + + def test_tree_returns_string(self): + """Test that tree returns a string""" + with tempfile.TemporaryDirectory() as tmpdir: + result = tree(tmpdir) + assert isinstance(result, str) + + def test_tree_single_file_no_directories(self): + """Test tree with only a single file""" + with tempfile.TemporaryDirectory() as tmpdir: + Path(tmpdir, "single_file.txt").touch() + + result = tree(tmpdir) + + assert "single_file.txt" in result + assert "0 directories" in result + assert "1 files" in result or "1 file" in result + + def test_tree_only_directories_no_files(self): + """Test tree with only directories, no files""" + with tempfile.TemporaryDirectory() as tmpdir: + Path(tmpdir, "dir1").mkdir() + Path(tmpdir, "dir2").mkdir() + Path(tmpdir, "dir3").mkdir() + + result = tree(tmpdir) + + assert "dir1" in result + assert "dir2" in result + assert "dir3" in result + assert "3 directories" in result