diff --git a/src/basic_memory/cli/commands/cloud/upload.py b/src/basic_memory/cli/commands/cloud/upload.py index 87cc6d6fa..5add0a785 100644 --- a/src/basic_memory/cli/commands/cloud/upload.py +++ b/src/basic_memory/cli/commands/cloud/upload.py @@ -11,13 +11,22 @@ from basic_memory.mcp.tools.utils import call_put -async def upload_path(local_path: Path, project_name: str) -> bool: +async def upload_path( + local_path: Path, + project_name: str, + verbose: bool = False, + use_gitignore: bool = True, + dry_run: bool = False, +) -> bool: """ Upload a file or directory to cloud project via WebDAV. Args: local_path: Path to local file or directory project_name: Name of cloud project (destination) + verbose: Show detailed information about filtering and upload + use_gitignore: If False, skip .gitignore patterns (still use .bmignore) + dry_run: If True, show what would be uploaded without uploading Returns: True if upload succeeded, False otherwise @@ -34,35 +43,54 @@ async def upload_path(local_path: Path, project_name: str) -> bool: # Get files to upload if local_path.is_file(): files_to_upload = [(local_path, local_path.name)] + if verbose: + print(f"Uploading single file: {local_path.name}") else: - files_to_upload = _get_files_to_upload(local_path) + files_to_upload = _get_files_to_upload(local_path, verbose, use_gitignore) if not files_to_upload: print("No files found to upload") + if verbose: + print( + "\nTip: Use --verbose to see which files are being filtered, " + "or --no-gitignore to skip .gitignore patterns" + ) return True print(f"Found {len(files_to_upload)} file(s) to upload") - # Upload files using httpx - total_bytes = 0 - - async with get_client() as client: - for i, (file_path, relative_path) in enumerate(files_to_upload, 1): - # Build remote path: /webdav/{project_name}/{relative_path} - remote_path = f"/webdav/{project_name}/{relative_path}" - print(f"Uploading {relative_path} ({i}/{len(files_to_upload)})") - - # Read file content asynchronously - async with aiofiles.open(file_path, "rb") as f: - content = await f.read() - - # Upload via HTTP PUT to WebDAV endpoint - response = await call_put(client, remote_path, content=content) - response.raise_for_status() - - total_bytes += file_path.stat().st_size - - # Format size based on magnitude + # Calculate total size + total_bytes = sum(file_path.stat().st_size for file_path, _ in files_to_upload) + + # If dry run, just show what would be uploaded + if dry_run: + print("\nFiles that would be uploaded:") + for file_path, relative_path in files_to_upload: + size = file_path.stat().st_size + if size < 1024: + size_str = f"{size} bytes" + elif size < 1024 * 1024: + size_str = f"{size / 1024:.1f} KB" + else: + size_str = f"{size / (1024 * 1024):.1f} MB" + print(f" {relative_path} ({size_str})") + else: + # Upload files using httpx + async with get_client() as client: + for i, (file_path, relative_path) in enumerate(files_to_upload, 1): + # Build remote path: /webdav/{project_name}/{relative_path} + remote_path = f"/webdav/{project_name}/{relative_path}" + print(f"Uploading {relative_path} ({i}/{len(files_to_upload)})") + + # Read file content asynchronously + async with aiofiles.open(file_path, "rb") as f: + content = await f.read() + + # Upload via HTTP PUT to WebDAV endpoint + response = await call_put(client, remote_path, content=content) + response.raise_for_status() + + # Format total size based on magnitude if total_bytes < 1024: size_str = f"{total_bytes} bytes" elif total_bytes < 1024 * 1024: @@ -70,7 +98,11 @@ async def upload_path(local_path: Path, project_name: str) -> bool: else: size_str = f"{total_bytes / (1024 * 1024):.1f} MB" - print(f"✓ Upload complete: {len(files_to_upload)} file(s) ({size_str})") + if dry_run: + print(f"\nTotal: {len(files_to_upload)} file(s) ({size_str})") + else: + print(f"✓ Upload complete: {len(files_to_upload)} file(s) ({size_str})") + return True except httpx.HTTPStatusError as e: @@ -81,22 +113,38 @@ async def upload_path(local_path: Path, project_name: str) -> bool: return False -def _get_files_to_upload(directory: Path) -> list[tuple[Path, str]]: +def _get_files_to_upload( + directory: Path, verbose: bool = False, use_gitignore: bool = True +) -> list[tuple[Path, str]]: """ Get list of files to upload from directory. - Uses .bmignore and .gitignore patterns for filtering. + Uses .bmignore and optionally .gitignore patterns for filtering. Args: directory: Directory to scan + verbose: Show detailed filtering information + use_gitignore: If False, skip .gitignore patterns (still use .bmignore) Returns: List of (absolute_path, relative_path) tuples """ files = [] - - # Load ignore patterns from .bmignore and .gitignore - ignore_patterns = load_gitignore_patterns(directory) + ignored_files = [] + + # Load ignore patterns from .bmignore and optionally .gitignore + ignore_patterns = load_gitignore_patterns(directory, use_gitignore=use_gitignore) + + if verbose: + gitignore_path = directory / ".gitignore" + gitignore_exists = gitignore_path.exists() and use_gitignore + print(f"\nScanning directory: {directory}") + print("Using .bmignore: Yes") + print(f"Using .gitignore: {'Yes' if gitignore_exists else 'No'}") + print(f"Ignore patterns loaded: {len(ignore_patterns)}") + if ignore_patterns and len(ignore_patterns) <= 20: + print(f"Patterns: {', '.join(sorted(ignore_patterns))}") + print() # Walk through directory for root, dirs, filenames in os.walk(directory): @@ -106,7 +154,11 @@ def _get_files_to_upload(directory: Path) -> list[tuple[Path, str]]: filtered_dirs = [] for d in dirs: dir_path = root_path / d - if not should_ignore_path(dir_path, directory, ignore_patterns): + if should_ignore_path(dir_path, directory, ignore_patterns): + if verbose: + rel_path = dir_path.relative_to(directory) + print(f" [IGNORED DIR] {rel_path}/") + else: filtered_dirs.append(d) dirs[:] = filtered_dirs @@ -114,15 +166,25 @@ def _get_files_to_upload(directory: Path) -> list[tuple[Path, str]]: for filename in filenames: file_path = root_path / filename + # Calculate relative path for display/remote + rel_path = file_path.relative_to(directory) + remote_path = str(rel_path).replace("\\", "/") + # Check if file should be ignored if should_ignore_path(file_path, directory, ignore_patterns): + ignored_files.append(remote_path) + if verbose: + print(f" [IGNORED] {remote_path}") continue - # Calculate relative path for remote - rel_path = file_path.relative_to(directory) - # Use forward slashes for WebDAV paths - remote_path = str(rel_path).replace("\\", "/") + if verbose: + print(f" [INCLUDE] {remote_path}") files.append((file_path, remote_path)) + if verbose: + print("\nSummary:") + print(f" Files to upload: {len(files)}") + print(f" Files ignored: {len(ignored_files)}") + return files diff --git a/src/basic_memory/cli/commands/cloud/upload_command.py b/src/basic_memory/cli/commands/cloud/upload_command.py index d88f5ab53..59a279d8d 100644 --- a/src/basic_memory/cli/commands/cloud/upload_command.py +++ b/src/basic_memory/cli/commands/cloud/upload_command.py @@ -43,6 +43,22 @@ def upload( "--sync/--no-sync", help="Sync project after upload (default: true)", ), + verbose: bool = typer.Option( + False, + "--verbose", + "-v", + help="Show detailed information about file filtering and upload", + ), + no_gitignore: bool = typer.Option( + False, + "--no-gitignore", + help="Skip .gitignore patterns (still respects .bmignore)", + ), + dry_run: bool = typer.Option( + False, + "--dry-run", + help="Show what would be uploaded without actually uploading", + ), ) -> None: """Upload local files or directories to cloud project via WebDAV. @@ -50,6 +66,9 @@ def upload( bm cloud upload ~/my-notes --project research bm cloud upload notes.md --project research --create-project bm cloud upload ~/docs --project work --no-sync + bm cloud upload ./history --project proto --verbose + bm cloud upload ./notes --project work --no-gitignore + bm cloud upload ./files --project test --dry-run """ async def _upload(): @@ -72,17 +91,28 @@ async def _upload(): ) raise typer.Exit(1) - # Perform upload - console.print(f"[blue]Uploading {path} to project '{project}'...[/blue]") - success = await upload_path(path, project) + # Perform upload (or dry run) + if dry_run: + console.print( + f"[yellow]DRY RUN: Showing what would be uploaded to '{project}'[/yellow]" + ) + else: + console.print(f"[blue]Uploading {path} to project '{project}'...[/blue]") + + success = await upload_path( + path, project, verbose=verbose, use_gitignore=not no_gitignore, dry_run=dry_run + ) if not success: console.print("[red]Upload failed[/red]") raise typer.Exit(1) - console.print(f"[green]✅ Successfully uploaded to '{project}'[/green]") + if dry_run: + console.print("[yellow]DRY RUN complete - no files were uploaded[/yellow]") + else: + console.print(f"[green]✅ Successfully uploaded to '{project}'[/green]") - # Sync project if requested - if sync: + # Sync project if requested (skip on dry run) + if sync and not dry_run: console.print(f"[blue]Syncing project '{project}'...[/blue]") try: await sync_project(project) diff --git a/src/basic_memory/ignore_utils.py b/src/basic_memory/ignore_utils.py index 54c6b4888..b4954a063 100644 --- a/src/basic_memory/ignore_utils.py +++ b/src/basic_memory/ignore_utils.py @@ -172,15 +172,16 @@ def load_bmignore_patterns() -> Set[str]: return patterns -def load_gitignore_patterns(base_path: Path) -> Set[str]: +def load_gitignore_patterns(base_path: Path, use_gitignore: bool = True) -> Set[str]: """Load gitignore patterns from .gitignore file and .bmignore. Combines patterns from: 1. ~/.basic-memory/.bmignore (user's global ignore patterns) - 2. {base_path}/.gitignore (project-specific patterns) + 2. {base_path}/.gitignore (project-specific patterns, if use_gitignore=True) Args: base_path: The base directory to search for .gitignore file + use_gitignore: If False, only load patterns from .bmignore (default: True) Returns: Set of patterns to ignore @@ -188,18 +189,19 @@ def load_gitignore_patterns(base_path: Path) -> Set[str]: # Start with patterns from .bmignore patterns = load_bmignore_patterns() - gitignore_file = base_path / ".gitignore" - if gitignore_file.exists(): - try: - with gitignore_file.open("r", encoding="utf-8") as f: - for line in f: - line = line.strip() - # Skip empty lines and comments - if line and not line.startswith("#"): - patterns.add(line) - except Exception: - # If we can't read .gitignore, just use default patterns - pass + if use_gitignore: + gitignore_file = base_path / ".gitignore" + if gitignore_file.exists(): + try: + with gitignore_file.open("r", encoding="utf-8") as f: + for line in f: + line = line.strip() + # Skip empty lines and comments + if line and not line.startswith("#"): + patterns.add(line) + except Exception: + # If we can't read .gitignore, just use default patterns + pass return patterns diff --git a/tests/cli/test_upload.py b/tests/cli/test_upload.py index 8a9d780a6..46a660277 100644 --- a/tests/cli/test_upload.py +++ b/tests/cli/test_upload.py @@ -20,7 +20,7 @@ def test_collects_files_from_directory(self, tmp_path): (tmp_path / "subdir" / "file3.py").write_text("content3") # Call with real ignore utils (no mocking) - result = _get_files_to_upload(tmp_path) + result = _get_files_to_upload(tmp_path, verbose=False, use_gitignore=True) # Should find all 3 files assert len(result) == 3 @@ -326,3 +326,138 @@ async def test_builds_correct_webdav_path(self, tmp_path): mock_put.assert_called_once() call_args = mock_put.call_args assert call_args[0][1] == "/webdav/my-project/subdir/file.txt" + + def test_no_gitignore_skips_gitignore_patterns(self, tmp_path): + """Test that --no-gitignore flag skips .gitignore patterns.""" + # Create test files + (tmp_path / "keep.txt").write_text("keep") + (tmp_path / "secret.bak").write_text("secret") # Use .bak instead of .pyc + + # Create .gitignore file that ignores .bak files + gitignore_file = tmp_path / ".gitignore" + gitignore_file.write_text("*.bak\n") + + # With use_gitignore=False, should include .bak files + result = _get_files_to_upload(tmp_path, verbose=False, use_gitignore=False) + + # Extract relative paths + relative_paths = [rel_path for _, rel_path in result] + + # Both files should be included when gitignore is disabled + assert "keep.txt" in relative_paths + assert "secret.bak" in relative_paths + + def test_no_gitignore_still_respects_bmignore(self, tmp_path): + """Test that --no-gitignore still respects .bmignore patterns.""" + # Create test files + (tmp_path / "keep.txt").write_text("keep") + (tmp_path / ".hidden").write_text( + "hidden" + ) # Should be ignored by .bmignore default pattern + + # Create .gitignore that would allow .hidden + gitignore_file = tmp_path / ".gitignore" + gitignore_file.write_text("# Allow all\n") + + # With use_gitignore=False, should still filter hidden files via .bmignore + result = _get_files_to_upload(tmp_path, verbose=False, use_gitignore=False) + + # Extract relative paths + relative_paths = [rel_path for _, rel_path in result] + + # keep.txt should be included, .hidden should be filtered by .bmignore + assert "keep.txt" in relative_paths + assert ".hidden" not in relative_paths + + def test_verbose_shows_filtering_info(self, tmp_path, capsys): + """Test that verbose mode shows filtering information.""" + # Create test files + (tmp_path / "keep.txt").write_text("keep") + (tmp_path / "ignore.pyc").write_text("ignore") + + # Create .gitignore + gitignore_file = tmp_path / ".gitignore" + gitignore_file.write_text("*.pyc\n") + + # Run with verbose=True + _get_files_to_upload(tmp_path, verbose=True, use_gitignore=True) + + # Capture output + captured = capsys.readouterr() + + # Should show scanning information + assert "Scanning directory:" in captured.out + assert "Using .bmignore: Yes" in captured.out + assert "Using .gitignore:" in captured.out + assert "Ignore patterns loaded:" in captured.out + + # Should show file status + assert "[INCLUDE]" in captured.out or "[IGNORED]" in captured.out + + # Should show summary + assert "Summary:" in captured.out + assert "Files to upload:" in captured.out + assert "Files ignored:" in captured.out + + def test_wildcard_gitignore_filters_all_files(self, tmp_path): + """Test that a wildcard * in .gitignore filters all files.""" + # Create test files + (tmp_path / "file1.txt").write_text("content1") + (tmp_path / "file2.md").write_text("content2") + + # Create .gitignore with wildcard + gitignore_file = tmp_path / ".gitignore" + gitignore_file.write_text("*\n") + + # Should filter all files + result = _get_files_to_upload(tmp_path, verbose=False, use_gitignore=True) + assert len(result) == 0 + + # With use_gitignore=False, should include files + result = _get_files_to_upload(tmp_path, verbose=False, use_gitignore=False) + assert len(result) == 2 + + @pytest.mark.asyncio + async def test_dry_run_shows_files_without_uploading(self, tmp_path, capsys): + """Test that --dry-run shows what would be uploaded without uploading.""" + # Create test files + (tmp_path / "file1.txt").write_text("content1") + (tmp_path / "file2.txt").write_text("content2") + + # Don't mock anything - we want to verify no actual upload happens + result = await upload_path(tmp_path, "test-project", dry_run=True) + + # Should return success + assert result is True + + # Check output shows dry run info + captured = capsys.readouterr() + assert "Found 2 file(s) to upload" in captured.out + assert "Files that would be uploaded:" in captured.out + assert "file1.txt" in captured.out + assert "file2.txt" in captured.out + assert "Total:" in captured.out + + @pytest.mark.asyncio + async def test_dry_run_with_verbose(self, tmp_path, capsys): + """Test that --dry-run works with --verbose.""" + # Create test files + (tmp_path / "keep.txt").write_text("keep") + (tmp_path / "ignore.pyc").write_text("ignore") + + # Create .gitignore + gitignore_file = tmp_path / ".gitignore" + gitignore_file.write_text("*.pyc\n") + + result = await upload_path(tmp_path, "test-project", verbose=True, dry_run=True) + + # Should return success + assert result is True + + # Check output shows both verbose and dry run info + captured = capsys.readouterr() + assert "Scanning directory:" in captured.out + assert "[INCLUDE] keep.txt" in captured.out + assert "[IGNORED] ignore.pyc" in captured.out + assert "Files that would be uploaded:" in captured.out + assert "keep.txt" in captured.out