diff --git a/docs/cloud-cli.md b/docs/cloud-cli.md index 98eeb96d7..236c3aec4 100644 --- a/docs/cloud-cli.md +++ b/docs/cloud-cli.md @@ -1,22 +1,39 @@ # Basic Memory Cloud CLI Guide -The Basic Memory Cloud CLI provides commands for interacting with Basic Memory Cloud instances, including authentication, project management, file synchronization, and local file access. This guide covers installation, configuration, and usage of the cloud features. +The Basic Memory Cloud CLI provides seamless integration between local and cloud knowledge bases using a **cloud mode toggle**. When cloud mode is enabled, all your regular `bm` commands work transparently with the cloud instead of locally. ## Overview The cloud CLI enables you to: -- Authenticate with Basic Memory Cloud using OAuth -- List and create projects on cloud instances -- Upload local files and directories to cloud projects via WebDAV -- Mount cloud files locally for real-time editing with rclone -- Check the health status of cloud instances -- Automatically filter uploads using gitignore patterns +- **Toggle cloud mode** with `bm cloud login` / `bm cloud logout` +- **Use regular commands in cloud mode**: `bm project`, `bm sync`, `bm tool` all work with cloud +- **Bidirectional sync** with rclone bisync (recommended for most users) +- **Direct file access** via rclone mount (alternative workflow) +- **Integrity verification** with `bm cloud check` +- **Automatic project creation** from local directories -## Authentication +## The Cloud Mode Paradigm -### Initial Setup +Basic Memory Cloud follows the **Dropbox/iCloud model** - a single cloud space containing all your projects, not per-project connections. -Before using cloud commands, you need to authenticate with Basic Memory Cloud: +**How it works:** +- One login per machine: `bm cloud login` +- One sync directory: `~/basic-memory-cloud-sync/` (all projects) +- Projects are folders within your cloud space +- All regular commands work in cloud mode + +**Why this model:** +- ✅ Single set of credentials (not N per project) +- ✅ One rclone process (not N processes) +- ✅ Familiar pattern (like Dropbox) +- ✅ Simple operations (setup once, sync anytime) +- ✅ Natural scaling (add projects = add folders) + +## Quick Start + +### 1. Enable Cloud Mode + +Authenticate and enable cloud mode for all commands: ```bash bm cloud login @@ -25,410 +42,587 @@ bm cloud login This command will: 1. Open your browser to the Basic Memory Cloud authentication page 2. Prompt you to authorize the CLI application -3. Store your authentication token locally for future use - -## Project Management +3. Store your authentication token locally +4. **Enable cloud mode** - all CLI commands now work against cloud -### Listing Projects +### 2. Set Up Sync -View all projects on a cloud instance: +Set up bidirectional file synchronization: ```bash -bm cloud project list +bm cloud setup ``` -Example output: -``` - Cloud Projects -┏━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Name ┃ Path ┃ -┡━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ my-research │ my-research │ -│ work-notes │ work-notes │ -└──────────────────┴──────────────────────────┘ +This will: +1. Install rclone automatically (if needed) +2. Configure sync credentials +3. Create `~/basic-memory-cloud-sync/` directory +4. Establish initial sync baseline -Found 2 project(s) -``` +**Alternative:** Use `bm cloud setup --mount` to set up mount instead of sync. -### Creating Projects +### 3. Verify Setup -Create a new project on a cloud instance: +Check that everything is working: ```bash -bm cloud project add my-new-project +bm cloud status ``` -Create and set as default: +You should see: +- `Mode: Cloud (enabled)` +- `Cloud instance is healthy` +- Bisync status showing `✓ Initialized` + +### 4. Start Using Cloud + +Now all your regular commands work with the cloud: + ```bash -bm cloud project add my-new-project --default -``` +# List cloud projects +bm project list -## File Upload +# Create cloud project +bm project add "my-research" -### Basic Upload +# Use MCP tools on cloud +bm tool write-note --title "Hello" --folder "my-research" --content "Test" -Upload files or directories to a cloud project: +# Sync with cloud +bm sync -```bash -# Upload a directory -bm cloud upload my-project /path/to/local/files +# Watch mode for continuous sync +bm sync --watch +``` + +### 5. Disable Cloud Mode -# Upload a single file -bm cloud upload my-project /path/to/file.md +Return to local mode: + +```bash +bm cloud logout ``` -### Upload Options +All commands now work locally again. -#### Timestamp Preservation +## Working with Cloud Projects -By default, file modification times are preserved during upload: +**Important:** When cloud mode is enabled, use regular `bm project` commands (not `bm cloud project`). + +### Listing Projects + +View all projects (cloud projects when cloud mode is enabled): ```bash -# Preserve timestamps (default) -bm cloud upload my-project ./docs +# In cloud mode - lists cloud projects +bm project list -# Don't preserve timestamps -bm cloud upload my-project ./docs --no-preserve-timestamps +# In local mode - lists local projects +bm project list ``` -#### Gitignore Filtering +### Creating Projects -The CLI automatically respects `.gitignore` patterns and includes smart defaults for development artifacts: +Create a new project (creates on cloud when cloud mode is enabled): ```bash -# Respect .gitignore and defaults (default behavior) -bm cloud upload my-project ./my-repo +# In cloud mode - creates cloud project +bm project add my-new-project -# Upload everything, ignore .gitignore -bm cloud upload my-project ./my-repo --no-gitignore +# Create and set as default +bm project add my-new-project --default ``` -**Default ignore patterns include:** -- `.git`, `.venv`, `venv`, `env`, `.env` -- `node_modules`, `__pycache__`, `.pytest_cache` -- `*.pyc`, `*.pyo`, `*.pyd` -- `.DS_Store`, `Thumbs.db` -- `.idea`, `.vscode` -- `build`, `dist`, `.tox`, `.cache` -- `.mypy_cache`, `.ruff_cache` +### Automatic Project Creation -### Upload Examples +**New in SPEC-9:** Projects are automatically created when you create local directories! ```bash -# Upload a local knowledge base -bm cloud upload my-research ~/Documents/research +# Create a local directory in your sync folder +mkdir ~/basic-memory-cloud-sync/new-project +echo "# Notes" > ~/basic-memory-cloud-sync/new-project/readme.md -# Upload specific documentation, preserving structure -bm cloud upload docs-project ./docs +# Sync - automatically creates cloud project +bm sync -# Upload without gitignore filtering for a complete backup -bm cloud upload backup-project ./ --no-gitignore +# Verify - project now exists on cloud +bm project list ``` -### Upload Output +This Dropbox-like workflow means you don't need to manually coordinate projects between local and cloud. + +## File Synchronization -During upload, you'll see progress and filtering information: +### The `bm sync` Command (Cloud Mode Aware) +The `bm sync` command automatically adapts based on cloud mode: + +**In local mode:** ```bash -$ bm cloud upload my-project ./my-repo +bm sync # Indexes local files into database +``` -Ignored 45 file(s) based on .gitignore and default patterns -Uploading 23 file(s) to project 'my-project' on https://cloud.basicmemory.com... - ✓ README.md - ✓ src/main.py - ✓ src/utils.py - ✓ docs/guide.md - ... -Successfully uploaded 23 file(s)! +**In cloud mode:** +```bash +bm sync # Runs bisync + indexes files +bm sync --watch # Continuous sync every 60 seconds +bm sync --interval 30 # Custom interval ``` -## Local File Access +The same command works everywhere - no need to remember different commands for local vs cloud! -Basic Memory Cloud supports mounting your cloud files locally using rclone, enabling real-time editing with your favorite text editor or IDE. Changes made locally are automatically synchronized to the cloud. +## Bidirectional Sync (bisync) - Recommended + +Bidirectional sync is the **recommended approach** for most users. It provides: +- ✅ Offline access to all files +- ✅ Automatic bidirectional synchronization +- ✅ Conflict detection and resolution +- ✅ Works with any editor or tool +- ✅ Background watch mode ### Setup -Before mounting files, you need to set up the local access system: +Set up bisync (runs automatically if you used `bm cloud setup`): ```bash bm cloud setup ``` -This command will: -1. Check if rclone is installed (and install it if needed) -2. Retrieve your tenant information from the cloud -3. Generate secure, scoped credentials for your tenant -4. Configure rclone with your tenant's storage settings -5. Display instructions for mounting your files +Or set up with custom directory: -### Mounting Files +```bash +bm cloud setup --dir ~/my-sync-folder +``` -Mount your cloud files to a local directory: +### Running Sync + +Use the cloud-aware `bm sync` command: ```bash -# Mount with default (balanced) profile -bm cloud mount +# Manual sync +bm sync -# Mount with specific profile -bm cloud mount --profile fast -bm cloud mount --profile balanced -bm cloud mount --profile safe +# Watch mode (continuous sync) +bm sync --watch + +# Custom interval (30 seconds) +bm sync --watch --interval 30 ``` -#### Mount Profiles +### Bisync Profiles + +Bisync supports three conflict resolution strategies with different safety levels: -Different profiles optimize for different use cases: +| Profile | Conflict Resolution | Max Deletes | Use Case | +|---------|-------------------|-------------|----------| +| **balanced** | newer | 25 | Default, recommended for most users | +| **safe** | none | 10 | Keep both versions on conflict | +| **fast** | newer | 50 | Rapid iteration, higher delete tolerance | -- **fast**: Ultra-fast development (5s sync, higher bandwidth) - - Cache time: 5s, Poll interval: 3s - - Best for: Active development, frequent file changes +**Profile Details:** -- **balanced**: Fast development (10-15s sync, recommended) - - Cache time: 10s, Poll interval: 5s - - Best for: General use, good balance of speed and reliability +- **safe**: + - Conflict resolution: `none` (creates `.conflict` files for both versions) + - Max delete: 10 files per sync + - Best for: Critical data where you want manual conflict resolution -- **safe**: Conflict-aware mount with backup (15s+ sync) - - Cache time: 15s, Poll interval: 10s - - Includes conflict detection and backup functionality - - Best for: Collaborative editing, important documents +- **balanced** (default): + - Conflict resolution: `newer` (auto-resolve to most recent file) + - Max delete: 25 files per sync + - Best for: General use with automatic conflict handling -### Mount Status +- **fast**: + - Conflict resolution: `newer` (auto-resolve to most recent file) + - Max delete: 50 files per sync + - Best for: Rapid development iteration with less restrictive safety checks -Check the current mount status: +**How to Select a Profile:** + +The default profile (`balanced`) is used automatically with `bm sync`: ```bash -bm cloud mount-status +# Uses balanced profile (default) +bm sync ``` -Example output: +For advanced control, use `bm cloud bisync` with the `--profile` flag: + +```bash +# Use safe mode +bm cloud bisync --profile safe + +# Use fast mode +bm cloud bisync --profile fast + +# Preview changes with specific profile +bm cloud bisync --profile safe --dry-run ``` - Cloud Mount Status -┏━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Property ┃ Value ┃ -┡━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ Tenant ID │ 63cd4020-2e31-5c53-bdbc-07182b129183 │ -│ Mount Path │ ~/basic-memory-63cd4020-2e31-5c53-bdbc-07182b129183 │ -│ Status │ ✓ Mounted │ -│ rclone Processes │ 1 running │ -└──────────────────┴───────────────────────────────────────────────────────────┘ -Available mount profiles: - fast: Ultra-fast development (5s sync, higher bandwidth) - balanced: Fast development (10-15s sync, recommended) - safe: Conflict-aware mount with backup +**Check Available Profiles:** + +```bash +bm cloud status ``` -### Unmounting Files +This shows all available profiles with their settings. + +**Current Limitations:** -Unmount your cloud files and clean up processes: +- Profiles are hardcoded and cannot be customized +- No config file option to change default profile +- Profile settings (max_delete, conflict_resolve) cannot be modified without code changes +- Profile selection only available via `bm cloud bisync --profile` (advanced command) + +### Establishing New Baseline + +If you need to force a complete resync: ```bash -bm cloud unmount +bm cloud bisync --resync ``` -This command will: -1. Unmount the filesystem -2. Kill any running rclone processes -3. Clean up temporary files +**Warning:** This overwrites the sync state. Use only when recovering from errors. -### Working with Mounted Files +### Checking Sync Status + +View current sync status: + +```bash +bm cloud status +``` -Once mounted, your cloud files appear as a regular directory on your local system. You can: +This shows: +- Cloud mode status +- Instance health +- Sync directory location +- Last sync time +- Available bisync profiles -- Edit files with any text editor or IDE -- Create new files and directories -- Move and rename files -- Use command-line tools like `grep`, `find`, etc. +### Verifying Sync Integrity -#### Example Workflow +Check that local and cloud files match: ```bash -# Set up local access -bm cloud setup +# Full integrity check +bm cloud check -# Mount your files -bm cloud mount --profile balanced +# Faster one-way check +bm cloud check --one-way +``` -# Navigate to your mounted files -cd ~/basic-memory-{your-tenant-id} +This uses `rclone check` to verify files match without transferring data. -# Edit files with your preferred editor -code my-notes.md -vim research/paper.md +### Working with Bisync -# Changes are automatically synced to the cloud -# Check sync status -bm cloud mount-status +Create and edit files in `~/basic-memory-cloud-sync/`: -# When done, unmount -bm cloud unmount +```bash +# Create a new note +echo "# My Research" > ~/basic-memory-cloud-sync/my-project/notes.md + +# Edit with your favorite editor +code ~/basic-memory-cloud-sync/my-project/ + +# Sync changes to cloud +bm sync ``` -### Technical Details +In watch mode, changes sync automatically: -- **Protocol**: Uses rclone with NFS mount (no FUSE dependencies) -- **Storage**: Files are stored in Tigris object storage (S3-compatible) -- **Sync**: Bidirectional synchronization with configurable cache settings -- **Security**: Uses scoped, time-limited credentials for your tenant only -- **Compatibility**: Works on macOS, Linux, and Windows +```bash +# Start watch mode +bm sync --watch -## Instance Management +# Edit files - they sync automatically every 60 seconds +code ~/basic-memory-cloud-sync/my-project/ +``` -### Health Check +### Filter Configuration -Check if a cloud instance is healthy and get version information: +Bisync uses `.bmignore` patterns from `~/.basic-memory/.bmignore`: ```bash -bm cloud status +# View current ignore patterns +cat ~/.basic-memory/.bmignore + +# Edit ignore patterns +code ~/.basic-memory/.bmignore ``` -Example output: +Example `.bmignore`: + +```gitignore +# This file is used by 'bm cloud bisync' and file sync +# Patterns use standard gitignore-style syntax + +# Hidden files (files starting with dot) +- .* + +# Basic Memory internal files +- memory.db/** +- memory.db-shm/** +- memory.db-wal/** +- config.json/** + +# Version control +- .git/** + +# Python +- __pycache__/** +- *.pyc +- .venv/** + +# Node.js +- node_modules/** ``` -Cloud instance is healthy - Status: ok - Version: 0.14.4 - Timestamp: 2024-01-15T10:30:00Z + +**Key points:** +- ✅ **Global configuration** - One ignore file for all projects +- ✅ **rclone filter syntax** - Patterns with `- ` prefix +- ✅ **Automatic creation** - Created with defaults on first use +- ✅ **Shared patterns** - Same patterns used by sync service + +## NFS Mount (Direct Access) - Alternative + +NFS mount provides direct file system access as an alternative to bisync. Use this if you prefer mounting files like a network drive. + +### Setup + +Set up mount instead of bisync: + +```bash +bm cloud setup --mount ``` -## WebDAV Protocol +### Mounting Files + +Mount your cloud files: + +```bash +# Mount with default settings +bm cloud mount -File uploads use the WebDAV protocol for efficient, resumable file transfers. The CLI handles: +# Mount with specific profile +bm cloud mount --profile fast +``` -- Directory structure preservation -- File metadata preservation (timestamps) -- Error handling and retry logic -- Progress reporting +#### Mount Profiles -### WebDAV Endpoints +- **balanced** (default): Balanced caching for general use +- **streaming**: Optimized for large files +- **fast**: Minimal verification for rapid access -Files are uploaded to: `{host_url}/{project}/webdav/{file_path}` +### Checking Mount Status -Example: -- Host: `https://cloud.basicmemory.com` -- Project: `my-research` -- File: `docs/notes.md` -- WebDAV URL: `https://cloud.basicmemory.com/proxy/my-research/webdav/docs/notes.md` +View current mount status: -### Authentication Configuration +```bash +bm cloud status --mount +``` -By default, the CLI uses production authentication settings. For development or custom deployments, you can override these settings. +### Unmounting Files -#### Production vs Development +Unmount when done: -- **Production** (default): Uses `client_01K4DGBWAZWP83N3H8VVEMRX6W` and `https://eloquent-lotus-05.authkit.app` -- **Development**: Uses `client_01K46RED2BW9YKYE4N7Y9BDN2V` and `https://exciting-aquarium-32-staging.authkit.app` +```bash +bm cloud unmount +``` -#### Environment Variables +### Working with Mounted Files +Once mounted, files appear at `~/basic-memory-cloud/`: ```bash -# For development environment -export BASIC_MEMORY_CLOUD_HOST="https://development.cloud.basicmemory.com" -export BASIC_MEMORY_CLOUD_CLIENT_ID="client_01K46RED2BW9YKYE4N7Y9BDN2V" -export BASIC_MEMORY_CLOUD_DOMAIN="https://exciting-aquarium-32-staging.authkit.app" +# List cloud files +ls ~/basic-memory-cloud/ -bm cloud login +# Edit with your favorite editor +code ~/basic-memory-cloud/my-project/ + +# Changes are immediately synced to cloud +echo "# Notes" > ~/basic-memory-cloud/my-project/readme.md ``` -#### Configuration File +**Note:** Changes are written through to cloud immediately. There's no "sync" step needed. + +## Instance Management + +### Health Check -You can also set the values in `~/.basic-memory/config.json`: +Check if your cloud instance is healthy: -development -```json -{ - "cloud_host": "http://development.cloud.basicmemory.com", - "cloud_client_id": "client_01K46RED2BW9YKYE4N7Y9BDN2V", - "cloud_domain": "https://exciting-aquarium-32-staging.authkit.app" -} +```bash +bm cloud status ``` +This shows: +- Cloud mode enabled/disabled +- Instance health status +- Instance version +- Sync or mount status + ## Troubleshooting ### Authentication Issues -**Problem**: "Not authenticated" errors -**Solution**: Re-run the login command: +**Problem**: "Authentication failed" or "Invalid token" + +**Solution**: Re-authenticate: + ```bash +bm cloud logout bm cloud login ``` -**Problem**: Wrong environment (dev vs prod) -**Solution**: Check and set the correct environment variables or config +### Sync Issues + +**Problem**: "Bisync not initialized" + +**Solution**: Run setup or initialize with resync: + +```bash +bm cloud setup +# or +bm cloud bisync --resync +``` + +**Problem**: "Too many deletes" error -### Upload Issues +**Solution**: Bisync detected many deletions (safety check). Review changes and use a higher delete limit profile or force resync: -**Problem**: "No files found to upload" -**Solution**: Check gitignore filtering or use `--no-gitignore`: ```bash -bm cloud upload my-project ./path --no-gitignore +bm cloud bisync --profile fast # Higher delete limit +# or +bm cloud bisync --resync # Force baseline ``` -**Problem**: Upload timeouts -**Solution**: The CLI uses a 5-minute timeout for large uploads. For very large files, consider breaking them into smaller chunks. +**Problem**: Conflicts detected + +**Solution**: Bisync found files changed in both locations. Check sync directory for `.conflict` files: + +```bash +ls ~/basic-memory-cloud-sync/**/*.conflict +``` + +Resolve conflicts manually, then sync again. ### Connection Issues -**Problem**: "API request failed" errors -**Solution**: -1. Verify the cloud instance is running: `bm cloud status` -2. Check your internet connection +**Problem**: "Cannot connect to cloud instance" + +**Solution**: Check cloud status: + +```bash +bm cloud status +``` + +If instance is down, wait a few minutes and retry. If problem persists, contact support. ### Mount Issues -**Problem**: "rclone not found" during setup -**Solution**: The setup command will attempt to install rclone automatically. If this fails: -- **macOS**: `brew install rclone` -- **Linux**: `sudo snap install rclone` or `sudo apt install rclone` -- **Windows**: `winget install Rclone.Rclone` +**Problem**: "Mount point is busy" + +**Solution**: Unmount and remount: + +```bash +bm cloud unmount +bm cloud mount +``` -**Problem**: Mount fails with permission errors -**Solution**: -- Ensure you have proper permissions for the mount directory -- On Linux, you may need to add your user to the `fuse` group -- Try unmounting any existing mounts: `bm cloud unmount` +**Problem**: "Permission denied" when accessing mounted files -**Problem**: Files not syncing or appearing outdated -**Solution**: -1. Check mount status: `bm cloud mount-status` -2. Try remounting with a faster profile: `bm cloud mount --profile fast` -3. Unmount and remount: `bm cloud unmount && bm cloud mount` +**Solution**: Check mount status and remount: -**Problem**: Multiple mount processes running -**Solution**: Clean up orphaned processes: ```bash -bm cloud unmount # This will clean up all processes -bm cloud mount # Fresh mount +bm cloud status --mount +bm cloud unmount +bm cloud mount ``` ## Security -- All communication uses HTTPS -- OAuth 2.1 with PKCE provides secure authentication -- Tokens automatically refresh when needed -- Tokens are stored locally in `~/.basic-memory/basic-memory-cloud.json` +- **Authentication**: OAuth 2.1 with PKCE flow +- **Tokens**: Stored securely in `~/.basic-memory/auth/token` +- **Transport**: All data encrypted in transit (HTTPS) +- **Credentials**: Scoped S3 credentials for sync/mount (read-write access to your tenant only) +- **Isolation**: Your data is isolated from other tenants +- **Ignore patterns**: Sensitive files (`.env`, credentials) automatically excluded ## Command Reference +### Cloud Mode Management + ```bash -# Authentication -bm cloud login +bm cloud login # Authenticate and enable cloud mode +bm cloud logout # Disable cloud mode +bm cloud status # Check cloud mode and sync status +bm cloud status --mount # Check cloud mode and mount status +``` + +### Setup + +```bash +bm cloud setup # Setup bisync (default, recommended) +bm cloud setup --mount # Setup mount (alternative) +bm cloud setup --dir ~/sync # Custom sync directory +``` -# Project management -bm cloud project list -bm cloud project add [--default] +### Project Management (Cloud Mode Aware) -# File operations -bm cloud upload [--no-preserve-timestamps] [--no-gitignore] +When cloud mode is enabled, these commands work with cloud: -# Local file access -bm cloud setup # Set up local access with rclone -bm cloud mount [--profile ] # Mount cloud files locally -bm cloud mount-status # Check mount status -bm cloud unmount # Unmount cloud files +```bash +bm project list # List projects +bm project add # Create project +bm project add --default # Create and set as default +bm project rm # Delete project +bm project set-default # Set default project +``` -# Instance management -bm cloud status +### File Synchronization + +```bash +bm sync # Sync files (local or cloud depending on mode) +bm sync --watch # Continuous sync (cloud mode only) +bm sync --interval 30 # Custom interval for watch mode + +# Advanced bisync commands +bm cloud bisync # Run bisync manually +bm cloud bisync --profile safe # Use specific profile +bm cloud bisync --dry-run # Preview changes +bm cloud bisync --resync # Force new baseline +bm cloud bisync --watch # Continuous sync +bm cloud bisync --verbose # Show detailed output + +# Integrity verification +bm cloud check # Full integrity check +bm cloud check --one-way # Faster one-way check +``` + +### Direct File Access (Mount) + +```bash +bm cloud mount # Mount cloud files +bm cloud mount --profile fast # Use specific profile +bm cloud unmount # Unmount files ``` -For more information about Basic Memory Cloud, visit the [official documentation](https://memory.basicmachines.co). +## Summary + +Basic Memory Cloud provides two workflows: + +### Recommended: Bidirectional Sync (bisync) +1. `bm cloud login` - Authenticate once +2. `bm cloud setup` - Configure sync once +3. `bm sync` - Sync anytime (or use `--watch`) +4. Work in `~/basic-memory-cloud-sync/` +5. Changes sync bidirectionally + +### Alternative: Direct Mount +1. `bm cloud login` - Authenticate once +2. `bm cloud setup --mount` - Configure mount once +3. `bm cloud mount` - Mount when needed +4. Work in `~/basic-memory-cloud/` +5. Changes write through immediately + +Both approaches work seamlessly with cloud mode - all your regular `bm` commands work with either workflow! diff --git a/specs/SPEC-8 TigrisFS Integration.md b/specs/SPEC-8 TigrisFS Integration.md new file mode 100644 index 000000000..819b6ddb2 --- /dev/null +++ b/specs/SPEC-8 TigrisFS Integration.md @@ -0,0 +1,886 @@ +--- +title: 'SPEC-8: TigrisFS Integration for Tenant API' +Date: September 22, 2025 +Status: Phase 3.6 Complete - Tenant Mount API Endpoints Ready for CLI Implementation +Priority: High +Goal: Replace Fly volumes with Tigris bucket provisioning in production tenant API +permalink: spec-8-tigris-fs-integration +--- + +## Executive Summary + +Based on SPEC-7 Phase 4 POC testing, this spec outlines productizing the TigrisFS/rclone implementation in the Basic Memory Cloud tenant API. +We're moving from proof-of-concept to production integration, replacing Fly volume storage with Tigris bucket-per-tenant architecture. + +## Current Architecture (Fly Volumes) + +### Tenant Provisioning Flow +```python +# apps/cloud/src/basic_memory_cloud/workflows/tenant_provisioning.py +async def provision_tenant_infrastructure(tenant_id: str): + # 1. Create Fly app + # 2. Create Fly volume ← REPLACE THIS + # 3. Deploy API container with volume mount + # 4. Configure health checks +``` + +### Storage Implementation +- Each tenant gets dedicated Fly volume (1GB-10GB) +- Volume mounted at `/app/data` in API container +- Local filesystem storage with Basic Memory indexing +- No global caching or edge distribution + +## Proposed Architecture (Tigris Buckets) + +### New Tenant Provisioning Flow +```python +async def provision_tenant_infrastructure(tenant_id: str): + # 1. Create Fly app + # 2. Create Tigris bucket with admin credentials ← NEW + # 3. Store bucket name in tenant record ← NEW + # 4. Deploy API container with TigrisFS mount using admin credentials + # 5. Configure health checks +``` + +### Storage Implementation +- Each tenant gets dedicated Tigris bucket +- TigrisFS mounts bucket at `/app/data` in API container +- Global edge caching and distribution +- Configurable cache TTL for sync performance + +## Implementation Plan + +### Phase 1: Bucket Provisioning Service + +**✅ IMPLEMENTED: StorageClient with Admin Credentials** +```python +# apps/cloud/src/basic_memory_cloud/clients/storage_client.py +class StorageClient: + async def create_tenant_bucket(self, tenant_id: UUID) -> TigrisBucketCredentials + async def delete_tenant_bucket(self, tenant_id: UUID, bucket_name: str) -> bool + async def list_buckets(self) -> list[TigrisBucketResponse] + async def test_tenant_credentials(self, credentials: TigrisBucketCredentials) -> bool +``` + +**Simplified Architecture Using Admin Credentials:** +- Single admin access key with full Tigris permissions (configured in console) +- No tenant-specific IAM user creation needed +- Bucket-per-tenant isolation for logical separation +- Admin credentials shared across all tenant operations + +**Integrate with Provisioning workflow:** +```python +# Update tenant_provisioning.py +async def provision_tenant_infrastructure(tenant_id: str): + storage_client = StorageClient(settings.aws_access_key_id, settings.aws_secret_access_key) + bucket_creds = await storage_client.create_tenant_bucket(tenant_id) + await store_bucket_name(tenant_id, bucket_creds.bucket_name) + await deploy_api_with_tigris(tenant_id, bucket_creds) +``` + +### Phase 2: Simplified Bucket Management + +**✅ SIMPLIFIED: Admin Credentials + Bucket Names Only** + +Since we use admin credentials for all operations, we only need to track bucket names per tenant: + +1. **Primary Storage (Fly Secrets)** + ```bash + flyctl secrets set -a basic-memory-{tenant_id} \ + AWS_ACCESS_KEY_ID="{admin_access_key}" \ + AWS_SECRET_ACCESS_KEY="{admin_secret_key}" \ + AWS_ENDPOINT_URL_S3="https://fly.storage.tigris.dev" \ + AWS_REGION="auto" \ + BUCKET_NAME="basic-memory-{tenant_id}" + ``` + +2. **Database Storage (Bucket Name Only)** + ```python + # apps/cloud/src/basic_memory_cloud/models/tenant.py + class Tenant(BaseModel): + # ... existing fields + tigris_bucket_name: Optional[str] = None # Just store bucket name + tigris_region: str = "auto" + created_at: datetime + ``` + +**Benefits of Simplified Approach:** +- No credential encryption/decryption needed +- Admin credentials managed centrally in environment +- Only bucket names stored in database (not sensitive) +- Simplified backup/restore scenarios +- Reduced security attack surface + +### Phase 3: API Container Updates + +**Update API container configuration:** +```dockerfile +# apps/api/Dockerfile +# Add TigrisFS installation +RUN curl -L https://github.com/tigrisdata/tigrisfs/releases/latest/download/tigrisfs-linux-amd64 \ + -o /usr/local/bin/tigrisfs && chmod +x /usr/local/bin/tigrisfs +``` + +**Startup script integration:** +```bash +# apps/api/tigrisfs-startup.sh (already exists) +# Mount TigrisFS → Start Basic Memory API +exec python -m basic_memory_cloud_api.main +``` + +**Fly.toml environment (optimized for < 5s startup):** +```toml +# apps/api/fly.tigris-production.toml +[env] + TIGRISFS_MEMORY_LIMIT = '1024' # Reduced for faster init + TIGRISFS_MAX_FLUSHERS = '16' # Fewer threads for faster startup + TIGRISFS_STAT_CACHE_TTL = '30s' # Balance sync speed vs startup + TIGRISFS_LAZY_INIT = 'true' # Enable lazy loading + BASIC_MEMORY_HOME = '/app/data' + +# Suspend optimization for wake-on-network +[machine] + auto_stop_machines = "suspend" # Faster than full stop + auto_start_machines = true + min_machines_running = 0 +``` + +### Phase 4: Local Access Features + +**CLI automation for local mounting:** +```python +# New CLI command: basic-memory cloud mount +async def setup_local_mount(tenant_id: str): + # 1. Fetch bucket credentials from cloud API + # 2. Configure rclone with scoped IAM policy + # 3. Mount via rclone nfsmount (macOS) or FUSE (Linux) + # 4. Start Basic Memory sync watcher +``` + +**Local mount configuration:** +```bash +# rclone config for tenant +rclone mount basic-memory-{tenant_id}: ~/basic-memory-{tenant_id} \ + --nfs-mount \ + --vfs-cache-mode writes \ + --cache-dir ~/.cache/rclone/basic-memory-{tenant_id} +``` + +### Phase 5: TigrisFS Cache Sync Solutions + +**Problem**: When files are uploaded via CLI/bisync, the tenant API container doesn't see them immediately due to TigrisFS cache (30s TTL) and lack of inotify events on mounted filesystems. + +**Multi-Layer Solution:** + +**Layer 1: API Sync Endpoint** (Immediate) +```python +# POST /sync - Force TigrisFS cache refresh +# Callable by CLI after uploads +subprocess.run(["sync", "fsync /app/data"], check=True) +``` + +**Layer 2: Tigris Webhook Integration** (Real-time) +https://www.tigrisdata.com/docs/buckets/object-notifications/#webhook +```python +# Webhook endpoint for bucket changes +@app.post("/webhooks/tigris/{tenant_id}") +async def handle_bucket_notification(tenant_id: str, event: TigrisEvent): + if event.eventName in ["OBJECT_CREATED_PUT", "OBJECT_DELETED"]: + await notify_container_sync(tenant_id, event.object.key) +``` + +**Layer 3: CLI Sync Notification** (User-triggered) +```bash +# CLI calls container sync endpoint after successful bisync +basic-memory cloud bisync # Automatically notifies container +curl -X POST https://basic-memory-{tenant-id}.fly.dev/sync +``` + +**Layer 4: Periodic Sync Fallback** (Safety net) +```python +# Background task: fsync /app/data every 30s as fallback +# Ensures eventual consistency even if other layers fail +``` + +**Implementation Priority:** +1. Layer 1 (API endpoint) - Quick testing capability +2. Layer 3 (CLI integration) - Improved UX +3. Layer 4 (Periodic fallback) - Safety net +4. Layer 2 (Webhooks) - Production real-time sync + + +## Performance Targets + +### Sync Latency +- **Target**: < 5 seconds local→cloud→container +- **Configuration**: `TIGRISFS_STAT_CACHE_TTL = '5s'` +- **Monitoring**: Track sync metrics in production + +### Container Startup +- **Target**: < 5 seconds including TigrisFS mount +- **Fast retry**: 0.5s intervals for mount verification +- **Fallback**: Container fails fast if mount fails + +### Memory Usage +- **TigrisFS cache**: 2GB memory limit per container +- **Concurrent uploads**: 32 flushers max +- **VM sizing**: shared-cpu-2x (2048mb) minimum + +## Security Considerations + +### Bucket Isolation +- Each tenant has dedicated bucket +- IAM policies prevent cross-tenant access +- No shared bucket with subdirectories + +### Credential Security +- Fly secrets for runtime access +- Encrypted database backup for disaster recovery +- Credential rotation capability + +### Data Residency +- Tigris global edge caching +- SOC2 Type II compliance +- Encryption at rest and in transit + +## Operational Benefits + +### Scalability +- Horizontal scaling with stateless API containers +- Global edge distribution +- Better resource utilization + +### Reliability +- No cold starts between tenants +- Built-in redundancy and caching +- Simplified backup strategy + +### Cost Efficiency +- Pay-per-use storage pricing +- Shared infrastructure benefits +- Reduced operational overhead + +## Risk Mitigation + +### Data Loss Prevention +- Dual credential storage (Fly + database) +- Automated backup workflows to R2/S3 +- Tigris built-in redundancy + +### Performance Degradation +- Configurable cache settings per tenant +- Monitoring and alerting on sync latency +- Fallback to volume storage if needed + +### Security Vulnerabilities +- Bucket-per-tenant isolation +- Regular credential rotation +- Security scanning and monitoring + +## Success Metrics + +### Technical Metrics +- Sync latency P50 < 5 seconds +- Container startup time < 5 seconds +- Zero data loss incidents +- 99.9% uptime per tenant + +### Business Metrics +- Reduced infrastructure costs vs volumes +- Improved user experience with faster sync +- Enhanced enterprise security posture +- Simplified operational overhead + +## Open Questions + +1. **Tigris rate limits**: What are the API limits for bucket creation? +2. **Cost analysis**: What's the break-even point vs Fly volumes? +3. **Regional preferences**: Should enterprise customers choose regions? +4. **Backup retention**: How long to keep automated backups? + +## Implementation Checklist + +### Phase 1: Bucket Provisioning Service ✅ COMPLETED +- [x] **Research Tigris bucket API** - Document bucket creation and S3 API compatibility +- [x] **Create StorageClient class** - Implemented with admin credentials and comprehensive integration tests +- [x] **Test bucket creation** - Full test suite validates API integration with real Tigris environment +- [x] **Add bucket provisioning to DBOS workflow** - Integrated StorageClient with tenant_provisioning.py + +### Phase 2: Simplified Bucket Management ✅ COMPLETED +- [x] **Update Tenant model** with tigris_bucket_name field (replaced fly_volume_id) +- [x] **Implement bucket name storage** - Database migration and model updates completed +- [x] **Test bucket provisioning integration** - Full test suite validates workflow from tenant creation to bucket assignment +- [x] **Remove volume logic from all tests** - Complete migration from volume-based to bucket-based architecture + +### Phase 3: API Container Integration ✅ COMPLETED +- [x] **Update Dockerfile** to install TigrisFS binary in API container with configurable version +- [x] **Optimize tigrisfs-startup.sh** with production-ready security and reliability improvements +- [x] **Create production-ready container** with proper signal handling and mount validation +- [x] **Implement security fixes** based on Claude code review (conditional debug, credential protection) +- [x] **Add proper process supervision** with cleanup traps and error handling +- [x] **Remove debug artifacts** - Cleaned up all debug Dockerfiles and test scripts + +### Phase 3.5: IAM Access Key Management ✅ COMPLETED +- [x] **Research Tigris IAM API** - Documented create_policy, attach_user_policy, delete_access_key operations +- [x] **Implement bucket-scoped credential generation** - StorageClient.create_tenant_access_keys() with IAM policies +- [x] **Add comprehensive security test suite** - 5 security-focused integration tests covering all attack vectors +- [x] **Verify cross-bucket access prevention** - Scoped credentials can ONLY access their designated bucket +- [x] **Test credential lifecycle management** - Create, validate, delete, and revoke access keys +- [x] **Validate admin vs scoped credential isolation** - Different access patterns and security boundaries +- [x] **Test multi-tenant isolation** - Multiple tenants cannot access each other's buckets + +### Phase 3.6: Tenant Mount API Endpoints ✅ COMPLETED +- [x] **Implement GET /tenant/mount/info** - Returns mount info without exposing credentials +- [x] **Implement POST /tenant/mount/credentials** - Creates new bucket-scoped credentials for CLI mounting +- [x] **Implement DELETE /tenant/mount/credentials/{cred_id}** - Revoke specific credentials with proper cleanup +- [x] **Implement GET /tenant/mount/credentials** - List active credentials without exposing secrets +- [x] **Add TenantMountCredentials database model** - Tracks credential metadata (no secret storage) +- [x] **Create comprehensive test suite** - 28 tests covering all scenarios including multi-session support +- [x] **Implement multi-session credential flow** - Multiple active credentials per tenant supported +- [x] **Secure credential handling** - Secret keys never stored, returned once only for immediate use +- [x] **Add dependency injection for StorageClient** - Clean integration with existing API architecture +- [x] **Fix Tigris configuration for cloud service** - Added AWS environment variables to fly.template.toml +- [x] **Update tenant machine configurations** - Include AWS credentials for TigrisFS mounting with clear credential strategy + +**Security Test Results:** +``` +✅ Cross-bucket access prevention - PASS +✅ Deleted credentials access revoked - PASS +✅ Invalid credentials rejected - PASS +✅ Admin vs scoped credential isolation - PASS +✅ Multiple scoped credentials isolation - PASS +``` + +**Implementation Details:** +- Uses Tigris IAM managed policies (create_policy + attach_user_policy) +- Bucket-scoped S3 policies with Actions: GetObject, PutObject, DeleteObject, ListBucket +- Resource ARNs limited to specific bucket: `arn:aws:s3:::bucket-name` and `arn:aws:s3:::bucket-name/*` +- Access keys follow Tigris format: `tid_` prefix with secure random suffix +- Complete cleanup on deletion removes both access keys and associated policies + +### Phase 4: Local Access CLI +- [x] **Design local mount CLI command** for automated rclone configuration +- [x] **Implement credential fetching** from cloud API for local setup +- [x] **Create rclone config automation** for tenant-specific bucket mounting +- [x] **Test local→cloud→container sync** with optimized cache settings +- [x] **Document local access setup** for beta users + +### Phase 5: Webhook Integration (Future) +- [ ] **Research Tigris webhook API** for object notifications and payload format +- [ ] **Design webhook endpoint** for real-time sync notifications +- [ ] **Implement notification handling** to trigger Basic Memory sync events +- [ ] **Test webhook delivery** and sync latency improvements + +## Success Metrics +- [ ] **Container startup < 5 seconds** including TigrisFS mount and Basic Memory init +- [ ] **Sync latency < 5 seconds** for local→cloud→container file changes +- [ ] **Zero data loss** during bucket provisioning and credential management +- [ ] **100% test coverage** for new TigrisBucketService and credential functions +- [ ] **Beta deployment** with internal users validating local-cloud workflow + + + +## Implementation Notes + +## Phase 4.1: Bidirectional Sync with rclone bisync (NEW) + +### Problem Statement +During testing, we discovered that some applications (particularly Obsidian) don't detect file changes over NFS mounts. Rather than building a custom sync daemon, we can leverage `rclone bisync` - rclone's built-in bidirectional synchronization feature. + +### Solution: rclone bisync +Use rclone's proven bidirectional sync instead of custom implementation: + +**Core Architecture:** +```bash +# rclone bisync handles all the complexity +rclone bisync ~/basic-memory-{tenant_id} basic-memory-{tenant_id}:{bucket_name} \ + --create-empty-src-dirs \ + --conflict-resolve newer \ + --resilient \ + --check-access +``` + +**Key Benefits:** +- ✅ **Battle-tested**: Production-proven rclone functionality +- ✅ **MIT licensed**: Open source with permissive licensing +- ✅ **No custom code**: Zero maintenance burden for sync logic +- ✅ **Built-in safety**: max-delete protection, conflict resolution +- ✅ **Simple installation**: Works with Homebrew rclone (no FUSE needed) +- ✅ **File watcher compatible**: Works with Obsidian and all applications +- ✅ **Offline support**: Can work offline and sync when connected + +### bisync Conflict Resolution Options + +**Built-in conflict strategies:** +```bash +--conflict-resolve none # Keep both files with .conflict suffixes (safest) +--conflict-resolve newer # Always pick the most recently modified file +--conflict-resolve larger # Choose based on file size +--conflict-resolve path1 # Always prefer local changes +--conflict-resolve path2 # Always prefer cloud changes +``` + +### Sync Profiles Using bisync + +**Profile configurations:** +```python +BISYNC_PROFILES = { + "safe": { + "conflict_resolve": "none", # Keep both versions + "max_delete": 10, # Prevent mass deletion + "check_access": True, # Verify sync integrity + "description": "Safe mode with conflict preservation" + }, + "balanced": { + "conflict_resolve": "newer", # Auto-resolve to newer file + "max_delete": 25, + "check_access": True, + "description": "Balanced mode (recommended default)" + }, + "fast": { + "conflict_resolve": "newer", + "max_delete": 50, + "check_access": False, # Skip verification for speed + "description": "Fast mode for rapid iteration" + } +} +``` + +### CLI Commands + +**Manual sync commands:** +```bash +basic-memory cloud bisync # Manual bidirectional sync +basic-memory cloud bisync --dry-run # Preview changes +basic-memory cloud bisync --profile safe # Use specific profile +basic-memory cloud bisync --resync # Force full baseline resync +``` + +**Watch mode (Step 1):** +```bash +basic-memory cloud bisync --watch # Long-running process, sync every 60s +basic-memory cloud bisync --watch --interval 30s # Custom interval +``` + +**System integration (Step 2 - Future):** +```bash +basic-memory cloud bisync-service install # Install as system service +basic-memory cloud bisync-service start # Start background service +basic-memory cloud bisync-service status # Check service status +``` + +### Implementation Strategy + +**Phase 4.1.1: Core bisync Implementation** +- [ ] Implement `run_bisync()` function wrapping rclone bisync +- [ ] Add profile-based configuration (safe/balanced/fast) +- [ ] Create conflict resolution and safety options +- [ ] Test with sample files and conflict scenarios + +**Phase 4.1.2: Watch Mode** +- [ ] Add `--watch` flag for continuous sync +- [ ] Implement configurable sync intervals +- [ ] Add graceful shutdown and signal handling +- [ ] Create status monitoring and progress indicators + +**Phase 4.1.3: User Experience** +- [ ] Add conflict reporting and resolution guidance +- [ ] Implement dry-run preview functionality +- [ ] Create troubleshooting and diagnostic commands +- [ ] Add filtering configuration (.gitignore-style) + +**Phase 4.1.4: System Integration (Future)** +- [ ] Generate platform-specific service files (launchd/systemd) +- [ ] Add service management commands +- [ ] Implement automatic startup and recovery +- [ ] Create monitoring and logging integration + +### Technical Implementation + +**Core bisync wrapper:** +```python +def run_bisync( + tenant_id: str, + bucket_name: str, + profile: str = "balanced", + dry_run: bool = False +) -> bool: + """Run rclone bisync with specified profile.""" + + local_path = Path.home() / f"basic-memory-{tenant_id}" + remote_path = f"basic-memory-{tenant_id}:{bucket_name}" + profile_config = BISYNC_PROFILES[profile] + + cmd = [ + "rclone", "bisync", + str(local_path), remote_path, + "--create-empty-src-dirs", + "--resilient", + f"--conflict-resolve={profile_config['conflict_resolve']}", + f"--max-delete={profile_config['max_delete']}", + "--filters-file", "~/.basic-memory/bisync-filters.txt" + ] + + if profile_config.get("check_access"): + cmd.append("--check-access") + + if dry_run: + cmd.append("--dry-run") + + return subprocess.run(cmd, check=True).returncode == 0 +``` + +**Default filter file (~/.basic-memory/bisync-filters.txt):** +``` +- .DS_Store +- .git/** +- __pycache__/** +- *.pyc +- .pytest_cache/** +- node_modules/** +- .conflict-* +- Thumbs.db +- desktop.ini +``` + +**Advantages Over Custom Daemon:** +- ✅ **Zero maintenance**: No custom sync logic to debug/maintain +- ✅ **Production proven**: Used by thousands in production +- ✅ **Safety features**: Built-in max-delete, conflict handling, recovery +- ✅ **Filtering**: Advanced exclude patterns and rules +- ✅ **Performance**: Optimized for various storage backends +- ✅ **Community support**: Extensive documentation and community + +## Phase 4.2: NFS Mount Support (Direct Access) + +### Solution: rclone nfsmount +Keep the existing NFS mount functionality for users who prefer direct file access: + +**Core Architecture:** +```bash +# rclone nfsmount provides transparent file access +rclone nfsmount basic-memory-{tenant_id}:{bucket_name} ~/basic-memory-{tenant_id} \ + --vfs-cache-mode writes \ + --dir-cache-time 10s \ + --daemon +``` + +**Key Benefits:** +- ✅ **Real-time access**: Files appear immediately as they're created/modified +- ✅ **Transparent**: Works with any application that reads/writes files +- ✅ **Low latency**: Direct access without sync delays +- ✅ **Simple**: No periodic sync commands needed +- ✅ **Homebrew compatible**: Works with Homebrew rclone (no FUSE required) + +**Limitations:** +- ❌ **File watcher compatibility**: Some apps (Obsidian) don't detect changes over NFS +- ❌ **Network dependency**: Requires active connection to cloud storage +- ❌ **Potential conflicts**: Simultaneous edits from multiple locations can cause issues + +### Mount Profiles (Existing) + +**Already implemented profiles from SPEC-7 testing:** +```python +MOUNT_PROFILES = { + "fast": { + "cache_time": "5s", + "poll_interval": "3s", + "description": "Ultra-fast development (5s sync)" + }, + "balanced": { + "cache_time": "10s", + "poll_interval": "5s", + "description": "Fast development (10-15s sync, recommended)" + }, + "safe": { + "cache_time": "15s", + "poll_interval": "10s", + "description": "Conflict-aware mount with backup", + "extra_args": ["--conflict-suffix", ".conflict-{DateTimeExt}"] + } +} +``` + +### CLI Commands (Existing) + +**Mount commands already implemented:** +```bash +basic-memory cloud mount # Mount with balanced profile +basic-memory cloud mount --profile fast # Ultra-fast caching +basic-memory cloud mount --profile safe # Conflict detection +basic-memory cloud unmount # Clean unmount +basic-memory cloud mount-status # Show mount status +``` + +## User Choice: Mount vs Bisync + +### When to Use Each Approach + +| Use Case | Recommended Solution | Why | +|----------|---------------------|-----| +| **Obsidian users** | `bisync` | File watcher support for live preview | +| **CLI/vim/emacs users** | `mount` | Direct file access, lower latency | +| **Offline work** | `bisync` | Can work offline, sync when connected | +| **Real-time collaboration** | `mount` | Immediate visibility of changes | +| **Multiple machines** | `bisync` | Better conflict handling | +| **Single machine** | `mount` | Simpler, more transparent | +| **Development work** | Either | Both work well, user preference | +| **Large files** | `mount` | Streaming access vs full download | + +### Installation Simplicity + +**Both approaches now use simple Homebrew installation:** +```bash +# Single installation command for both approaches +brew install rclone + +# No macFUSE, no system modifications needed +# Works immediately with both mount and bisync +``` + +### Implementation Status + +**Phase 4.1: bisync** (NEW) +- [ ] Implement bisync command wrapper +- [ ] Add watch mode with configurable intervals +- [ ] Create conflict resolution workflows +- [ ] Add filtering and safety options + +**Phase 4.2: mount** (EXISTING - ✅ IMPLEMENTED) +- [x] NFS mount commands with profile support +- [x] Mount management and cleanup +- [x] Process monitoring and health checks +- [x] Credential integration with cloud API + +**Both approaches share:** +- [x] Credential management via cloud API +- [x] Secure rclone configuration +- [x] Tenant isolation and bucket scoping +- [x] Simple Homebrew rclone installation + + +Key Features: + +1. Cross-Platform rclone Installation (rclone_installer.py): +- macOS: Homebrew → official script fallback +- Linux: snap → apt → official script fallback +- Windows: winget → chocolatey → scoop fallback +- Automatic version detection and verification + +2. Smart rclone Configuration (rclone_config.py): +- Automatic tenant-specific config generation +- Three optimized mount profiles from your SPEC-7 testing: +- fast: 5s sync (ultra-performance) +- balanced: 10-15s sync (recommended default) +- safe: 15s sync + conflict detection +- Backup existing configs before modification + +3. Robust Mount Management (mount_commands.py): +- Automatic tenant credential generation +- Mount path management (~/basic-memory-{tenant-id}) +- Process lifecycle management (prevent duplicate mounts) +- Orphaned process cleanup +- Mount verification and health checking + +4. Clean Architecture (api_client.py): +- Separated API client to avoid circular imports +- Reuses existing authentication infrastructure +- Consistent error handling and logging + +User Experience: + +One-Command Setup: +basic-memory cloud setup +```bash +# 1. Installs rclone automatically +# 2. Authenticates with existing login +# 3. Generates secure credentials +# 4. Configures rclone +# 5. Performs initial mount +``` + +Profile-Based Mounting: +basic-memory cloud mount --profile fast # 5s sync +basic-memory cloud mount --profile balanced # 15s sync (default) +basic-memory cloud mount --profile safe # conflict detection + +Status Monitoring: +basic-memory cloud mount-status +```bash +# Shows: tenant info, mount path, sync profile, rclone processes +``` +### local mount api + +Endpoint 1: Get Tenant Info for user +Purpose: Get tenant details for mounting +- pass in jwt +- service returns mount info + +**✅ IMPLEMENTED API Specification:** + +**Endpoint 1: GET /tenant/mount/info** +- Purpose: Get tenant mount information without exposing credentials +- Authentication: JWT token (tenant_id extracted from claims) + +Request: +``` +GET /tenant/mount/info +Authorization: Bearer {jwt_token} +``` + +Response: +```json +{ + "tenant_id": "434252dd-d83b-4b20-bf70-8a950ff875c4", + "bucket_name": "basic-memory-434252dd", + "has_credentials": true, + "credentials_created_at": "2025-09-22T16:48:50.414694" +} +``` + +**Endpoint 2: POST /tenant/mount/credentials** +- Purpose: Generate NEW bucket-scoped S3 credentials for rclone mounting +- Authentication: JWT token (tenant_id extracted from claims) +- Multi-session: Creates new credentials without revoking existing ones + +Request: +``` +POST /tenant/mount/credentials +Authorization: Bearer {jwt_token} +Content-Type: application/json +``` +*Note: No request body needed - tenant_id extracted from JWT* + +Response: +```json +{ + "tenant_id": "434252dd-d83b-4b20-bf70-8a950ff875c4", + "bucket_name": "basic-memory-434252dd", + "access_key": "test_access_key_12345", + "secret_key": "test_secret_key_abcdef", + "endpoint_url": "https://fly.storage.tigris.dev", + "region": "auto" +} +``` + +**🔒 Security Notes:** +- Secret key returned ONCE only - never stored in database +- Credentials are bucket-scoped (cannot access other tenants' buckets) +- Multiple active credentials supported per tenant (work laptop + personal machine) + +Implementation Notes + +Security: +- Both endpoints require JWT authentication +- Extract tenant_id from JWT claims (not request body) +- Generate scoped credentials (not admin credentials) +- Credentials should have bucket-specific access only + +Integration Points: +- Use your existing StorageClient from SPEC-8 implementation +- Leverage existing JWT middleware for tenant extraction +- Return same credential format as your Tigris bucket provisioning + +Error Handling: +- 401 if not authenticated +- 403 if tenant doesn't exist +- 500 if credential generation fails + +**🔄 Design Decisions:** + +1. **Secure Credential Flow (No Secret Storage)** + +Based on CLI flow analysis, we follow security best practices: +- ✅ API generates both access_key + secret_key via Tigris IAM +- ✅ Returns both in API response for immediate use +- ✅ CLI uses credentials immediately to configure rclone +- ✅ Database stores only metadata (access_key + policy_arn for cleanup) +- ✅ rclone handles secure local credential storage +- ❌ **Never store secret_key in database (even encrypted)** + +2. **CLI Credential Flow** +```bash +# CLI calls API +POST /tenant/mount/credentials → {access_key, secret_key, ...} + +# CLI immediately configures rclone +rclone config create basic-memory-{tenant_id} s3 \ + access_key_id={access_key} \ + secret_access_key={secret_key} \ + endpoint=https://fly.storage.tigris.dev + +# Database tracks metadata only +INSERT INTO tenant_mount_credentials (tenant_id, access_key, policy_arn, ...) +``` + +3. **Multiple Sessions Supported** + +- Users can have multiple active credential sets (work laptop, personal machine, etc.) +- Each credential generation creates a new Tigris access key +- List active credentials via API (shows access_key but never secret) + +4. **Failure Handling & Cleanup** + +- **Happy Path**: Credentials created → Used immediately → rclone configured +- **Orphaned Credentials**: Background job revokes unused credentials +- **API Failure Recovery**: Retry Tigris deletion with stored policy_arn +- **Status Tracking**: Track tigris_deletion_status (pending/completed/failed) + +5. **Event Sourcing & Audit** + +- MountCredentialCreatedEvent +- MountCredentialRevokedEvent +- MountCredentialOrphanedEvent (for cleanup) +- Full audit trail for security compliance + +6. **Tenant/Bucket Validation** + +- Verify tenant exists and has valid bucket before credential generation +- Use existing StorageClient to validate bucket access +- Prevent credential generation for inactive/invalid tenants + +📋 **Implemented API Endpoints:** + +``` +✅ IMPLEMENTED: +GET /tenant/mount/info # Get tenant/bucket info (no credentials exposed) +POST /tenant/mount/credentials # Generate new credentials (returns secret once) +GET /tenant/mount/credentials # List active credentials (no secrets) +DELETE /tenant/mount/credentials/{cred_id} # Revoke specific credentials +``` + +**API Implementation Status:** +- ✅ **GET /tenant/mount/info**: Returns tenant_id, bucket_name, has_credentials, credentials_created_at +- ✅ **POST /tenant/mount/credentials**: Creates new bucket-scoped access keys, returns access_key + secret_key once +- ✅ **GET /tenant/mount/credentials**: Lists active credentials without exposing secret keys +- ✅ **DELETE /tenant/mount/credentials/{cred_id}**: Revokes specific credentials with proper Tigris IAM cleanup +- ✅ **Multi-session support**: Multiple active credentials per tenant (work laptop + personal machine) +- ✅ **Security**: Secret keys never stored in database, returned once only for immediate use +- ✅ **Comprehensive test suite**: 28 tests covering all scenarios including error handling and multi-session flows +- ✅ **Dependency injection**: Clean integration with existing FastAPI architecture +- ✅ **Production-ready configuration**: Tigris credentials properly configured for tenant machines + +🗄️ **Secure Database Schema:** + +```sql +CREATE TABLE tenant_mount_credentials ( + id UUID PRIMARY KEY, + tenant_id UUID REFERENCES tenant(id), + access_key VARCHAR(255) NOT NULL, + -- secret_key REMOVED - never store secrets (security best practice) + policy_arn VARCHAR(255) NOT NULL, -- For Tigris IAM cleanup + tigris_deletion_status VARCHAR(20) DEFAULT 'pending', -- Track cleanup + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW(), + revoked_at TIMESTAMP NULL, + last_used_at TIMESTAMP NULL, -- Track usage for orphan cleanup + description VARCHAR(255) DEFAULT 'CLI mount credentials' +); +``` + +**Security Benefits:** +- ✅ Database breach cannot expose secrets +- ✅ Follows "secrets don't persist" security principle +- ✅ Meets compliance requirements (SOC2, etc.) +- ✅ Reduced attack surface +- ✅ CLI gets credentials once and stores securely via rclone diff --git a/specs/SPEC-9 Multi-Project Bidirectional Sync Architecture.md b/specs/SPEC-9 Multi-Project Bidirectional Sync Architecture.md new file mode 100644 index 000000000..a962ee621 --- /dev/null +++ b/specs/SPEC-9 Multi-Project Bidirectional Sync Architecture.md @@ -0,0 +1,1114 @@ +--- +title: 'SPEC-9: Multi-Project Bidirectional Sync Architecture' +type: spec +permalink: specs/spec-9-multi-project-bisync +tags: +- cloud +- bisync +- architecture +- multi-project +--- + +# SPEC-9: Multi-Project Bidirectional Sync Architecture + +## Status: ✅ Implementation Complete + +**Completed Phases:** +- ✅ Phase 1: Cloud Mode Toggle & Config +- ✅ Phase 2: Bisync Updates (Multi-Project) +- ✅ Phase 3: Sync Command Dual Mode +- ✅ Phase 4: Remove Duplicate Commands & Cloud Mode Auth +- ✅ Phase 5: Mount Updates +- ✅ Phase 6: Safety & Validation +- ⏸️ Phase 7: Cloud-Side Implementation (Deferred to cloud repo) +- ✅ Phase 8.1: Testing (All test scenarios validated) +- ✅ Phase 8.2: Documentation (Core docs complete, demos pending) + +**Key Achievements:** +- Unified CLI: `bm sync`, `bm project`, `bm tool` work transparently in both local and cloud modes +- Multi-project sync: Single `bm sync` operation handles all projects bidirectionally +- Cloud mode toggle: `bm cloud login` / `bm cloud logout` switches modes seamlessly +- Integrity checking: `bm cloud check` verifies file matching without data transfer +- Directory isolation: Mount and bisync use separate directories with conflict prevention +- Clean UX: No RCLONE_TEST files, clear error messages, transparent implementation + +## Why + +**Current State:** +SPEC-8 implemented rclone bisync for cloud file synchronization, but has several architectural limitations: +1. Syncs only a single project subdirectory (`bucket:/basic-memory`) +2. Requires separate `bm cloud` command namespace, duplicating existing CLI commands +3. Users must learn different commands for local vs cloud operations +4. RCLONE_TEST marker files clutter user directories + +**Problems:** +1. **Duplicate Commands**: `bm project` vs `bm cloud project`, `bm tool` vs (no cloud equivalent) +2. **Inconsistent UX**: Same operations require different command syntax depending on mode +3. **Single Project Sync**: Users can only sync one project at a time +4. **Manual Coordination**: Creating new projects requires manual coordination between local and cloud +5. **Confusing Artifacts**: RCLONE_TEST marker files confuse users + +**Goals:** +- **Unified CLI**: All existing `bm` commands work in both local and cloud mode via toggle +- **Multi-Project Sync**: Single sync operation handles all projects bidirectionally +- **Simple Mode Switch**: `bm cloud login` enables cloud mode, `logout` returns to local +- **Automatic Registration**: Projects auto-register on both local and cloud sides +- **Clean UX**: Remove unnecessary safety checks and confusing artifacts + +## Cloud Access Paradigm: The Dropbox Model + +**Mental Model Shift:** + +Basic Memory cloud access follows the **Dropbox/iCloud paradigm** - not a per-project cloud connection model. + +**What This Means:** + +``` +Traditional Project-Based Model (❌ Not This): + bm cloud mount --project work # Mount individual project + bm cloud mount --project personal # Mount another project + bm cloud sync --project research # Sync specific project + → Multiple connections, multiple credentials, complex management + +Dropbox Model (✅ This): + bm cloud mount # One mount, all projects + bm sync # One sync, all projects + ~/basic-memory-cloud/ # One folder, all content + → Single connection, organized by folders (projects) +``` + +**Key Principles:** + +1. **Mount/Bisync = Access Methods, Not Project Tools** + - Mount: Read-through cache to cloud (like Dropbox folder) + - Bisync: Bidirectional sync with cloud (like Dropbox sync) + - Both operate at **bucket level** (all projects) + +2. **Projects = Organization Within Cloud Space** + - Projects are folders within your cloud storage + - Creating a folder creates a project (auto-discovered) + - Projects are managed via `bm project` commands + +3. **One Cloud Space Per Machine** + - One set of IAM credentials per tenant + - One mount point: `~/basic-memory-cloud/` + - One bisync directory: `~/basic-memory-cloud-sync/` (default) + - All projects accessible through this single entry point + +4. **Why This Works Better** + - **Credential Management**: One credential set, not N sets per project + - **Resource Efficiency**: One rclone process, not N processes + - **Familiar Pattern**: Users already understand Dropbox/iCloud + - **Operational Simplicity**: `mount` once, `unmount` once + - **Scales Naturally**: Add projects by creating folders, not reconfiguring cloud access + +**User Journey:** + +```bash +# Setup cloud access (once) +bm cloud login +bm cloud mount # or: bm cloud setup for bisync + +# Work with projects (create folders as needed) +cd ~/basic-memory-cloud/ +mkdir my-new-project +echo "# Notes" > my-new-project/readme.md + +# Cloud auto-discovers and registers project +# No additional cloud configuration needed +``` + +This paradigm shift means **mount and bisync are infrastructure concerns**, while **projects are content organization**. Users think about their knowledge, not about cloud plumbing. + +## What + +This spec affects: + +1. **Cloud Mode Toggle** (`config.py`, `async_client.py`): + - Add `cloud_mode` flag to `~/.basic-memory/config.json` + - Set/unset `BASIC_MEMORY_PROXY_URL` based on cloud mode + - `bm cloud login` enables cloud mode, `logout` disables it + - All CLI commands respect cloud mode via existing async_client + +2. **Unified CLI Commands**: + - **Remove**: `bm cloud project` commands (duplicate of `bm project`) + - **Enhance**: `bm sync` co-opted for bisync in cloud mode + - **Keep**: `bm cloud login/logout/status/setup` for mode management + - **Result**: `bm project`, `bm tool`, `bm sync` work in both modes + +3. **Bisync Integration** (`bisync_commands.py`): + - Remove `--check-access` (no RCLONE_TEST files) + - Sync bucket root (all projects), not single subdirectory + - Project auto-registration before sync + - `bm sync` triggers bisync in cloud mode + - `bm sync --watch` for continuous sync + +4. **Config Structure**: + ```json + { + "cloud_mode": true, + "cloud_host": "https://cloud.basicmemory.com", + "auth_tokens": {...}, + "bisync_config": { + "profile": "balanced", + "sync_dir": "~/basic-memory-cloud-sync" + } + } + ``` + +5. **User Workflows**: + - **Enable cloud**: `bm cloud login` → all commands work remotely + - **Create projects**: `bm project add "name"` creates on cloud + - **Sync files**: `bm sync` runs bisync (all projects) + - **Use tools**: `bm tool write-note` creates notes on cloud + - **Disable cloud**: `bm cloud logout` → back to local mode + +## Implementation Tasks + +### Phase 1: Cloud Mode Toggle & Config (Foundation) ✅ + +**1.1 Update Config Schema** +- [x] Add `cloud_mode: bool = False` to Config model +- [x] Add `bisync_config: dict` with `profile` and `sync_dir` fields +- [x] Ensure `cloud_host` field exists +- [x] Add config migration for existing users (defaults handle this) + +**1.2 Update async_client.py** +- [x] Read `cloud_mode` from config (not just environment) +- [x] Set `BASIC_MEMORY_PROXY_URL` from config when `cloud_mode=true` +- [x] Priority: env var > config.cloud_host (if cloud_mode) > None (local ASGI) +- [ ] Test both local and cloud mode routing + +**1.3 Update Login/Logout Commands** +- [x] `bm cloud login`: Set `cloud_mode=true` and save config +- [x] `bm cloud login`: Set `BASIC_MEMORY_PROXY_URL` environment variable +- [x] `bm cloud logout`: Set `cloud_mode=false` and save config +- [x] `bm cloud logout`: Clear `BASIC_MEMORY_PROXY_URL` environment variable +- [x] `bm cloud status`: Show current mode (local/cloud), connection status + +**1.4 Skip Initialization in Cloud Mode** ✅ +- [x] Update `ensure_initialization()` to check `cloud_mode` and return early +- [x] Document that `config.projects` is only used in local mode +- [x] Cloud manages its own projects via API, no local reconciliation needed + +### Phase 2: Bisync Updates (Multi-Project) + +**2.1 Remove RCLONE_TEST Files** ✅ +- [x] Update all bisync profiles: `check_access=False` +- [x] Remove RCLONE_TEST creation from `setup_cloud_bisync()` +- [x] Remove RCLONE_TEST upload logic +- [ ] Update documentation + +**2.2 Sync Bucket Root (All Projects)** ✅ +- [x] Change remote path from `bucket:/basic-memory` to `bucket:/` in `build_bisync_command()` +- [x] Update `setup_cloud_bisync()` to use bucket root +- [ ] Test with multiple projects + +**2.3 Project Auto-Registration (Bisync)** ✅ +- [x] Add `fetch_cloud_projects()` function (GET /proxy/projects/projects) +- [x] Add `scan_local_directories()` function +- [x] Add `create_cloud_project()` function (POST /proxy/projects/projects) +- [x] Integrate into `run_bisync()`: fetch → scan → create missing → sync +- [x] Wait for API 201 response before syncing + +**2.4 Bisync Directory Configuration** ✅ +- [x] Add `--dir` parameter to `bm cloud bisync-setup` +- [x] Store bisync directory in config +- [x] Default to `~/basic-memory-cloud-sync/` +- [x] Add `validate_bisync_directory()` safety check +- [x] Update `get_default_mount_path()` to return fixed `~/basic-memory-cloud/` + +**2.5 Sync/Status API Infrastructure** ✅ (commit d48b1dc) +- [x] Create `POST /{project}/project/sync` endpoint for background sync +- [x] Create `POST /{project}/project/status` endpoint for scan-only status +- [x] Create `SyncReportResponse` Pydantic schema +- [x] Refactor CLI `sync` command to use API endpoint +- [x] Refactor CLI `status` command to use API endpoint +- [x] Create `command_utils.py` with shared `run_sync()` function +- [x] Update `notify_container_sync()` to call `run_sync()` for each project +- [x] Update all tests to match new API-based implementation + +### Phase 3: Sync Command Dual Mode ✅ + +**3.1 Update `bm sync` Command** ✅ +- [x] Check `config.cloud_mode` at start +- [x] If `cloud_mode=false`: Run existing local sync +- [x] If `cloud_mode=true`: Run bisync +- [x] Add `--watch` parameter for continuous sync +- [x] Add `--interval` parameter (default 60 seconds) +- [x] Error if `--watch` used in local mode with helpful message + +**3.2 Watch Mode for Bisync** ✅ +- [x] Implement `run_bisync_watch()` with interval loop +- [x] Add `--interval` parameter (default 60 seconds) +- [x] Handle errors gracefully, continue on failure +- [x] Show sync progress and status + +**3.3 Integrity Check Command** ✅ +- [x] Implement `bm cloud check` command using `rclone check` +- [x] Read-only operation that verifies file matching +- [x] Error with helpful messages if rclone/bisync not set up +- [x] Support `--one-way` flag for faster checks +- [x] Transparent about rclone implementation +- [x] Suggest `bm sync` to resolve differences + +**Implementation Notes:** +- `bm sync` adapts to cloud mode automatically - users don't need separate commands +- `bm cloud bisync` kept for power users with full options (--dry-run, --resync, --profile, --verbose) +- `bm cloud check` provides integrity verification without transferring data +- Design philosophy: Simplicity for everyday use, transparency about implementation + +### Phase 4: Remove Duplicate Commands & Cloud Mode Auth ✅ + +**4.0 Cloud Mode Authentication** ✅ +- [x] Update `async_client.py` to support dual auth sources +- [x] FastMCP context auth (cloud service mode) via `inject_auth_header()` +- [x] JWT token file auth (CLI cloud mode) via `CLIAuth.get_valid_token()` +- [x] Automatic token refresh for CLI cloud mode +- [x] Remove `BASIC_MEMORY_PROXY_URL` environment variable dependency +- [x] Simplify to use only `config.cloud_mode` + `config.cloud_host` + +**4.1 Delete `bm cloud project` Commands** ✅ +- [x] Remove `bm cloud project list` (use `bm project list`) +- [x] Remove `bm cloud project add` (use `bm project add`) +- [x] Update `core_commands.py` to remove project_app subcommands +- [x] Keep only: `login`, `logout`, `status`, `setup`, `mount`, `unmount`, bisync commands +- [x] Remove unused imports (Table, generate_permalink, os) +- [x] Clean up environment variable references in login/logout + +**4.2 CLI Command Cloud Mode Integration** ✅ +- [x] Add runtime `cloud_mode_enabled` checks to all CLI commands +- [x] Update `list_projects()` to conditionally authenticate based on cloud mode +- [x] Update `remove_project()` to conditionally authenticate based on cloud mode +- [x] Update `run_sync()` to conditionally authenticate based on cloud mode +- [x] Update `get_project_info()` to conditionally authenticate based on cloud mode +- [x] Update `run_status()` to conditionally authenticate based on cloud mode +- [x] Remove auth from `set_default_project()` (local-only command, no cloud version) +- [x] Create CLI integration tests (`test-int/cli/`) to validate both local and cloud modes +- [x] Replace mock-heavy CLI tests with integration tests (deleted 5 mock test files) + +**4.3 OAuth Authentication Fixes** ✅ +- [x] Restore missing `SettingsConfigDict` in `BasicMemoryConfig` +- [x] Fix environment variable reading with `BASIC_MEMORY_` prefix +- [x] Fix `.env` file loading +- [x] Fix extra field handling for config files +- [x] Resolve `bm cloud login` OAuth failure ("Something went wrong" error) +- [x] Implement PKCE (Proof Key for Code Exchange) for device flow +- [x] Generate code verifier and SHA256 challenge for device authorization +- [x] Send code_verifier with token polling requests +- [x] Support both PKCE-required and PKCE-optional OAuth clients +- [x] Verify authentication flow works end-to-end with staging and production +- [x] Document WorkOS requirement: redirect URI must be configured even for device flow + +**4.4 Update Documentation** +- [ ] Update `cloud-cli.md` with cloud mode toggle workflow +- [ ] Document `bm cloud login` → use normal commands +- [ ] Add examples of cloud mode usage +- [ ] Document mount vs bisync directory isolation +- [ ] Add troubleshooting section + +### Phase 5: Mount Updates ✅ + +**5.1 Fixed Mount Directory** ✅ +- [x] Change mount path to `~/basic-memory-cloud/` (fixed, no tenant ID) +- [x] Update `get_default_mount_path()` function +- [x] Remove configurability (fixed location) +- [x] Update mount commands to use new path + +**5.2 Mount at Bucket Root** ✅ +- [x] Ensure mount uses bucket root (not subdirectory) +- [x] Test with multiple projects +- [x] Verify all projects visible in mount + +**Implementation:** Mount uses fixed `~/basic-memory-cloud/` directory and syncs entire bucket root `basic-memory-{tenant_id}:{bucket_name}` for all projects. + +### Phase 6: Safety & Validation ✅ + +**6.1 Directory Conflict Prevention** ✅ +- [x] Implement `validate_bisync_directory()` check +- [x] Detect if bisync dir == mount dir +- [x] Detect if bisync dir is currently mounted +- [x] Show clear error messages with solutions + +**6.2 State Management** ✅ +- [x] Use `--workdir` for bisync state +- [x] Store state in `~/.basic-memory/bisync-state/{tenant-id}/` +- [x] Ensure state directory created before bisync + +**Implementation:** `validate_bisync_directory()` prevents conflicts by checking directory equality and mount status. State managed in isolated `~/.basic-memory/bisync-state/{tenant-id}/` directory using `--workdir` flag. + +### Phase 7: Cloud-Side Implementation (Deferred to Cloud Repo) + +**7.1 Project Discovery Service (Cloud)** - Deferred +- [ ] Create `ProjectDiscoveryService` background job +- [ ] Scan `/app/data/` every 2 minutes +- [ ] Auto-register new directories as projects +- [ ] Log discovery events +- [ ] Handle errors gracefully + +**7.2 Project API Updates (Cloud)** - Deferred +- [ ] Ensure `POST /proxy/projects/projects` creates directory synchronously +- [ ] Return 201 with project details +- [ ] Ensure directory ready immediately after creation + +**Note:** Phase 7 is cloud-side work that belongs in the basic-memory-cloud repository. The CLI-side implementation (Phase 2.3 auto-registration) is complete and working - it calls the existing cloud API endpoints. + +### Phase 8: Testing & Documentation + +**8.1 Test Scenarios** +- [x] Test: Cloud mode toggle (login/logout) +- [x] Test: Local-first project creation (bisync) +- [x] Test: Cloud-first project creation (API) +- [x] Test: Multi-project bidirectional sync +- [x] Test: MCP tools in cloud mode +- [x] Test: Watch mode continuous sync +- [x] Test: Safety profile protection (max_delete implemented) +- [x] Test: No RCLONE_TEST files (check_access=False in all profiles) +- [x] Test: Mount/bisync directory isolation (validate_bisync_directory) +- [x] Test: Integrity check command (bm cloud check) + +**8.2 Documentation** +- [x] Update cloud-cli.md with cloud mode instructions +- [x] Document Dropbox model paradigm +- [x] Update command reference with new commands +- [x] Document `bm sync` dual mode behavior +- [x] Document `bm cloud check` command +- [x] Document directory structure and fixed paths +- [ ] Update README with quick start +- [ ] Create migration guide for existing users +- [ ] Create video/GIF demos + +### Success Criteria Checklist + +- [x] `bm cloud login` enables cloud mode for all commands +- [x] `bm cloud logout` reverts to local mode +- [x] `bm project`, `bm tool`, `bm sync` work transparently in both modes +- [x] `bm sync` runs bisync in cloud mode, local sync in local mode +- [x] Single sync operation handles all projects bidirectionally +- [x] Local directories auto-create cloud projects via API +- [x] Cloud projects auto-sync to local directories +- [x] No RCLONE_TEST files in user directories +- [x] Bisync profiles provide safety via `max_delete` limits +- [x] `bm sync --watch` enables continuous sync +- [x] No duplicate `bm cloud project` commands (removed) +- [x] `bm cloud check` command for integrity verification +- [ ] Documentation covers cloud mode toggle and workflows +- [ ] Edge cases handled gracefully with clear errors + +## How (High Level) + +### Architecture Overview + +**Cloud Mode Toggle:** +``` +┌─────────────────────────────────────┐ +│ bm cloud login │ +│ ├─ Authenticate via OAuth │ +│ ├─ Set cloud_mode: true in config │ +│ └─ Set BASIC_MEMORY_PROXY_URL │ +└─────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────┐ +│ All CLI commands use async_client │ +│ ├─ async_client checks proxy URL │ +│ ├─ If set: HTTP to cloud │ +│ └─ If not: Local ASGI │ +└─────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────┐ +│ bm project add "work" │ +│ bm tool write-note ... │ +│ bm sync (triggers bisync) │ +│ → All work against cloud │ +└─────────────────────────────────────┘ +``` + +**Storage Hierarchy:** +``` +Cloud Container: Bucket: Local Sync Dir: +/app/data/ (mounted) ←→ production-tenant-{id}/ ←→ ~/basic-memory-cloud-sync/ +├── basic-memory/ ├── basic-memory/ ├── basic-memory/ +│ ├── notes/ │ ├── notes/ │ ├── notes/ +│ └── concepts/ │ └── concepts/ │ └── concepts/ +├── work-project/ ├── work-project/ ├── work-project/ +│ └── tasks/ │ └── tasks/ │ └── tasks/ +└── personal/ └── personal/ └── personal/ + └── journal/ └── journal/ └── journal/ + +Bidirectional sync via rclone bisync +``` + +### Sync Flow + +**`bm sync` execution (in cloud mode):** + +1. **Check cloud mode** + ```python + if not config.cloud_mode: + # Run normal local file sync + run_local_sync() + return + + # Cloud mode: Run bisync + ``` + +2. **Fetch cloud projects** + ```python + # GET /proxy/projects/projects (via async_client) + cloud_projects = fetch_cloud_projects() + cloud_project_names = {p["name"] for p in cloud_projects["projects"]} + ``` + +3. **Scan local sync directory** + ```python + sync_dir = config.bisync_config["sync_dir"] # ~/basic-memory-cloud-sync + local_dirs = [d.name for d in sync_dir.iterdir() + if d.is_dir() and not d.name.startswith('.')] + ``` + +4. **Create missing cloud projects** + ```python + for dir_name in local_dirs: + if dir_name not in cloud_project_names: + # POST /proxy/projects/projects (via async_client) + create_cloud_project(name=dir_name) + # Blocks until 201 response + ``` + +5. **Run bisync on bucket root** + ```bash + rclone bisync \ + ~/basic-memory-cloud-sync \ + basic-memory-{tenant}:{bucket} \ + --filters-file ~/.basic-memory/.bmignore.rclone \ + --conflict-resolve=newer \ + --max-delete=25 + # Syncs ALL project subdirectories bidirectionally + ``` + +6. **Notify cloud to refresh** (commit d48b1dc) + ```python + # After rclone bisync completes, sync each project's database + for project in cloud_projects: + # POST /{project}/project/sync (via async_client) + # Triggers background sync for this project + await run_sync(project=project_name) + ``` + +### Key Changes + +**1. Cloud Mode via Config** + +**Config changes:** +```python +class Config: + cloud_mode: bool = False + cloud_host: str = "https://cloud.basicmemory.com" + bisync_config: dict = { + "profile": "balanced", + "sync_dir": "~/basic-memory-cloud-sync" + } +``` + +**async_client.py behavior:** +```python +def create_client() -> AsyncClient: + # Check config first, then environment + config = ConfigManager().config + proxy_url = os.getenv("BASIC_MEMORY_PROXY_URL") or \ + (config.cloud_host if config.cloud_mode else None) + + if proxy_url: + return AsyncClient(base_url=proxy_url) # HTTP to cloud + else: + return AsyncClient(transport=ASGITransport(...)) # Local ASGI +``` + +**2. Login/Logout Sets Cloud Mode** + +```python +# bm cloud login +async def login(): + # Existing OAuth flow... + success = await auth.login() + if success: + config.cloud_mode = True + config.save() + os.environ["BASIC_MEMORY_PROXY_URL"] = config.cloud_host +``` + +```python +# bm cloud logout +def logout(): + config.cloud_mode = False + config.save() + os.environ.pop("BASIC_MEMORY_PROXY_URL", None) +``` + +**3. Remove Duplicate Commands** + +**Delete:** +- `bm cloud project list` → use `bm project list` +- `bm cloud project add` → use `bm project add` + +**Keep:** +- `bm cloud login` - Enable cloud mode +- `bm cloud logout` - Disable cloud mode +- `bm cloud status` - Show current mode & connection +- `bm cloud setup` - Initial bisync setup +- `bm cloud bisync` - Power-user command with full options +- `bm cloud check` - Verify file integrity between local and cloud + +**4. Sync Command Dual Mode** + +```python +# bm sync +def sync_command(watch: bool = False, profile: str = "balanced"): + config = ConfigManager().config + + if config.cloud_mode: + # Run bisync for cloud sync + run_bisync(profile=profile, watch=watch) + else: + # Run local file sync + run_local_sync() +``` + +**5. Remove RCLONE_TEST Files** + +```python +# All profiles: check_access=False +BISYNC_PROFILES = { + "safe": RcloneBisyncProfile(check_access=False, max_delete=10), + "balanced": RcloneBisyncProfile(check_access=False, max_delete=25), + "fast": RcloneBisyncProfile(check_access=False, max_delete=50), +} +``` + +**6. Sync Bucket Root (All Projects)** + +```python +# Sync entire bucket, not subdirectory +rclone_remote = f"basic-memory-{tenant_id}:{bucket_name}" +``` + +## How to Evaluate + +### Test Scenarios + +**1. Cloud Mode Toggle** +```bash +# Start in local mode +bm project list +# → Shows local projects + +# Enable cloud mode +bm cloud login +# → Authenticates, sets cloud_mode=true + +bm project list +# → Now shows cloud projects (same command!) + +# Disable cloud mode +bm cloud logout + +bm project list +# → Back to local projects +``` + +**Expected:** ✅ Single command works in both modes + +**2. Local-First Project Creation (Cloud Mode)** +```bash +# Enable cloud mode +bm cloud login + +# Create new project locally in sync dir +mkdir ~/basic-memory-cloud-sync/my-research +echo "# Research Notes" > ~/basic-memory-cloud-sync/my-research/index.md + +# Run sync (triggers bisync in cloud mode) +bm sync + +# Verify: +# - Cloud project created automatically via API +# - Files synced to bucket:/my-research/ +# - Cloud database updated +# - `bm project list` shows new project +``` + +**Expected:** ✅ Project visible in cloud project list + +**3. Cloud-First Project Creation** +```bash +# In cloud mode +bm project add "work-notes" +# → Creates project on cloud (via async_client HTTP) + +# Run sync +bm sync + +# Verify: +# - Local directory ~/basic-memory-cloud-sync/work-notes/ created +# - Files sync bidirectionally +# - Can use `bm tool write-note` to add content remotely +``` + +**Expected:** ✅ Project accessible via all CLI commands + +**4. Multi-Project Bidirectional Sync** +```bash +# Setup: 3 projects in cloud mode +# Modify files in all 3 locally and remotely + +bm sync + +# Verify: +# - All 3 projects sync simultaneously +# - Changes propagate correctly +# - No cross-project interference +``` + +**Expected:** ✅ All projects in sync state + +**5. MCP Tools Work in Cloud Mode** +```bash +# In cloud mode +bm tool write-note \ + --title "Meeting Notes" \ + --folder "work-notes" \ + --content "Discussion points..." + +# Verify: +# - Note created on cloud (via async_client HTTP) +# - Next `bm sync` pulls note to local +# - Note appears in ~/basic-memory-cloud-sync/work-notes/ +``` + +**Expected:** ✅ Tools work transparently in cloud mode + +**6. Watch Mode Continuous Sync** +```bash +# In cloud mode +bm sync --watch + +# While running: +# - Create local folder → auto-creates cloud project +# - Edit files locally → syncs to cloud +# - Edit files remotely → syncs to local +# - Create project via API → appears locally + +# Verify: +# - Continuous bidirectional sync +# - New projects handled automatically +# - No manual intervention needed +``` + +**Expected:** ✅ Seamless continuous sync + +**7. Safety Profile Protection** +```bash +# Create project with 15 files locally +# Delete project from cloud (simulate error) + +bm sync --profile safe + +# Verify: +# - Bisync detects 15 pending deletions +# - Exceeds max_delete=10 limit +# - Aborts with clear error +# - No files deleted locally +``` + +**Expected:** ✅ Safety limit prevents data loss + +**8. No RCLONE_TEST Files** +```bash +# After setup and multiple syncs +ls -la ~/basic-memory-cloud-sync/ + +# Verify: +# - No RCLONE_TEST files +# - No .rclone state files (in ~/.basic-memory/bisync-state/) +# - Clean directory structure +``` + +**Expected:** ✅ User directory stays clean + +### Success Criteria + +- [x] `bm cloud login` enables cloud mode for all commands +- [x] `bm cloud logout` reverts to local mode +- [x] `bm project`, `bm tool`, `bm sync` work in both modes transparently +- [x] `bm sync` runs bisync in cloud mode, local sync in local mode +- [x] Single sync operation handles all projects bidirectionally +- [x] Local directories auto-create cloud projects via API +- [x] Cloud projects auto-sync to local directories +- [x] No RCLONE_TEST files in user directories +- [x] Bisync profiles provide safety via `max_delete` limits +- [x] `bm sync --watch` enables continuous sync +- [x] No duplicate `bm cloud project` commands (removed) +- [x] `bm cloud check` command for integrity verification +- [ ] Documentation covers cloud mode toggle and workflows +- [ ] Edge cases handled gracefully with clear errors + +## Notes + +### API Contract + +**Cloud must provide:** + +1. **Project Management APIs:** + - `GET /proxy/projects/projects` - List all projects + - `POST /proxy/projects/projects` - Create project synchronously + - `POST /proxy/sync` - Trigger cache refresh + +2. **Project Discovery Service (Background):** + - **Purpose**: Auto-register projects created via mount, direct bucket uploads, or any non-API method + - **Interval**: Every 2 minutes + - **Behavior**: + - Scan `/app/data/` for directories + - Register any directory not already in project database + - Log discovery events + - **Implementation**: + ```python + class ProjectDiscoveryService: + """Background service to auto-discover projects from filesystem.""" + + async def run(self): + """Scan /app/data/ and register new project directories.""" + data_path = Path("/app/data") + + for dir_path in data_path.iterdir(): + # Skip hidden and special directories + if not dir_path.is_dir() or dir_path.name.startswith('.'): + continue + + project_name = dir_path.name + + # Check if project already registered + project = await self.project_repo.get_by_name(project_name) + if not project: + # Auto-register new project + await self.project_repo.create( + name=project_name, + path=str(dir_path) + ) + logger.info(f"Auto-discovered project: {project_name}") + ``` + +**Project Creation (API-based):** +- API creates `/app/data/{project-name}/` directory +- Registers project in database +- Returns 201 with project details +- Directory ready for bisync immediately + +**Project Creation (Discovery-based):** +- User creates folder via mount: `~/basic-memory-cloud/new-project/` +- Files appear in `/app/data/new-project/` (mounted bucket) +- Discovery service finds directory on next scan (within 2 minutes) +- Auto-registers as project +- User sees project in `bm project list` after discovery + +**Why Both Methods:** +- **API**: Immediate registration when using bisync (client-side scan + API call) +- **Discovery**: Delayed registration when using mount (no API call hook) +- **Result**: Projects created ANY way (API, mount, bisync, WebDAV) eventually registered +- **Trade-off**: 2-minute delay for mount-created projects is acceptable + +### Mount vs Bisync Directory Isolation + +**Critical Safety Requirement**: Mount and bisync MUST use different directories to prevent conflicts. + +**The Dropbox Model Applied:** + +Both mount and bisync operate at **bucket level** (all projects), following the Dropbox/iCloud paradigm: + +``` +~/basic-memory-cloud/ # Mount: Read-through cache (like Dropbox folder) +├── work-notes/ +├── personal/ +└── research/ + +~/basic-memory-cloud-sync/ # Bisync: Bidirectional sync (like Dropbox sync folder) +├── work-notes/ +├── personal/ +└── research/ +``` + +**Mount Directory (Fixed):** +```bash +# Fixed location, not configurable +~/basic-memory-cloud/ +``` +- **Scope**: Entire bucket (all projects) +- **Method**: NFS mount via `rclone nfsmount` +- **Behavior**: Read-through cache to cloud bucket +- **Credentials**: One IAM credential set per tenant +- **Process**: One rclone mount process +- **Use Case**: Quick access, browsing, light editing +- **Known Issue**: Obsidian compatibility problems with NFS +- **Not Configurable**: Fixed location prevents user error + +**Why Fixed Location:** +- One mount point per machine (like `/Users/you/Dropbox`) +- Prevents credential proliferation (one credential set, not N) +- Prevents multiple mount processes (resource efficiency) +- Familiar pattern users already understand +- Simple operations: `mount` once, `unmount` once + +**Bisync Directory (User Configurable):** +```bash +# Default location +~/basic-memory-cloud-sync/ + +# User can override +bm cloud setup --dir ~/my-knowledge-base +``` +- **Scope**: Entire bucket (all projects) +- **Method**: Bidirectional sync via `rclone bisync` +- **Behavior**: Full local copy with periodic sync +- **Credentials**: Same IAM credential set as mount +- **Use Case**: Full offline access, reliable editing, Obsidian support +- **Configurable**: Users may want specific locations (external drive, existing folder structure) + +**Why User Configurable:** +- Users have preferences for where local copies live +- May want sync folder on external drive +- May want to integrate with existing folder structure +- Default works for most, option available for power users + +**Conflict Prevention:** +```python +def validate_bisync_directory(bisync_dir: Path): + """Ensure bisync directory doesn't conflict with mount.""" + mount_dir = Path.home() / "basic-memory-cloud" + + if bisync_dir.resolve() == mount_dir.resolve(): + raise BisyncError( + f"Cannot use {bisync_dir} for bisync - it's the mount directory!\n" + f"Mount and bisync must use different directories.\n\n" + f"Options:\n" + f" 1. Use default: ~/basic-memory-cloud-sync/\n" + f" 2. Specify different directory: --dir ~/my-sync-folder" + ) + + # Check if mount is active at this location + result = subprocess.run(["mount"], capture_output=True, text=True) + if str(bisync_dir) in result.stdout and "rclone" in result.stdout: + raise BisyncError( + f"{bisync_dir} is currently mounted via 'bm cloud mount'\n" + f"Cannot use mounted directory for bisync.\n\n" + f"Either:\n" + f" 1. Unmount first: bm cloud unmount\n" + f" 2. Use different directory for bisync" + ) +``` + +**Why This Matters:** +- Mounting and syncing the SAME directory would create infinite loops +- rclone mount → bisync detects changes → syncs to bucket → mount sees changes → triggers bisync → ∞ +- Separate directories = clean separation of concerns +- Mount is read-heavy caching layer, bisync is write-heavy bidirectional sync + +### Future Enhancements + +**Phase 2 (Not in this spec):** +- **Near Real-Time Sync**: Integrate `watch_service.py` with cloud mode + - Watch service detects local changes (already battle-tested) + - Queue changes in memory + - Use `rclone copy` for individual file sync (near instant) + - Example: `rclone copyto ~/sync/project/file.md tenant:{bucket}/project/file.md` + - Fallback to full `rclone bisync` every N seconds for bidirectional changes + - Provides near real-time sync without polling overhead +- Per-project bisync profiles (different safety levels per project) +- Selective project sync (exclude specific projects from sync) +- Project deletion workflow (cascade to cloud/local) +- Conflict resolution UI/CLI + +**Phase 3:** +- Project sharing between tenants +- Incremental backup/restore +- Sync statistics and bandwidth monitoring +- Mobile app integration with cloud mode + +### Related Specs + +- **SPEC-8**: TigrisFS Integration - Original bisync implementation +- **SPEC-6**: Explicit Project Parameter Architecture - Multi-project foundations +- **SPEC-5**: CLI Cloud Upload via WebDAV - Cloud file operations + +### Implementation Notes + +**Architectural Simplifications:** +- **Unified CLI**: Eliminated duplicate commands by using mode toggle +- **Single Entry Point**: All commands route through `async_client` which handles mode +- **Config-Driven**: Cloud mode stored in persistent config, not just environment +- **Transparent Routing**: Existing commands work without modification in cloud mode + +**Complexity Trade-offs:** +- Removed: Separate `bm cloud project` command namespace +- Removed: Complex state detection for new projects +- Removed: RCLONE_TEST marker file management +- Added: Simple cloud_mode flag and config integration +- Added: Simple project list comparison before sync +- Relied on: Existing bisync profile safety mechanisms +- Result: Significantly simpler, more maintainable code + +**User Experience:** +- **Mental Model**: "Toggle cloud mode, use normal commands" +- **No Learning Curve**: Same commands work locally and in cloud +- **Minimal Config**: Just login/logout to switch modes +- **Safety**: Profile system gives users control over safety/speed trade-offs +- **"Just Works"**: Create folders anywhere, they sync automatically + +**Migration Path:** +- Existing `bm cloud project` users: Use `bm project` instead +- Existing `bm cloud bisync` becomes `bm sync` in cloud mode +- Config automatically migrates on first `bm cloud login` + + +## Testing + + +Initial Setup (One Time) + +1. Login to cloud and enable cloud mode: +bm cloud login +# → Authenticates via OAuth +# → Sets cloud_mode=true in config +# → Sets BASIC_MEMORY_PROXY_URL environment variable +# → All CLI commands now route to cloud + +2. Check cloud mode status: +bm cloud status +# → Shows: Mode: Cloud (enabled) +# → Shows: Host: https://cloud.basicmemory.com +# → Checks cloud health + +3. Set up bidirectional sync: +bm cloud bisync-setup +# Or with custom directory: +bm cloud bisync-setup --dir ~/my-sync-folder + +# This will: +# → Install rclone (if not already installed) +# → Get tenant info (tenant_id, bucket_name) +# → Generate scoped IAM credentials +# → Configure rclone with credentials +# → Create sync directory (default: ~/basic-memory-cloud-sync/) +# → Validate no conflict with mount directory +# → Run initial --resync to establish baseline + +Normal Usage + +4. Create local project and sync: +# Create a local project directory +mkdir ~/basic-memory-cloud-sync/my-research +echo "# Research Notes" > ~/basic-memory-cloud-sync/my-research/readme.md + +# Run sync +bm cloud bisync + +# Auto-magic happens: +# → Checks for new local directories +# → Finds "my-research" not in cloud +# → Creates project on cloud via POST /proxy/projects/projects +# → Runs bidirectional sync (all projects) +# → Syncs to bucket root (all projects synced together) + +5. Watch mode for continuous sync: +bm cloud bisync --watch +# Or with custom interval: +bm cloud bisync --watch --interval 30 + +# → Syncs every 60 seconds (or custom interval) +# → Auto-registers new projects on each run +# → Press Ctrl+C to stop + +6. Check bisync status: +bm cloud bisync-status +# → Shows tenant ID +# → Shows sync directory path +# → Shows initialization status +# → Shows last sync time +# → Lists available profiles (safe/balanced/fast) + +7. Manual sync with different profiles: +# Safe mode (max 10 deletes, preserves conflicts) +bm cloud bisync --profile safe + +# Balanced mode (max 25 deletes, auto-resolve to newer) - default +bm cloud bisync --profile balanced + +# Fast mode (max 50 deletes, skip verification) +bm cloud bisync --profile fast + +8. Dry run to preview changes: +bm cloud bisync --dry-run +# → Shows what would be synced without making changes + +9. Force resync (if needed): +bm cloud bisync --resync +# → Establishes new baseline +# → Use if sync state is corrupted + +10. Check file integrity: +bm cloud check +# → Verifies all files match between local and cloud +# → Read-only operation (no data transfer) +# → Shows differences if any found + +# Faster one-way check +bm cloud check --one-way +# → Only checks for missing files on destination + +Verify Cloud Mode Integration + +11. Test that all commands work in cloud mode: +# List cloud projects (not local) +bm project list + +# Create project on cloud +bm project add "work-notes" + +# Use MCP tools against cloud +bm tool write-note --title "Test" --folder "my-research" --content "Hello" + +# All of these work against cloud because cloud_mode=true + +12. Switch back to local mode: +bm cloud logout +# → Sets cloud_mode=false +# → Clears BASIC_MEMORY_PROXY_URL +# → All commands now work locally again + +Expected Directory Structure + +~/basic-memory-cloud-sync/ # Your local sync directory +├── my-research/ # Auto-created cloud project +│ ├── readme.md +│ └── notes.md +├── work-notes/ # Another project +│ └── tasks.md +└── personal/ # Another project + └── journal.md + +# All sync bidirectionally with: +bucket:/ # Cloud bucket root +├── my-research/ +├── work-notes/ +└── personal/ + +Key Points to Test + +1. ✅ Cloud mode toggle works (login/logout) +2. ✅ Bisync setup validates directory (no conflict with mount) +3. ✅ Local directories auto-create cloud projects +4. ✅ All projects sync together (bucket root) +5. ✅ No RCLONE_TEST files created +6. ✅ Changes sync bidirectionally +7. ✅ Watch mode continuous sync works +8. ✅ Profile safety limits work (max_delete) +9. ✅ `bm sync` adapts to cloud mode automatically +10. ✅ `bm cloud check` verifies file integrity without side effects diff --git a/src/basic_memory/api/app.py b/src/basic_memory/api/app.py index 626a91a18..cc2e23479 100644 --- a/src/basic_memory/api/app.py +++ b/src/basic_memory/api/app.py @@ -19,7 +19,6 @@ resource, search, prompt_router, - webdav, ) from basic_memory.config import ConfigManager from basic_memory.services.initialization import initialize_file_sync, initialize_app @@ -77,7 +76,6 @@ async def lifespan(app: FastAPI): # pragma: no cover app.include_router(directory_router.router, prefix="/{project}") app.include_router(prompt_router.router, prefix="/{project}") app.include_router(importer_router.router, prefix="/{project}") -app.include_router(webdav.router, prefix="/{project}") # Project resource router works accross projects app.include_router(project.project_resource_router) diff --git a/src/basic_memory/api/routers/__init__.py b/src/basic_memory/api/routers/__init__.py index c71c1f717..e08b61c0f 100644 --- a/src/basic_memory/api/routers/__init__.py +++ b/src/basic_memory/api/routers/__init__.py @@ -7,6 +7,5 @@ from . import resource_router as resource from . import search_router as search from . import prompt_router as prompt -from . import webdav_router as webdav -__all__ = ["knowledge", "management", "memory", "project", "resource", "search", "prompt", "webdav"] +__all__ = ["knowledge", "management", "memory", "project", "resource", "search", "prompt"] diff --git a/src/basic_memory/api/routers/project_router.py b/src/basic_memory/api/routers/project_router.py index 8c871f6e8..b78234624 100644 --- a/src/basic_memory/api/routers/project_router.py +++ b/src/basic_memory/api/routers/project_router.py @@ -1,11 +1,17 @@ """Router for project management.""" import os -from fastapi import APIRouter, HTTPException, Path, Body +from fastapi import APIRouter, HTTPException, Path, Body, BackgroundTasks from typing import Optional +from loguru import logger -from basic_memory.deps import ProjectServiceDep, ProjectPathDep -from basic_memory.schemas import ProjectInfoResponse +from basic_memory.deps import ( + ProjectConfigDep, + ProjectServiceDep, + ProjectPathDep, + SyncServiceDep, +) +from basic_memory.schemas import ProjectInfoResponse, SyncReportResponse from basic_memory.schemas.project_info import ( ProjectList, ProjectItem, @@ -97,6 +103,54 @@ async def update_project( raise HTTPException(status_code=400, detail=str(e)) +# Sync project filesystem +@project_router.post("/sync") +async def sync_project( + background_tasks: BackgroundTasks, + sync_service: SyncServiceDep, + project_config: ProjectConfigDep, +): + """Force project filesystem sync to database. + + Scans the project directory and updates the database with any new or modified files. + + Args: + background_tasks: FastAPI background tasks + sync_service: Sync service for this project + project_config: Project configuration + + Returns: + Response confirming sync was initiated + """ + background_tasks.add_task(sync_service.sync, project_config.home, project_config.name) + logger.info(f"Filesystem sync initiated for project: {project_config.name}") + + return { + "status": "sync_started", + "message": f"Filesystem sync initiated for project '{project_config.name}'", + } + + +@project_router.post("/status", response_model=SyncReportResponse) +async def project_sync_status( + sync_service: SyncServiceDep, + project_config: ProjectConfigDep, +) -> SyncReportResponse: + """Scan directory for changes compared to database state. + + Args: + sync_service: Sync service for this project + project_config: Project configuration + + Returns: + Scan report with details on files that need syncing + """ + logger.info(f"Scanning filesystem for project: {project_config.name}") + sync_report = await sync_service.scan(project_config.home) + + return SyncReportResponse.from_sync_report(sync_report) + + # List all available projects @project_resource_router.get("/projects", response_model=ProjectList) async def list_projects( @@ -259,7 +313,7 @@ async def get_default_project( # Synchronize projects between config and database -@project_resource_router.post("/sync", response_model=ProjectStatusResponse) +@project_resource_router.post("/config/sync", response_model=ProjectStatusResponse) async def synchronize_projects( project_service: ProjectServiceDep, ) -> ProjectStatusResponse: diff --git a/src/basic_memory/api/routers/webdav_router.py b/src/basic_memory/api/routers/webdav_router.py deleted file mode 100644 index f01d35f6f..000000000 --- a/src/basic_memory/api/routers/webdav_router.py +++ /dev/null @@ -1,353 +0,0 @@ -"""WebDAV router for basic-memory project uploads.""" - -import os -import shutil -from datetime import datetime -from pathlib import Path -import aiofiles -from loguru import logger - -from basic_memory.deps import ProjectPathDep, ProjectServiceDep -from fastapi import APIRouter, Request, Response, HTTPException -from fastapi.responses import StreamingResponse - -router = APIRouter( - prefix="/webdav", - tags=["webdav"], -) - - -async def get_project_path_or_404(project_service: ProjectServiceDep, project: str) -> Path: - found_project = await project_service.get_project(project) - if not found_project: - raise HTTPException(status_code=404, detail=f"Project: '{project}' does not exist") - return Path(found_project.path) - - -async def get_project_file_path_or_404(project_path: Path, path: str) -> Path: - file_path = Path(project_path / path) - if not file_path.exists(): - raise HTTPException(status_code=404, detail=f"File: '{path}' does not exist") - return file_path - - -@router.api_route("/{path:path}", methods=["OPTIONS"]) -async def webdav_options( - path: str, - project: ProjectPathDep, - project_service: ProjectServiceDep, -): - """WebDAV OPTIONS endpoint.""" - project_path = await get_project_path_or_404(project_service, project) - file_path = await get_project_file_path_or_404(project_path, path) - return await _webdav_options(file_path) - - -@router.api_route("/{path:path}", methods=["PROPFIND"]) -async def webdav_propfind( - path: str, - project: ProjectPathDep, - project_service: ProjectServiceDep, -): - """WebDAV PROPFIND endpoint.""" - project_path = await get_project_path_or_404(project_service, project) - file_path = await get_project_file_path_or_404(project_path, path) - return await _webdav_propfind(project, project_path, file_path) - - -@router.api_route("/{path:path}", methods=["GET"]) -async def webdav_get( - path: str, - project: ProjectPathDep, - project_service: ProjectServiceDep, -): - """WebDAV GET endpoint.""" - project_path = await get_project_path_or_404(project_service, project) - file_path = await get_project_file_path_or_404(project_path, path) - return await _webdav_get(file_path) - - -@router.api_route("/{path:path}", methods=["PUT"]) -async def webdav_put( - request: Request, - path: str, - project: ProjectPathDep, - project_service: ProjectServiceDep, -): - """WebDAV PUT endpoint.""" - project_path = await get_project_path_or_404(project_service, project) - file_path = Path(project_path / path) - return await _webdav_put(request, project, file_path) - - -@router.api_route("/{path:path}", methods=["DELETE"]) -async def webdav_delete( - path: str, - project: ProjectPathDep, - project_service: ProjectServiceDep, -): - """WebDAV DELETE endpoint.""" - project_path = await get_project_path_or_404(project_service, project) - file_path = await get_project_file_path_or_404(project_path, path) - return await _webdav_delete(project, file_path) - - -@router.api_route("/{path:path}", methods=["MKCOL"]) -async def webdav_mkcol( - path: str, - project: ProjectPathDep, - project_service: ProjectServiceDep, -): - """WebDAV MKCOL endpoint.""" - project_path = await get_project_path_or_404(project_service, project) - dir_path = Path(project_path / path) - return await _webdav_mkcol(project, dir_path) - - -# Handle WebDAV root -@router.api_route("/", methods=["OPTIONS", "PROPFIND"]) -async def webdav_root( - request: Request, - project: ProjectPathDep, - project_service: ProjectServiceDep, -): - """WebDAV root endpoint.""" - project_path = await get_project_path_or_404(project_service, project) - method = request.method - if method == "OPTIONS": - return await _webdav_options(project_path) - else: - return await _webdav_propfind(project, project_path, project_path) - - -async def _webdav_options(file_path: Path) -> Response: - """Handle WebDAV OPTIONS request.""" - file_size = file_path.stat().st_size - return Response( - status_code=204, - headers={ - "DAV": "1,2", - "MS-Author-Via": "DAV", - "Allow": "OPTIONS,GET,HEAD,POST,DELETE,TRACE,PROPFIND,PROPPATCH,COPY,MOVE,LOCK,UNLOCK,PUT", - "Content-Length": f"{file_size}", - }, - ) - - -async def _webdav_propfind(project: str, project_path: Path, file_path: Path) -> Response: - """Handle WebDAV PROPFIND request to list directory contents.""" - - # Calculate relative path from project root - try: - relative_path = file_path.relative_to(project_path) - relative_path_str = str(relative_path).replace("\\", "/") - if relative_path_str == ".": - relative_path_str = "" - except ValueError: - # file_path is not under project_path - relative_path_str = "" - - # Build minimal PROPFIND response - if file_path.is_dir(): - # Directory listing - href_path = ( - f"/{project}/webdav/{relative_path_str}/" - if relative_path_str - else f"/{project}/webdav/" - ) - xml_response = f""" - - - {href_path} - - - - {file_path.name if file_path.name else project} - - HTTP/1.1 200 OK - - -""" - - # Add child items - for child in file_path.iterdir(): - # Calculate child relative path - child_relative = child.relative_to(project_path) - child_relative_str = str(child_relative).replace("\\", "/") - - if child.is_dir(): - child_href = f"/{project}/webdav/{child_relative_str}/" - xml_response += f""" - {child_href} - - - - {child.name} - - HTTP/1.1 200 OK - - -""" - else: - child_href = f"/{project}/webdav/{child_relative_str}" - file_size = child.stat().st_size - xml_response += f""" - {child_href} - - - - {child.name} - {file_size} - - HTTP/1.1 200 OK - - -""" - - xml_response += "" - else: - # File properties - href_path = f"/{project}/webdav/{relative_path_str}" - file_size = file_path.stat().st_size - xml_response = f""" - - - {href_path} - - - - {file_path.name} - {file_size} - - HTTP/1.1 200 OK - - -""" - - return Response(content=xml_response, status_code=207, media_type="text/xml; charset=utf-8") - - -async def _webdav_get(file_path: Path) -> Response: - """Handle WebDAV GET request to download file.""" - - async def file_generator(): - async with aiofiles.open(file_path, "rb") as file: - while chunk := await file.read(8192): - yield chunk - - file_size = file_path.stat().st_size - headers = {"Content-Length": str(file_size), "Content-Type": "application/octet-stream"} - - return StreamingResponse(file_generator(), status_code=200, headers=headers) - - -async def _webdav_put(request: Request, project: str, file_path: Path) -> Response: - """Handle WebDAV PUT request to upload file.""" - - # Check if file exists before writing (for correct HTTP status) - file_existed = file_path.exists() - - # Ensure parent directory exists - file_path.parent.mkdir(parents=True, exist_ok=True) - - # Write file content - try: - async with aiofiles.open(file_path, "wb") as file: - async for chunk in request.stream(): - await file.write(chunk) - - # Preserve timestamps if provided in headers - await _preserve_file_timestamps(request, file_path) - - logger.info(f"WebDAV: Uploaded file {file_path} to project {project}.") - - return Response(status_code=204 if file_existed else 201) - - except Exception as e: - logger.error(f"WebDAV: Failed to upload file {file_path}: {e}") - raise HTTPException(status_code=500, detail=f"Failed to upload file: {e}") from e - - -async def _preserve_file_timestamps(request: Request, file_path: Path) -> None: - """Preserve file timestamps from WebDAV headers if provided. - - Supports multiple timestamp header formats: - - X-OC-Mtime: Unix timestamp (ownCloud/Nextcloud format) - - X-Timestamp: Unix timestamp - - X-Mtime: Unix timestamp - - Last-Modified: HTTP date format - """ - - # Try different header formats for modification time - mtime_timestamp = None - - # Check for custom timestamp headers (Unix timestamp) - for header_name in ["X-OC-Mtime", "X-Timestamp", "X-Mtime"]: - if header_name in request.headers: - try: - mtime_timestamp = float(request.headers[header_name]) - logger.debug(f"Using {header_name} timestamp: {mtime_timestamp}") - break - except (ValueError, TypeError) as e: - logger.warning(f"Invalid timestamp in {header_name} header: {e}") - continue - - # Fall back to Last-Modified header if no custom timestamp found - if mtime_timestamp is None and "Last-Modified" in request.headers: - try: - # Parse HTTP date format - last_modified_str = request.headers["Last-Modified"] - dt = datetime.strptime(last_modified_str, "%a, %d %b %Y %H:%M:%S GMT") - # Replace with UTC timezone to ensure correct timestamp calculation - from datetime import timezone - - dt = dt.replace(tzinfo=timezone.utc) - mtime_timestamp = dt.timestamp() - logger.debug(f"Using Last-Modified timestamp: {mtime_timestamp}") - except (ValueError, TypeError) as e: - logger.warning(f"Invalid Last-Modified header format: {e}") - - # Apply timestamp if we found one - if mtime_timestamp is not None: - try: - # Use os.utime to set both access and modification times - # Set access time to modification time to keep them consistent - os.utime(file_path, (mtime_timestamp, mtime_timestamp)) - logger.debug(f"Set file timestamps for {file_path} to {mtime_timestamp}") - except OSError as e: - logger.warning(f"Failed to set timestamps for {file_path}: {e}") - else: - logger.debug(f"No timestamp headers found, using current time for {file_path}") - - -async def _webdav_delete(project: str, file_path: Path) -> Response: - """Handle WebDAV DELETE request to delete file or directory.""" - - try: - if file_path.is_dir(): - shutil.rmtree(file_path) - else: - file_path.unlink() - - logger.info(f"WebDAV: Deleted {file_path} for project {project}") - return Response(status_code=204) - - except Exception as e: - logger.error(f"WebDAV: Failed to delete {file_path}: {e}") - raise HTTPException(status_code=500, detail=f"Failed to delete: {e}") from e - - -async def _webdav_mkcol(project: str, dir_path: Path) -> Response: - """Handle WebDAV MKCOL request to create directory.""" - - if dir_path.exists(): - raise HTTPException(status_code=405, detail="Directory already exists") - - try: - dir_path.mkdir(parents=True, exist_ok=False) - logger.info(f"WebDAV: Created directory {dir_path} for project {project}") - return Response(status_code=201) - - except Exception as e: - logger.error(f"WebDAV: Failed to create directory {dir_path}: {e}") - raise HTTPException(status_code=500, detail=f"Failed to create directory: {e}") from e diff --git a/src/basic_memory/cli/auth.py b/src/basic_memory/cli/auth.py index 25628f247..86eca8154 100644 --- a/src/basic_memory/cli/auth.py +++ b/src/basic_memory/cli/auth.py @@ -1,7 +1,10 @@ """WorkOS OAuth Device Authorization for CLI.""" +import base64 +import hashlib import json import os +import secrets import time import webbrowser @@ -22,12 +25,36 @@ def __init__(self, client_id: str, authkit_domain: str): app_config = ConfigManager().config # Store tokens in data dir self.token_file = app_config.data_dir_path / "basic-memory-cloud.json" + # PKCE parameters + self.code_verifier = None + self.code_challenge = None + + def generate_pkce_pair(self) -> tuple[str, str]: + """Generate PKCE code verifier and challenge.""" + # Generate code verifier (43-128 characters) + code_verifier = base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("utf-8") + code_verifier = code_verifier.rstrip("=") + + # Generate code challenge (SHA256 hash of verifier) + challenge_bytes = hashlib.sha256(code_verifier.encode("utf-8")).digest() + code_challenge = base64.urlsafe_b64encode(challenge_bytes).decode("utf-8") + code_challenge = code_challenge.rstrip("=") + + return code_verifier, code_challenge async def request_device_authorization(self) -> dict | None: - """Request device authorization from WorkOS.""" + """Request device authorization from WorkOS with PKCE.""" device_auth_url = f"{self.authkit_domain}/oauth2/device_authorization" - data = {"client_id": self.client_id, "scope": "openid profile email offline_access"} + # Generate PKCE pair + self.code_verifier, self.code_challenge = self.generate_pkce_pair() + + data = { + "client_id": self.client_id, + "scope": "openid profile email offline_access", + "code_challenge": self.code_challenge, + "code_challenge_method": "S256", + } try: async with httpx.AsyncClient() as client: @@ -76,6 +103,7 @@ async def poll_for_token(self, device_code: str, interval: int = 5) -> dict | No "client_id": self.client_id, "device_code": device_code, "grant_type": "urn:ietf:params:oauth:grant-type:device_code", + "code_verifier": self.code_verifier, } max_attempts = 60 # 5 minutes with 5-second intervals diff --git a/src/basic_memory/cli/commands/cloud/__init__.py b/src/basic_memory/cli/commands/cloud/__init__.py index bb7637833..85b431990 100644 --- a/src/basic_memory/cli/commands/cloud/__init__.py +++ b/src/basic_memory/cli/commands/cloud/__init__.py @@ -2,3 +2,4 @@ # Import all commands to register them with typer from basic_memory.cli.commands.cloud.core_commands import * # noqa: F401,F403 +from basic_memory.cli.commands.cloud.api_client import get_authenticated_headers # noqa: F401 diff --git a/src/basic_memory/cli/commands/cloud/api_client.py b/src/basic_memory/cli/commands/cloud/api_client.py index c0ed3b905..e19e6192e 100644 --- a/src/basic_memory/cli/commands/cloud/api_client.py +++ b/src/basic_memory/cli/commands/cloud/api_client.py @@ -26,7 +26,10 @@ def get_cloud_config() -> tuple[str, str, str]: async def get_authenticated_headers() -> dict[str, str]: - """Get authentication headers with JWT token.""" + """ + Get authentication headers with JWT token. + handles jwt refresh if needed. + """ client_id, domain, _ = get_cloud_config() auth = CLIAuth(client_id=client_id, authkit_domain=domain) token = await auth.get_valid_token() @@ -54,12 +57,12 @@ async def make_api_request( async with httpx.AsyncClient(timeout=timeout) as client: try: console.print(f"[dim]Making {method} request to {url}[/dim]") - console.print(f"[dim]Headers: {dict(headers)}[/dim]") + # console.print(f"[dim]Headers: {dict(headers)}[/dim]") response = await client.request(method=method, url=url, headers=headers, json=json_data) console.print(f"[dim]Response status: {response.status_code}[/dim]") - console.print(f"[dim]Response headers: {dict(response.headers)}[/dim]") + # console.print(f"[dim]Response headers: {dict(response.headers)}[/dim]") response.raise_for_status() return response diff --git a/src/basic_memory/cli/commands/cloud/bisync_commands.py b/src/basic_memory/cli/commands/cloud/bisync_commands.py new file mode 100644 index 000000000..4fe4d26ea --- /dev/null +++ b/src/basic_memory/cli/commands/cloud/bisync_commands.py @@ -0,0 +1,818 @@ +"""Cloud bisync commands for Basic Memory CLI.""" + +import asyncio +import subprocess +import time +from datetime import datetime +from pathlib import Path +from typing import Optional + +import typer +from rich.console import Console +from rich.table import Table + +from basic_memory.cli.commands.cloud.api_client import CloudAPIError, make_api_request +from basic_memory.cli.commands.cloud.rclone_config import ( + add_tenant_to_rclone_config, +) +from basic_memory.cli.commands.cloud.rclone_installer import RcloneInstallError, install_rclone +from basic_memory.config import ConfigManager +from basic_memory.ignore_utils import get_bmignore_path, create_default_bmignore +from basic_memory.schemas.cloud import ( + TenantMountInfo, + MountCredentials, + CloudProjectList, + CloudProjectCreateRequest, + CloudProjectCreateResponse, +) +from basic_memory.utils import generate_permalink + +console = Console() + + +class BisyncError(Exception): + """Exception raised for bisync-related errors.""" + + pass + + +class RcloneBisyncProfile: + """Bisync profile with safety settings.""" + + def __init__( + self, + name: str, + conflict_resolve: str, + max_delete: int, + check_access: bool, + description: str, + extra_args: Optional[list[str]] = None, + ): + self.name = name + self.conflict_resolve = conflict_resolve + self.max_delete = max_delete + self.check_access = check_access + self.description = description + self.extra_args = extra_args or [] + + +# Bisync profiles based on SPEC-9 Phase 2.1 +BISYNC_PROFILES = { + "safe": RcloneBisyncProfile( + name="safe", + conflict_resolve="none", + max_delete=10, + check_access=False, + description="Safe mode with conflict preservation (keeps both versions)", + ), + "balanced": RcloneBisyncProfile( + name="balanced", + conflict_resolve="newer", + max_delete=25, + check_access=False, + description="Balanced mode - auto-resolve to newer file (recommended)", + ), + "fast": RcloneBisyncProfile( + name="fast", + conflict_resolve="newer", + max_delete=50, + check_access=False, + description="Fast mode for rapid iteration (skip verification)", + ), +} + + +async def get_mount_info() -> TenantMountInfo: + """Get current tenant information from cloud API.""" + try: + config_manager = ConfigManager() + config = config_manager.config + host_url = config.cloud_host.rstrip("/") + + response = await make_api_request(method="GET", url=f"{host_url}/tenant/mount/info") + + return TenantMountInfo.model_validate(response.json()) + except Exception as e: + raise BisyncError(f"Failed to get tenant info: {e}") from e + + +async def generate_mount_credentials(tenant_id: str) -> MountCredentials: + """Generate scoped credentials for syncing.""" + try: + config_manager = ConfigManager() + config = config_manager.config + host_url = config.cloud_host.rstrip("/") + + response = await make_api_request(method="POST", url=f"{host_url}/tenant/mount/credentials") + + return MountCredentials.model_validate(response.json()) + except Exception as e: + raise BisyncError(f"Failed to generate credentials: {e}") from e + + +async def fetch_cloud_projects() -> CloudProjectList: + """Fetch list of projects from cloud API. + + Returns: + CloudProjectList with projects from cloud + """ + try: + config_manager = ConfigManager() + config = config_manager.config + host_url = config.cloud_host.rstrip("/") + + response = await make_api_request(method="GET", url=f"{host_url}/proxy/projects/projects") + + return CloudProjectList.model_validate(response.json()) + except Exception as e: + raise BisyncError(f"Failed to fetch cloud projects: {e}") from e + + +def scan_local_directories(sync_dir: Path) -> list[str]: + """Scan local sync directory for project folders. + + Args: + sync_dir: Path to bisync directory + + Returns: + List of directory names (project names) + """ + if not sync_dir.exists(): + return [] + + directories = [] + for item in sync_dir.iterdir(): + if item.is_dir() and not item.name.startswith("."): + directories.append(item.name) + + return directories + + +async def create_cloud_project(project_name: str) -> CloudProjectCreateResponse: + """Create a new project on cloud. + + Args: + project_name: Name of project to create + + Returns: + CloudProjectCreateResponse with project details from API + """ + try: + config_manager = ConfigManager() + config = config_manager.config + host_url = config.cloud_host.rstrip("/") + + # Use generate_permalink to ensure consistent naming + project_path = generate_permalink(project_name) + + project_data = CloudProjectCreateRequest( + name=project_name, + path=project_path, + set_default=False, + ) + + response = await make_api_request( + method="POST", + url=f"{host_url}/proxy/projects/projects", + headers={"Content-Type": "application/json"}, + json_data=project_data.model_dump(), + ) + + return CloudProjectCreateResponse.model_validate(response.json()) + except Exception as e: + raise BisyncError(f"Failed to create cloud project '{project_name}': {e}") from e + + +def get_bisync_state_path(tenant_id: str) -> Path: + """Get path to bisync state directory.""" + return Path.home() / ".basic-memory" / "bisync-state" / tenant_id + + +def get_bisync_directory() -> Path: + """Get bisync directory from config. + + Returns: + Path to bisync directory (default: ~/basic-memory-cloud-sync) + """ + config_manager = ConfigManager() + config = config_manager.config + + sync_dir = config.bisync_config.get("sync_dir", str(Path.home() / "basic-memory-cloud-sync")) + return Path(sync_dir).expanduser().resolve() + + +def validate_bisync_directory(bisync_dir: Path) -> None: + """Validate bisync directory doesn't conflict with mount. + + Raises: + BisyncError: If bisync directory conflicts with mount directory + """ + # Get fixed mount directory + mount_dir = (Path.home() / "basic-memory-cloud").resolve() + + # Check if bisync dir is the same as mount dir + if bisync_dir == mount_dir: + raise BisyncError( + f"Cannot use {bisync_dir} for bisync - it's the mount directory!\n" + f"Mount and bisync must use different directories.\n\n" + f"Options:\n" + f" 1. Use default: ~/basic-memory-cloud-sync/\n" + f" 2. Specify different directory: --dir ~/my-sync-folder" + ) + + # Check if mount is active at this location + result = subprocess.run(["mount"], capture_output=True, text=True) + if str(bisync_dir) in result.stdout and "rclone" in result.stdout: + raise BisyncError( + f"{bisync_dir} is currently mounted via 'bm cloud mount'\n" + f"Cannot use mounted directory for bisync.\n\n" + f"Either:\n" + f" 1. Unmount first: bm cloud unmount\n" + f" 2. Use different directory for bisync" + ) + + +def convert_bmignore_to_rclone_filters() -> Path: + """Convert .bmignore patterns to rclone filter format. + + Reads ~/.basic-memory/.bmignore (gitignore-style) and converts to + ~/.basic-memory/.bmignore.rclone (rclone filter format). + + Only regenerates if .bmignore has been modified since last conversion. + + Returns: + Path to converted rclone filter file + """ + # Ensure .bmignore exists + create_default_bmignore() + + bmignore_path = get_bmignore_path() + # Create rclone filter path: ~/.basic-memory/.bmignore -> ~/.basic-memory/.bmignore.rclone + rclone_filter_path = bmignore_path.parent / f"{bmignore_path.name}.rclone" + + # Skip regeneration if rclone file is newer than bmignore + if rclone_filter_path.exists(): + bmignore_mtime = bmignore_path.stat().st_mtime + rclone_mtime = rclone_filter_path.stat().st_mtime + if rclone_mtime >= bmignore_mtime: + return rclone_filter_path + + # Read .bmignore patterns + patterns = [] + try: + with bmignore_path.open("r", encoding="utf-8") as f: + for line in f: + line = line.strip() + # Keep comments and empty lines + if not line or line.startswith("#"): + patterns.append(line) + continue + + # Convert gitignore pattern to rclone filter syntax + # gitignore: node_modules → rclone: - node_modules/** + # gitignore: *.pyc → rclone: - *.pyc + if "*" in line: + # Pattern already has wildcard, just add exclude prefix + patterns.append(f"- {line}") + else: + # Directory pattern - add /** for recursive exclude + patterns.append(f"- {line}/**") + + except Exception: + # If we can't read the file, create a minimal filter + patterns = ["# Error reading .bmignore, using minimal filters", "- .git/**"] + + # Write rclone filter file + rclone_filter_path.write_text("\n".join(patterns) + "\n") + + return rclone_filter_path + + +def get_bisync_filter_path() -> Path: + """Get path to bisync filter file. + + Uses ~/.basic-memory/.bmignore (converted to rclone format). + The file is automatically created with default patterns on first use. + + Returns: + Path to rclone filter file + """ + return convert_bmignore_to_rclone_filters() + + +def bisync_state_exists(tenant_id: str) -> bool: + """Check if bisync state exists (has been initialized).""" + state_path = get_bisync_state_path(tenant_id) + return state_path.exists() and any(state_path.iterdir()) + + +def build_bisync_command( + tenant_id: str, + bucket_name: str, + local_path: Path, + profile: RcloneBisyncProfile, + dry_run: bool = False, + resync: bool = False, + verbose: bool = False, +) -> list[str]: + """Build rclone bisync command with profile settings.""" + + # Sync with the entire bucket root (all projects) + rclone_remote = f"basic-memory-{tenant_id}:{bucket_name}" + filter_path = get_bisync_filter_path() + state_path = get_bisync_state_path(tenant_id) + + # Ensure state directory exists + state_path.mkdir(parents=True, exist_ok=True) + + cmd = [ + "rclone", + "bisync", + str(local_path), + rclone_remote, + "--create-empty-src-dirs", + "--resilient", + f"--conflict-resolve={profile.conflict_resolve}", + f"--max-delete={profile.max_delete}", + "--filters-file", + str(filter_path), + "--workdir", + str(state_path), + ] + + # Add verbosity flags + if verbose: + cmd.append("--verbose") # Full details with file-by-file output + else: + # Show progress bar during transfers + cmd.append("--progress") + + if profile.check_access: + cmd.append("--check-access") + + if dry_run: + cmd.append("--dry-run") + + if resync: + cmd.append("--resync") + + cmd.extend(profile.extra_args) + + return cmd + + +def setup_cloud_bisync(sync_dir: Optional[str] = None) -> None: + """Set up cloud bisync with rclone installation and configuration. + + Args: + sync_dir: Optional custom sync directory path. If not provided, uses config default. + """ + console.print("[bold blue]Basic Memory Cloud Bisync Setup[/bold blue]") + console.print("Setting up bidirectional sync to your cloud tenant...\n") + + try: + # Step 1: Install rclone + console.print("[blue]Step 1: Installing rclone...[/blue]") + install_rclone() + + # Step 2: Get mount info (for tenant_id, bucket) + console.print("\n[blue]Step 2: Getting tenant information...[/blue]") + tenant_info = asyncio.run(get_mount_info()) + + tenant_id = tenant_info.tenant_id + bucket_name = tenant_info.bucket_name + + console.print(f"[green]✓ Found tenant: {tenant_id}[/green]") + console.print(f"[green]✓ Bucket: {bucket_name}[/green]") + + # Step 3: Generate credentials + console.print("\n[blue]Step 3: Generating sync credentials...[/blue]") + creds = asyncio.run(generate_mount_credentials(tenant_id)) + + access_key = creds.access_key + secret_key = creds.secret_key + + console.print("[green]✓ Generated secure credentials[/green]") + + # Step 4: Configure rclone + console.print("\n[blue]Step 4: Configuring rclone...[/blue]") + add_tenant_to_rclone_config( + tenant_id=tenant_id, + bucket_name=bucket_name, + access_key=access_key, + secret_key=secret_key, + ) + + # Step 5: Configure and create local directory + console.print("\n[blue]Step 5: Configuring sync directory...[/blue]") + + # If custom sync_dir provided, save to config + if sync_dir: + config_manager = ConfigManager() + config = config_manager.load_config() + config.bisync_config["sync_dir"] = sync_dir + config_manager.save_config(config) + console.print("[green]✓ Saved custom sync directory to config[/green]") + + # Get bisync directory (from config or default) + local_path = get_bisync_directory() + + # Validate bisync directory + validate_bisync_directory(local_path) + + # Create directory + local_path.mkdir(parents=True, exist_ok=True) + console.print(f"[green]✓ Created sync directory: {local_path}[/green]") + + # Step 6: Perform initial resync + console.print("\n[blue]Step 6: Performing initial sync...[/blue]") + console.print("[yellow]This will establish the baseline for bidirectional sync.[/yellow]") + + run_bisync( + tenant_id=tenant_id, + bucket_name=bucket_name, + local_path=local_path, + profile_name="balanced", + resync=True, + ) + + console.print("\n[bold green]✓ Bisync setup completed successfully![/bold green]") + console.print("\nYour local files will now sync bidirectionally with the cloud!") + console.print(f"\nLocal directory: {local_path}") + console.print("\nUseful commands:") + console.print(" bm sync # Run sync (recommended)") + console.print(" bm sync --watch # Start watch mode") + console.print(" bm cloud status # Check sync status") + console.print(" bm cloud check # Verify file integrity") + console.print(" bm cloud bisync --dry-run # Preview changes (advanced)") + + except (RcloneInstallError, BisyncError, CloudAPIError) as e: + console.print(f"\n[red]Setup failed: {e}[/red]") + raise typer.Exit(1) + except Exception as e: + console.print(f"\n[red]Unexpected error during setup: {e}[/red]") + raise typer.Exit(1) + + +def run_bisync( + tenant_id: Optional[str] = None, + bucket_name: Optional[str] = None, + local_path: Optional[Path] = None, + profile_name: str = "balanced", + dry_run: bool = False, + resync: bool = False, + verbose: bool = False, +) -> bool: + """Run rclone bisync with specified profile.""" + + try: + # Get tenant info if not provided + if not tenant_id or not bucket_name: + tenant_info = asyncio.run(get_mount_info()) + tenant_id = tenant_info.tenant_id + bucket_name = tenant_info.bucket_name + + # Set default local path if not provided + if not local_path: + local_path = get_bisync_directory() + + # Validate bisync directory + validate_bisync_directory(local_path) + + # Check if local path exists + if not local_path.exists(): + raise BisyncError( + f"Local directory {local_path} does not exist. Run 'basic-memory cloud bisync-setup' first." + ) + + # Get bisync profile + if profile_name not in BISYNC_PROFILES: + raise BisyncError( + f"Unknown profile: {profile_name}. Available: {list(BISYNC_PROFILES.keys())}" + ) + + profile = BISYNC_PROFILES[profile_name] + + # Auto-register projects before sync (unless dry-run or resync) + if not dry_run and not resync: + try: + console.print("[dim]Checking for new projects...[/dim]") + + # Fetch cloud projects and extract directory names from paths + cloud_data = asyncio.run(fetch_cloud_projects()) + cloud_projects = cloud_data.projects + + # Extract directory names from cloud project paths + # Compare directory names, not project names + # Cloud path /app/data/basic-memory -> directory name "basic-memory" + cloud_dir_names = set() + for p in cloud_projects: + path = p.path + # Strip /app/data/ prefix if present (cloud mode) + if path.startswith("/app/data/"): + path = path[len("/app/data/") :] + # Get the last segment (directory name) + dir_name = Path(path).name + cloud_dir_names.add(dir_name) + + # Scan local directories + local_dirs = scan_local_directories(local_path) + + # Create missing cloud projects + new_projects = [] + for dir_name in local_dirs: + if dir_name not in cloud_dir_names: + new_projects.append(dir_name) + + if new_projects: + console.print( + f"[blue]Found {len(new_projects)} new local project(s), creating on cloud...[/blue]" + ) + for project_name in new_projects: + try: + asyncio.run(create_cloud_project(project_name)) + console.print(f"[green] ✓ Created project: {project_name}[/green]") + except BisyncError as e: + console.print( + f"[yellow] ⚠ Could not create {project_name}: {e}[/yellow]" + ) + else: + console.print("[dim]All local projects already registered on cloud[/dim]") + + except Exception as e: + console.print(f"[yellow]Warning: Project auto-registration failed: {e}[/yellow]") + console.print("[yellow]Continuing with sync anyway...[/yellow]") + + # Check if first run and require resync + if not resync and not bisync_state_exists(tenant_id) and not dry_run: + raise BisyncError( + "First bisync requires --resync to establish baseline. " + "Run: basic-memory cloud bisync --resync" + ) + + # Build and execute bisync command + bisync_cmd = build_bisync_command( + tenant_id, + bucket_name, + local_path, + profile, + dry_run=dry_run, + resync=resync, + verbose=verbose, + ) + + if dry_run: + console.print("[yellow]DRY RUN MODE - No changes will be made[/yellow]") + + console.print( + f"[blue]Running bisync with profile '{profile_name}' ({profile.description})...[/blue]" + ) + console.print(f"[dim]Command: {' '.join(bisync_cmd)}[/dim]") + console.print() # Blank line before output + + # Stream output in real-time so user sees progress + result = subprocess.run(bisync_cmd, text=True) + + if result.returncode != 0: + raise BisyncError(f"Bisync command failed with code {result.returncode}") + + console.print() # Blank line after output + + if dry_run: + console.print("[green]✓ Dry run completed successfully[/green]") + elif resync: + console.print("[green]✓ Initial sync baseline established[/green]") + else: + console.print("[green]✓ Sync completed successfully[/green]") + + # Notify container to refresh cache (if not dry run) + if not dry_run: + try: + asyncio.run(notify_container_sync(tenant_id)) + except Exception as e: + console.print(f"[yellow]Warning: Could not notify container: {e}[/yellow]") + + return True + + except BisyncError: + raise + except Exception as e: + raise BisyncError(f"Unexpected error during bisync: {e}") from e + + +async def notify_container_sync(tenant_id: str) -> None: + """Sync all projects after bisync completes.""" + try: + from basic_memory.cli.commands.command_utils import run_sync + + # Fetch all projects and sync each one + cloud_data = await fetch_cloud_projects() + projects = cloud_data.projects + + if not projects: + console.print("[dim]No projects to sync[/dim]") + return + + console.print(f"[blue]Notifying cloud to index {len(projects)} project(s)...[/blue]") + + for project in projects: + project_name = project.name + if project_name: + try: + await run_sync(project=project_name) + except Exception as e: + # Non-critical, log and continue + console.print(f"[yellow] ⚠ Sync failed for {project_name}: {e}[/yellow]") + + console.print("[dim]Note: Cloud indexing has started and may take a few moments[/dim]") + + except Exception as e: + # Non-critical, don't fail the bisync + console.print(f"[yellow]Warning: Post-sync failed: {e}[/yellow]") + + +def run_bisync_watch( + tenant_id: Optional[str] = None, + bucket_name: Optional[str] = None, + local_path: Optional[Path] = None, + profile_name: str = "balanced", + interval_seconds: int = 60, +) -> None: + """Run bisync in watch mode with periodic syncs.""" + + console.print("[bold blue]Starting bisync watch mode[/bold blue]") + console.print(f"Sync interval: {interval_seconds} seconds") + console.print("Press Ctrl+C to stop\n") + + try: + while True: + try: + start_time = time.time() + + run_bisync( + tenant_id=tenant_id, + bucket_name=bucket_name, + local_path=local_path, + profile_name=profile_name, + ) + + elapsed = time.time() - start_time + console.print(f"[dim]Sync completed in {elapsed:.1f}s[/dim]") + + # Wait for next interval + time.sleep(interval_seconds) + + except BisyncError as e: + console.print(f"[red]Sync error: {e}[/red]") + console.print(f"[yellow]Retrying in {interval_seconds} seconds...[/yellow]") + time.sleep(interval_seconds) + + except KeyboardInterrupt: + console.print("\n[yellow]Watch mode stopped[/yellow]") + + +def show_bisync_status() -> None: + """Show current bisync status and configuration.""" + + try: + # Get tenant info + tenant_info = asyncio.run(get_mount_info()) + tenant_id = tenant_info.tenant_id + + local_path = get_bisync_directory() + state_path = get_bisync_state_path(tenant_id) + + # Create status table + table = Table(title="Cloud Bisync Status", show_header=True, header_style="bold blue") + table.add_column("Property", style="green", min_width=20) + table.add_column("Value", style="dim", min_width=30) + + # Check initialization status + is_initialized = bisync_state_exists(tenant_id) + init_status = ( + "[green]✓ Initialized[/green]" if is_initialized else "[red]✗ Not initialized[/red]" + ) + + table.add_row("Tenant ID", tenant_id) + table.add_row("Local Directory", str(local_path)) + table.add_row("Status", init_status) + table.add_row("State Directory", str(state_path)) + + # Check for last sync info + if is_initialized: + # Look for most recent state file + state_files = list(state_path.glob("*.lst")) + if state_files: + latest = max(state_files, key=lambda p: p.stat().st_mtime) + last_sync = datetime.fromtimestamp(latest.stat().st_mtime) + table.add_row("Last Sync", last_sync.strftime("%Y-%m-%d %H:%M:%S")) + + console.print(table) + + # Show bisync profiles + console.print("\n[bold]Available bisync profiles:[/bold]") + for name, profile in BISYNC_PROFILES.items(): + console.print(f" {name}: {profile.description}") + console.print(f" - Conflict resolution: {profile.conflict_resolve}") + console.print(f" - Max delete: {profile.max_delete} files") + + console.print("\n[dim]To use a profile: bm cloud bisync --profile [/dim]") + + # Show setup instructions if not initialized + if not is_initialized: + console.print("\n[yellow]To initialize bisync, run:[/yellow]") + console.print(" bm cloud setup") + console.print(" or") + console.print(" bm cloud bisync --resync") + + except Exception as e: + console.print(f"[red]Error getting bisync status: {e}[/red]") + raise typer.Exit(1) + + +def run_check( + tenant_id: Optional[str] = None, + bucket_name: Optional[str] = None, + local_path: Optional[Path] = None, + one_way: bool = False, +) -> bool: + """Check file integrity between local and cloud using rclone check. + + Args: + tenant_id: Cloud tenant ID (auto-detected if not provided) + bucket_name: S3 bucket name (auto-detected if not provided) + local_path: Local bisync directory (uses config default if not provided) + one_way: If True, only check for missing files on destination (faster) + + Returns: + True if check passed (files match), False if differences found + """ + try: + # Check if rclone is installed + from basic_memory.cli.commands.cloud.rclone_installer import is_rclone_installed + + if not is_rclone_installed(): + raise BisyncError( + "rclone is not installed. Run 'bm cloud bisync-setup' first to set up cloud sync." + ) + + # Get tenant info if not provided + if not tenant_id or not bucket_name: + tenant_info = asyncio.run(get_mount_info()) + tenant_id = tenant_id or tenant_info.tenant_id + bucket_name = bucket_name or tenant_info.bucket_name + + # Get local path from config + if not local_path: + local_path = get_bisync_directory() + + # Check if bisync is initialized + if not bisync_state_exists(tenant_id): + raise BisyncError( + "Bisync not initialized. Run 'bm cloud bisync --resync' to establish baseline." + ) + + # Build rclone check command + rclone_remote = f"basic-memory-{tenant_id}:{bucket_name}" + filter_path = get_bisync_filter_path() + + cmd = [ + "rclone", + "check", + str(local_path), + rclone_remote, + "--filter-from", + str(filter_path), + ] + + if one_way: + cmd.append("--one-way") + + console.print("[bold blue]Checking file integrity between local and cloud[/bold blue]") + console.print(f"[dim]Local: {local_path}[/dim]") + console.print(f"[dim]Remote: {rclone_remote}[/dim]") + console.print(f"[dim]Command: {' '.join(cmd)}[/dim]") + console.print() + + # Run check command + result = subprocess.run(cmd, capture_output=True, text=True) + + # rclone check returns: + # 0 = success (all files match) + # non-zero = differences found or error + if result.returncode == 0: + console.print("[green]✓ All files match between local and cloud[/green]") + return True + else: + console.print("[yellow]⚠ Differences found:[/yellow]") + if result.stderr: + console.print(result.stderr) + if result.stdout: + console.print(result.stdout) + console.print("\n[dim]To sync differences, run: bm sync[/dim]") + return False + + except BisyncError: + raise + except Exception as e: + raise BisyncError(f"Check failed: {e}") from e diff --git a/src/basic_memory/cli/commands/cloud/core_commands.py b/src/basic_memory/cli/commands/cloud/core_commands.py index 9bb16122f..ae248ff23 100644 --- a/src/basic_memory/cli/commands/cloud/core_commands.py +++ b/src/basic_memory/cli/commands/cloud/core_commands.py @@ -1,21 +1,18 @@ """Core cloud commands for Basic Memory CLI.""" import asyncio -from pathlib import Path from typing import Optional -import httpx import typer from rich.console import Console -from rich.table import Table from basic_memory.cli.app import cloud_app from basic_memory.cli.auth import CLIAuth +from basic_memory.config import ConfigManager from basic_memory.cli.commands.cloud.api_client import ( CloudAPIError, get_cloud_config, make_api_request, - get_authenticated_headers, ) from basic_memory.cli.commands.cloud.mount_commands import ( mount_cloud_files, @@ -23,19 +20,25 @@ show_mount_status, unmount_cloud_files, ) +from basic_memory.cli.commands.cloud.bisync_commands import ( + run_bisync, + run_bisync_watch, + run_check, + setup_cloud_bisync, + show_bisync_status, +) from basic_memory.cli.commands.cloud.rclone_config import MOUNT_PROFILES -from basic_memory.ignore_utils import load_gitignore_patterns, should_ignore_path -from basic_memory.utils import generate_permalink +from basic_memory.cli.commands.cloud.bisync_commands import BISYNC_PROFILES console = Console() @cloud_app.command() def login(): - """Authenticate with WorkOS using OAuth Device Authorization flow.""" + """Authenticate with WorkOS using OAuth Device Authorization flow and enable cloud mode.""" async def _login(): - client_id, domain, _ = get_cloud_config() + client_id, domain, host_url = get_cloud_config() auth = CLIAuth(client_id=client_id, authkit_domain=domain) success = await auth.login() @@ -43,293 +46,69 @@ async def _login(): console.print("[red]Login failed[/red]") raise typer.Exit(1) - asyncio.run(_login()) - - -# Project commands - -project_app = typer.Typer(help="Manage Basic Memory Cloud Projects") -cloud_app.add_typer(project_app, name="project") - - -@project_app.command("list") -def list_projects() -> None: - """List projects on the cloud instance.""" - - try: - # Get cloud configuration - _, _, host_url = get_cloud_config() - host_url = host_url.rstrip("/") - - console.print(f"[blue]Fetching projects from {host_url}...[/blue]") - - # Make API request to list projects - response = asyncio.run( - make_api_request(method="GET", url=f"{host_url}/proxy/projects/projects") - ) - - projects_data = response.json() - - if not projects_data.get("projects"): - console.print("[yellow]No projects found on the cloud instance.[/yellow]") - return - - # Create table for display - table = Table( - title="Cloud Projects", show_header=True, header_style="bold blue", min_width=60 - ) - table.add_column("Name", style="green", min_width=20) - table.add_column("Path", style="dim", min_width=30) - - for project in projects_data["projects"]: - # Format the path for display - path = project.get("path", "") - if path.startswith("/"): - path = f"~{path}" if path.startswith(str(Path.home())) else path - - table.add_row( - project.get("name", "unnamed"), - path, - ) - - console.print(table) - console.print(f"\n[green]Found {len(projects_data['projects'])} project(s)[/green]") - - except CloudAPIError as e: - console.print(f"[red]Error: {e}[/red]") - raise typer.Exit(1) - except Exception as e: - console.print(f"[red]Unexpected error: {e}[/red]") - raise typer.Exit(1) - - -@project_app.command("add") -def add_project( - name: str = typer.Argument(..., help="Name of the project to add"), - set_default: bool = typer.Option(False, "--default", "-d", help="Set as default project"), -) -> None: - """Create a new project on the cloud instance.""" - - # Get cloud configuration - _, _, host_url = get_cloud_config() - host_url = host_url.rstrip("/") - - # Prepare headers - headers = {"Content-Type": "application/json"} - - project_path = generate_permalink(name) - # Prepare project data - project_data = { - "name": name, - "path": project_path, - "set_default": set_default, - } + # Enable cloud mode after successful login + config_manager = ConfigManager() + config = config_manager.load_config() + config.cloud_mode = True + config_manager.save_config(config) - console.print(project_data) - - try: - console.print(f"[blue]Creating project '{name}' on {host_url}...[/blue]") + console.print("[green]✓ Cloud mode enabled[/green]") + console.print(f"[dim]All CLI commands now work against {host_url}[/dim]") - # Make API request to create project - response = asyncio.run( - make_api_request( - method="POST", - url=f"{host_url}/proxy/projects/projects", - headers=headers, - json_data=project_data, - ) - ) + asyncio.run(_login()) - result = response.json() - console.print(f"[green]Project '{name}' created successfully![/green]") +@cloud_app.command() +def logout(): + """Disable cloud mode and return to local mode.""" - # Display project details - if "project" in result: - project = result["project"] - console.print(f" Name: {project.get('name', name)}") - console.print(f" Path: {project.get('path', 'unknown')}") - if project.get("id"): - console.print(f" ID: {project['id']}") + # Disable cloud mode + config_manager = ConfigManager() + config = config_manager.load_config() + config.cloud_mode = False + config_manager.save_config(config) - except CloudAPIError as e: - console.print(f"[red]Error creating project: {e}[/red]") - raise typer.Exit(1) - except Exception as e: - console.print(f"[red]Unexpected error: {e}[/red]") - raise typer.Exit(1) + console.print("[green]✓ Cloud mode disabled[/green]") + console.print("[dim]All CLI commands now work locally[/dim]") -@cloud_app.command("upload") -def upload_files( - project: str = typer.Argument(..., help="Project name to upload to"), - path_to_files: str = typer.Argument(..., help="Local path to files or directory to upload"), - preserve_timestamps: bool = typer.Option( - True, - "--preserve-timestamps/--no-preserve-timestamps", - help="Preserve file modification times", - ), - respect_gitignore: bool = typer.Option( +@cloud_app.command("status") +def status( + bisync: bool = typer.Option( True, - "--respect-gitignore/--no-gitignore", - help="Respect .gitignore patterns and skip common development artifacts", + "--bisync/--mount", + help="Show bisync status (default) or mount status", ), ) -> None: - """Upload files to a cloud project using WebDAV.""" + """Check cloud mode status and cloud instance health. + + Shows cloud mode status, instance health, and sync/mount status. + Use --bisync (default) to show bisync status or --mount for mount status. + """ + # Check cloud mode + config_manager = ConfigManager() + config = config_manager.load_config() + + console.print("[bold blue]Cloud Mode Status[/bold blue]") + if config.cloud_mode: + console.print(" Mode: [green]Cloud (enabled)[/green]") + console.print(f" Host: {config.cloud_host}") + console.print(" [dim]All CLI commands work against cloud[/dim]") + else: + console.print(" Mode: [yellow]Local (disabled)[/yellow]") + console.print(" [dim]All CLI commands work locally[/dim]") + console.print("\n[dim]To enable cloud mode, run: bm cloud login[/dim]") + return # Get cloud configuration _, _, host_url = get_cloud_config() host_url = host_url.rstrip("/") - # Validate local path - local_path = Path(path_to_files).expanduser().resolve() - if not local_path.exists(): - console.print(f"[red]Error: Path '{path_to_files}' does not exist[/red]") - raise typer.Exit(1) - # Prepare headers headers = {} try: - # Load gitignore patterns (only if enabled) - ignore_patterns = load_gitignore_patterns(local_path) if respect_gitignore else set() - - # Collect files to upload - files_to_upload = [] - ignored_count = 0 - - if local_path.is_file(): - # Single file upload - check if it should be ignored - if not respect_gitignore or not should_ignore_path( - local_path, local_path.parent, ignore_patterns - ): - files_to_upload.append(local_path) - else: - ignored_count += 1 - else: - # Recursively collect all files - for file_path in local_path.rglob("*"): - if file_path.is_file(): - if not respect_gitignore or not should_ignore_path( - file_path, local_path, ignore_patterns - ): - files_to_upload.append(file_path) - else: - ignored_count += 1 - - # Show summary - if ignored_count > 0 and respect_gitignore: - console.print( - f"[dim]Ignored {ignored_count} file(s) based on .gitignore and default patterns[/dim]" - ) - - if not files_to_upload: - console.print("[yellow]No files found to upload[/yellow]") - return - - console.print( - f"[blue]Uploading {len(files_to_upload)} file(s) to project '{project}' on {host_url}...[/blue]" - ) - - # Upload files using WebDAV - asyncio.run( - _upload_files_webdav( - files_to_upload=files_to_upload, - local_base_path=local_path, - project=project, - host_url=host_url, - headers=headers, - preserve_timestamps=preserve_timestamps, - ) - ) - - console.print(f"[green]Successfully uploaded {len(files_to_upload)} file(s)![/green]") - - except CloudAPIError as e: - console.print(f"[red]Error uploading files: {e}[/red]") - raise typer.Exit(1) - except Exception as e: - console.print(f"[red]Unexpected error: {e}[/red]") - raise typer.Exit(1) - - -async def _upload_files_webdav( - files_to_upload: list[Path], - local_base_path: Path, - project: str, - host_url: str, - headers: dict, - preserve_timestamps: bool, -) -> None: - """Upload files using WebDAV protocol.""" - - # Get authentication headers for WebDAV uploads - auth_headers = await get_authenticated_headers() - - async with httpx.AsyncClient(timeout=300.0) as client: - for file_path in files_to_upload: - # Calculate relative path for WebDAV outside try block - if local_base_path.is_file(): - # Single file upload - use just the filename - relative_path = file_path.name - else: - # Directory upload - preserve structure - relative_path = file_path.relative_to(local_base_path) - - try: - # WebDAV URL - webdav_url = f"{host_url}/proxy/{project}/webdav/{relative_path}" - - # Prepare upload headers - upload_headers = dict(headers) - upload_headers.update(auth_headers) - - # Add timestamp preservation header if requested - if preserve_timestamps: - mtime = file_path.stat().st_mtime - upload_headers["X-OC-Mtime"] = str(mtime) - - # Disable compression for WebDAV as well - upload_headers.setdefault("Accept-Encoding", "identity") - - # Read file content - file_content = file_path.read_bytes() - - # console.print(f"[dim]Uploading {relative_path} to {webdav_url}[/dim]") - - # Upload file - response = await client.put( - webdav_url, content=file_content, headers=upload_headers - ) - - # console.print(f"[dim]WebDAV response status: {response.status_code}[/dim]") - response.raise_for_status() - - # Show file upload progress - console.print(f" ✓ {relative_path}") - - except httpx.HTTPError as e: - console.print(f" ✗ {relative_path} - {e}") - if hasattr(e, "response") and e.response is not None: # pyright: ignore [reportAttributeAccessIssue] - response = e.response # type: ignore - console.print(f"[red]WebDAV Response status: {response.status_code}[/red]") - console.print(f"[red]WebDAV Response headers: {dict(response.headers)}[/red]") - raise CloudAPIError(f"Failed to upload {file_path.name}: {e}") from e - - -@cloud_app.command("status") -def status() -> None: - """Check the status of the cloud instance.""" - - # Get cloud configuration - _, _, host_url = get_cloud_config() - host_url = host_url.rstrip("/") - - # Prepare headers - headers = {} - - try: - console.print(f"[blue]Checking status of {host_url}...[/blue]") + console.print("\n[blue]Checking cloud instance health...[/blue]") # Make API request to check health response = asyncio.run( @@ -348,8 +127,15 @@ def status() -> None: if "timestamp" in health_data: console.print(f" Timestamp: {health_data['timestamp']}") + # Show sync/mount status based on flag + console.print() + if bisync: + show_bisync_status() + else: + show_mount_status() + except CloudAPIError as e: - console.print(f"[red]Error checking status: {e}[/red]") + console.print(f"[red]Error checking cloud health: {e}[/red]") raise typer.Exit(1) except Exception as e: console.print(f"[red]Unexpected error: {e}[/red]") @@ -360,9 +146,32 @@ def status() -> None: @cloud_app.command("setup") -def setup() -> None: - """Set up local file access with automatic rclone installation and configuration.""" - setup_cloud_mount() +def setup( + bisync: bool = typer.Option( + True, + "--bisync/--mount", + help="Use bidirectional sync (recommended) or mount as network drive", + ), + sync_dir: Optional[str] = typer.Option( + None, + "--dir", + help="Custom sync directory for bisync (default: ~/basic-memory-cloud-sync)", + ), +) -> None: + """Set up cloud file access with automatic rclone installation and configuration. + + Default: Sets up bidirectional sync (recommended).\n + Use --mount: Sets up mount as network drive (alternative workflow).\n + + Examples:\n + bm cloud setup # Setup bisync (default)\n + bm cloud setup --mount # Setup mount instead\n + bm cloud setup --dir ~/sync # Custom bisync directory\n + """ + if bisync: + setup_cloud_bisync(sync_dir=sync_dir) + else: + setup_cloud_mount() @cloud_app.command("mount") @@ -392,7 +201,73 @@ def unmount() -> None: raise typer.Exit(1) -@cloud_app.command("mount-status") -def mount_status() -> None: - """Show current mount status.""" - show_mount_status() +# Bisync commands + + +@cloud_app.command("bisync") +def bisync( + profile: str = typer.Option( + "balanced", help=f"Bisync profile: {', '.join(BISYNC_PROFILES.keys())}" + ), + dry_run: bool = typer.Option(False, "--dry-run", help="Preview changes without syncing"), + resync: bool = typer.Option(False, "--resync", help="Force resync to establish new baseline"), + watch: bool = typer.Option(False, "--watch", help="Run continuous sync in watch mode"), + interval: int = typer.Option(60, "--interval", help="Sync interval in seconds for watch mode"), + verbose: bool = typer.Option(False, "--verbose", "-v", help="Show detailed sync output"), +) -> None: + """Run bidirectional sync between local files and cloud storage. + + Examples: + basic-memory cloud bisync # Manual sync with balanced profile + basic-memory cloud bisync --dry-run # Preview what would be synced + basic-memory cloud bisync --resync # Establish new baseline + basic-memory cloud bisync --watch # Continuous sync every 60s + basic-memory cloud bisync --watch --interval 30 # Continuous sync every 30s + basic-memory cloud bisync --profile safe # Use safe profile (keep conflicts) + basic-memory cloud bisync --verbose # Show detailed file sync output + """ + try: + if watch: + run_bisync_watch(profile_name=profile, interval_seconds=interval) + else: + run_bisync(profile_name=profile, dry_run=dry_run, resync=resync, verbose=verbose) + except Exception as e: + console.print(f"[red]Bisync failed: {e}[/red]") + raise typer.Exit(1) + + +@cloud_app.command("bisync-status") +def bisync_status() -> None: + """Show current bisync status and configuration. + + DEPRECATED: Use 'bm cloud status' instead (bisync is now the default). + """ + console.print( + "[yellow]Note: 'bisync-status' is deprecated. Use 'bm cloud status' instead.[/yellow]" + ) + console.print("[dim]Showing bisync status...[/dim]\n") + show_bisync_status() + + +@cloud_app.command("check") +def check( + one_way: bool = typer.Option( + False, + "--one-way", + help="Only check for missing files on destination (faster)", + ), +) -> None: + """Check file integrity between local and cloud storage using rclone check. + + Verifies that files match between your local bisync directory and cloud storage + without transferring any data. This is useful for validating sync integrity. + + Examples: + bm cloud check # Full integrity check + bm cloud check --one-way # Faster check (missing files only) + """ + try: + run_check(one_way=one_way) + except Exception as e: + console.print(f"[red]Check failed: {e}[/red]") + raise typer.Exit(1) diff --git a/src/basic_memory/cli/commands/cloud/mount_commands.py b/src/basic_memory/cli/commands/cloud/mount_commands.py index ad8da8074..01e115697 100644 --- a/src/basic_memory/cli/commands/cloud/mount_commands.py +++ b/src/basic_memory/cli/commands/cloud/mount_commands.py @@ -107,7 +107,7 @@ def setup_cloud_mount() -> None: # Step 5: Perform initial mount console.print("\n[blue]Step 5: Mounting cloud files...[/blue]") - mount_path = get_default_mount_path(tenant_id) + mount_path = get_default_mount_path() MOUNT_PROFILES["balanced"] mount_cloud_files( @@ -154,7 +154,7 @@ def mount_cloud_files( # Set default mount path if not provided if not mount_path: - mount_path = get_default_mount_path(tenant_id) + mount_path = get_default_mount_path() # Get mount profile if profile_name not in MOUNT_PROFILES: @@ -215,7 +215,7 @@ def unmount_cloud_files(tenant_id: Optional[str] = None) -> None: if not tenant_id: raise MountError("Could not determine tenant ID") - mount_path = get_default_mount_path(tenant_id) + mount_path = get_default_mount_path() if not is_path_mounted(mount_path): console.print(f"[yellow]Path {mount_path} is not mounted[/yellow]") @@ -255,7 +255,7 @@ def show_mount_status() -> None: console.print("[red]Could not determine tenant ID[/red]") return - mount_path = get_default_mount_path(tenant_id) + mount_path = get_default_mount_path() # Create status table table = Table(title="Cloud Mount Status", show_header=True, header_style="bold blue") diff --git a/src/basic_memory/cli/commands/cloud/rclone_config.py b/src/basic_memory/cli/commands/cloud/rclone_config.py index fa638383e..2f648fb25 100644 --- a/src/basic_memory/cli/commands/cloud/rclone_config.py +++ b/src/basic_memory/cli/commands/cloud/rclone_config.py @@ -168,9 +168,13 @@ def remove_tenant_from_rclone_config(tenant_id: str) -> bool: return False -def get_default_mount_path(tenant_id: str) -> Path: - """Get default mount path for a tenant.""" - return Path.home() / f"basic-memory-{tenant_id}" +def get_default_mount_path() -> Path: + """Get default mount path (fixed location per SPEC-9). + + Returns: + Fixed mount path: ~/basic-memory-cloud/ + """ + return Path.home() / "basic-memory-cloud" def build_mount_command( diff --git a/src/basic_memory/cli/commands/command_utils.py b/src/basic_memory/cli/commands/command_utils.py new file mode 100644 index 000000000..ab69cff4e --- /dev/null +++ b/src/basic_memory/cli/commands/command_utils.py @@ -0,0 +1,60 @@ +"""utility functions for commands""" + +from typing import Optional + +from mcp.server.fastmcp.exceptions import ToolError +import typer + +from rich.console import Console + +from basic_memory.cli.commands.cloud import get_authenticated_headers +from basic_memory.mcp.async_client import client + +from basic_memory.mcp.tools.utils import call_post, call_get +from basic_memory.mcp.project_context import get_active_project +from basic_memory.schemas import ProjectInfoResponse + +console = Console() + + +async def run_sync(project: Optional[str] = None): + """Run sync operation via API endpoint.""" + + try: + from basic_memory.config import ConfigManager + + config = ConfigManager().config + auth_headers = {} + if config.cloud_mode_enabled: + auth_headers = await get_authenticated_headers() + + project_item = await get_active_project(client, project, None, headers=auth_headers) + response = await call_post( + client, f"{project_item.project_url}/project/sync", headers=auth_headers + ) + data = response.json() + console.print(f"[green]✓ {data['message']}[/green]") + except (ToolError, ValueError) as e: + console.print(f"[red]✗ Sync failed: {e}[/red]") + raise typer.Exit(1) + + +async def get_project_info(project: str): + """Run sync operation via API endpoint.""" + + try: + from basic_memory.config import ConfigManager + + config = ConfigManager().config + auth_headers = {} + if config.cloud_mode_enabled: + auth_headers = await get_authenticated_headers() + + project_item = await get_active_project(client, project, None, headers=auth_headers) + response = await call_get( + client, f"{project_item.project_url}/project/info", headers=auth_headers + ) + return ProjectInfoResponse.model_validate(response.json()) + except (ToolError, ValueError) as e: + console.print(f"[red]✗ Sync failed: {e}[/red]") + raise typer.Exit(1) diff --git a/src/basic_memory/cli/commands/project.py b/src/basic_memory/cli/commands/project.py index d9fac5d29..20463ebee 100644 --- a/src/basic_memory/cli/commands/project.py +++ b/src/basic_memory/cli/commands/project.py @@ -9,12 +9,13 @@ from rich.table import Table from basic_memory.cli.app import app -from basic_memory.mcp.resources.project_info import project_info +from basic_memory.cli.commands.cloud import get_authenticated_headers +from basic_memory.cli.commands.command_utils import get_project_info +from basic_memory.config import ConfigManager import json from datetime import datetime from rich.panel import Panel -from rich.tree import Tree from basic_memory.mcp.async_client import client from basic_memory.mcp.tools.utils import call_get from basic_memory.schemas.project_info import ProjectList @@ -31,6 +32,8 @@ project_app = typer.Typer(help="Manage multiple Basic Memory projects") app.add_typer(project_app, name="project") +config = ConfigManager().config + def format_path(path: str) -> str: """Format a path for display, using ~ for home directory.""" @@ -42,10 +45,14 @@ def format_path(path: str) -> str: @project_app.command("list") def list_projects() -> None: - """List all configured projects.""" + """List all Basic Memory projects.""" # Use API to list projects try: - response = asyncio.run(call_get(client, "/projects/projects")) + auth_headers = {} + if config.cloud_mode_enabled: + auth_headers = asyncio.run(get_authenticated_headers()) + + response = asyncio.run(call_get(client, "/projects/projects", headers=auth_headers)) result = ProjectList.model_validate(response.json()) table = Table(title="Basic Memory Projects") @@ -63,42 +70,75 @@ def list_projects() -> None: raise typer.Exit(1) -@project_app.command("add") -def add_project( - name: str = typer.Argument(..., help="Name of the project"), - path: str = typer.Argument(..., help="Path to the project directory"), - set_default: bool = typer.Option(False, "--default", help="Set as default project"), -) -> None: - """Add a new project.""" - # Resolve to absolute path - resolved_path = Path(os.path.abspath(os.path.expanduser(path))).as_posix() +if config.cloud_mode_enabled: - try: - data = {"name": name, "path": resolved_path, "set_default": set_default} + @project_app.command("add") + def add_project_cloud( + name: str = typer.Argument(..., help="Name of the project"), + set_default: bool = typer.Option(False, "--default", help="Set as default project"), + ) -> None: + """Add a new project to Basic Memory Cloud""" - response = asyncio.run(call_post(client, "/projects/projects", json=data)) - result = ProjectStatusResponse.model_validate(response.json()) + try: + auth_headers = asyncio.run(get_authenticated_headers()) - console.print(f"[green]{result.message}[/green]") - except Exception as e: - console.print(f"[red]Error adding project: {str(e)}[/red]") - raise typer.Exit(1) + data = {"name": name, "path": generate_permalink(name), "set_default": set_default} - # Display usage hint - console.print("\nTo use this project:") - console.print(f" basic-memory --project={name} ") - console.print(" # or") - console.print(f" basic-memory project default {name}") + response = asyncio.run( + call_post(client, "/projects/projects", json=data, headers=auth_headers) + ) + result = ProjectStatusResponse.model_validate(response.json()) + + console.print(f"[green]{result.message}[/green]") + except Exception as e: + console.print(f"[red]Error adding project: {str(e)}[/red]") + raise typer.Exit(1) + + # Display usage hint + console.print("\nTo use this project:") + console.print(f" basic-memory --project={name} ") +else: + + @project_app.command("add") + def add_project( + name: str = typer.Argument(..., help="Name of the project"), + path: str = typer.Argument(..., help="Path to the project directory"), + set_default: bool = typer.Option(False, "--default", help="Set as default project"), + ) -> None: + """Add a new project.""" + # Resolve to absolute path + resolved_path = Path(os.path.abspath(os.path.expanduser(path))).as_posix() + + try: + data = {"name": name, "path": resolved_path, "set_default": set_default} + + response = asyncio.run(call_post(client, "/projects/projects", json=data)) + result = ProjectStatusResponse.model_validate(response.json()) + + console.print(f"[green]{result.message}[/green]") + except Exception as e: + console.print(f"[red]Error adding project: {str(e)}[/red]") + raise typer.Exit(1) + + # Display usage hint + console.print("\nTo use this project:") + console.print(f" basic-memory --project={name} ") @project_app.command("remove") def remove_project( name: str = typer.Argument(..., help="Name of the project to remove"), ) -> None: - """Remove a project from configuration.""" + """Remove a project.""" try: + auth_headers = {} + if config.cloud_mode_enabled: + auth_headers = asyncio.run(get_authenticated_headers()) + project_permalink = generate_permalink(name) - response = asyncio.run(call_delete(client, f"/projects/{project_permalink}")) + response = asyncio.run( + call_delete(client, f"/projects/{project_permalink}", headers=auth_headers) + ) result = ProjectStatusResponse.model_validate(response.json()) console.print(f"[green]{result.message}[/green]") @@ -110,102 +150,96 @@ def remove_project( console.print("[yellow]Note: The project files have not been deleted from disk.[/yellow]") -@project_app.command("default") -def set_default_project( - name: str = typer.Argument(..., help="Name of the project to set as CLI default"), -) -> None: - """Set the default project for CLI operations (when no --project flag is specified).""" - try: - project_permalink = generate_permalink(name) - response = asyncio.run(call_put(client, f"/projects/{project_permalink}/default")) - result = ProjectStatusResponse.model_validate(response.json()) - - console.print(f"[green]{result.message}[/green]") - except Exception as e: - console.print(f"[red]Error setting default project: {str(e)}[/red]") - raise typer.Exit(1) - - # The API call above updates the config file default - console.print( - f"[green]CLI commands will now use '{name}' when no --project flag is specified[/green]" - ) - - -@project_app.command("sync-config") -def synchronize_projects() -> None: - """Synchronize project config between configuration file and database.""" - # Call the API to synchronize projects - - try: - response = asyncio.run(call_post(client, "/projects/sync")) - result = ProjectStatusResponse.model_validate(response.json()) - - console.print(f"[green]{result.message}[/green]") - except Exception as e: # pragma: no cover - console.print(f"[red]Error synchronizing projects: {str(e)}[/red]") - raise typer.Exit(1) - - -@project_app.command("move") -def move_project( - name: str = typer.Argument(..., help="Name of the project to move"), - new_path: str = typer.Argument(..., help="New absolute path for the project"), -) -> None: - """Move a project to a new location.""" - # Resolve to absolute path - resolved_path = Path(os.path.abspath(os.path.expanduser(new_path))).as_posix() - - try: - data = {"path": resolved_path} - - project_permalink = generate_permalink(name) - - # TODO fix route to use ProjectPathDep - response = asyncio.run( - call_patch(client, f"/{name}/project/{project_permalink}", json=data) - ) - result = ProjectStatusResponse.model_validate(response.json()) +if not config.cloud_mode_enabled: + + @project_app.command("default") + def set_default_project( + name: str = typer.Argument(..., help="Name of the project to set as CLI default"), + ) -> None: + """Set the default project when 'config.default_project_mode' is set.""" + try: + project_permalink = generate_permalink(name) + response = asyncio.run(call_put(client, f"/projects/{project_permalink}/default")) + result = ProjectStatusResponse.model_validate(response.json()) + + console.print(f"[green]{result.message}[/green]") + except Exception as e: + console.print(f"[red]Error setting default project: {str(e)}[/red]") + raise typer.Exit(1) + + @project_app.command("sync-config") + def synchronize_projects() -> None: + """Synchronize project config between configuration file and database.""" + # Call the API to synchronize projects + + try: + response = asyncio.run(call_post(client, "/projects/config/sync")) + result = ProjectStatusResponse.model_validate(response.json()) + + console.print(f"[green]{result.message}[/green]") + except Exception as e: # pragma: no cover + console.print(f"[red]Error synchronizing projects: {str(e)}[/red]") + raise typer.Exit(1) + + @project_app.command("move") + def move_project( + name: str = typer.Argument(..., help="Name of the project to move"), + new_path: str = typer.Argument(..., help="New absolute path for the project"), + ) -> None: + """Move a project to a new location.""" + # Resolve to absolute path + resolved_path = Path(os.path.abspath(os.path.expanduser(new_path))).as_posix() + + try: + data = {"path": resolved_path} + + project_permalink = generate_permalink(name) + + # TODO fix route to use ProjectPathDep + response = asyncio.run( + call_patch(client, f"/{name}/project/{project_permalink}", json=data) + ) + result = ProjectStatusResponse.model_validate(response.json()) - console.print(f"[green]{result.message}[/green]") + console.print(f"[green]{result.message}[/green]") - # Show important file movement reminder - console.print() # Empty line for spacing - console.print( - Panel( - "[bold red]IMPORTANT:[/bold red] Project configuration updated successfully.\n\n" - "[yellow]You must manually move your project files from the old location to:[/yellow]\n" - f"[cyan]{resolved_path}[/cyan]\n\n" - "[dim]Basic Memory has only updated the configuration - your files remain in their original location.[/dim]", - title="⚠️ Manual File Movement Required", - border_style="yellow", - expand=False, + # Show important file movement reminder + console.print() # Empty line for spacing + console.print( + Panel( + "[bold red]IMPORTANT:[/bold red] Project configuration updated successfully.\n\n" + "[yellow]You must manually move your project files from the old location to:[/yellow]\n" + f"[cyan]{resolved_path}[/cyan]\n\n" + "[dim]Basic Memory has only updated the configuration - your files remain in their original location.[/dim]", + title="⚠️ Manual File Movement Required", + border_style="yellow", + expand=False, + ) ) - ) - except Exception as e: - console.print(f"[red]Error moving project: {str(e)}[/red]") - raise typer.Exit(1) + except Exception as e: + console.print(f"[red]Error moving project: {str(e)}[/red]") + raise typer.Exit(1) @project_app.command("info") def display_project_info( + name: str = typer.Argument(..., help="Name of the project"), json_output: bool = typer.Option(False, "--json", help="Output in JSON format"), ): """Display detailed information and statistics about the current project.""" try: # Get project info - info = asyncio.run(project_info.fn()) # type: ignore # pyright: ignore [reportAttributeAccessIssue] + info = asyncio.run(get_project_info(name)) if json_output: # Convert to JSON and print print(json.dumps(info.model_dump(), indent=2, default=str)) else: - # Create rich display - console = Console() - # Project configuration section console.print( Panel( + f"Basic Memory version: [bold green]{info.system.version}[/bold green]\n" f"[bold]Project:[/bold] {info.project_name}\n" f"[bold]Path:[/bold] {info.project_path}\n" f"[bold]Default Project:[/bold] {info.default_project}\n", @@ -275,42 +309,6 @@ def display_project_info( console.print(recent_table) - # System status - system_tree = Tree("🖥️ System Status") - system_tree.add(f"Basic Memory version: [bold green]{info.system.version}[/bold green]") - system_tree.add( - f"Database: [cyan]{info.system.database_path}[/cyan] ([green]{info.system.database_size}[/green])" - ) - - # Watch status - if info.system.watch_status: # pragma: no cover - watch_branch = system_tree.add("Watch Service") - running = info.system.watch_status.get("running", False) - status_color = "green" if running else "red" - watch_branch.add( - f"Status: [bold {status_color}]{'Running' if running else 'Stopped'}[/bold {status_color}]" - ) - - if running: - start_time = ( - datetime.fromisoformat(info.system.watch_status.get("start_time", "")) - if isinstance(info.system.watch_status.get("start_time"), str) - else info.system.watch_status.get("start_time") - ) - watch_branch.add( - f"Running since: [cyan]{start_time.strftime('%Y-%m-%d %H:%M')}[/cyan]" - ) - watch_branch.add( - f"Files synced: [green]{info.system.watch_status.get('synced_files', 0)}[/green]" - ) - watch_branch.add( - f"Errors: [{'red' if info.system.watch_status.get('error_count', 0) > 0 else 'green'}]{info.system.watch_status.get('error_count', 0)}[/{'red' if info.system.watch_status.get('error_count', 0) > 0 else 'green'}]" - ) - else: - system_tree.add("[yellow]Watch service not running[/yellow]") - - console.print(system_tree) - # Available projects projects_table = Table(title="📁 Available Projects") projects_table.add_column("Name", style="blue") diff --git a/src/basic_memory/cli/commands/status.py b/src/basic_memory/cli/commands/status.py index 5f696075d..7d6399052 100644 --- a/src/basic_memory/cli/commands/status.py +++ b/src/basic_memory/cli/commands/status.py @@ -2,19 +2,21 @@ import asyncio from typing import Set, Dict +from typing import Annotated, Optional +from mcp.server.fastmcp.exceptions import ToolError import typer from loguru import logger from rich.console import Console from rich.panel import Panel from rich.tree import Tree -from basic_memory import db from basic_memory.cli.app import app -from basic_memory.cli.commands.sync import get_sync_service -from basic_memory.config import ConfigManager, get_project_config -from basic_memory.repository import ProjectRepository -from basic_memory.sync.sync_service import SyncReport +from basic_memory.cli.commands.cloud import get_authenticated_headers +from basic_memory.mcp.async_client import client +from basic_memory.mcp.tools.utils import call_post +from basic_memory.schemas import SyncReportResponse +from basic_memory.mcp.project_context import get_active_project # Create rich console console = Console() @@ -47,7 +49,7 @@ def add_files_to_tree( branch.add(f"[{style}]{file_name}[/{style}]") -def group_changes_by_directory(changes: SyncReport) -> Dict[str, Dict[str, int]]: +def group_changes_by_directory(changes: SyncReportResponse) -> Dict[str, Dict[str, int]]: """Group changes by directory for summary view.""" by_dir = {} for change_type, paths in [ @@ -87,7 +89,9 @@ def build_directory_summary(counts: Dict[str, int]) -> str: return " ".join(parts) -def display_changes(project_name: str, title: str, changes: SyncReport, verbose: bool = False): +def display_changes( + project_name: str, title: str, changes: SyncReportResponse, verbose: bool = False +): """Display changes using Rich for better visualization.""" tree = Tree(f"{project_name}: {title}") @@ -122,33 +126,41 @@ def display_changes(project_name: str, title: str, changes: SyncReport, verbose: console.print(Panel(tree, expand=False)) -async def run_status(verbose: bool = False): # pragma: no cover +async def run_status(project: Optional[str] = None, verbose: bool = False): # pragma: no cover """Check sync status of files vs database.""" - # Check knowledge/ directory - app_config = ConfigManager().config - config = get_project_config() + try: + from basic_memory.config import ConfigManager + + config = ConfigManager().config + auth_headers = {} + if config.cloud_mode_enabled: + auth_headers = await get_authenticated_headers() + + project_item = await get_active_project(client, project, None) + response = await call_post( + client, f"{project_item.project_url}/project/status", headers=auth_headers + ) + sync_report = SyncReportResponse.model_validate(response.json()) - _, session_maker = await db.get_or_create_db( - db_path=app_config.database_path, db_type=db.DatabaseType.FILESYSTEM - ) - project_repository = ProjectRepository(session_maker) - project = await project_repository.get_by_name(config.project) - if not project: # pragma: no cover - raise Exception(f"Project '{config.project}' not found") + display_changes(project_item.name, "Status", sync_report, verbose) - sync_service = await get_sync_service(project) - knowledge_changes = await sync_service.scan(config.home) - display_changes(project.name, "Status", knowledge_changes, verbose) + except (ValueError, ToolError) as e: + console.print(f"[red]✗ Error: {e}[/red]") + raise typer.Exit(1) @app.command() def status( + project: Annotated[ + Optional[str], + typer.Option(help="The project name."), + ] = None, verbose: bool = typer.Option(False, "--verbose", "-v", help="Show detailed file information"), ): """Show sync status between files and database.""" try: - asyncio.run(run_status(verbose)) # pragma: no cover + asyncio.run(run_status(project, verbose)) # pragma: no cover except Exception as e: logger.error(f"Error checking status: {e}") typer.echo(f"Error checking status: {e}", err=True) diff --git a/src/basic_memory/cli/commands/sync.py b/src/basic_memory/cli/commands/sync.py index ad49d3824..6a5557aba 100644 --- a/src/basic_memory/cli/commands/sync.py +++ b/src/basic_memory/cli/commands/sync.py @@ -1,242 +1,59 @@ """Command module for basic-memory sync operations.""" import asyncio -from collections import defaultdict -from dataclasses import dataclass -from pathlib import Path -from typing import List, Dict +from typing import Annotated, Optional import typer -from loguru import logger -from rich.console import Console -from rich.tree import Tree -from basic_memory import db from basic_memory.cli.app import app -from basic_memory.config import ConfigManager, get_project_config -from basic_memory.markdown import EntityParser -from basic_memory.markdown.markdown_processor import MarkdownProcessor -from basic_memory.models import Project -from basic_memory.repository import ( - EntityRepository, - ObservationRepository, - RelationRepository, - ProjectRepository, -) -from basic_memory.repository.search_repository import SearchRepository -from basic_memory.services import EntityService, FileService -from basic_memory.services.link_resolver import LinkResolver -from basic_memory.services.search_service import SearchService -from basic_memory.sync import SyncService -from basic_memory.sync.sync_service import SyncReport - -console = Console() - - -@dataclass -class ValidationIssue: - file_path: str - error: str - - -async def get_sync_service(project: Project) -> SyncService: # pragma: no cover - """Get sync service instance with all dependencies.""" - - app_config = ConfigManager().config - _, session_maker = await db.get_or_create_db( - db_path=app_config.database_path, db_type=db.DatabaseType.FILESYSTEM - ) - - project_path = Path(project.path) - entity_parser = EntityParser(project_path) - markdown_processor = MarkdownProcessor(entity_parser) - file_service = FileService(project_path, markdown_processor) - - # Initialize repositories - entity_repository = EntityRepository(session_maker, project_id=project.id) - observation_repository = ObservationRepository(session_maker, project_id=project.id) - relation_repository = RelationRepository(session_maker, project_id=project.id) - search_repository = SearchRepository(session_maker, project_id=project.id) - - # Initialize services - search_service = SearchService(search_repository, entity_repository, file_service) - link_resolver = LinkResolver(entity_repository, search_service) - - # Initialize services - entity_service = EntityService( - entity_parser, - entity_repository, - observation_repository, - relation_repository, - file_service, - link_resolver, - ) - - # Create sync service - sync_service = SyncService( - app_config=app_config, - entity_service=entity_service, - entity_parser=entity_parser, - entity_repository=entity_repository, - relation_repository=relation_repository, - search_service=search_service, - file_service=file_service, - ) - - return sync_service - - -def group_issues_by_directory(issues: List[ValidationIssue]) -> Dict[str, List[ValidationIssue]]: - """Group validation issues by directory.""" - grouped = defaultdict(list) - for issue in issues: - dir_name = Path(issue.file_path).parent.name - grouped[dir_name].append(issue) - return dict(grouped) - - -def display_sync_summary(knowledge: SyncReport): - """Display a one-line summary of sync changes.""" - config = get_project_config() - total_changes = knowledge.total - project_name = config.project - - if total_changes == 0: - console.print(f"[green]Project '{project_name}': Everything up to date[/green]") - return - - # Format as: "Synced X files (A new, B modified, C moved, D deleted)" - changes = [] - new_count = len(knowledge.new) - mod_count = len(knowledge.modified) - move_count = len(knowledge.moves) - del_count = len(knowledge.deleted) - - if new_count: - changes.append(f"[green]{new_count} new[/green]") - if mod_count: - changes.append(f"[yellow]{mod_count} modified[/yellow]") - if move_count: - changes.append(f"[blue]{move_count} moved[/blue]") - if del_count: - changes.append(f"[red]{del_count} deleted[/red]") - - console.print(f"Project '{project_name}': Synced {total_changes} files ({', '.join(changes)})") - - -def display_detailed_sync_results(knowledge: SyncReport): - """Display detailed sync results with trees.""" - config = get_project_config() - project_name = config.project - - if knowledge.total == 0: - console.print(f"\n[green]Project '{project_name}': Everything up to date[/green]") - return - - console.print(f"\n[bold]Sync Results for Project '{project_name}'[/bold]") - - if knowledge.total > 0: - knowledge_tree = Tree("[bold]Knowledge Files[/bold]") - if knowledge.new: - created = knowledge_tree.add("[green]Created[/green]") - for path in sorted(knowledge.new): - checksum = knowledge.checksums.get(path, "") - created.add(f"[green]{path}[/green] ({checksum[:8]})") - if knowledge.modified: - modified = knowledge_tree.add("[yellow]Modified[/yellow]") - for path in sorted(knowledge.modified): - checksum = knowledge.checksums.get(path, "") - modified.add(f"[yellow]{path}[/yellow] ({checksum[:8]})") - if knowledge.moves: - moved = knowledge_tree.add("[blue]Moved[/blue]") - for old_path, new_path in sorted(knowledge.moves.items()): - checksum = knowledge.checksums.get(new_path, "") - moved.add(f"[blue]{old_path}[/blue] → [blue]{new_path}[/blue] ({checksum[:8]})") - if knowledge.deleted: - deleted = knowledge_tree.add("[red]Deleted[/red]") - for path in sorted(knowledge.deleted): - deleted.add(f"[red]{path}[/red]") - console.print(knowledge_tree) - - -async def run_sync(verbose: bool = False): - """Run sync operation.""" - app_config = ConfigManager().config - config = get_project_config() - - _, session_maker = await db.get_or_create_db( - db_path=app_config.database_path, db_type=db.DatabaseType.FILESYSTEM - ) - project_repository = ProjectRepository(session_maker) - project = await project_repository.get_by_name(config.project) - if not project: # pragma: no cover - raise Exception(f"Project '{config.project}' not found") - - import time - - start_time = time.time() - - logger.info( - "Sync command started", - project=config.project, - verbose=verbose, - directory=str(config.home), - ) - - sync_service = await get_sync_service(project) - - logger.info("Running one-time sync") - knowledge_changes = await sync_service.sync(config.home, project_name=project.name) - - # Log results - duration_ms = int((time.time() - start_time) * 1000) - logger.info( - "Sync command completed", - project=config.project, - total_changes=knowledge_changes.total, - new_files=len(knowledge_changes.new), - modified_files=len(knowledge_changes.modified), - deleted_files=len(knowledge_changes.deleted), - moved_files=len(knowledge_changes.moves), - duration_ms=duration_ms, - ) - - # Display results - if verbose: - display_detailed_sync_results(knowledge_changes) - else: - display_sync_summary(knowledge_changes) # pragma: no cover +from basic_memory.cli.commands.command_utils import run_sync +from basic_memory.config import ConfigManager @app.command() def sync( - verbose: bool = typer.Option( - False, - "--verbose", - "-v", - help="Show detailed sync information.", - ), + project: Annotated[ + Optional[str], + typer.Option(help="The project name."), + ] = None, + watch: Annotated[ + bool, + typer.Option("--watch", help="Run continuous sync (cloud mode only)"), + ] = False, + interval: Annotated[ + int, + typer.Option("--interval", help="Sync interval in seconds for watch mode (default: 60)"), + ] = 60, ) -> None: - """Sync knowledge files with the database.""" - config = get_project_config() - - try: - # Show which project we're syncing - typer.echo(f"Syncing project: {config.project}") - typer.echo(f"Project path: {config.home}") - - # Run sync - asyncio.run(run_sync(verbose=verbose)) - - except Exception as e: # pragma: no cover - if not isinstance(e, typer.Exit): - logger.exception( - "Sync command failed", - f"project={config.project}," - f"error={str(e)}," - f"error_type={type(e).__name__}," - f"directory={str(config.home)}", + """Sync knowledge files with the database. + + In local mode: Scans filesystem and updates database. + In cloud mode: Runs bidirectional file sync (bisync) then updates database. + + Examples: + bm sync # One-time sync + bm sync --watch # Continuous sync every 60s + bm sync --watch --interval 30 # Continuous sync every 30s + """ + config = ConfigManager().config + + if config.cloud_mode_enabled: + # Cloud mode: run bisync which includes database sync + from basic_memory.cli.commands.cloud.bisync_commands import run_bisync, run_bisync_watch + + try: + if watch: + run_bisync_watch(interval_seconds=interval) + else: + run_bisync() + except Exception: + raise typer.Exit(1) + else: + # Local mode: just database sync + if watch: + typer.echo( + "Error: --watch is only available in cloud mode. Run 'bm cloud login' first." ) - typer.echo(f"Error during sync: {e}", err=True) raise typer.Exit(1) - raise + + asyncio.run(run_sync(project)) diff --git a/src/basic_memory/config.py b/src/basic_memory/config.py index 6cac3668b..cfd73e3fb 100644 --- a/src/basic_memory/config.py +++ b/src/basic_memory/config.py @@ -98,15 +98,9 @@ class BasicMemoryConfig(BaseSettings): description="Skip expensive initialization synchronization. Useful for cloud/stateless deployments where project reconciliation is not needed.", ) - # API connection configuration - api_url: Optional[str] = Field( - default=None, - description="URL of remote Basic Memory API. If set, MCP will connect to this API instead of using local ASGI transport.", - ) - # Cloud configuration cloud_client_id: str = Field( - default="client_01K4DGBWAZWP83N3H8VVEMRX6W", + default="client_01K6KWQPW6J1M8VV7R3TZP5A6M", description="OAuth client ID for Basic Memory Cloud", ) @@ -116,8 +110,39 @@ class BasicMemoryConfig(BaseSettings): ) cloud_host: str = Field( - default="https://cloud.basicmemory.com", - description="Basic Memory Cloud proxy host URL", + default_factory=lambda: os.getenv( + "BASIC_MEMORY_CLOUD_HOST", "https://cloud.basicmemory.com" + ), + description="Basic Memory Cloud host URL", + ) + + cloud_mode: bool = Field( + default=False, + description="Enable cloud mode - all requests go to cloud instead of local (config file value)", + ) + + @property + def cloud_mode_enabled(self) -> bool: + """Check if cloud mode is enabled. + + Priority: + 1. BASIC_MEMORY_CLOUD_MODE environment variable + 2. Config file value (cloud_mode) + """ + env_value = os.environ.get("BASIC_MEMORY_CLOUD_MODE", "").lower() + if env_value in ("true", "1", "yes"): + return True + elif env_value in ("false", "0", "no"): + return False + # Fall back to config file value + return self.cloud_mode + + bisync_config: Dict[str, Any] = Field( + default_factory=lambda: { + "profile": "balanced", + "sync_dir": str(Path.home() / "basic-memory-cloud-sync"), + }, + description="Bisync configuration for cloud sync", ) model_config = SettingsConfigDict( diff --git a/src/basic_memory/ignore_utils.py b/src/basic_memory/ignore_utils.py index 84f6915a1..0753b2208 100644 --- a/src/basic_memory/ignore_utils.py +++ b/src/basic_memory/ignore_utils.py @@ -6,37 +6,178 @@ # Common directories and patterns to ignore by default +# These are used as fallback if .bmignore doesn't exist DEFAULT_IGNORE_PATTERNS = { + # Hidden files (files starting with dot) + ".*", + # Basic Memory internal files + "memory.db", + "memory.db-shm", + "memory.db-wal", + "config.json", + # Version control ".git", - ".venv", - "venv", - "env", - ".env", - "node_modules", + ".svn", + # Python "__pycache__", - ".pytest_cache", - ".coverage", "*.pyc", "*.pyo", "*.pyd", - ".DS_Store", - "Thumbs.db", - ".idea", - ".vscode", + ".pytest_cache", + ".coverage", "*.egg-info", - "build", - "dist", ".tox", - ".cache", ".mypy_cache", ".ruff_cache", - ".obsidian", + # Virtual environments + ".venv", + "venv", + "env", + ".env", + # Node.js + "node_modules", + # Build artifacts + "build", + "dist", + ".cache", + # IDE ".idea", + ".vscode", + # OS files + ".DS_Store", + "Thumbs.db", + "desktop.ini", + # Obsidian + ".obsidian", + # Temporary files + "*.tmp", + "*.swp", + "*.swo", + "*~", } +def get_bmignore_path() -> Path: + """Get path to .bmignore file. + + Returns: + Path to ~/.basic-memory/.bmignore + """ + return Path.home() / ".basic-memory" / ".bmignore" + + +def create_default_bmignore() -> None: + """Create default .bmignore file if it doesn't exist. + + This ensures users have a file they can customize for all Basic Memory operations. + """ + bmignore_path = get_bmignore_path() + + if bmignore_path.exists(): + return + + bmignore_path.parent.mkdir(parents=True, exist_ok=True) + bmignore_path.write_text("""# Basic Memory Ignore Patterns +# This file is used by both 'bm cloud upload', 'bm cloud bisync', and file sync +# Patterns use standard gitignore-style syntax + +# Hidden files (files starting with dot) +.* + +# Basic Memory internal files +memory.db +memory.db-shm +memory.db-wal +config.json + +# Version control +.git +.svn + +# Python +__pycache__ +*.pyc +*.pyo +*.pyd +.pytest_cache +.coverage +*.egg-info +.tox +.mypy_cache +.ruff_cache + +# Virtual environments +.venv +venv +env +.env + +# Node.js +node_modules + +# Build artifacts +build +dist +.cache + +# IDE +.idea +.vscode + +# OS files +.DS_Store +Thumbs.db +desktop.ini + +# Obsidian +.obsidian + +# Temporary files +*.tmp +*.swp +*.swo +*~ +""") + + +def load_bmignore_patterns() -> Set[str]: + """Load patterns from .bmignore file. + + Returns: + Set of patterns from .bmignore, or DEFAULT_IGNORE_PATTERNS if file doesn't exist + """ + bmignore_path = get_bmignore_path() + + # Create default file if it doesn't exist + if not bmignore_path.exists(): + create_default_bmignore() + + patterns = set() + + try: + with bmignore_path.open("r", encoding="utf-8") as f: + for line in f: + line = line.strip() + # Skip empty lines and comments + if line and not line.startswith("#"): + patterns.add(line) + except Exception: + # If we can't read .bmignore, fall back to defaults + return set(DEFAULT_IGNORE_PATTERNS) + + # If no patterns were loaded, use defaults + if not patterns: + return set(DEFAULT_IGNORE_PATTERNS) + + return patterns + + def load_gitignore_patterns(base_path: Path) -> Set[str]: - """Load gitignore patterns from .gitignore file and add default patterns. + """Load gitignore patterns from .gitignore file and .bmignore. + + Combines patterns from: + 1. ~/.basic-memory/.bmignore (user's global ignore patterns) + 2. {base_path}/.gitignore (project-specific patterns) Args: base_path: The base directory to search for .gitignore file @@ -44,7 +185,8 @@ def load_gitignore_patterns(base_path: Path) -> Set[str]: Returns: Set of patterns to ignore """ - patterns = set(DEFAULT_IGNORE_PATTERNS) + # Start with patterns from .bmignore + patterns = load_bmignore_patterns() gitignore_file = base_path / ".gitignore" if gitignore_file.exists(): @@ -109,7 +251,13 @@ def should_ignore_path(file_path: Path, base_path: Path, ignore_patterns: Set[st if pattern in relative_path.parts: return True - # Glob pattern match + # Check if any individual path part matches the glob pattern + # This handles cases like ".*" matching ".hidden.md" in "concept/.hidden.md" + for part in relative_path.parts: + if fnmatch.fnmatch(part, pattern): + return True + + # Glob pattern match on full path if fnmatch.fnmatch(relative_posix, pattern) or fnmatch.fnmatch(relative_str, pattern): return True diff --git a/src/basic_memory/mcp/async_client.py b/src/basic_memory/mcp/async_client.py index 882c627cb..905b12f87 100644 --- a/src/basic_memory/mcp/async_client.py +++ b/src/basic_memory/mcp/async_client.py @@ -1,8 +1,8 @@ -import os from httpx import ASGITransport, AsyncClient, Timeout from loguru import logger from basic_memory.api.app import app as fastapi_app +from basic_memory.config import ConfigManager def create_client() -> AsyncClient: @@ -11,8 +11,8 @@ def create_client() -> AsyncClient: Returns: AsyncClient configured for either local ASGI or remote proxy """ - proxy_base_url = os.getenv("BASIC_MEMORY_PROXY_URL", None) - logger.info(f"BASIC_MEMORY_PROXY_URL: {proxy_base_url}") + config_manager = ConfigManager() + config = config_manager.config # Configure timeout for longer operations like write_note # Default httpx timeout is 5 seconds which is too short for file operations @@ -23,13 +23,14 @@ def create_client() -> AsyncClient: pool=30.0, # 30 seconds for connection pool ) - if proxy_base_url: + if config.cloud_mode_enabled: # Use HTTP transport to proxy endpoint + proxy_base_url = f"{config.cloud_host}/proxy" logger.info(f"Creating HTTP client for proxy at: {proxy_base_url}") return AsyncClient(base_url=proxy_base_url, timeout=timeout) else: # Default: use ASGI transport for local API (development mode) - logger.debug("Creating ASGI client for local Basic Memory API") + logger.info("Creating ASGI client for local Basic Memory API") return AsyncClient( transport=ASGITransport(app=fastapi_app), base_url="http://test", timeout=timeout ) diff --git a/src/basic_memory/mcp/project_context.py b/src/basic_memory/mcp/project_context.py index dea7d93a0..2617b09c3 100644 --- a/src/basic_memory/mcp/project_context.py +++ b/src/basic_memory/mcp/project_context.py @@ -5,24 +5,30 @@ """ import os -from typing import Optional +from typing import Optional, List from httpx import AsyncClient +from httpx._types import ( + HeaderTypes, +) from loguru import logger from fastmcp import Context from basic_memory.config import ConfigManager from basic_memory.mcp.tools.utils import call_get -from basic_memory.schemas.project_info import ProjectItem +from basic_memory.schemas.project_info import ProjectItem, ProjectList from basic_memory.utils import generate_permalink async def resolve_project_parameter(project: Optional[str] = None) -> Optional[str]: """Resolve project parameter using three-tier hierarchy. - Resolution order: - 1. Single Project Mode (--project cli arg, or BASIC_MEMORY_MCP_PROJECT env var) - highest priority - 2. Explicit project parameter - medium priority - 3. Default project if default_project_mode=true - lowest priority + if config.cloud_mode: + project is required + else: + Resolution order: + 1. Single Project Mode (--project cli arg, or BASIC_MEMORY_MCP_PROJECT env var) - highest priority + 2. Explicit project parameter - medium priority + 3. Default project if default_project_mode=true - lowest priority Args: project: Optional explicit project parameter @@ -30,6 +36,16 @@ async def resolve_project_parameter(project: Optional[str] = None) -> Optional[s Returns: Resolved project name or None if no resolution possible """ + + config = ConfigManager().config + # if cloud_mode, project is required + if config.cloud_mode: + if project: + logger.debug(f"project: {project}, cloud_mode: {config.cloud_mode}") + return project + else: + raise ValueError("No project specified. Project is required for cloud mode.") + # Priority 1: CLI constraint overrides everything (--project arg sets env var) constrained_project = os.environ.get("BASIC_MEMORY_MCP_PROJECT") if constrained_project: @@ -42,7 +58,6 @@ async def resolve_project_parameter(project: Optional[str] = None) -> Optional[s return project # Priority 3: Default project mode - config = ConfigManager().config if config.default_project_mode: logger.debug(f"Using default project from config: {config.default_project}") return config.default_project @@ -51,16 +66,20 @@ async def resolve_project_parameter(project: Optional[str] = None) -> Optional[s return None +async def get_project_names(client: AsyncClient, headers: HeaderTypes | None = None) -> List[str]: + response = await call_get(client, "/projects/projects", headers=headers) + project_list = ProjectList.model_validate(response.json()) + return [project.name for project in project_list.projects] + + async def get_active_project( - client: AsyncClient, project: Optional[str] = None, context: Optional[Context] = None + client: AsyncClient, + project: Optional[str] = None, + context: Optional[Context] = None, + headers: HeaderTypes | None = None, ) -> ProjectItem: """Get and validate project, setting it in context if available. - Uses three-tier resolution: - 1. CLI constraint (BASIC_MEMORY_MCP_PROJECT env var) - 2. Explicit project parameter - 3. Default project if default_project_mode=true - Args: client: HTTP client for API calls project: Optional project name (resolved using hierarchy) @@ -73,12 +92,13 @@ async def get_active_project( ValueError: If no project can be resolved HTTPError: If project doesn't exist or is inaccessible """ - # Resolve project using three-tier hierarchy resolved_project = await resolve_project_parameter(project) if not resolved_project: + project_names = await get_project_names(client, headers) raise ValueError( - "No project specified. Either provide project parameter, " - "set default_project_mode=true in config, or use --project constraint." + "No project specified. " + "Either set 'default_project_mode=true' in config, or use 'project' argument.\n" + f"Available projects: {project_names}" ) project = resolved_project @@ -93,7 +113,7 @@ async def get_active_project( # Validate project exists by calling API logger.debug(f"Validating project: {project}") permalink = generate_permalink(project) - response = await call_get(client, f"/{permalink}/project/item") + response = await call_get(client, f"/{permalink}/project/item", headers=headers) active_project = ProjectItem.model_validate(response.json()) # Cache in context if available diff --git a/src/basic_memory/mcp/tools/sync_status.py b/src/basic_memory/mcp/tools/sync_status.py index 13c1165fd..6d39ba540 100644 --- a/src/basic_memory/mcp/tools/sync_status.py +++ b/src/basic_memory/mcp/tools/sync_status.py @@ -222,7 +222,6 @@ async def sync_status(project: Optional[str] = None, context: Context | None = N [ "", "**Note**: All configured projects will be automatically synced during startup.", - "You don't need to manually switch projects - Basic Memory handles this for you.", ] ) diff --git a/src/basic_memory/schemas/__init__.py b/src/basic_memory/schemas/__init__.py index 0c2455091..0717d72c8 100644 --- a/src/basic_memory/schemas/__init__.py +++ b/src/basic_memory/schemas/__init__.py @@ -48,6 +48,10 @@ DirectoryNode, ) +from basic_memory.schemas.sync_report import ( + SyncReportResponse, +) + # For convenient imports, export all models __all__ = [ # Base @@ -77,4 +81,6 @@ "ProjectInfoResponse", # Directory "DirectoryNode", + # Sync + "SyncReportResponse", ] diff --git a/src/basic_memory/schemas/cloud.py b/src/basic_memory/schemas/cloud.py new file mode 100644 index 000000000..4d33bd947 --- /dev/null +++ b/src/basic_memory/schemas/cloud.py @@ -0,0 +1,46 @@ +"""Schemas for cloud-related API responses.""" + +from pydantic import BaseModel, Field + + +class TenantMountInfo(BaseModel): + """Response from /tenant/mount/info endpoint.""" + + tenant_id: str = Field(..., description="Unique identifier for the tenant") + bucket_name: str = Field(..., description="S3 bucket name for the tenant") + + +class MountCredentials(BaseModel): + """Response from /tenant/mount/credentials endpoint.""" + + access_key: str = Field(..., description="S3 access key for mount") + secret_key: str = Field(..., description="S3 secret key for mount") + + +class CloudProject(BaseModel): + """Representation of a cloud project.""" + + name: str = Field(..., description="Project name") + path: str = Field(..., description="Project path on cloud") + + +class CloudProjectList(BaseModel): + """Response from /proxy/projects/projects endpoint.""" + + projects: list[CloudProject] = Field(default_factory=list, description="List of cloud projects") + + +class CloudProjectCreateRequest(BaseModel): + """Request to create a new cloud project.""" + + name: str = Field(..., description="Project name") + path: str = Field(..., description="Project path (permalink)") + set_default: bool = Field(default=False, description="Set as default project") + + +class CloudProjectCreateResponse(BaseModel): + """Response from creating a cloud project.""" + + name: str = Field(..., description="Created project name") + path: str = Field(..., description="Created project path") + message: str = Field(default="", description="Success message") diff --git a/src/basic_memory/schemas/sync_report.py b/src/basic_memory/schemas/sync_report.py new file mode 100644 index 000000000..267bf669a --- /dev/null +++ b/src/basic_memory/schemas/sync_report.py @@ -0,0 +1,48 @@ +"""Pydantic schemas for sync report responses.""" + +from typing import TYPE_CHECKING, Dict, Set + +from pydantic import BaseModel, Field + +# avoid cirular imports +if TYPE_CHECKING: + from basic_memory.sync.sync_service import SyncReport + + +class SyncReportResponse(BaseModel): + """Report of file changes found compared to database state. + + Used for API responses when scanning or syncing files. + """ + + new: Set[str] = Field(default_factory=set, description="Files on disk but not in database") + modified: Set[str] = Field(default_factory=set, description="Files with different checksums") + deleted: Set[str] = Field(default_factory=set, description="Files in database but not on disk") + moves: Dict[str, str] = Field( + default_factory=dict, description="Files moved (old_path -> new_path)" + ) + checksums: Dict[str, str] = Field( + default_factory=dict, description="Current file checksums (path -> checksum)" + ) + total: int = Field(description="Total number of changes") + + @classmethod + def from_sync_report(cls, report: "SyncReport") -> "SyncReportResponse": + """Convert SyncReport dataclass to Pydantic model. + + Args: + report: SyncReport dataclass from sync service + + Returns: + SyncReportResponse with same data + """ + return cls( + new=report.new, + modified=report.modified, + deleted=report.deleted, + moves=report.moves, + checksums=report.checksums, + total=report.total, + ) + + model_config = {"from_attributes": True} diff --git a/src/basic_memory/services/initialization.py b/src/basic_memory/services/initialization.py index 7038f8fd3..f45aa632e 100644 --- a/src/basic_memory/services/initialization.py +++ b/src/basic_memory/services/initialization.py @@ -11,7 +11,10 @@ from basic_memory import db from basic_memory.config import BasicMemoryConfig -from basic_memory.repository import ProjectRepository +from basic_memory.models import Project +from basic_memory.repository import ( + ProjectRepository, +) async def initialize_database(app_config: BasicMemoryConfig) -> None: @@ -102,14 +105,16 @@ async def initialize_file_sync( active_projects = await project_repository.get_active_projects() # Start sync for all projects as background tasks (non-blocking) - async def sync_project_background(project): + async def sync_project_background(project: Project): """Sync a single project in the background.""" # avoid circular imports - from basic_memory.cli.commands.sync import get_sync_service + from basic_memory.sync.sync_service import get_sync_service logger.info(f"Starting background sync for project: {project.name}") try: + # Create sync service sync_service = await get_sync_service(project) + sync_dir = Path(project.path) await sync_service.sync(sync_dir, project_name=project.name) logger.info(f"Background sync completed successfully for project: {project.name}") @@ -176,9 +181,16 @@ def ensure_initialization(app_config: BasicMemoryConfig) -> None: This is a wrapper for the async initialize_app function that can be called from synchronous code like CLI entry points. + No-op if app_config.cloud_mode == True. Cloud basic memory manages it's own projects + Args: app_config: The Basic Memory project configuration """ + # Skip initialization in cloud mode - cloud manages its own projects + if app_config.cloud_mode_enabled: + logger.debug("Skipping initialization in cloud mode - projects managed by cloud") + return + try: result = asyncio.run(initialize_app(app_config)) logger.info(f"Initialization completed successfully: result={result}") diff --git a/src/basic_memory/services/project_service.py b/src/basic_memory/services/project_service.py index 9e35c411f..2fc8e4266 100644 --- a/src/basic_memory/services/project_service.py +++ b/src/basic_memory/services/project_service.py @@ -21,6 +21,9 @@ from basic_memory.utils import generate_permalink +config = ConfigManager().config + + class ProjectService: """Service for managing Basic Memory projects.""" @@ -96,11 +99,16 @@ async def add_project(self, name: str, path: str, set_default: bool = False) -> Raises: ValueError: If the project already exists """ - if not self.repository: # pragma: no cover - raise ValueError("Repository is required for add_project") - - # Resolve to absolute path - resolved_path = Path(os.path.abspath(os.path.expanduser(path))).as_posix() + # in cloud mode, don't allow arbitrary paths. + if config.cloud_mode: + basic_memory_home = os.getenv("BASIC_MEMORY_HOME") + assert basic_memory_home is not None + base_path = Path(basic_memory_home) + + # Resolve to absolute path + resolved_path = Path(os.path.abspath(os.path.expanduser(base_path / path))).as_posix() + else: + resolved_path = Path(os.path.abspath(os.path.expanduser(path))).as_posix() # First add to config file (this will validate the project doesn't exist) project_config = self.config_manager.add_project(name, resolved_path) diff --git a/src/basic_memory/sync/sync_service.py b/src/basic_memory/sync/sync_service.py index 4aa52b247..1e7f19839 100644 --- a/src/basic_memory/sync/sync_service.py +++ b/src/basic_memory/sync/sync_service.py @@ -12,12 +12,16 @@ from loguru import logger from sqlalchemy.exc import IntegrityError -from basic_memory.config import BasicMemoryConfig +from basic_memory import db +from basic_memory.config import BasicMemoryConfig, ConfigManager from basic_memory.file_utils import has_frontmatter -from basic_memory.markdown import EntityParser -from basic_memory.models import Entity -from basic_memory.repository import EntityRepository, RelationRepository +from basic_memory.ignore_utils import load_bmignore_patterns, should_ignore_path +from basic_memory.markdown import EntityParser, MarkdownProcessor +from basic_memory.models import Entity, Project +from basic_memory.repository import EntityRepository, RelationRepository, ObservationRepository +from basic_memory.repository.search_repository import SearchRepository from basic_memory.services import EntityService, FileService +from basic_memory.services.link_resolver import LinkResolver from basic_memory.services.search_service import SearchService from basic_memory.services.sync_status_service import sync_status_tracker, SyncStatus @@ -83,6 +87,8 @@ def __init__( self.search_service = search_service self.file_service = file_service self._thread_pool = ThreadPoolExecutor(max_workers=app_config.sync_thread_pool_size) + # Load ignore patterns once at initialization for performance + self._ignore_patterns = load_bmignore_patterns() async def _read_file_async(self, file_path: Path) -> str: """Read file content in thread pool to avoid blocking the event loop.""" @@ -660,17 +666,33 @@ async def scan_directory(self, directory: Path) -> ScanResult: logger.debug(f"Scanning directory {directory}") result = ScanResult() + ignored_count = 0 for root, dirnames, filenames in os.walk(str(directory)): - # Skip dot directories in-place - dirnames[:] = [d for d in dirnames if not d.startswith(".")] + # Convert root to Path for easier manipulation + root_path = Path(root) + + # Filter out ignored directories in-place + dirnames_to_remove = [] + for dirname in dirnames: + dir_path = root_path / dirname + if should_ignore_path(dir_path, directory, self._ignore_patterns): + dirnames_to_remove.append(dirname) + ignored_count += 1 + + # Remove ignored directories from dirnames to prevent os.walk from descending + for dirname in dirnames_to_remove: + dirnames.remove(dirname) for filename in filenames: - # Skip dot files - if filename.startswith("."): + path = root_path / filename + + # Check if file should be ignored + if should_ignore_path(path, directory, self._ignore_patterns): + ignored_count += 1 + logger.trace(f"Ignoring file per .bmignore: {path.relative_to(directory)}") continue - path = Path(root) / filename rel_path = path.relative_to(directory).as_posix() checksum = await self._compute_checksum_async(rel_path) result.files[rel_path] = checksum @@ -683,7 +705,55 @@ async def scan_directory(self, directory: Path) -> ScanResult: f"{directory} scan completed " f"directory={str(directory)} " f"files_found={len(result.files)} " + f"files_ignored={ignored_count} " f"duration_ms={duration_ms}" ) return result + + +async def get_sync_service(project: Project) -> SyncService: # pragma: no cover + """Get sync service instance with all dependencies.""" + + app_config = ConfigManager().config + _, session_maker = await db.get_or_create_db( + db_path=app_config.database_path, db_type=db.DatabaseType.FILESYSTEM + ) + + project_path = Path(project.path) + entity_parser = EntityParser(project_path) + markdown_processor = MarkdownProcessor(entity_parser) + file_service = FileService(project_path, markdown_processor) + + # Initialize repositories + entity_repository = EntityRepository(session_maker, project_id=project.id) + observation_repository = ObservationRepository(session_maker, project_id=project.id) + relation_repository = RelationRepository(session_maker, project_id=project.id) + search_repository = SearchRepository(session_maker, project_id=project.id) + + # Initialize services + search_service = SearchService(search_repository, entity_repository, file_service) + link_resolver = LinkResolver(entity_repository, search_service) + + # Initialize services + entity_service = EntityService( + entity_parser, + entity_repository, + observation_repository, + relation_repository, + file_service, + link_resolver, + ) + + # Create sync service + sync_service = SyncService( + app_config=app_config, + entity_service=entity_service, + entity_parser=entity_parser, + entity_repository=entity_repository, + relation_repository=relation_repository, + search_service=search_service, + file_service=file_service, + ) + + return sync_service diff --git a/src/basic_memory/sync/watch_service.py b/src/basic_memory/sync/watch_service.py index e7d793c5d..890a2ab0c 100644 --- a/src/basic_memory/sync/watch_service.py +++ b/src/basic_memory/sync/watch_service.py @@ -15,6 +15,7 @@ from rich.console import Console from watchfiles import awatch from watchfiles.main import FileChange, Change +import time class WatchEvent(BaseModel): @@ -210,11 +211,8 @@ def is_project_path(self, project: Project, path): async def handle_changes(self, project: Project, changes: Set[FileChange]) -> None: """Process a batch of file changes""" - import time - from typing import List, Set - - # Lazily initialize sync service for project changes - from basic_memory.cli.commands.sync import get_sync_service + # avoid circular imports + from basic_memory.sync.sync_service import get_sync_service sync_service = await get_sync_service(project) file_service = sync_service.file_service diff --git a/src/basic_memory/utils.py b/src/basic_memory/utils.py index 9aa6b76c9..d73964fc7 100644 --- a/src/basic_memory/utils.py +++ b/src/basic_memory/utils.py @@ -331,8 +331,8 @@ def detect_potential_file_conflicts(file_path: str, existing_paths: List[str]) - return conflicts -def validate_project_path(path: str, project_path: Path) -> bool: - """Ensure path stays within project boundaries.""" +def valid_project_path_value(path: str): + """Ensure project path is valid.""" # Allow empty strings as they resolve to the project root if not path: return True @@ -353,6 +353,15 @@ def validate_project_path(path: str, project_path: Path) -> bool: if path.strip() and any(ord(c) < 32 and c not in [" ", "\t"] for c in path): return False + return True + + +def validate_project_path(path: str, project_path: Path) -> bool: + """Ensure path is valid and stays within project boundaries.""" + + if not valid_project_path_value(path): + return False + try: resolved = (project_path / path).resolve() return resolved.is_relative_to(project_path.resolve()) diff --git a/test-int/cli/test_project_commands_integration.py b/test-int/cli/test_project_commands_integration.py new file mode 100644 index 000000000..5f9247367 --- /dev/null +++ b/test-int/cli/test_project_commands_integration.py @@ -0,0 +1,112 @@ +"""Integration tests for project CLI commands.""" + +from typer.testing import CliRunner + +from basic_memory.cli.main import app + + +def test_project_list(app_config, test_project, config_manager): + """Test 'bm project list' command shows projects.""" + runner = CliRunner() + result = runner.invoke(app, ["project", "list"]) + + if result.exit_code != 0: + print(f"STDOUT: {result.stdout}") + print(f"STDERR: {result.stderr}") + print(f"Exception: {result.exception}") + assert result.exit_code == 0 + assert "test-project" in result.stdout + assert "✓" in result.stdout # default marker + + +def test_project_info(app_config, test_project, config_manager): + """Test 'bm project info' command shows project details.""" + runner = CliRunner() + result = runner.invoke(app, ["project", "info", "test-project"]) + + if result.exit_code != 0: + print(f"STDOUT: {result.stdout}") + print(f"STDERR: {result.stderr}") + assert result.exit_code == 0 + assert "Basic Memory Project Info" in result.stdout + assert "test-project" in result.stdout + assert "Statistics" in result.stdout + + +def test_project_info_json(app_config, test_project, config_manager): + """Test 'bm project info --json' command outputs valid JSON.""" + import json + + runner = CliRunner() + result = runner.invoke(app, ["project", "info", "test-project", "--json"]) + + if result.exit_code != 0: + print(f"STDOUT: {result.stdout}") + print(f"STDERR: {result.stderr}") + assert result.exit_code == 0 + + # Parse JSON to verify it's valid + data = json.loads(result.stdout) + assert data["project_name"] == "test-project" + assert "statistics" in data + assert "system" in data + + +def test_project_add_and_remove(app_config, tmp_path, config_manager): + """Test adding and removing a project.""" + runner = CliRunner() + new_project_path = tmp_path / "new-project" + new_project_path.mkdir() + + # Add project + result = runner.invoke(app, ["project", "add", "new-project", str(new_project_path)]) + + if result.exit_code != 0: + print(f"STDOUT: {result.stdout}") + print(f"STDERR: {result.stderr}") + assert result.exit_code == 0 + assert ( + "Project 'new-project' added successfully" in result.stdout + or "added" in result.stdout.lower() + ) + + # Verify it shows up in list + result = runner.invoke(app, ["project", "list"]) + assert result.exit_code == 0 + assert "new-project" in result.stdout + + # Remove project + result = runner.invoke(app, ["project", "remove", "new-project"]) + assert result.exit_code == 0 + assert "removed" in result.stdout.lower() or "deleted" in result.stdout.lower() + + +def test_project_set_default(app_config, tmp_path, config_manager): + """Test setting default project.""" + runner = CliRunner() + new_project_path = tmp_path / "another-project" + new_project_path.mkdir() + + # Add a second project + result = runner.invoke(app, ["project", "add", "another-project", str(new_project_path)]) + if result.exit_code != 0: + print(f"STDOUT: {result.stdout}") + print(f"STDERR: {result.stderr}") + assert result.exit_code == 0 + + # Set as default + result = runner.invoke(app, ["project", "default", "another-project"]) + if result.exit_code != 0: + print(f"STDOUT: {result.stdout}") + print(f"STDERR: {result.stderr}") + assert result.exit_code == 0 + assert "default" in result.stdout.lower() + + # Verify in list + result = runner.invoke(app, ["project", "list"]) + assert result.exit_code == 0 + # The new project should have the checkmark now + lines = result.stdout.split("\n") + for line in lines: + if "another-project" in line: + assert "✓" in line diff --git a/test-int/cli/test_sync_commands_integration.py b/test-int/cli/test_sync_commands_integration.py new file mode 100644 index 000000000..8578abc01 --- /dev/null +++ b/test-int/cli/test_sync_commands_integration.py @@ -0,0 +1,61 @@ +"""Integration tests for sync CLI commands.""" + +from pathlib import Path +from typer.testing import CliRunner + +from basic_memory.cli.main import app + + +def test_sync_command(app_config, test_project, config_manager, config_home): + """Test 'bm sync' command successfully syncs files.""" + runner = CliRunner() + + # Create a test file + test_file = Path(config_home) / "test-note.md" + test_file.write_text("# Test Note\n\nThis is a test.") + + # Run sync + result = runner.invoke(app, ["sync", "--project", "test-project"]) + + if result.exit_code != 0: + print(f"STDOUT: {result.stdout}") + print(f"STDERR: {result.stderr}") + assert result.exit_code == 0 + assert "sync" in result.stdout.lower() or "initiated" in result.stdout.lower() + + +def test_status_command(app_config, test_project, config_manager, config_home): + """Test 'bm status' command shows sync status.""" + runner = CliRunner() + + # Create a test file + test_file = Path(config_home) / "unsynced.md" + test_file.write_text("# Unsynced Note\n\nThis file hasn't been synced yet.") + + # Run status + result = runner.invoke(app, ["status", "--project", "test-project"]) + + if result.exit_code != 0: + print(f"STDOUT: {result.stdout}") + print(f"STDERR: {result.stderr}") + assert result.exit_code == 0 + # Should show some status output + assert len(result.stdout) > 0 + + +def test_status_verbose(app_config, test_project, config_manager, config_home): + """Test 'bm status --verbose' shows detailed status.""" + runner = CliRunner() + + # Create a test file + test_file = Path(config_home) / "test.md" + test_file.write_text("# Test\n\nContent.") + + # Run status with verbose + result = runner.invoke(app, ["status", "--project", "test-project", "--verbose"]) + + if result.exit_code != 0: + print(f"STDOUT: {result.stdout}") + print(f"STDERR: {result.stderr}") + assert result.exit_code == 0 + assert len(result.stdout) > 0 diff --git a/test-int/cli/test_version_integration.py b/test-int/cli/test_version_integration.py new file mode 100644 index 000000000..553c6d6d8 --- /dev/null +++ b/test-int/cli/test_version_integration.py @@ -0,0 +1,15 @@ +"""Integration tests for version command.""" + +from typer.testing import CliRunner + +from basic_memory.cli.main import app +import basic_memory + + +def test_version_command(): + """Test 'bm --version' command shows version.""" + runner = CliRunner() + result = runner.invoke(app, ["--version"]) + + assert result.exit_code == 0 + assert basic_memory.__version__ in result.stdout diff --git a/test-int/conftest.py b/test-int/conftest.py index b96f84d71..37e8af88a 100644 --- a/test-int/conftest.py +++ b/test-int/conftest.py @@ -116,6 +116,9 @@ def config_home(tmp_path, monkeypatch) -> Path: @pytest.fixture(scope="function", autouse=True) def app_config(config_home, tmp_path, monkeypatch) -> BasicMemoryConfig: """Create test app configuration.""" + # Disable cloud mode for CLI tests + monkeypatch.setenv("BASIC_MEMORY_CLOUD_MODE", "false") + # Create a basic config with test-project like unit tests do projects = {"test-project": str(config_home)} app_config = BasicMemoryConfig( @@ -124,6 +127,7 @@ def app_config(config_home, tmp_path, monkeypatch) -> BasicMemoryConfig: default_project="test-project", default_project_mode=True, update_permalinks_on_move=True, + cloud_mode=False, # Explicitly disable cloud mode ) return app_config diff --git a/tests/api/test_async_client.py b/tests/api/test_async_client.py index 1b53461c4..c39f6a55e 100644 --- a/tests/api/test_async_client.py +++ b/tests/api/test_async_client.py @@ -1,14 +1,20 @@ """Tests for async_client configuration.""" +import os from unittest.mock import patch from httpx import AsyncClient, ASGITransport, Timeout +from basic_memory.config import ConfigManager from basic_memory.mcp.async_client import create_client def test_create_client_uses_asgi_when_no_remote_env(): """Test that create_client uses ASGI transport when BASIC_MEMORY_USE_REMOTE_API is not set.""" - with patch.dict("os.environ", {}, clear=True): + # Ensure env vars are not set (pop if they exist) + with patch.dict("os.environ", clear=False): + os.environ.pop("BASIC_MEMORY_USE_REMOTE_API", None) + os.environ.pop("BASIC_MEMORY_CLOUD_MODE", None) + client = create_client() assert isinstance(client, AsyncClient) @@ -16,20 +22,26 @@ def test_create_client_uses_asgi_when_no_remote_env(): assert str(client.base_url) == "http://test" -def test_create_client_uses_http_when_proxy_env_set(): - """Test that create_client uses HTTP transport when BASIC_MEMORY_USE_REMOTE_API is set.""" - with patch.dict("os.environ", {"BASIC_MEMORY_PROXY_URL": "http://localhost:8000"}): +def test_create_client_uses_http_when_cloud_mode_env_set(): + """Test that create_client uses HTTP transport when BASIC_MEMORY_CLOUD_MODE is set.""" + + config = ConfigManager().config + with patch.dict("os.environ", {"BASIC_MEMORY_CLOUD_MODE": "True"}): client = create_client() assert isinstance(client, AsyncClient) assert not isinstance(client._transport, ASGITransport) - # When using remote API, no base_url is set (dynamic from headers) - assert str(client.base_url) == "http://localhost:8000" + # Cloud mode uses cloud_host/proxy as base_url + assert str(client.base_url) == f"{config.cloud_host}/proxy/" def test_create_client_configures_extended_timeouts(): """Test that create_client configures 30-second timeouts for long operations.""" - with patch.dict("os.environ", {}, clear=True): + # Ensure env vars are not set (pop if they exist) + with patch.dict("os.environ", clear=False): + os.environ.pop("BASIC_MEMORY_USE_REMOTE_API", None) + os.environ.pop("BASIC_MEMORY_CLOUD_MODE", None) + client = create_client() # Verify timeout configuration @@ -38,12 +50,3 @@ def test_create_client_configures_extended_timeouts(): assert client.timeout.read == 30.0 # 30 seconds for reading assert client.timeout.write == 30.0 # 30 seconds for writing assert client.timeout.pool == 30.0 # 30 seconds for pool - - # Also test with proxy URL - with patch.dict("os.environ", {"BASIC_MEMORY_PROXY_URL": "http://localhost:8000"}): - client = create_client() - - # Same timeout configuration should apply - assert isinstance(client.timeout, Timeout) - assert client.timeout.read == 30.0 - assert client.timeout.write == 30.0 diff --git a/tests/api/test_project_router.py b/tests/api/test_project_router.py index f451bded3..2f742ebfa 100644 --- a/tests/api/test_project_router.py +++ b/tests/api/test_project_router.py @@ -442,3 +442,30 @@ async def test_update_project_empty_path_endpoint( await project_service.remove_project(test_project_name) except Exception: pass + + +@pytest.mark.asyncio +async def test_sync_project_endpoint(test_graph, client, project_url): + """Test the project sync endpoint initiates background sync.""" + # Call the sync endpoint + response = await client.post(f"{project_url}/project/sync") + + # Verify response + assert response.status_code == 200 + data = response.json() + + # Check response structure + assert "status" in data + assert "message" in data + assert data["status"] == "sync_started" + assert "Filesystem sync initiated" in data["message"] + + +@pytest.mark.asyncio +async def test_sync_project_endpoint_not_found(client): + """Test the project sync endpoint with nonexistent project.""" + # Call the sync endpoint for a project that doesn't exist + response = await client.post("/nonexistent-project/project/sync") + + # Should return 404 + assert response.status_code == 404 diff --git a/tests/api/test_webdav_router.py b/tests/api/test_webdav_router.py deleted file mode 100644 index c669b082c..000000000 --- a/tests/api/test_webdav_router.py +++ /dev/null @@ -1,708 +0,0 @@ -"""Tests for WebDAV router endpoints.""" - -import json -import time -import pytest - - -@pytest.mark.asyncio -async def test_webdav_options_file(client, project_config, project_url): - """Test WebDAV OPTIONS for a file.""" - # Create a test file - content = "# Test Content\n\nThis is a test file." - test_file = project_config.home / "test.md" - test_file.write_text(content) - - # Test OPTIONS request - response = await client.request("OPTIONS", f"{project_url}/webdav/test.md") - assert response.status_code == 204 - assert "DAV" in response.headers - assert response.headers["DAV"] == "1,2" - assert "Allow" in response.headers - assert "PUT" in response.headers["Allow"] - assert "GET" in response.headers["Allow"] - assert "DELETE" in response.headers["Allow"] - - -@pytest.mark.asyncio -async def test_webdav_options_directory(client, project_config, project_url): - """Test WebDAV OPTIONS for a directory.""" - # Create a test directory - test_dir = project_config.home / "test_dir" - test_dir.mkdir(exist_ok=True) - - # Test OPTIONS request - response = await client.request("OPTIONS", f"{project_url}/webdav/test_dir") - assert response.status_code == 204 - assert response.headers["DAV"] == "1,2" - - -@pytest.mark.asyncio -async def test_webdav_propfind_root(client, project_config, project_url): - """Test WebDAV PROPFIND for project root directory.""" - # Create some test files and directories - (project_config.home / "test.md").write_text("# Test") - (project_config.home / "subdir").mkdir(exist_ok=True) - (project_config.home / "subdir" / "nested.md").write_text("# Nested") - - # Test PROPFIND request for root - response = await client.request("PROPFIND", f"{project_url}/webdav/") - assert response.status_code == 207 - assert response.headers["content-type"] == "text/xml; charset=utf-8" - - # Check XML response contains directory listing - xml_content = response.text - assert "" in xml_content # For directory - - -@pytest.mark.asyncio -async def test_webdav_propfind_subdirectory(client, project_config, project_url): - """Test WebDAV PROPFIND for a subdirectory.""" - # Create test structure - subdir = project_config.home / "docs" - subdir.mkdir(exist_ok=True) - (subdir / "readme.md").write_text("# README") - (subdir / "guide.md").write_text("# Guide") - - # Test PROPFIND request for subdirectory - response = await client.request("PROPFIND", f"{project_url}/webdav/docs") - assert response.status_code == 207 - - xml_content = response.text - assert "readme.md" in xml_content - assert "guide.md" in xml_content - assert "" in xml_content # File size information - - -@pytest.mark.asyncio -async def test_webdav_propfind_file(client, project_config, project_url): - """Test WebDAV PROPFIND for a single file.""" - # Create a test file - test_file = project_config.home / "single.md" - test_file.write_text("# Single File") - - # Test PROPFIND request for the file - response = await client.request("PROPFIND", f"{project_url}/webdav/single.md") - assert response.status_code == 207 - - xml_content = response.text - assert "single.md" in xml_content - assert "" in xml_content - assert "" in xml_content # Empty for files - - -@pytest.mark.asyncio -async def test_webdav_get_file(client, project_config, project_url): - """Test WebDAV GET for downloading a file.""" - # Create a test file - content = "# Test Content\n\nThis is a test file for download." - test_file = project_config.home / "download.md" - test_file.write_text(content) - - # Test GET request - response = await client.get(f"{project_url}/webdav/download.md") - assert response.status_code == 200 - assert response.headers["content-type"] == "application/octet-stream" - assert "Content-Length" in response.headers - # Normalize line endings for cross-platform compatibility - assert content.replace("\n", "\r\n") in response.text or content in response.text - - -@pytest.mark.asyncio -async def test_webdav_get_binary_file(client, project_config, project_url): - """Test WebDAV GET for downloading a binary file.""" - # Create a test binary file - binary_content = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01" - test_file = project_config.home / "test.png" - test_file.write_bytes(binary_content) - - # Test GET request - response = await client.get(f"{project_url}/webdav/test.png") - assert response.status_code == 200 - assert response.headers["content-type"] == "application/octet-stream" - assert response.content == binary_content - - -@pytest.mark.asyncio -async def test_webdav_put_new_file(client, project_config, project_url): - """Test WebDAV PUT for creating a new file.""" - # Test data - content = "# New File\n\nThis file was uploaded via WebDAV PUT." - file_path = "uploads/new.md" - - # Ensure the file doesn't exist - full_path = project_config.home / file_path - if full_path.exists(): - full_path.unlink() - - # Test PUT request - response = await client.put( - f"{project_url}/webdav/{file_path}", - content=content, - headers={"Content-Type": "text/markdown"}, - ) - assert response.status_code == 201 # Created - - # Verify file was created - assert full_path.exists() - assert full_path.read_text() == content - - -@pytest.mark.asyncio -async def test_webdav_put_update_existing_file(client, project_config, project_url): - """Test WebDAV PUT for updating an existing file.""" - # Create initial file - file_path = "updates/existing.md" - full_path = project_config.home / file_path - full_path.parent.mkdir(parents=True, exist_ok=True) - initial_content = "# Original Content" - full_path.write_text(initial_content) - - # Update with new content - updated_content = "# Updated Content\n\nThis content was updated via PUT." - response = await client.put( - f"{project_url}/webdav/{file_path}", - content=updated_content, - headers={"Content-Type": "text/markdown"}, - ) - assert response.status_code == 204 # No Content (updated) - - # Verify file was updated - assert full_path.read_text() == updated_content - - -@pytest.mark.asyncio -async def test_webdav_put_binary_file(client, project_config, project_url): - """Test WebDAV PUT for uploading a binary file.""" - # Test binary data - binary_content = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01" - file_path = "images/uploaded.png" - - # Test PUT request with binary data - response = await client.put( - f"{project_url}/webdav/{file_path}", - content=binary_content, - headers={"Content-Type": "image/png"}, - ) - assert response.status_code == 201 - - # Verify binary file was created correctly - full_path = project_config.home / file_path - assert full_path.exists() - assert full_path.read_bytes() == binary_content - - -@pytest.mark.asyncio -async def test_webdav_put_nested_directory_creation(client, project_config, project_url): - """Test WebDAV PUT creates parent directories automatically.""" - # Test creating a file in a nested path that doesn't exist - content = "# Deep nested file" - file_path = "very/deep/nested/path/file.md" - - # Test PUT request - response = await client.put(f"{project_url}/webdav/{file_path}", content=content) - assert response.status_code == 201 - - # Verify directory structure was created - full_path = project_config.home / file_path - assert full_path.exists() - assert full_path.read_text() == content - assert full_path.parent.exists() - - -@pytest.mark.asyncio -async def test_webdav_delete_file(client, project_config, project_url): - """Test WebDAV DELETE for removing a file.""" - # Create a test file - file_path = "delete_me.md" - full_path = project_config.home / file_path - full_path.write_text("# File to delete") - - # Verify file exists - assert full_path.exists() - - # Test DELETE request - response = await client.delete(f"{project_url}/webdav/{file_path}") - assert response.status_code == 204 - - # Verify file was deleted - assert not full_path.exists() - - -@pytest.mark.asyncio -async def test_webdav_delete_directory(client, project_config, project_url): - """Test WebDAV DELETE for removing a directory.""" - # Create a test directory with content - dir_path = "delete_dir" - full_path = project_config.home / dir_path - full_path.mkdir(exist_ok=True) - (full_path / "file1.md").write_text("# File 1") - (full_path / "file2.md").write_text("# File 2") - - # Verify directory exists - assert full_path.exists() - assert full_path.is_dir() - - # Test DELETE request - response = await client.delete(f"{project_url}/webdav/{dir_path}") - assert response.status_code == 204 - - # Verify directory was deleted - assert not full_path.exists() - - -@pytest.mark.asyncio -async def test_webdav_mkcol_create_directory(client, project_config, project_url): - """Test WebDAV MKCOL for creating a directory.""" - # Test directory path - dir_path = "new_collection" - full_path = project_config.home / dir_path - - # Ensure directory doesn't exist - if full_path.exists(): - full_path.rmdir() - - # Test MKCOL request - response = await client.request("MKCOL", f"{project_url}/webdav/{dir_path}") - assert response.status_code == 201 - - # Verify directory was created - assert full_path.exists() - assert full_path.is_dir() - - -@pytest.mark.asyncio -async def test_webdav_mkcol_create_nested_directory(client, project_config, project_url): - """Test WebDAV MKCOL for creating nested directories.""" - # Test nested directory path - dir_path = "nested/directory/structure" - full_path = project_config.home / dir_path - - # Test MKCOL request - response = await client.request("MKCOL", f"{project_url}/webdav/{dir_path}") - assert response.status_code == 201 - - # Verify nested directory structure was created - assert full_path.exists() - assert full_path.is_dir() - - -@pytest.mark.asyncio -async def test_webdav_mkcol_existing_directory(client, project_config, project_url): - """Test WebDAV MKCOL for directory that already exists.""" - # Create directory first - dir_path = "existing_dir" - full_path = project_config.home / dir_path - full_path.mkdir(exist_ok=True) - - # Test MKCOL request on existing directory - response = await client.request("MKCOL", f"{project_url}/webdav/{dir_path}") - assert response.status_code == 405 # Method Not Allowed - - -# Error cases - - -@pytest.mark.asyncio -async def test_webdav_get_nonexistent_file(client, project_url): - """Test WebDAV GET for file that doesn't exist.""" - response = await client.get(f"{project_url}/webdav/nonexistent.md") - assert response.status_code == 404 - assert "does not exist" in response.json()["detail"] - - -@pytest.mark.asyncio -async def test_webdav_propfind_nonexistent_path(client, project_url): - """Test WebDAV PROPFIND for path that doesn't exist.""" - response = await client.request("PROPFIND", f"{project_url}/webdav/nonexistent") - assert response.status_code == 404 - - -@pytest.mark.asyncio -async def test_webdav_delete_nonexistent_file(client, project_url): - """Test WebDAV DELETE for file that doesn't exist.""" - response = await client.delete(f"{project_url}/webdav/nonexistent.md") - assert response.status_code == 404 - - -@pytest.mark.asyncio -async def test_webdav_options_nonexistent_file(client, project_url): - """Test WebDAV OPTIONS for file that doesn't exist.""" - response = await client.request("OPTIONS", f"{project_url}/webdav/nonexistent.md") - assert response.status_code == 404 - - -@pytest.mark.asyncio -async def test_webdav_nonexistent_project(client): - """Test WebDAV endpoints with nonexistent project.""" - response = await client.get("/nonexistent-project/webdav/test.md") - assert response.status_code == 404 - assert "Project" in response.json()["detail"] - - -# Integration tests - - -@pytest.mark.asyncio -async def test_webdav_full_upload_workflow(client, project_config, project_url): - """Test complete workflow: create directory, upload file, verify, download.""" - # Step 1: Create directory - response = await client.request("MKCOL", f"{project_url}/webdav/workflow") - assert response.status_code == 201 - - # Step 2: Upload file - content = "# Workflow Test\n\nComplete WebDAV workflow test." - response = await client.put(f"{project_url}/webdav/workflow/test.md", content=content) - assert response.status_code == 201 - - # Step 3: Verify with PROPFIND - response = await client.request("PROPFIND", f"{project_url}/webdav/workflow") - assert response.status_code == 207 - assert "test.md" in response.text - - # Step 4: Download and verify content - response = await client.get(f"{project_url}/webdav/workflow/test.md") - assert response.status_code == 200 - assert response.text == content - - # Step 5: Update file - updated_content = content + "\n\nUpdated content." - response = await client.put(f"{project_url}/webdav/workflow/test.md", content=updated_content) - assert response.status_code == 204 - - # Step 6: Verify update - response = await client.get(f"{project_url}/webdav/workflow/test.md") - assert response.status_code == 200 - assert response.text == updated_content - - # Step 7: Clean up - response = await client.delete(f"{project_url}/webdav/workflow/test.md") - assert response.status_code == 204 - - response = await client.delete(f"{project_url}/webdav/workflow") - assert response.status_code == 204 - - -@pytest.mark.asyncio -async def test_webdav_mixed_content_types(client, project_config, project_url): - """Test WebDAV with various file types.""" - # Test files of different types - test_files = { - "document.md": "# Markdown Document", - "data.json": json.dumps({"key": "value"}, indent=2), - "config.txt": "key=value\nother=setting", - "binary.dat": b"\x00\x01\x02\x03\xff", - } - - # Upload all files - for filename, content in test_files.items(): - if isinstance(content, bytes): - response = await client.put(f"{project_url}/webdav/{filename}", content=content) - else: - response = await client.put(f"{project_url}/webdav/{filename}", content=content) - assert response.status_code == 201 - - # Verify all files with PROPFIND - response = await client.request("PROPFIND", f"{project_url}/webdav/") - assert response.status_code == 207 - for filename in test_files.keys(): - assert filename in response.text - - # Download and verify each file - for filename, original_content in test_files.items(): - response = await client.get(f"{project_url}/webdav/{filename}") - assert response.status_code == 200 - - if isinstance(original_content, bytes): - assert response.content == original_content - else: - assert response.text == original_content - - -# Timestamp preservation tests - - -@pytest.mark.asyncio -async def test_webdav_put_with_xoc_mtime_header(client, project_config, project_url): - """Test WebDAV PUT preserves timestamps from X-OC-Mtime header.""" - # Test data - content = "# Test File\n\nTesting timestamp preservation." - file_path = "timestamps/xoc_mtime.md" - - # Set a specific timestamp (Jan 1, 2020 12:00:00 UTC) - target_timestamp = 1577880000.0 - - # Test PUT request with X-OC-Mtime header - response = await client.put( - f"{project_url}/webdav/{file_path}", - content=content, - headers={"X-OC-Mtime": str(target_timestamp)}, - ) - assert response.status_code == 201 - - # Verify file was created and timestamp was preserved - full_path = project_config.home / file_path - assert full_path.exists() - - stat = full_path.stat() - # Allow small tolerance for timestamp precision - assert abs(stat.st_mtime - target_timestamp) < 1.0 - - -@pytest.mark.asyncio -async def test_webdav_put_with_x_timestamp_header(client, project_config, project_url): - """Test WebDAV PUT preserves timestamps from X-Timestamp header.""" - content = "# Test File\n\nTesting X-Timestamp header." - file_path = "timestamps/x_timestamp.md" - - # Set a specific timestamp (Dec 25, 2021 08:30:45 UTC) - target_timestamp = 1640422245.0 - - response = await client.put( - f"{project_url}/webdav/{file_path}", - content=content, - headers={"X-Timestamp": str(target_timestamp)}, - ) - assert response.status_code == 201 - - full_path = project_config.home / file_path - stat = full_path.stat() - assert abs(stat.st_mtime - target_timestamp) < 1.0 - - -@pytest.mark.asyncio -async def test_webdav_put_with_x_mtime_header(client, project_config, project_url): - """Test WebDAV PUT preserves timestamps from X-Mtime header.""" - content = "# Test File\n\nTesting X-Mtime header." - file_path = "timestamps/x_mtime.md" - - # Set a specific timestamp (July 4, 2022 15:45:30 UTC) - target_timestamp = 1656946530.0 - - response = await client.put( - f"{project_url}/webdav/{file_path}", - content=content, - headers={"X-Mtime": str(target_timestamp)}, - ) - assert response.status_code == 201 - - full_path = project_config.home / file_path - stat = full_path.stat() - assert abs(stat.st_mtime - target_timestamp) < 1.0 - - -@pytest.mark.asyncio -async def test_webdav_put_with_last_modified_header(client, project_config, project_url): - """Test WebDAV PUT preserves timestamps from Last-Modified header.""" - content = "# Test File\n\nTesting Last-Modified header." - file_path = "timestamps/last_modified.md" - - # HTTP date format timestamp (March 15, 2023 10:20:30 GMT) - last_modified_str = "Wed, 15 Mar 2023 10:20:30 GMT" - # Expected timestamp (correct calculation) - target_timestamp = 1678875630.0 - - response = await client.put( - f"{project_url}/webdav/{file_path}", - content=content, - headers={"Last-Modified": last_modified_str}, - ) - assert response.status_code == 201 - - full_path = project_config.home / file_path - stat = full_path.stat() - assert abs(stat.st_mtime - target_timestamp) < 1.0 - - -@pytest.mark.asyncio -async def test_webdav_put_without_timestamp_headers(client, project_config, project_url): - """Test WebDAV PUT uses current time when no timestamp headers provided.""" - content = "# Test File\n\nNo timestamp headers." - file_path = "timestamps/no_headers.md" - - # Record current time before upload - before_upload = time.time() - - response = await client.put(f"{project_url}/webdav/{file_path}", content=content) - assert response.status_code == 201 - - # Record current time after upload - after_upload = time.time() - - full_path = project_config.home / file_path - stat = full_path.stat() - - # File timestamp should be between before and after upload times - # Allow small tolerance for file system timestamp precision differences - assert before_upload - 0.1 <= stat.st_mtime <= after_upload + 0.1 - - -@pytest.mark.asyncio -async def test_webdav_put_header_priority(client, project_config, project_url): - """Test header priority: X-OC-Mtime takes precedence over others.""" - content = "# Test File\n\nTesting header priority." - file_path = "timestamps/priority_test.md" - - # Set multiple timestamp headers - xoc_timestamp = 1577880000.0 # Jan 1, 2020 - x_timestamp = 1640422245.0 # Dec 25, 2021 - last_modified = "Wed, 15 Mar 2023 10:20:30 GMT" # March 15, 2023 - - response = await client.put( - f"{project_url}/webdav/{file_path}", - content=content, - headers={ - "X-OC-Mtime": str(xoc_timestamp), - "X-Timestamp": str(x_timestamp), - "Last-Modified": last_modified, - }, - ) - assert response.status_code == 201 - - full_path = project_config.home / file_path - stat = full_path.stat() - # Should use X-OC-Mtime (highest priority) - assert abs(stat.st_mtime - xoc_timestamp) < 1.0 - - -@pytest.mark.asyncio -async def test_webdav_put_invalid_timestamp_headers(client, project_config, project_url): - """Test WebDAV PUT handles invalid timestamp headers gracefully.""" - content = "# Test File\n\nTesting invalid timestamps." - file_path = "timestamps/invalid_headers.md" - - before_upload = time.time() - - response = await client.put( - f"{project_url}/webdav/{file_path}", - content=content, - headers={ - "X-OC-Mtime": "not_a_number", - "X-Timestamp": "invalid", - "Last-Modified": "Not a valid date format", - }, - ) - assert response.status_code == 201 - - after_upload = time.time() - - full_path = project_config.home / file_path - stat = full_path.stat() - - # Should fall back to current time when all headers are invalid - # Allow small tolerance for file system timestamp precision differences - assert before_upload - 0.1 <= stat.st_mtime <= after_upload + 0.1 - - -@pytest.mark.asyncio -async def test_webdav_put_update_preserves_timestamp(client, project_config, project_url): - """Test WebDAV PUT preserves timestamp when updating existing file.""" - content1 = "# Original Content" - content2 = "# Updated Content" - file_path = "timestamps/update_test.md" - - # Create file with specific timestamp - original_timestamp = 1577880000.0 # Jan 1, 2020 - - response = await client.put( - f"{project_url}/webdav/{file_path}", - content=content1, - headers={"X-OC-Mtime": str(original_timestamp)}, - ) - assert response.status_code == 201 - - # Update file with different timestamp - updated_timestamp = 1640422245.0 # Dec 25, 2021 - - response = await client.put( - f"{project_url}/webdav/{file_path}", - content=content2, - headers={"X-OC-Mtime": str(updated_timestamp)}, - ) - assert response.status_code == 204 # Updated existing file - - full_path = project_config.home / file_path - stat = full_path.stat() - - # Should have the updated timestamp - assert abs(stat.st_mtime - updated_timestamp) < 1.0 - # Content should be updated - assert full_path.read_text() == content2 - - -@pytest.mark.asyncio -async def test_webdav_put_binary_file_with_timestamp(client, project_config, project_url): - """Test WebDAV PUT preserves timestamps for binary files.""" - binary_content = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01" - file_path = "timestamps/binary_with_timestamp.png" - - target_timestamp = 1656946530.0 # July 4, 2022 - - response = await client.put( - f"{project_url}/webdav/{file_path}", - content=binary_content, - headers={"X-OC-Mtime": str(target_timestamp), "Content-Type": "image/png"}, - ) - assert response.status_code == 201 - - full_path = project_config.home / file_path - stat = full_path.stat() - assert abs(stat.st_mtime - target_timestamp) < 1.0 - assert full_path.read_bytes() == binary_content - - -@pytest.mark.asyncio -async def test_webdav_timestamp_integration_workflow(client, project_config, project_url): - """Test complete workflow with timestamp preservation.""" - # Create multiple files with different timestamps to simulate project upload - files_data = [ - { - "path": "notes/readme.md", - "content": "# Project README", - "timestamp": 1577880000.0, # Jan 1, 2020 - }, - { - "path": "notes/changelog.md", - "content": "# Changelog\n\n## v1.0.0", - "timestamp": 1640422245.0, # Dec 25, 2021 - }, - { - "path": "docs/guide.md", - "content": "# User Guide", - "timestamp": 1656946530.0, # July 4, 2022 - }, - ] - - # Upload all files with their original timestamps - for file_data in files_data: - response = await client.put( - f"{project_url}/webdav/{file_data['path']}", - content=file_data["content"], - headers={"X-OC-Mtime": str(file_data["timestamp"])}, - ) - assert response.status_code == 201 - - # Verify all files have correct timestamps and content - for file_data in files_data: - full_path = project_config.home / file_data["path"] - assert full_path.exists() - assert full_path.read_text() == file_data["content"] - - stat = full_path.stat() - assert abs(stat.st_mtime - file_data["timestamp"]) < 1.0 - - # Verify directory structure - response = await client.request("PROPFIND", f"{project_url}/webdav/") - assert response.status_code == 207 - assert "notes" in response.text - assert "docs" in response.text - - # Verify file listing in subdirectories - response = await client.request("PROPFIND", f"{project_url}/webdav/notes") - assert response.status_code == 207 - assert "readme.md" in response.text - assert "changelog.md" in response.text diff --git a/tests/cli/test_bisync_commands.py b/tests/cli/test_bisync_commands.py new file mode 100644 index 000000000..30c5354ae --- /dev/null +++ b/tests/cli/test_bisync_commands.py @@ -0,0 +1,465 @@ +"""Tests for bisync_commands module.""" + +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest + +from basic_memory.cli.commands.cloud.bisync_commands import ( + BisyncError, + convert_bmignore_to_rclone_filters, + scan_local_directories, + validate_bisync_directory, + build_bisync_command, + get_bisync_directory, + get_bisync_state_path, + bisync_state_exists, + BISYNC_PROFILES, +) + + +class TestConvertBmignoreToRcloneFilters: + """Tests for convert_bmignore_to_rclone_filters().""" + + def test_converts_basic_patterns(self, tmp_path): + """Test conversion of basic gitignore patterns to rclone format.""" + bmignore_dir = tmp_path / ".basic-memory" + bmignore_dir.mkdir(exist_ok=True) + bmignore_file = bmignore_dir / ".bmignore" + + # Write test patterns + bmignore_file.write_text("# Comment line\nnode_modules\n*.pyc\n.git\n**/*.log\n") + + with patch( + "basic_memory.cli.commands.cloud.bisync_commands.get_bmignore_path", + return_value=bmignore_file, + ): + convert_bmignore_to_rclone_filters() + + # Read the generated rclone filter file + rclone_filter = bmignore_dir / ".bmignore.rclone" + assert rclone_filter.exists() + + content = rclone_filter.read_text() + lines = content.strip().split("\n") + + # Check comment preserved + assert "# Comment line" in lines + + # Check patterns converted correctly + assert "- node_modules/**" in lines # Directory without wildcard + assert "- *.pyc" in lines # Wildcard pattern unchanged + assert "- .git/**" in lines # Directory pattern + assert "- **/*.log" in lines # Wildcard pattern unchanged + + def test_handles_empty_bmignore(self, tmp_path): + """Test handling of empty .bmignore file.""" + bmignore_dir = tmp_path / ".basic-memory" + bmignore_dir.mkdir(exist_ok=True) + bmignore_file = bmignore_dir / ".bmignore" + bmignore_file.write_text("") + + with patch( + "basic_memory.cli.commands.cloud.bisync_commands.get_bmignore_path", + return_value=bmignore_file, + ): + convert_bmignore_to_rclone_filters() + + rclone_filter = bmignore_dir / ".bmignore.rclone" + assert rclone_filter.exists() + + def test_handles_missing_bmignore(self, tmp_path): + """Test handling when .bmignore doesn't exist.""" + bmignore_dir = tmp_path / ".basic-memory" + bmignore_dir.mkdir(exist_ok=True) + bmignore_file = bmignore_dir / ".bmignore" + + # Ensure file doesn't exist + if bmignore_file.exists(): + bmignore_file.unlink() + + with patch( + "basic_memory.cli.commands.cloud.bisync_commands.get_bmignore_path", + return_value=bmignore_file, + ): + with patch("basic_memory.cli.commands.cloud.bisync_commands.create_default_bmignore"): + convert_bmignore_to_rclone_filters() + + # Should create minimal filter with .git + rclone_filter = bmignore_dir / ".bmignore.rclone" + assert rclone_filter.exists() + content = rclone_filter.read_text() + assert "- .git/**" in content + + +class TestScanLocalDirectories: + """Tests for scan_local_directories().""" + + def test_scans_existing_directories(self, tmp_path): + """Test scanning existing project directories.""" + # Use a subdirectory to avoid interference from test fixtures + scan_dir = tmp_path / "scan_test" + scan_dir.mkdir() + + # Create test directories + (scan_dir / "project1").mkdir() + (scan_dir / "project2").mkdir() + (scan_dir / "project3").mkdir() + + # Create a hidden directory (should be ignored) + (scan_dir / ".hidden").mkdir() + + # Create a file (should be ignored) + (scan_dir / "file.txt").write_text("test") + + result = scan_local_directories(scan_dir) + + assert len(result) == 3 + assert "project1" in result + assert "project2" in result + assert "project3" in result + assert ".hidden" not in result + + def test_handles_empty_directory(self, tmp_path): + """Test scanning empty directory.""" + scan_dir = tmp_path / "empty_test" + scan_dir.mkdir() + result = scan_local_directories(scan_dir) + assert result == [] + + def test_handles_nonexistent_directory(self, tmp_path): + """Test scanning nonexistent directory.""" + nonexistent = tmp_path / "does-not-exist" + result = scan_local_directories(nonexistent) + assert result == [] + + def test_ignores_hidden_directories(self, tmp_path): + """Test that hidden directories are ignored.""" + scan_dir = tmp_path / "hidden_test" + scan_dir.mkdir() + + (scan_dir / ".git").mkdir() + (scan_dir / ".cache").mkdir() + (scan_dir / "visible").mkdir() + + result = scan_local_directories(scan_dir) + + assert len(result) == 1 + assert "visible" in result + assert ".git" not in result + assert ".cache" not in result + + +class TestValidateBisyncDirectory: + """Tests for validate_bisync_directory().""" + + def test_allows_valid_directory(self, tmp_path): + """Test that valid directory passes validation.""" + bisync_dir = tmp_path / "sync" + bisync_dir.mkdir() + + # Should not raise + validate_bisync_directory(bisync_dir) + + def test_rejects_mount_directory(self, tmp_path): + """Test that mount directory is rejected.""" + mount_dir = Path.home() / "basic-memory-cloud" + + with pytest.raises(BisyncError) as exc_info: + validate_bisync_directory(mount_dir) + + assert "mount directory" in str(exc_info.value).lower() + + @patch("subprocess.run") + def test_rejects_mounted_directory(self, mock_run, tmp_path): + """Test that currently mounted directory is rejected.""" + bisync_dir = tmp_path / "sync" + bisync_dir.mkdir() + + # Mock mount command showing this directory is mounted + mock_run.return_value = Mock( + stdout=f"rclone on {bisync_dir} type fuse.rclone", + stderr="", + returncode=0, + ) + + with pytest.raises(BisyncError) as exc_info: + validate_bisync_directory(bisync_dir) + + assert "currently mounted" in str(exc_info.value).lower() + + +class TestBuildBisyncCommand: + """Tests for build_bisync_command().""" + + def test_builds_basic_command(self, tmp_path): + """Test building basic bisync command.""" + tenant_id = "test-tenant" + bucket_name = "test-bucket" + local_path = tmp_path / "sync" + local_path.mkdir() + profile = BISYNC_PROFILES["balanced"] + + with patch( + "basic_memory.cli.commands.cloud.bisync_commands.get_bisync_filter_path" + ) as mock_filter: + mock_filter.return_value = Path("/test/filter") + + cmd = build_bisync_command( + tenant_id=tenant_id, + bucket_name=bucket_name, + local_path=local_path, + profile=profile, + ) + + assert cmd[0] == "rclone" + assert cmd[1] == "bisync" + assert str(local_path) in cmd + assert f"basic-memory-{tenant_id}:{bucket_name}" in cmd + assert "--create-empty-src-dirs" in cmd + assert "--resilient" in cmd + assert f"--conflict-resolve={profile.conflict_resolve}" in cmd + assert f"--max-delete={profile.max_delete}" in cmd + assert "--progress" in cmd + + def test_adds_dry_run_flag(self, tmp_path): + """Test that dry-run flag is added when requested.""" + tenant_id = "test-tenant" + bucket_name = "test-bucket" + local_path = tmp_path / "sync" + local_path.mkdir() + profile = BISYNC_PROFILES["safe"] + + with patch( + "basic_memory.cli.commands.cloud.bisync_commands.get_bisync_filter_path" + ) as mock_filter: + mock_filter.return_value = Path("/test/filter") + + cmd = build_bisync_command( + tenant_id=tenant_id, + bucket_name=bucket_name, + local_path=local_path, + profile=profile, + dry_run=True, + ) + + assert "--dry-run" in cmd + + def test_adds_resync_flag(self, tmp_path): + """Test that resync flag is added when requested.""" + tenant_id = "test-tenant" + bucket_name = "test-bucket" + local_path = tmp_path / "sync" + local_path.mkdir() + profile = BISYNC_PROFILES["balanced"] + + with patch( + "basic_memory.cli.commands.cloud.bisync_commands.get_bisync_filter_path" + ) as mock_filter: + mock_filter.return_value = Path("/test/filter") + + cmd = build_bisync_command( + tenant_id=tenant_id, + bucket_name=bucket_name, + local_path=local_path, + profile=profile, + resync=True, + ) + + assert "--resync" in cmd + + def test_adds_verbose_flag(self, tmp_path): + """Test that verbose flag is added when requested.""" + tenant_id = "test-tenant" + bucket_name = "test-bucket" + local_path = tmp_path / "sync" + local_path.mkdir() + profile = BISYNC_PROFILES["fast"] + + with patch( + "basic_memory.cli.commands.cloud.bisync_commands.get_bisync_filter_path" + ) as mock_filter: + mock_filter.return_value = Path("/test/filter") + + cmd = build_bisync_command( + tenant_id=tenant_id, + bucket_name=bucket_name, + local_path=local_path, + profile=profile, + verbose=True, + ) + + assert "--verbose" in cmd + assert "--progress" not in cmd # Progress replaced by verbose + + def test_creates_state_directory(self, tmp_path): + """Test that state directory is created.""" + tenant_id = "test-tenant" + bucket_name = "test-bucket" + local_path = tmp_path / "sync" + local_path.mkdir() + profile = BISYNC_PROFILES["balanced"] + + with patch( + "basic_memory.cli.commands.cloud.bisync_commands.get_bisync_filter_path" + ) as mock_filter: + with patch( + "basic_memory.cli.commands.cloud.bisync_commands.get_bisync_state_path" + ) as mock_state: + state_path = tmp_path / "state" + mock_filter.return_value = Path("/test/filter") + mock_state.return_value = state_path + + build_bisync_command( + tenant_id=tenant_id, + bucket_name=bucket_name, + local_path=local_path, + profile=profile, + ) + + # State directory should be created + assert state_path.exists() + assert state_path.is_dir() + + +class TestBisyncStateManagement: + """Tests for bisync state functions.""" + + def test_get_bisync_state_path(self): + """Test state path generation.""" + tenant_id = "test-tenant-123" + result = get_bisync_state_path(tenant_id) + + expected = Path.home() / ".basic-memory" / "bisync-state" / tenant_id + assert result == expected + + def test_bisync_state_exists_true(self, tmp_path): + """Test checking for existing state.""" + state_dir = tmp_path / "state" + state_dir.mkdir() + (state_dir / "test.lst").write_text("test") + + with patch( + "basic_memory.cli.commands.cloud.bisync_commands.get_bisync_state_path", + return_value=state_dir, + ): + result = bisync_state_exists("test-tenant") + + assert result is True + + def test_bisync_state_exists_false_no_dir(self, tmp_path): + """Test checking for nonexistent state directory.""" + state_dir = tmp_path / "nonexistent" + + with patch( + "basic_memory.cli.commands.cloud.bisync_commands.get_bisync_state_path", + return_value=state_dir, + ): + result = bisync_state_exists("test-tenant") + + assert result is False + + def test_bisync_state_exists_false_empty_dir(self, tmp_path): + """Test checking for empty state directory.""" + state_dir = tmp_path / "state" + state_dir.mkdir() + + with patch( + "basic_memory.cli.commands.cloud.bisync_commands.get_bisync_state_path", + return_value=state_dir, + ): + result = bisync_state_exists("test-tenant") + + assert result is False + + +class TestGetBisyncDirectory: + """Tests for get_bisync_directory().""" + + def test_returns_default_directory(self): + """Test that default directory is returned when not configured.""" + with patch("basic_memory.cli.commands.cloud.bisync_commands.ConfigManager") as mock_config: + mock_config.return_value.config.bisync_config = {} + + result = get_bisync_directory() + + expected = Path.home() / "basic-memory-cloud-sync" + assert result == expected + + def test_returns_configured_directory(self, tmp_path): + """Test that configured directory is returned.""" + custom_dir = tmp_path / "custom-sync" + + with patch("basic_memory.cli.commands.cloud.bisync_commands.ConfigManager") as mock_config: + mock_config.return_value.config.bisync_config = {"sync_dir": str(custom_dir)} + + result = get_bisync_directory() + + assert result == custom_dir + + +class TestCloudProjectAutoRegistration: + """Tests for project auto-registration logic.""" + + @pytest.mark.asyncio + async def test_extracts_directory_names_from_cloud_paths(self): + """Test extraction of directory names from cloud project paths.""" + from basic_memory.cli.commands.cloud.bisync_commands import fetch_cloud_projects + + with patch( + "basic_memory.cli.commands.cloud.bisync_commands.make_api_request" + ) as mock_request: + mock_response = Mock() + mock_response.json.return_value = { + "projects": [ + {"name": "Main Project", "path": "/app/data/basic-memory"}, + {"name": "Work", "path": "/app/data/work-notes"}, + {"name": "Personal", "path": "/app/data/personal"}, + ] + } + mock_request.return_value = mock_response + + result = await fetch_cloud_projects() + + # Extract directory names as the code does + cloud_dir_names = set() + for p in result.projects: + path = p.path + if path.startswith("/app/data/"): + path = path[len("/app/data/") :] + dir_name = Path(path).name + cloud_dir_names.add(dir_name) + + assert cloud_dir_names == {"basic-memory", "work-notes", "personal"} + + @pytest.mark.asyncio + async def test_create_cloud_project_generates_permalink(self): + """Test that create_cloud_project generates correct permalink.""" + from basic_memory.cli.commands.cloud.bisync_commands import create_cloud_project + + with patch( + "basic_memory.cli.commands.cloud.bisync_commands.make_api_request" + ) as mock_request: + with patch( + "basic_memory.cli.commands.cloud.bisync_commands.generate_permalink" + ) as mock_permalink: + mock_permalink.return_value = "my-new-project" + mock_response = Mock() + mock_response.json.return_value = { + "name": "My New Project", + "path": "my-new-project", + "message": "Created", + } + mock_request.return_value = mock_response + + await create_cloud_project("My New Project") + + # Verify permalink was generated + mock_permalink.assert_called_once_with("My New Project") + + # Verify request was made with correct data + call_args = mock_request.call_args + json_data = call_args.kwargs["json_data"] + assert json_data["name"] == "My New Project" + assert json_data["path"] == "my-new-project" + assert json_data["set_default"] is False diff --git a/tests/cli/test_cli_tools.py b/tests/cli/test_cli_tools.py index f59b1593c..98513de95 100644 --- a/tests/cli/test_cli_tools.py +++ b/tests/cli/test_cli_tools.py @@ -467,19 +467,19 @@ def test_continue_conversation_no_results(cli_env): @patch("basic_memory.services.initialization.initialize_database") -def test_ensure_migrations_functionality(mock_initialize_database, project_config, monkeypatch): +def test_ensure_migrations_functionality(mock_initialize_database, app_config, monkeypatch): """Test the database initialization functionality.""" from basic_memory.services.initialization import ensure_initialization # Call the function - ensure_initialization(project_config) + ensure_initialization(app_config) # The underlying asyncio.run should call our mocked function mock_initialize_database.assert_called_once() @patch("basic_memory.services.initialization.initialize_database") -def test_ensure_migrations_handles_errors(mock_initialize_database, project_config, monkeypatch): +def test_ensure_migrations_handles_errors(mock_initialize_database, app_config, monkeypatch): """Test that initialization handles errors gracefully.""" from basic_memory.services.initialization import ensure_initialization @@ -487,6 +487,6 @@ def test_ensure_migrations_handles_errors(mock_initialize_database, project_conf mock_initialize_database.side_effect = Exception("Test error") # Call the function - should not raise exception - ensure_initialization(project_config) + ensure_initialization(app_config) # We're just making sure it doesn't crash by calling it diff --git a/tests/cli/test_project_commands.py b/tests/cli/test_project_commands.py deleted file mode 100644 index 28c22574a..000000000 --- a/tests/cli/test_project_commands.py +++ /dev/null @@ -1,227 +0,0 @@ -"""Tests for the project CLI commands.""" - -import os -from unittest.mock import patch, MagicMock -from typer.testing import CliRunner - -from basic_memory.cli.main import app as cli_app - - -@patch("basic_memory.cli.commands.project.asyncio.run") -def test_project_list_command(mock_run, cli_env): - """Test the 'project list' command with mocked API.""" - # Mock the API response - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = { - "projects": [{"name": "test", "path": "/path/to/test", "is_default": True}], - "default_project": "test", - "current_project": "test", - } - mock_run.return_value = mock_response - - runner = CliRunner() - result = runner.invoke(cli_app, ["project", "list"]) - - # Just verify it runs without exception - assert result.exit_code == 0 - - -@patch("basic_memory.cli.commands.project.asyncio.run") -def test_project_add_command(mock_run, cli_env): - """Test the 'project add' command with mocked API.""" - # Mock the API response - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = { - "message": "Project 'test-project' added successfully", - "status": "success", - "default": False, - } - mock_run.return_value = mock_response - - runner = CliRunner() - result = runner.invoke(cli_app, ["project", "add", "test-project", "/path/to/project"]) - - # Just verify it runs without exception - assert result.exit_code == 0 - - -@patch("basic_memory.cli.commands.project.asyncio.run") -def test_project_remove_command(mock_run, cli_env): - """Test the 'project remove' command with mocked API.""" - # Mock the API response - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = { - "message": "Project 'test-project' removed successfully", - "status": "success", - "default": False, - } - mock_run.return_value = mock_response - - runner = CliRunner() - result = runner.invoke(cli_app, ["project", "remove", "test-project"]) - - # Just verify it runs without exception - assert result.exit_code == 0 - - -@patch("basic_memory.cli.commands.project.asyncio.run") -@patch("importlib.reload") -def test_project_default_command(mock_reload, mock_run, cli_env): - """Test the 'project default' command with mocked API.""" - # Mock the API response - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = { - "message": "Project 'test-project' set as default successfully", - "status": "success", - "default": True, - } - mock_run.return_value = mock_response - - # Mock necessary config methods to have the test-project handled - # Patching call_put directly since it's imported at the module level - - # Patch the os.environ for checking - # On Windows, preserve USERPROFILE to allow home directory detection - env_vars = {} - if os.name == "nt" and "USERPROFILE" in os.environ: - env_vars["USERPROFILE"] = os.environ["USERPROFILE"] - - with patch.dict(os.environ, env_vars, clear=True): - # Patch ConfigManager.set_default_project to prevent validation error - with patch("basic_memory.config.ConfigManager.set_default_project"): - runner = CliRunner() - result = runner.invoke(cli_app, ["project", "default", "test-project"]) - - # Just verify it runs without exception and environment is set - assert result.exit_code == 0 - - -@patch("basic_memory.cli.commands.project.asyncio.run") -def test_project_sync_command(mock_run, cli_env): - """Test the 'project sync' command with mocked API.""" - # Mock the API response - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = { - "message": "Projects synchronized successfully between configuration and database", - "status": "success", - "default": False, - } - mock_run.return_value = mock_response - - runner = CliRunner() - result = runner.invoke(cli_app, ["project", "sync-config"]) - - # Just verify it runs without exception - assert result.exit_code == 0 - - -@patch("basic_memory.cli.commands.project.asyncio.run") -def test_project_failure_exits_with_error(mock_run, cli_env): - """Test that CLI commands properly exit with error code on API failures.""" - # Mock an exception being raised - mock_run.side_effect = Exception("API server not running") - - runner = CliRunner() - - # Test various commands for proper error handling - list_result = runner.invoke(cli_app, ["project", "list"]) - add_result = runner.invoke(cli_app, ["project", "add", "test-project", "/path/to/project"]) - remove_result = runner.invoke(cli_app, ["project", "remove", "test-project"]) - default_result = runner.invoke(cli_app, ["project", "default", "test-project"]) - - # All should exit with code 1 and show error message - assert list_result.exit_code == 1 - assert "Error listing projects" in list_result.output - - assert add_result.exit_code == 1 - assert "Error adding project" in add_result.output - - assert remove_result.exit_code == 1 - assert "Error removing project" in remove_result.output - - assert default_result.exit_code == 1 - assert "Error setting default project" in default_result.output - - -@patch("basic_memory.cli.commands.project.asyncio.run") -def test_project_move_command(mock_run, cli_env): - """Test the 'project move' command with mocked API.""" - # Mock the API response - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = { - "message": "Project 'test-project' updated successfully", - "status": "success", - "default": False, - } - mock_run.return_value = mock_response - - runner = CliRunner() - result = runner.invoke(cli_app, ["project", "move", "test-project", "/new/path/to/project"]) - - # Verify it runs without exception - assert result.exit_code == 0 - # Verify the important warning message is displayed - assert "Manual File Movement Required" in result.output - assert "You must manually move your project files" in result.output - assert "/new/path/to/project" in result.output - - -@patch("basic_memory.cli.commands.project.asyncio.run") -def test_project_move_command_failure(mock_run, cli_env): - """Test the 'project move' command with API failure.""" - # Mock an exception being raised - mock_run.side_effect = Exception("Project not found") - - runner = CliRunner() - result = runner.invoke(cli_app, ["project", "move", "nonexistent-project", "/new/path"]) - - # Should exit with code 1 and show error message - assert result.exit_code == 1 - assert "Error moving project" in result.output - - -@patch("basic_memory.cli.commands.project.call_patch") -def test_project_move_command_uses_permalink(mock_call_patch, cli_env): - """Test that the 'project move' command correctly generates and uses permalink in API call.""" - # Mock successful API response - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = { - "message": "Project 'Test Project Name' updated successfully", - "status": "success", - "default": False, - } - mock_call_patch.return_value = mock_response - - runner = CliRunner() - - # Test with a project name that needs normalization (spaces, mixed case) - project_name = "Test Project Name" - new_path = os.path.join("new", "path", "to", "project") - - result = runner.invoke(cli_app, ["project", "move", project_name, new_path]) - - # Verify command executed successfully - assert result.exit_code == 0 - - # Verify call_patch was called with the correct permalink-formatted project name - mock_call_patch.assert_called_once() - args, kwargs = mock_call_patch.call_args - - # Check the API endpoint uses the original name and normalized permalink - # The actual implementation uses f"/{name}/project/{project_permalink}" - expected_endpoint = "/Test Project Name/project/test-project-name" - assert args[1] == expected_endpoint # Second argument is the endpoint URL - - # Verify the data contains the resolved path (using same normalization as the function) - from pathlib import Path - - expected_path = Path(os.path.abspath(os.path.expanduser(new_path))).as_posix() - expected_data = {"path": expected_path} - assert kwargs["json"] == expected_data diff --git a/tests/cli/test_project_info.py b/tests/cli/test_project_info.py deleted file mode 100644 index 177510a3b..000000000 --- a/tests/cli/test_project_info.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Tests for the project_info CLI command.""" - -import json -from datetime import datetime -from unittest.mock import patch, AsyncMock - -from typer.testing import CliRunner - -from basic_memory.cli.main import app as cli_app -from basic_memory.schemas.project_info import ( - ProjectInfoResponse, - ProjectStatistics, - ActivityMetrics, - SystemStatus, -) - - -def test_info_stats(): - """Test the 'project info' command with default output.""" - runner = CliRunner() - - # Create mock project info data - mock_info = ProjectInfoResponse( - project_name="test-project", - project_path="/test/path", - default_project="test-project", - statistics=ProjectStatistics( - total_entities=10, - total_observations=20, - total_relations=5, - total_unresolved_relations=1, - isolated_entities=2, - entity_types={"note": 8, "concept": 2}, - observation_categories={"tech": 15, "note": 5}, - relation_types={"connects_to": 3, "references": 2}, - most_connected_entities=[], - ), - activity=ActivityMetrics(recently_created=[], recently_updated=[], monthly_growth={}), - system=SystemStatus( - version="0.13.0", - database_path="/test/db.sqlite", - database_size="1.2 MB", - watch_status=None, - timestamp=datetime.now(), - ), - available_projects={"test-project": {"path": "/test/path"}}, - ) - - # Mock the async project_info function - with patch( - "basic_memory.cli.commands.project.project_info.fn", new_callable=AsyncMock - ) as mock_func: - mock_func.return_value = mock_info - - # Run the command - result = runner.invoke(cli_app, ["project", "info"]) - - # Verify exit code - assert result.exit_code == 0 - - # Check that key data is included in the output - assert "Basic Memory Project Info" in result.stdout - assert "test-project" in result.stdout - assert "Statistics" in result.stdout - - -def test_info_stats_json(): - """Test the 'project info --json' command for JSON output.""" - runner = CliRunner() - - # Create mock project info data - mock_info = ProjectInfoResponse( - project_name="test-project", - project_path="/test/path", - default_project="test-project", - statistics=ProjectStatistics( - total_entities=10, - total_observations=20, - total_relations=5, - total_unresolved_relations=1, - isolated_entities=2, - entity_types={"note": 8, "concept": 2}, - observation_categories={"tech": 15, "note": 5}, - relation_types={"connects_to": 3, "references": 2}, - most_connected_entities=[], - ), - activity=ActivityMetrics(recently_created=[], recently_updated=[], monthly_growth={}), - system=SystemStatus( - version="0.13.0", - database_path="/test/db.sqlite", - database_size="1.2 MB", - watch_status=None, - timestamp=datetime.now(), - ), - available_projects={"test-project": {"path": "/test/path"}}, - ) - - # Mock the async project_info function - with patch( - "basic_memory.cli.commands.project.project_info.fn", new_callable=AsyncMock - ) as mock_func: - mock_func.return_value = mock_info - - # Run the command with --json flag - result = runner.invoke(cli_app, ["project", "info", "--json"]) - - # Verify exit code - assert result.exit_code == 0 - - # Parse JSON output - output = json.loads(result.stdout) - - # Verify JSON structure matches our mock data - assert output["default_project"] == "test-project" - assert output["project_name"] == "test-project" - assert output["statistics"]["total_entities"] == 10 diff --git a/tests/cli/test_status.py b/tests/cli/test_status.py deleted file mode 100644 index 066b3d0fb..000000000 --- a/tests/cli/test_status.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Tests for CLI status command.""" - -from unittest.mock import patch, AsyncMock - -from typer.testing import CliRunner - -from basic_memory.cli.app import app -from basic_memory.cli.commands.status import ( - add_files_to_tree, - build_directory_summary, - group_changes_by_directory, - display_changes, -) -from basic_memory.sync.sync_service import SyncReport - -# Set up CLI runner -runner = CliRunner() - - -def test_status_command(): - """Test CLI status command.""" - # Mock the async run_status function to avoid event loop issues - with patch( - "basic_memory.cli.commands.status.run_status", new_callable=AsyncMock - ) as mock_run_status: - # Mock successful execution (no return value needed since it just prints) - mock_run_status.return_value = None - - # Should exit with code 0 - result = runner.invoke(app, ["status", "--verbose"]) - assert result.exit_code == 0 - - # Verify the function was called with verbose=True - mock_run_status.assert_called_once_with(True) - - -def test_status_command_error(): - """Test CLI status command error handling.""" - # Mock the async run_status function to raise an exception - with patch( - "basic_memory.cli.commands.status.run_status", new_callable=AsyncMock - ) as mock_run_status: - # Mock an error - mock_run_status.side_effect = Exception("Database connection failed") - - # Should exit with code 1 when error occurs - result = runner.invoke(app, ["status", "--verbose"]) - assert result.exit_code == 1 - assert "Error checking status: Database connection failed" in result.stderr - - -def test_display_changes_no_changes(): - """Test displaying no changes.""" - changes = SyncReport(set(), set(), set(), {}, {}) - display_changes("test", "Test", changes, verbose=True) - display_changes("test", "Test", changes, verbose=False) - - -def test_display_changes_with_changes(): - """Test displaying various changes.""" - changes = SyncReport( - new={"dir1/new.md"}, - modified={"dir1/mod.md"}, - deleted={"dir2/del.md"}, - moves={"old.md": "new.md"}, - checksums={"dir1/new.md": "abcd1234"}, - ) - display_changes("test", "Test", changes, verbose=True) - display_changes("test", "Test", changes, verbose=False) - - -def test_build_directory_summary(): - """Test building directory change summary.""" - counts = { - "new": 2, - "modified": 1, - "moved": 1, - "deleted": 1, - } - summary = build_directory_summary(counts) - assert "+2" in summary - assert "~1" in summary - assert "↔1" in summary - assert "-1" in summary - - -def test_build_directory_summary_empty(): - """Test summary with no changes.""" - counts = { - "new": 0, - "modified": 0, - "moved": 0, - "deleted": 0, - } - summary = build_directory_summary(counts) - assert summary == "" - - -def test_group_changes_by_directory(): - """Test grouping changes by directory.""" - changes = SyncReport( - new={"dir1/new.md", "dir2/new2.md"}, - modified={"dir1/mod.md"}, - deleted={"dir2/del.md"}, - moves={"dir1/old.md": "dir2/new.md"}, - checksums={}, - ) - - grouped = group_changes_by_directory(changes) - - assert grouped["dir1"]["new"] == 1 - assert grouped["dir1"]["modified"] == 1 - assert grouped["dir1"]["moved"] == 1 - - assert grouped["dir2"]["new"] == 1 - assert grouped["dir2"]["deleted"] == 1 - assert grouped["dir2"]["moved"] == 1 - - -def test_add_files_to_tree(): - """Test adding files to tree visualization.""" - from rich.tree import Tree - - # Test with various path patterns - paths = { - "dir1/file1.md", # Normal nested file - "dir1/file2.md", # Another in same dir - "dir2/subdir/file3.md", # Deeper nesting - "root.md", # Root level file - } - - # Test without checksums - tree = Tree("Test") - add_files_to_tree(tree, paths, "green") - - # Test with checksums - checksums = {"dir1/file1.md": "abcd1234", "dir1/file2.md": "efgh5678"} - - tree = Tree("Test with checksums") - add_files_to_tree(tree, paths, "green", checksums) diff --git a/tests/cli/test_sync.py b/tests/cli/test_sync.py deleted file mode 100644 index 165a26d63..000000000 --- a/tests/cli/test_sync.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Tests for CLI sync command.""" - -import pytest -from typer.testing import CliRunner - -from basic_memory.cli.app import app -from basic_memory.cli.commands.sync import ( - display_sync_summary, - display_detailed_sync_results, - run_sync, - group_issues_by_directory, - ValidationIssue, -) -from basic_memory.config import get_project_config -from basic_memory.sync.sync_service import SyncReport - -# Set up CLI runner -runner = CliRunner() - - -def test_group_issues_by_directory(): - """Test grouping validation issues by directory.""" - issues = [ - ValidationIssue("dir1/file1.md", "error1"), - ValidationIssue("dir1/file2.md", "error2"), - ValidationIssue("dir2/file3.md", "error3"), - ] - - grouped = group_issues_by_directory(issues) - - assert len(grouped["dir1"]) == 2 - assert len(grouped["dir2"]) == 1 - assert grouped["dir1"][0].error == "error1" - assert grouped["dir2"][0].error == "error3" - - -def test_display_sync_summary_no_changes(): - """Test displaying sync summary with no changes.""" - changes = SyncReport(set(), set(), set(), {}, {}) - display_sync_summary(changes) - - -def test_display_sync_summary_with_changes(): - """Test displaying sync summary with various changes.""" - changes = SyncReport( - new={"new.md"}, - modified={"mod.md"}, - deleted={"del.md"}, - moves={"old.md": "new.md"}, - checksums={"new.md": "abcd1234"}, - ) - display_sync_summary(changes) - - -def test_display_detailed_sync_results_no_changes(): - """Test displaying detailed results with no changes.""" - changes = SyncReport(set(), set(), set(), {}, {}) - display_detailed_sync_results(changes) - - -def test_display_detailed_sync_results_with_changes(): - """Test displaying detailed results with various changes.""" - changes = SyncReport( - new={"new.md"}, - modified={"mod.md"}, - deleted={"del.md"}, - moves={"old.md": "new.md"}, - checksums={"new.md": "abcd1234", "mod.md": "efgh5678"}, - ) - display_detailed_sync_results(changes) - - -@pytest.mark.asyncio -async def test_run_sync_basic(sync_service, project_config, test_project): - """Test basic sync operation.""" - # Set up test environment - config = get_project_config() - config.home = project_config.home - config.name = test_project.name - - # Create test files - test_file = project_config.home / "test.md" - test_file.write_text("""--- -title: Test ---- -# Test -Some content""") - - # Run sync - should detect new file - await run_sync(verbose=True) - - -def test_sync_command(): - """Test the sync command.""" - from unittest.mock import patch, AsyncMock - - # Mock the async run_sync function to avoid event loop issues - with patch("basic_memory.cli.commands.sync.run_sync", new_callable=AsyncMock) as mock_run_sync: - # Mock successful execution (no return value needed since it just prints) - mock_run_sync.return_value = None - - # Mock config values that the sync command prints - result = runner.invoke(app, ["sync", "--verbose"]) - assert result.exit_code == 0 - - # Verify output contains project info - assert "Syncing project: test-project" in result.stdout - - # Verify the function was called with verbose=True - mock_run_sync.assert_called_once_with(verbose=True) - - -def test_sync_command_error(): - """Test the sync command error handling.""" - from unittest.mock import patch, AsyncMock - - # Mock the async run_sync function to raise an exception - with patch("basic_memory.cli.commands.sync.run_sync", new_callable=AsyncMock) as mock_run_sync: - # Mock an error - mock_run_sync.side_effect = Exception("Sync failed") - - result = runner.invoke(app, ["sync", "--verbose"]) - assert result.exit_code == 1 - assert "Error during sync: Sync failed" in result.stderr diff --git a/tests/cli/test_version.py b/tests/cli/test_version.py deleted file mode 100644 index 816cccc2a..000000000 --- a/tests/cli/test_version.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Tests for CLI sync command.""" - -from typer.testing import CliRunner - -from basic_memory.cli.app import app - -# Set up CLI runner -runner = CliRunner() - - -def test_version_arg(): - """Test the version arg.""" - result = runner.invoke(app, ["--version"]) - assert result.exit_code == 0 diff --git a/tests/services/test_initialization.py b/tests/services/test_initialization.py index 1e7bb8732..92bca4276 100644 --- a/tests/services/test_initialization.py +++ b/tests/services/test_initialization.py @@ -31,9 +31,9 @@ async def test_initialize_database_error(mock_get_or_create_db, app_config): @patch("basic_memory.services.initialization.asyncio.run") -def test_ensure_initialization(mock_run, project_config): +def test_ensure_initialization(mock_run, app_config): """Test synchronous initialization wrapper.""" - ensure_initialization(project_config) + ensure_initialization(app_config) mock_run.assert_called_once() @@ -124,7 +124,7 @@ async def test_reconcile_projects_with_error_handling(mock_get_db, app_config): @pytest.mark.asyncio @patch("basic_memory.services.initialization.db.get_or_create_db") -@patch("basic_memory.cli.commands.sync.get_sync_service") +@patch("basic_memory.sync.sync_service.get_sync_service") @patch("basic_memory.sync.WatchService") @patch("basic_memory.services.initialization.asyncio.create_task") async def test_initialize_file_sync_background_tasks( diff --git a/tests/services/test_project_service_operations.py b/tests/services/test_project_service_operations.py index 12614d70d..84692a1ea 100644 --- a/tests/services/test_project_service_operations.py +++ b/tests/services/test_project_service_operations.py @@ -1,7 +1,6 @@ """Additional tests for ProjectService operations.""" import os -import json from unittest.mock import patch import pytest @@ -102,33 +101,3 @@ async def test_update_project_path(project_service: ProjectService, tmp_path, co config_manager.remove_project(test_project) except Exception: pass - - -@pytest.mark.asyncio -async def test_system_status_with_watch(project_service: ProjectService): - """Test system status with watch status.""" - # Mock watch status file - mock_watch_status = { - "running": True, - "start_time": "2025-03-05T18:00:42.752435", - "pid": 7321, - "error_count": 0, - "last_error": None, - "last_scan": "2025-03-05T19:59:02.444416", - "synced_files": 6, - "recent_events": [], - } - - # Patch Path.exists and Path.read_text - with ( - patch("pathlib.Path.exists", return_value=True), - patch("pathlib.Path.read_text", return_value=json.dumps(mock_watch_status)), - ): - # Get system status - status = project_service.get_system_status() - - # Verify watch status is included - assert status.watch_status is not None - assert status.watch_status["running"] is True - assert status.watch_status["pid"] == 7321 - assert status.watch_status["synced_files"] == 6