-
Notifications
You must be signed in to change notification settings - Fork 180
Expand file tree
/
Copy pathproject_sync.py
More file actions
375 lines (301 loc) · 13.9 KB
/
project_sync.py
File metadata and controls
375 lines (301 loc) · 13.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
"""Cloud sync commands for Basic Memory projects.
Commands for syncing, bisyncing, and checking integrity between local and cloud
project instances. These were previously in project.py but belong here since
they are cloud-specific operations.
"""
import os
from datetime import datetime
import typer
from rich.console import Console
from basic_memory.cli.app import cloud_app
from basic_memory.cli.commands.cloud.bisync_commands import get_mount_info
from basic_memory.cli.commands.cloud.rclone_commands import (
RcloneError,
SyncProject,
get_project_bisync_state,
project_bisync,
project_check,
project_sync,
)
from basic_memory.cli.commands.command_utils import run_with_cleanup
from basic_memory.cli.commands.routing import force_routing
from basic_memory.config import ConfigManager, ProjectEntry
from basic_memory.mcp.async_client import get_client
from basic_memory.mcp.clients import ProjectClient
from basic_memory.schemas.project_info import ProjectItem
from basic_memory.utils import generate_permalink, normalize_project_path
console = Console()
# --- Shared helpers ---
def _has_cloud_credentials(config) -> bool:
"""Return whether cloud credentials are available (API key or OAuth token)."""
from basic_memory.config import has_cloud_credentials
return has_cloud_credentials(config)
def _require_cloud_credentials(config) -> None:
"""Exit with actionable guidance when cloud credentials are missing."""
if _has_cloud_credentials(config):
return
console.print("[red]Error: cloud credentials are required for this command[/red]")
console.print("[dim]Run 'bm cloud login' or 'bm cloud api-key save <key>' first[/dim]")
raise typer.Exit(1)
async def _get_cloud_project(name: str) -> ProjectItem | None:
"""Fetch a project by name from the cloud API."""
async with get_client(project_name=name) as client:
projects_list = await ProjectClient(client).list_projects()
for proj in projects_list.projects:
if generate_permalink(proj.name) == generate_permalink(name):
return proj
return None
def _get_sync_project(
name: str, config, project_data: ProjectItem
) -> tuple[SyncProject, str | None]:
"""Build a SyncProject and resolve local_sync_path from config.
Returns (sync_project, local_sync_path). Exits if no local_sync_path configured.
"""
sync_entry = config.projects.get(name)
# Support both new (path) and legacy (local_sync_path) configs
local_sync_path = (sync_entry.local_sync_path or sync_entry.path) if sync_entry else None
if not local_sync_path or not os.path.isabs(local_sync_path):
console.print(f"[red]Error: Project '{name}' has no local sync path configured[/red]")
console.print(f"\nConfigure sync with: bm cloud sync-setup {name} ~/path/to/local")
raise typer.Exit(1)
sync_project = SyncProject(
name=project_data.name,
path=normalize_project_path(project_data.path),
local_sync_path=local_sync_path,
)
return sync_project, local_sync_path
# --- Commands ---
@cloud_app.command("sync")
def sync_project_command(
name: str = typer.Option(..., "--name", help="Project name to sync"),
dry_run: bool = typer.Option(False, "--dry-run", help="Preview changes without syncing"),
verbose: bool = typer.Option(False, "--verbose", "-v", help="Show detailed output"),
) -> None:
"""One-way sync: local -> cloud (make cloud identical to local).
Example:
bm cloud sync --name research
bm cloud sync --name research --dry-run
"""
config = ConfigManager().config
_require_cloud_credentials(config)
try:
# Get tenant info for bucket name
tenant_info = run_with_cleanup(get_mount_info())
bucket_name = tenant_info.bucket_name
# Get project info
with force_routing(cloud=True):
project_data = run_with_cleanup(_get_cloud_project(name))
if not project_data:
console.print(f"[red]Error: Project '{name}' not found[/red]")
raise typer.Exit(1)
sync_project, local_sync_path = _get_sync_project(name, config, project_data)
# Run sync
console.print(f"[blue]Syncing {name} (local -> cloud)...[/blue]")
success = project_sync(sync_project, bucket_name, dry_run=dry_run, verbose=verbose)
if success:
console.print(f"[green]{name} synced successfully[/green]")
# Trigger database sync if not a dry run
if not dry_run:
async def _trigger_db_sync():
async with get_client(project_name=name) as client:
return await ProjectClient(client).sync(
project_data.external_id, force_full=False
)
try:
with force_routing(cloud=True):
result = run_with_cleanup(_trigger_db_sync())
console.print(f"[dim]Database sync initiated: {result.get('message')}[/dim]")
except Exception as e:
console.print(f"[yellow]Warning: Could not trigger database sync: {e}[/yellow]")
else:
console.print(f"[red]{name} sync failed[/red]")
raise typer.Exit(1)
except RcloneError as e:
console.print(f"[red]Sync error: {e}[/red]")
raise typer.Exit(1)
except Exception as e:
console.print(f"[red]Error: {e}[/red]")
raise typer.Exit(1)
@cloud_app.command("bisync")
def bisync_project_command(
name: str = typer.Option(..., "--name", help="Project name to bisync"),
dry_run: bool = typer.Option(False, "--dry-run", help="Preview changes without syncing"),
resync: bool = typer.Option(False, "--resync", help="Force new baseline"),
verbose: bool = typer.Option(False, "--verbose", "-v", help="Show detailed output"),
) -> None:
"""Two-way sync: local <-> cloud (bidirectional sync).
Examples:
bm cloud bisync --name research --resync # First time
bm cloud bisync --name research # Subsequent syncs
bm cloud bisync --name research --dry-run # Preview changes
"""
config = ConfigManager().config
_require_cloud_credentials(config)
try:
# Get tenant info for bucket name
tenant_info = run_with_cleanup(get_mount_info())
bucket_name = tenant_info.bucket_name
# Get project info
with force_routing(cloud=True):
project_data = run_with_cleanup(_get_cloud_project(name))
if not project_data:
console.print(f"[red]Error: Project '{name}' not found[/red]")
raise typer.Exit(1)
sync_project, local_sync_path = _get_sync_project(name, config, project_data)
# Run bisync
console.print(f"[blue]Bisync {name} (local <-> cloud)...[/blue]")
success = project_bisync(
sync_project, bucket_name, dry_run=dry_run, resync=resync, verbose=verbose
)
if success:
console.print(f"[green]{name} bisync completed successfully[/green]")
# Update config — sync_entry is guaranteed non-None because
# _get_sync_project validated local_sync_path (which comes from sync_entry)
sync_entry = config.projects.get(name)
if sync_entry is None:
raise RuntimeError(
f"Sync entry for project '{name}' unexpectedly missing after validation"
)
sync_entry.last_sync = datetime.now()
sync_entry.bisync_initialized = True
ConfigManager().save_config(config)
# Trigger database sync if not a dry run
if not dry_run:
async def _trigger_db_sync():
async with get_client(project_name=name) as client:
return await ProjectClient(client).sync(
project_data.external_id, force_full=False
)
try:
with force_routing(cloud=True):
result = run_with_cleanup(_trigger_db_sync())
console.print(f"[dim]Database sync initiated: {result.get('message')}[/dim]")
except Exception as e:
console.print(f"[yellow]Warning: Could not trigger database sync: {e}[/yellow]")
else:
console.print(f"[red]{name} bisync failed[/red]")
raise typer.Exit(1)
except RcloneError as e:
console.print(f"[red]Bisync error: {e}[/red]")
raise typer.Exit(1)
except Exception as e:
console.print(f"[red]Error: {e}[/red]")
raise typer.Exit(1)
@cloud_app.command("check")
def check_project_command(
name: str = typer.Option(..., "--name", help="Project name to check"),
one_way: bool = typer.Option(False, "--one-way", help="Check one direction only (faster)"),
) -> None:
"""Verify file integrity between local and cloud.
Example:
bm cloud check --name research
"""
config = ConfigManager().config
_require_cloud_credentials(config)
try:
# Get tenant info for bucket name
tenant_info = run_with_cleanup(get_mount_info())
bucket_name = tenant_info.bucket_name
# Get project info
with force_routing(cloud=True):
project_data = run_with_cleanup(_get_cloud_project(name))
if not project_data:
console.print(f"[red]Error: Project '{name}' not found[/red]")
raise typer.Exit(1)
sync_project, local_sync_path = _get_sync_project(name, config, project_data)
# Run check
console.print(f"[blue]Checking {name} integrity...[/blue]")
match = project_check(sync_project, bucket_name, one_way=one_way)
if match:
console.print(f"[green]{name} files match[/green]")
else:
console.print(f"[yellow]!{name} has differences[/yellow]")
except RcloneError as e:
console.print(f"[red]Check error: {e}[/red]")
raise typer.Exit(1)
except Exception as e:
console.print(f"[red]Error: {e}[/red]")
raise typer.Exit(1)
@cloud_app.command("bisync-reset")
def bisync_reset(
name: str = typer.Argument(..., help="Project name to reset bisync state for"),
) -> None:
"""Clear bisync state for a project.
This removes the bisync metadata files, forcing a fresh --resync on next bisync.
Useful when bisync gets into an inconsistent state or when remote path changes.
"""
import shutil
try:
state_path = get_project_bisync_state(name)
if not state_path.exists():
console.print(f"[yellow]No bisync state found for project '{name}'[/yellow]")
return
# Remove the entire state directory
shutil.rmtree(state_path)
console.print(f"[green]Cleared bisync state for project '{name}'[/green]")
console.print("\nNext steps:")
console.print(f" 1. Preview: bm cloud bisync --name {name} --resync --dry-run")
console.print(f" 2. Sync: bm cloud bisync --name {name} --resync")
except Exception as e:
console.print(f"[red]Error clearing bisync state: {str(e)}[/red]")
raise typer.Exit(1)
@cloud_app.command("sync-setup")
def setup_project_sync(
name: str = typer.Argument(..., help="Project name"),
local_path: str = typer.Argument(..., help="Local sync directory"),
) -> None:
"""Configure local sync for an existing cloud project.
Example:
bm cloud sync-setup research ~/Documents/research
"""
import os
from pathlib import Path
config_manager = ConfigManager()
config = config_manager.config
_require_cloud_credentials(config)
async def _verify_project_exists():
"""Verify the project exists on cloud by listing all projects."""
async with get_client(project_name=name) as client:
projects_list = await ProjectClient(client).list_projects()
project_names = [p.name for p in projects_list.projects]
if name not in project_names:
raise ValueError(f"Project '{name}' not found on cloud")
return True
try:
# Verify project exists on cloud
with force_routing(cloud=True):
run_with_cleanup(_verify_project_exists())
# Resolve and create local path
resolved_path = Path(os.path.abspath(os.path.expanduser(local_path)))
resolved_path.mkdir(parents=True, exist_ok=True)
# Update project entry with sync path — path is always the local directory
entry = config.projects.get(name)
if entry:
entry.path = resolved_path.as_posix()
entry.local_sync_path = resolved_path.as_posix()
entry.bisync_initialized = False
entry.last_sync = None
else:
config.projects[name] = ProjectEntry(
path=resolved_path.as_posix(),
local_sync_path=resolved_path.as_posix(),
)
config_manager.save_config(config)
# Create the project in the local DB so the MCP server can immediately use it
async def _create_local_project():
async with get_client() as client:
data = {"name": name, "path": resolved_path.as_posix(), "set_default": False}
return await ProjectClient(client).create_project(data)
with force_routing(local=True):
try:
run_with_cleanup(_create_local_project())
except Exception:
pass # Project may already exist locally; reconcile on next startup
console.print(f"[green]Sync configured for project '{name}'[/green]")
console.print(f"\nLocal sync path: {resolved_path}")
console.print("\nNext steps:")
console.print(f" 1. Preview: bm cloud bisync --name {name} --resync --dry-run")
console.print(f" 2. Sync: bm cloud bisync --name {name} --resync")
except Exception as e:
console.print(f"[red]Error configuring sync: {str(e)}[/red]")
raise typer.Exit(1)