fix: improve utf-8 support for file reading/writing (#32)

fixes #29

Signed-off-by: phernandez <paul@basicmachines.co>

Author: phernandez

src/basic_memory/__init__.py

@@ -1,3 +1,3 @@
 """basic-memory - Local-first knowledge management combining Zettelkasten with knowledge graphs"""
 
-__version__ = "0.9.0"
+__version__ = "0.9.0"

src/basic_memory/api/routers/project_info_router.py

@@ -3,9 +3,6 @@
 import json
 from datetime import datetime
 
-from fastapi import APIRouter
-from sqlalchemy import text
-
 from basic_memory.config import config, config_manager
 from basic_memory.deps import (
     ProjectInfoRepositoryDep,
@@ -18,6 +15,8 @@
     SystemStatus,
 )
 from basic_memory.sync.watch_service import WATCH_STATUS_JSON
+from fastapi import APIRouter
+from sqlalchemy import text
 
 router = APIRouter(prefix="/stats", tags=["statistics"])
 
@@ -262,7 +261,7 @@ async def get_system_status() -> SystemStatus:
     watch_status_path = config.home / ".basic-memory" / WATCH_STATUS_JSON
     if watch_status_path.exists():
         try:
-            watch_status = json.loads(watch_status_path.read_text())
+            watch_status = json.loads(watch_status_path.read_text(encoding="utf-8"))
         except Exception:  # pragma: no cover
             pass
 

src/basic_memory/cli/commands/import_chatgpt.py

@@ -7,15 +7,14 @@
 from typing import Dict, Any, List, Annotated, Set, Optional
 
 import typer
-from loguru import logger
-from rich.console import Console
-from rich.panel import Panel
-from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
-
 from basic_memory.cli.app import import_app
 from basic_memory.config import config
 from basic_memory.markdown import EntityParser, MarkdownProcessor
 from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter
+from loguru import logger
+from rich.console import Console
+from rich.panel import Panel
+from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
 
 console = Console()
 
@@ -167,7 +166,7 @@ async def process_chatgpt_json(
         read_task = progress.add_task("Reading chat data...", total=None)
 
         # Read conversations
-        conversations = json.loads(json_path.read_text())
+        conversations = json.loads(json_path.read_text(encoding="utf-8"))
         progress.update(read_task, total=len(conversations))
 
         # Process each conversation

src/basic_memory/cli/commands/import_claude_conversations.py

@@ -7,15 +7,14 @@
 from typing import Dict, Any, List, Annotated
 
 import typer
-from loguru import logger
-from rich.console import Console
-from rich.panel import Panel
-from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
-
 from basic_memory.cli.app import claude_app
 from basic_memory.config import config
 from basic_memory.markdown import EntityParser, MarkdownProcessor
 from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter
+from loguru import logger
+from rich.console import Console
+from rich.panel import Panel
+from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
 
 console = Console()
 
@@ -124,7 +123,7 @@ async def process_conversations_json(
         read_task = progress.add_task("Reading chat data...", total=None)
 
         # Read chat data - handle array of arrays format
-        data = json.loads(json_path.read_text())
+        data = json.loads(json_path.read_text(encoding="utf-8"))
         conversations = [chat for chat in data]
         progress.update(read_task, total=len(conversations))
 

src/basic_memory/cli/commands/import_claude_projects.py

@@ -6,15 +6,14 @@
 from typing import Dict, Any, Annotated, Optional
 
 import typer
-from loguru import logger
-from rich.console import Console
-from rich.panel import Panel
-from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
-
 from basic_memory.cli.app import claude_app
 from basic_memory.config import config
 from basic_memory.markdown import EntityParser, MarkdownProcessor
 from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter
+from loguru import logger
+from rich.console import Console
+from rich.panel import Panel
+from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
 
 console = Console()
 
@@ -103,7 +102,7 @@ async def process_projects_json(
         read_task = progress.add_task("Reading project data...", total=None)
 
         # Read project data
-        data = json.loads(json_path.read_text())
+        data = json.loads(json_path.read_text(encoding="utf-8"))
         progress.update(read_task, total=len(data))
 
         # Track import counts

src/basic_memory/config.py

@@ -5,13 +5,12 @@
 from pathlib import Path
 from typing import Any, Dict, Literal, Optional
 
+import basic_memory
+from basic_memory.utils import setup_logging
 from loguru import logger
 from pydantic import Field, field_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
-import basic_memory
-from basic_memory.utils import setup_logging
-
 DATABASE_NAME = "memory.db"
 DATA_DIR_NAME = ".basic-memory"
 CONFIG_FILE_NAME = "config.json"
@@ -111,7 +110,7 @@ def load_config(self) -> BasicMemoryConfig:
         """Load configuration from file or create default."""
         if self.config_file.exists():
             try:
-                data = json.loads(self.config_file.read_text())
+                data = json.loads(self.config_file.read_text(encoding="utf-8"))
                 return BasicMemoryConfig(**data)
             except Exception as e:
                 logger.error(f"Failed to load config: {e}")

src/basic_memory/file_utils.py

@@ -85,7 +85,7 @@ async def write_file_atomic(path: FilePath, content: str) -> None:
     temp_path = path_obj.with_suffix(".tmp")
 
     try:
-        temp_path.write_text(content)
+        temp_path.write_text(content, encoding="utf-8")
         temp_path.replace(path_obj)
         logger.debug("Wrote file atomically", path=str(path_obj), content_length=len(content))
     except Exception as e:  # pragma: no cover
@@ -203,7 +203,7 @@ async def update_frontmatter(path: FilePath, updates: Dict[str, Any]) -> str:
         path_obj = Path(path) if isinstance(path, str) else path
 
         # Read current content
-        content = path_obj.read_text()
+        content = path_obj.read_text(encoding="utf-8")
 
         # Parse current frontmatter
         current_fm = {}
@@ -215,7 +215,7 @@ async def update_frontmatter(path: FilePath, updates: Dict[str, Any]) -> str:
         new_fm = {**current_fm, **updates}
 
         # Write new file with updated frontmatter
-        yaml_fm = yaml.dump(new_fm, sort_keys=False)
+        yaml_fm = yaml.dump(new_fm, sort_keys=False, allow_unicode=True)
         final_content = f"---\n{yaml_fm}---\n\n{content.strip()}"
 
         logger.debug("Updating frontmatter", path=str(path_obj), update_keys=list(updates.keys()))

src/basic_memory/markdown/markdown_processor.py

@@ -83,7 +83,7 @@ async def write_file(
         """
         # Dirty check if needed
         if expected_checksum is not None:
-            current_content = path.read_text()
+            current_content = path.read_text(encoding="utf-8")
             current_checksum = await file_utils.compute_checksum(current_content)
             if current_checksum != expected_checksum:
                 raise DirtyFileError(f"File {path} has been modified")

src/basic_memory/mcp/prompts/ai_assistant_guide.py

@@ -1,8 +1,7 @@
 from pathlib import Path
 
-from loguru import logger
-
 from basic_memory.mcp.server import mcp
+from loguru import logger
 
 
 @mcp.resource(
@@ -20,7 +19,9 @@ def ai_assistant_guide() -> str:
         A focused guide on Basic Memory usage.
     """
     logger.info("Loading AI assistant guide resource")
-    guide_doc = Path(__file__).parent.parent.parent.parent.parent / "static" / "ai_assistant_guide.md"
-    content = guide_doc.read_text()
+    guide_doc = (
+        Path(__file__).parent.parent.parent.parent.parent / "static" / "ai_assistant_guide.md"
+    )
+    content = guide_doc.read_text(encoding="utf-8")
     logger.info(f"Loaded AI assistant guide ({len(content)} chars)")
     return content

src/basic_memory/services/file_service.py

@@ -5,15 +5,14 @@
 from pathlib import Path
 from typing import Any, Dict, Tuple, Union
 
-from loguru import logger
-
 from basic_memory import file_utils
 from basic_memory.file_utils import FileError
 from basic_memory.markdown.markdown_processor import MarkdownProcessor
 from basic_memory.models import Entity as EntityModel
 from basic_memory.schemas import Entity as EntitySchema
 from basic_memory.services.exceptions import FileOperationError
 from basic_memory.utils import FilePath
+from loguru import logger
 
 
 class FileService:
@@ -171,8 +170,7 @@ async def read_file(self, path: FilePath) -> Tuple[str, str]:
 
         try:
             logger.debug("Reading file", operation="read_file", path=str(full_path))
-
-            content = full_path.read_text()
+            content = full_path.read_text(encoding="utf-8")
             checksum = await file_utils.compute_checksum(content)
 
             logger.debug(
@@ -236,7 +234,7 @@ async def compute_checksum(self, path: FilePath) -> str:
         try:
             if self.is_markdown(path):
                 # read str
-                content = full_path.read_text()
+                content = full_path.read_text(encoding="utf-8")
             else:
                 # read bytes
                 content = full_path.read_bytes()