fix: utf8 for all file reads/write/open instead of default platform encoding (#91)

Signed-off-by: phernandez <paul@basicmachines.co>

Author: phernandez

installer/installer.py

@@ -44,7 +44,7 @@ def update_claude_config():
 
     # Load existing config or create new
     if config_path.exists():
-        config = json.loads(config_path.read_text())
+        config = json.loads(config_path.read_text(encoding="utf-8"))
     else:
         config = {"mcpServers": {}}
 

src/basic_memory/cli/commands/import_memory_json.py

@@ -38,7 +38,7 @@ async def process_memory_json(
         read_task = progress.add_task("Reading memory.json...", total=None)
 
         # First pass - collect entities and relations
-        with open(json_path) as f:
+        with open(json_path, encoding="utf-8") as f:
             lines = f.readlines()
             progress.update(read_task, total=len(lines))
 

src/basic_memory/markdown/entity_parser.py

@@ -104,7 +104,7 @@ async def parse_file(self, path: Path | str) -> EntityMarkdown:
             absolute_path = self.base_path / path
 
         # Parse frontmatter and content using python-frontmatter
-        file_content = absolute_path.read_text()
+        file_content = absolute_path.read_text(encoding="utf-8")
         return await self.parse_file_content(absolute_path, file_content)
 
     async def parse_file_content(self, absolute_path, file_content):

src/basic_memory/sync/sync_service.py

@@ -1,9 +1,8 @@
 """Service for syncing files between filesystem and database."""
 
 import os
-
-from dataclasses import dataclass
-from dataclasses import field
+import time
+from dataclasses import dataclass, field
 from datetime import datetime
 from pathlib import Path
 from typing import Dict, Optional, Set, Tuple
@@ -18,7 +17,6 @@
 from basic_memory.repository import EntityRepository, RelationRepository
 from basic_memory.services import EntityService, FileService
 from basic_memory.services.search_service import SearchService
-import time
 
 
 @dataclass
@@ -237,7 +235,7 @@ async def sync_markdown_file(self, path: str, new: bool = True) -> Tuple[Optiona
         logger.debug(f"Parsing markdown file, path: {path}, new: {new}")
 
         file_path = self.entity_parser.base_path / path
-        file_content = file_path.read_text()
+        file_content = file_path.read_text(encoding="utf-8")
         file_contains_frontmatter = has_frontmatter(file_content)
 
         # entity markdown will always contain front matter, so it can be used up create/update the entity

tests/api/test_resource_router.py

@@ -2,9 +2,9 @@
 
 import json
 from datetime import datetime, timezone
+from pathlib import Path
 
 import pytest
-from pathlib import Path
 
 from basic_memory.schemas import EntityResponse
 
@@ -346,7 +346,7 @@ async def test_put_resource_new_file(client, test_config, entity_repository, sea
     assert full_path.exists()
 
     # Verify file content
-    file_content = full_path.read_text()
+    file_content = full_path.read_text(encoding="utf-8")
     assert json.loads(file_content) == canvas_data
 
     # Verify entity was created in DB
@@ -420,7 +420,7 @@ async def test_put_resource_update_existing(client, test_config, entity_reposito
     assert response.status_code == 200
 
     # Verify file was updated
-    updated_content = full_path.read_text()
+    updated_content = full_path.read_text(encoding="utf-8")
     assert json.loads(updated_content) == updated_data
 
     # Verify entity was updated

tests/cli/test_import_chatgpt.py

@@ -1,10 +1,11 @@
 """Tests for import_chatgpt command."""
 
 import json
+
 import pytest
 from typer.testing import CliRunner
 
-from basic_memory.cli.app import import_app, app
+from basic_memory.cli.app import app, import_app
 from basic_memory.cli.commands import import_chatgpt
 from basic_memory.config import config
 from basic_memory.markdown import EntityParser, MarkdownProcessor
@@ -144,7 +145,7 @@ def sample_conversation_with_hidden():
 def sample_chatgpt_json(tmp_path, sample_conversation):
     """Create a sample ChatGPT JSON file."""
     json_file = tmp_path / "conversations.json"
-    with open(json_file, "w") as f:
+    with open(json_file, "w", encoding="utf-8") as f:
         json.dump([sample_conversation], f)
     return json_file
 
@@ -167,7 +168,7 @@ async def test_process_chatgpt_json(tmp_path, sample_chatgpt_json):
     assert conv_path.exists()
 
     # Check content formatting
-    content = conv_path.read_text()
+    content = conv_path.read_text(encoding="utf-8")
     assert "# Test Conversation" in content
     assert "### User" in content
     assert "Hello, this is a test message" in content
@@ -183,14 +184,14 @@ async def test_process_code_blocks(tmp_path, sample_conversation_with_code):
 
     # Create test file
     json_file = tmp_path / "code_test.json"
-    with open(json_file, "w") as f:
+    with open(json_file, "w", encoding="utf-8") as f:
         json.dump([sample_conversation_with_code], f)
 
     await import_chatgpt.process_chatgpt_json(json_file, tmp_path, processor)
 
     # Check content
     conv_path = tmp_path / "20250111-code-test.md"
-    content = conv_path.read_text()
+    content = conv_path.read_text(encoding="utf-8")
     assert "```python" in content
     assert "def hello():" in content
     assert "```" in content
@@ -204,7 +205,7 @@ async def test_hidden_messages(tmp_path, sample_conversation_with_hidden):
 
     # Create test file
     json_file = tmp_path / "hidden_test.json"
-    with open(json_file, "w") as f:
+    with open(json_file, "w", encoding="utf-8") as f:
         json.dump([sample_conversation_with_hidden], f)
 
     results = await import_chatgpt.process_chatgpt_json(json_file, tmp_path, processor)
@@ -214,7 +215,7 @@ async def test_hidden_messages(tmp_path, sample_conversation_with_hidden):
 
     # Check content
     conv_path = tmp_path / "20250111-hidden-test.md"
-    content = conv_path.read_text()
+    content = conv_path.read_text(encoding="utf-8")
     assert "Visible message" in content
     assert "Hidden message" not in content
 

tests/cli/test_import_claude_conversations.py

@@ -1,6 +1,7 @@
 """Tests for import_claude command (chat conversations)."""
 
 import json
+
 import pytest
 from typer.testing import CliRunner
 
@@ -44,7 +45,7 @@ def sample_conversation():
 def sample_conversations_json(tmp_path, sample_conversation):
     """Create a sample conversations.json file."""
     json_file = tmp_path / "conversations.json"
-    with open(json_file, "w") as f:
+    with open(json_file, "w", encoding="utf-8") as f:
         json.dump([sample_conversation], f)
     return json_file
 
@@ -65,7 +66,7 @@ async def test_process_chat_json(tmp_path, sample_conversations_json):
     # Check conversation file
     conv_path = tmp_path / "20250105-test-conversation.md"
     assert conv_path.exists()
-    content = conv_path.read_text()
+    content = conv_path.read_text(encoding="utf-8")
 
     # Check content formatting
     assert "### Human" in content
@@ -156,7 +157,7 @@ def test_import_conversation_with_attachments(tmp_path):
     }
 
     json_file = tmp_path / "with_attachments.json"
-    with open(json_file, "w") as f:
+    with open(json_file, "w", encoding="utf-8") as f:
         json.dump([conversation], f)
 
     # Set up environment
@@ -168,7 +169,7 @@ def test_import_conversation_with_attachments(tmp_path):
 
     # Check attachment formatting
     conv_path = tmp_path / "conversations/20250105-test-with-attachments.md"
-    content = conv_path.read_text()
+    content = conv_path.read_text(encoding="utf-8")
     assert "**Attachment: test.txt**" in content
     assert "```" in content
     assert "Test file content" in content

tests/cli/test_import_claude_projects.py

@@ -1,6 +1,7 @@
 """Tests for import_claude_projects command."""
 
 import json
+
 import pytest
 from typer.testing import CliRunner
 
@@ -43,7 +44,7 @@ def sample_project():
 def sample_projects_json(tmp_path, sample_project):
     """Create a sample projects.json file."""
     json_file = tmp_path / "projects.json"
-    with open(json_file, "w") as f:
+    with open(json_file, "w", encoding="utf-8") as f:
         json.dump([sample_project], f)
     return json_file
 
@@ -70,14 +71,14 @@ async def test_process_projects_json(tmp_path, sample_projects_json):
     # Check document files
     doc1 = project_dir / "docs/test-document.md"
     assert doc1.exists()
-    content1 = doc1.read_text()
+    content1 = doc1.read_text(encoding="utf-8")
     assert "# Test Document" in content1
     assert "This is test content" in content1
 
     # Check prompt template
     prompt = project_dir / "prompt-template.md"
     assert prompt.exists()
-    prompt_content = prompt.read_text()
+    prompt_content = prompt.read_text(encoding="utf-8")
     assert "# Test Prompt" in prompt_content
     assert "This is a test prompt" in prompt_content
 
@@ -160,7 +161,7 @@ def test_import_project_without_prompt(tmp_path):
     }
 
     json_file = tmp_path / "no_prompt.json"
-    with open(json_file, "w") as f:
+    with open(json_file, "w", encoding="utf-8") as f:
         json.dump([project], f)
 
     # Set up environment

tests/cli/test_import_memory_json.py

@@ -1,6 +1,7 @@
 """Tests for import_memory_json command."""
 
 import json
+
 import pytest
 from typer.testing import CliRunner
 
@@ -35,7 +36,7 @@ def sample_entities():
 def sample_json_file(tmp_path, sample_entities):
     """Create a sample memory.json file."""
     json_file = tmp_path / "memory.json"
-    with open(json_file, "w") as f:
+    with open(json_file, "w", encoding="utf-8") as f:
         for entity in sample_entities:
             f.write(json.dumps(entity) + "\n")
     return json_file
@@ -55,7 +56,7 @@ async def test_process_memory_json(tmp_path, sample_json_file):
     # Check file was created
     entity_file = tmp_path / "test/test_entity.md"
     assert entity_file.exists()
-    content = entity_file.read_text()
+    content = entity_file.read_text(encoding="utf-8")
     assert "Test observation 1" in content
     assert "Test observation 2" in content
     assert "test_relation [[related_entity]]" in content
@@ -120,7 +121,7 @@ def test_import_json_command_handle_old_format(tmp_path):
     ]
 
     json_file = tmp_path / "old_format.json"
-    with open(json_file, "w") as f:
+    with open(json_file, "w", encoding="utf-8") as f:
         for item in old_format:
             f.write(json.dumps(item) + "\n")
 

tests/markdown/test_markdown_processor.py

@@ -8,10 +8,10 @@
 
 import pytest
 
-from basic_memory.markdown.markdown_processor import MarkdownProcessor, DirtyFileError
+from basic_memory.markdown.markdown_processor import DirtyFileError, MarkdownProcessor
 from basic_memory.markdown.schemas import (
-    EntityMarkdown,
     EntityFrontmatter,
+    EntityMarkdown,
     Observation,
     Relation,
 )
@@ -41,7 +41,7 @@ async def test_write_new_minimal_file(markdown_processor: MarkdownProcessor, tmp
     await markdown_processor.write_file(path, markdown)
 
     # Read back and verify
-    content = path.read_text()
+    content = path.read_text(encoding="utf-8")
     assert "---" in content  # Has frontmatter
     assert "type: note" in content
     assert "permalink: test" in content
@@ -90,7 +90,7 @@ async def test_write_new_file_with_content(markdown_processor: MarkdownProcessor
     await markdown_processor.write_file(path, markdown)
 
     # Read back and verify
-    content = path.read_text()
+    content = path.read_text(encoding="utf-8")
 
     # Check content preserved exactly
     assert "# Custom Title" in content
@@ -169,7 +169,7 @@ async def test_dirty_file_detection(markdown_processor: MarkdownProcessor, tmp_p
     checksum = await markdown_processor.write_file(path, initial)
 
     # Modify file directly
-    path.write_text(path.read_text() + "\nModified!")
+    path.write_text(path.read_text(encoding="utf-8") + "\nModified!")
 
     # Try to update with old checksum
     update = EntityMarkdown(