Skip to content

Commit f13742a

Browse files
committed
feat(core): add note_content tenant schema primitive
- add the shared note_content model, migration, and repository - cover CRUD, migration, and Postgres-backed repository behavior - fix the Alembic async fallback coroutine warning Signed-off-by: phernandez <paul@basicmachines.co>
1 parent 367fcaa commit f13742a

File tree

9 files changed

+771
-25
lines changed

9 files changed

+771
-25
lines changed

src/basic_memory/alembic/env.py

Lines changed: 52 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,54 @@ async def run_async_migrations(connectable):
118118
await connectable.dispose()
119119

120120

121+
def _run_async_migrations_with_asyncio_run(connectable) -> None:
122+
"""Run async migrations with asyncio.run while closing failed coroutines.
123+
124+
Trigger: asyncio.run() may reject execution when another event loop is already active.
125+
Why: Python raises before awaiting the coroutine, which otherwise leaks a
126+
RuntimeWarning about an un-awaited coroutine.
127+
Outcome: close the pending coroutine before bubbling the RuntimeError to the
128+
fallback path.
129+
"""
130+
migration_coro = run_async_migrations(connectable)
131+
try:
132+
asyncio.run(migration_coro)
133+
except RuntimeError:
134+
migration_coro.close()
135+
raise
136+
137+
138+
def _run_async_migrations_in_thread(connectable) -> None:
139+
"""Run async migrations in a dedicated thread with its own event loop."""
140+
import concurrent.futures
141+
142+
def run_in_thread():
143+
"""Run async migrations in a new event loop in a separate thread."""
144+
new_loop = asyncio.new_event_loop()
145+
asyncio.set_event_loop(new_loop)
146+
try:
147+
new_loop.run_until_complete(run_async_migrations(connectable))
148+
finally:
149+
new_loop.close()
150+
151+
with concurrent.futures.ThreadPoolExecutor() as executor:
152+
future = executor.submit(run_in_thread)
153+
future.result() # Wait for completion and re-raise any exceptions
154+
155+
156+
def _run_async_engine_migrations(connectable) -> None:
157+
"""Run async-engine migrations with a running-loop fallback."""
158+
try:
159+
_run_async_migrations_with_asyncio_run(connectable)
160+
except RuntimeError as e:
161+
if "cannot be called from a running event loop" in str(e):
162+
# We're in a running event loop (likely uvloop or Python 3.14+ tests).
163+
# Switch to a dedicated thread so Alembic can finish without nesting loops.
164+
_run_async_migrations_in_thread(connectable)
165+
else:
166+
raise
167+
168+
121169
def run_migrations_online() -> None:
122170
"""Run migrations in 'online' mode.
123171
@@ -148,30 +196,10 @@ def run_migrations_online() -> None:
148196

149197
# Handle async engines (PostgreSQL with asyncpg)
150198
if isinstance(connectable, AsyncEngine):
151-
# Try to run async migrations
152-
# nest_asyncio allows asyncio.run() from within event loops, but doesn't work with uvloop
153-
try:
154-
asyncio.run(run_async_migrations(connectable))
155-
except RuntimeError as e:
156-
if "cannot be called from a running event loop" in str(e):
157-
# We're in a running event loop (likely uvloop) - need to use a different approach
158-
# Create a new thread to run the async migrations
159-
import concurrent.futures
160-
161-
def run_in_thread():
162-
"""Run async migrations in a new event loop in a separate thread."""
163-
new_loop = asyncio.new_event_loop()
164-
asyncio.set_event_loop(new_loop)
165-
try:
166-
new_loop.run_until_complete(run_async_migrations(connectable))
167-
finally:
168-
new_loop.close()
169-
170-
with concurrent.futures.ThreadPoolExecutor() as executor:
171-
future = executor.submit(run_in_thread)
172-
future.result() # Wait for completion and re-raise any exceptions
173-
else:
174-
raise
199+
# Trigger: async engines need Alembic work to cross the sync/async boundary.
200+
# Why: most callers can use asyncio.run(), but running-loop contexts need a thread fallback.
201+
# Outcome: migrations complete without leaking un-awaited coroutines.
202+
_run_async_engine_migrations(connectable)
175203
else:
176204
# Handle sync engines (SQLite) or sync connections
177205
if hasattr(connectable, "connect"):
Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
"""Add note_content table
2+
3+
Revision ID: l5g6h7i8j9k0
4+
Revises: k4e5f6g7h8i9
5+
Create Date: 2026-04-04 12:00:00.000000
6+
7+
"""
8+
9+
from typing import Sequence, Union
10+
11+
import sqlalchemy as sa
12+
from alembic import op
13+
14+
15+
# revision identifiers, used by Alembic.
16+
revision: str = "l5g6h7i8j9k0"
17+
down_revision: Union[str, None] = "k4e5f6g7h8i9"
18+
branch_labels: Union[str, Sequence[str], None] = None
19+
depends_on: Union[str, Sequence[str], None] = None
20+
21+
22+
def upgrade() -> None:
23+
"""Create note_content for materialized note content and sync state."""
24+
op.create_table(
25+
"note_content",
26+
sa.Column("entity_id", sa.Integer(), nullable=False),
27+
sa.Column("project_id", sa.Integer(), nullable=False),
28+
sa.Column("external_id", sa.String(), nullable=False),
29+
sa.Column("file_path", sa.String(), nullable=False),
30+
sa.Column("markdown_content", sa.Text(), nullable=False),
31+
sa.Column("db_version", sa.BigInteger(), nullable=False),
32+
sa.Column("db_checksum", sa.String(), nullable=False),
33+
sa.Column("file_version", sa.BigInteger(), nullable=True),
34+
sa.Column("file_checksum", sa.String(), nullable=True),
35+
sa.Column("file_write_status", sa.String(), nullable=False),
36+
sa.Column("last_source", sa.String(), nullable=True),
37+
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
38+
sa.Column("file_updated_at", sa.DateTime(timezone=True), nullable=True),
39+
sa.Column("last_materialization_error", sa.Text(), nullable=True),
40+
sa.Column("last_materialization_attempt_at", sa.DateTime(timezone=True), nullable=True),
41+
sa.CheckConstraint(
42+
"file_write_status IN ("
43+
"'pending', "
44+
"'writing', "
45+
"'synced', "
46+
"'failed', "
47+
"'external_change_detected'"
48+
")",
49+
name="ck_note_content_file_write_status",
50+
),
51+
sa.ForeignKeyConstraint(["entity_id"], ["entity.id"], ondelete="CASCADE"),
52+
sa.ForeignKeyConstraint(["project_id"], ["project.id"], ondelete="CASCADE"),
53+
sa.PrimaryKeyConstraint("entity_id"),
54+
)
55+
op.create_index("ix_note_content_project_id", "note_content", ["project_id"], unique=False)
56+
op.create_index("ix_note_content_file_path", "note_content", ["file_path"], unique=False)
57+
op.create_index("ix_note_content_external_id", "note_content", ["external_id"], unique=True)
58+
59+
60+
def downgrade() -> None:
61+
"""Drop note_content and its supporting indexes."""
62+
op.drop_index("ix_note_content_external_id", table_name="note_content")
63+
op.drop_index("ix_note_content_file_path", table_name="note_content")
64+
op.drop_index("ix_note_content_project_id", table_name="note_content")
65+
op.drop_table("note_content")

src/basic_memory/models/__init__.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,13 @@
22

33
import basic_memory
44
from basic_memory.models.base import Base
5-
from basic_memory.models.knowledge import Entity, Observation, Relation
5+
from basic_memory.models.knowledge import Entity, NoteContent, Observation, Relation
66
from basic_memory.models.project import Project
77

88
__all__ = [
99
"Base",
1010
"Entity",
11+
"NoteContent",
1112
"Observation",
1213
"Relation",
1314
"Project",

src/basic_memory/models/knowledge.py

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
from typing import Optional
77

88
from sqlalchemy import (
9+
BigInteger,
10+
CheckConstraint,
911
Integer,
1012
String,
1113
Text,
@@ -116,6 +118,12 @@ class Entity(Base):
116118
foreign_keys="[Relation.to_id]",
117119
cascade="all, delete-orphan",
118120
)
121+
note_content = relationship(
122+
"NoteContent",
123+
back_populates="entity",
124+
cascade="all, delete-orphan",
125+
uselist=False,
126+
)
119127

120128
@property
121129
def relations(self):
@@ -141,6 +149,74 @@ def __repr__(self) -> str:
141149
return f"Entity(id={self.id}, external_id='{self.external_id}', name='{self.title}', type='{self.note_type}', checksum='{self.checksum}')"
142150

143151

152+
class NoteContent(Base):
153+
"""Materialized markdown content and sync state for a note entity."""
154+
155+
__tablename__ = "note_content"
156+
__table_args__ = (
157+
CheckConstraint(
158+
"file_write_status IN ("
159+
"'pending', "
160+
"'writing', "
161+
"'synced', "
162+
"'failed', "
163+
"'external_change_detected'"
164+
")",
165+
name="ck_note_content_file_write_status",
166+
),
167+
Index("ix_note_content_project_id", "project_id"),
168+
Index("ix_note_content_file_path", "file_path"),
169+
Index("ix_note_content_external_id", "external_id", unique=True),
170+
)
171+
172+
# Core identity mirrored from entity for hot note reads
173+
entity_id: Mapped[int] = mapped_column(
174+
Integer,
175+
ForeignKey("entity.id", ondelete="CASCADE"),
176+
primary_key=True,
177+
)
178+
project_id: Mapped[int] = mapped_column(
179+
Integer,
180+
ForeignKey("project.id", ondelete="CASCADE"),
181+
nullable=False,
182+
)
183+
external_id: Mapped[str] = mapped_column(String, nullable=False)
184+
file_path: Mapped[str] = mapped_column(String, nullable=False)
185+
186+
# Materialized content version tracked in the tenant database
187+
markdown_content: Mapped[str] = mapped_column(Text, nullable=False)
188+
db_version: Mapped[int] = mapped_column(BigInteger, nullable=False)
189+
db_checksum: Mapped[str] = mapped_column(String, nullable=False)
190+
191+
# File materialization state tracked against the latest write attempts
192+
file_version: Mapped[Optional[int]] = mapped_column(BigInteger, nullable=True)
193+
file_checksum: Mapped[Optional[str]] = mapped_column(String, nullable=True)
194+
file_write_status: Mapped[str] = mapped_column(String, nullable=False, default="pending")
195+
last_source: Mapped[Optional[str]] = mapped_column(String, nullable=True)
196+
updated_at: Mapped[datetime] = mapped_column(
197+
DateTime(timezone=True),
198+
default=lambda: datetime.now().astimezone(),
199+
onupdate=lambda: datetime.now().astimezone(),
200+
)
201+
file_updated_at: Mapped[Optional[datetime]] = mapped_column(
202+
DateTime(timezone=True),
203+
nullable=True,
204+
)
205+
last_materialization_error: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
206+
last_materialization_attempt_at: Mapped[Optional[datetime]] = mapped_column(
207+
DateTime(timezone=True),
208+
nullable=True,
209+
)
210+
211+
entity = relationship("Entity", back_populates="note_content")
212+
213+
def __repr__(self) -> str: # pragma: no cover
214+
return (
215+
f"NoteContent(entity_id={self.entity_id}, external_id='{self.external_id}', "
216+
f"file_path='{self.file_path}', file_write_status='{self.file_write_status}')"
217+
)
218+
219+
144220
class Observation(Base):
145221
"""An observation about an entity.
146222

src/basic_memory/repository/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
from .entity_repository import EntityRepository
2+
from .note_content_repository import NoteContentRepository
23
from .observation_repository import ObservationRepository
34
from .project_repository import ProjectRepository
45
from .relation_repository import RelationRepository
56

67
__all__ = [
78
"EntityRepository",
9+
"NoteContentRepository",
810
"ObservationRepository",
911
"ProjectRepository",
1012
"RelationRepository",

0 commit comments

Comments
 (0)