Hotfix/many to many updates (#22)

* WIP: Fix many-to-many relationships updates

* WIP: Clean up the SQL CRUD methods

* WIP: Enable nested updates in SQL

* WIP: Clean up the SQL update method

* Fix 'AttributeError: 'int' object has no attribute 'id'' with SQL update

* Fix failing tests

* Fix dependency errors in python 3.13

* Fix failing CI tests for examples

* Fix 'TypeError: field 'TodoList.description' was not initialized with a Field() or Relationship()'

* Add tests for blog example in github CI

Author: Tinitto

.github/workflows/ci.yml

@@ -69,7 +69,7 @@ jobs:
     strategy:
       matrix:
         python-version: [ "3.10", "3.11", "3.12", "3.13" ]
-        example_app: ["todos"]
+        example_app: ["todos", "blog"]
 
     services:
       mongodb:

.gitignore

@@ -135,6 +135,7 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+.venv3_13
 
 # Spyder project settings
 .spyderproject

CONTRIBUTING.md

@@ -77,7 +77,7 @@ By contributing, you agree that your contributions will be licensed under its MI
 - Install the dependencies
 
   ```bash
-  pip install -r requirements.txt
+  pip install -r ."[all,test]"
   ```
 
 - Run the pre-commit installation

LIMITATIONS.md

@@ -0,0 +1,15 @@
+# Limitations
+
+## Filtering
+
+### Redis
+
+- Mongo-style regular expression filtering is not supported. 
+  This is because native redis regular expression filtering is limited to the most basic text based search.
+
+## Update Operation
+
+### SQL 
+
+- Even though one can update a model to theoretically infinite number of levels deep,
+  the returned results can only contain 1-level-deep nested models and no more.

README.md

@@ -336,6 +336,11 @@ libraries = await redis_store.delete(
 
 - [ ] Add documentation site
 
+## Limitations
+
+This library is limited in some specific cases.
+Read through the [`LIMITATIONS.md`](./LIMITATIONS.md) file for more.
+
 ## Contributions
 
 Contributions are welcome. The docs have to maintained, the code has to be made cleaner, more idiomatic and faster,

examples/blog/conftest.py

@@ -7,7 +7,7 @@
 import pytest_asyncio
 import pytest_mock
 from fastapi.testclient import TestClient
-from models import (  # SqlAuthor,
+from models import (
     MongoAuthor,
     MongoComment,
     MongoInternalAuthor,

examples/blog/main.py

@@ -13,7 +13,7 @@
 from fastapi.security import OAuth2PasswordRequestForm
 from models import MongoPost, RedisPost, SqlInternalAuthor, SqlPost
 from pydantic import BaseModel
-from schemas import InternalAuthor, Post, TokenResponse
+from schemas import InternalAuthor, PartialPost, Post, TokenResponse
 from stores import MongoStoreDep, RedisStoreDep, SqlStoreDep, clear_stores
 
 _ACCESS_TOKEN_EXPIRE_MINUTES = 30
@@ -112,8 +112,13 @@ async def search(
 
         if redis:
             # redis's regex search is not mature so we use its full text search
+            # Unfortunately, redis search does not permit us to search fields that are arrays.
             redis_query = [
-                (_get_redis_field(RedisPost, k) % f"*{v}*")
+                (
+                    (_get_redis_field(RedisPost, k) == f"{v}")
+                    if k == "tags.title"
+                    else (_get_redis_field(RedisPost, k) % f"*{v}*")
+                )
                 for k, v in query_dict.items()
             ]
             results += await redis.find(RedisPost, *redis_query)
@@ -194,7 +199,7 @@ async def update_one(
     mongo: MongoStoreDep,
     current_user: CurrentUserDep,
     id_: int | str,
-    payload: Post,
+    payload: PartialPost,
 ):
     """Update a post"""
     results = []

examples/blog/models.py

@@ -21,7 +21,7 @@
     "MongoPost",
     Post,
     embedded_models={
-        "author": MongoAuthor,
+        "author": MongoAuthor | None,
         "comments": list[MongoComment],
         "tags": list[MongoTag],
     },
@@ -39,15 +39,14 @@
     "RedisPost",
     Post,
     embedded_models={
-        "author": RedisAuthor,
+        "author": RedisAuthor | None,
         "comments": list[RedisComment],
         "tags": list[RedisTag],
     },
 )
 
 # sqlite models
 SqlInternalAuthor = SQLModel("SqlInternalAuthor", InternalAuthor)
-# SqlAuthor = SQLModel("SqlAuthor", Author, table=False)
 SqlComment = SQLModel(
     "SqlComment", Comment, relationships={"author": SqlInternalAuthor | None}
 )

examples/blog/schemas.py

@@ -3,7 +3,7 @@
 from datetime import datetime
 
 from pydantic import BaseModel
-from utils import current_timestamp
+from utils import Partial, current_timestamp
 
 from nqlstore import Field, Relationship
 
@@ -33,15 +33,8 @@ class Post(BaseModel):
         disable_on_redis=True,
     )
     author: Author | None = Relationship(default=None)
-    comments: list["Comment"] = Relationship(
-        default=[],
-        disable_on_redis=True,
-    )
-    tags: list["Tag"] = Relationship(
-        default=[],
-        link_model="TagLink",
-        disable_on_redis=True,
-    )
+    comments: list["Comment"] = Relationship(default=[])
+    tags: list["Tag"] = Relationship(default=[], link_model="TagLink")
     created_at: str = Field(index=True, default_factory=current_timestamp)
     updated_at: str = Field(index=True, default_factory=current_timestamp)
 
@@ -89,11 +82,15 @@ class TagLink(BaseModel):
 class Tag(BaseModel):
     """The tags to help searching for posts"""
 
-    title: str = Field(index=True, unique=True, full_text_search=True)
+    title: str = Field(index=True, unique=True)
 
 
 class TokenResponse(BaseModel):
     """HTTP-only response"""
 
     access_token: str
     token_type: str
+
+
+# Partial models
+PartialPost = Partial("PartialPost", Post)

examples/blog/test_main.py

@@ -11,6 +11,7 @@
 
 _TITLE_SEARCH_TERMS = ["ho", "oo", "work"]
 _TAG_SEARCH_TERMS = ["art", "om"]
+_HEADERS = {"Authorization": f"Bearer {ACCESS_TOKEN}"}
 
 
 @pytest.mark.asyncio
@@ -21,9 +22,7 @@ async def test_create_sql_post(
     """POST to /posts creates a post in sql and returns it"""
     timestamp = datetime.now().isoformat()
     with client_with_sql as client:
-        response = client.post(
-            "/posts", json=post, headers={"Authorization": f"Bearer {ACCESS_TOKEN}"}
-        )
+        response = client.post("/posts", json=post, headers=_HEADERS)
 
         got = response.json()
         post_id = got["id"]
@@ -61,12 +60,12 @@ async def test_create_redis_post(
     client_with_redis: TestClient,
     redis_store: RedisStore,
     post: dict,
+    freezer,
 ):
     """POST to /posts creates a post in redis and returns it"""
+    timestamp = datetime.now().isoformat()
     with client_with_redis as client:
-        response = client.post(
-            "/posts", json=post, headers={"Authorization": f"Bearer {ACCESS_TOKEN}"}
-        )
+        response = client.post("/posts", json=post, headers=_HEADERS)
 
         got = response.json()
         post_id = got["id"]
@@ -75,7 +74,8 @@ async def test_create_redis_post(
         expected = {
             "id": post_id,
             "title": post["title"],
-            "content": post.get("content"),
+            "content": post.get("content", ""),
+            "author": {**got["author"], **AUTHOR},
             "pk": post_id,
             "tags": [
                 {
@@ -86,6 +86,8 @@ async def test_create_redis_post(
                 for raw, resp in zip(raw_tags, resp_tags)
             ],
             "comments": [],
+            "created_at": timestamp,
+            "updated_at": timestamp,
         }
 
         db_query = {"id": {"$eq": post_id}}
@@ -102,25 +104,25 @@ async def test_create_mongo_post(
     client_with_mongo: TestClient,
     mongo_store: MongoStore,
     post: dict,
+    freezer,
 ):
     """POST to /posts creates a post in redis and returns it"""
+    timestamp = datetime.now().isoformat()
     with client_with_mongo as client:
-        response = client.post("/posts", json=post)
+        response = client.post("/posts", json=post, headers=_HEADERS)
 
         got = response.json()
         post_id = got["id"]
         raw_tags = post.get("tags", [])
         expected = {
             "id": post_id,
             "title": post["title"],
-            "content": post.get("content"),
-            "tags": [
-                {
-                    **raw,
-                }
-                for raw in raw_tags
-            ],
+            "content": post.get("content", ""),
+            "author": {"name": AUTHOR["name"]},
+            "tags": raw_tags,
             "comments": [],
+            "created_at": timestamp,
+            "updated_at": timestamp,
         }
 
         db_query = {"_id": {"$eq": ObjectId(post_id)}}
@@ -138,22 +140,26 @@ async def test_update_sql_post(
     sql_store: SQLStore,
     sql_posts: list[SqlPost],
     index: int,
+    freezer,
 ):
     """PUT to /posts/{id} updates the sql post of given id and returns updated version"""
+    timestamp = datetime.now().isoformat()
     with client_with_sql as client:
         post = sql_posts[index]
+        post_dict = post.model_dump(mode="json", exclude_none=True, exclude_unset=True)
         id_ = post.id
         update = {
-            "name": "some other name",
-            "todos": [
-                *post.tags,
+            **post_dict,
+            "title": "some other title",
+            "tags": [
+                *post_dict["tags"],
                 {"title": "another one"},
                 {"title": "another one again"},
             ],
-            "comments": [*post.comments, *COMMENT_LIST[index:]],
+            "comments": [*post_dict["comments"], *COMMENT_LIST[index:]],
         }
 
-        response = client.put(f"/posts/{id_}", json=update)
+        response = client.put(f"/posts/{id_}", json=update, headers=_HEADERS)
 
         got = response.json()
         expected = {
@@ -164,8 +170,9 @@ async def test_update_sql_post(
                     **raw,
                     "id": final["id"],
                     "post_id": final["post_id"],
-                    "author": final["author"],
-                    "author_id": final["author_id"],
+                    "author_id": 1,
+                    "created_at": timestamp,
+                    "updated_at": timestamp,
                 }
                 for raw, final in zip(update["comments"], got["comments"])
             ],
@@ -192,22 +199,25 @@ async def test_update_redis_post(
     redis_store: RedisStore,
     redis_posts: list[RedisPost],
     index: int,
+    freezer,
 ):
     """PUT to /posts/{id} updates the redis post of given id and returns updated version"""
+    timestamp = datetime.now().isoformat()
     with client_with_redis as client:
         post = redis_posts[index]
+        post_dict = post.model_dump(mode="json", exclude_none=True, exclude_unset=True)
         id_ = post.id
         update = {
-            "name": "some other name",
-            "todos": [
-                *post.tags,
+            "title": "some other title",
+            "tags": [
+                *post_dict.get("tags", []),
                 {"title": "another one"},
                 {"title": "another one again"},
             ],
-            "comments": [*post.comments, *COMMENT_LIST[index:]],
+            "comments": [*post_dict.get("comments", []), *COMMENT_LIST[index:]],
         }
 
-        response = client.put(f"/posts/{id_}", json=update)
+        response = client.put(f"/posts/{id_}", json=update, headers=_HEADERS)
 
         got = response.json()
         expected = {
@@ -219,6 +229,8 @@ async def test_update_redis_post(
                     "id": final["id"],
                     "author": final["author"],
                     "pk": final["pk"],
+                    "created_at": timestamp,
+                    "updated_at": timestamp,
                 }
                 for raw, final in zip(update["comments"], got["comments"])
             ],
@@ -265,22 +277,25 @@ async def test_update_mongo_post(
     mongo_store: MongoStore,
     mongo_posts: list[MongoPost],
     index: int,
+    freezer,
 ):
     """PUT to /posts/{id} updates the mongo post of given id and returns updated version"""
+    timestamp = datetime.now().isoformat()
     with client_with_mongo as client:
         post = mongo_posts[index]
+        post_dict = post.model_dump(mode="json", exclude_none=True, exclude_unset=True)
         id_ = post.id
         update = {
-            "name": "some other name",
-            "todos": [
-                *post.tags,
+            "title": "some other title",
+            "tags": [
+                *post_dict.get("tags", []),
                 {"title": "another one"},
                 {"title": "another one again"},
             ],
-            "comments": [*post.comments, *COMMENT_LIST[index:]],
+            "comments": [*post_dict.get("comments", []), *COMMENT_LIST[index:]],
         }
 
-        response = client.put(f"/posts/{id_}", json=update)
+        response = client.put(f"/posts/{id_}", json=update, headers=_HEADERS)
 
         got = response.json()
         expected = {
@@ -290,6 +305,8 @@ async def test_update_mongo_post(
                 {
                     **raw,
                     "author": final["author"],
+                    "created_at": timestamp,
+                    "updated_at": timestamp,
                 }
                 for raw, final in zip(update["comments"], got["comments"])
             ],
@@ -538,14 +555,14 @@ async def test_search_sql_by_tag(
 
 
 @pytest.mark.asyncio
-@pytest.mark.parametrize("q", _TAG_SEARCH_TERMS)
+@pytest.mark.parametrize("q", ["random", "another one", "another one again"])
 async def test_search_redis_by_tag(
     client_with_redis: TestClient,
     redis_store: RedisStore,
     redis_posts: list[RedisPost],
     q: str,
 ):
-    """GET /posts?tag={} gets all redis posts with tag containing search item"""
+    """GET /posts?tag={} gets all redis posts with tag containing search item. Partial searches nit supported."""
     with client_with_redis as client:
         response = client.get(f"/posts?tag={q}")