🥦 [GH-83] Split up celery beat and worker options (#85)

* chore: split celery beat/worker options [#83]

* chore: bump packages and include redis extra [#83]

* docs: update documentation [#83]

* chore: drop fastapi-cache2 [#83]

* chore: lint [#83]

* chore: more lint [#83]

* chore: more more lint [#83]

* meh

Author: nickatnight

.github/workflows/test.yml

@@ -48,8 +48,10 @@ jobs:
             args: ""
           - name: Deployments[render], Example Api, py3.11, & celery[yes]
             args: "deployments='render' py_version=3.11 use_celery='yes'"
-          - name: Deployments[render], Example Api, py3.9, celery[yes], & sentry[yes]
-            args: "deployments='render' py_version=3.9 use_celery='yes' use_sentry='yes'"
+          - name: Deployments[digitalocean], Example Api, py3.9, celery[yes], & sentry[yes]
+            args: "deployments='digitalocean' py_version=3.9 use_celery='yes' use_sentry='yes'"
+          - name: Deployments[render], Example Api, py3.10, celery[yes], beat[yes], & sentry[no]
+            args: "deployments='digitalocean' py_version=3.9 use_celery='yes' periodic_tasks='yes' use_sentry='no'"
     name: "Docker ${{ matrix.script.name }}"
     runs-on: ubuntu-latest
     env:

cookiecutter.json

@@ -10,8 +10,17 @@
     "db_container_name": "db",
     "backend_container_name": "backend",
     "use_celery": ["no", "yes"],
+    "periodic_tasks": ["no", "yes"],
     "use_sentry": ["no", "yes"],
     "github_username": "change.me",
 
-    "deployments": ["none", "render", "digitalocean"]
+    "deployments": ["none", "render", "digitalocean"],
+
+    "__prompts__": {
+        "use_celery": "Do you want to enable Celery for background tasks?",
+        "periodic_tasks": "Do you want to enable Celery Beat for periodic tasks (requires Celery)?",
+        "use_sentry": "Do you want to enable Sentry for error tracking?",
+        "github_username": "What is your GitHub username?",
+        "deployments": "Do you want to enable deployment to a platform-as-a-service (PaaS) provider?"
+    }
 }

docs/source/project_features/celery.rst

@@ -1,8 +1,65 @@
 Asynchronous Tasks with Celery
 ==============================
 
-The project uses Celery for asynchronous tasks, with Celery beat for task scheduling.
+The project uses Celery for asynchronous tasks, with Celery beat for task scheduling, and Redis is used as the broker by default.
 
-Redis is used as the broker by default.
+.. note::
 
-You can read more here on how to get started with `Celery <https://docs.celeryq.dev/en/stable/getting-started/first-steps-with-celery.html>`_
+   You can read more here on how to get started with `Celery <https://docs.celeryq.dev/en/stable/getting-started/first-steps-with-celery.html>`_
+
+FastAPI comes with built-in `BackgroundTasks <https://fastapi.tiangolo.com/tutorial/background-tasks/>`_, however, they run in a single
+process. This won't work well for more complex tasks, especially at scale. The Celery worker and beat service can scale independently of
+the FastAPI application, which is how typical async applications are deployed.
+
+Configuring Celery Workers
+--------------------------
+
+Celery workers can be configured to run on a specific number of workers. This is useful for scaling the application to handle more requests.
+
+.. code-block:: python
+
+  ...
+
+  @router.get("/ping", tags=["health"])
+    def pong() -> Response:
+        hello_world_task.delay()
+        # or a task with arguments
+        my_task.delay(id=1)
+        return JSONResponse({"ping": "pong!"})
+
+You can read more about the different application settings `here <https://docs.celeryq.dev/en/v5.2.7/userguide/application.html#application>`_.
+
+
+Configuring Celery Beat
+-----------------------
+
+Celery beat is used to schedule periodic tasks. It can be configured to run on a schedule, or to run on a fixed time interval.
+
+.. code-block:: python
+
+  # worker.py
+
+   from celery.schedules import crontab
+
+   ...
+   app.conf.beat_schedule = {
+        # Executes every Monday morning at 7:30 a.m.
+        'add-every-monday-morning': {
+            'task': 'src.core.tasks.add',
+            'schedule': crontab(hour=7, minute=30, day_of_week=1),
+            'args': (16, 16),
+        },
+        # Executes every 3rd minute
+        'hello-every-third-minute': {
+            'task': 'src.core.tasks.hello_world_task',
+            'schedule': crontab(minute="*/3"),
+        },
+        # Executes at sunset in Melbourne
+        'add-at-melbourne-sunset': {
+            'task': 'src.core.tasks.melbourne_add',
+            'schedule': solar('sunset', -37.81753, 144.96715),
+            'args': (16, 16),
+        },
+    }
+
+You can read more about the different schedule `types <https://docs.celeryq.dev/en/v5.2.7/userguide/periodic-tasks.html#crontab-schedules>`_.

hooks/pre_gen_project.py

@@ -15,3 +15,13 @@
 
     # Exit to cancel project
     sys.exit(1)
+
+
+if (
+    "{{ cookiecutter.use_celery}}" == "no"
+    and "{{ cookiecutter.periodic_tasks}}" == "yes"
+):
+    print(
+        "ERROR: Celery is not enabled, but periodic tasks are requested. Please enable Celery."
+    )
+    sys.exit(1)

tests/test_bake_project.py

@@ -5,6 +5,7 @@
 
 import pytest
 from binaryornot.check import is_binary
+from cookiecutter.exceptions import FailedHookException
 
 try:
     import sh
@@ -21,14 +22,18 @@
     {"deployments": "none"},
     {"deployments": "render"},
     {"deployments": "digitalocean"},
-    {"use_celery": "no"},
-    {"use_celery": "yes"},
+    {"use_celery": "no", "periodic_tasks": "no"},
+    {"use_celery": "yes", "periodic_tasks": "no"},
+    {"use_celery": "yes", "periodic_tasks": "yes"},
     {"py_version": "3.9"},
     {"py_version": "3.10"},
     {"py_version": "3.11"},
     {"use_sentry": "no"},
     {"use_sentry": "yes"},
 ]
+UNSUPPORTED_COMBINATIONS = [
+    {"use_celery": "no", "periodic_tasks": "yes"},
+]
 PATTERN = r"{{(\s?cookiecutter)[.](.*?)}}"
 RE_OBJ = re.compile(PATTERN)
 
@@ -88,6 +93,27 @@ def test_project_generation(cookies, context, context_override) -> None:
     check_paths(paths)
 
 
+@pytest.mark.parametrize("slug", ["project slug", "Project_Slug"])
+def test_invalid_slug(cookies, context, slug):
+    """Invalid slug should fail pre-generation hook."""
+    context.update({"project_slug": slug})
+
+    result = cookies.bake(extra_context=context)
+
+    assert result.exit_code != 0
+    assert isinstance(result.exception, FailedHookException)
+
+
+@pytest.mark.parametrize("invalid_context", UNSUPPORTED_COMBINATIONS)
+def test_error_if_incompatible(cookies, context, invalid_context):
+    """It should not generate project an incompatible combination is selected."""
+    context.update(invalid_context)
+    result = cookies.bake(extra_context=context)
+
+    assert result.exit_code != 0
+    assert isinstance(result.exception, FailedHookException)
+
+
 @pytest.mark.parametrize("context_override", SUPPORTED_COMBINATIONS, ids=_fixture_id)
 def test_pre_commit_hooks(cookies, context_override) -> None:
     """Generated project pre-commit hooks run successfully."""

{{ cookiecutter.project_slug }}/docker-compose.yml

@@ -48,9 +48,7 @@ services:
 {%- if cookiecutter.use_celery == "yes" %}
   redis:
     restart: always
-    image: redis:latest
-    ports:
-      - "6479:6379"
+    image: redis:7-alpine
     volumes:
       - redis-data:/data
 
@@ -63,7 +61,8 @@ services:
     depends_on:
       - {{ cookiecutter.backend_container_name }}
       - redis
-
+{%- endif %}
+{%- if cookiecutter.periodic_tasks == "yes" %}
   beat:
     restart: always
     build: ./{{ cookiecutter.backend_container_name }}

{{ cookiecutter.project_slug }}/{{ cookiecutter.backend_container_name }}/pyproject.toml

@@ -14,10 +14,12 @@ dependencies = [
     "fastapi==0.116.1",
     "pydantic-settings==2.10.1",
     "sqlmodel==0.0.24",
-    {%- if cookiecutter.use_celery == "yes" %}
-    "redis==4.6.0",
-    "fastapi-cache2[redis]==0.2.2",
-    "celery[beat]==5.2.7",{%- endif %}
+    {%- if cookiecutter.use_celery == "yes" and cookiecutter.periodic_tasks == "yes" %}
+    "redis==6.2.0",
+    "celery[beat,redis]==5.2.7",
+    {%- elif cookiecutter.use_celery == "yes" %}
+    "redis==6.2.0",
+    "celery[redis]==5.2.7",{%- endif %}
     "uvicorn[standard]==0.35.0",
     "PyYAML==6.0.1",
     "httpx==0.28.1",

{{ cookiecutter.project_slug }}/{{ cookiecutter.backend_container_name }}/src/api/deps.py

@@ -1,3 +1,5 @@
+from typing import cast
+
 from redis import Redis
 
 from src.core.config import settings
@@ -14,4 +16,4 @@ def get_redis_client() -> Redis:
         encoding="utf8",
         decode_responses=True,
     )
-    return redis
+    return cast(Redis, redis)

{{ cookiecutter.project_slug }}/{{ cookiecutter.backend_container_name }}/src/core/tasks.py

@@ -0,0 +1,6 @@
+from src.worker import celery_app
+
+
+@celery_app.task  # type: ignore
+def hello_world_task() -> None:
+    print("Hello World")

{{ cookiecutter.project_slug }}/{{ cookiecutter.backend_container_name }}/src/main.py

@@ -3,13 +3,8 @@
 import sentry_sdk{% endif %}
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
-{%- if cookiecutter.use_celery == "yes" %}
-from fastapi_cache import FastAPICache
-from fastapi_cache.backends.redis import RedisBackend{%- endif %}
 
 from src.api import routes
-{%- if cookiecutter.use_celery == "yes" %}
-from src.api.deps import get_redis_client{%- endif %}
 from src.core.config import settings
 from src.db.session import add_postgresql_extension
 
@@ -35,9 +30,6 @@
 
 def on_startup() -> None:
     add_postgresql_extension()
-    {%- if cookiecutter.use_celery == "yes" %}
-    redis_client = get_redis_client()
-    FastAPICache.init(RedisBackend(redis_client), prefix="fastapi-cache"){%- endif %}
     {%- if cookiecutter.use_sentry == "yes" %}
     if settings.SENTRY_DSN:
         sentry_sdk.init(