Skip to content

Commit f57413a

Browse files
mike-bestcatmike-bestcat
committed
Worky
1 parent 01159d9 commit f57413a

28 files changed

Lines changed: 377 additions & 37 deletions

File tree

‎infra/self-hosted/compose.yaml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,32 @@
11
name: jupiter
22

3+
x-webapi-cron-env: &webapi-cron-env
4+
SQLITE_DB_URL: ${SQLITE_DB_URL:-sqlite+aiosqlite:////data/jupiter.sqlite}
5+
POSTGRES_DB_URL: ${POSTGRES_DB_URL:-}
6+
ALEMBIC_INI_PATH: ${ALEMBIC_INI_PATH:-../../core/migrations/alembic.sqlite.ini}
7+
ALEMBIC_MIGRATIONS_PATH: ${ALEMBIC_MIGRATIONS_PATH:-../../core/migrations/sqlite}
8+
UNIVERSE: ${UNIVERSE:?Check https://docs.get-thriving.com/how-tos/self-hosting}
9+
ENV: ${ENV:-production}
10+
INSTANCE: ${INSTANCE:?Check https://docs.get-thriving.com/how-tos/self-hosting}
11+
AUTH_TOKEN_SECRET: ${AUTH_TOKEN_SECRET:?Check https://docs.get-thriving.com/how-tos/self-hosting}
12+
WEBAPI_STORAGE_ENGINE: ${WEBAPI_STORAGE_ENGINE:-sqlite}
13+
WEBAPI_TELEMETRY: ${WEBAPI_TELEMETRY:-local}
14+
WEBAPI_SEARCH: ${WEBAPI_SEARCH:-sql}
15+
WEBAPI_CRM: ${WEBAPI_CRM:-noop}
16+
POSTGRES_VERSION: ${POSTGRES_VERSION:-18}
17+
WEBAPI_CRON_EXECUTION_MODE: "run-forever"
18+
19+
x-webapi-cron-service: &webapi-cron-service
20+
restart: always
21+
volumes:
22+
- sqlite_data:/data
23+
depends_on:
24+
webapi-postgres:
25+
condition: service_healthy
26+
required: false
27+
webapi:
28+
condition: service_healthy
29+
330
services:
431
webapi-postgres:
532
profiles:
@@ -54,6 +81,41 @@ services:
5481
interval: 30s
5582
timeout: 10s
5683
retries: 5
84+
webapi-gc-do-all:
85+
<<: *webapi-cron-service
86+
image: ${DOCKER_IMAGE_WEBAPI_GC_DO_ALL:-getthriving/webapi-gc-do-all:${VERSION:-latest}}
87+
environment:
88+
<<: *webapi-cron-env
89+
webapi-gen-do-all:
90+
<<: *webapi-cron-service
91+
image: ${DOCKER_IMAGE_WEBAPI_GEN_DO_ALL:-getthriving/webapi-gen-do-all:${VERSION:-latest}}
92+
environment:
93+
<<: *webapi-cron-env
94+
webapi-schedule-external-sync-do-all:
95+
<<: *webapi-cron-service
96+
image: ${DOCKER_IMAGE_WEBAPI_SCHEDULE_EXTERNAL_SYNC_DO_ALL:-getthriving/webapi-schedule-external-sync-do-all:${VERSION:-latest}}
97+
environment:
98+
<<: *webapi-cron-env
99+
webapi-search-index-backfill-do-all:
100+
<<: *webapi-cron-service
101+
image: ${DOCKER_IMAGE_WEBAPI_SEARCH_INDEX_BACKFILL_DO_ALL:-getthriving/webapi-search-index-backfill-do-all:${VERSION:-latest}}
102+
environment:
103+
<<: *webapi-cron-env
104+
webapi-search-mutation-log-drain-do-all:
105+
<<: *webapi-cron-service
106+
image: ${DOCKER_IMAGE_WEBAPI_SEARCH_MUTATION_LOG_DRAIN_DO_ALL:-getthriving/webapi-search-mutation-log-drain-do-all:${VERSION:-latest}}
107+
environment:
108+
<<: *webapi-cron-env
109+
webapi-search-mutation-log-processing-requeue-do-all:
110+
<<: *webapi-cron-service
111+
image: ${DOCKER_IMAGE_WEBAPI_SEARCH_MUTATION_LOG_PROCESSING_REQUEUE_DO_ALL:-getthriving/webapi-search-mutation-log-processing-requeue-do-all:${VERSION:-latest}}
112+
environment:
113+
<<: *webapi-cron-env
114+
webapi-stats-do-all:
115+
<<: *webapi-cron-service
116+
image: ${DOCKER_IMAGE_WEBAPI_STATS_DO_ALL:-getthriving/webapi-stats-do-all:${VERSION:-latest}}
117+
environment:
118+
<<: *webapi-cron-env
57119
api:
58120
environment:
59121
- UNIVERSE=${UNIVERSE:?Check https://docs.get-thriving.com/how-tos/self-hosting}

‎render.yaml‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,42 @@ previews:
22
generation: automatic
33
expireAfterDays: 1
44

5+
# Background mutations (formerly inside jupiter-webapi-srv); Render cron + WEBAPI_CRON_EXECUTION_MODE=start-run-stop.
6+
envVarGroups:
7+
- name: jupiter-webapi-cron
8+
envVars:
9+
- key: UNIVERSE
10+
value: thrive
11+
- key: ENV
12+
value: production
13+
previewValue: staging
14+
- key: INSTANCE
15+
value: Main
16+
previewValue: TO-FILL
17+
# Ephemeral SQLite URL (domain data uses Postgres); webapi-srv owns the /data disk.
18+
- key: SQLITE_DB_URL
19+
value: sqlite+aiosqlite:////tmp/jupiter.sqlite
20+
- key: POSTGRES_DB_URL
21+
fromDatabase:
22+
name: jupiter-webapi-srv-db
23+
property: connectionString
24+
- key: ALEMBIC_INI_PATH
25+
value: /jupiter/src/core/migrations/alembic.postgres.ini
26+
- key: ALEMBIC_MIGRATIONS_PATH
27+
value: /jupiter/src/core/migrations/postgres
28+
- key: WEBAPI_STORAGE_ENGINE
29+
value: postgres
30+
- key: WEBAPI_TELEMETRY
31+
value: sentry
32+
previewValue: sentry
33+
- key: WEBAPI_SEARCH
34+
value: algolia
35+
- key: WEBAPI_CRM
36+
value: wix
37+
previewValue: noop
38+
- key: WEBAPI_CRON_EXECUTION_MODE
39+
value: start-run-stop
40+
541
services:
642
- type: web
743
name: jupiter-webui
@@ -102,6 +138,113 @@ services:
102138
- key: WEBAPI_CRM
103139
value: wix
104140
previewValue: noop
141+
# Schedules match ``get_background_mutation_crontab()`` on each use case (see ``background_mutation_use_case``).
142+
- type: cron
143+
name: jupiter-webapi-gc-do-all
144+
runtime: docker
145+
dockerfilePath: ./src/webapi/gc-do-all/Dockerfile
146+
dockerContext: .
147+
region: frankfurt
148+
plan: starter
149+
schedule: "0 * * * *"
150+
autoDeploy: true
151+
rootDir: .
152+
previews:
153+
generation: automatic
154+
plan: starter
155+
envVarGroups:
156+
- jupiter-webapi-cron
157+
- type: cron
158+
name: jupiter-webapi-gen-do-all
159+
runtime: docker
160+
dockerfilePath: ./src/webapi/gen-do-all/Dockerfile
161+
dockerContext: .
162+
region: frankfurt
163+
plan: starter
164+
schedule: "0 * * * *"
165+
autoDeploy: true
166+
rootDir: .
167+
previews:
168+
generation: automatic
169+
plan: starter
170+
envVarGroups:
171+
- jupiter-webapi-cron
172+
- type: cron
173+
name: jupiter-webapi-schedule-external-sync-do-all
174+
runtime: docker
175+
dockerfilePath: ./src/webapi/schedule-external-sync-do-all/Dockerfile
176+
dockerContext: .
177+
region: frankfurt
178+
plan: starter
179+
schedule: "0 * * * *"
180+
autoDeploy: true
181+
rootDir: .
182+
previews:
183+
generation: automatic
184+
plan: starter
185+
envVarGroups:
186+
- jupiter-webapi-cron
187+
- type: cron
188+
name: jupiter-webapi-search-index-backfill-do-all
189+
runtime: docker
190+
dockerfilePath: ./src/webapi/search-index-backfill-do-all/Dockerfile
191+
dockerContext: .
192+
region: frankfurt
193+
plan: starter
194+
schedule: "0 * * * *"
195+
autoDeploy: true
196+
rootDir: .
197+
previews:
198+
generation: automatic
199+
plan: starter
200+
envVarGroups:
201+
- jupiter-webapi-cron
202+
- type: cron
203+
name: jupiter-webapi-search-mutation-log-drain-do-all
204+
runtime: docker
205+
dockerfilePath: ./src/webapi/search-mutation-log-drain-do-all/Dockerfile
206+
dockerContext: .
207+
region: frankfurt
208+
plan: starter
209+
# SearchMutationLogDrainDoAllUseCase: */30 * * * * * (Render: every minute, no sub-minute cron)
210+
schedule: "* * * * *"
211+
autoDeploy: true
212+
rootDir: .
213+
previews:
214+
generation: automatic
215+
plan: starter
216+
envVarGroups:
217+
- jupiter-webapi-cron
218+
- type: cron
219+
name: jupiter-webapi-search-mutation-log-processing-requeue-do-all
220+
runtime: docker
221+
dockerfilePath: ./src/webapi/search-mutation-log-processing-requeue-do-all/Dockerfile
222+
dockerContext: .
223+
region: frankfurt
224+
plan: starter
225+
schedule: "*/5 * * * *"
226+
autoDeploy: true
227+
rootDir: .
228+
previews:
229+
generation: automatic
230+
plan: starter
231+
envVarGroups:
232+
- jupiter-webapi-cron
233+
- type: cron
234+
name: jupiter-webapi-stats-do-all
235+
runtime: docker
236+
dockerfilePath: ./src/webapi/stats-do-all/Dockerfile
237+
dockerContext: .
238+
region: frankfurt
239+
plan: starter
240+
schedule: "0 * * * *"
241+
autoDeploy: true
242+
rootDir: .
243+
previews:
244+
generation: automatic
245+
plan: starter
246+
envVarGroups:
247+
- jupiter-webapi-cron
105248
- type: web
106249
name: jupiter-api
107250
domains:

‎src/alib/py/framework/jupiter/framework/appform/cron/appform.py‎

Lines changed: 9 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
"""Cron-style application form: a single background mutation use case."""
22

33
import asyncio
4-
import enum
54
import types
65
from collections.abc import Iterator
76
from json import JSONDecodeError
@@ -24,6 +23,7 @@
2423
UnavailableForContextHandler,
2524
UnavailableGloballyHandler,
2625
)
26+
from jupiter.framework.appform.cron.execution_mode import CronExecutionMode
2727
from jupiter.framework.appform.cron.trigger import cron_trigger_from_crontab
2828
from jupiter.framework.auth.auth_token import (
2929
ExpiredAuthTokenError,
@@ -69,16 +69,6 @@
6969
)
7070

7171

72-
class CronExecutionMode(enum.Enum):
73-
"""How the cron process runs after startup."""
74-
75-
START_RUN_STOP = "start_run_stop"
76-
"""Start, run the use case once, then exit."""
77-
78-
RUN_FOREVER = "run_forever"
79-
"""Start and keep running the use case on its crontab (same model as WebAPI)."""
80-
81-
8272
class Cron(
8373
AppForm[_PortsT, _GlobalPropertiesT, _ServicePropertiesT, _ComponentPropertiesT],
8474
Generic[_PortsT, _GlobalPropertiesT, _ServicePropertiesT, _ComponentPropertiesT],
@@ -140,6 +130,11 @@ def __init__(
140130
use_case=use_case,
141131
)
142132

133+
@property
134+
def execution_mode(self) -> CronExecutionMode:
135+
"""How this cron process runs (once and exit vs scheduled loop)."""
136+
return self._execution_mode
137+
143138
@classmethod
144139
def build_from_module_root(
145140
cls: type[_CronT],
@@ -152,8 +147,8 @@ def build_from_module_root(
152147
invocation_recorder: MutationInvocationRecorder,
153148
use_case_type: type[BackgroundMutationUseCase[Any, Any, Any, Any, Any]], # type: ignore[explicit-any]
154149
exception_handler_base: type[_CronExceptionHandlerT],
150+
execution_mode: CronExecutionMode,
155151
*module_root: types.ModuleType,
156-
execution_mode: CronExecutionMode = CronExecutionMode.RUN_FOREVER,
157152
) -> _CronT:
158153
"""Build the cron app form and register exception handlers from ``module_root``."""
159154

@@ -163,9 +158,7 @@ def extract_exception_handler(
163158
tuple[
164159
type[Exception],
165160
type[
166-
CronExceptionHandler[
167-
GlobalProperties, ServiceProperties, Exception
168-
]
161+
CronExceptionHandler[GlobalProperties, ServiceProperties, Exception]
169162
],
170163
]
171164
]:
@@ -212,9 +205,7 @@ def extract_exception_handler(
212205
):
213206
if exception_type in cron_app._exception_handlers:
214207
continue
215-
cron_app._add_exception_handler(
216-
exception_type, exception_handler_type
217-
)
208+
cron_app._add_exception_handler(exception_type, exception_handler_type)
218209

219210
return cron_app
220211

‎src/alib/py/framework/jupiter/framework/appform/cron/execution_mode.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
"""Cron process execution mode.
2+
3+
Set ``WEBAPI_CRON_EXECUTION_MODE`` to ``start-run-stop`` (Render) or ``run-forever`` (PM2, Compose).
4+
"""
5+
6+
import enum
7+
8+
9+
class CronExecutionMode(enum.Enum):
10+
"""How the cron process runs after startup."""
11+
12+
START_RUN_STOP = "start-run-stop"
13+
"""Start, run the use case once, then exit."""
14+
15+
RUN_FOREVER = "run-forever"
16+
"""Start and keep running the use case on its crontab (in-process scheduler)."""

‎src/alib/py/framework/jupiter/framework/appform/webapi/appform.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
get_args,
2020
get_origin,
2121
)
22+
2223
import uvicorn
2324
from fastapi import Depends, FastAPI, HTTPException, Request, Response
2425
from fastapi.openapi.utils import get_openapi
@@ -368,7 +369,7 @@ async def simple_login(
368369
continue
369370
app._add_use_case_type(use_case_type, mr)
370371

371-
for use_case_type, command in app._use_case_commands.items():
372+
for _, command in app._use_case_commands.items():
372373
if isinstance(command, UseCaseCommand):
373374
command.attach_route(app._fast_app)
374375
else:

‎src/webapi/gc-do-all/jupiter_webapi_gc_do_all/config.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
)
2121
from jupiter.framework.appform.cron.appform import Cron
2222
from jupiter.framework.appform.cron.exception import CronExceptionHandler
23+
from jupiter.framework.appform.cron.execution_mode import CronExecutionMode
2324
from jupiter.framework.service_properties import ServiceProperties
2425
from jupiter.framework.sqlalchemy_async_url import normalized_async_sqlalchemy_db_url
2526

@@ -34,6 +35,7 @@ class JupiterWebApiProperties(ServiceProperties):
3435
telemetry: JupiterWebApiTelemetry
3536
search_backend: JupiterWebApiSearchBackend
3637
crm_backend: JupiterWebApiCrmBackend
38+
execution_mode: CronExecutionMode
3739
sqlite_db_url: str
3840
postgres_db_url: str
3941
alembic_ini_path: Path
@@ -99,6 +101,9 @@ def find_up_the_dir_tree(partial_path: Union[str, Path]) -> Path:
99101
telemetry = JupiterWebApiTelemetry(cast(str, os.getenv("WEBAPI_TELEMETRY")))
100102
search_backend = JupiterWebApiSearchBackend(cast(str, os.getenv("WEBAPI_SEARCH")))
101103
crm_backend = JupiterWebApiCrmBackend(cast(str, os.getenv("WEBAPI_CRM")))
104+
execution_mode = CronExecutionMode(
105+
cast(str, os.getenv("WEBAPI_CRON_EXECUTION_MODE"))
106+
)
102107

103108
if not alembic_ini_path.is_absolute():
104109
alembic_ini_path = find_up_the_dir_tree(alembic_ini_path)
@@ -110,6 +115,7 @@ def find_up_the_dir_tree(partial_path: Union[str, Path]) -> Path:
110115
telemetry=telemetry,
111116
search_backend=search_backend,
112117
crm_backend=crm_backend,
118+
execution_mode=execution_mode,
113119
sentry_dsn=sentry_dsn,
114120
sqlite_db_url=sqlite_db_url,
115121
postgres_db_url=postgres_db_url,

‎src/webapi/gc-do-all/jupiter_webapi_gc_do_all/jupiter.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ async def main() -> None:
188188
invocation_recorder,
189189
GCDoAllUseCase,
190190
JupiterExceptionHandler,
191+
service_properties.execution_mode,
191192
jupiter_webapi_gc_do_all.exceptions,
192193
)
193194

@@ -203,6 +204,7 @@ async def main() -> None:
203204
rich_print(f" Environment: {global_properties.env}")
204205
rich_print(f" Instance: {global_properties.instance}")
205206
rich_print(f" Hosting: {global_properties.universe.hosting}")
207+
rich_print(f" Execution mode: {service_properties.execution_mode.value}")
206208
rich_print("-" * 80)
207209
rich_print("Component Classes:")
208210
rich_print(f" Telemetry: {telemetry.__class__.__name__}")

‎src/webapi/gen-do-all/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,4 +37,4 @@ RUN uv sync --all-packages --active
3737

3838
WORKDIR /jupiter/src/webapi/gen-do-all
3939

40-
ENTRYPOINT [ "dumb-init", "python", "-c", "import jupiter_webapi_gen_do_all" ]
40+
ENTRYPOINT [ "dumb-init", "python", "-m", "jupiter_webapi_gen_do_all.jupiter" ]

0 commit comments

Comments
 (0)