Skip to content

Commit 6d2fcb7

Browse files
iziangiziang
authored
feat: centralize dynamic env var construction in config.py, remove composite vars from .env (#1105)
- Added PostgreSQL, Redis, and Elasticsearch configuration fields to `Config` class in `config.py`. - Updated `env.template` to reflect new environment variables for Redis and Elasticsearch. - Removed redundant URL construction from `entrypoint.sh`, as URLs are now built within the `Config` class. - Adjusted `start-celery-flower.sh` to simplify Celery command execution.
1 parent 2f84677 commit 6d2fcb7

4 files changed

Lines changed: 68 additions & 45 deletions

File tree

‎aperag/config.py‎

Lines changed: 58 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,28 @@ class Config(BaseSettings):
4848
# Debug mode
4949
debug: bool = Field(False, alias="DEBUG")
5050

51+
# Postgres atomic fields
52+
postgres_host: str = Field("127.0.0.1", alias="POSTGRES_HOST")
53+
postgres_port: int = Field(5432, alias="POSTGRES_PORT")
54+
postgres_db: str = Field("postgres", alias="POSTGRES_DB")
55+
postgres_user: str = Field("postgres", alias="POSTGRES_USER")
56+
postgres_password: str = Field("postgres", alias="POSTGRES_PASSWORD")
57+
58+
# Redis atomic fields
59+
redis_host: str = Field("127.0.0.1", alias="REDIS_HOST")
60+
redis_port: int = Field(6379, alias="REDIS_PORT")
61+
redis_user: str = Field("default", alias="REDIS_USER")
62+
redis_password: str = Field("password", alias="REDIS_PASSWORD")
63+
64+
# Elasticsearch atomic fields
65+
es_host_name: str = Field("127.0.0.1", alias="ES_HOST_NAME")
66+
es_port: int = Field(9200, alias="ES_PORT")
67+
es_user: str = Field("", alias="ES_USER")
68+
es_password: str = Field("", alias="ES_PASSWORD")
69+
es_protocol: str = Field("http", alias="ES_PROTOCOL")
70+
5171
# Database
52-
database_url: str = Field(f"sqlite:///{BASE_DIR}/db.sqlite3", alias="DATABASE_URL")
72+
database_url: Optional[str] = Field(None, alias="DATABASE_URL")
5373

5474
# Database connection pool settings
5575
db_pool_size: int = Field(20, alias="DB_POOL_SIZE")
@@ -68,7 +88,7 @@ class Config(BaseSettings):
6888
logto_app_id: str = Field("", alias="LOGTO_APP_ID")
6989

7090
# Celery
71-
celery_broker_url: str = Field("redis://localhost:6379/0", alias="CELERY_BROKER_URL")
91+
celery_broker_url: Optional[str] = Field(None, alias="CELERY_BROKER_URL")
7292
celery_result_backend: Optional[str] = None # Will be set in __post_init__
7393
celery_beat_scheduler: str = "django_celery_beat.schedulers:DatabaseScheduler"
7494
celery_worker_send_task_events: bool = True
@@ -84,7 +104,7 @@ class Config(BaseSettings):
84104
embedding_max_chunks_in_batch: int = Field(10, alias="EMBEDDING_MAX_CHUNKS_IN_BATCH")
85105

86106
# Memory backend
87-
memory_redis_url: str = Field("redis://127.0.0.1:6379/1", alias="MEMORY_REDIS_URL")
107+
memory_redis_url: Optional[str] = Field(None, alias="MEMORY_REDIS_URL")
88108

89109
# Vector DB
90110
vector_db_type: str = Field("qdrant", alias="VECTOR_DB_TYPE")
@@ -108,14 +128,8 @@ class Config(BaseSettings):
108128
chunk_size: int = Field(400, alias="CHUNK_SIZE")
109129
chunk_overlap_size: int = Field(20, alias="CHUNK_OVERLAP_SIZE")
110130

111-
# Redis
112-
redis_host: str = Field("localhost", alias="REDIS_HOST")
113-
redis_port: str = Field("6379", alias="REDIS_PORT")
114-
redis_username: str = Field("", alias="REDIS_USERNAME")
115-
redis_password: str = Field("", alias="REDIS_PASSWORD")
116-
117131
# Fulltext search
118-
es_host: str = Field("http://localhost:9200", alias="ES_HOST")
132+
es_host: Optional[str] = Field(None, alias="ES_HOST")
119133
es_timeout: int = Field(30, alias="ES_TIMEOUT") # ES request timeout in seconds
120134
es_max_retries: int = Field(3, alias="ES_MAX_RETRIES") # Max retries for ES requests
121135

@@ -141,10 +155,6 @@ class Config(BaseSettings):
141155

142156
def __init__(self, **kwargs):
143157
super().__init__(**kwargs)
144-
# Set celery_result_backend if not set
145-
if not self.celery_result_backend:
146-
self.celery_result_backend = self.celery_broker_url
147-
148158
# Load model configs from file
149159
import json
150160
import os
@@ -154,6 +164,40 @@ def __init__(self, **kwargs):
154164
with open(json_path, "r", encoding="utf-8") as f:
155165
self.model_configs = json.load(f)
156166

167+
# DATABASE_URL
168+
if not self.database_url:
169+
self.database_url = (
170+
f"postgresql://{self.postgres_user}:{self.postgres_password}"
171+
f"@{self.postgres_host}:{self.postgres_port}/{self.postgres_db}"
172+
)
173+
# CELERY_BROKER_URL
174+
if not self.celery_broker_url:
175+
self.celery_broker_url = (
176+
f"redis://{self.redis_user}:{self.redis_password}"
177+
f"@{self.redis_host}:{self.redis_port}/0"
178+
)
179+
180+
# CELERY_RESULT_BACKEND
181+
if not self.celery_result_backend:
182+
self.celery_result_backend = self.celery_broker_url
183+
184+
# MEMORY_REDIS_URL
185+
if not self.memory_redis_url:
186+
self.memory_redis_url = (
187+
f"redis://{self.redis_user}:{self.redis_password}"
188+
f"@{self.redis_host}:{self.redis_port}/1"
189+
)
190+
# ES_HOST
191+
if not self.es_host:
192+
if self.es_user and self.es_password:
193+
self.es_host = (
194+
f"{self.es_protocol}://{self.es_user}:{self.es_password}"
195+
f"@{self.es_host_name}:{self.es_port}"
196+
)
197+
else:
198+
self.es_host = (
199+
f"{self.es_protocol}://{self.es_host_name}:{self.es_port}"
200+
)
157201
# Object store config
158202
if self.object_store_type == "local":
159203
self.object_store_local_config = LocalObjectStoreConfig()

‎envs/env.template‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ POSTGRES_PORT=5432
44
POSTGRES_DB=postgres
55
POSTGRES_USER=postgres
66
POSTGRES_PASSWORD=postgres
7-
DATABASE_URL=postgresql://postgres:postgres@127.0.0.1:5432/postgres
87

98
# Database Connection Pool Settings
109
# Adjust these values based on your server resources and expected load
@@ -15,17 +14,21 @@ DB_POOL_RECYCLE=3600 # Recycle connections after 1 hour (in seconds)
1514
DB_POOL_PRE_PING=True # Validate connections before use
1615

1716
# Redis
18-
MEMORY_REDIS_URL=redis://default:password@127.0.0.1:6379
19-
20-
# Celery
21-
CELERY_BROKER_URL=redis://default:password@127.0.0.1:6379/0
17+
REDIS_HOST=127.0.0.1
18+
REDIS_PORT=6379
19+
REDIS_USER=default
20+
REDIS_PASSWORD=password
2221

2322
# Vector DB
2423
VECTOR_DB_TYPE=qdrant
2524
VECTOR_DB_CONTEXT={"url":"http://127.0.0.1","port":6333,"distance":"Cosine","timeout":1000}
2625

2726
# Elasticsearch
28-
ES_HOST=http://127.0.0.1:9200
27+
ES_HOST_NAME=127.0.0.1
28+
ES_PORT=9200
29+
ES_USER=
30+
ES_PASSWORD=
31+
ES_PROTOCOL=http
2932

3033
# Neo4J
3134
NEO4J_HOST=127.0.0.1

‎scripts/entrypoint.sh‎

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -58,28 +58,4 @@ except Exception as error:
5858
sys.stderr.write("Failed to create pgvector extension (this is non-critical): {}\n".format(error))
5959
END
6060

61-
# Build DATABASE_URL from components
62-
if [[ -n "${POSTGRES_HOST:-}" && -n "${POSTGRES_USER:-}" && -n "${POSTGRES_PASSWORD:-}" ]]; then
63-
export DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-postgres}"
64-
fi
65-
66-
# Build CELERY_BROKER_URL from Redis components
67-
if [[ -n "${REDIS_HOST:-}" && -n "${REDIS_USER:-}" && -n "${REDIS_PASSWORD:-}" ]]; then
68-
export CELERY_BROKER_URL="redis://${REDIS_USER}:${REDIS_PASSWORD}@${REDIS_HOST}:${REDIS_PORT:-6379}/0"
69-
fi
70-
71-
# Build MEMORY_REDIS_URL from Redis components
72-
if [[ -n "${REDIS_HOST:-}" && -n "${REDIS_USER:-}" && -n "${REDIS_PASSWORD:-}" ]]; then
73-
export MEMORY_REDIS_URL="redis://${REDIS_USER}:${REDIS_PASSWORD}@${REDIS_HOST}:${REDIS_PORT:-6379}/1"
74-
fi
75-
76-
# Build ES_HOST from Elasticsearch components
77-
if [[ -n "${ES_HOST_NAME:-}" ]]; then
78-
if [[ -n "${ES_USER:-}" && -n "${ES_PASSWORD:-}" ]]; then
79-
export ES_HOST="${ES_PROTOCOL:-http}://${ES_USER}:${ES_PASSWORD}@${ES_HOST_NAME}:${ES_PORT:-9200}"
80-
else
81-
export ES_HOST="${ES_PROTOCOL:-http}://${ES_HOST_NAME}:${ES_PORT:-9200}"
82-
fi
83-
fi
84-
8561
exec "$@"

‎scripts/start-celery-flower.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@ set -o nounset
55

66
exec watchfiles celery.__main__.main \
77
--args \
8-
"-A config.celery -b \"${CELERY_BROKER_URL}\" flower --basic_auth=\"${CELERY_FLOWER_USER}:${CELERY_FLOWER_PASSWORD}\""
8+
"-A config.celery flower --basic_auth=\"${CELERY_FLOWER_USER}:${CELERY_FLOWER_PASSWORD}\""

0 commit comments

Comments
 (0)