feat(Logging): Unify logging format (#7050)

Author: khvn26

api/.env-ci

@@ -1,5 +1,4 @@
 DATABASE_URL=postgresql://postgres:password@localhost:5432/flagsmith
 ANALYTICS_DATABASE_URL=postgresql://postgres:password@localhost:5433/analytics
 PYTEST_ADDOPTS=--cov . --cov-report xml -n auto --ci
-GUNICORN_LOGGER_CLASS=util.logging.GunicornJsonCapableLogger
 COVERAGE_CORE=sysmon

api/app/settings/common.py

@@ -11,15 +11,14 @@
 """
 
 import importlib
-import json
+import logging
 import os
 import warnings
 from datetime import datetime, time, timedelta
 from typing import Any
 
 import dj_database_url
 import django_stubs_ext
-import prometheus_client
 import pytz
 from common.core import ReplicaReadStrategy
 from corsheaders.defaults import default_headers  # type: ignore[import-untyped]
@@ -610,72 +609,19 @@
 CHARGEBEE_API_KEY = env("CHARGEBEE_API_KEY", default=None)
 CHARGEBEE_SITE = env("CHARGEBEE_SITE", default=None)
 
-# Logging configuration
-ACCESS_LOG_EXTRA_ITEMS = env.list("ACCESS_LOG_EXTRA_ITEMS", subcast=str, default=[])
-LOGGING_CONFIGURATION_FILE = env.str("LOGGING_CONFIGURATION_FILE", default=None)
-if LOGGING_CONFIGURATION_FILE:
-    with open(LOGGING_CONFIGURATION_FILE, "r") as f:
-        LOGGING = json.loads(f.read())
-else:
-    LOG_FORMAT = env.str("LOG_FORMAT", default="generic")
-    LOG_LEVEL = env.str("LOG_LEVEL", default="WARNING")
-    LOGGING = {
-        "version": 1,
-        "disable_existing_loggers": False,
-        "formatters": {
-            "generic": {"format": "%(name)-12s %(levelname)-8s %(message)s"},
-            "json": {
-                "()": "util.logging.JsonFormatter",
-                "datefmt": "%Y-%m-%d %H:%M:%S",
-            },
-        },
-        "handlers": {
-            "console": {
-                "level": LOG_LEVEL,
-                "class": "logging.StreamHandler",
-                "formatter": LOG_FORMAT,
-            }
-        },
-        "loggers": {
-            "": {"level": LOG_LEVEL, "handlers": ["console"]},
-            # Not sure why the following loggers are necessary, but it doesn't seem to
-            # write log messages for e.g. features.workflows.core.models without adding
-            # them explicitly.
-            # TODO: move all apps to a parent 'apps' directory and configure the logger
-            #  for that dir
-            "features": {
-                "level": LOG_LEVEL,
-                "handlers": ["console"],
-                "propagate": False,
-            },
-            "task_processor": {
-                "level": LOG_LEVEL,
-                "handlers": ["console"],
-                "propagate": False,
-            },
-            "app_analytics": {
-                "level": LOG_LEVEL,
-                "handlers": ["console"],
-                "propagate": False,
-            },
-            "webhooks": {
-                "level": LOG_LEVEL,
-                "handlers": ["console"],
-                "propagate": False,
-            },
-        },
-    }
+# Logging is configured by flagsmith-common's setup_logging() in ensure_cli_env(),
+# before Django loads. Disable Django's logging config to preserve our setup.
+LOGGING_CONFIG = None
+LOG_FORMAT = env.str("LOG_FORMAT", default="generic")
+LOG_LEVEL = env.str("LOG_LEVEL", default="WARNING")
 
 ENABLE_DB_LOGGING = env.bool("DJANGO_ENABLE_DB_LOGGING", default=False)
 if ENABLE_DB_LOGGING:
     if not DEBUG:
         warnings.warn("Setting DEBUG=True to ensure DB logging functions correctly.")
         DEBUG = True
 
-    LOGGING["loggers"]["django.db.backends"] = {
-        "level": "DEBUG",
-        "handlers": ["console"],
-    }
+    logging.getLogger("django.db.backends").setLevel(logging.DEBUG)
 
 CACHE_FLAGS_SECONDS = env.int("CACHE_FLAGS_SECONDS", default=0)
 FLAGS_CACHE_LOCATION = "environment-flags"
@@ -1442,12 +1388,6 @@
     INSTALLED_APPS.append("licensing")
 
 PROMETHEUS_ENABLED = env.bool("PROMETHEUS_ENABLED", False)
-PROMETHEUS_HISTOGRAM_BUCKETS = tuple(
-    env.list(
-        "PROMETHEUS_HISTOGRAM_BUCKETS",
-        default=prometheus_client.Histogram.DEFAULT_BUCKETS,
-    )
-)
 
 DOCGEN_MODE = env.bool("DOCGEN_MODE", default=False)
 

api/organisations/urls.py

@@ -161,7 +161,7 @@
 ]
 
 if settings.LICENSING_INSTALLED:  # pragma: no cover
-    from licensing.views import (  # type: ignore[import-not-found]
+    from licensing.views import (  # type: ignore[import-not-found,import-untyped,unused-ignore]
         create_or_update_licence,
     )
 

api/poetry.lock

@@ -163,7 +163,7 @@ pygithub = "~2.8"
 hubspot-api-client = "^12.0.0"
 djangorestframework-dataclasses = "^1.3.1"
 pyotp = "^2.9.0"
-flagsmith-common = { version = "^3.3.0", extras = [
+flagsmith-common = { version = ">=3.5.0,<4", extras = [
   "common-core",
   "flagsmith-schemas",
   "task-processor",

api/pyproject.toml

@@ -3,6 +3,7 @@ set -e
 
 # common environment variables
 ACCESS_LOG_FORMAT=${ACCESS_LOG_FORMAT:-'%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" %({origin}i)s %({access-control-allow-origin}o)s'}
+APPLICATION_LOGGERS=${APPLICATION_LOGGERS:-"common,features,task_processor,app_analytics,webhooks"}
 
 waitfordb() {
   if [ -z "${SKIP_WAIT_FOR_DB}" ]; then