feat: Initial Prometheus support (#5254)

Co-authored-by: Matthew Elwell <matthew.elwell@flagsmith.com>

Author: khvn26

api/Makefile

@@ -9,8 +9,6 @@ DOTENV_OVERRIDE_FILE ?= .env
 
 POETRY_VERSION ?= 2.1.1
 
-GUNICORN_LOGGER_CLASS ?= util.logging.GunicornJsonCapableLogger
-
 SAML_REVISION ?= v1.6.4
 RBAC_REVISION ?= v0.11.2
 
@@ -98,10 +96,8 @@ django-collect-static:
 
 .PHONY: serve
 serve:
-	poetry run gunicorn --bind 0.0.0.0:8000 \
-		--logger-class ${GUNICORN_LOGGER_CLASS} \
-		--reload \
-		app.wsgi
+	flagsmith start --reload api
+		
 
 .PHONY: generate-ld-client-types
 generate-ld-client-types:

api/app/settings/common.py

@@ -10,24 +10,27 @@
 https://docs.djangoproject.com/en/1.9/ref/settings/
 """
 
-import importlib.util
+import importlib
 import json
 import os
 import warnings
 from datetime import datetime, time, timedelta
 
 import dj_database_url  # type: ignore[import-untyped]
+import django_stubs_ext
 import prometheus_client
 import pytz
 from corsheaders.defaults import default_headers  # type: ignore[import-untyped]
 from django.core.exceptions import ImproperlyConfigured
 from django.core.management.utils import get_random_secret_key
 from environs import Env
-from task_processor.task_run_method import TaskRunMethod  # type: ignore[import-untyped]
+from task_processor.task_run_method import TaskRunMethod
 
 from app.routers import ReplicaReadStrategy
 from app.utils import get_numbered_env_vars_with_prefix
 
+django_stubs_ext.monkeypatch()
+
 env = Env()
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -69,6 +72,7 @@
 # Application definition
 
 INSTALLED_APPS = [
+    "common.core",
     "core.custom_admin.apps.CustomAdminConfig",
     "django.contrib.auth",
     "django.contrib.contenttypes",
@@ -279,6 +283,7 @@
     ],
 }
 MIDDLEWARE = [
+    "common.gunicorn.middleware.RouteLoggerMiddleware",
     "django.middleware.security.SecurityMiddleware",
     "whitenoise.middleware.WhiteNoiseMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
@@ -1351,5 +1356,10 @@
 if LICENSING_INSTALLED:  # pragma: no cover
     INSTALLED_APPS.append("licensing")
 
-PROMETHEUS_ENABLED = True
-PROMETHEUS_HISTOGRAM_BUCKETS = prometheus_client.Histogram.DEFAULT_BUCKETS
+PROMETHEUS_ENABLED = env.bool("PROMETHEUS_ENABLED", False)
+PROMETHEUS_HISTOGRAM_BUCKETS = tuple(
+    env.list(
+        "PROMETHEUS_HISTOGRAM_BUCKETS",
+        default=prometheus_client.Histogram.DEFAULT_BUCKETS,
+    )
+)

api/app_analytics/tasks.py

@@ -4,7 +4,7 @@
 from django.conf import settings
 from django.db.models import Q, Sum
 from django.utils import timezone
-from task_processor.decorators import (  # type: ignore[import-untyped]
+from task_processor.decorators import (
     register_recurring_task,
     register_task_handler,
 )
@@ -24,9 +24,9 @@
 )
 from environments.models import Environment
 
-if settings.USE_POSTGRES_FOR_ANALYTICS:
+if settings.USE_POSTGRES_FOR_ANALYTICS:  # pragma: no cover
 
-    @register_recurring_task(  # type: ignore[misc]
+    @register_recurring_task(
         run_every=timedelta(minutes=60),
         kwargs={
             "bucket_size": ANALYTICS_READ_BUCKET_SIZE,
@@ -68,7 +68,7 @@ def clean_up_old_analytics_data():  # type: ignore[no-untyped-def]
     ).delete()
 
 
-@register_task_handler()  # type: ignore[misc]
+@register_task_handler()
 def track_feature_evaluation_v2(
     environment_id: int, feature_evaluations: list[dict[str, int | str | bool]]
 ) -> None:
@@ -86,7 +86,7 @@ def track_feature_evaluation_v2(
     FeatureEvaluationRaw.objects.bulk_create(feature_evaluation_objects)
 
 
-@register_task_handler()  # type: ignore[misc]
+@register_task_handler()
 def track_feature_evaluation(
     environment_id: int,
     feature_evaluations: dict[str, int],
@@ -103,7 +103,7 @@ def track_feature_evaluation(
     FeatureEvaluationRaw.objects.bulk_create(feature_evaluation_objects)
 
 
-@register_task_handler()  # type: ignore[misc]
+@register_task_handler()
 def track_request(resource: int, host: str, environment_key: str, count: int = 1):  # type: ignore[no-untyped-def]
     environment = Environment.get_from_cache(environment_key)  # type: ignore[no-untyped-call]
     if environment is None:

api/audit/tasks.py

@@ -4,10 +4,10 @@
 
 from django.contrib.auth import get_user_model
 from django.utils import timezone
-from task_processor.decorators import (  # type: ignore[import-untyped]
+from task_processor.decorators import (
     register_task_handler,
 )
-from task_processor.models import TaskPriority  # type: ignore[import-untyped]
+from task_processor.models import TaskPriority
 
 from audit.constants import (
     FEATURE_STATE_UPDATED_BY_CHANGE_REQUEST_MESSAGE,
@@ -18,14 +18,14 @@
 logger = logging.getLogger(__name__)
 
 
-@register_task_handler(priority=TaskPriority.HIGHEST)  # type: ignore[misc]
+@register_task_handler(priority=TaskPriority.HIGHEST)
 def create_feature_state_went_live_audit_log(feature_state_id: int):  # type: ignore[no-untyped-def]
     _create_feature_state_audit_log_for_change_request(
         feature_state_id, FEATURE_STATE_WENT_LIVE_MESSAGE
     )
 
 
-@register_task_handler(priority=TaskPriority.HIGHEST)  # type: ignore[misc]
+@register_task_handler(priority=TaskPriority.HIGHEST)
 def create_feature_state_updated_by_change_request_audit_log(feature_state_id: int):  # type: ignore[no-untyped-def]
     _create_feature_state_audit_log_for_change_request(
         feature_state_id, FEATURE_STATE_UPDATED_BY_CHANGE_REQUEST_MESSAGE
@@ -63,7 +63,7 @@ def _create_feature_state_audit_log_for_change_request(  # type: ignore[no-untyp
     )
 
 
-@register_task_handler(priority=TaskPriority.HIGHEST)  # type: ignore[misc]
+@register_task_handler(priority=TaskPriority.HIGHEST)
 def create_audit_log_from_historical_record(  # type: ignore[no-untyped-def]
     history_instance_id: int,
     history_user_id: typing.Optional[int],
@@ -123,7 +123,7 @@ def create_audit_log_from_historical_record(  # type: ignore[no-untyped-def]
     )
 
 
-@register_task_handler()  # type: ignore[misc]
+@register_task_handler()
 def create_segment_priorities_changed_audit_log(  # type: ignore[no-untyped-def]
     previous_id_priority_pairs: typing.List[typing.Tuple[int, int]],
     feature_segment_ids: typing.List[int],

api/conftest.py

@@ -24,7 +24,7 @@
 from pytest_mock import MockerFixture
 from rest_framework.authtoken.models import Token
 from rest_framework.test import APIClient
-from task_processor.task_run_method import TaskRunMethod  # type: ignore[import-untyped]
+from task_processor.task_run_method import TaskRunMethod
 from urllib3 import BaseHTTPResponse
 from urllib3.connectionpool import HTTPConnectionPool
 from xdist import get_xdist_worker_id  # type: ignore[import-untyped]

api/core/signals.py

@@ -4,7 +4,7 @@
 from django.core.exceptions import ObjectDoesNotExist
 from django.utils import timezone
 from simple_history.models import HistoricalRecords  # type: ignore[import-untyped]
-from task_processor.task_run_method import TaskRunMethod  # type: ignore[import-untyped]
+from task_processor.task_run_method import TaskRunMethod
 
 from audit import tasks
 from core.models import AbstractBaseAuditableModel

api/custom_auth/tasks.py

@@ -2,7 +2,7 @@
 
 from django.conf import settings
 from django.utils import timezone
-from task_processor.decorators import (  # type: ignore[import-untyped]
+from task_processor.decorators import (
     register_recurring_task,
 )
 

api/edge_api/identities/tasks.py

@@ -2,10 +2,10 @@
 import typing
 
 from django.utils import timezone
-from task_processor.decorators import (  # type: ignore[import-untyped]
+from task_processor.decorators import (
     register_task_handler,
 )
-from task_processor.models import TaskPriority  # type: ignore[import-untyped]
+from task_processor.models import TaskPriority
 
 from audit.models import AuditLog
 from audit.related_object_type import RelatedObjectType
@@ -29,7 +29,7 @@
 logger = logging.getLogger(__name__)
 
 
-@register_task_handler()  # type: ignore[misc]
+@register_task_handler()
 def call_environment_webhook_for_feature_state_change(  # type: ignore[no-untyped-def]
     feature_id: int,
     environment_api_key: str,
@@ -90,7 +90,7 @@ def call_environment_webhook_for_feature_state_change(  # type: ignore[no-untype
     call_environment_webhooks(environment.id, data, event_type=event_type.value)
 
 
-@register_task_handler(priority=TaskPriority.HIGH)  # type: ignore[misc]
+@register_task_handler(priority=TaskPriority.HIGH)
 def sync_identity_document_features(identity_uuid: str):  # type: ignore[no-untyped-def]
     from .models import EdgeIdentity
 
@@ -108,7 +108,7 @@ def sync_identity_document_features(identity_uuid: str):  # type: ignore[no-unty
     identity.save()
 
 
-@register_task_handler()  # type: ignore[misc]
+@register_task_handler()
 def generate_audit_log_records(
     environment_api_key: str,
     identifier: str,
@@ -148,7 +148,7 @@ def generate_audit_log_records(
     AuditLog.objects.bulk_create(audit_records)
 
 
-@register_task_handler()  # type: ignore[misc]
+@register_task_handler()
 def update_flagsmith_environments_v2_identity_overrides(
     environment_api_key: str,
     identity_uuid: str,
@@ -172,7 +172,7 @@ def update_flagsmith_environments_v2_identity_overrides(
     dynamodb_wrapper_v2.update_identity_overrides(identity_override_changeset)
 
 
-@register_task_handler()  # type: ignore[misc]
+@register_task_handler()
 def delete_environments_v2_identity_overrides_by_feature(feature_id: int) -> None:
     dynamodb_wrapper_v2 = DynamoEnvironmentV2Wrapper()
 

api/environments/tasks.py

@@ -1,7 +1,7 @@
-from task_processor.decorators import (  # type: ignore[import-untyped]
+from task_processor.decorators import (
     register_task_handler,
 )
-from task_processor.models import TaskPriority  # type: ignore[import-untyped]
+from task_processor.models import TaskPriority
 
 from audit.models import AuditLog
 from environments.dynamodb import DynamoIdentityWrapper
@@ -17,12 +17,12 @@
 )
 
 
-@register_task_handler(priority=TaskPriority.HIGH)  # type: ignore[misc]
+@register_task_handler(priority=TaskPriority.HIGH)
 def rebuild_environment_document(environment_id: int) -> None:
     Environment.write_environments_to_dynamodb(environment_id=environment_id)
 
 
-@register_task_handler(priority=TaskPriority.HIGHEST)  # type: ignore[misc]
+@register_task_handler(priority=TaskPriority.HIGHEST)
 def process_environment_update(audit_log_id: int):  # type: ignore[no-untyped-def]
     audit_log = AuditLog.objects.get(id=audit_log_id)
 
@@ -38,7 +38,7 @@ def process_environment_update(audit_log_id: int):  # type: ignore[no-untyped-de
         send_environment_update_message_for_project(audit_log.project)
 
 
-@register_task_handler()  # type: ignore[misc]
+@register_task_handler()
 def delete_environment_from_dynamo(api_key: str, environment_id: str):  # type: ignore[no-untyped-def]
     # Delete environment
     environment_wrapper.delete_environment(api_key)
@@ -51,12 +51,12 @@ def delete_environment_from_dynamo(api_key: str, environment_id: str):  # type:
     environment_v2_wrapper.delete_environment(environment_id)  # type: ignore[arg-type]
 
 
-@register_task_handler()  # type: ignore[misc]
+@register_task_handler()
 def delete_environment(environment_id: int) -> None:
     Environment.objects.get(id=environment_id).delete()
 
 
-@register_task_handler()  # type: ignore[misc]
+@register_task_handler()
 def clone_environment_feature_states(
     source_environment_id: int, clone_environment_id: int
 ) -> None:

api/features/feature_health/tasks.py

@@ -1,12 +1,12 @@
-from task_processor.decorators import (  # type: ignore[import-untyped]
+from task_processor.decorators import (
     register_task_handler,
 )
 
 from features.feature_health import services
 from features.models import Feature
 
 
-@register_task_handler()  # type: ignore[misc]
+@register_task_handler()
 def update_feature_unhealthy_tag(feature_id: int) -> None:
     if feature := Feature.objects.filter(id=feature_id).first():
         services.update_feature_unhealthy_tag(feature)