[ENG-9706] support osfmetrics migration

.github/workflows/test_djelme.yml

@@ -12,11 +12,11 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
         id: setup-py
         with:
-          python-version: '3.11'
+          python-version: '3.13'
       - run: pip install poetry
       - run: poetry install --no-root --with=lint
       - run: poetry run python -m elasticsearch_metrics.tests --lint
@@ -27,7 +27,6 @@ jobs:
       matrix:
         python: ['3.10', '3.11', '3.12', '3.13', '3.14']
         django: ['4.2', '5.1', '5.2']
-        # TODO: elasticsearch: ['6', '7', '8', '9']
     services:
       elasticsearch6:
         image: elasticsearch:6.8.23
@@ -42,8 +41,8 @@ jobs:
           node.name: singlenode
           cluster.initial_master_nodes: singlenode
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
         id: setup-py
         with:
           python-version: ${{ matrix.python }}

README.md

@@ -61,15 +61,9 @@ class UsageRecord(EventRecord):
         using = "my-es8-backend"  # backend name -- required if multiple backends use the same imp
 ```
 
-Either enable autosetup...
-```python
-# ... in your django project settings file ...
-DJELME_AUTOSETUP = True
-```
-
-...or be sure to run the `djelme_backend_setup` management command before trying to store anything.
+and be sure to run the `djelme_backend_setup` management command before trying to store anything:
 ```shell
-# This will create an index template for usagerecord timeseries indexes
+# This will create an index template for each timeseries record type
 python manage.py djelme_backend_setup
 ```
 
@@ -100,18 +94,18 @@ UsageRecord.search_timeseries_range(datetime.date(2030, 1, 1), datetime.date(203
 
 By default, behind the scenes, a new elasticsearch index is created for each record type for each day
 in which a record is saved (using UTC timezone). You can change this for a record type by setting
-`Meta.timedepth`, or change the default timedepth with the setting `DJELME_DEFAULT_TIMEDEPTH` (see below).
+`Meta.timeseries_index_timedepth`, or change the default timedepth with the setting `DJELME_DEFAULT_TIMEDEPTH` (see below).
 
 ```python
 class MyEventWithMonthlyIndexes(EventRecord):
     class Meta:
-        timedepth = 2  # year and month
+        timeseries_index_timedepth = 2  # year and month
 ```
 
-- index per year: `timedepth = 1`
-- index per month: `timedepth = 2`
-- index per day: `timedepth = 3` (default)
-- index per hour: `timedepth = 4`
+- index per year: `timeseries_index_timedepth = 1`
+- index per month: `timeseries_index_timedepth = 2`
+- index per day: `timeseries_index_timedepth = 3` (default)
+- index per hour: `timeseries_index_timedepth = 4`
 
 
 ## Index settings
@@ -192,18 +186,14 @@ class UsageRecord(MyBaseMetric):
   }
   ```
 
-* `DJELME_AUTOSETUP`: Optional feature, default `False` --
-    set `True` to run backend setup automatically when your django app starts
-    (like creating index templates in elasticsearch, if they don't already exist)
-
 * `DJELME_DEFAULT_TIMEDEPTH`: Set the granularity of timeseries indexes by the number of "time parts" in index names
     ```
     DJELME_DEFAULT_TIMEDEPTH = 1  # yearly indexes; YYYY
     DJELME_DEFAULT_TIMEDEPTH = 2  # monthly indexes; YYYY.MM
     DJELME_DEFAULT_TIMEDEPTH = 3  # daily indexes; YYYY.MM.DD (this is the default)
     DJELME_DEFAULT_TIMEDEPTH = 4  # hourly indexes; YYYY.MM.DD.HH
     ```
-    you can also set `Meta.timedepth` on a specific record type; this will take precedence
+    you can also set `Meta.timeseries_index_timedepth` on a specific record type; this will take precedence
 
 ## Management commands
 

elasticsearch_metrics/apps.py

@@ -1,38 +1,25 @@
 import collections
 
 from django.apps import AppConfig
-from django.conf import settings
 from django.utils.module_loading import autodiscover_modules
 
 from elasticsearch_metrics.registry import djelme_registry
 
-AUTOSETUP_SETTING = "DJELME_AUTOSETUP"
-
 
 class ElasticsearchMetricsConfig(AppConfig):
     name = "elasticsearch_metrics"
 
     def ready(self) -> None:
-        # load backends settings
-        _backend_names_by_module = collections.defaultdict(list)
-        for (
-            _backend_name,
-            _imp_module_name,
-            _,
-        ) in djelme_registry.each_backend_settings():
-            _backend_names_by_module[_imp_module_name].append(_backend_name)
         # discover any `foo.metrics` in installed apps
         autodiscover_modules("metrics")
-        # call `djelme_when_ready` for each imp module (only once)
-        for _imp_module_name, _backend_names in _backend_names_by_module.items():
+        # call `djelme_when_ready` once for each djelme imp module used by a backend
+        _backends_by_imp: dict[str, list[str]] = collections.defaultdict(list)
+        for _backend_name, _imp_name, _ in djelme_registry.each_backend_settings():
+            _backends_by_imp[_imp_name].append(_backend_name)
+        for _imp_module_name, _backend_names in _backends_by_imp.items():
             _imp_module = djelme_registry.get_imp_module(_imp_module_name)
             _imp_module.djelme_when_ready(
                 backends=[
                     djelme_registry.get_backend(_name) for _name in _backend_names
                 ]
             )
-        # autosetup? (default no)
-        if getattr(settings, AUTOSETUP_SETTING, False) is True:
-            _types_by_backend = djelme_registry.recordtypes_by_backend()
-            for _backend_name, _recordtypes in _types_by_backend.items():
-                djelme_registry.get_backend(_backend_name).djelme_setup(_recordtypes)

elasticsearch_metrics/exceptions.py

@@ -2,17 +2,33 @@ class DjelmeError(Exception):
     """Base class from which all django-elasticsearch-metrics -related exceptions inherit."""
 
 
-class TimeseriesSetupError(DjelmeError):
+class DjelmeSetupError(DjelmeError):
     """for errors that might be solved by `djelme_backend_setup`"""
 
 
-class IndexTemplateNotFoundError(TimeseriesSetupError):
+class IndexNotFoundError(DjelmeSetupError):
+    """specific index not found"""
+
+    def __init__(self, message, client_error):
+        self.client_error = client_error
+        super().__init__(message, client_error)
+
+
+class IndexOutOfSyncError(DjelmeSetupError):
+    """specific index has different mappings than expected"""
+
+
+class IndexTemplateNotFoundError(DjelmeSetupError):
+    """index template not found"""
+
     def __init__(self, message, client_error):
         self.client_error = client_error
         super().__init__(message, client_error)
 
 
-class IndexTemplateOutOfSyncError(TimeseriesSetupError):
+class IndexTemplateOutOfSyncError(DjelmeSetupError):
+    """index template has different mappings, settings, or patterns than expected"""
+
     def __init__(self, message, mappings_in_sync, patterns_in_sync, settings_in_sync):
         self.mappings_in_sync = mappings_in_sync
         self.patterns_in_sync = patterns_in_sync

elasticsearch_metrics/imps/elastic6.py

@@ -8,6 +8,7 @@
 from collections.abc import Iterator
 import dataclasses
 import datetime
+import functools
 import logging
 
 from django.apps import apps
@@ -137,12 +138,12 @@ def construct_index(cls, opts, bases):
 
     @property
     def _template_name(self):
-        _prefix = self.get_timeseries_name_prefix()
+        _prefix = self.get_index_name_prefix()
         return f"{_prefix}{self.__template_name}"
 
     @property
     def _template_pattern(self):
-        _prefix = self.get_timeseries_name_prefix()
+        _prefix = self.get_index_name_prefix()
         return f"{_prefix}{self.__template_pattern}"
 
 
@@ -188,7 +189,7 @@ def sync_index_template(cls, using=None):
         return index_template
 
     @classmethod
-    def check_index_template(cls, using: str | None = None) -> bool:
+    def check_index_template(cls, using: str | None = None) -> None:
         """Check if class is in sync with index template in Elasticsearch.
 
         :raise: IndexTemplateNotFoundError if index template does not exist.
@@ -204,7 +205,7 @@ def check_index_template(cls, using: str | None = None) -> bool:
             template_name = cls._template_name
             metric_name = cls.__name__
             raise exceptions.IndexTemplateNotFoundError(
-                "{template_name} does not exist for {metric_name}".format(**locals()),
+                f"Index template {template_name!r} does not exist for {metric_name}",
                 client_error=client_error,
             ) from client_error
         else:
@@ -241,18 +242,21 @@ def check_index_template(cls, using: str | None = None) -> bool:
                     [key for key, value in word_map.items() if not value]
                 )
                 raise exceptions.IndexTemplateOutOfSyncError(
-                    "{template_name} is out of sync with {metric_name} ({out_of_sync})".format(
-                        **locals()
-                    ),
+                    f"Index template {template_name!r} is out of sync with {metric_name} ({out_of_sync})",
                     mappings_in_sync=mappings_in_sync,
                     patterns_in_sync=patterns_in_sync,
                     settings_in_sync=settings_in_sync,
                 )
-            return True
 
     @classmethod
-    def check_djelme_setup(cls, using: str | None = None) -> bool:
-        return cls.check_index_template(using)
+    def check_djelme_setup(cls, using: str | None = None) -> None:
+        cls.check_index_template(using)
+
+    @classmethod
+    @functools.cache
+    def require_been_setup(cls, using: str | None = None) -> None:
+        """check setup once -- raise on failure, remember success"""
+        cls.check_djelme_setup(using)
 
     @classmethod
     def get_timeseries_index_template(cls):
@@ -262,7 +266,7 @@ def get_timeseries_index_template(cls):
         )
 
     @classmethod
-    def get_timeseries_name_prefix(cls) -> str:
+    def get_index_name_prefix(cls) -> str:
         return ""
 
     @classmethod
@@ -302,6 +306,7 @@ def save(self, using=None, index=None, validate=True, **kwargs):
         """Same as `Document.save`, except will save into the index determined
         by the metric's timestamp field.
         """
+        self.require_been_setup(using=using)  # prevent automapped indexes
         self.timestamp = self.timestamp or timezone.now()
         if not index:
             index = self.get_index_name(date=self.timestamp)
@@ -351,7 +356,7 @@ class DjelmeElastic6Backend:
     imp_kwargs: dict[str, str]
 
     @property
-    def elastic6_client(self):
+    def elastic_client(self):
         # assumes `connections.configure` was already called
         return connections.get_connection(self.backend_name)
 
@@ -373,9 +378,10 @@ def djelme_teardown(self, recordtypes: collections.abc.Iterable[type]) -> None:
             logger.info("tearing down %r", _metric_type)
             _indexname_wildcard = _metric_type._template_pattern
             _templatename = _metric_type._template_name
-            self.elastic6_client.indices.delete(index=_indexname_wildcard)
+            _client = self.elastic_client
+            _client.indices.delete(index=_indexname_wildcard)
             try:
-                self.elastic6_client.indices.delete_template(_templatename)
+                _client.indices.delete_template(_templatename)
             except NotFoundError:
                 pass