[ENG-11458] fix: osfmetrics data lost in migration (#11774)

* fix: migrate_osfmetrics_fix_6to8

make sure all items get a recent monthly usage report with cumulative
counts up to this point -- previous data migration incorrectly assumed
monthly usage reports were either complete or completely missing

- restore some previously-removed elastic6 config and dsl types in
  osf.metrics.es6_metrics
- change django-elasticsearch-metrics dependency to a version with
  elastic6 support added back in
- add migrate_osfmetrics_fix_6to8 management command
    - shows diagnostic counts from es6 and es8 (unless --no-counts)
    - starts a task for each item with any past usage to add a usage
      report for 2026-05 with accurate cumulative view/download counts
    - allow running via osf-admin web ui

* add es6 back to github actions

* fix: orderable YearMonth

* more correct counts

* loggable error on non-existant osfid

* more accurate comparison count

* move usage report epoch to setting

* finding anomalies...

* further inspecting anomalies...

* fix up whoopsies

* better handle preprint osfids

* fix some older usage reports too

* use date_histogram to skip empty months

* remove debugging

Author: aaxelb

.docker-compose.env

@@ -6,6 +6,7 @@ DOMAIN=http://localhost:5000/
 INTERNAL_DOMAIN=http://192.168.168.167:5000/
 API_DOMAIN=http://localhost:8000/
 ELASTIC_URI=192.168.168.167:9200
+ELASTIC6_URI=192.168.168.167:9201
 ELASTIC8_URI=http://192.168.168.167:9202
 ELASTIC8_USERNAME=elastic
 OSF_DB_HOST=192.168.168.167

.github/workflows/test-build.yml

@@ -48,6 +48,15 @@ jobs:
           --health-interval 10s
           --health-timeout 30s
           --health-retries 5
+      elasticsearch6: &ES6_SERVICE
+        image: elasticsearch:6.8.23
+        ports:
+          - 9201:9200
+        options: >-
+          --health-cmd "curl -sf http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=30s"
+          --health-interval 10s
+          --health-timeout 30s
+          --health-retries 5
       postgres: &POSTGRES_SERVICE
         image: postgres
         env:
@@ -67,6 +76,7 @@ jobs:
       run: poetry run python3 -m invoke test-ci-addons --junit
       env:
         ELASTIC8_URI: http://localhost:9202
+        ELASTIC6_URI: http://localhost:9201
     - name: Upload report
       if: (success() || failure())    # run this step even if previous step failed
       uses: ./.github/actions/gen-report
@@ -94,6 +104,7 @@ jobs:
       checks: write
     services:
       elasticsearch8: *ES8_SERVICE
+      elasticsearch6: *ES6_SERVICE
       postgres: *POSTGRES_SERVICE
     steps:
       - uses: actions/checkout@v6
@@ -104,6 +115,7 @@ jobs:
         run: poetry run python3 -m invoke test-ci-api1-and-js --junit
         env:
           ELASTIC8_URI: http://localhost:9202
+          ELASTIC6_URI: http://localhost:9201
       - name: Upload report
         if: (success() || failure())    # run this step even if previous step failed
         uses: ./.github/actions/gen-report
@@ -115,6 +127,7 @@ jobs:
       checks: write
     services:
       elasticsearch8: *ES8_SERVICE
+      elasticsearch6: *ES6_SERVICE
       postgres: *POSTGRES_SERVICE
     steps:
     - uses: actions/checkout@v6
@@ -123,6 +136,7 @@ jobs:
       run: poetry run python3 -m invoke test-ci-api2 --junit
       env:
         ELASTIC8_URI: http://localhost:9202
+        ELASTIC6_URI: http://localhost:9201
     - name: Upload report
       if: (success() || failure())    # run this step even if previous step failed
       uses: ./.github/actions/gen-report
@@ -141,6 +155,7 @@ jobs:
       run: poetry run python3 -m invoke test-ci-api3-and-osf --junit
       env:
         ELASTIC8_URI: http://localhost:9202
+        ELASTIC6_URI: http://localhost:9201
     - name: Upload report
       if: (success() || failure())    # run this step even if previous step failed
       uses: ./.github/actions/gen-report

admin/management/urls.py

@@ -1,4 +1,4 @@
-from django.urls import re_path
+from django.urls import re_path, path
 
 from admin.management import views
 
@@ -22,4 +22,5 @@
             name='sync_notification_templates'),
     re_path(r'^remove_orcid_from_user_social', views.RemoveOrcidFromUserSocial.as_view(),
             name='remove_orcid_from_user_social'),
+    path('migrate_osfmetrics_fix_6to8', views.MigrateOsfmetricsFix6to8.as_view(), name='migrate_osfmetrics_fix_6to8'),
 ]

admin/management/views.py

@@ -1,3 +1,5 @@
+from io import StringIO
+
 from dateutil.parser import isoparse
 from django.views.generic import TemplateView, View
 from django.contrib import messages
@@ -203,3 +205,18 @@ def post(self, request):
         remove_orcid_from_user_social()
         messages.success(request, 'Orcid from user social have been successfully removed.')
         return redirect(reverse('management:commands'))
+
+
+class MigrateOsfmetricsFix6to8(ManagementCommandPermissionView):
+    def post(self, request):
+        _command_kwargs = {
+            'no_color': True,
+            'no_counts': request.POST.get('no_counts'),
+            'delete_es8_usage_reports': request.POST.get('delete_es8_usage_reports'),
+            'start': request.POST.get('start'),
+        }
+        _out_io = StringIO()
+        call_command('migrate_osfmetrics_fix_6to8', **_command_kwargs, stdout=_out_io)
+        for _line in _out_io.getvalue().split('\n'):
+            messages.info(request, _line)
+        return redirect(reverse('management:commands'))

admin/templates/management/commands.html

@@ -190,6 +190,23 @@ <h4><u>Remove existing orcid info from user social</u></h4>
                     </nav>
                 </form>
             </section>
+            <section>
+                <h4><u>migrate osf-metrics (fix) 6to8</u></h4>
+                <p>
+                    view progress of the osf-metrics migration (fixup) from elastic6 to elastic8 (or start it)
+                </p>
+                <form method="post"
+                      action="{% url 'management:migrate_osfmetrics_fix_6to8'%}"
+                      style="display: flex; flex-direction: column;">
+                    {% csrf_token %}
+                    <label><input type="checkbox" name="no_counts"> no counts</label>
+                    <label><input type="checkbox" name="delete_es8_usage_reports"> delete es8 usage reports (caution)</label>
+                    <label><input type="checkbox" name="start"> start tasks (caution)</label>
+                    <nav>
+                        <input class="btn btn-success" type="submit" value="Run" />
+                    </nav>
+                </form>
+            </section>
         </div>
     </section>
 {% endblock %}

api/base/settings/defaults.py

@@ -321,6 +321,12 @@
 
 # django-elasticsearch-metrics
 DJELME_BACKENDS = {
+    'osfmetrics_es6': {
+        'elasticsearch_metrics.imps.elastic6': {
+            'hosts': osf_settings.ELASTIC6_URI,
+            'retry_on_timeout': True,
+        },
+    },
     'osfmetrics_es8': {
         'elasticsearch_metrics.imps.elastic8': {
             # passthru kwargs to elasticsearch8 connection constructor
@@ -338,6 +344,8 @@
     },
 }
 OSF_USAGEEVENT_EXPIRATION_DAYS = 90
+ELASTICSEARCH_METRICS_DATE_FORMAT = '%Y'
+MONTHLY_USAGE_REPORT_EPOCH = '2026-05'  # cannot create monthly usage reports before this point
 
 WAFFLE_CACHE_NAME = 'waffle_cache'
 STORAGE_USAGE_CACHE_NAME = 'storage_usage'

api/metrics/views.py

@@ -136,8 +136,9 @@ def _do_es_request(self, django_request, djelme_backend_name, method, path):
             {'Content-Type': _content_type, 'Accept': 'application/json'}
             if _content_type else None
         )
+        _perform_fn = getattr(_client, 'perform_request', None) or _client.transport.perform_request
         try:
-            _response = _client.perform_request(
+            _response = _perform_fn(
                 method,
                 f'/{path}',
                 params=django_request.GET.dict(),
@@ -150,7 +151,7 @@ def _do_es_request(self, django_request, djelme_backend_name, method, path):
                 content_type='text/plain; charset=utf-8',
                 status=_api_error.status_code,
             )
-        return JsonResponse(_response.body)
+        return JsonResponse(_response if isinstance(_response, dict) else _response.body)
 
     def _get_es_client(self, djelme_backend_name):
         try:

docker-compose.yml

@@ -13,6 +13,8 @@ volumes:
     external: false
   elasticsearch8_data_vol:
     external: false
+  elasticsearch6_data_vol:
+    external: false
   rabbitmq_vol:
     external: false
   preprints_dist_vol:
@@ -83,6 +85,22 @@ services:
       retries: 30
     stdin_open: true
 
+  # Temporary: Remove when done with es6
+  elasticsearch6:
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.3.1
+    environment:
+      - ES_JAVA_OPTS=-Xms512m -Xmx512m  # reduce memory usage
+    ports:
+      - 9201:9200
+    volumes:
+      - elasticsearch6_data_vol:/usr/share/elasticsearch/data
+    healthcheck:
+      start_period: 15s
+      test: curl -s http://localhost:9200/_cluster/health | grep -vq '"status":"red"'
+      interval: 10s
+      retries: 30
+    stdin_open: true
+
   postgres:
     image: postgres:15.4
     command: