Merge pull request #11644 from bodintsov/feature/add-djelme

[ENG-9702] Feature/OSF to use latest djelme

Author: aaxelb

.docker-compose.env

@@ -7,6 +7,7 @@ INTERNAL_DOMAIN=http://192.168.168.167:5000/
 API_DOMAIN=http://localhost:8000/
 ELASTIC_URI=192.168.168.167:9200
 ELASTIC6_URI=192.168.168.167:9201
+ELASTIC8_URI=http://192.168.168.167:9202
 OSF_DB_HOST=192.168.168.167
 DB_HOST=192.168.168.167
 REDIS_HOST=redis://192.168.168.167:6379

addons/base/views.py

@@ -14,7 +14,7 @@
 import waffle
 from django.db import transaction
 from django.contrib.contenttypes.models import ContentType
-from elasticsearch import exceptions as es_exceptions
+from elasticsearch6 import exceptions as es_exceptions
 from rest_framework import status as http_status
 
 from api.caching.tasks import update_storage_usage_with_size

api/base/elasticsearch_dsl_views.py

@@ -3,7 +3,7 @@
 import datetime
 import typing
 
-import elasticsearch_dsl as edsl
+import elasticsearch6_dsl as edsl
 from rest_framework import generics, exceptions as drf_exceptions
 from rest_framework.settings import api_settings as drf_settings
 from api.base.settings.defaults import REPORT_FILENAME_FORMAT
@@ -23,7 +23,7 @@
 
 
 class ElasticsearchListView(FilterMixin, JSONAPIBaseView, generics.ListAPIView, abc.ABC):
-    '''abstract view class using `elasticsearch_dsl.Search` as a queryset-analogue
+    '''abstract view class using `elasticsearch6_dsl.Search` as a queryset-analogue
 
     builds a `Search` based on `self.get_default_search()` and the request's
     query parameters for filtering, sorting, and pagination -- fetches only
@@ -36,7 +36,7 @@ class ElasticsearchListView(FilterMixin, JSONAPIBaseView, generics.ListAPIView,
 
     @abc.abstractmethod
     def get_default_search(self) -> edsl.Search | None:
-        '''the base `elasticsearch_dsl.Search` for this list, based on url path
+        '''the base `elasticsearch6_dsl.Search` for this list, based on url path
 
         (common jsonapi query parameters will be considered automatically)
         '''
@@ -95,7 +95,7 @@ def finalize_response(self, request, response, *args, **kwargs):
     # (filtering handled in-view to reuse logic from FilterMixin)
     filter_backends = ()
 
-    # note: because elasticsearch_dsl.Search supports slicing and gives results when iterated on,
+    # note: because elasticsearch6_dsl.Search supports slicing and gives results when iterated on,
     #       it works fine with default pagination
 
     # override rest_framework.generics.GenericAPIView

api/base/settings/defaults.py

@@ -316,10 +316,18 @@
 HASHIDS_SALT = 'pinkhimalayan'
 
 # django-elasticsearch-metrics
-ELASTICSEARCH_DSL = {
-    'default': {
-        'hosts': osf_settings.ELASTIC6_URI,
-        'retry_on_timeout': True,
+DJELME_AUTOSETUP = True
+DJELME_BACKENDS = {
+    'osfmetrics_es6': {
+        'elasticsearch_metrics.imps.elastic6': {
+            'hosts': osf_settings.ELASTIC6_URI,
+            'retry_on_timeout': True,
+        },
+    },
+    'osfmetrics_es8': {
+        'elasticsearch_metrics.imps.elastic8': {
+            'hosts': osf_settings.ELASTIC8_URI,
+        },
     },
 }
 # Store yearly indices for time-series metrics

api/metrics/views.py

@@ -6,8 +6,8 @@
 from django.http import JsonResponse, HttpResponse, Http404
 from django.utils import timezone
 
-from elasticsearch.exceptions import NotFoundError, RequestError
-from elasticsearch_dsl.connections import get_connection
+from elasticsearch6.exceptions import NotFoundError, RequestError
+from elasticsearch6_dsl.connections import get_connection
 
 from framework.auth.oauth_scopes import CoreScopes
 

api_tests/institutions/views/test_institution_department_list.py

@@ -44,7 +44,7 @@ def populate_counts(self, user, user2, user3, user4, admin, institution):
             department_name='Old Department',
             public_project_count=1,
             private_project_count=1,
-        ).save(refresh=True)
+        ).save()
 
         _this_month = YearMonth.from_date(datetime.date.today())
 
@@ -56,7 +56,7 @@ def populate_counts(self, user, user2, user3, user4, admin, institution):
             department_name='New Department',
             public_project_count=1,
             private_project_count=1,
-        ).save(refresh=True)
+        ).save()
 
         # A second user entered the department
         InstitutionalUserReport(
@@ -66,7 +66,7 @@ def populate_counts(self, user, user2, user3, user4, admin, institution):
             department_name='New Department',
             public_project_count=1,
             private_project_count=1,
-        ).save(refresh=True)
+        ).save()
 
         # A new department with a single user to test sorting
         InstitutionalUserReport(
@@ -76,7 +76,7 @@ def populate_counts(self, user, user2, user3, user4, admin, institution):
             department_name='Smaller Department',
             public_project_count=1,
             private_project_count=1,
-        ).save(refresh=True)
+        ).save()
 
         # A user with no department
         InstitutionalUserReport(
@@ -85,7 +85,7 @@ def populate_counts(self, user, user2, user3, user4, admin, institution):
             institution_id=institution._id,
             public_project_count=1,
             private_project_count=1,
-        ).save(refresh=True)
+        ).save()
 
     @pytest.fixture()
     def admin(self, institution):
@@ -113,6 +113,7 @@ def test_auth(self, app, url, user, admin):
         assert resp.json['data'] == []
 
     def test_get(self, app, url, admin, institution, populate_counts):
+        InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern)
         resp = app.get(url, auth=admin.auth)
 
         assert resp.json['data'] == [{

api_tests/institutions/views/test_institution_summary_metrics.py

@@ -84,6 +84,7 @@ def test_get_empty(self, app, url, institutional_admin):
         assert resp.json['meta'] == {'version': '2.0'}
 
     def test_get_report(self, app, url, institutional_admin, institution, reports, unshown_reports):
+        InstitutionMonthlySummaryReport._get_connection().indices.refresh(InstitutionMonthlySummaryReport._template_pattern)
         resp = app.get(url, auth=institutional_admin.auth)
         assert resp.status_code == 200
 
@@ -149,6 +150,7 @@ def test_get_report_with_multiple_months_and_institutions(
             monthly_logged_in_user_count=270,
             monthly_active_user_count=260,
         )
+        InstitutionMonthlySummaryReport._get_connection().indices.refresh(InstitutionMonthlySummaryReport._template_pattern)
 
         resp = app.get(url, auth=institutional_admin.auth)
         assert resp.status_code == 200
@@ -189,6 +191,7 @@ def test_get_with_valid_report_dates(self, app, url, institution, institutional_
             institution,
             user_count=4133,
         )
+        InstitutionMonthlySummaryReport._get_connection().indices.refresh(InstitutionMonthlySummaryReport._template_pattern)
 
         resp = app.get(f'{url}?report_yearmonth=2024-08', auth=institutional_admin.auth)
         assert resp.status_code == 200
@@ -213,6 +216,7 @@ def test_get_with_invalid_report_date(self, app, url, institution, institutional
             institution,
             user_count=999,
         )
+        InstitutionMonthlySummaryReport._get_connection().indices.refresh(InstitutionMonthlySummaryReport._template_pattern)
 
         # Request with an invalid report_date format
         resp = app.get(f'{url}?report_yearmonth=invalid-date', auth=institutional_admin.auth)
@@ -233,6 +237,7 @@ def test_get_without_report_date_uses_most_recent(self, app, url, institution, i
             institution,
             user_count=999,
         )
+        InstitutionMonthlySummaryReport._get_connection().indices.refresh(InstitutionMonthlySummaryReport._template_pattern)
 
         resp = app.get(url, auth=institutional_admin.auth)
         assert resp.status_code == 200
@@ -247,5 +252,5 @@ def _summary_report_factory(yearmonth, institution, **kwargs):
         institution_id=institution._id,
         **kwargs,
     )
-    report.save(refresh=True)
+    report.save()
     return report

api_tests/institutions/views/test_institution_user_metric_list.py

@@ -89,6 +89,7 @@ def test_get_empty(self, app, url, institutional_admin):
         assert _resp.json['data'] == []
 
     def test_get_reports(self, app, url, institutional_admin, institution, reports, unshown_reports):
+        InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern)
         _resp = app.get(url, auth=institutional_admin.auth)
         assert _resp.status_code == 200
         assert len(_resp.json['data']) == len(reports)
@@ -100,6 +101,7 @@ def test_get_reports(self, app, url, institutional_admin, institution, reports,
             assert len(response_object['attributes']['contacts']) == 0
 
     def test_filter_reports(self, app, url, institutional_admin, institution, reports, unshown_reports):
+        InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern)
         for _query, _expected_user_ids in (
             ({'filter[department]': 'nunavum'}, set()),
             ({'filter[department]': 'incidentally'}, set()),
@@ -135,6 +137,7 @@ def test_filter_reports(self, app, url, institutional_admin, institution, report
             assert set(_user_ids(_resp)) == _expected_user_ids
 
     def test_sort_reports(self, app, url, institutional_admin, institution, reports, unshown_reports):
+        InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern)
         for _query, _expected_user_id_list in (
             ({'sort': 'storage_byte_count'}, ['u_sparse', 'u_orc', 'u_blargl', 'u_orcomma']),
             ({'sort': '-storage_byte_count'}, ['u_orcomma', 'u_blargl', 'u_orc', 'u_sparse']),
@@ -144,6 +147,7 @@ def test_sort_reports(self, app, url, institutional_admin, institution, reports,
             assert list(_user_ids(_resp)) == _expected_user_id_list
 
     def test_paginate_reports(self, app, url, institutional_admin, institution, reports, unshown_reports):
+        InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern)
         for _query, _expected_user_id_list in (
             ({'sort': 'storage_byte_count', 'page[size]': 2}, ['u_sparse', 'u_orc']),
             ({'sort': 'storage_byte_count', 'page[size]': 2, 'page': 2}, ['u_blargl', 'u_orcomma']),
@@ -178,6 +182,7 @@ def test_get_report_formats_csv_tsv(self, app, url, institutional_admin, institu
             month_last_active='2018-02',
             month_last_login='2018-02',
         )
+        InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern)
 
         resp = app.get(f'{url}?format={format_type}', auth=institutional_admin.auth)
         assert resp.status_code == 200
@@ -281,6 +286,7 @@ def test_csv_tsv_ignores_pagination(self, app, url, institutional_admin, institu
                 str(736662999298 + i),
                 f'Jalen Hurts #{i}',
             ])
+        InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern)
 
         # Make request for CSV format with page[size]=10
         resp = app.get(f'{url}?format={format_type}', auth=institutional_admin.auth)
@@ -346,6 +352,7 @@ def test_get_report_format_table_json(self, app, url, institutional_admin, insti
             month_last_active='2018-02',
             month_last_login='2018-02',
         )
+        InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern)
 
         resp = app.get(f'{url}?format=json_report', auth=institutional_admin.auth)
         assert resp.status_code == 200
@@ -411,6 +418,7 @@ def test_correct_number_of_contact_messages(self, app, url, institutional_admin,
             department_name='a department, or so, that happens, incidentally, to have commas',
             storage_byte_count=736662999298,
         )
+        InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern)
 
         receiver = user1
         with capture_notifications():
@@ -477,5 +485,5 @@ def _report_factory(yearmonth, institution, **kwargs):
         institution_id=institution._id,
         **kwargs,
     )
-    _report.save(refresh=True)
+    _report.save()
     return _report

api_tests/metrics/test_composite_query.py

@@ -1,4 +1,3 @@
-import time
 import pytest
 from datetime import datetime
 from osf_tests.factories import (
@@ -75,7 +74,7 @@ def test_elasticsearch_agg_query(self, app, user, base_url, preprint):
             path=preprint.primary_file.path,
             timestamp=datetime(year=2020, month=2, day=1)
         )
-        time.sleep(1)  # gives ES some time to update
+        PreprintDownload._get_connection().indices.refresh(PreprintDownload._template_pattern)
 
         resp = app.post_json_api(post_url, payload, auth=user.auth)
         assert resp.status_code == 200

api_tests/metrics/test_preprint_metrics.py

@@ -8,7 +8,7 @@
 
 from django.utils import timezone
 from waffle.testutils import override_switch
-from elasticsearch.exceptions import RequestError
+from elasticsearch6.exceptions import RequestError
 
 from osf import features
 from api.base.settings import API_PRIVATE_BASE as API_BASE