[Feature][ENG-9691] osfmetrics elastic-6-to-8 migration (part 1)

.docker-compose.env

@@ -7,6 +7,8 @@ INTERNAL_DOMAIN=http://192.168.168.167:5000/
 API_DOMAIN=http://localhost:8000/
 ELASTIC_URI=192.168.168.167:9200
 ELASTIC6_URI=192.168.168.167:9201
+ELASTIC8_URI=http://192.168.168.167:9202
+ELASTIC8_USERNAME=elastic
 OSF_DB_HOST=192.168.168.167
 DB_HOST=192.168.168.167
 REDIS_HOST=redis://192.168.168.167:6379

.github/workflows/test-build.yml

@@ -37,7 +37,19 @@ jobs:
     permissions:
       checks: write
     services:
-      postgres:
+      elasticsearch8: &ES8_SERVICE
+        image: elasticsearch:8.19.14
+        ports:
+          - 9202:9200
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: false
+        options: >-
+          --health-cmd "curl -sf http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=30s"
+          --health-interval 10s
+          --health-timeout 30s
+          --health-retries 5
+      postgres: &POSTGRES_SERVICE
         image: postgres
         env:
           POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
@@ -54,6 +66,8 @@ jobs:
     - uses: ./.github/actions/start-build
     - name: Run tests
       run: poetry run python3 -m invoke test-ci-addons --junit
+      env:
+        ELASTIC8_URI: http://localhost:9202
     - name: Upload report
       if: (success() || failure())    # run this step even if previous step failed
       uses: ./.github/actions/gen-report
@@ -64,18 +78,7 @@ jobs:
     permissions:
       checks: write
     services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+      postgres: *POSTGRES_SERVICE
     steps:
     - uses: actions/checkout@v2
     - uses: ./.github/actions/start-build
@@ -91,25 +94,17 @@ jobs:
     permissions:
       checks: write
     services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+      elasticsearch8: *ES8_SERVICE
+      postgres: *POSTGRES_SERVICE
     steps:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/start-build
       - name: NVM & yarn install
         run: poetry run python3 -m invoke assets --dev
       - name: Run test
         run: poetry run python3 -m invoke test-ci-api1-and-js --junit
+        env:
+          ELASTIC8_URI: http://localhost:9202
       - name: Upload report
         if: (success() || failure())    # run this step even if previous step failed
         uses: ./.github/actions/gen-report
@@ -120,23 +115,15 @@ jobs:
     permissions:
       checks: write
     services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+      elasticsearch8: *ES8_SERVICE
+      postgres: *POSTGRES_SERVICE
     steps:
     - uses: actions/checkout@v2
     - uses: ./.github/actions/start-build
     - name: Run tests
       run: poetry run python3 -m invoke test-ci-api2 --junit
+      env:
+        ELASTIC8_URI: http://localhost:9202
     - name: Upload report
       if: (success() || failure())    # run this step even if previous step failed
       uses: ./.github/actions/gen-report
@@ -147,19 +134,7 @@ jobs:
       checks: write
     needs: build-cache
     services:
-      postgres:
-        image: postgres
-
-        env:
-          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+      postgres: *POSTGRES_SERVICE
     steps:
     - uses: actions/checkout@v2
     - uses: ./.github/actions/start-build
@@ -175,19 +150,7 @@ jobs:
       checks: write
     needs: build-cache
     services:
-      postgres:
-        image: postgres
-
-        env:
-          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+      postgres: *POSTGRES_SERVICE
       mailhog:
         image: mailhog/mailhog
         ports:
@@ -208,19 +171,7 @@ jobs:
       checks: write
     needs: build-cache
     services:
-      postgres:
-        image: postgres
-
-        env:
-          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+      postgres: *POSTGRES_SERVICE
     steps:
     - uses: actions/checkout@v2
     - uses: ./.github/actions/start-build

addons/base/views.py

@@ -14,7 +14,7 @@
 import waffle
 from django.db import transaction
 from django.contrib.contenttypes.models import ContentType
-from elasticsearch import exceptions as es_exceptions
+from elasticsearch6 import exceptions as es_exceptions
 from rest_framework import status as http_status
 
 from api.caching.tasks import update_storage_usage_with_size

api/base/elasticsearch_dsl_views.py

@@ -3,7 +3,7 @@
 import datetime
 import typing
 
-import elasticsearch_dsl as edsl
+import elasticsearch6_dsl as edsl
 from rest_framework import generics, exceptions as drf_exceptions
 from rest_framework.settings import api_settings as drf_settings
 from api.base.settings.defaults import REPORT_FILENAME_FORMAT
@@ -23,7 +23,7 @@
 
 
 class ElasticsearchListView(FilterMixin, JSONAPIBaseView, generics.ListAPIView, abc.ABC):
-    '''abstract view class using `elasticsearch_dsl.Search` as a queryset-analogue
+    '''abstract view class using `elasticsearch6_dsl.Search` as a queryset-analogue
 
     builds a `Search` based on `self.get_default_search()` and the request's
     query parameters for filtering, sorting, and pagination -- fetches only
@@ -36,7 +36,7 @@ class ElasticsearchListView(FilterMixin, JSONAPIBaseView, generics.ListAPIView,
 
     @abc.abstractmethod
     def get_default_search(self) -> edsl.Search | None:
-        '''the base `elasticsearch_dsl.Search` for this list, based on url path
+        '''the base `elasticsearch6_dsl.Search` for this list, based on url path
 
         (common jsonapi query parameters will be considered automatically)
         '''
@@ -95,7 +95,7 @@ def finalize_response(self, request, response, *args, **kwargs):
     # (filtering handled in-view to reuse logic from FilterMixin)
     filter_backends = ()
 
-    # note: because elasticsearch_dsl.Search supports slicing and gives results when iterated on,
+    # note: because elasticsearch6_dsl.Search supports slicing and gives results when iterated on,
     #       it works fine with default pagination
 
     # override rest_framework.generics.GenericAPIView

api/base/settings/defaults.py

@@ -316,10 +316,23 @@
 HASHIDS_SALT = 'pinkhimalayan'
 
 # django-elasticsearch-metrics
-ELASTICSEARCH_DSL = {
-    'default': {
-        'hosts': osf_settings.ELASTIC6_URI,
-        'retry_on_timeout': True,
+DJELME_BACKENDS = {
+    'osfmetrics_es6': {
+        'elasticsearch_metrics.imps.elastic6': {
+            'hosts': osf_settings.ELASTIC6_URI,
+            'retry_on_timeout': True,
+        },
+    },
+    'osfmetrics_es8': {
+        'elasticsearch_metrics.imps.elastic8': {
+            'hosts': osf_settings.ELASTIC8_URI,
+            'ca_certs': osf_settings.ELASTIC8_CERT_PATH,
+            'basic_auth': (
+                (osf_settings.ELASTIC8_USERNAME, osf_settings.ELASTIC8_SECRET)
+                if osf_settings.ELASTIC8_SECRET is not None
+                else None
+            ),
+        },
     },
 }
 # Store yearly indices for time-series metrics

api/metrics/views.py

@@ -6,8 +6,8 @@
 from django.http import JsonResponse, HttpResponse, Http404
 from django.utils import timezone
 
-from elasticsearch.exceptions import NotFoundError, RequestError
-from elasticsearch_dsl.connections import get_connection
+from elasticsearch6.exceptions import NotFoundError, RequestError
+from elasticsearch6_dsl.connections import get_connection
 
 from framework.auth.oauth_scopes import CoreScopes
 

api_tests/institutions/views/test_institution_department_list.py

@@ -44,7 +44,7 @@ def populate_counts(self, user, user2, user3, user4, admin, institution):
             department_name='Old Department',
             public_project_count=1,
             private_project_count=1,
-        ).save(refresh=True)
+        ).save()
 
         _this_month = YearMonth.from_date(datetime.date.today())
 
@@ -56,7 +56,7 @@ def populate_counts(self, user, user2, user3, user4, admin, institution):
             department_name='New Department',
             public_project_count=1,
             private_project_count=1,
-        ).save(refresh=True)
+        ).save()
 
         # A second user entered the department
         InstitutionalUserReport(
@@ -66,7 +66,7 @@ def populate_counts(self, user, user2, user3, user4, admin, institution):
             department_name='New Department',
             public_project_count=1,
             private_project_count=1,
-        ).save(refresh=True)
+        ).save()
 
         # A new department with a single user to test sorting
         InstitutionalUserReport(
@@ -76,7 +76,7 @@ def populate_counts(self, user, user2, user3, user4, admin, institution):
             department_name='Smaller Department',
             public_project_count=1,
             private_project_count=1,
-        ).save(refresh=True)
+        ).save()
 
         # A user with no department
         InstitutionalUserReport(
@@ -85,7 +85,7 @@ def populate_counts(self, user, user2, user3, user4, admin, institution):
             institution_id=institution._id,
             public_project_count=1,
             private_project_count=1,
-        ).save(refresh=True)
+        ).save()
 
     @pytest.fixture()
     def admin(self, institution):
@@ -113,6 +113,7 @@ def test_auth(self, app, url, user, admin):
         assert resp.json['data'] == []
 
     def test_get(self, app, url, admin, institution, populate_counts):
+        InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern)
         resp = app.get(url, auth=admin.auth)
 
         assert resp.json['data'] == [{

api_tests/institutions/views/test_institution_summary_metrics.py

@@ -84,6 +84,7 @@ def test_get_empty(self, app, url, institutional_admin):
         assert resp.json['meta'] == {'version': '2.0'}
 
     def test_get_report(self, app, url, institutional_admin, institution, reports, unshown_reports):
+        InstitutionMonthlySummaryReport._get_connection().indices.refresh(InstitutionMonthlySummaryReport._template_pattern)
         resp = app.get(url, auth=institutional_admin.auth)
         assert resp.status_code == 200
 
@@ -149,6 +150,7 @@ def test_get_report_with_multiple_months_and_institutions(
             monthly_logged_in_user_count=270,
             monthly_active_user_count=260,
         )
+        InstitutionMonthlySummaryReport._get_connection().indices.refresh(InstitutionMonthlySummaryReport._template_pattern)
 
         resp = app.get(url, auth=institutional_admin.auth)
         assert resp.status_code == 200
@@ -189,6 +191,7 @@ def test_get_with_valid_report_dates(self, app, url, institution, institutional_
             institution,
             user_count=4133,
         )
+        InstitutionMonthlySummaryReport._get_connection().indices.refresh(InstitutionMonthlySummaryReport._template_pattern)
 
         resp = app.get(f'{url}?report_yearmonth=2024-08', auth=institutional_admin.auth)
         assert resp.status_code == 200
@@ -213,6 +216,7 @@ def test_get_with_invalid_report_date(self, app, url, institution, institutional
             institution,
             user_count=999,
         )
+        InstitutionMonthlySummaryReport._get_connection().indices.refresh(InstitutionMonthlySummaryReport._template_pattern)
 
         # Request with an invalid report_date format
         resp = app.get(f'{url}?report_yearmonth=invalid-date', auth=institutional_admin.auth)
@@ -233,6 +237,7 @@ def test_get_without_report_date_uses_most_recent(self, app, url, institution, i
             institution,
             user_count=999,
         )
+        InstitutionMonthlySummaryReport._get_connection().indices.refresh(InstitutionMonthlySummaryReport._template_pattern)
 
         resp = app.get(url, auth=institutional_admin.auth)
         assert resp.status_code == 200
@@ -247,5 +252,5 @@ def _summary_report_factory(yearmonth, institution, **kwargs):
         institution_id=institution._id,
         **kwargs,
     )
-    report.save(refresh=True)
+    report.save()
     return report