Skip to content

Commit 6c45a66

Browse files
aaxelbaaxelb
authored
Merge pull request #11672 from bodintsov/feature/add-new-es8-metrics
[ENG-9703] Feature/add new es8 metrics
2 parents af0dfae + 1cef7d3 commit 6c45a66

File tree

17 files changed

+481
-118
lines changed

17 files changed

+481
-118
lines changed

‎.docker-compose.env‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ API_DOMAIN=http://localhost:8000/
88
ELASTIC_URI=192.168.168.167:9200
99
ELASTIC6_URI=192.168.168.167:9201
1010
ELASTIC8_URI=http://192.168.168.167:9202
11+
ELASTIC8_USERNAME=elastic
1112
OSF_DB_HOST=192.168.168.167
1213
DB_HOST=192.168.168.167
1314
REDIS_HOST=redis://192.168.168.167:6379

‎.github/workflows/test-build.yml‎

Lines changed: 27 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,19 @@ jobs:
3737
permissions:
3838
checks: write
3939
services:
40-
postgres:
40+
elasticsearch8: &ES8_SERVICE
41+
image: elasticsearch:8.19.14
42+
ports:
43+
- 9202:9200
44+
env:
45+
discovery.type: single-node
46+
xpack.security.enabled: false
47+
options: >-
48+
--health-cmd "curl -sf http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=30s"
49+
--health-interval 10s
50+
--health-timeout 30s
51+
--health-retries 5
52+
postgres: &POSTGRES_SERVICE
4153
image: postgres
4254
env:
4355
POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
@@ -54,6 +66,8 @@ jobs:
5466
- uses: ./.github/actions/start-build
5567
- name: Run tests
5668
run: poetry run python3 -m invoke test-ci-addons --junit
69+
env:
70+
ELASTIC8_URI: http://localhost:9202
5771
- name: Upload report
5872
if: (success() || failure()) # run this step even if previous step failed
5973
uses: ./.github/actions/gen-report
@@ -64,18 +78,7 @@ jobs:
6478
permissions:
6579
checks: write
6680
services:
67-
postgres:
68-
image: postgres
69-
env:
70-
POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
71-
options: >-
72-
--health-cmd pg_isready
73-
--health-interval 10s
74-
--health-timeout 5s
75-
--health-retries 5
76-
ports:
77-
# Maps tcp port 5432 on service container to the host
78-
- 5432:5432
81+
postgres: *POSTGRES_SERVICE
7982
steps:
8083
- uses: actions/checkout@v2
8184
- uses: ./.github/actions/start-build
@@ -91,25 +94,17 @@ jobs:
9194
permissions:
9295
checks: write
9396
services:
94-
postgres:
95-
image: postgres
96-
env:
97-
POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
98-
options: >-
99-
--health-cmd pg_isready
100-
--health-interval 10s
101-
--health-timeout 5s
102-
--health-retries 5
103-
ports:
104-
# Maps tcp port 5432 on service container to the host
105-
- 5432:5432
97+
elasticsearch8: *ES8_SERVICE
98+
postgres: *POSTGRES_SERVICE
10699
steps:
107100
- uses: actions/checkout@v2
108101
- uses: ./.github/actions/start-build
109102
- name: NVM & yarn install
110103
run: poetry run python3 -m invoke assets --dev
111104
- name: Run test
112105
run: poetry run python3 -m invoke test-ci-api1-and-js --junit
106+
env:
107+
ELASTIC8_URI: http://localhost:9202
113108
- name: Upload report
114109
if: (success() || failure()) # run this step even if previous step failed
115110
uses: ./.github/actions/gen-report
@@ -120,23 +115,15 @@ jobs:
120115
permissions:
121116
checks: write
122117
services:
123-
postgres:
124-
image: postgres
125-
env:
126-
POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
127-
options: >-
128-
--health-cmd pg_isready
129-
--health-interval 10s
130-
--health-timeout 5s
131-
--health-retries 5
132-
ports:
133-
# Maps tcp port 5432 on service container to the host
134-
- 5432:5432
118+
elasticsearch8: *ES8_SERVICE
119+
postgres: *POSTGRES_SERVICE
135120
steps:
136121
- uses: actions/checkout@v2
137122
- uses: ./.github/actions/start-build
138123
- name: Run tests
139124
run: poetry run python3 -m invoke test-ci-api2 --junit
125+
env:
126+
ELASTIC8_URI: http://localhost:9202
140127
- name: Upload report
141128
if: (success() || failure()) # run this step even if previous step failed
142129
uses: ./.github/actions/gen-report
@@ -147,19 +134,7 @@ jobs:
147134
checks: write
148135
needs: build-cache
149136
services:
150-
postgres:
151-
image: postgres
152-
153-
env:
154-
POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
155-
options: >-
156-
--health-cmd pg_isready
157-
--health-interval 10s
158-
--health-timeout 5s
159-
--health-retries 5
160-
ports:
161-
# Maps tcp port 5432 on service container to the host
162-
- 5432:5432
137+
postgres: *POSTGRES_SERVICE
163138
steps:
164139
- uses: actions/checkout@v2
165140
- uses: ./.github/actions/start-build
@@ -175,19 +150,7 @@ jobs:
175150
checks: write
176151
needs: build-cache
177152
services:
178-
postgres:
179-
image: postgres
180-
181-
env:
182-
POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
183-
options: >-
184-
--health-cmd pg_isready
185-
--health-interval 10s
186-
--health-timeout 5s
187-
--health-retries 5
188-
ports:
189-
# Maps tcp port 5432 on service container to the host
190-
- 5432:5432
153+
postgres: *POSTGRES_SERVICE
191154
mailhog:
192155
image: mailhog/mailhog
193156
ports:
@@ -208,19 +171,7 @@ jobs:
208171
checks: write
209172
needs: build-cache
210173
services:
211-
postgres:
212-
image: postgres
213-
214-
env:
215-
POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
216-
options: >-
217-
--health-cmd pg_isready
218-
--health-interval 10s
219-
--health-timeout 5s
220-
--health-retries 5
221-
ports:
222-
# Maps tcp port 5432 on service container to the host
223-
- 5432:5432
174+
postgres: *POSTGRES_SERVICE
224175
steps:
225176
- uses: actions/checkout@v2
226177
- uses: ./.github/actions/start-build

‎api/base/settings/defaults.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -316,7 +316,6 @@
316316
HASHIDS_SALT = 'pinkhimalayan'
317317

318318
# django-elasticsearch-metrics
319-
DJELME_AUTOSETUP = True
320319
DJELME_BACKENDS = {
321320
'osfmetrics_es6': {
322321
'elasticsearch_metrics.imps.elastic6': {
@@ -327,6 +326,12 @@
327326
'osfmetrics_es8': {
328327
'elasticsearch_metrics.imps.elastic8': {
329328
'hosts': osf_settings.ELASTIC8_URI,
329+
'ca_certs': osf_settings.ELASTIC8_CERT_PATH,
330+
'basic_auth': (
331+
(osf_settings.ELASTIC8_USERNAME, osf_settings.ELASTIC8_SECRET)
332+
if osf_settings.ELASTIC8_SECRET is not None
333+
else None
334+
),
330335
},
331336
},
332337
}

‎api_tests/metrics/test_counted_usage.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,9 @@ def assert_saved_with(mock_save, *, expected_doc_id=None, expected_attrs):
3838

3939
@pytest.fixture
4040
def mock_save():
41-
with mock.patch('elasticsearch6_dsl.Document.save', autospec=True) as mock_save:
42-
yield mock_save
41+
with mock.patch('elasticsearch_metrics.imps.elastic6.BaseMetric.check_index_template'):
42+
with mock.patch('elasticsearch6_dsl.Document.save', autospec=True) as mock_save:
43+
yield mock_save
4344

4445

4546
@pytest.mark.django_db

‎api_tests/metrics/test_registries_moderation_metrics.py‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
import pytest
2-
from waffle.testutils import override_switch
32

4-
from osf import features
53
from osf_tests.factories import RegistrationFactory, AuthUserFactory
64
from osf.utils.workflows import RegistrationModerationStates, RegistrationModerationTriggers
75
from osf.metrics import RegistriesModerationMetrics
@@ -17,11 +15,6 @@ class TestRegistrationModerationMetrics:
1715
def registration(self):
1816
return RegistrationFactory()
1917

20-
@pytest.fixture(autouse=True)
21-
def enable_elasticsearch_metrics(self):
22-
with override_switch(features.ELASTICSEARCH_METRICS, active=True):
23-
yield
24-
2518
@pytest.mark.es_metrics
2619
def test_record_transitions(self, registration):
2720
with capture_notifications():
@@ -50,11 +43,6 @@ class TestRegistrationModerationMetricsView:
5043
def registration(self):
5144
return RegistrationFactory()
5245

53-
@pytest.fixture(autouse=True)
54-
def enable_elasticsearch_metrics(self):
55-
with override_switch(features.ELASTICSEARCH_METRICS, active=True):
56-
yield
57-
5846
@pytest.fixture
5947
def user(self):
6048
user = AuthUserFactory()

‎conftest.py‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,20 @@
55

66
from django.db import transaction
77
from elasticsearch6_dsl.connections import connections
8-
from website import settings as osf_settings
9-
from elasticsearch_metrics.tests._test_util import RealElasticTestCase
8+
from elasticsearch_metrics.tests.util import djelme_test_backends
109
from faker import Factory
1110
import pytest
1211
import responses
1312
import xml.etree.ElementTree as ET
13+
from waffle.testutils import override_switch
1414

1515
from api_tests.share import _utils as shtrove_test_utils
1616
from framework.celery_tasks import app as celery_app
1717
from osf.external.spam import tasks as spam_tasks
1818
from website import settings as website_settings
1919
from osf.management.commands.populate_notification_types import populate_notification_types
20+
from osf import features
21+
2022

2123
def pytest_configure(config):
2224
if not os.getenv('GITHUB_ACTIONS') == 'true':
@@ -146,19 +148,12 @@ def _es_metrics_marker(request):
146148
yield
147149
return
148150

149-
connections.create_connection(
150-
alias='osfmetrics_es6',
151-
hosts=osf_settings.ELASTIC6_URI,
152-
)
153-
154-
class _Es6TestCase(RealElasticTestCase, autosetup_djelme_backends=True):
155-
...
156-
es6_test_case = _Es6TestCase()
157-
es6_test_case.setUp()
158-
try:
151+
with (
152+
override_switch(features.ELASTICSEARCH_METRICS, active=True),
153+
djelme_test_backends(),
154+
):
159155
yield
160-
finally:
161-
es6_test_case.tearDown()
156+
162157

163158
@pytest.fixture
164159
def mock_share_responses():

‎docker-compose.yml‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -79,15 +79,22 @@ services:
7979
stdin_open: true
8080

8181
elasticsearch8:
82-
image: docker.elastic.co/elasticsearch/elasticsearch:8.19.11
83-
platform: linux/arm64
82+
image: elasticsearch:8.19.14
8483
environment:
85-
- xpack.security.enabled=false
8684
- discovery.type=single-node
85+
- xpack.security.enabled=false
86+
- ES_JAVA_OPTS=-Xms512m -Xmx512m # reduce memory usage
87+
- xpack.ml.enabled=false
8788
ports:
8889
- 9202:9200
8990
volumes:
9091
- elasticsearch8_data_vol:/usr/share/elasticsearch/data
92+
healthcheck:
93+
start_period: 15s
94+
test: ["CMD", "curl", "-sf", "http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=30s"]
95+
interval: 10s
96+
timeout: 30s
97+
retries: 5
9198
stdin_open: true
9299

93100
postgres:

‎osf/metrics/__init__.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
StorageAddonUsage,
1818
UserSummaryReport,
1919
)
20+
from . import es8_metrics
21+
2022

2123
DAILY_REPORTS = (
2224
DownloadCountReport,
@@ -36,4 +38,5 @@
3638
'PreprintView',
3739
'PreprintDownload',
3840
'RegistriesModerationMetrics',
41+
'es8_metrics',
3942
)

0 commit comments

Comments
 (0)