Cleanup elasticsearch testing (#2464)

* Drop testing against ancient elasticsearch versions

* Bump elasticsearch versions to latest of the major

* Drop docker-compose.yml version

* Drop unused volumes

* Test against Elastic stack 9

Author: xrmx

.ci/.matrix_exclude.yml

@@ -37,6 +37,9 @@ exclude:
     FRAMEWORK: celery-5-django-5
   - VERSION: python-3.9
     FRAMEWORK: celery-5-django-5
+  # Elasticsearch
+  - VERSION: python-3.6
+    FRAMEWORK: elasticsearch-9
   # Flask
   - VERSION: pypy-3
     FRAMEWORK: flask-0.11 # see https://github.com/pallets/flask/commit/6e46d0cd, 0.11.2 was never released

.ci/.matrix_framework.yml

@@ -29,9 +29,9 @@ FRAMEWORK:
   - pyodbc-newest
   - memcached-newest
   - pylibmc-newest
-  - elasticsearch-2
   - elasticsearch-7
   - elasticsearch-8
+  - elasticsearch-9
   - cassandra-newest
   - psutil-newest
   #- eventlet-newest

.ci/.matrix_framework_full.yml

@@ -63,11 +63,9 @@ FRAMEWORK:
   - psutil-5.0
   - psutil-4.0
   #- eventlet-newest
-  - elasticsearch-2
-  - elasticsearch-5
-  - elasticsearch-6
   - elasticsearch-7
   - elasticsearch-8
+  - elasticsearch-9
   - gevent-newest
   - aiohttp-3.0
   - aiohttp-newest

tests/docker-compose.yml

@@ -1,5 +1,3 @@
-version: '2.1'
-
 services:
   postgres:
     user: postgres
@@ -59,21 +57,8 @@ services:
   redis:
     image: redis
 
-  elasticsearch6:
-    image: docker.elastic.co/elasticsearch/elasticsearch:6.8.0
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9200"]
-    environment:
-      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-      - "network.host="
-      - "transport.host=127.0.0.1"
-      - "http.host=0.0.0.0"
-      - "xpack.security.enabled=false"
-    volumes:
-      - pyesdata6:/usr/share/elasticsearch/data
-
   elasticsearch7:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.29
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9200"]
     environment:
@@ -86,7 +71,7 @@ services:
       - pyesdata7:/usr/share/elasticsearch/data
 
   elasticsearch8:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.19.6
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9200"]
     ports:
@@ -101,31 +86,21 @@ services:
     volumes:
       - pyesdata8:/usr/share/elasticsearch/data
 
-  elasticsearch5:
-    image: docker.elastic.co/elasticsearch/elasticsearch:5.6.16
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9200"]
-    environment:
-      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-      - "network.host="
-      - "transport.host=127.0.0.1"
-      - "http.host=0.0.0.0"
-      - "xpack.security.enabled=false"
-    volumes:
-      - pyesdata5:/usr/share/elasticsearch/data
-
-  elasticsearch2:
-    image: elasticsearch:2
+  elasticsearch9:
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.2.0
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9200"]
+    ports:
+      - "9200:9200"
     environment:
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-      - "network.host="
+      - "network.host=_site_"
       - "transport.host=127.0.0.1"
       - "http.host=0.0.0.0"
       - "xpack.security.enabled=false"
+      - "action.destructive_requires_name=false" # allow for easy cleanup by calling DELETE *
     volumes:
-      - pyesdata2:/usr/share/elasticsearch/data
+      - pyesdata9:/usr/share/elasticsearch/data
 
   mssql:
     image: mcr.microsoft.com/mssql/server
@@ -185,11 +160,9 @@ services:
   run_tests:
     image: ${REGISTRY:-elasticobservability}/${IMAGE_NAME:-apm-agent-python-testing}:${PYTHON_VERSION}
     environment:
+      ES_9_URL: 'http://elasticsearch9:9200'
       ES_8_URL: 'http://elasticsearch8:9200'
       ES_7_URL: 'http://elasticsearch7:9200'
-      ES_6_URL: 'http://elasticsearch6:9200'
-      ES_5_URL: 'http://elasticsearch5:9200'
-      ES_2_URL: 'http://elasticsearch2:9200'
 
 
 volumes:
@@ -209,11 +182,7 @@ volumes:
     driver: local
   pyesdata8:
     driver: local
-  pyesdata6:
-    driver: local
-  pyesdata5:
-    driver: local
-  pyesdata2:
+  pyesdata9:
     driver: local
   pycassandradata3:
     driver: local

tests/requirements/reqs-elasticsearch-2.txt

@@ -0,0 +1,3 @@
+elasticsearch>=9.0,<10.0
+aiohttp
+-r reqs-base.txt