From 8dae849bc8ce3517fefeb05233ce2f4738a4d88f Mon Sep 17 00:00:00 2001 From: jokiefer Date: Tue, 14 Nov 2023 08:12:37 +0100 Subject: [PATCH 01/28] fixing test setup and documentation --- .env | 41 ++++++ .gitignore | 1 - README.md | 22 +++- docker-compose.yml | 122 ++++++++++++++++++ tests/backends/elasticsearch/test_evaluate.py | 41 +++--- 5 files changed, 205 insertions(+), 22 deletions(-) create mode 100644 .env create mode 100644 docker-compose.yml diff --git a/.env b/.env new file mode 100644 index 0000000..5013ac6 --- /dev/null +++ b/.env @@ -0,0 +1,41 @@ +# Project namespace (defaults to the current folder name if not set) +#COMPOSE_PROJECT_NAME=myproject + + +# Password for the 'elastic' user (at least 6 characters) +ELASTIC_PASSWORD=changeme + + +# Password for the 'kibana_system' user (at least 6 characters) +KIBANA_PASSWORD=changeme + + +# Version of Elastic products +STACK_VERSION=8.7.1 + + +# Set the cluster name +CLUSTER_NAME=docker-cluster + + +# Set to 'basic' or 'trial' to automatically start the 30-day trial +LICENSE=basic +#LICENSE=trial + + +# Port to expose Elasticsearch HTTP API to the host +ES_PORT=9200 + + +# Port to expose Kibana to the host +KIBANA_PORT=5601 + + +# Increase or decrease based on the available host memory (in bytes) +ES_MEM_LIMIT=1073741824 +KB_MEM_LIMIT=1073741824 +LS_MEM_LIMIT=1073741824 + + +# SAMPLE Predefined Key only to be used in POC environments +ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2 diff --git a/.gitignore b/.gitignore index 0964046..607cf71 100644 --- a/.gitignore +++ b/.gitignore @@ -102,7 +102,6 @@ celerybeat.pid *.sage.py # Environments -.env .venv env/ venv/ diff --git a/README.md b/README.md index ad770cc..ab2f23d 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,14 @@ class MyAPIEvaluator(Evaluator): ## Testing +### Proconditions + +1. Install [gdal](https://gdal.org/download.html#binaries) on your system. This package and some site-packages will use it to calculate geospatial data. +2. Provide an elasticsearch instance. You can simply use the provided docker compose setup. + + +### Python dependencies + For testing, several requirements must be satisfied. These can be installed, via pip: ```bash @@ -180,13 +188,24 @@ pip install -r requirements-dev.txt pip install -r requirements-test.txt ``` +### Docker + +Start the elasticsearch instance: + +```bash +docker compose up + +``` + +### Excecuting tests + + The functionality can be tested using `pytest`. ```bash python -m pytest ``` -### Docker To execute tests in Docker: @@ -195,6 +214,7 @@ docker build -t pygeofilter/test -f Dockerfile-3.9 . docker run --rm pygeofilter/test ``` + ## Backends The following backends are shipped with `pygeofilter`. Some require additional dependencies, refer to the [installation](#installation) section for further details. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..bfe1270 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,122 @@ +version: '3.8' +services: + es01: + depends_on: + setup: + condition: service_healthy + + image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0 + ports: + - ${ES_PORT}:9200 + env_file: + - .env + environment: + - node.name=es01 + - cluster.name=${CLUSTER_NAME} + - discovery.type=single-node + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - bootstrap.memory_lock=true + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=certs/es01/es01.key + - xpack.security.http.ssl.certificate=certs/es01/es01.crt + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.key=certs/es01/es01.key + - xpack.security.transport.ssl.certificate=certs/es01/es01.crt + - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.license.self_generated.type=${LICENSE} + mem_limit: ${ES_MEM_LIMIT} + volumes: + - certs:/usr/share/elasticsearch/config/certs + - esdata01:/usr/share/elasticsearch/data + + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'", + ] + interval: 10s + timeout: 10s + retries: 120 + + + setup: + image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0 + volumes: + - certs:/usr/share/elasticsearch/config/certs + user: "0" + env_file: + - .env + command: > + bash -c ' + if [ x${ELASTIC_PASSWORD} == x ]; then + echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; + exit 1; + elif [ x${KIBANA_PASSWORD} == x ]; then + echo "Set the KIBANA_PASSWORD environment variable in the .env file"; + exit 1; + fi; + if [ ! -f config/certs/ca.zip ]; then + echo "Creating CA"; + bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip; + unzip config/certs/ca.zip -d config/certs; + fi; + if [ ! -f config/certs/certs.zip ]; then + echo "Creating certs"; + echo -ne \ + "instances:\n"\ + " - name: es01\n"\ + " dns:\n"\ + " - es01\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + " - name: kibana\n"\ + " dns:\n"\ + " - kibana\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + > config/certs/instances.yml; + bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key; + unzip config/certs/certs.zip -d config/certs; + fi; + echo "Setting file permissions" + chown -R root:root config/certs; + find . -type d -exec chmod 750 \{\} \;; + find . -type f -exec chmod 640 \{\} \;; + echo "Waiting for Elasticsearch availability"; + until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done; + echo "Setting kibana_system password"; + until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; + echo "All done!"; + ' + healthcheck: + test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"] + interval: 1s + timeout: 5s + retries: 120 + + +volumes: + certs: + driver: local + esdata01: + driver: local + kibanadata: + driver: local + metricbeatdata01: + driver: local + filebeatdata01: + driver: local + logstashdata01: + driver: local + + +networks: + default: + name: elastic + external: false \ No newline at end of file diff --git a/tests/backends/elasticsearch/test_evaluate.py b/tests/backends/elasticsearch/test_evaluate.py index e3e8904..d1a2cfc 100644 --- a/tests/backends/elasticsearch/test_evaluate.py +++ b/tests/backends/elasticsearch/test_evaluate.py @@ -54,7 +54,10 @@ class Index: @pytest.fixture(autouse=True, scope="session") def connection(): connections.create_connection( - hosts=["http://localhost:9200"], + hosts=["https://localhost:9200"], + ca_certs=False, + verify_certs=False, + basic_auth=("elastic", "changeme") ) @@ -107,9 +110,7 @@ def data(index): def filter_(ast_): query = to_filter(ast_, version="8.2") - print(query) result = Record.search().query(query).execute() - print([r.identifier for r in result]) return result @@ -135,35 +136,35 @@ def test_comparison(data): def test_combination(data): result = filter_(parse("int_attribute = 5 AND float_attribute < 6.0")) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier result = filter_(parse("int_attribute = 6 OR float_attribute < 6.0")) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier def test_between(data): result = filter_(parse("float_attribute BETWEEN -1 AND 1")) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier result = filter_(parse("int_attribute NOT BETWEEN 4 AND 6")) - assert len(result) == 1 and result[0].identifier is data[1].identifier + assert len(result) == 1 and result[0].identifier == data[1].identifier def test_like(data): result = filter_(parse("str_attribute LIKE 'this is a test'")) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier result = filter_(parse("str_attribute LIKE 'this is % test'")) assert len(result) == 2 result = filter_(parse("str_attribute NOT LIKE '% another test'")) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier result = filter_(parse("str_attribute NOT LIKE 'this is . test'")) - assert len(result) == 1 and result[0].identifier is data[1].identifier + assert len(result) == 1 and result[0].identifier == data[1].identifier result = filter_(parse("str_attribute ILIKE 'THIS IS . TEST'")) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier result = filter_(parse("str_attribute ILIKE 'THIS IS % TEST'")) assert len(result) == 2 @@ -171,18 +172,18 @@ def test_like(data): def test_in(data): result = filter_(parse("int_attribute IN ( 1, 2, 3, 4, 5 )")) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier result = filter_(parse("int_attribute NOT IN ( 1, 2, 3, 4, 5 )")) - assert len(result) == 1 and result[0].identifier is data[1].identifier + assert len(result) == 1 and result[0].identifier == data[1].identifier def test_null(data): result = filter_(parse("maybe_str_attribute IS NULL")) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier result = filter_(parse("maybe_str_attribute IS NOT NULL")) - assert len(result) == 1 and result[0].identifier is data[1].identifier + assert len(result) == 1 and result[0].identifier == data[1].identifier def test_has_attr(): @@ -203,17 +204,17 @@ def test_temporal(data): ], ) ) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier result = filter_( parse("datetime_attribute BEFORE 2000-01-01T00:00:05.00Z"), ) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier result = filter_( parse("datetime_attribute AFTER 2000-01-01T00:00:05.00Z"), ) - assert len(result) == 1 and result[0].identifier is data[1].identifier + assert len(result) == 1 and result[0].identifier == data[1].identifier # def test_array(): @@ -258,14 +259,14 @@ def test_spatial(data): result = filter_( parse("INTERSECTS(geometry, ENVELOPE (0.0 1.0 0.0 1.0))"), ) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier # TODO: test more spatial queries result = filter_( parse("BBOX(center, 2, 2, 3, 3)"), ) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier # def test_arithmetic(): From 8309d585e81239f51e297de737b830104155cd73 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Tue, 14 Nov 2023 08:53:12 +0100 Subject: [PATCH 02/28] * fixing docker file and docker testing docs * fixing geoalchemy deprecations --- README.md | 5 ++--- Dockerfile-3.9 => bullseye-3.9.Dockerfile | 8 ++++---- docker-compose.yml | 13 +++++++++++++ tests/backends/elasticsearch/test_evaluate.py | 5 +++-- tests/backends/sqlalchemy/test_evaluate.py | 1 - 5 files changed, 22 insertions(+), 10 deletions(-) rename Dockerfile-3.9 => bullseye-3.9.Dockerfile (81%) diff --git a/README.md b/README.md index ab2f23d..d8a36d8 100644 --- a/README.md +++ b/README.md @@ -209,9 +209,8 @@ python -m pytest To execute tests in Docker: -``` -docker build -t pygeofilter/test -f Dockerfile-3.9 . -docker run --rm pygeofilter/test +```bash +docker compose up --build test ``` diff --git a/Dockerfile-3.9 b/bullseye-3.9.Dockerfile similarity index 81% rename from Dockerfile-3.9 rename to bullseye-3.9.Dockerfile index 94c019a..9b8cd40 100644 --- a/Dockerfile-3.9 +++ b/bullseye-3.9.Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-buster +FROM python:3.9-bullseye LABEL description="Test executor" @@ -8,6 +8,7 @@ RUN apt-get update --fix-missing \ binutils \ libproj-dev \ gdal-bin \ + libgdal-dev \ libsqlite3-mod-spatialite \ spatialite-bin \ && rm -rf /var/lib/apt/lists/* @@ -19,6 +20,7 @@ COPY requirements-test.txt . COPY requirements-dev.txt . RUN pip install -r requirements-test.txt RUN pip install -r requirements-dev.txt +RUN pip install pygdal=="`gdal-config --version`.*" COPY pygeofilter pygeofilter COPY tests tests @@ -26,6 +28,4 @@ COPY README.md . COPY setup.py . RUN pip install -e . -RUN chmod +x tests/execute-tests.sh - -CMD ["tests/execute-tests.sh"] +CMD ["python", "-m", "pytest"] diff --git a/docker-compose.yml b/docker-compose.yml index bfe1270..682637c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,18 @@ version: '3.8' services: + + test: + build: + context: . + dockerfile: bullseye-3.9.Dockerfile + tty: true # To support colorized log output. + networks: + - default + depends_on: + - es01 + environment: + - eshostname=es01 + es01: depends_on: setup: diff --git a/tests/backends/elasticsearch/test_evaluate.py b/tests/backends/elasticsearch/test_evaluate.py index d1a2cfc..bdec193 100644 --- a/tests/backends/elasticsearch/test_evaluate.py +++ b/tests/backends/elasticsearch/test_evaluate.py @@ -1,5 +1,5 @@ # pylint: disable=W0621,C0114,C0115,C0116 - +import os import pytest from elasticsearch_dsl import ( Date, @@ -53,8 +53,9 @@ class Index: @pytest.fixture(autouse=True, scope="session") def connection(): + hostname = os.environ.get("eshostname", "localhost") connections.create_connection( - hosts=["https://localhost:9200"], + hosts=[f"https://{hostname}:9200"], ca_certs=False, verify_certs=False, basic_auth=("elastic", "changeme") diff --git a/tests/backends/sqlalchemy/test_evaluate.py b/tests/backends/sqlalchemy/test_evaluate.py index 705f3af..b12e019 100644 --- a/tests/backends/sqlalchemy/test_evaluate.py +++ b/tests/backends/sqlalchemy/test_evaluate.py @@ -31,7 +31,6 @@ class Record(Base): geometry_type="MULTIPOLYGON", srid=4326, spatial_index=False, - management=True, ) ) float_attribute = Column(Float) From a2acfc1de32c24e0303d73be7ad245d962634168 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 09:20:24 +0100 Subject: [PATCH 03/28] fixes #81, #80 --- pygeofilter/backends/django/evaluate.py | 4 + pygeofilter/parsers/fes/base.py | 4 - pygeofilter/parsers/fes/v11.py | 4 + ...te.py => test_django_evaluate_with_cql.py} | 0 .../django/test_django_evaluate_with_fes11.py | 205 ++++++++++++++++++ tests/parsers/fes/test_v11.py | 38 ++-- 6 files changed, 232 insertions(+), 23 deletions(-) rename tests/backends/django/{test_django_evaluate.py => test_django_evaluate_with_cql.py} (100%) create mode 100644 tests/backends/django/test_django_evaluate_with_fes11.py diff --git a/pygeofilter/backends/django/evaluate.py b/pygeofilter/backends/django/evaluate.py index 3498641..bc39d8d 100644 --- a/pygeofilter/backends/django/evaluate.py +++ b/pygeofilter/backends/django/evaluate.py @@ -57,6 +57,10 @@ def between(self, node, lhs, low, high): @handle(ast.Like) def like(self, node, lhs): + if node.wildcard != "%": + # FIXME: this will also replace the singlechar and escapechar, + # but they shall not be replaced + node.pattern = node.pattern.replace(node.wildcard, "%") return filters.like( lhs, node.pattern, node.nocase, node.not_, self.mapping_choices ) diff --git a/pygeofilter/parsers/fes/base.py b/pygeofilter/parsers/fes/base.py index 29fb945..ca4b2a1 100644 --- a/pygeofilter/parsers/fes/base.py +++ b/pygeofilter/parsers/fes/base.py @@ -134,10 +134,6 @@ def distance(self, node: Element): # # TODO: ast.BBox() seems incompatible # pass - @handle("ValueReference") - def value_reference(self, node): - return ast.Attribute(node.text) - @handle("Literal") def literal(self, node): type_ = node.get("type", "").rpartition(":")[2] diff --git a/pygeofilter/parsers/fes/v11.py b/pygeofilter/parsers/fes/v11.py index 264aa64..a899c93 100644 --- a/pygeofilter/parsers/fes/v11.py +++ b/pygeofilter/parsers/fes/v11.py @@ -29,6 +29,10 @@ def div( self, node: Element, lhs: ast.ScalarAstType, rhs: ast.ScalarAstType ) -> ast.Node: return ast.Div(lhs, rhs) + + @handle("PropertyName") + def property_name(self, node): + return ast.Attribute(node.text) def parse(input_: ParseInput) -> ast.Node: diff --git a/tests/backends/django/test_django_evaluate.py b/tests/backends/django/test_django_evaluate_with_cql.py similarity index 100% rename from tests/backends/django/test_django_evaluate.py rename to tests/backends/django/test_django_evaluate_with_cql.py diff --git a/tests/backends/django/test_django_evaluate_with_fes11.py b/tests/backends/django/test_django_evaluate_with_fes11.py new file mode 100644 index 0000000..f7e485d --- /dev/null +++ b/tests/backends/django/test_django_evaluate_with_fes11.py @@ -0,0 +1,205 @@ +# ------------------------------------------------------------------------------ +# +# Project: pygeofilter +# Authors: Jonas Kiefer +# +# ------------------------------------------------------------------------------ +# Copyright (C) 2019 EOX IT Services GmbH +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies of this Software or works derived from this Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# ------------------------------------------------------------------------------ + +import pytest +from testapp import models + +from pygeofilter.backends.django.evaluate import to_filter +from pygeofilter.parsers.fes.parser import parse + + +def evaluate(fes_expr, expected_ids, model_type=None): + model_type = model_type or models.Record + mapping = models.FIELD_MAPPING + mapping_choices = models.MAPPING_CHOICES + + ast = parse(fes_expr) + filters = to_filter(ast, mapping, mapping_choices) + + qs = model_type.objects.filter(filters) + + assert expected_ids == type(expected_ids)(qs.values_list("identifier", flat=True)) + + +FILTER_FRAME = """ + + {0} + +""" + + +# common comparisons + +@pytest.mark.django_db +def test_common_value_like(): + CONSTRAINT = ''' + + strAttribute + AA* + + ''' + + evaluate(FILTER_FRAME.format(CONSTRAINT), ("A",)) + + +@pytest.mark.django_db +def test_common_value_like_middle(): + CONSTRAINT = ''' + + strAttribute + A*A + + ''' + + evaluate(FILTER_FRAME.format(CONSTRAINT), ("A",)) + + +@pytest.mark.django_db +def test_like_beginswith(): + CONSTRAINT = ''' + + strAttribute + A* + + ''' + + evaluate(FILTER_FRAME.format(CONSTRAINT), ("A",)) + + +@pytest.mark.django_db +def test_like_endswith(): + CONSTRAINT = ''' + + strAttribute + *a + + ''' + + evaluate(FILTER_FRAME.format(CONSTRAINT), ("A",)) + + +@pytest.mark.django_db +def test_like_middle(): + CONSTRAINT = ''' + + strMetaAttribute + *parent* + + ''' + + evaluate(FILTER_FRAME.format(CONSTRAINT), ("A", "B")) + + +@pytest.mark.django_db +def test_like_startswith_middle(): + CONSTRAINT = ''' + + strMetaAttribute + A*rent* + + ''' + + evaluate(FILTER_FRAME.format(CONSTRAINT), ("A",)) + + +@pytest.mark.django_db +def test_like_startswith_middle_endswith(): + CONSTRAINT = ''' + + strMetaAttribute + A%ren%A + + ''' + + evaluate(FILTER_FRAME.format(CONSTRAINT), ("A",)) + + +@pytest.mark.django_db +def test_not_like_beginswith(): + CONSTRAINT = ''' + + + strMetaAttribute + B% + + + ''' + + evaluate(FILTER_FRAME.format(CONSTRAINT), ("A",)) + + +@pytest.mark.django_db +def test_not_like_endswith(): + CONSTRAINT = ''' + + + strMetaAttribute + %B + + + ''' + + evaluate(FILTER_FRAME.format(CONSTRAINT), ("A",)) + + +# spatial predicates + +@pytest.mark.django_db +def test_intersects_point(): + CONSTRAINT = ''' + + geometry + + 1.0 1.0 + + + + ''' + + evaluate(FILTER_FRAME.format(CONSTRAINT), ("A",)) + + +@pytest.mark.django_db +def test_intersects_mulitipoint_1(): + CONSTRAINT = ''' + + geometry + + + + 1.0 1.0 + + + + + ''' + + evaluate(FILTER_FRAME.format(CONSTRAINT), ("A",)) diff --git a/tests/parsers/fes/test_v11.py b/tests/parsers/fes/test_v11.py index c54b3cd..b348cac 100644 --- a/tests/parsers/fes/test_v11.py +++ b/tests/parsers/fes/test_v11.py @@ -9,11 +9,11 @@ def test_and(): xmlns:xsd="http://www.w3.org/2001/XMLSchema-datatypes"> - attr + attr 30 - attr + attr 10 @@ -39,11 +39,11 @@ def test_or(): xmlns:xsd="http://www.w3.org/2001/XMLSchema-datatypes"> - attr + attr 30.5 - attr + attr 10.5 @@ -69,7 +69,7 @@ def test_not(): xmlns:xsd="http://www.w3.org/2001/XMLSchema-datatypes"> - attr + attr value @@ -90,7 +90,7 @@ def test_not_equal(): - attr + attr value @@ -112,7 +112,7 @@ def test_is_like(): singleChar="." escapeChar="\\" matchCase="true"> - attr + attr some% @@ -138,7 +138,7 @@ def test_is_like(): singleChar="." escapeChar="\\" matchCase="false"> - attr + attr some% @@ -161,7 +161,7 @@ def test_is_null(): - attr + attr """ @@ -178,7 +178,7 @@ def test_is_between(): - attr + attr 10.5 @@ -203,7 +203,7 @@ def test_geom_equals(): - attr + attr @@ -236,7 +236,7 @@ def test_geom_disjoint(): - attr + attr 1.0 1.0 2.0 2.0 @@ -264,7 +264,7 @@ def test_geom_touches(): - attr + attr @@ -301,7 +301,7 @@ def test_geom_within(): - attr + attr 0.0 1.0 2.0 3.0 @@ -335,7 +335,7 @@ def test_geom_overlaps(): - attr + attr @@ -402,7 +402,7 @@ def test_geom_crosses(): - attr + attr 1.0 2.0 2.0 1.0 @@ -424,7 +424,7 @@ def test_geom_intersects(): - attr + attr 1.0 0.5 2.0 1.5 @@ -452,7 +452,7 @@ def test_geom_contains(): - attr + attr 1.0 0.5 2.0 0.5 2.0 1.5 1.0 1.5 1.0 0.5 @@ -479,7 +479,7 @@ def test_geom_dwithin(): - attr + attr 1.0 1.0 From 8b427b9a995b339ba49c4fd24254fa29b87d43a7 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 09:25:11 +0100 Subject: [PATCH 04/28] fixes github workflow for testing --- .github/workflows/main.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a48923f..7a70d2b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,15 +29,9 @@ jobs: sudo sysctl -w vm.swappiness=1 sudo sysctl -w fs.file-max=262144 sudo sysctl -w vm.max_map_count=262144 - - name: Install and run Elasticsearch 📦 - uses: getong/elasticsearch-action@v1.2 - with: - elasticsearch version: '8.2.2' - host port: 9200 - container port: 9200 - host node port: 9300 - node port: 9300 - discovery type: 'single-node' + - name: Run elasticsearch docker container 📦 + run: | + docker compose up --build es01 - name: Run unit tests run: | pytest From 6ceb118ddbd8ed86482f335f5eb7eec4195814e8 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 10:26:11 +0100 Subject: [PATCH 05/28] add tox setup for testing --- README.md | 7 +++--- requirements-base.txt | 13 +++++++++++ requirements-dev.txt | 3 +-- requirements-test.txt | 16 +++----------- .../django/test_django_evaluate_with_fes11.py | 20 ----------------- tox.ini | 22 +++++++++++++++++++ 6 files changed, 43 insertions(+), 38 deletions(-) create mode 100644 requirements-base.txt create mode 100644 tox.ini diff --git a/README.md b/README.md index d8a36d8..ef91494 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,7 @@ class MyAPIEvaluator(Evaluator): For testing, several requirements must be satisfied. These can be installed, via pip: ```bash +pip install -r requirements-base.txt pip install -r requirements-dev.txt pip install -r requirements-test.txt ``` @@ -193,17 +194,17 @@ pip install -r requirements-test.txt Start the elasticsearch instance: ```bash -docker compose up +docker compose up es01 ``` ### Excecuting tests -The functionality can be tested using `pytest`. +The functionality can be tested using `tox`. To run tests in a specific python and django version run ```bash -python -m pytest +tox -e py311-django41 ``` diff --git a/requirements-base.txt b/requirements-base.txt new file mode 100644 index 0000000..d8accea --- /dev/null +++ b/requirements-base.txt @@ -0,0 +1,13 @@ +geoalchemy2 +sqlalchemy<2.0.0 +geopandas +fiona +pyproj +rtree +pygml +dateparser +pygeoif +lark +elasticsearch +elasticsearch-dsl +gdal<=3.6.4 \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index 66c2576..3cac3fe 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,4 @@ flake8 -pytest -pytest-django wheel mypy<=0.982 +django>=3.2,<4.3 \ No newline at end of file diff --git a/requirements-test.txt b/requirements-test.txt index 13f35b9..89a3ffa 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,13 +1,3 @@ -django -geoalchemy2 -sqlalchemy<2.0.0 -geopandas -fiona -pyproj -rtree -pygml -dateparser -pygeoif==0.7 -lark -elasticsearch -elasticsearch-dsl +tox==4.5.1 +pytest +pytest-django \ No newline at end of file diff --git a/tests/backends/django/test_django_evaluate_with_fes11.py b/tests/backends/django/test_django_evaluate_with_fes11.py index f7e485d..c2d0b9c 100644 --- a/tests/backends/django/test_django_evaluate_with_fes11.py +++ b/tests/backends/django/test_django_evaluate_with_fes11.py @@ -183,23 +183,3 @@ def test_intersects_point(): ''' evaluate(FILTER_FRAME.format(CONSTRAINT), ("A",)) - - -@pytest.mark.django_db -def test_intersects_mulitipoint_1(): - CONSTRAINT = ''' - - geometry - - - - 1.0 1.0 - - - - - ''' - - evaluate(FILTER_FRAME.format(CONSTRAINT), ("A",)) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..af6e20b --- /dev/null +++ b/tox.ini @@ -0,0 +1,22 @@ +[tox] +requires = + tox>=4 +env_list = + py{37,38,39,310,311}-django{32,40,41,42} + +[testenv] +description = run unit tests +deps= + django32: Django>=3.2,<4.0 + django40: Django>=4.0,<4.1 + django41: Django>=4.1,<4.2 + django42: Django>=4.2,<4.3 + -r requirements-base.txt + -r requirements-test.txt + +setenv = + PYTHONPATH = {toxinidir} + +commands = + python -m pytest {posargs} + From 66feb704992451d1d024c1efbe36e277281472c9 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 10:26:15 +0100 Subject: [PATCH 06/28] Update main.yml --- .github/workflows/main.yml | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7a70d2b..1fbb858 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,11 +6,14 @@ jobs: test: runs-on: ubuntu-20.04 strategy: + fail-fast: false matrix: - python-version: ['3.7', '3.8', '3.9'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + env: + PYTHON: ${{ matrix.python-version }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 name: Setup Python ${{ matrix.python-version }} with: python-version: ${{ matrix.python-version }} @@ -19,23 +22,13 @@ jobs: run: | sudo apt-get update sudo apt-get install -y binutils libproj-dev gdal-bin libgdal-dev libsqlite3-mod-spatialite spatialite-bin - pip install -r requirements-test.txt - pip install -r requirements-dev.txt + pip install tox pip install pygdal=="`gdal-config --version`.*" - pip install . - - name: Configure sysctl limits - run: | - sudo swapoff -a - sudo sysctl -w vm.swappiness=1 - sudo sysctl -w fs.file-max=262144 - sudo sysctl -w vm.max_map_count=262144 + - name: Run elasticsearch docker container 📦 run: | - docker compose up --build es01 - - name: Run unit tests - run: | - pytest - # - name: run pre-commit (code formatting, lint and type checking) - # run: | - # python -m pip install pre-commit - # pre-commit run --all-files + docker compose up --build --detach es01 + + - name: Run tox targets for ${{ matrix.python-version }} + run: tox run -f py$(echo ${{ matrix.python-version }} | tr -d .) + From 051f4cad7a2370c43c0b3f2999259eeac6370b67 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 10:29:42 +0100 Subject: [PATCH 07/28] Update main.yml and tox.ini --- .github/workflows/main.yml | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1fbb858..a671a5a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11'] env: PYTHON: ${{ matrix.python-version }} steps: diff --git a/tox.ini b/tox.ini index af6e20b..780da0e 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ requires = tox>=4 env_list = - py{37,38,39,310,311}-django{32,40,41,42} + py{38,39,310,311}-django{32,40,41,42} [testenv] description = run unit tests From b7db7da252092191aad47bd1e8472e6436945351 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 10:30:23 +0100 Subject: [PATCH 08/28] Update setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 47ac4ce..cf3fb6b 100644 --- a/setup.py +++ b/setup.py @@ -74,10 +74,10 @@ "Intended Audience :: Developers", "Topic :: Scientific/Engineering :: GIS", "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], tests_require=["pytest"], ) From dcc3cdf07aace66cf484e1f8af9e4e815eda605c Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 10:32:43 +0100 Subject: [PATCH 09/28] Update requirements-base.txt --- requirements-base.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-base.txt b/requirements-base.txt index d8accea..839cac4 100644 --- a/requirements-base.txt +++ b/requirements-base.txt @@ -10,4 +10,5 @@ pygeoif lark elasticsearch elasticsearch-dsl -gdal<=3.6.4 \ No newline at end of file +gdal<=3.6.4 +pygdal<=3.6.4 \ No newline at end of file From e800458c3cda79061ba1ffb1ef3b03fe0cc8dc9c Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 11:09:01 +0100 Subject: [PATCH 10/28] Update requirements-base.txt and tox.ini --- requirements-base.txt | 3 +-- tox.ini | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements-base.txt b/requirements-base.txt index 839cac4..727ea4f 100644 --- a/requirements-base.txt +++ b/requirements-base.txt @@ -10,5 +10,4 @@ pygeoif lark elasticsearch elasticsearch-dsl -gdal<=3.6.4 -pygdal<=3.6.4 \ No newline at end of file +gdal \ No newline at end of file diff --git a/tox.ini b/tox.ini index 780da0e..1b28f8f 100644 --- a/tox.ini +++ b/tox.ini @@ -11,12 +11,13 @@ deps= django40: Django>=4.0,<4.1 django41: Django>=4.1,<4.2 django42: Django>=4.2,<4.3 - -r requirements-base.txt - -r requirements-test.txt + -rrequirements-base.txt + -rrequirements-test.txt setenv = PYTHONPATH = {toxinidir} + commands = python -m pytest {posargs} From f3aa49d3c8cddfcdd170d5460b2b11c670c24560 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 11:27:55 +0100 Subject: [PATCH 11/28] Update main.yml, requirements-base.txt, and tox.ini --- .github/workflows/main.yml | 2 +- requirements-base.txt | 1 - tox.ini | 3 +-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a671a5a..38860de 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,7 +23,7 @@ jobs: sudo apt-get update sudo apt-get install -y binutils libproj-dev gdal-bin libgdal-dev libsqlite3-mod-spatialite spatialite-bin pip install tox - pip install pygdal=="`gdal-config --version`.*" + sudo -H pip install gdal=="`gdal-config --version`.*" - name: Run elasticsearch docker container 📦 run: | diff --git a/requirements-base.txt b/requirements-base.txt index 727ea4f..87e8452 100644 --- a/requirements-base.txt +++ b/requirements-base.txt @@ -10,4 +10,3 @@ pygeoif lark elasticsearch elasticsearch-dsl -gdal \ No newline at end of file diff --git a/tox.ini b/tox.ini index 1b28f8f..3473e75 100644 --- a/tox.ini +++ b/tox.ini @@ -12,12 +12,11 @@ deps= django41: Django>=4.1,<4.2 django42: Django>=4.2,<4.3 -rrequirements-base.txt - -rrequirements-test.txt + -r requirements-test.txt setenv = PYTHONPATH = {toxinidir} - commands = python -m pytest {posargs} From a6a4b5cca535b9292c8ef90b8fde19b8d1abc68e Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 11:33:56 +0100 Subject: [PATCH 12/28] Update main.yml --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 38860de..98018b9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,7 +4,7 @@ on: [ push, pull_request ] jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: @@ -24,6 +24,7 @@ jobs: sudo apt-get install -y binutils libproj-dev gdal-bin libgdal-dev libsqlite3-mod-spatialite spatialite-bin pip install tox sudo -H pip install gdal=="`gdal-config --version`.*" + sudo -H pip install pygdal=="`gdal-config --version`.*" - name: Run elasticsearch docker container 📦 run: | From 5d98e3aebab6a7acbda5559044735ad7baf1c330 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 12:12:51 +0100 Subject: [PATCH 13/28] Update main.yml, requirements-gdal.txt, and tox.ini --- .github/workflows/main.yml | 3 +-- tox.ini | 5 ++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 98018b9..a1e4c55 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,8 +23,7 @@ jobs: sudo apt-get update sudo apt-get install -y binutils libproj-dev gdal-bin libgdal-dev libsqlite3-mod-spatialite spatialite-bin pip install tox - sudo -H pip install gdal=="`gdal-config --version`.*" - sudo -H pip install pygdal=="`gdal-config --version`.*" + echo "gdal==`gdal-config --version`.*" >> requirements-gdal.txt - name: Run elasticsearch docker container 📦 run: | diff --git a/tox.ini b/tox.ini index 3473e75..99a5319 100644 --- a/tox.ini +++ b/tox.ini @@ -4,6 +4,8 @@ requires = env_list = py{38,39,310,311}-django{32,40,41,42} +passenv = GDAL_VERSION + [testenv] description = run unit tests deps= @@ -11,8 +13,9 @@ deps= django40: Django>=4.0,<4.1 django41: Django>=4.1,<4.2 django42: Django>=4.2,<4.3 + -rrequirements-gdal.txt -rrequirements-base.txt - -r requirements-test.txt + -rrequirements-test.txt setenv = PYTHONPATH = {toxinidir} From 22243b59909552f3abfa123d7621d053c96a3201 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 12:19:58 +0100 Subject: [PATCH 14/28] Update main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a1e4c55..716d5ca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,7 +4,7 @@ on: [ push, pull_request ] jobs: test: - runs-on: ubuntu-22.04 + runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: From 7401e21ef21c23f9542fa854d0b1d40223a7f218 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 12:24:35 +0100 Subject: [PATCH 15/28] Update main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 716d5ca..a1e4c55 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,7 +4,7 @@ on: [ push, pull_request ] jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: From bd583af46e126286ca075b7ab785d310597d183d Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 12:37:11 +0100 Subject: [PATCH 16/28] Update main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a1e4c55..716d5ca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,7 +4,7 @@ on: [ push, pull_request ] jobs: test: - runs-on: ubuntu-22.04 + runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: From d37f361e2b9c8b6fa9544365b2374691fb33fe76 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 12:41:57 +0100 Subject: [PATCH 17/28] Update tox.ini --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 99a5319..f4dd78c 100644 --- a/tox.ini +++ b/tox.ini @@ -13,6 +13,7 @@ deps= django40: Django>=4.0,<4.1 django41: Django>=4.1,<4.2 django42: Django>=4.2,<4.3 + setuptools==57.5.0 -rrequirements-gdal.txt -rrequirements-base.txt -rrequirements-test.txt From c7884c2a24c606df6c5de8b9d8ec7d4df13f9d9e Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 12:51:32 +0100 Subject: [PATCH 18/28] Update main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 716d5ca..45beb07 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,7 +23,7 @@ jobs: sudo apt-get update sudo apt-get install -y binutils libproj-dev gdal-bin libgdal-dev libsqlite3-mod-spatialite spatialite-bin pip install tox - echo "gdal==`gdal-config --version`.*" >> requirements-gdal.txt + echo -e "gdal==`gdal-config --version`.*\npygdal==`gdal-config --version`.*" >> requirements-gdal.txt - name: Run elasticsearch docker container 📦 run: | From 238165aa58743521ed76daee929fc1b7248d52c0 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 12:54:40 +0100 Subject: [PATCH 19/28] Update main.yml --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 45beb07..e7cc32e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,6 +23,8 @@ jobs: sudo apt-get update sudo apt-get install -y binutils libproj-dev gdal-bin libgdal-dev libsqlite3-mod-spatialite spatialite-bin pip install tox + pip install --upgrade pip setuptools==57.5.0 + echo -e "gdal==`gdal-config --version`.*\npygdal==`gdal-config --version`.*" >> requirements-gdal.txt - name: Run elasticsearch docker container 📦 From 8e4b5d8dfe9d9515eecaa1c2f02cd1b956bbad31 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 13:34:15 +0100 Subject: [PATCH 20/28] Update main.yml and tox.ini --- .github/workflows/main.yml | 3 +-- tox.ini | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e7cc32e..ebb86fd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,7 +4,7 @@ on: [ push, pull_request ] jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: @@ -23,7 +23,6 @@ jobs: sudo apt-get update sudo apt-get install -y binutils libproj-dev gdal-bin libgdal-dev libsqlite3-mod-spatialite spatialite-bin pip install tox - pip install --upgrade pip setuptools==57.5.0 echo -e "gdal==`gdal-config --version`.*\npygdal==`gdal-config --version`.*" >> requirements-gdal.txt diff --git a/tox.ini b/tox.ini index f4dd78c..99a5319 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,6 @@ deps= django40: Django>=4.0,<4.1 django41: Django>=4.1,<4.2 django42: Django>=4.2,<4.3 - setuptools==57.5.0 -rrequirements-gdal.txt -rrequirements-base.txt -rrequirements-test.txt From 5018aec9548d35d9dfc5ba8086bab9dabc77ece4 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 13:46:51 +0100 Subject: [PATCH 21/28] Update main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ebb86fd..91593e9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,7 +24,7 @@ jobs: sudo apt-get install -y binutils libproj-dev gdal-bin libgdal-dev libsqlite3-mod-spatialite spatialite-bin pip install tox - echo -e "gdal==`gdal-config --version`.*\npygdal==`gdal-config --version`.*" >> requirements-gdal.txt + echo -e "gdal==`gdal-config --version`.*" >> requirements-gdal.txt - name: Run elasticsearch docker container 📦 run: | From 2589efce7780f32fd65c550ff73bba0563031817 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 15:19:11 +0100 Subject: [PATCH 22/28] Update main.yml, setup.py, and tox.ini --- .github/workflows/main.yml | 2 +- setup.py | 2 +- tox.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 91593e9..5528e66 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,7 +4,7 @@ on: [ push, pull_request ] jobs: test: - runs-on: ubuntu-22.04 + runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: diff --git a/setup.py b/setup.py index cf3fb6b..4ed5c0f 100644 --- a/setup.py +++ b/setup.py @@ -63,7 +63,7 @@ if not on_rtd else [], extras_require={ - "backend-django": ["django"], + "backend-django": ["django>=4.0"], "backend-sqlalchemy": ["geoalchemy2", "sqlalchemy<2.0.0"], "backend-native": ["shapely"], "backend-elasticsearch": ["elasticsearch", "elasticsearch-dsl"], diff --git a/tox.ini b/tox.ini index 99a5319..181bcf5 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ requires = tox>=4 env_list = - py{38,39,310,311}-django{32,40,41,42} + py{3.8,3.9,3.10,3.11}-django{40,41,42} passenv = GDAL_VERSION From 08ef222987b63502412e470c258892929e80f7b3 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 15:21:37 +0100 Subject: [PATCH 23/28] Update main.yml and tox.ini --- .github/workflows/main.yml | 2 +- tox.ini | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5528e66..e0a9427 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,5 +31,5 @@ jobs: docker compose up --build --detach es01 - name: Run tox targets for ${{ matrix.python-version }} - run: tox run -f py$(echo ${{ matrix.python-version }} | tr -d .) + run: tox run -f py$(echo ${{ matrix.python-version }}) diff --git a/tox.ini b/tox.ini index 181bcf5..0c2f43d 100644 --- a/tox.ini +++ b/tox.ini @@ -4,8 +4,6 @@ requires = env_list = py{3.8,3.9,3.10,3.11}-django{40,41,42} -passenv = GDAL_VERSION - [testenv] description = run unit tests deps= From 26ce488db9f9cb2ee48217eff96cc39d3ca6e6ce Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 15:25:26 +0100 Subject: [PATCH 24/28] Update requirements-test.txt and tox.ini --- requirements-test.txt | 3 ++- tox.ini | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements-test.txt b/requirements-test.txt index 89a3ffa..06a2c25 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1,4 @@ tox==4.5.1 pytest -pytest-django \ No newline at end of file +pytest-django +setuptools=>58.0.3 \ No newline at end of file diff --git a/tox.ini b/tox.ini index 0c2f43d..9800813 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,7 @@ env_list = [testenv] description = run unit tests deps= + django32: Django>=3.2,<4.0 django40: Django>=4.0,<4.1 django41: Django>=4.1,<4.2 From 56f5899e9575d98d12026989e6afa70ec2c59c92 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Wed, 15 Nov 2023 15:27:52 +0100 Subject: [PATCH 25/28] Update requirements-test.txt --- requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-test.txt b/requirements-test.txt index 06a2c25..b773494 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,4 @@ tox==4.5.1 pytest pytest-django -setuptools=>58.0.3 \ No newline at end of file +setuptools>=58.0.3 \ No newline at end of file From b74d2b74f9bb39e939402898a1a775704d19869e Mon Sep 17 00:00:00 2001 From: jokiefer Date: Thu, 16 Nov 2023 15:02:48 +0100 Subject: [PATCH 26/28] Update .env, main.yml, and 7 more files... --- .env | 3 +- .github/workflows/main.yml | 6 +-- README.md | 5 ++- bullseye-3.9.Dockerfile | 31 --------------- docker-compose.yml | 39 ++++++++++++++----- docker/test/bullseye.Dockerfile | 37 ++++++++++++++++++ setup.py | 2 - tests/backends/elasticsearch/test_evaluate.py | 2 +- tox.ini | 6 +-- 9 files changed, 76 insertions(+), 55 deletions(-) delete mode 100644 bullseye-3.9.Dockerfile create mode 100644 docker/test/bullseye.Dockerfile diff --git a/.env b/.env index 5013ac6..69feecb 100644 --- a/.env +++ b/.env @@ -1,7 +1,8 @@ +# This env file is used by docker compose to inject env variables inside the docker-compose.yml file (not provided to container) + # Project namespace (defaults to the current folder name if not set) #COMPOSE_PROJECT_NAME=myproject - # Password for the 'elastic' user (at least 6 characters) ELASTIC_PASSWORD=changeme diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e0a9427..e925a31 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,8 +28,4 @@ jobs: - name: Run elasticsearch docker container 📦 run: | - docker compose up --build --detach es01 - - - name: Run tox targets for ${{ matrix.python-version }} - run: tox run -f py$(echo ${{ matrix.python-version }}) - + docker compose up --build test-gdal-322 diff --git a/README.md b/README.md index ef91494..6b18855 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ pygeofilter is a pure Python parser implementation of OGC filtering standards [![Build Status](https://github.com/geopython/pygeofilter/workflows/build%20%E2%9A%99%EF%B8%8F/badge.svg)](https://github.com/geopython/pygeofilter/actions) [![Documentation Status](https://readthedocs.org/projects/pygeofilter/badge/?version=latest)](https://pygeofilter.readthedocs.io/en/latest/?badge=latest) +* Tested on debian 11 with gdal version 3.2.2 with python versions 3.9, 3.10 in combination with django versions 4.0, 4.1, 4.2. ## Features @@ -208,10 +209,10 @@ tox -e py311-django41 ``` -To execute tests in Docker: +To execute tests with Docker: ```bash -docker compose up --build test +docker compose up --build test-gdal-322 ``` diff --git a/bullseye-3.9.Dockerfile b/bullseye-3.9.Dockerfile deleted file mode 100644 index 9b8cd40..0000000 --- a/bullseye-3.9.Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM python:3.9-bullseye - -LABEL description="Test executor" - -ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update --fix-missing \ - && apt-get install -y --no-install-recommends \ - binutils \ - libproj-dev \ - gdal-bin \ - libgdal-dev \ - libsqlite3-mod-spatialite \ - spatialite-bin \ - && rm -rf /var/lib/apt/lists/* - -RUN mkdir /app -WORKDIR /app - -COPY requirements-test.txt . -COPY requirements-dev.txt . -RUN pip install -r requirements-test.txt -RUN pip install -r requirements-dev.txt -RUN pip install pygdal=="`gdal-config --version`.*" - -COPY pygeofilter pygeofilter -COPY tests tests -COPY README.md . -COPY setup.py . -RUN pip install -e . - -CMD ["python", "-m", "pytest"] diff --git a/docker-compose.yml b/docker-compose.yml index 682637c..c70e9be 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,28 +1,46 @@ version: '3.8' services: - test: + test-gdal-322: build: context: . - dockerfile: bullseye-3.9.Dockerfile + dockerfile: docker/test/bullseye.Dockerfile tty: true # To support colorized log output. networks: - default + volumes: + - type: bind + source: ./pygeofilter + target: /app/pygeofilter + - type: bind + source: ./tests + target: /app/tests + environment: + - ESHOSTNAME=es01 + depends_on: - es01 - environment: - - eshostname=es01 - + + # to get debug information run in verbose mode + # command: > + # /bin/sh -c "tox run -v -v -v -v" + + # to debug container + # command: > + # tail -f /dev/null + + command: > + /bin/sh -c "tox run" + + es01: depends_on: - setup: - condition: service_healthy + - setup image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0 ports: - ${ES_PORT}:9200 - env_file: - - .env + environment: - node.name=es01 - cluster.name=${CLUSTER_NAME} @@ -44,7 +62,8 @@ services: volumes: - certs:/usr/share/elasticsearch/config/certs - esdata01:/usr/share/elasticsearch/data - + networks: + - default healthcheck: test: [ diff --git a/docker/test/bullseye.Dockerfile b/docker/test/bullseye.Dockerfile new file mode 100644 index 0000000..cac1e2b --- /dev/null +++ b/docker/test/bullseye.Dockerfile @@ -0,0 +1,37 @@ +FROM python:3.10-bullseye +# finally there will be python version 3.9, 3.10 to run matrix tests with tox + +LABEL description="Test executor" + + +RUN set -x \ + && pythonVersions='python3.9 python3.9-dev' \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + # diverse python versions + software-properties-common gpg-agent\ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get purge -y --autoremove software-properties-common gpg-agent \ + && apt-get install -y --no-install-recommends $pythonVersions \ + # Project os dependencies + && apt-get install -y --no-install-recommends \ + binutils \ + libproj-dev \ + gdal-bin \ + libgdal-dev \ + libsqlite3-mod-spatialite \ + spatialite-bin \ + # cleanup + && rm -rf /var/lib/apt/lists/* + +RUN mkdir /app +WORKDIR /app + +COPY requirements-test.txt requirements-base.txt README.md setup.py tox.ini ./ + +# Code base will binded by docker compose. Otherwise the container needs to rebuild on any code change. + +# create dynamic requirements file with the current os gdal version +RUN echo "gdal==`gdal-config --version`.*" >> requirements-gdal.txt \ + && pip install tox + diff --git a/setup.py b/setup.py index 4ed5c0f..060875f 100644 --- a/setup.py +++ b/setup.py @@ -74,10 +74,8 @@ "Intended Audience :: Developers", "Topic :: Scientific/Engineering :: GIS", "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", ], tests_require=["pytest"], ) diff --git a/tests/backends/elasticsearch/test_evaluate.py b/tests/backends/elasticsearch/test_evaluate.py index bdec193..9b25fb3 100644 --- a/tests/backends/elasticsearch/test_evaluate.py +++ b/tests/backends/elasticsearch/test_evaluate.py @@ -53,7 +53,7 @@ class Index: @pytest.fixture(autouse=True, scope="session") def connection(): - hostname = os.environ.get("eshostname", "localhost") + hostname = os.environ.get("ESHOSTNAME", "localhost") connections.create_connection( hosts=[f"https://{hostname}:9200"], ca_certs=False, diff --git a/tox.ini b/tox.ini index 9800813..0e8ba36 100644 --- a/tox.ini +++ b/tox.ini @@ -2,13 +2,11 @@ requires = tox>=4 env_list = - py{3.8,3.9,3.10,3.11}-django{40,41,42} + py{3.9,3.10}-django{40,41,42} [testenv] description = run unit tests deps= - - django32: Django>=3.2,<4.0 django40: Django>=4.0,<4.1 django41: Django>=4.1,<4.2 django42: Django>=4.2,<4.3 @@ -22,3 +20,5 @@ setenv = commands = python -m pytest {posargs} +passenv = * + From 8b801bb2e87d83371d194405f648572613314346 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Thu, 16 Nov 2023 15:04:19 +0100 Subject: [PATCH 27/28] Update main.yml --- .github/workflows/main.yml | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e925a31..2a23ce8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,31 +1,14 @@ -name: build ⚙️ +name: test ⚙️ on: [ push, pull_request ] jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false - matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] - env: - PYTHON: ${{ matrix.python-version }} steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - name: Setup Python ${{ matrix.python-version }} - with: - python-version: ${{ matrix.python-version }} - cache: pip - - name: Install requirements - run: | - sudo apt-get update - sudo apt-get install -y binutils libproj-dev gdal-bin libgdal-dev libsqlite3-mod-spatialite spatialite-bin - pip install tox - - echo -e "gdal==`gdal-config --version`.*" >> requirements-gdal.txt - - name: Run elasticsearch docker container 📦 run: | docker compose up --build test-gdal-322 From 6d01ab061c3f8bb771e105899887355f3e72d135 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Thu, 16 Nov 2023 15:09:02 +0100 Subject: [PATCH 28/28] Update main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2a23ce8..9d1ed6c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,6 +9,6 @@ jobs: fail-fast: false steps: - uses: actions/checkout@v3 - - name: Run elasticsearch docker container 📦 + - name: Run test suite docker container 📦 run: | docker compose up --build test-gdal-322