Remove gdal (#638)

* removed gdal dependency

* run on ubuntu latest

* Update pywps/validator/complexvalidator.py

Co-authored-by: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com>

* clean up test_ows

* update exception in complex validator

Co-authored-by: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com>

Author: cehbrecht

.github/workflows/main.yml

@@ -4,18 +4,18 @@ on: [ push, pull_request ]
 
 jobs:
   main:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [3.7, 3.8, 3.9]
     env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         COVERALLS_SERVICE_NAME: github
     steps:
     - uses: actions/checkout@v2
     - name: Install packages
       run: |
-        sudo apt-get update && sudo apt-get -y install gdal-bin libgdal-dev libnetcdf-dev libhdf5-dev
+        sudo apt-get update && sudo apt-get -y install libnetcdf-dev libhdf5-dev
     - uses: actions/setup-python@v2
       name: Setup Python ${{ matrix.python-version }}
       with:
@@ -26,17 +26,16 @@ jobs:
         pip3 install -r requirements.txt
         pip3 install -r requirements-dev.txt
         pip3 install -r requirements-extra.txt
-        pip3 install -r requirements-gdal.txt
     - name: run tests ⚙️
       run: pytest -v tests
     - name: run coveralls ⚙️
       run: coveralls
-      if: matrix.python-version == 3.6
+      if: matrix.python-version == 3.7
     - name: build docs 🏗️
       run: |
         pip3 install -e .
         cd docs && make html
-      if: matrix.python-version == 3.6
+      if: matrix.python-version == 3.7
     - name: run flake8 ⚙️
       run: flake8 pywps
-      if: matrix.python-version == 3.6
+      if: matrix.python-version == 3.7

pywps/dependencies.py

@@ -6,13 +6,6 @@
 import warnings
 
 try:
-    import osgeo
-    from osgeo import gdal, ogr
-except ImportError:
-    warnings.warn('Complex validation requires GDAL/OGR support.')
-    ogr = None
-
-try:
-    import netCDF4
+    import netCDF4  # noqa
 except ImportError:
     warnings.warn('Complex validation requires netCDF4 support.')

pywps/validator/complexvalidator.py

@@ -35,7 +35,7 @@ def validategml(data_input, mode):
     `MODE.SIMPLE`
         the mimetype will be checked
     `MODE.STRICT`
-        `GDAL/OGR <http://gdal.org/>`_ is used for getting the proper format.
+        `Fiona` is used for getting the proper format.
     `MODE.VERYSTRICT`
         the :class:`lxml.etree` is used along with given input `schema` and the
         GML file is properly validated against given schema.
@@ -55,11 +55,11 @@ def validategml(data_input, mode):
 
     if mode >= MODE.STRICT:
 
-        from pywps.dependencies import ogr
-        data_source = ogr.Open(data_input.file)
-        if data_source:
-            passed = (data_source.GetDriver().GetName() == "GML")
-        else:
+        try:
+            import fiona
+            data_source = fiona.open(data_input.file)
+            passed = (data_source.driver == "GML")
+        except (ModuleNotFoundError, ImportError):
             passed = False
 
     if mode >= MODE.VERYSTRICT:
@@ -89,7 +89,7 @@ def validategpx(data_input, mode):
     `MODE.SIMPLE`
         the mimetype will be checked
     `MODE.STRICT`
-        `GDAL/OGR <http://gdal.org/>`_ is used for getting the proper format.
+        `Fiona` is used for getting the proper format.
     `MODE.VERYSTRICT`
         the :class:`lxml.etree` is used along with given input `schema` and the
         GPX file is properly validated against given schema.
@@ -109,11 +109,11 @@ def validategpx(data_input, mode):
 
     if mode >= MODE.STRICT:
 
-        from pywps.dependencies import ogr
-        data_source = ogr.Open(data_input.file)
-        if data_source:
-            passed = (data_source.GetDriver().GetName() == "GPX")
-        else:
+        try:
+            import fiona
+            data_source = fiona.open(data_input.file)
+            passed = (data_source.driver == "GPX")
+        except (ModuleNotFoundError, ImportError):
             passed = False
 
     if mode >= MODE.VERYSTRICT:
@@ -249,11 +249,11 @@ def validategeojson(data_input, mode):
 
     if mode >= MODE.STRICT:
 
-        from pywps.dependencies import ogr
-        data_source = ogr.Open(data_input.file)
-        if data_source:
-            passed = (data_source.GetDriver().GetName() == "GeoJSON")
-        else:
+        try:
+            import fiona
+            data_source = fiona.open(data_input.file)
+            passed = (data_source.driver == "GeoJSON")
+        except (ModuleNotFoundError, ImportError):
             passed = False
 
     if mode >= MODE.VERYSTRICT:
@@ -317,22 +317,11 @@ def validateshapefile(data_input, mode):
 
     if mode >= MODE.STRICT:
 
-        from pywps.dependencies import ogr
-
-        import zipfile
-        z = zipfile.ZipFile(data_input.file)
-        shape_name = None
-        for name in z.namelist():
-            z.extract(name, data_input.tempdir)
-            if os.path.splitext(name)[1].lower() == '.shp':
-                shape_name = name
-
-        if shape_name:
-            data_source = ogr.Open(os.path.join(data_input.tempdir, shape_name))
-
-        if data_source:
-            passed = (data_source.GetDriver().GetName() == "ESRI Shapefile")
-        else:
+        try:
+            import fiona
+            sf = fiona.open(data_input.file)
+            passed = (sf.driver == "ESRI Shapefile")
+        except (ModuleNotFoundError, ImportError):
             passed = False
 
     return passed
@@ -357,10 +346,10 @@ def validategeotiff(data_input, mode):
     if mode >= MODE.STRICT:
 
         try:
-            from pywps.dependencies import osgeo # noqa
-            data_source = osgeo.gdal.Open(data_input.file)
-            passed = (data_source.GetDriver().ShortName == "GTiff")
-        except ImportError:
+            from geotiff import GeoTiff
+            data_source = GeoTiff(data_input.file)
+            passed = (data_source.crs_code > 0)
+        except (ModuleNotFoundError, ImportError):
             passed = False
 
     return passed

requirements-extra.txt

@@ -1 +1 @@
-netCDF4
+netCDF4

requirements-gdal.txt

@@ -8,3 +8,5 @@ SQLAlchemy
 werkzeug
 MarkupSafe
 humanize
+geotiff
+fiona

requirements.txt

@@ -10,7 +10,6 @@
 import tempfile
 import unittest
 from pywps import Service, Process, ComplexInput, ComplexOutput, Format, FORMATS, get_format
-from pywps.dependencies import ogr
 from pywps.exceptions import NoApplicableCode
 from pywps import get_ElementMakerForVersion
 import pywps.configuration as config
@@ -26,41 +25,8 @@ def create_feature():
 
     def feature(request, response):
         input = request.inputs['input'][0].file
-        # What do we need to assert a Complex input?
-        # assert type(input) is str
-
-        # open the input file
-        try:
-            inSource = ogr.Open(input)
-        except Exception as e:
-            return "Could not open given vector file: {}".format(e)
-        inLayer = inSource.GetLayer()
-
-        # create output file
-        out = 'point'
-        outPath = os.path.join(tempfile.gettempdir(), out)
-
-        driver = ogr.GetDriverByName('GML')
-        outSource = driver.CreateDataSource(
-            outPath,
-            ["XSISCHEMAURI=http://schemas.opengis.net/gml/2.1.2/feature.xsd"])
-        outLayer = outSource.CreateLayer(out, None, ogr.wkbUnknown)
-
-        # get the first feature
-        inFeature = inLayer.GetNextFeature()
-        inGeometry = inFeature.GetGeometryRef()
-
-        # make the buffer
-        buff = inGeometry.Buffer(float(100000))
-
-        # create output feature to the file
-        outFeature = ogr.Feature(feature_def=outLayer.GetLayerDefn())
-        outFeature.SetGeometryDirectly(buff)
-        outLayer.CreateFeature(outFeature)
-        outFeature.Destroy()
-
         response.outputs['output'].data_format = FORMATS.GML
-        response.outputs['output'].file = outPath
+        response.outputs['output'].file = input
         return response
 
     return Process(handler=feature,

tests/test_ows.py

@@ -8,34 +8,40 @@
 
 import unittest
 import pytest
-import sys
-from pywps.validator.complexvalidator import *
+from pywps.validator.mode import MODE
+from pywps.validator.complexvalidator import (
+    validategml,
+    # validategpx,
+    # validatexml,
+    validatejson,
+    validategeojson,
+    validateshapefile,
+    validategeotiff,
+    validatenetcdf,
+    validatedods,
+)
 from pywps.inout.formats import FORMATS
 from pywps import ComplexInput
 from pywps.inout.basic import SOURCE_TYPE
 import tempfile
 import os
 
-try:
-    import osgeo
-except ImportError:
-    WITH_GDAL = False
-else:
-    WITH_GDAL = True
 
 try:
-    import netCDF4
+    import netCDF4  # noqa
 except ImportError:
     WITH_NC4 = False
 else:
     WITH_NC4 = True
 
+
 def get_input(name, schema, mime_type):
 
     class FakeFormat(object):
         mimetype = 'text/plain'
         schema = None
         units = None
+
         def validate(self, data):
             return True
 
@@ -67,20 +73,17 @@ class ValidateTest(unittest.TestCase):
     def setUp(self):
         pass
 
-
     def tearDown(self):
         pass
 
-    @unittest.skip('long')
     def test_gml_validator(self):
         """Test GML validator
         """
         gml_input = get_input('gml/point.gml', 'point.xsd', FORMATS.GML.mime_type)
         self.assertTrue(validategml(gml_input, MODE.NONE), 'NONE validation')
         self.assertTrue(validategml(gml_input, MODE.SIMPLE), 'SIMPLE validation')
-        if WITH_GDAL:
-            self.assertTrue(validategml(gml_input, MODE.STRICT), 'STRICT validation')
-            self.assertTrue(validategml(gml_input, MODE.VERYSTRICT), 'VERYSTRICT validation')
+        self.assertTrue(validategml(gml_input, MODE.STRICT), 'STRICT validation')
+        self.assertTrue(validategml(gml_input, MODE.VERYSTRICT), 'VERYSTRICT validation')
         gml_input.stream.close()
 
     def test_json_validator(self):
@@ -99,9 +102,8 @@ def test_geojson_validator(self):
                                   FORMATS.GEOJSON.mime_type)
         self.assertTrue(validategeojson(geojson_input, MODE.NONE), 'NONE validation')
         self.assertTrue(validategeojson(geojson_input, MODE.SIMPLE), 'SIMPLE validation')
-        if WITH_GDAL:
-            self.assertTrue(validategeojson(geojson_input, MODE.STRICT), 'STRICT validation')
-            self.assertTrue(validategeojson(geojson_input, MODE.VERYSTRICT), 'VERYSTRICT validation')
+        self.assertTrue(validategeojson(geojson_input, MODE.STRICT), 'STRICT validation')
+        self.assertTrue(validategeojson(geojson_input, MODE.VERYSTRICT), 'VERYSTRICT validation')
         geojson_input.stream.close()
 
     def test_shapefile_validator(self):
@@ -111,8 +113,7 @@ def test_shapefile_validator(self):
                 FORMATS.SHP.mime_type)
         self.assertTrue(validateshapefile(shapefile_input, MODE.NONE), 'NONE validation')
         self.assertTrue(validateshapefile(shapefile_input, MODE.SIMPLE), 'SIMPLE validation')
-        if WITH_GDAL:
-            self.assertTrue(validateshapefile(shapefile_input, MODE.STRICT), 'STRICT validation')
+        self.assertTrue(validateshapefile(shapefile_input, MODE.STRICT), 'STRICT validation')
         shapefile_input.stream.close()
 
     def test_geotiff_validator(self):
@@ -122,8 +123,6 @@ def test_geotiff_validator(self):
                                   FORMATS.GEOTIFF.mime_type)
         self.assertTrue(validategeotiff(geotiff_input, MODE.NONE), 'NONE validation')
         self.assertTrue(validategeotiff(geotiff_input, MODE.SIMPLE), 'SIMPLE validation')
-        if not WITH_GDAL:
-            self.skipTest('GDAL not Installed')
         self.assertTrue(validategeotiff(geotiff_input, MODE.STRICT), 'STRICT validation')
         geotiff_input.stream.close()
 
@@ -161,12 +160,12 @@ def test_dods_default(self):
                                      default_type=SOURCE_TYPE.URL,
                                      mode=MODE.SIMPLE)
 
-
     def test_fail_validator(self):
         fake_input = get_input('point.xsd', 'point.xsd', FORMATS.SHP.mime_type)
         self.assertFalse(validategml(fake_input, MODE.SIMPLE), 'SIMPLE validation invalid')
         fake_input.stream.close()
 
+
 def load_tests(loader=None, tests=None, pattern=None):
     if not loader:
         loader = unittest.TestLoader()