[Fixes #14354] Cannot upload multiple layers from GPKG file (#14355)

* [Fixes #14354] Cannot upload multiple layers from GPKG file

Author: mattiagiupponi

.github/workflows/tests.yml

@@ -98,7 +98,7 @@ jobs:
     with:
       suite_name: "Upload Tests"
       test_command: >
-        ./test.sh geonode.upload --duration=30 --failfast
+        ./test.sh geonode.upload --duration=30
 
   # -------------------------------------------------
   # CLEANUP JOB: REMOVE ALL ARTIFACTS AT THE END

geonode/settings.py

@@ -968,8 +968,7 @@
     MIDDLEWARE += ("geonode.security.middleware.AdminAllowedMiddleware",)
 
 # LOCKDOWN API endpoints to prevent unauthenticated access.
-# If set to True, search won't deliver results and filtering ResourceBase-objects is not possible for anonymous users
-API_LOCKDOWN = ast.literal_eval(os.getenv("API_LOCKDOWN", "False"))
+API_LOCKDOWN = ast.literal_eval(os.getenv("API_LOCKDOWN", "True"))
 
 # Require users to authenticate before using Geonode
 LOCKDOWN_GEONODE = ast.literal_eval(os.getenv("LOCKDOWN_GEONODE", "False"))

geonode/upload/handlers/base.py

@@ -478,12 +478,17 @@ def identify_authority(self, layer):
                 if _code is None:
                     raise Exception("CRS authority code not found, fallback to default behaviour")
         except Exception:
-            spatial_ref = layer.GetSpatialRef()
-            spatial_ref.AutoIdentifyEPSG()
-            _name = spatial_ref.GetAuthorityName(None) or spatial_ref.GetAttrValue("AUTHORITY", 0)
-            _code = (
-                spatial_ref.GetAuthorityCode("PROJCS")
-                or spatial_ref.GetAuthorityCode("GEOGCS")
-                or spatial_ref.GetAttrValue("AUTHORITY", 1)
-            )
+            try:
+                spatial_ref = layer.GetSpatialRef()
+                spatial_ref.AutoIdentifyEPSG()
+                _name = spatial_ref.GetAuthorityName(None) or spatial_ref.GetAttrValue("AUTHORITY", 0)
+                _code = (
+                    spatial_ref.GetAuthorityCode("PROJCS")
+                    or spatial_ref.GetAuthorityCode("GEOGCS")
+                    or spatial_ref.GetAttrValue("AUTHORITY", 1)
+                )
+            except Exception:
+                logger.error(
+                    f"The following layer {layer.GetName()} does not have a Coordinate Reference System (CRS) and will be skipped."
+                )
         return f"{_name}:{_code}"

geonode/upload/handlers/common/tests_vector.py

@@ -518,7 +518,7 @@ def test_select_valid_layers(self):
         The function should return only the datasets with a geometry
         The other one are discarded
         """
-        all_layers = GPKGFileHandler().get_ogr2ogr_driver().Open(self.no_crs_gpkg)
+        all_layers = GPKGFileHandler().open_source_file({"base_file": self.no_crs_gpkg})
 
         with self.assertLogs(level="ERROR") as _log:
             valid_layer = GPKGFileHandler()._select_valid_layers(all_layers)

geonode/upload/handlers/common/vector.py

@@ -443,7 +443,7 @@ def extract_resource_to_publish(self, files, action, layer_name, alternate, **kw
                 }
             ]
 
-        layers = self.open_source_file(files)
+        layers = self._select_valid_layers(self.open_source_file(files), filter_layer=layer_name)
         if not layers:
             return []
         return [
@@ -589,18 +589,39 @@ def open_source_file(self, files):
         return [gdal_proxy]
 
     def _select_valid_layers(self, all_layers, **kwargs):
+        """
+        Select valid layers from GDAL datasource objects.
+        If more than one layer is found, it loop over all the possibility
+        to extract all the layers.
+        Is possible to pass a filter_layer argument with the name of the layer
+        to retrieve only the needed one
+        """
+        filter_layer = kwargs.get("filter_layer", None)
         layers = []
-        for layer in all_layers:
-            try:
-                layer = self._extract_layer(layer)
-                self.identify_authority(layer)
-                layers.append(layer)
-            except Exception as e:
-                logger.error(e)
-                logger.error(
-                    f"The following layer {layer.GetName()} does not have a Coordinate Reference System (CRS) and will be skipped."
-                )
-                pass
+
+        for ds in all_layers:
+            if not ds:
+                continue
+            if ds.GetLayerCount() > 0:
+                candidates_layers = [ds.GetLayerByIndex(i) for i in range(ds.GetLayerCount())]
+            else:
+                candidates_layers = [ds]
+            for layer in candidates_layers:
+                try:
+                    lry = self._extract_layer(layer)
+                    if not lry:
+                        continue
+                    lry._parent_ds = ds
+                    self.identify_authority(lry)
+                    if filter_layer and self.fixup_name(lry.GetName()) == filter_layer:
+                        return [lry]
+                    layers.append(lry)
+                except Exception as e:
+                    logger.error(f"Layer skipped due to error: {e}")
+
+        if filter_layer and not layers:
+            logger.warning(f"No layer matching filter '{filter_layer}' was found.")
+
         return layers
 
     def can_overwrite(self, _exec_obj, dataset):

geonode/upload/handlers/gpkg/handler.py

@@ -107,12 +107,12 @@ def is_valid(files, user, **kwargs):
         actual_upload = upload_validator._get_parallel_uploads_count()
         max_upload = upload_validator._get_max_parallel_uploads()
 
-        layers = GPKGFileHandler().get_ogr2ogr_driver().Open(files.get("base_file"))
+        layers = GPKGFileHandler().open_source_file(files)
 
         if not layers:
             raise InvalidGeopackageException("The geopackage provided is invalid")
 
-        layers_count = len(layers)
+        layers_count = layers[0].GetLayerCount()
 
         if layers_count >= max_upload:
             raise UploadParallelismLimitException(
@@ -152,7 +152,7 @@ def handle_xml_file(self, saved_dataset, _exec):
         pass
 
     def _select_valid_layers(self, all_layers, **kwargs):
-        layers = super()._select_valid_layers(all_layers=all_layers)
+        layers = super()._select_valid_layers(all_layers=all_layers, **kwargs)
         if execution_id := kwargs.get("execution_id", None):
             exec_obj = orchestrator.get_execution_object(execution_id)
             if exec_obj.action in (ira.REPLACE.value, ira.UPSERT.value):

geonode/upload/handlers/gpkg/tests.py

@@ -41,7 +41,7 @@ def setUpClass(cls):
         super().setUpClass()
         cls.handler = GPKGFileHandler()
         cls.valid_gpkg = f"{project_dir}/tests/fixture/valid.gpkg"
-        cls.multiple_layer = f"{project_dir}/tests/fixture/multiple_layer.gpkg"
+        cls.multiple_layer = f"{project_dir}/tests/fixture/multiple_layers.gpkg"
         cls.invalid_gpkg = f"{project_dir}/tests/fixture/invalid.gpkg"
         cls.user, _ = get_user_model().objects.get_or_create(username="admin")
         cls.invalid_files = {"base_file": cls.invalid_gpkg}
@@ -181,14 +181,14 @@ def test_select_valid_layers(self):
             step="step",
             action="replace",
             input_params={
-                "files": {"base_file": "/tmp/multiple_layer.gpkg"},
+                "files": {"base_file": "/tmp/multiple_layers.gpkg"},
                 "skip_existing_layer": True,
                 "store_spatial_file": False,
                 "handler_module_path": str(self.handler),
             },
         )
 
-        all_layers = GPKGFileHandler().get_ogr2ogr_driver().Open("/tmp/multiple_layer.gpkg")
+        all_layers = GPKGFileHandler().open_source_file({"base_file": ("/tmp/multiple_layers.gpkg")})
 
         with self.assertRaises(Exception) as exp:
             GPKGFileHandler()._select_valid_layers(all_layers, execution_id=str(exec_id))
@@ -198,4 +198,4 @@ def test_select_valid_layers(self):
             exp.exception.args[0],
         )
 
-        os.remove("/tmp/multiple_layer.gpkg")
+        os.remove("/tmp/multiple_layers.gpkg")

geonode/upload/tests/end2end/test_end2end.py

@@ -55,6 +55,7 @@ def setUpClass(cls) -> None:
         super().setUpClass()
         cls.user = get_user_model().objects.exclude(username="Anonymous").first()
         cls.valid_gkpg = f"{project_dir}/tests/fixture/valid.gpkg"
+        cls.valid_multiple_layers_gkpg = f"{project_dir}/tests/fixture/multiple_layers.gpkg"
         cls.valid_geojson = f"{project_dir}/tests/fixture/valid.geojson"
         cls.no_crs_gpkg = f"{project_dir}/tests/fixture/noCrsTable.gpkg"
         file_path = gisdata.PROJECT_ROOT
@@ -132,9 +133,9 @@ def _assertimport(
                 while (
                     ExecutionRequest.objects.get(exec_id=response.json().get("execution_id"))
                     != ExecutionRequest.STATUS_FINISHED
-                    and tentative <= 10
+                    and tentative <= 15
                 ):
-                    time.sleep(10)
+                    time.sleep(2)
                     tentative += 1
             exc_obj = ExecutionRequest.objects.get(exec_id=response.json().get("execution_id"))
             if exc_obj.status != ExecutionRequest.STATUS_FINISHED:
@@ -228,6 +229,27 @@ def test_file_upload_permissions_none_uploads_successfully(self):
         self._cleanup_layers(name="stazioni_metropolitana")
 
 
+class ImporterMultupleGeoPackageImportTest(BaseImporterEndToEndTest):
+    @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"})
+    @override_settings(GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data")
+    def test_import_multiple_layers_from_geopackage(self):
+        self._cleanup_layers(name="area")
+        self._cleanup_layers(name="example")
+        try:
+            resource = None
+            payload = {"base_file": open(self.valid_multiple_layers_gkpg, "rb"), "action": "upload"}
+            self._assertimport(payload, "area")
+            resource = ResourceBase.objects.filter(
+                Q(alternate__icontains="geonode:example") | Q(alternate__icontains="example")
+            )
+            self.assertTrue(resource.exists())
+        finally:
+            if resource.first():
+                resource.first().delete()
+            self._cleanup_layers(name="area")
+            self._cleanup_layers(name="example")
+
+
 class ImporterNoCRSImportTest(BaseImporterEndToEndTest):
     @override_settings(ASYNC_SIGNALS=False)
     @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"})