Implement optional count in Feature Providers (#2215)

* Implement optional count in Feature Providers

* Revert collection entry

* Add count for SQLite.GPKG

* Update docs

Author: webb-ben

docs/source/configuration.rst

@@ -273,6 +273,9 @@ default.
                 include_extra_query_parameters: false  # include extra query parameters that are not part of the collection properties (default: false)
                 # editable transactions: DO NOT ACTIVATE unless you have setup access control beyond pygeoapi
                 editable: true  # optional: if backend is writable, default is false
+                # count: include `numberMatched` in collection results responses, for providers
+                # that require an additional query to calculate this value (e.g. a SQL COUNT query).
+                count: true  # optional: perform additional count on queries, default is true
                 # coordinate reference systems (CRS) section is optional
                 # default CRSs are http://www.opengis.net/def/crs/OGC/1.3/CRS84 (coordinates without height)
                 # and http://www.opengis.net/def/crs/OGC/1.3/CRS84h (coordinates with ellipsoidal height)

docs/source/publishing/ogcapi-features.rst

@@ -16,30 +16,36 @@ parameters.
 
 
 .. csv-table::
-   :header: Provider, property filters/display, resulttype, bbox, datetime, sortby, skipGeometry, domains, CQL, transactions, crs
+   :header: Provider, property filters/display, resulttype hits/count, bbox, datetime, sortby, skipGeometry, domains, CQL, transactions, crs
    :align: left
 
-   `CSV`_,✅/✅,results/hits,✅,❌,❌,✅,❌,❌,❌,✅
-   `Elasticsearch`_,✅/✅,results/hits,✅,✅,✅,✅,✅,✅,✅,✅
-   `ERDDAP Tabledap Service`_,❌/❌,results/hits,✅,✅,❌,❌,❌,❌,❌,✅
-   `ESRI Feature Service`_,✅/✅,results/hits,✅,✅,✅,✅,❌,❌,❌,✅
-   `GeoJSON`_,✅/✅,results/hits,✅,❌,❌,✅,❌,❌,❌,✅
-   `MongoDB`_,✅/❌,results,✅,✅,✅,✅,❌,❌,❌,✅
-   `MySQL`_,✅/✅,results/hits,✅,✅,✅,✅,❌,✅,✅,✅
-   `OGR`_,✅/❌,results/hits,✅,❌,❌,✅,❌,❌,❌,✅
-   `OpenSearch`_,✅/✅,results/hits,✅,✅,✅,✅,❌,✅,✅,✅
-   `Oracle`_,✅/✅,results/hits,✅,❌,✅,✅,❌,❌,❌,✅
-   `Parquet`_,✅/✅,results/hits,✅,✅,❌,✅,❌,❌,❌,✅
-   `PostgreSQL`_,✅/✅,results/hits,✅,✅,✅,✅,❌,✅,✅,✅
-   `SQLiteGPKG`_,✅/❌,results/hits,✅,❌,❌,✅,❌,❌,❌,✅
-   `SensorThings API`_,✅/✅,results/hits,✅,✅,✅,✅,❌,❌,✅,✅
-   `Socrata`_,✅/✅,results/hits,✅,✅,✅,✅,❌,❌,❌,✅
-   `TinyDB`_,✅/✅,results/hits,✅,✅,✅,✅,✅,❌,✅,✅
+   `CSV`_,✅/✅,✅/✅,✅,❌,❌,✅,❌,❌,❌,✅
+   `Elasticsearch`_,✅/✅,✅/❌,✅,✅,✅,✅,✅,✅,✅,✅
+   `ERDDAP Tabledap Service`_,❌/❌,❌/❌,✅,✅,❌,❌,❌,❌,❌,✅
+   `ESRI Feature Service`_,✅/✅,✅/✅,✅,✅,✅,✅,❌,❌,❌,✅
+   `GeoJSON`_,✅/✅,✅/✅,✅,❌,❌,✅,❌,❌,❌,✅
+   `MongoDB`_,✅/❌,✅/✅,✅,✅,✅,✅,❌,❌,❌,✅
+   `MySQL`_,✅/✅,✅/✅,✅,✅,✅,✅,❌,✅,✅,✅
+   `OGR`_,✅/❌,✅/❌,✅,❌,❌,✅,❌,❌,❌,✅
+   `OpenSearch`_,✅/✅,✅/❌,✅,✅,✅,✅,❌,✅,✅,✅
+   `Oracle`_,✅/✅,✅/✅,✅,❌,✅,✅,❌,❌,❌,✅
+   `Parquet`_,✅/✅,✅/❌,✅,✅,❌,✅,❌,❌,❌,✅
+   `PostgreSQL`_,✅/✅,✅/✅,✅,✅,✅,✅,❌,✅,✅,✅
+   `SQLiteGPKG`_,✅/❌,✅/✅,✅,❌,❌,✅,❌,❌,❌,✅
+   `SensorThings API`_,✅/✅,✅/❌,✅,✅,✅,✅,❌,❌,✅,✅
+   `Socrata`_,✅/✅,✅/✅,✅,✅,✅,✅,❌,❌,❌,✅
+   `TinyDB`_,✅/✅,✅/✅,✅,✅,✅,✅,✅,❌,✅,✅
 
 .. note::
    For more information on CRS transformations, see :ref:`crs`.
 
 
+.. note::
+   Providers that support the query parameter ``resulttype=hits`` will also
+   include ``numberMatched`` in the default ``resulttype=results`` response.
+   Providers that support the configuration option ``count: false`` will 
+   not include the ``numberMatched`` property in the ``results`` response.
+
 Connection examples
 -------------------
 

pygeoapi/provider/csv_.py

@@ -194,8 +194,9 @@ def _load(self, offset=0, limit=10, resulttype='results',
 
                 feature_collection['features'].append(feature)
 
-                feature_collection['numberMatched'] = \
-                    len(feature_collection['features'])
+                if self.count:
+                    feature_collection['numberMatched'] = \
+                        len(feature_collection['features'])
 
         if identifier is not None and not found:
             return None

pygeoapi/provider/esri.py

@@ -156,9 +156,13 @@ def query(self, offset=0, limit=10, resulttype='results',
         fc = {
             'type': 'FeatureCollection',
             'features': [],
-            'numberMatched': self._get_count(params)
         }
 
+        if self.count or resulttype == 'hits':
+            matched = self._get_count(params)
+            LOGGER.debug(f'Found {matched} result(s)')
+            fc['numberMatched'] = matched
+
         if resulttype == 'hits':
             return fc
 
@@ -168,7 +172,7 @@ def query(self, offset=0, limit=10, resulttype='results',
         params['resultOffset'] = offset
         params['resultRecordCount'] = limit
 
-        hits_ = min(limit, fc['numberMatched'])
+        hits_ = min(limit, matched) if self.count else limit
         fc['features'] = self._get_all(params, hits_)
 
         fc['numberReturned'] = len(fc['features'])

pygeoapi/provider/geojson.py

@@ -188,7 +188,8 @@ def query(self, offset=0, limit=10, resulttype='results',
                           properties=properties,
                           select_properties=select_properties)
 
-        data['numberMatched'] = len(data['features'])
+        if self.count or resulttype == 'hits':
+            data['numberMatched'] = len(data['features'])
 
         if resulttype == 'hits':
             data['features'] = []

pygeoapi/provider/mongo.py

@@ -102,7 +102,6 @@ def _get_feature_list(self, filterObj, sortList=[], skip=0, maxitems=1,
         if sortList:
             featurecursor = featurecursor.sort(sortList)
 
-        matchCount = self.featuredb[self.collection].count_documents(filterObj)
         featurecursor.skip(skip)
         featurecursor.limit(maxitems)
         featurelist = list(featurecursor)
@@ -111,7 +110,7 @@ def _get_feature_list(self, filterObj, sortList=[], skip=0, maxitems=1,
             if skip_geometry:
                 item['geometry'] = None
 
-        return featurelist, matchCount
+        return featurelist
 
     @crs_transform
     def query(self, offset=0, limit=10, resulttype='results',
@@ -144,20 +143,28 @@ def query(self, offset=0, limit=10, resulttype='results',
                       ASCENDING if (sort['order'] == '+') else DESCENDING)
                      for sort in sortby]
 
-        featurelist, matchcount = self._get_feature_list(
-            filterobj, sortList=sort_list, skip=offset, maxitems=limit,
-            skip_geometry=skip_geometry)
-
-        if resulttype == 'hits':
-            featurelist = []
-
         feature_collection = {
             'type': 'FeatureCollection',
-            'features': featurelist,
-            'numberMatched': matchcount,
-            'numberReturned': len(featurelist)
+            'features': []
         }
 
+        if self.count or resulttype == 'hits':
+            matched = self.featuredb[self.collection].count_documents(
+                filterobj)
+            LOGGER.debug(f'Found {matched} result(s)')
+            feature_collection['numberMatched'] = matched
+
+        if resulttype == 'hits':
+            return feature_collection
+
+        featurelist = self._get_feature_list(
+            filterobj, sortList=sort_list, skip=offset, maxitems=limit,
+            skip_geometry=skip_geometry
+        )
+
+        feature_collection['features'] = featurelist
+        feature_collection['numberReturned'] = len(featurelist)
+
         return feature_collection
 
     @crs_transform
@@ -168,8 +175,7 @@ def get(self, identifier, **kwargs):
         :param identifier: feature id
         :returns: dict of single GeoJSON feature
         """
-        featurelist, matchcount = self._get_feature_list(
-                                    {'_id': ObjectId(identifier)})
+        featurelist = self._get_feature_list({'_id': ObjectId(identifier)})
         if featurelist:
             return featurelist[0]
         else:

pygeoapi/provider/socrata.py

@@ -123,10 +123,14 @@ def query(self, offset=0, limit=10, resulttype='results',
 
         fc = {
             'type': 'FeatureCollection',
-            'features': [],
-            'numberMatched': self._get_count(params)
+            'features': []
         }
 
+        if self.count or resulttype == 'hits':
+            matched = self._get_count(params)
+            LOGGER.debug(f'Found {matched} result(s)')
+            fc['numberMatched'] = matched
+
         if resulttype == 'hits':
             # Return hits
             LOGGER.debug('Returning hits')

pygeoapi/provider/sql.py

@@ -220,6 +220,7 @@ def query(
 
             crs_transform_out = get_transform_from_spec(crs_transform_spec)
 
+            response['numberReturned'] = 0
             for item in (
                 results.order_by(*order_by_clauses).offset(offset).limit(limit)
             ):

pygeoapi/provider/sqlite.py

@@ -169,18 +169,6 @@ def __response_feature(self, row_data, skip_geometry=False):
         else:
             return None
 
-    def __response_feature_hits(self, hits):
-        """Assembles GeoJSON/Feature number
-
-        :returns: GeoJSON FeaturesCollection
-        """
-
-        feature_collection = {"features": [],
-                              "type": "FeatureCollection"}
-        feature_collection['numberMatched'] = hits
-
-        return feature_collection
-
     def __load(self):
         """
         Private method for loading spatiallite,
@@ -305,14 +293,25 @@ def query(self, offset=0, limit=10, resulttype='results',
         where_clause, where_values = self.__get_where_clauses(
             properties=properties, bbox=bbox)
 
-        if resulttype == 'hits':
+        feature_collection = {
+            'type': 'FeatureCollection',
+            'features': []
+        }
+
+        if self.count or resulttype == 'hits':
 
-            sql_query = f"SELECT COUNT(*) as hits FROM {self.table} {where_clause} "  # noqa
+            sql_query = (
+                'SELECT COUNT(*) as hits '
+                f'FROM {self.table} {where_clause}'
+            )
 
             res = self.cursor.execute(sql_query, where_values)
 
-            hits = res.fetchone()["hits"]
-            return self.__response_feature_hits(hits)
+            hits = res.fetchone()['hits']
+            feature_collection['numberMatched'] = hits
+
+        if resulttype == 'hits':
+            return feature_collection
 
         sql_query = f"SELECT DISTINCT {self.columns} from \
             {self.table} {where_clause} limit ? offset ?"
@@ -326,14 +325,12 @@ def query(self, offset=0, limit=10, resulttype='results',
         row_data = self.cursor.execute(
             sql_query, where_values + (limit, offset))
 
-        feature_collection = {
-            'type': 'FeatureCollection',
-            'features': []
-        }
-
+        feature_collection['numberReturned'] = 0
         for rd in row_data:
             feature_collection['features'].append(
-                self.__response_feature(rd, skip_geometry=skip_geometry))
+                self.__response_feature(rd, skip_geometry=skip_geometry)
+            )
+            feature_collection['numberReturned'] += 1
 
         return feature_collection
 

pygeoapi/provider/tinydb_.py

@@ -229,7 +229,10 @@ def query(self, offset=0, limit=10, resulttype='results',
             LOGGER.error(f'{msg}: {err}')
             raise ProviderInvalidQueryError(msg)
 
-        feature_collection['numberMatched'] = len(results)
+        if self.count or resulttype == 'hits':
+            matched = len(results)
+            LOGGER.debug(f'Found {matched} result(s)')
+            feature_collection['numberMatched'] = matched
 
         if resulttype == 'hits':
             return feature_collection