enh: convenience kwarg since_time for search_dataset_via_api

Author: paulmueller

CHANGELOG

@@ -3,6 +3,7 @@
  - fix: '_condensed' suffix for file stem randomly missing in downloads
  - enh: add `CKANAPI.hostname`
  - enh: sort search results by creation date instead of score and modified date
+ - enh: convenience kwarg `since_time` for `search_dataset_via_api`
  - ref: replace `user_list` with `user_autocomplete`
  - ref: replace `search_dataset` with `search_dataset_via_api`
    in `APIInterrogator`; `search_dataset` is now only free text search.

dcoraid/dbmodel/db_api.py

@@ -1,5 +1,6 @@
 from itertools import islice
 import sys
+import time
 import urllib.parse
 
 import numpy as np
@@ -137,6 +138,7 @@ def search_dataset_via_api(self,
                                circles: list[str] = None,
                                collections: list[str] = None,
                                circle_collection_union: bool = False,
+                               since_time: float = None,
                                sort_solr: str = "metadata_created desc",
                                limit: int = 100):
         """Search datasets via the CKAN API
@@ -146,7 +148,9 @@ def search_dataset_via_api(self,
         query: str
             search query
         filter_queries: list of str
-            SOLR `fq` filter queries (are joined with 'AND')
+            SOLR `fq` filter queries (are joined with 'AND'). The `circles`,
+            `collections`, `circle_collection_union`, and `since_date`
+            convenience kwargs are appended to the query list.
         circles: list of str
             list of circles (organizations) to search in
         collections: list of str
@@ -156,6 +160,9 @@ def search_dataset_via_api(self,
             sets. Otherwise (default), search only for datasets that
             are at least member of one of the circles and one of the
             collections.
+        since_time: float
+            Return only datasets that have been modified after this time
+            since the epoch.
         sort_solr: str
             SOLR search ordering. By default, sort according to dataset
             creation date `'metadata_created desc'`. The CKAN default is
@@ -185,6 +192,7 @@ def search_dataset_via_api(self,
         else:
             solr_collections_query = None
 
+        # collections and/or circles filter query
         if solr_circle_query and solr_collections_query:
             if circle_collection_union:
                 fq = f"({solr_circle_query} OR {solr_collections_query})"
@@ -198,6 +206,16 @@ def search_dataset_via_api(self,
             fq = ""
         if fq:
             filter_queries.append(fq)
+
+        # time filter query
+        if since_time is not None:
+            gm_time_str = time.strftime(r"%Y-%m-%dT%H\:%M\:%SZ",
+                                        time.gmtime(since_time-60))
+            # Use "metadata_modified", since datasets that were previously
+            # drafts or private datasets made public would not show up
+            # if "metadata_created" was used.
+            filter_queries.append(f"metadata_modified:[{gm_time_str} TO NOW]")
+
         if len(filter_queries) == 0:
             final_fq = ""
         elif len(filter_queries) == 1:

tests/test_dbmodel_api.py

@@ -1,5 +1,6 @@
 import pathlib
 import random
+import time
 
 import pytest
 
@@ -174,6 +175,31 @@ def test_search_dataset_only_one_filter_query():
         assert False, "Search did not return figshare-7771184-v2!"
 
 
+def test_search_dataset_since_time():
+    """Search datasets that have been modified after some time"""
+    api = common.get_api()
+    db = db_api.APIInterrogator(api=api)
+    tstart = time.time()
+
+    # Normally, this should not return anything, except for a race
+    # condition when multiple tests are running at the same time.
+    de0 = db.search_dataset_via_api(since_time=tstart)
+
+    # Create a dataset
+    ds_dict = common.make_dataset_for_download()
+
+    # Run the query again
+    de1 = db.search_dataset_via_api(since_time=tstart)
+    # The new dataset should be in the results.
+    assert len(de1) > len(de0)
+
+    for item in de1:
+        if item["id"] == ds_dict["id"]:
+            break
+    else:
+        assert False, "created dataset not found"
+
+
 @pytest.mark.skipif(not HAS_FIGSHARE_ACCESS,
                     reason="No access to figshare-import circle")
 def test_get_datasets_user_shared_figshare():