Reference period (#41)

* Change to reference period

Author: mcarans

doc/main.md

@@ -21,7 +21,7 @@ upload your datasets to HDX.
     -   [Configuring Logging](#configuring-logging)
     -   [Operations on HDX Objects](#operations-on-hdx-objects)
     -   [Dataset Specific Operations](#dataset-specific-operations)
-        -   [Dataset Date](#dataset-date)
+        -   [Reference Period](#reference-period)
         -   [Expected Update Frequency](#expected-update-frequency)
         -   [Location](#location)
         -   [Tags](#tags)
@@ -48,6 +48,10 @@ The library has detailed API documentation which can be found in the menu at the
 
 
 ## Breaking Changes
+From 5.9.8, get_date_of_dataset has become get_reference_period, 
+set_date_of_dataset has become set_reference_period and set_dataset_year_range 
+has become set_reference_period_year_range
+
 From 5.9.7, Python 3.7 no longer supported
 
 From 5.8.2, date handling uses timezone aware dates instead of naive dates and defaults
@@ -208,7 +212,7 @@ virtualenv if not installed:
      from HDX and view the date of the dataset:
 
          dataset = Dataset.read_from_hdx("novel-coronavirus-2019-ncov-cases")
-         print(dataset.get_date_of_dataset())
+         print(dataset.get_reference_period())
 
 11. You can search for datasets on HDX and get their resources:
 
@@ -226,14 +230,14 @@ virtualenv if not installed:
     server. With a dataset to which you have permissions, change the dataset date:
 
          dataset = Dataset.read_from_hdx("ID OR NAME OF DATASET")
-         print(dataset.get_date_of_dataset())  # record this
-         dataset.set_date_of_dataset("2015-07-26")
-         print(dataset.get_date_of_dataset())
+         print(dataset.get_reference_period())  # record this
+         dataset.set_reference_period("2015-07-26")
+         print(dataset.get_reference_period())
          dataset.update_in_hdx()
 
 14. You can view it on HDX before changing it back (if you have an API key):
 
-         dataset.set_date_of_dataset("PREVIOUS DATE")
+         dataset.set_reference_period("PREVIOUS DATE")
          dataset.update_in_hdx()
 
 15. Exit and remove virtualenv:
@@ -561,35 +565,37 @@ example:
 
     dataset.remove_showcase(showcase)
 
-### Dataset Date
+### Reference Period
 
-Dataset date is a mandatory field in HDX. This date is the date of the data in the 
-dataset, not to be confused with when data was last added/changed in the dataset. It can 
-be a single date or a range.
+Reference Period is a mandatory field in HDX. It is the time period for which 
+data are collected or calculated and to which, as a result, they refer. The 
+reference period may be of any length: a year, a month, or even a day. It 
+should not to be confused with when data was last added/changed in the dataset. 
+It can be a single date or a range. 
 
-To get the dataset date, you can do as shown below. It returns a dictionary containing 
+To get the reference period, you can do as shown below. It returns a dictionary containing 
 keys "startdate" (start date as datetime), "enddate" (end date as datetime), 
 "startdate_str" (start date as string), "enddate_str" (end date as string) and ongoing 
 (whether the end date is a rolls forward every day). You can supply a 
 [date format](https://docs.python.org/3/library/datetime.html#strftime-strptime-behavior). 
 If you do not, the output format will be an 
 [ISO 8601 date](https://en.wikipedia.org/wiki/ISO_8601) eg. 2007-01-25.
 
-    dataset_date = dataset.get_date_of_dataset("OPTIONAL FORMAT")
+    reference_period = dataset.get_reference_period("OPTIONAL FORMAT")
 
-To set the dataset date, you must pass either datetime.datetime objects or strings to 
+To set the reference period, you must pass either datetime.datetime objects or strings to 
 the function below. It accepts a start date and an optional end date which if not 
 supplied is assumed to be the same as the start date. Instead of the end date, the flag 
 "ongoing" which by default is False can be set to True which indicates that the end date 
 rolls forward every day. 
 
-    dataset.set_date_of_dataset("START DATE", "END DATE")
+    dataset.set_reference_period("START DATE", "END DATE")
 
-The method below allows you to set the dataset's date using a year range. The start and 
+The method below allows you to set the reference period using a year range. The start and 
 end year can be supplied as integers or strings. If no end year is supplied then the 
 range will be from the beginning of the start year to the end of that year.
 
-    dataset.set_dataset_year_range(START YEAR, END YEAR)
+    dataset.set_reference_period_year_range(START YEAR, END YEAR)
 
 ### Expected Update Frequency
 

src/hdx/data/dataset.py

@@ -1232,7 +1232,7 @@ def autocomplete(
         """
         return cls._autocomplete(name, limit, configuration)
 
-    def get_date_of_dataset(
+    def get_reference_period(
         self,
         date_format: Optional[str] = None,
         today: datetime = now_utc(),
@@ -1253,14 +1253,14 @@ def get_date_of_dataset(
             self.data.get("dataset_date"), date_format, today
         )
 
-    def set_date_of_dataset(
+    def set_reference_period(
         self,
         startdate: Union[datetime, str],
         enddate: Union[datetime, str, None] = None,
         ongoing: bool = False,
         ignore_timeinfo: bool = True,
     ) -> None:
-        """Set dataset date from either datetime objects or strings. Any time and time
+        """Set reference period from either datetime objects or strings. Any time and time
         zone information will be ignored by default (meaning that the time of the start
         date is set to 00:00:00, the time of any end date is set to 23:59:59 and the
         time zone is set to UTC). To have the time and time zone accounted for, set
@@ -1279,12 +1279,12 @@ def set_date_of_dataset(
             startdate, enddate, ongoing, ignore_timeinfo
         )
 
-    def set_dataset_year_range(
+    def set_reference_period_year_range(
         self,
         dataset_year: Union[str, int, Iterable],
         dataset_end_year: Optional[Union[str, int]] = None,
     ) -> List[int]:
-        """Set dataset date as a range from year or start and end year.
+        """Set reference period as a range from year or start and end year.
 
         Args:
             dataset_year (Union[str, int, Iterable]): Dataset year given as string or int or range in an iterable
@@ -2307,17 +2307,17 @@ def get_hdx_url(self) -> Optional[str]:
         return f"{self.configuration.get_hdx_site_url()}/dataset/{name}"
 
     def remove_dates_from_title(
-        self, change_title: bool = True, set_dataset_date: bool = False
+        self, change_title: bool = True, set_reference_period: bool = False
     ) -> List[Tuple[datetime, datetime]]:
         """Remove dates from dataset title returning sorted the dates that were found in
         title. The title in the dataset metadata will be changed by default. The
-        dataset's metadata field dataset date will not be changed by default, but if
-        set_dataset_date is True, then the range with the lowest start date will be used
-        to set the dataset date field.
+        dataset's metadata field reference period will not be changed by default, but if
+        set_reference_period is True, then the range with the lowest start date will be used
+        to set the reference period field.
 
         Args:
             change_title (bool): Whether to change the dataset title. Defaults to True.
-            set_dataset_date (bool): Whether to set dataset date from date or range in title. Defaults to False.
+            set_reference_period (bool): Whether to set reference period from date or range in title. Defaults to False.
 
         Returns:
             List[Tuple[datetime,datetime]]: Date ranges found in title
@@ -2328,9 +2328,9 @@ def remove_dates_from_title(
         newtitle, ranges = DatasetTitleHelper.get_dates_from_title(title)
         if change_title:
             self.data["title"] = newtitle
-        if set_dataset_date and len(ranges) != 0:
+        if set_reference_period and len(ranges) != 0:
             startdate, enddate = ranges[0]
-            self.set_date_of_dataset(startdate, enddate)
+            self.set_reference_period(startdate, enddate)
         return ranges
 
     def generate_resource_from_rows(
@@ -2431,7 +2431,7 @@ def generate_resource_from_iterator(
         to it the dataset. The returned dictionary will contain the resource in the key
         resource, headers in the key headers and list of rows in the key rows.
 
-        The date of dataset can optionally be set by supplying a column in which the
+        The reference period can optionally be set by supplying a column in which the
         date or year is to be looked up. Note that any timezone information is ignored
         and UTC assumed. Alternatively, a function can be supplied to handle any dates
         in a row. It should accept a row and should return None to ignore the row or a
@@ -2465,7 +2465,7 @@ def generate_resource_from_iterator(
             folder (str): Folder to which to write file containing rows
             filename (str): Filename of file to write rows
             resourcedata (Dict): Resource data
-            datecol (Optional[Union[int,str]]): Date column for setting dataset date. Defaults to None (don't set).
+            datecol (Optional[Union[int,str]]): Date column for setting reference period. Defaults to None (don't set).
             yearcol (Optional[Union[int,str]]): Year column for setting dataset year range. Defaults to None (don't set).
             date_function (Optional[Callable[[Dict],Optional[Dict]]]): Date function to call for each row. Defaults to None.
             quickcharts (Optional[Dict]): Dictionary containing optional keys: hashtag, values, cutdown and/or cutdownhashtags
@@ -2588,7 +2588,7 @@ def datecol_function(row):
             else:
                 retdict["startdate"] = dates[0]
                 retdict["enddate"] = dates[1]
-                self.set_date_of_dataset(dates[0], dates[1])
+                self.set_reference_period(dates[0], dates[1])
         resource = self.generate_resource_from_rows(
             folder,
             filename,
@@ -2648,7 +2648,7 @@ def download_and_generate_resource(
         (which will be in dict or list form depending upon the dict_rows argument) and
         outputs a modified row.
 
-        The date of dataset can optionally be set by supplying a column in which the
+        The reference period can optionally be set by supplying a column in which the
         date or year is to be looked up. Note that any timezone information is ignored
         and UTC assumed. Alternatively, a function can be supplied to handle any dates
         in a row. It should accept a row and should return None to ignore the row or a
@@ -2684,7 +2684,7 @@ def download_and_generate_resource(
             resourcedata (Dict): Resource data
             header_insertions (Optional[ListTuple[Tuple[int,str]]]): List of (position, header) to insert. Defaults to None.
             row_function (Optional[Callable[[List[str],Dict],Dict]]): Function to call for each row. Defaults to None.
-            datecol (Optional[str]): Date column for setting dataset date. Defaults to None (don't set).
+            datecol (Optional[str]): Date column for setting reference period. Defaults to None (don't set).
             yearcol (Optional[str]): Year column for setting dataset year range. Defaults to None (don't set).
             date_function (Optional[Callable[[Dict],Optional[Dict]]]): Date function to call for each row. Defaults to None.
             quickcharts (Optional[Dict]): Dictionary containing optional keys: hashtag, values, cutdown and/or cutdownhashtags

test-requirements.txt

@@ -1,4 +1,4 @@
 pytest==7.2.1
 pytest-cov==4.0.0
-tox==4.4.4
+tox==4.4.5
 -r requirements.txt

tests/hdx/data/test_dataset_noncore.py

@@ -194,34 +194,34 @@ def test_get_hdx_url(
 
     def test_get_set_date_of_dataset(self):
         dataset = Dataset({"dataset_date": "[2020-01-07T00:00:00 TO *]"})
-        result = dataset.get_date_of_dataset(today=datetime(2020, 11, 17))
+        result = dataset.get_reference_period(today=datetime(2020, 11, 17))
         assert result == {
             "startdate": datetime(2020, 1, 7, 0, 0, tzinfo=timezone.utc),
             "enddate": datetime(2020, 11, 17, 23, 59, 59, tzinfo=timezone.utc),
             "startdate_str": "2020-01-07T00:00:00+00:00",
             "enddate_str": "2020-11-17T23:59:59+00:00",
             "ongoing": True,
         }
-        dataset.set_date_of_dataset("2020-02-09")
-        result = dataset.get_date_of_dataset("%d/%m/%Y")
+        dataset.set_reference_period("2020-02-09")
+        result = dataset.get_reference_period("%d/%m/%Y")
         assert result == {
             "startdate": datetime(2020, 2, 9, 0, 0, tzinfo=timezone.utc),
             "enddate": datetime(2020, 2, 9, 23, 59, 59, tzinfo=timezone.utc),
             "startdate_str": "09/02/2020",
             "enddate_str": "09/02/2020",
             "ongoing": False,
         }
-        dataset.set_date_of_dataset("2020-02-09", "2020-10-20")
-        result = dataset.get_date_of_dataset("%d/%m/%Y")
+        dataset.set_reference_period("2020-02-09", "2020-10-20")
+        result = dataset.get_reference_period("%d/%m/%Y")
         assert result == {
             "startdate": datetime(2020, 2, 9, 0, 0, tzinfo=timezone.utc),
             "enddate": datetime(2020, 10, 20, 23, 59, 59, tzinfo=timezone.utc),
             "startdate_str": "09/02/2020",
             "enddate_str": "20/10/2020",
             "ongoing": False,
         }
-        dataset.set_date_of_dataset("2020-02-09", ongoing=True)
-        result = dataset.get_date_of_dataset(
+        dataset.set_reference_period("2020-02-09", ongoing=True)
+        result = dataset.get_reference_period(
             "%d/%m/%Y", today=datetime(2020, 3, 9, 0, 0)
         )
         assert result == {
@@ -234,17 +234,17 @@ def test_get_set_date_of_dataset(self):
 
     def test_set_dataset_year_range(self, configuration):
         dataset = Dataset()
-        retval = dataset.set_dataset_year_range(2001, 2015)
+        retval = dataset.set_reference_period_year_range(2001, 2015)
         assert retval == [2001, 2015]
-        retval = dataset.set_dataset_year_range("2010", "2017")
+        retval = dataset.set_reference_period_year_range("2010", "2017")
         assert retval == [2010, 2017]
-        retval = dataset.set_dataset_year_range("2013")
+        retval = dataset.set_reference_period_year_range("2013")
         assert retval == [2013]
-        retval = dataset.set_dataset_year_range({2005, 2002, 2003})
+        retval = dataset.set_reference_period_year_range({2005, 2002, 2003})
         assert retval == [2002, 2003, 2005]
-        retval = dataset.set_dataset_year_range([2005, 2002, 2003])
+        retval = dataset.set_reference_period_year_range([2005, 2002, 2003])
         assert retval == [2002, 2003, 2005]
-        retval = dataset.set_dataset_year_range((2005, 2002, 2003))
+        retval = dataset.set_reference_period_year_range((2005, 2002, 2003))
         assert retval == [2002, 2003, 2005]
 
     def test_is_set_subnational(self):
@@ -879,7 +879,10 @@ def test_remove_dates_from_title(self):
         assert dataset.remove_dates_from_title() == list()
         assert dataset["title"] == title
         assert "dataset_date" not in dataset
-        assert dataset.remove_dates_from_title(set_dataset_date=True) == list()
+        assert (
+            dataset.remove_dates_from_title(set_reference_period=True)
+            == list()
+        )
         title = "ICA Armenia, 2017 - Drought Risk, 1981-2015"
         dataset["title"] = title
         expected = [
@@ -901,7 +904,8 @@ def test_remove_dates_from_title(self):
         assert "dataset_date" not in dataset
         dataset["title"] = title
         assert (
-            dataset.remove_dates_from_title(set_dataset_date=True) == expected
+            dataset.remove_dates_from_title(set_reference_period=True)
+            == expected
         )
         assert dataset["title"] == newtitle
         assert (
@@ -917,7 +921,8 @@ def test_remove_dates_from_title(self):
             )
         ]
         assert (
-            dataset.remove_dates_from_title(set_dataset_date=True) == expected
+            dataset.remove_dates_from_title(set_reference_period=True)
+            == expected
         )
         assert dataset["title"] == "Mon_State_Village_Tract_Boundaries 9999"
         assert (
@@ -926,7 +931,8 @@ def test_remove_dates_from_title(self):
         )
         dataset["title"] = "Mon_State_Village_Tract_Boundaries 2001 99"
         assert (
-            dataset.remove_dates_from_title(set_dataset_date=True) == expected
+            dataset.remove_dates_from_title(set_reference_period=True)
+            == expected
         )
         assert dataset["title"] == "Mon_State_Village_Tract_Boundaries 99"
         assert (
@@ -935,7 +941,8 @@ def test_remove_dates_from_title(self):
         )
         dataset["title"] = "Mon_State_Village_Tract_Boundaries 9999 2001 99"
         assert (
-            dataset.remove_dates_from_title(set_dataset_date=True) == expected
+            dataset.remove_dates_from_title(set_reference_period=True)
+            == expected
         )
         assert dataset["title"] == "Mon_State_Village_Tract_Boundaries 9999 99"
         assert (
@@ -1702,7 +1709,7 @@ def test_load_save_to_json(self, vocabulary_read):
             dataset.set_organization("fb7c2910-6080-4b66-8b4f-0be9b6dc4d8e")
             start_date = "2020-02-09"
             end_date = "2020-10-20"
-            dataset.set_date_of_dataset(start_date, end_date)
+            dataset.set_reference_period(start_date, end_date)
             expected_update_frequency = "Every day"
             dataset.set_expected_update_frequency(expected_update_frequency)
             dataset.set_subnational(False)
@@ -1721,7 +1728,7 @@ def test_load_save_to_json(self, vocabulary_read):
             dataset = Dataset.load_from_json(path)
             assert dataset["name"] == name
             assert dataset["maintainer"] == maintainer
-            dateinfo = dataset.get_date_of_dataset()
+            dateinfo = dataset.get_reference_period()
             assert dateinfo["startdate_str"][:10] == start_date
             assert dateinfo["enddate_str"][:10] == end_date
             assert (

tests/hdx/data/test_update_logic.py

@@ -70,7 +70,7 @@ def new_dataset(self, dataset_data):
         new_dataset.set_organization("c021f6be-3598-418e-8f7f-c7a799194dba")
         new_dataset.set_expected_update_frequency("Every month")
         new_dataset.set_subnational(False)
-        new_dataset.set_dataset_year_range(1961, 2019)
+        new_dataset.set_reference_period_year_range(1961, 2019)
         new_dataset.add_country_location("zmb")
         new_dataset.add_tag("hxl")
         return new_dataset
@@ -82,7 +82,7 @@ def dataset(self, dataset_data, resources_yaml):
         dataset.set_organization("c021f6be-3598-418e-8f7f-c7a799194dba")
         dataset.set_expected_update_frequency("Every month")
         dataset.set_subnational(False)
-        dataset.set_dataset_year_range(1961, 2019)
+        dataset.set_reference_period_year_range(1961, 2019)
         dataset.add_country_location("zmb")
         dataset.add_tag("hxl")
         dataset["id"] = "3adc4bb0-faef-42ae-bd67-0ea08918a629"