remove freq parameter from gaps.completeness functions

docs/examples/pvfleets-qa-pipeline/pvfleets-irradiance-qa.py

@@ -167,8 +167,7 @@
 # Trim the series based on daily completeness score
 trim_series = pvanalytics.quality.gaps.trim_incomplete(
     time_series,
-    minimum_completeness=.25,
-    freq=data_freq)
+    minimum_completeness=.25)
 first_valid_date, last_valid_date = \
     pvanalytics.quality.gaps.start_stop_dates(trim_series)
 time_series = time_series[first_valid_date.tz_convert(time_series.index.tz):

docs/examples/pvfleets-qa-pipeline/pvfleets-power-qa.py

@@ -163,7 +163,7 @@
 
 # Trim the series based on daily completeness score
 trim_series = pvanalytics.quality.gaps.trim_incomplete(
-    time_series, minimum_completeness=.25, freq=data_freq)
+    time_series, minimum_completeness=.25)
 first_valid_date, last_valid_date = \
     pvanalytics.quality.gaps.start_stop_dates(trim_series)
 time_series = time_series[first_valid_date.tz_convert(time_series.index.tz):

docs/examples/pvfleets-qa-pipeline/pvfleets-temperature-qa.py

@@ -178,8 +178,7 @@
 # Trim the series based on daily completeness score
 trim_series = pvanalytics.quality.gaps.trim_incomplete(
     time_series,
-    minimum_completeness=.25,
-    freq=data_freq)
+    minimum_completeness=.25)
 first_valid_date, last_valid_date = \
     pvanalytics.quality.gaps.start_stop_dates(trim_series)
 time_series = time_series[first_valid_date.tz_convert(time_series.index.tz):

docs/whatsnew/v0.2.3.rst

@@ -12,6 +12,9 @@ Bug Fixes
 ~~~~~~~~~
 * :py:func:`pvanalytics.features.clearsky.reno` now correctly passes
   ``window_length`` to the underlying pvlib function. (:pull:`221`)
+* Remove freq parameter from :py:func:`pvanalytics.quality.gaps.completeness`
+  and :py:func:`pvanalytics.quality.gaps.completeness_score`. Frequency is now
+  always calculated from the input data's DatetimeIndex. (:pull:`236`) 
 
 Requirements
 ~~~~~~~~~~~~
@@ -35,4 +38,4 @@ Testing
 
 Contributors
 ~~~~~~~~~~~~
-
+* Cliff Hansen (:ghuser:`cwhanse`)

pvanalytics/quality/gaps.py

@@ -250,13 +250,11 @@ def _freq_to_seconds(freq):
     return delta.days * (1440 * 60) + delta.seconds
 
 
-def completeness_score(series, freq=None, keep_index=True):
+def completeness_score(series, keep_index=True):
     """Calculate a data completeness score for each day.
 
     The completeness score for a given day is the fraction of time in
-    the day for which there is data (a value other than NaN). The time
-    duration attributed to each value is equal to the timestamp
-    spacing of `series`, or `freq` if it is specified. For example, a
+    the day for which there is data (a value other than NaN). For example, a
     24-hour time series with 30 minute timestamp spacing and 24
     non-NaN values would have data for a total of 12 hours and
     therefore a completeness score of 0.5.
@@ -265,10 +263,6 @@ def completeness_score(series, freq=None, keep_index=True):
     ----------
     series : Series
         A DatetimeIndexed series.
-    freq : str, default None
-        Interval between samples in the series as a pandas frequency
-        string. If None, the frequency is inferred using
-        :py:func:`pandas.infer_freq`.
     keep_index : boolean, default True
         Whether or not the returned series has the same index as
         `series`. If False the returned series will be indexed by day.
@@ -279,30 +273,16 @@ def completeness_score(series, freq=None, keep_index=True):
         A series of floats giving the completeness score for each day
         (fraction of the day for which `series` has data).
 
-    Raises
-    ------
-    ValueError
-        If `freq` is longer than the frequency inferred from `series`.
-
     """
-    inferred_seconds = _freq_to_seconds(pd.infer_freq(series.index))
-    if freq:
-        freq_seconds = _freq_to_seconds(freq)
-        seconds_per_sample = freq_seconds
-    else:
-        seconds_per_sample = inferred_seconds
-
-    if freq and inferred_seconds < freq_seconds:
-        raise ValueError("freq must be less than or equal to the"
-                         + " frequency of the series")
+    seconds_per_sample = series.index.diff().total_seconds()[1]
     daily_counts = series.resample('D').count()
     daily_completeness = (daily_counts * seconds_per_sample) / (1440*60)
     if keep_index:
         return daily_completeness.reindex(series.index, method='pad')
     return daily_completeness
 
 
-def complete(series, minimum_completeness=0.333, freq=None):
+def complete(series, minimum_completeness=0.333):
     """Select data points that are part of days with complete data.
 
     A day has complete data if its completeness score is greater than
@@ -312,12 +292,9 @@ def complete(series, minimum_completeness=0.333, freq=None):
     Parameters
     ----------
     series : Series
-        The data to be checked for completeness.
+        A DatetimeIndexed series to be checked for completeness.
     minimum_completeness : float, default 0.333
         Fraction of the day that must have data.
-    freq : str, default None
-        The expected frequency of the data in `series`. If none then
-        the frequency is inferred from the data.
 
     Returns
     -------
@@ -335,7 +312,7 @@ def complete(series, minimum_completeness=0.333, freq=None):
     completeness_score
 
     """
-    return completeness_score(series, freq=freq) >= minimum_completeness
+    return completeness_score(series) >= minimum_completeness
 
 
 def start_stop_dates(series, days=10):
@@ -415,7 +392,7 @@ def trim(series, days=10):
     return mask
 
 
-def trim_incomplete(series, minimum_completeness=0.333333, days=10, freq=None):
+def trim_incomplete(series, minimum_completeness=0.333333, days=10):
     """Trim the series based on the completeness score.
 
     Combines :py:func:`completeness_score` and :py:func:`trim`.
@@ -430,9 +407,6 @@ def trim_incomplete(series, minimum_completeness=0.333333, days=10, freq=None):
         The number of consecutive days with completeness greater than
         `minumum_completeness` for the 'good' data to start or
         end. See :py:func:`start_stop_dates` for more information.
-    freq : str, default None
-        The expected frequency of the series. See
-        :py:func:`completeness_score` fore more information.
 
     Returns
     -------
@@ -448,6 +422,6 @@ def trim_incomplete(series, minimum_completeness=0.333333, days=10, freq=None):
     completeness_score
 
     """
-    completeness = completeness_score(series, freq=freq)
+    completeness = completeness_score(series)
     complete_days = completeness >= minimum_completeness
     return trim(complete_days, days=days)

pvanalytics/tests/quality/test_gaps.py

@@ -492,33 +492,15 @@ def test_completeness_score_all_nans():
 def test_completeness_score_no_data():
     """A data set with completely missing timestamps and NaNs has
     completeness 0."""
-    four_days = pd.date_range(start='01/01/2020', freq='D', periods=4)
-    completeness = gaps.completeness_score(
-        pd.Series(index=four_days, dtype='float64'),
-        freq='15min',
-        keep_index=False
-    )
+    four_days = pd.date_range(start='01/01/2020', end='01/04/2020', periods=4)
+    test_data = pd.Series(index=four_days, dtype='float64')
+    completeness = gaps.completeness_score(test_data, keep_index=False)
+    # have to exclude freq because completeness is returned with freq='D'
+    # due to resampling, but test_data is not constructed with freq
     assert_series_equal(
         pd.Series(0.0, index=four_days),
-        completeness
-    )
-
-
-def test_completeness_score_incomplete_index():
-    """A series with one data point per hour has 25% completeness at
-    15-minute sample frequency"""
-    data = pd.Series(
-        1,
-        index=pd.date_range(start='01/01/2020', freq='1h', periods=72),
-    )
-    completeness = gaps.completeness_score(data, freq='15min',
-                                           keep_index=False)
-    assert_series_equal(
-        pd.Series(
-            0.25,
-            index=pd.date_range(start='01/01/2020', freq='D', periods=3)
-        ),
-        completeness
+        completeness,
+        check_freq=False
     )
 
 
@@ -538,19 +520,6 @@ def test_completeness_score_complete():
     )
 
 
-def test_completeness_score_freq_too_high():
-    """If the infered freq is shorter than the passed freq an exception is
-    raised."""
-    data = pd.Series(
-        1,
-        index=pd.date_range(start='1/1/2020', freq='15min', periods=24*4*4)
-    )
-    with pytest.raises(ValueError):
-        gaps.completeness_score(data, freq='16min')
-    with pytest.raises(ValueError):
-        gaps.completeness_score(data, freq='1h')
-
-
 def test_completeness_score_reindex():
     """Every timestamp is marked with completeness for the day when
     keep_index=True"""
@@ -586,7 +555,7 @@ def test_complete_threshold_zero():
     data.dropna()
     assert_series_equal(
         pd.Series(True, index=data.index),
-        gaps.complete(data, minimum_completeness=0, freq='15min')
+        gaps.complete(data, minimum_completeness=0)
     )
     data = pd.Series(1.0, index=ten_days)
     assert_series_equal(
@@ -627,7 +596,7 @@ def test_complete_threshold_one():
     )
     assert_series_equal(
         gaps.complete(data, minimum_completeness=1.0),
-        gaps.complete(data, minimum_completeness=1.0, freq='15min')
+        gaps.complete(data, minimum_completeness=1.0)
     )