diff --git a/docs/examples/pvfleets-qa-pipeline/pvfleets-irradiance-qa.py b/docs/examples/pvfleets-qa-pipeline/pvfleets-irradiance-qa.py index 96610437..f4eb8e0a 100644 --- a/docs/examples/pvfleets-qa-pipeline/pvfleets-irradiance-qa.py +++ b/docs/examples/pvfleets-qa-pipeline/pvfleets-irradiance-qa.py @@ -167,8 +167,7 @@ # Trim the series based on daily completeness score trim_series = pvanalytics.quality.gaps.trim_incomplete( time_series, - minimum_completeness=.25, - freq=data_freq) + minimum_completeness=.25) first_valid_date, last_valid_date = \ pvanalytics.quality.gaps.start_stop_dates(trim_series) time_series = time_series[first_valid_date.tz_convert(time_series.index.tz): diff --git a/docs/examples/pvfleets-qa-pipeline/pvfleets-power-qa.py b/docs/examples/pvfleets-qa-pipeline/pvfleets-power-qa.py index 5f1fbec3..cf0bcaa8 100644 --- a/docs/examples/pvfleets-qa-pipeline/pvfleets-power-qa.py +++ b/docs/examples/pvfleets-qa-pipeline/pvfleets-power-qa.py @@ -163,7 +163,7 @@ # Trim the series based on daily completeness score trim_series = pvanalytics.quality.gaps.trim_incomplete( - time_series, minimum_completeness=.25, freq=data_freq) + time_series, minimum_completeness=.25) first_valid_date, last_valid_date = \ pvanalytics.quality.gaps.start_stop_dates(trim_series) time_series = time_series[first_valid_date.tz_convert(time_series.index.tz): diff --git a/docs/examples/pvfleets-qa-pipeline/pvfleets-temperature-qa.py b/docs/examples/pvfleets-qa-pipeline/pvfleets-temperature-qa.py index 4e9b59e0..841df795 100644 --- a/docs/examples/pvfleets-qa-pipeline/pvfleets-temperature-qa.py +++ b/docs/examples/pvfleets-qa-pipeline/pvfleets-temperature-qa.py @@ -178,8 +178,7 @@ # Trim the series based on daily completeness score trim_series = pvanalytics.quality.gaps.trim_incomplete( time_series, - minimum_completeness=.25, - freq=data_freq) + minimum_completeness=.25) first_valid_date, last_valid_date = \ pvanalytics.quality.gaps.start_stop_dates(trim_series) time_series = time_series[first_valid_date.tz_convert(time_series.index.tz): diff --git a/docs/whatsnew/v0.2.3.rst b/docs/whatsnew/v0.2.3.rst index af198872..216f9f59 100644 --- a/docs/whatsnew/v0.2.3.rst +++ b/docs/whatsnew/v0.2.3.rst @@ -12,6 +12,9 @@ Bug Fixes ~~~~~~~~~ * :py:func:`pvanalytics.features.clearsky.reno` now correctly passes ``window_length`` to the underlying pvlib function. (:pull:`221`) +* Remove freq parameter from :py:func:`pvanalytics.quality.gaps.completeness` + and :py:func:`pvanalytics.quality.gaps.completeness_score`. Frequency is now + always calculated from the input data's DatetimeIndex. (:pull:`236`) Requirements ~~~~~~~~~~~~ @@ -35,4 +38,4 @@ Testing Contributors ~~~~~~~~~~~~ - +* Cliff Hansen (:ghuser:`cwhanse`) diff --git a/pvanalytics/quality/gaps.py b/pvanalytics/quality/gaps.py index 642a2db5..2e16d83e 100644 --- a/pvanalytics/quality/gaps.py +++ b/pvanalytics/quality/gaps.py @@ -250,13 +250,11 @@ def _freq_to_seconds(freq): return delta.days * (1440 * 60) + delta.seconds -def completeness_score(series, freq=None, keep_index=True): +def completeness_score(series, keep_index=True): """Calculate a data completeness score for each day. The completeness score for a given day is the fraction of time in - the day for which there is data (a value other than NaN). The time - duration attributed to each value is equal to the timestamp - spacing of `series`, or `freq` if it is specified. For example, a + the day for which there is data (a value other than NaN). For example, a 24-hour time series with 30 minute timestamp spacing and 24 non-NaN values would have data for a total of 12 hours and therefore a completeness score of 0.5. @@ -265,10 +263,6 @@ def completeness_score(series, freq=None, keep_index=True): ---------- series : Series A DatetimeIndexed series. - freq : str, default None - Interval between samples in the series as a pandas frequency - string. If None, the frequency is inferred using - :py:func:`pandas.infer_freq`. keep_index : boolean, default True Whether or not the returned series has the same index as `series`. If False the returned series will be indexed by day. @@ -279,22 +273,8 @@ def completeness_score(series, freq=None, keep_index=True): A series of floats giving the completeness score for each day (fraction of the day for which `series` has data). - Raises - ------ - ValueError - If `freq` is longer than the frequency inferred from `series`. - """ - inferred_seconds = _freq_to_seconds(pd.infer_freq(series.index)) - if freq: - freq_seconds = _freq_to_seconds(freq) - seconds_per_sample = freq_seconds - else: - seconds_per_sample = inferred_seconds - - if freq and inferred_seconds < freq_seconds: - raise ValueError("freq must be less than or equal to the" - + " frequency of the series") + seconds_per_sample = series.index.diff().total_seconds()[1] daily_counts = series.resample('D').count() daily_completeness = (daily_counts * seconds_per_sample) / (1440*60) if keep_index: @@ -302,7 +282,7 @@ def completeness_score(series, freq=None, keep_index=True): return daily_completeness -def complete(series, minimum_completeness=0.333, freq=None): +def complete(series, minimum_completeness=0.333): """Select data points that are part of days with complete data. A day has complete data if its completeness score is greater than @@ -312,12 +292,9 @@ def complete(series, minimum_completeness=0.333, freq=None): Parameters ---------- series : Series - The data to be checked for completeness. + A DatetimeIndexed series to be checked for completeness. minimum_completeness : float, default 0.333 Fraction of the day that must have data. - freq : str, default None - The expected frequency of the data in `series`. If none then - the frequency is inferred from the data. Returns ------- @@ -335,7 +312,7 @@ def complete(series, minimum_completeness=0.333, freq=None): completeness_score """ - return completeness_score(series, freq=freq) >= minimum_completeness + return completeness_score(series) >= minimum_completeness def start_stop_dates(series, days=10): @@ -415,7 +392,7 @@ def trim(series, days=10): return mask -def trim_incomplete(series, minimum_completeness=0.333333, days=10, freq=None): +def trim_incomplete(series, minimum_completeness=0.333333, days=10): """Trim the series based on the completeness score. Combines :py:func:`completeness_score` and :py:func:`trim`. @@ -430,9 +407,6 @@ def trim_incomplete(series, minimum_completeness=0.333333, days=10, freq=None): The number of consecutive days with completeness greater than `minumum_completeness` for the 'good' data to start or end. See :py:func:`start_stop_dates` for more information. - freq : str, default None - The expected frequency of the series. See - :py:func:`completeness_score` fore more information. Returns ------- @@ -448,6 +422,6 @@ def trim_incomplete(series, minimum_completeness=0.333333, days=10, freq=None): completeness_score """ - completeness = completeness_score(series, freq=freq) + completeness = completeness_score(series) complete_days = completeness >= minimum_completeness return trim(complete_days, days=days) diff --git a/pvanalytics/tests/quality/test_gaps.py b/pvanalytics/tests/quality/test_gaps.py index 51cce2e9..6f360132 100644 --- a/pvanalytics/tests/quality/test_gaps.py +++ b/pvanalytics/tests/quality/test_gaps.py @@ -492,33 +492,15 @@ def test_completeness_score_all_nans(): def test_completeness_score_no_data(): """A data set with completely missing timestamps and NaNs has completeness 0.""" - four_days = pd.date_range(start='01/01/2020', freq='D', periods=4) - completeness = gaps.completeness_score( - pd.Series(index=four_days, dtype='float64'), - freq='15min', - keep_index=False - ) + four_days = pd.date_range(start='01/01/2020', end='01/04/2020', periods=4) + test_data = pd.Series(index=four_days, dtype='float64') + completeness = gaps.completeness_score(test_data, keep_index=False) + # have to exclude freq because completeness is returned with freq='D' + # due to resampling, but test_data is not constructed with freq assert_series_equal( pd.Series(0.0, index=four_days), - completeness - ) - - -def test_completeness_score_incomplete_index(): - """A series with one data point per hour has 25% completeness at - 15-minute sample frequency""" - data = pd.Series( - 1, - index=pd.date_range(start='01/01/2020', freq='1h', periods=72), - ) - completeness = gaps.completeness_score(data, freq='15min', - keep_index=False) - assert_series_equal( - pd.Series( - 0.25, - index=pd.date_range(start='01/01/2020', freq='D', periods=3) - ), - completeness + completeness, + check_freq=False ) @@ -538,19 +520,6 @@ def test_completeness_score_complete(): ) -def test_completeness_score_freq_too_high(): - """If the infered freq is shorter than the passed freq an exception is - raised.""" - data = pd.Series( - 1, - index=pd.date_range(start='1/1/2020', freq='15min', periods=24*4*4) - ) - with pytest.raises(ValueError): - gaps.completeness_score(data, freq='16min') - with pytest.raises(ValueError): - gaps.completeness_score(data, freq='1h') - - def test_completeness_score_reindex(): """Every timestamp is marked with completeness for the day when keep_index=True""" @@ -586,7 +555,7 @@ def test_complete_threshold_zero(): data.dropna() assert_series_equal( pd.Series(True, index=data.index), - gaps.complete(data, minimum_completeness=0, freq='15min') + gaps.complete(data, minimum_completeness=0) ) data = pd.Series(1.0, index=ten_days) assert_series_equal( @@ -627,7 +596,7 @@ def test_complete_threshold_one(): ) assert_series_equal( gaps.complete(data, minimum_completeness=1.0), - gaps.complete(data, minimum_completeness=1.0, freq='15min') + gaps.complete(data, minimum_completeness=1.0) )