Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions src/data_profiling/model/pandas/describe_timeseries_pandas.py
Original file line number Diff line number Diff line change
Expand Up @@ -173,12 +173,30 @@ def compute_gap_stats(series: pd.Series) -> pd.Series:
A series with the gaps intervals.
"""

empty_stats = {
"min": 0,
"max": 0,
"mean": 0,
"std": 0,
"series": series,
"gaps": [],
"n_gaps": 0,
}

gap = series.dropna()
index_name = gap.index.name if gap.index.name else "index"
gap = gap.reset_index()[index_name]
gap.index.name = None

is_datetime = isinstance(series.index, pd.DatetimeIndex)
if not is_datetime and not pd.api.types.is_numeric_dtype(gap):
datetime_gap = pd.to_datetime(gap, errors="coerce")
if datetime_gap.notna().all():
gap = datetime_gap
is_datetime = True
else:
return empty_stats

gap_stats, gaps = identify_gaps(gap, is_datetime)
has_gaps = len(gap_stats) > 0

Expand Down
16 changes: 16 additions & 0 deletions tests/unit/test_time_series.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import pytest

from data_profiling import ProfileReport
from data_profiling.model.pandas.describe_timeseries_pandas import compute_gap_stats


@pytest.fixture
Expand Down Expand Up @@ -78,6 +79,21 @@ def test_timeseries_with_sortby(sample_ts_df):
assert profile.config.vars.timeseries.sortby == "date"


def test_timeseries_gap_stats_with_string_date_index():
# GH#1773
dates = pd.date_range("2023-01-01", periods=10, freq="D").tolist()
dates.append(pd.Timestamp("2023-01-20"))
series = pd.Series(
np.arange(11, dtype=float),
index=pd.Index([date.strftime("%Y-%m-%d") for date in dates], name="Date Local"),
)

gap_stats = compute_gap_stats(series)

assert gap_stats["n_gaps"] == 1
assert gap_stats["min"] == pd.Timedelta(days=10)


def test_timeseries_without_sortby(sample_ts_df):
# Test time series without explicit sort column
profile = ProfileReport(sample_ts_df, tsmode=True)
Expand Down