Fix: Improve Relative Date Categorical Expressions (#1523)

* fix: improve relative expressions

* fix: improve relative expressions

Author: eakmanrq

setup.py

@@ -67,6 +67,7 @@
             "dbt-core",
             "dbt-duckdb>=1.4.2",
             "Faker",
+            "freezegun",
             "google-auth",
             "isort==5.10.1",
             "mkdocs-include-markdown-plugin==4.0.3",

sqlmesh/utils/date.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import re
 import time
 import typing as t
 import warnings
@@ -11,6 +12,8 @@
 from datetime import date, datetime, timedelta, timezone
 
 import dateparser
+from dateparser import freshness_date_parser as freshness_date_parser_module
+from dateparser.freshness_date_parser import freshness_date_parser
 from sqlglot import exp
 
 UTC = timezone.utc
@@ -22,6 +25,14 @@
     from sqlmesh.core.scheduler import Interval
 
 
+# The Freshness Date Data Parser doesn't support plural units so we add the `s?` to the expression
+freshness_date_parser_module.PATTERN = re.compile(
+    r"(\d+[.,]?\d*)\s*(%s)s?\b" % freshness_date_parser_module._UNITS, re.I | re.S | re.U  # type: ignore
+)
+DAY_SHORTCUT_EXPRESSIONS = {"today", "yesterday", "tomorrow"}
+TIME_UNITS = {"hours", "minutes", "seconds"}
+
+
 def now(minute_floor: bool = True) -> datetime:
     """
     Current utc datetime with optional minute level accuracy / granularity.
@@ -134,7 +145,11 @@ def to_datetime(value: TimeLike, relative_base: t.Optional[datetime] = None) ->
             epoch = None
 
         if epoch is None:
-            dt = dateparser.parse(str(value), settings={"RELATIVE_BASE": relative_base or now()})
+            relative_base = relative_base or now()
+            expression = str(value)
+            if is_catagorical_relative_expression(expression):
+                relative_base = relative_base.replace(hour=0, minute=0, second=0, microsecond=0)
+            dt = dateparser.parse(expression, settings={"RELATIVE_BASE": relative_base})
         else:
             try:
                 dt = datetime.strptime(str(value), DATE_INT_FMT)
@@ -279,3 +294,12 @@ def time_like_to_str(time_like: TimeLike) -> str:
     if is_date(time_like):
         return to_ds(time_like)
     return to_ts(time_like)
+
+
+def is_catagorical_relative_expression(expression: str) -> bool:
+    if expression.strip().lower() in DAY_SHORTCUT_EXPRESSIONS:
+        return True
+    grain_kwargs = freshness_date_parser.get_kwargs(expression)
+    if not grain_kwargs:
+        return False
+    return not any(k in TIME_UNITS for k in grain_kwargs)

tests/core/test_integration.py

@@ -698,6 +698,7 @@ def test_incremental_time_self_reference(
             SnapshotIntervals(
                 snapshot_name="sushi.customer_revenue_lifetime",
                 intervals=[
+                    (to_timestamp(to_date("7 days ago")), to_timestamp(to_date("6 days ago"))),
                     (to_timestamp(to_date("6 days ago")), to_timestamp(to_date("5 days ago"))),
                     (to_timestamp(to_date("5 days ago")), to_timestamp(to_date("4 days ago"))),
                     (to_timestamp(to_date("4 days ago")), to_timestamp(to_date("3 days ago"))),
@@ -709,6 +710,7 @@ def test_incremental_time_self_reference(
             SnapshotIntervals(
                 snapshot_name="sushi.customer_revenue_by_day",
                 intervals=[
+                    (to_timestamp(to_date("7 days ago")), to_timestamp(to_date("6 days ago"))),
                     (to_timestamp(to_date("6 days ago")), to_timestamp(to_date("5 days ago"))),
                 ],
             ),
@@ -720,9 +722,9 @@ def test_incremental_time_self_reference(
     num_batch_calls = Counter(
         [x[0][0] for x in sushi_context.console.update_snapshot_evaluation_progress.call_args_list]  # type: ignore
     )
-    # Validate that we made 6 calls to the customer_revenue_lifetime snapshot and 1 call to the customer_revenue_by_day snapshot
+    # Validate that we made 7 calls to the customer_revenue_lifetime snapshot and 1 call to the customer_revenue_by_day snapshot
     assert num_batch_calls == {
-        sushi_context.snapshots["sushi.customer_revenue_lifetime"]: 6,
+        sushi_context.snapshots["sushi.customer_revenue_lifetime"]: 7,
         sushi_context.snapshots["sushi.customer_revenue_by_day"]: 1,
     }
     # Validate that the results are the same as before the restate

tests/utils/test_date.py

@@ -1,8 +1,15 @@
 from datetime import date, datetime
 
 import pytest
+from freezegun import freeze_time
 
-from sqlmesh.utils.date import UTC, make_inclusive, to_datetime, to_timestamp
+from sqlmesh.utils.date import (
+    UTC,
+    is_catagorical_relative_expression,
+    make_inclusive,
+    to_datetime,
+    to_timestamp,
+)
 
 
 def test_to_datetime() -> None:
@@ -21,6 +28,31 @@ def test_to_datetime() -> None:
     assert to_datetime("0") == datetime(1970, 1, 1, tzinfo=UTC)
 
 
+@pytest.mark.parametrize(
+    "expression, result",
+    [
+        ("1 second ago", datetime(2023, 1, 20, 12, 29, 59, tzinfo=UTC)),
+        ("1 minute ago", datetime(2023, 1, 20, 12, 29, 00, tzinfo=UTC)),
+        ("1 hour ago", datetime(2023, 1, 20, 11, 30, 00, tzinfo=UTC)),
+        ("1 day ago", datetime(2023, 1, 19, 00, 00, 00, tzinfo=UTC)),
+        ("1 week ago", datetime(2023, 1, 13, 00, 00, 00, tzinfo=UTC)),
+        ("1 month ago", datetime(2022, 12, 20, 00, 00, 00, tzinfo=UTC)),
+        ("1 year ago", datetime(2022, 1, 20, 00, 00, 00, tzinfo=UTC)),
+        ("1 decade ago", datetime(2013, 1, 20, 00, 00, 00, tzinfo=UTC)),
+        ("3 days 2 hours ago", datetime(2023, 1, 17, 10, 30, 00, tzinfo=UTC)),
+        ("2 years 5 second ago", datetime(2021, 1, 20, 12, 29, 55, tzinfo=UTC)),
+        ("24 hours ago", datetime(2023, 1, 19, 12, 30, 00, tzinfo=UTC)),
+        ("1 year 5 days ago", datetime(2022, 1, 15, 00, 00, 00, tzinfo=UTC)),
+        ("yesterday", datetime(2023, 1, 19, 00, 00, 00, tzinfo=UTC)),
+        ("today", datetime(2023, 1, 20, 00, 00, 00, tzinfo=UTC)),
+        ("tomorrow", datetime(2023, 1, 21, 00, 00, 00, tzinfo=UTC)),
+    ],
+)
+def test_to_datetime_with_expressions(expression, result) -> None:
+    with freeze_time("2023-01-20 12:30:30"):
+        assert to_datetime(expression) == result
+
+
 def test_to_timestamp() -> None:
     assert to_timestamp("2020-01-01") == 1577836800000
 
@@ -55,3 +87,29 @@ def test_make_inclusive(start_in, end_in, start_out, end_out) -> None:
         to_datetime(start_out),
         to_datetime(end_out),
     )
+
+
+@pytest.mark.parametrize(
+    "expression, result",
+    [
+        ("1 second ago", False),
+        ("1 minute ago", False),
+        ("1 hour ago", False),
+        ("1 day ago", True),
+        ("1 week ago", True),
+        ("1 month ago", True),
+        ("1 year ago", True),
+        ("1 decade ago", True),
+        ("3 hours ago", False),
+        ("24 hours ago", False),
+        ("1 day 5 hours ago", False),
+        ("1 year 5 minutes ago", False),
+        ("2023-01-01", False),
+        ("2023-01-01 12:00:00", False),
+        ("yesterday", True),
+        ("today", True),
+        ("tomorrow", True),
+    ],
+)
+def test_is_catagorical_relative_expression(expression, result):
+    assert is_catagorical_relative_expression(expression) == result