feat: Date range preset based filtering for execution / file_execution (#1173)

* refactor: Moved file centric view and serializer to file execution app

* feat: Date range preset based filtering and other fields support for exec and file_exec
Refactored date time processing functions to utils/

* feat: Filtering support by file name for file execution, order by execution time

* feat: Filter changes for file execution status to support multi choice
Execution log filter support added for log level, file execution ID

* fix: Addressed review comments, fixed minor issue with date range preset for today, yesterday. Added preset for last_2_days

* feat: Data migration to update total_files column of WorkflowExecution (#1181)

* feat: Data migration to update total_files column of WorkflowExecution

* feat: Removed redundant execution statuses (#1185)

* feat: Modified execution log to use foreign key for execution (#1187)

* feat: Data migration to update total_files column of WorkflowExecution

* feat: Removed redundant execution statuses

* feat: Modified execution_id to be a foreign key in ExecutionLog table

* minor: Refactored schema migrations under 1 file
Handled data migration of execution_log for last 30 days
Added schema migration to remove the unused field

* feat: Avoided data migration on execution logs, backward compatible changes for viewing logs

* minor: Removed minor change on an old migration

* minor: Log message updated

---------

Signed-off-by: Chandrasekharan M <117059509+chandrasekharan-zipstack@users.noreply.github.com>
Co-authored-by: Rahul Johny <116638720+johnyrahul@users.noreply.github.com>

Author: chandrasekharan-zipstack

backend/usage_v2/dto.py

@@ -3,7 +3,7 @@
 from django.db.models import Q, QuerySet
 from django_filters import rest_framework as filters
 from usage_v2.models import Usage, UsageType
-from usage_v2.utils import DateTimeProcessor
+from utils.date import DateTimeProcessor
 from workflow_manager.file_execution.models import WorkflowFileExecution
 from workflow_manager.workflow_v2.models.execution import WorkflowExecution
 

backend/usage_v2/enums.py

@@ -9,7 +9,7 @@
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.response import Response
 from usage_v2.filter import UsageFilter
-from usage_v2.utils import DateTimeProcessor
+from utils.date import DateTimeProcessor
 from utils.pagination import CustomPagination
 from utils.user_context import UserContext
 

backend/usage_v2/filter.py

@@ -1,2 +1,5 @@
 from .constants import DateRangeKeys  # noqa: F401
+from .enums import DateRangePresets  # noqa: F401
+from .exceptions import InvalidDateRange, InvalidDatetime  # noqa: F401
+from .processor import DateRange, DateTimeProcessor  # noqa: F401
 from .serializer import DateRangeSerializer  # noqa: F401

backend/usage_v2/views.py

@@ -0,0 +1,58 @@
+from datetime import datetime, timedelta
+from enum import Enum
+from typing import Optional
+
+from django.utils import timezone
+from utils.date.exceptions import InvalidDateRange
+
+
+class DateRangePresets(Enum):
+    """Represents relative time presets.
+
+    Can be used for filtering entities
+    """
+
+    TODAY = ("today", 0, "Today")
+    YESTERDAY = ("yesterday", 1, "Yesterday")
+    LAST_2_DAYS = ("last_2_days", 2, "Last 2 Days")
+    LAST_7_DAYS = ("last_7_days", 7, "Last 7 Days")
+    LAST_30_DAYS = ("last_30_days", 30, "Last 30 Days")
+
+    def __init__(self, key: str, days: int, display_name: str):
+        self.key = key
+        self.days = days
+        self.display_name = display_name
+
+    def get_start_date(self) -> datetime:
+        now = timezone.now()
+        if self == DateRangePresets.TODAY:
+            return now.replace(hour=0, minute=0, second=0, microsecond=0)
+        elif self == DateRangePresets.YESTERDAY:
+            return (now - timedelta(days=1)).replace(
+                hour=0, minute=0, second=0, microsecond=0
+            )
+        return now - timedelta(days=self.days)
+
+    def get_end_date(self) -> datetime:
+        now = timezone.now()
+        if self == DateRangePresets.YESTERDAY:
+            return now.replace(hour=0, minute=0, second=0, microsecond=0)
+        return now
+
+    def get_date_range(self) -> tuple[datetime, datetime]:
+        return self.get_start_date(), self.get_end_date()
+
+    @classmethod
+    def from_value(cls, value: str) -> Optional["DateRangePresets"]:
+        try:
+            return next(preset for preset in cls if preset.key == value)
+        except StopIteration as e:
+            valid_values = [preset.key for preset in cls]
+            raise InvalidDateRange(
+                f"Invalid date range value: '{value}'. "
+                f"Valid values are: {', '.join(valid_values)}"
+            ) from e
+
+    @classmethod
+    def choices(cls) -> list[tuple[str, str]]:
+        return [(preset.key, preset.display_name) for preset in cls]

backend/utils/date/__init__.py

@@ -1,11 +1,11 @@
 from rest_framework.exceptions import APIException
 
 
-class InvalidDatetime(APIException):
+class InvalidDateRange(APIException):
     status_code = 400
-    default_detail = "Invalid datetime format"
+    default_detail = "Invalid date range"
 
 
-class InvalidDateRange(APIException):
+class InvalidDatetime(APIException):
     status_code = 400
-    default_detail = "Invalid date range"
+    default_detail = "Invalid datetime format"

backend/utils/date/enums.py

@@ -1,17 +1,31 @@
 import logging
+from dataclasses import dataclass
 from datetime import datetime, timedelta
 from typing import Optional, Union
 
 from dateutil.parser import parse
 from django.utils import timezone
 from isodate import parse_datetime
-from usage_v2.dto import DateRange
-from usage_v2.enums import DateRangePresets
-from usage_v2.exceptions import InvalidDatetime
+from utils.date.enums import DateRangePresets
+from utils.date.exceptions import InvalidDatetime
 
 logger = logging.getLogger(__name__)
 
 
+@dataclass
+class DateRange:
+    """
+    Represents a validated date range with start and end dates.
+
+    Attributes:
+        start_date: Beginning of the date range
+        end_date: End of the date range
+    """
+
+    start_date: datetime
+    end_date: datetime
+
+
 class DateTimeProcessor:
     DEFAULT_DAYS_RANGE = 1
     MAX_DAYS_RANGE = 60

backend/usage_v2/exceptions.py backend/utils/date/exceptions.pybackend/usage_v2/exceptions.py renamed to backend/utils/date/exceptions.py

@@ -0,0 +1,73 @@
+from api_v2.models import APIDeployment
+from django.db.models import QuerySet
+from django_filters import rest_framework as filters
+from pipeline_v2.models import Pipeline
+from utils.date import DateRangePresets, DateTimeProcessor
+from workflow_manager.execution.enum import ExecutionEntity
+from workflow_manager.workflow_v2.enums import ExecutionStatus
+from workflow_manager.workflow_v2.models import Workflow, WorkflowExecution
+
+
+class ExecutionFilter(filters.FilterSet):
+    execution_entity = filters.ChoiceFilter(
+        choices=[
+            (ExecutionEntity.API.value, "API"),
+            (ExecutionEntity.ETL.value, "ETL"),
+            (ExecutionEntity.TASK.value, "TASK"),
+            (ExecutionEntity.WORKFLOW.value, "WF"),
+        ],
+        method="filter_execution_entity",
+    )
+    # Lookup with query params: created_at_after, created_at_before
+    created_at = filters.DateTimeFromToRangeFilter()
+    status = filters.MultipleChoiceFilter(
+        field_name="status", choices=ExecutionStatus.choices
+    )
+    date_range = filters.ChoiceFilter(
+        choices=DateRangePresets.choices(),
+        method="filter_by_date_range",
+    )
+
+    class Meta:
+        model = WorkflowExecution
+        fields = []
+
+    def filter_execution_entity(
+        self, queryset: QuerySet, name: str, value: str
+    ) -> QuerySet:
+        if value == ExecutionEntity.API.value:
+            return queryset.filter(
+                pipeline_id__in=APIDeployment.objects.values_list("id", flat=True)
+            )
+        elif value == ExecutionEntity.ETL.value:
+            return queryset.filter(
+                pipeline_id__in=Pipeline.objects.filter(
+                    pipeline_type=Pipeline.PipelineType.ETL
+                ).values_list("id", flat=True)
+            )
+        elif value == ExecutionEntity.TASK.value:
+            return queryset.filter(
+                pipeline_id__in=Pipeline.objects.filter(
+                    pipeline_type=Pipeline.PipelineType.TASK
+                ).values_list("id", flat=True)
+            )
+        elif value == ExecutionEntity.WORKFLOW.value:
+            return queryset.filter(
+                pipeline_id=None,
+                workflow_id__in=Workflow.objects.values_list("id", flat=True),
+            )
+        return queryset
+
+    def filter_by_date_range(
+        self, queryset: QuerySet, name: str, value: str
+    ) -> QuerySet:
+        """
+        Filters Usages based on the provided date range.
+        """
+        date_span = DateTimeProcessor.filter_date_range(value)
+        if date_span:
+            queryset = queryset.filter(
+                created_at__gte=date_span.start_date,
+                created_at__lte=date_span.end_date,
+            )
+        return queryset

backend/usage_v2/utils.py backend/utils/date/processor.pybackend/usage_v2/utils.py renamed to backend/utils/date/processor.py

@@ -1,17 +1,13 @@
 import logging
-from typing import Optional
 
-from api_v2.models import APIDeployment
-from django.db.models import Q, QuerySet
-from pipeline_v2.models import Pipeline
+from django_filters.rest_framework import DjangoFilterBackend
 from rest_framework import viewsets
 from rest_framework.filters import OrderingFilter
 from rest_framework.permissions import IsAuthenticated
-from utils.date import DateRangeKeys, DateRangeSerializer
 from utils.pagination import CustomPagination
-from workflow_manager.execution.enum import ExecutionEntity
+from workflow_manager.execution.filter import ExecutionFilter
 from workflow_manager.execution.serializer import ExecutionSerializer
-from workflow_manager.workflow_v2.models import Workflow, WorkflowExecution
+from workflow_manager.workflow_v2.models import WorkflowExecution
 
 logger = logging.getLogger(__name__)
 
@@ -20,50 +16,8 @@ class ExecutionViewSet(viewsets.ReadOnlyModelViewSet):
     permission_classes = [IsAuthenticated]
     serializer_class = ExecutionSerializer
     pagination_class = CustomPagination
-    filter_backends = [OrderingFilter]
-    ordering_fields = ["created_at"]
+    filter_backends = [DjangoFilterBackend, OrderingFilter]
+    ordering_fields = ["created_at", "execution_time"]
     ordering = ["-created_at"]
-
-    def get_queryset(self) -> Optional[QuerySet]:
-        execution_entity = self.request.query_params.get("execution_entity")
-
-        queryset = WorkflowExecution.objects.all()
-
-        # Filter based on execution entity
-        if execution_entity == ExecutionEntity.API.value:
-            queryset = queryset.filter(
-                pipeline_id__in=APIDeployment.objects.values_list("id", flat=True)
-            )
-        elif execution_entity == ExecutionEntity.ETL.value:
-            queryset = queryset.filter(
-                pipeline_id__in=Pipeline.objects.filter(
-                    pipeline_type=Pipeline.PipelineType.ETL
-                ).values_list("id", flat=True)
-            )
-        elif execution_entity == ExecutionEntity.TASK.value:
-            queryset = queryset.filter(
-                pipeline_id__in=Pipeline.objects.filter(
-                    pipeline_type=Pipeline.PipelineType.TASK
-                ).values_list("id", flat=True)
-            )
-        elif execution_entity == ExecutionEntity.WORKFLOW.value:
-            queryset = queryset.filter(
-                pipeline_id=None,
-                workflow_id__in=Workflow.objects.values_list("id", flat=True),
-            )
-
-        # Parse and apply date filters
-        date_range_serializer = DateRangeSerializer(data=self.request.query_params)
-        date_range_serializer.is_valid(raise_exception=True)
-
-        filters = Q()
-        if start_date := date_range_serializer.validated_data.get(
-            DateRangeKeys.START_DATE
-        ):
-            filters &= Q(created_at__gte=start_date)
-        if end_date := date_range_serializer.validated_data.get(DateRangeKeys.END_DATE):
-            filters &= Q(created_at__lte=end_date)
-
-        queryset = queryset.filter(filters)
-
-        return queryset
+    filterset_class = ExecutionFilter
+    queryset = WorkflowExecution.objects.all()