feat: make max_done_report_age_hours configurable via user config

Add max_done_report_age_hours to connector spec (integer, min=0, max=24, default=0).
When 0 (default), DONE reports are never reused — new reports are always created.
When set to a positive value, DONE reports younger than that threshold are reused.
IN_PROGRESS/IN_QUEUE/CANCELLED/FATAL reports are unaffected by this setting.

Updated 48 unit tests covering all configurable threshold scenarios.

Co-Authored-By: bot_apk <apk@cognition.ai>

Author: devin-ai-integration[bot]

airbyte-integrations/connectors/source-amazon-seller-partner/components.py

@@ -569,10 +569,11 @@ class ReportCreationRequester(HttpRequester):
     1. Before creating a new report via POST, call GET /reports to check for existing reports
        of the same reportType, matching date range, and marketplaceIds.
     2. If matching reports are found, select the most recently created one (by createdTime).
-       DONE reports older than 1 day are skipped since their data snapshot may be stale and
-       the report document may have expired. Status filtering (e.g. CANCELLED, FATAL) is NOT
-       done here — the manifest's status_mapping is the single source of truth for which
-       statuses are retryable, skippable, or terminal.
+       DONE reports older than `max_done_report_age_hours` (from config, default 0) are skipped
+       since their data snapshot may be stale and the report document may have expired. When the
+       config value is 0 (default), DONE reports are never reused. Status filtering (e.g.
+       CANCELLED, FATAL) is NOT done here — the manifest's status_mapping is the single source
+       of truth for which statuses are retryable, skippable, or terminal.
     3. If no suitable report is found, fall through to super().send_request() to create a new one.
     """
 
@@ -682,7 +683,8 @@ def _find_existing_report(
         # DONE reports older than this threshold are considered stale (data snapshot may be outdated,
         # report document may have expired). IN_QUEUE and IN_PROGRESS reports are not subject to
         # this check because they are still being processed.
-        max_done_report_age_hours = 24
+        # When max_done_report_age_hours is 0 (default), DONE reports are never reused.
+        max_done_report_age_hours = self.config.get("max_done_report_age_hours", 0)
         now = ab_datetime_now()
 
         # Collect all matching candidates, then pick the most recently created one
@@ -699,22 +701,31 @@ def _find_existing_report(
             if not self._date_ranges_match(requested_start, requested_end, report_start, report_end):
                 continue
 
-            # Check createdTime freshness for DONE reports
+            # Check createdTime freshness for DONE reports.
+            # When max_done_report_age_hours is 0 (default), DONE reports are never reused.
             report_status = report.get("processingStatus", "")
             created_time_str = report.get("createdTime", "")
-            if report_status == "DONE" and created_time_str:
-                try:
-                    created_time = ab_datetime_parse(created_time_str)
-                    age_hours = (now - created_time).total_seconds() / 3600
-                    if age_hours > max_done_report_age_hours:
-                        report_id = report.get("reportId", "")
-                        logger.info(
-                            f"Skipping stale DONE report {report_id} (created {age_hours:.1f}h ago) "
-                            f"for {report_type}. Will look for a newer one."
-                        )
-                        continue
-                except (ValueError, TypeError):
-                    pass  # If we can't parse createdTime, don't skip — still usable
+            if report_status == "DONE":
+                if max_done_report_age_hours == 0:
+                    report_id = report.get("reportId", "")
+                    logger.info(
+                        f"Skipping DONE report {report_id} for {report_type} "
+                        f"because max_done_report_age_hours is 0 (always create new reports)."
+                    )
+                    continue
+                if created_time_str:
+                    try:
+                        created_time = ab_datetime_parse(created_time_str)
+                        age_hours = (now - created_time).total_seconds() / 3600
+                        if age_hours > max_done_report_age_hours:
+                            report_id = report.get("reportId", "")
+                            logger.info(
+                                f"Skipping stale DONE report {report_id} (created {age_hours:.1f}h ago) "
+                                f"for {report_type}. Will look for a newer one."
+                            )
+                            continue
+                    except (ValueError, TypeError):
+                        pass  # If we can't parse createdTime, don't skip — still usable
 
             # Parse createdTime to compare candidates
             report_created_time = None

airbyte-integrations/connectors/source-amazon-seller-partner/manifest.yaml

@@ -242,6 +242,18 @@ spec:
         default: false
         order: 13
         airbyte_hidden: true
+      max_done_report_age_hours:
+        title: "Max Age of Completed Reports to Reuse (Hours)"
+        type: integer
+        description: "When the connector finds an existing completed (DONE) report
+          matching the same date range and marketplace, it can reuse that report
+          instead of creating a new one. This setting controls how old (in hours)
+          a completed report can be and still be reused. Set to 0 to always create
+          new reports. Maximum is 24 hours."
+        default: 0
+        minimum: 0
+        maximum: 24
+        order: 14
       financial_events_step:
         title: Financial Events Step Size
         description: 'The time window size for fetching financial events data in chunks

airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/config.py

@@ -48,5 +48,9 @@ def with_asin_granularity(self, granularity: str) -> ConfigBuilder:
         self._config["sales_and_traffic_report_asin_granularity"] = granularity
         return self
 
+    def with_max_done_report_age_hours(self, hours: int) -> ConfigBuilder:
+        self._config["max_done_report_age_hours"] = hours
+        return self
+
     def build(self) -> Dict[str, str]:
         return self._config

airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_report_creation_requester.py

@@ -46,10 +46,13 @@ def _make_report(
     return report
 
 
-def _make_requester() -> ReportCreationRequester:
+def _make_requester(config: dict = None) -> ReportCreationRequester:
     """Create a ReportCreationRequester instance with mocked internals for unit testing."""
     requester = object.__new__(ReportCreationRequester)
 
+    # Set config — defaults to empty dict (max_done_report_age_hours defaults to 0 in component)
+    requester.config = config or {}
+
     # Mock _request_body_json to return a controlled body
     requester._request_body_json = MagicMock()
 
@@ -270,7 +273,7 @@ class TestFindExistingReport:
 
     def test_returns_matching_report(self):
         requester = _make_requester()
-        matching_report = _make_report(report_id="rpt-match", marketplace_ids=["ATVPDKIKX0DER"])
+        matching_report = _make_report(report_id="rpt-match", status="IN_PROGRESS", marketplace_ids=["ATVPDKIKX0DER"])
         get_response = _make_get_reports_response([matching_report])
         requester._http_client.send_request.return_value = (None, get_response)
 
@@ -476,9 +479,28 @@ def test_reuses_in_queue_report(self):
         assert result is not None
         assert result.json()["reportId"] == "rpt-in-queue"
 
+    def test_skips_done_report_when_max_age_is_zero(self):
+        """When max_done_report_age_hours is 0 (default), DONE reports should never be reused."""
+        requester = _make_requester()  # default config, max_done_report_age_hours=0
+        now = datetime.now(tz=timezone.utc)
+        report = _make_report(report_id="rpt-done", status="DONE", created_time=now.isoformat())
+        get_response = _make_get_reports_response([report])
+        requester._http_client.send_request.return_value = (None, get_response)
+
+        result = requester._find_existing_report(
+            stream_state=None,
+            stream_slice=None,
+            report_type="GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL",
+            requested_start="2023-01-01T00:00:00Z",
+            requested_end="2023-01-30T00:00:00Z",
+            requested_marketplace_ids=["ATVPDKIKX0DER"],
+        )
+
+        assert result is None
+
     def test_skips_stale_done_report(self):
-        """DONE reports older than 24h should be skipped."""
-        requester = _make_requester()
+        """DONE reports older than max_done_report_age_hours should be skipped."""
+        requester = _make_requester(config={"max_done_report_age_hours": 24})
         # Report created 48 hours ago
         old_time = datetime(2023, 1, 1, 0, 0, 0, tzinfo=timezone.utc).isoformat()
         report = _make_report(report_id="rpt-stale", status="DONE", created_time=old_time)
@@ -496,9 +518,9 @@ def test_skips_stale_done_report(self):
 
         assert result is None
 
-    def test_reuses_fresh_done_report(self):
-        """DONE reports created within 24h should be reusable."""
-        requester = _make_requester()
+    def test_reuses_fresh_done_report_when_max_age_set(self):
+        """DONE reports created within max_done_report_age_hours should be reusable."""
+        requester = _make_requester(config={"max_done_report_age_hours": 24})
         # Use a very recent timestamp
         now = datetime.now(tz=timezone.utc)
         fresh_time = now.isoformat()
@@ -519,8 +541,8 @@ def test_reuses_fresh_done_report(self):
         assert result.json()["reportId"] == "rpt-fresh"
 
     def test_in_progress_report_not_subject_to_staleness(self):
-        """IN_PROGRESS reports should not be subject to the 24h staleness check."""
-        requester = _make_requester()
+        """IN_PROGRESS reports should not be subject to the staleness check even when max_age is 0."""
+        requester = _make_requester()  # default config, max_done_report_age_hours=0
         old_time = datetime(2023, 1, 1, 0, 0, 0, tzinfo=timezone.utc).isoformat()
         report = _make_report(report_id="rpt-old-ip", status="IN_PROGRESS", created_time=old_time)
         get_response = _make_get_reports_response([report])
@@ -540,7 +562,7 @@ def test_in_progress_report_not_subject_to_staleness(self):
 
     def test_returns_latest_report_by_created_time(self):
         """When multiple matching reports exist, return the most recently created one."""
-        requester = _make_requester()
+        requester = _make_requester(config={"max_done_report_age_hours": 24})
         now = datetime.now(tz=timezone.utc)
         older_time = (now - timedelta(hours=2)).isoformat()
         newer_time = (now - timedelta(hours=1)).isoformat()
@@ -565,7 +587,7 @@ def test_returns_latest_report_by_created_time(self):
 
     def test_returns_latest_in_progress_over_older_done(self):
         """A newer IN_PROGRESS report should be preferred over an older DONE report."""
-        requester = _make_requester()
+        requester = _make_requester(config={"max_done_report_age_hours": 24})
         now = datetime.now(tz=timezone.utc)
         done_time = (now - timedelta(hours=10)).isoformat()
         ip_time = (now - timedelta(hours=1)).isoformat()
@@ -587,9 +609,9 @@ def test_returns_latest_in_progress_over_older_done(self):
         assert result is not None
         assert result.json()["reportId"] == "rpt-ip"
 
-    def test_done_report_without_created_time_is_still_usable(self):
-        """DONE reports without createdTime should still be reusable (no staleness check)."""
-        requester = _make_requester()
+    def test_done_report_without_created_time_is_still_usable_when_max_age_set(self):
+        """DONE reports without createdTime should still be reusable when max_age > 0."""
+        requester = _make_requester(config={"max_done_report_age_hours": 24})
         report = _make_report(report_id="rpt-no-time", status="DONE")
         get_response = _make_get_reports_response([report])
         requester._http_client.send_request.return_value = (None, get_response)
@@ -606,6 +628,96 @@ def test_done_report_without_created_time_is_still_usable(self):
         assert result is not None
         assert result.json()["reportId"] == "rpt-no-time"
 
+    def test_done_report_without_created_time_skipped_when_max_age_is_zero(self):
+        """DONE reports without createdTime should be skipped when max_age is 0."""
+        requester = _make_requester()  # default config, max_done_report_age_hours=0
+        report = _make_report(report_id="rpt-no-time", status="DONE")
+        get_response = _make_get_reports_response([report])
+        requester._http_client.send_request.return_value = (None, get_response)
+
+        result = requester._find_existing_report(
+            stream_state=None,
+            stream_slice=None,
+            report_type="GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL",
+            requested_start="2023-01-01T00:00:00Z",
+            requested_end="2023-01-30T00:00:00Z",
+            requested_marketplace_ids=["ATVPDKIKX0DER"],
+        )
+
+        assert result is None
+
+    def test_reuses_done_report_with_custom_max_age(self):
+        """DONE report created 3h ago should be reused when max_done_report_age_hours=6."""
+        requester = _make_requester(config={"max_done_report_age_hours": 6})
+        now = datetime.now(tz=timezone.utc)
+        report = _make_report(
+            report_id="rpt-3h",
+            status="DONE",
+            created_time=(now - timedelta(hours=3)).isoformat(),
+        )
+        get_response = _make_get_reports_response([report])
+        requester._http_client.send_request.return_value = (None, get_response)
+
+        result = requester._find_existing_report(
+            stream_state=None,
+            stream_slice=None,
+            report_type="GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL",
+            requested_start="2023-01-01T00:00:00Z",
+            requested_end="2023-01-30T00:00:00Z",
+            requested_marketplace_ids=["ATVPDKIKX0DER"],
+        )
+
+        assert result is not None
+        assert result.json()["reportId"] == "rpt-3h"
+
+    def test_skips_done_report_exceeding_custom_max_age(self):
+        """DONE report created 10h ago should be skipped when max_done_report_age_hours=6."""
+        requester = _make_requester(config={"max_done_report_age_hours": 6})
+        now = datetime.now(tz=timezone.utc)
+        report = _make_report(
+            report_id="rpt-10h",
+            status="DONE",
+            created_time=(now - timedelta(hours=10)).isoformat(),
+        )
+        get_response = _make_get_reports_response([report])
+        requester._http_client.send_request.return_value = (None, get_response)
+
+        result = requester._find_existing_report(
+            stream_state=None,
+            stream_slice=None,
+            report_type="GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL",
+            requested_start="2023-01-01T00:00:00Z",
+            requested_end="2023-01-30T00:00:00Z",
+            requested_marketplace_ids=["ATVPDKIKX0DER"],
+        )
+
+        assert result is None
+
+    def test_default_config_skips_done_but_reuses_in_progress(self):
+        """With default config (max_age=0), DONE reports are skipped but IN_PROGRESS is reused."""
+        requester = _make_requester()  # default config, max_done_report_age_hours=0
+        now = datetime.now(tz=timezone.utc)
+        done_report = _make_report(report_id="rpt-done", status="DONE", created_time=now.isoformat())
+        ip_report = _make_report(
+            report_id="rpt-ip",
+            status="IN_PROGRESS",
+            created_time=(now - timedelta(hours=1)).isoformat(),
+        )
+        get_response = _make_get_reports_response([done_report, ip_report])
+        requester._http_client.send_request.return_value = (None, get_response)
+
+        result = requester._find_existing_report(
+            stream_state=None,
+            stream_slice=None,
+            report_type="GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL",
+            requested_start="2023-01-01T00:00:00Z",
+            requested_end="2023-01-30T00:00:00Z",
+            requested_marketplace_ids=["ATVPDKIKX0DER"],
+        )
+
+        assert result is not None
+        assert result.json()["reportId"] == "rpt-ip"
+
 
 class TestSendRequest:
     """Tests for ReportCreationRequester.send_request integration."""
@@ -618,7 +730,7 @@ def test_reuses_existing_report_when_found(self):
             "dataEndTime": "2023-01-30T00:00:00Z",
             "marketplaceIds": ["ATVPDKIKX0DER"],
         }
-        matching_report = _make_report(report_id="rpt-existing")
+        matching_report = _make_report(report_id="rpt-existing", status="IN_PROGRESS")
         get_response = _make_get_reports_response([matching_report])
         requester._http_client.send_request.return_value = (None, get_response)