fix: Prevent infinite loop in legacy edx course export op

The export_edx_courses op in the legacy IRx pipeline had two bugs that
caused the courses/ folder to silently go missing in the S3 data packages
(observed since ~2026-01-21 for mitx-etl-mitxonline-production):

1. Infinite loop in the status-polling while loop
   When check_course_export_status() raised httpx.HTTPStatusError (e.g.,
   HTTP 404 from django-user-tasks after record cleanup, or a deleted
   course), the bare 'continue' swallowed the error without adding the
   course to failed_exports. The while-loop condition
   len(successful U failed) < len(tasks) was never satisfied, so the
   job hung indefinitely and the courses/ folder was never written.

   Fix: catch HTTPStatusError, log the status code and course id, and
   add the course to failed_exports so the loop can terminate. Also add
   a 60-minute hard timeout (matching the openedx asset-based code) and
   skip courses already resolved to avoid redundant API calls.

2. Abort on partial-failure from the export trigger API
   The ol_openedx_course_export plugin returns HTTP 400 when any course
   fails to queue, even if other courses succeeded. The unconditional
   raise_for_status() call caused export_courses() to raise immediately,
   aborting the entire op before any successfully-queued tarballs could
   be copied.

   Fix: allow HTTP 400 through so the caller receives the response body
   (upload_task_ids + failed_uploads) and can still process the courses
   that were queued successfully. Log the initially-failed courses as a
   warning.

Fixes: mitodl/hq#11104

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: quazi-h

dg_projects/legacy_openedx/legacy_openedx/ops/open_edx.py

@@ -31,6 +31,8 @@
 from pypika import MySQLQuery as Query
 from pypika import Table, Tables
 
+COURSE_EXPORT_TIMEOUT = timedelta(minutes=60)
+
 
 class ListCoursesConfig(Config):
     edx_course_api_page_size: int = Field(
@@ -577,22 +579,47 @@ def export_edx_courses(
     exported_courses = context.resources.openedx.export_courses(
         course_ids=edx_course_ids,
     )
+    failed_initial = exported_courses.get("failed_uploads", {})
+    if failed_initial:
+        context.log.warning(
+            "The following courses failed to queue for export and will be skipped: %s",
+            list(failed_initial.keys()),
+        )
     successful_exports: set[str] = set()
     failed_exports: set[str] = set()
     tasks = exported_courses["upload_task_ids"]
     context.log.info("Exporting %s tasks from Open edX", len(tasks))
     # Possible status values found here:
     # https://github.com/openedx/django-user-tasks/blob/master/user_tasks/models.py
+    start_time = datetime.now(tz=UTC)
     while len(successful_exports.union(failed_exports)) < len(tasks):
+        if datetime.now(tz=UTC) - start_time > COURSE_EXPORT_TIMEOUT:
+            timed_out = set(tasks.keys()) - successful_exports - failed_exports
+            context.log.error(
+                "Timed out waiting for course exports after %s. Unresolved courses: %s",
+                COURSE_EXPORT_TIMEOUT,
+                timed_out,
+            )
+            break
         time.sleep(timedelta(seconds=60).seconds)
         for course_id, task_id in tasks.items():
+            if course_id in successful_exports or course_id in failed_exports:
+                continue
             try:
                 task_status = context.resources.openedx.check_course_export_status(
                     course_id,
                     task_id,
                 )
-            except httpx.HTTPStatusError:
-                # Don't fail the whole job if one task status yields an error
+            except httpx.HTTPStatusError as e:
+                context.log.warning(
+                    "HTTP %s error checking export status for course %s "
+                    "(task %s), marking as failed: %s",
+                    e.response.status_code,
+                    course_id,
+                    task_id,
+                    e,
+                )
+                failed_exports.add(course_id)
                 continue
             if task_status["state"] == "Succeeded":
                 successful_exports.add(course_id)

packages/ol-orchestrate-lib/src/ol_orchestrate/resources/openedx.py

@@ -64,6 +64,13 @@ def export_courses(self, course_ids: list[str]) -> dict[str, dict[str, str]]:
 
         :returns: A dictionary of course IDs and the S3 URL where it will be exported
                   to.
+
+        .. note::
+            The ol_openedx_course_export plugin returns HTTP 400 when some (but
+            not all) courses fail to queue. We intentionally allow 400 through so
+            that callers can still process the successfully-queued tasks from the
+            ``upload_task_ids`` field and inspect ``failed_uploads`` themselves.
+            Any other non-2xx status is still raised as an error.
         """
         request_url = f"{self.studio_url}/api/courses/v0/export/"
         response = self.http_client.post(
@@ -72,7 +79,10 @@ def export_courses(self, course_ids: list[str]) -> dict[str, dict[str, str]]:
             headers={"Authorization": f"JWT {self._fetch_access_token()}"},
             timeout=60,
         )
-        response.raise_for_status()
+        # HTTP 400 is returned on partial failure (some courses failed to queue);
+        # allow the caller to handle failed_uploads gracefully.
+        if response.status_code != 400:  # noqa: PLR2004
+            response.raise_for_status()
         return response.json()
 
     def check_course_export_status(