Retry transient grading poll failures

Keep polling after failed submission-status requests until the existing
bounded poll budget is exhausted. Extend the E2E submission wait to cover
that window and add regression coverage for a transient 503 response.

Author: ihalaij1

e2e_tests/helpers.py

@@ -35,8 +35,7 @@ def upload_submission(page: Page, chapter_name: str, exercise_name: str, files:
     for file in files:
         page.get_by_label(file.label).set_input_files(os.path.join(assets_path, file.name))
     page.locator(exercise_name).get_by_role("button", name="Submit").click()
-    expect(page.locator("#page-modal")
-           ).to_contain_text("Total points:", timeout=10000)
+    expect(page.locator("#page-modal")).to_contain_text("Total points:", timeout=65000)
     page.get_by_role("button", name="Close", exact=True).click()
 
 def navigate_to_default_course(page: Page):

e2e_tests/test_compare_submissions.py

@@ -2,7 +2,7 @@
 from e2e_tests.helpers import upload_submission, login, logout, navigate_to_default_course, File
 
 
-def test_compare_submissions(page: Page) -> None:
+def test_compare_submissions(page: Page) -> None: # pylint: disable=too-many-statements
     ASSISTANT_FEEDBACK_TEXT = "ASSISTANT_FEEDBACK"
     FEEDBACK_TEXT = "FEEDBACK"
     POINTS = "5"
@@ -14,12 +14,26 @@ def test_compare_submissions(page: Page) -> None:
     red = "rgb(248, 215, 218)"
     login(page, "student", "student")
     navigate_to_default_course(page)
+
+    first_poll_failed = False
+
+    def fail_first_poll(route):
+        nonlocal first_poll_failed
+        if first_poll_failed:
+            route.continue_()
+        else:
+            first_poll_failed = True
+            route.fulfill(status=503, body="Temporary polling failure")
+
+    page.route("**/poll/**", fail_first_poll)
     upload_submission(
         page,
         chapter_name=chapter_name,
         exercise_name=exercise_name,
         files=(File("wallet.py"), File("wallet_program.py"))
     )
+    page.unroute("**/poll/**", fail_first_poll)
+    assert first_poll_failed
     upload_submission(
         page,
         chapter_name=chapter_name,

exercise/static/exercise/poll.js

@@ -41,8 +41,14 @@
       var poller = this;
       $.ajax(this.url, {dataType: "html"})
         .fail(function() {
-          poller.message("error");
-          poller.ready(true);
+          poller.count++;
+          if (poller.element.is(":visible") &&
+              poller.count < poller.settings.poll_delays.length) {
+            poller.schedule();
+          } else {
+            poller.message("error");
+            poller.ready(true);
+          }
         })
         .done(function(data) {
           poller.count++;