add competition tests; update account creation, submission tests

Author: Obada Haddad

.circleci/config.yml

@@ -43,6 +43,13 @@ jobs:
             docker pull codalab/codalab-legacy:py37 
             docker pull codalab/codalab-legacy:py3
 
+      # TODO: Putting E2E tests earlier to make playwright testing faster. Will remove after the PR is merged
+      - run:
+          name: "Tests: Run end-to-end (E2E) tests"
+          command: |
+            cd playwrightPython && $HOME/.local/bin/uv run pytest test_auth.py test_account_creation.py test_competition.py test_submission.py
+          no_output_timeout: 30m
+
       - run:
           name: "Tests: Run unit/integration tests (excluding e2e)"
           command: docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django py.test src/ -m "not e2e"
@@ -51,18 +58,21 @@ jobs:
       #     name: "Tests: Run end-to-end (E2E) tests"
       #     command: docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django py.test src/tests/functional/ -m e2e
       #     no_output_timeout: 60m
-      - run:
-          name: "Tests: Run end-to-end (E2E) tests"
-          command: |
-            cd playwrightPython && $HOME/.local/bin/uv run pytest test_auth.py test_account_creation.py test_submission.py
-          no_output_timeout: 30m
+
+      # TODO: uncomment this block just before the playwright PR is merged
+      #- run:
+      #    name: "Tests: Run end-to-end (E2E) tests"
+      #    command: |
+      #      cd playwrightPython && $HOME/.local/bin/uv run pytest test_auth.py test_account_creation.py test_competition.py test_submission.py
+      #    no_output_timeout: 30m
 
       # Example to run specific set of tests (for debugging individual tests from a batch of tests)
       # - run:
       #     name: e2e tests - competitions
       #     command: docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django py.test src/tests/functional/test_competitions.py -m e2e
       #     no_output_timeout: 60m
-
+      - store_artifacts:
+          path: playwrightPython/test-results
       - store_artifacts:
           path: artifacts/
 

.github/workflows/tests_pytest.yml

@@ -36,7 +36,7 @@ jobs:
     needs: [build]
     steps:
       - name: "Lint: Check code style with flake8"
-        run: docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django flake8 src/
+        run: docker compose exec django flake8 src/
   unit_tests:
     name: Unit test with Selenium
     runs-on: self-hosted
@@ -50,7 +50,7 @@ jobs:
     needs: [linter,build]
     steps:
       - name: "Tests: Run end-to-end (E2E) tests"
-        run: cd playwrightPython && $HOME/.local/bin/uv run pytest test_auth.py test_account_creation.py test_submission.py
+        run: cd playwrightPython && $HOME/.local/bin/uv run pytest test_auth.py test_account_creation.py test_competition.py test_submission.py
   cleanup:
     name: Cleanup
     runs-on: self-hosted

playwrightPython/test_account_creation.py

@@ -2,10 +2,16 @@
 import toml
 from loguru import logger
 from playwright.sync_api import expect, Page
+import random
+
+
+def randomNumber():
+    return str(random.randint(0, 1000))
+
 
 data = toml.load("config/config.toml")
-test_user = data["test_user"]["username"]
-test_password = data["test_user"]["password"]
+test_user = data["test_user"]["username"] + randomNumber()
+test_password = data["test_user"]["password"] + randomNumber()
 test_email = test_user + "@email.com"
 test_failed_user = data["test_failed_user"]["username"]
 db_host = data["database"]["host"]
@@ -55,8 +61,8 @@ def test_account_creation(page: Page):
             # Execute a command: this creates a new table
             cur.execute("SELECT username FROM profiles_user;")
             for row in cur:
-                logger.debug(row)
-                if row[0] in test_user:
+                logger.debug(row[0])
+                if row[0] == test_user:
                     assert row[0] == test_user
                     found = "true"
                     break

playwrightPython/test_competition.py

@@ -0,0 +1,120 @@
+from playwright.sync_api import expect, Page
+import toml
+import pytest
+import random
+
+data = toml.load("config/config.toml")
+
+titleNum = random.randint(0, 1000)
+
+
+# This allows us to autologin with a cookie in all tests instead of having to login each time
+@pytest.fixture(scope="session")
+def browser_context_args(browser_context_args):
+    """  """
+    browser_context_args.update(storage_state="config/state.json",)
+    return browser_context_args
+
+
+def test_competition_upload(page: Page):
+    page.goto("http://localhost/")
+    page.get_by_role("link", name=" Benchmarks/Competitions").click()
+    page.get_by_role("link", name=" Upload").click()
+    with page.expect_file_chooser() as fc_info:
+        page.get_by_role("button", name="").click()
+    file_chooser = fc_info.value
+    file_chooser.set_files("test_files/competitions/competition.zip")
+    expect(page.get_by_text("Competition created!")).to_be_visible()
+
+
+def task_creation(page):
+    page.goto("http://localhost/")
+    page.get_by_text("codabench Admin management").click()
+    page.get_by_role("link", name=" Resources").click()
+    page.get_by_text("Datasets and programs").first.click()
+    page.get_by_role("button", name=" Add Dataset/Program").click()
+    page.get_by_role("textbox", name="Name").click()
+    page.get_by_role("textbox", name="Name").fill(str(titleNum) + "Playwright Scoring Program")
+    page.get_by_role("textbox", name="Description").click()
+    page.get_by_role("textbox", name="Description").fill("Test Dataset Description")
+    page.locator(".ui.form > div > .ui").click()
+    page.get_by_text("Scoring Program", exact=True).nth(5).click()
+    with page.expect_file_chooser() as fc_info:
+        page.get_by_role("button", name="").click()
+    file_chooser = fc_info.value
+    file_chooser.set_files("test_files/Task/Scoring Program/scoring_program.zip")
+    page.get_by_role("button", name=" Upload").click()
+    page.get_by_text("Tasks", exact=True).click()
+    page.get_by_text("Create Task").first.click()
+    page.get_by_role("textbox", name="Name", exact=True).click()
+    page.get_by_role("textbox", name="Name", exact=True).fill(str(titleNum) + "Playwright Task")
+    page.get_by_role("textbox", name="Description").click()
+    page.get_by_role("textbox", name="Description").fill("Test Descirption Task")
+    page.get_by_text("Datasets and programs").nth(3).click()
+    page.locator("#scoring_program").click()
+    page.locator("#scoring_program").fill("test")
+    page.locator("a").filter(has_text=str(titleNum) + "Playwright Scoring Program").click()
+    page.get_by_text("Create", exact=True).click()
+
+
+def test_manual_competition_creation(page: Page):
+    task_creation(page)
+    page.goto("http://localhost/")
+    page.get_by_role("link", name=" Benchmarks/Competitions").click()
+    page.get_by_role("link", name=" Create").click()
+    page.get_by_role("textbox").nth(1).click()
+    page.get_by_role("textbox").nth(1).fill("Test")
+    with page.expect_file_chooser() as fc_info:
+        page.get_by_role("button", name="").click()
+    file_chooser = fc_info.value
+    file_chooser.set_files("test_files/competition/test_logo.png")
+    page.locator(".CodeMirror-scroll").first.click()
+    page.get_by_role("application").get_by_role("textbox").fill("on")
+    page.get_by_role("textbox", name="Example: codalab/codalab-").click()
+    page.get_by_role("textbox", name="Example: codalab/codalab-").fill("codalab/codalab-legacy:py37")
+    page.get_by_text("Participation").click()
+    page.locator("pre").nth(3).click()
+    page.get_by_role("application").filter(has_text="|||xxxxxxxxxx 101:").get_by_role("textbox").fill("Test Terms")
+    page.locator("input[name=\"registration_auto_approve\"]").check()
+    page.locator("a").filter(has_text="Pages").click()
+    page.get_by_role("button", name=" Add page").click()
+    page.get_by_role("textbox").nth(1).fill("Test Title")
+    page.locator("div:nth-child(2) > .EasyMDEContainer > .CodeMirror > .CodeMirror-scroll").click()
+    page.get_by_role("application").get_by_role("textbox").fill("Test Content")
+    page.get_by_text("Save").nth(1).click()
+    page.locator("a").filter(has_text="Phases").click()
+    page.get_by_role("button", name=" Add phase").click()
+    page.locator("input[name=\"name\"]").click()
+    page.locator("input[name=\"name\"]").fill(str(titleNum) + "Playwright Task")
+    page.locator("input[name=\"start_date\"]").click()
+    page.get_by_role("cell", name="12").click()
+    page.locator("input[name=\"start_time\"]").click()
+    page.get_by_role("cell", name="10:").click()
+    page.get_by_role("cell", name=":00").click()
+    page.locator("input[name=\"end_date\"]").click()
+    page.locator("input[name=\"end_date\"]").fill("")
+    page.locator("div:nth-child(7) > .EasyMDEContainer > .CodeMirror > .CodeMirror-scroll").click()
+    page.locator(".CodeMirror.cm-s-easymde.CodeMirror-wrap.CodeMirror-focused > div > textarea").fill("Phase")
+    page.locator(".ui.search.selection.dropdown.multiple").first.click()
+    page.locator(".ui.search.selection.dropdown.multiple > .search").first.fill(str(titleNum) + "Playwright Task")
+    page.get_by_text(str(titleNum) + "Playwright Task").nth(1).click()
+    page.get_by_text("Save").nth(3).click()
+    page.get_by_text("Leaderboard", exact=True).click()
+    page.get_by_role("button", name=" Add leaderboard").click()
+    page.locator("input[name=\"title\"]").click()
+    page.locator("input[name=\"title\"]").fill("Test title LEaderboard")
+    page.locator("input[name=\"key\"]").click()
+    page.locator("input[name=\"key\"]").fill("Test Key")
+    page.locator(".ui.plus").click()
+    page.locator("input[name=\"column_key_0\"]").click()
+    page.locator("input[name=\"column_key_0\"]").fill("Test Key")
+    page.locator("input[name=\"key\"]").click()
+    page.locator("input[name=\"key\"]").press("ControlOrMeta+a")
+    page.locator("input[name=\"key\"]").fill("Key")
+    page.locator("input[name=\"column_key_0\"]").click()
+    page.get_by_text("Save").nth(4).click()
+    page.get_by_role("button", name="Save").click()
+    expect(page.get_by_text("Test", exact=True)).to_be_visible()
+    expect(page.get_by_text("Test Content")).to_be_visible()
+
+# TODO Verify a few more items from the created competition

playwrightPython/test_submission.py

@@ -22,6 +22,7 @@ def wait_for_finished(page, time_out):
 
 
 def test_submission_basic(page: Page) -> None:
+    page.set_default_timeout(300000)
     page.goto("http://localhost/")
     page.get_by_role("link", name=" Benchmarks/Competitions").click()
     page.get_by_role("link", name=" Upload").click()
@@ -124,7 +125,7 @@ def test_submission_v18(page: Page) -> None:
     with page.expect_file_chooser() as fc_info:
         page.get_by_role("button", name="").click()
     file_chooser = fc_info.value
-    file_chooser.set_files("test_files/competitions/competition_18.zip`")
+    file_chooser.set_files("test_files/competitions/competition_18.zip")
     page.get_by_role("link", name="View").click()
     page.get_by_text("My Submissions").click()
     with page.expect_file_chooser() as fc_info: