Added new Playwright tests to replace the old Selenium tests

Author: reppuli92

.github/workflows/Dockerfile

@@ -1,9 +1,5 @@
 FROM ubuntu:22.04
 
-ARG GECKODRIVER_VERSION=0.33.0
-ARG GECKODRIVER_FILE=geckodriver-v${GECKODRIVER_VERSION}-linux64.tar.gz
-ARG GECKODRIVER_LINK=https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER_VERSION}/${GECKODRIVER_FILE}
-
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt update -y
@@ -17,9 +13,4 @@ RUN pip install -r requirements.txt
 RUN pip install -r requirements_testing.txt
 RUN rm requirements.txt requirements_testing.txt
 
-# install geckodriver for selenium
-RUN curl -s -L $GECKODRIVER_LINK | tar -xz
-RUN chmod +x geckodriver
-RUN mv geckodriver /usr/bin/
-
 ENV APLUS_BASE_URL="-"

.github/workflows/tests.yml

@@ -41,17 +41,6 @@ jobs:
           key: v3-${{ hashFiles('.github/workflows/Dockerfile', 'requirements_testing.txt', 'requirements.txt') }}
       - run: docker load -i .docker-img.tar
       - run: docker run -v ${{ github.workspace }}:${{ github.workspace }} -w ${{ github.workspace }} testimg bash -c 'python3 manage.py compilemessages && python3 manage.py test'
-  selenium-tests:
-    runs-on: ubuntu-latest
-    needs: docker-build
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/cache@v3
-        with:
-          path: .docker-img.tar
-          key: v3-${{ hashFiles('.github/workflows/Dockerfile', 'requirements_testing.txt', 'requirements.txt') }}
-      - run: docker load -i .docker-img.tar
-      - run: docker run -v ${{ github.workspace }}:${{ github.workspace }} -w ${{ github.workspace }} testimg bash -c 'python3 manage.py compilemessages && selenium_test/run_servers_and_tests.sh'
   playwright-tests:
     timeout-minutes: 60
     runs-on: ubuntu-latest

.gitignore

@@ -37,11 +37,6 @@ dump.json
 
 test_results/
 
-selenium_test/aplus.out
-selenium_test/example_grader.out
-selenium_test/server.ports
-selenium_test/grader/db.sqlite3
-
 *~
 *.swp
 *.swo

README.md

@@ -28,7 +28,7 @@ The [doc/GRADERS.md](doc/GRADERS.md) describes the assessment protocol supported
 Additionally, there is a minimal example grader in [doc/example_grader.py](doc/example_grader.py), which can be used to start a new service.
 A list of existing assessment services and other tools can be found in [the project github page](https://apluslms.github.io/components/).
 
-The [selenium_test/](selenium_test) offers an integration test suite using the Selenium Firefox driver.
+The [e2e_tests/](e2e_tests) folder offers an integration test suite using Playwright.
 
 Code Organization
 -----------------

doc/README.md

@@ -139,9 +139,6 @@ Therefore they are run and updated outside the containers, within a virtual envi
 
 ### Prerequisities
 
-- For Selenium tests:
-  * Firefox browser installed
-  * xvfb virtual framebuffer installed (`sudo apt-get install xvfb` or `aptdcon --install xvfb`)
 - python3 installed
 - [Python module venv (or virtualenv)](https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/#installing-virtualenv) installed
 - For translations: gettext (`aptdcon --install gettext` or `sudo apt-get install gettext`)
@@ -193,44 +190,6 @@ If you need to be able to login or use the admin-page while running the code wit
    $ python3 manage.py test
    ```
 
-### Selenium integration tests
-
-Selenium tests check end-to-end -functioning from a user-perspective.
-The tests will automatically open and direct Firefox browser windows.
-Currently, the tests depend on a Unix type shell and are run within a virtual environment created above.
-
-1. Install the necessary requirements:
-
-   ```sh
-   $ pip3 install -r requirements_testing.txt
-   ```
-
-2. Download the latest release of geckodriver (choose the appropriate tar.gz file based on your machine) from https://github.com/mozilla/geckodriver/releases and extract it.
-
-3. Check the path to your extracted geckodriver and add it to PATH:
-
-   ```sh
-   $ export PATH=$PATH:/path-to-your-extracted-geckodriver/
-   ```
-
-4. To setup the servers and run all the tests at one go:
-
-   ```sh
-   $ selenium_test/run_servers_and_tests.sh
-   ```
-
-5. Alternatively you can run individual tests:
-
-   ```sh
-   $ cd selenium_test/test/
-   $ ../run_servers.sh
-
-   $ python3 login_test.py
-   $ python3 home_page_test.py
-
-   $ ../kill_servers.sh
-   ```
-
 ### Updating translations for Finnish and English versions
 
 Before creating a pull request, you need to ensure that the [English strings](https://github.com/apluslms/a-plus/blob/master/locale/en/LC_MESSAGES/django.po) and [Finnish translations](https://github.com/apluslms/a-plus/blob/master/locale/fi/LC_MESSAGES/django.po) are up-to-date.

e2e_tests/test_ajax_exercise_grader.py

@@ -0,0 +1,37 @@
+import re
+from playwright.sync_api import Page, expect
+from e2e_tests.helpers import navigate_to_default_course, login
+
+def test_should_not_count_empty_submit(page: Page):
+    page.goto("http://localhost:8000/?hl=en")
+    login(page, "student", "student")
+    navigate_to_default_course(page)
+    page.get_by_role("link", name="Course materials").click()
+    first_link = page.get_by_role("link", name="AJAX exercises: grading in browser JavaScript").first
+    first_link.click()
+    page.get_by_role("button", name="Submit").click()
+
+    submissions = page.get_by_role("button", name=re.compile("My submissions"))
+    expect(submissions).to_contain_text("0 / 20")
+    #empty submit should not count as submission
+
+    points = page.locator('#chapter-exercise-1').get_by_role("button", name=re.compile("Points"))
+    expect(points).to_contain_text("0 / 10")
+
+
+def test_should_give_full_points_on_correct_answer(page: Page):
+    page.goto("http://localhost:8000/?hl=en")
+    login(page, "student", "student")
+    navigate_to_default_course(page)
+    page.get_by_role("link", name="Course materials").click()
+    first_link = page.get_by_role("link", name="AJAX exercises: grading in browser JavaScript").first
+    first_link.click()
+
+    page.get_by_role("textbox").fill("abc")
+    page.get_by_role("button", name="Submit").click()
+
+    submissions = page.get_by_role("button", name=re.compile("My submissions"))
+    expect(submissions).to_contain_text("1 / 20")
+
+    points = page.locator('#chapter-exercise-1').get_by_role("button", name=re.compile("Points"))
+    expect(points).to_contain_text("10 / 10")

e2e_tests/test_compare_submissions.py

@@ -35,7 +35,9 @@ def test_compare_submissions(page: Page) -> None:
     first_link.click()
     page.get_by_label("6.3.6 Wallet").get_by_role(
         "button", name="View all submissions").click()
-    page.locator("#submission-2").get_by_role("link", name="Inspect").click()
+
+    latest_submission = page.locator('[id^="submission-"]').first
+    latest_submission.get_by_role("link", name="Inspect").click()
 
     def assert_line(filename: str, line_number: int, text: str, color: str):
         line = page.get_by_test_id(f"{filename}-line-{line_number}")

e2e_tests/test_file_upload_grader.py

@@ -0,0 +1,25 @@
+import re
+from playwright.sync_api import Page, expect
+from e2e_tests.helpers import navigate_to_default_course, login
+
+
+def test_should_not_submit_without_files(page: Page):
+    page.goto("http://localhost:8000/?hl=en")
+    login(page, "student", "student")
+    navigate_to_default_course(page)
+    page.get_by_role("link", name="Course materials").click()
+    first_link = page.get_by_role("link", name="Exercises with Python grader utils").first
+    first_link.click()
+
+    exercise = page.locator('#chapter-exercise-1')
+
+    exercise.get_by_role("button", name="Submit").click()
+
+    submissions = exercise.get_by_role("button", name=re.compile("My submissions"))
+    points = exercise.get_by_role("button", name=re.compile("Points"))
+
+    expect(submissions).to_contain_text("0 / 10")
+    #clicking submit without files will not result in a submission
+    expect(points).to_contain_text("0 / 10")
+
+#TODO more tests ?

e2e_tests/test_example.py e2e_tests/test_homepage.pye2e_tests/test_example.py renamed to e2e_tests/test_homepage.py

@@ -1,14 +1,13 @@
 import re
 from playwright.sync_api import Page, expect
-from e2e_tests.helpers import navigate_to_default_course
+from e2e_tests.helpers import navigate_to_default_course, login
 
 
-def test_frontpage_has_title(page: Page):
+def test_homepage(page: Page):
     page.goto("http://localhost:8000/?hl=en")
     expect(page).to_have_title(re.compile("A+"))
 
-
-def test_course_has_heading(page: Page) -> None:
-    page.goto("http://localhost:8000/?hl=en")
+    login(page, "student", "student")
     navigate_to_default_course(page)
     expect(page.get_by_role("heading", name="A+ Manual")).to_be_visible()
+    #TODO figure out more testing for the homepage

e2e_tests/test_login.py

@@ -0,0 +1,20 @@
+import re
+from playwright.sync_api import Page, expect
+from e2e_tests.helpers import login
+
+
+def test_login(page: Page):
+    page.goto("http://localhost:8000/?hl=en")
+    login(page, "student", "student")
+    expect(page).to_have_title(re.compile("A+"))
+    expect(page.get_by_label("Main")).to_contain_text("Stacy Student")
+
+def test_false_login(page: Page):
+    page.goto("http://localhost:8000/?hl=en")
+    login(page, "fake", "fake")
+    expect(page).to_have_title(re.compile("Log in to A+ | A+"))
+
+    error_message = "Please enter a correct username and password. Note that both fields may be case-sensitive."
+    expect(
+        page.locator("div.alert.alert-danger.alert-dismissible")
+    ).to_contain_text(error_message)