chore: replace broken-link-checker with linkinator

Broken-link-checker (BLC) is basically dead -- no releases for 8 years,
no updates to master for 5. The trigger for this change is, however,
that BLC was not honoring exclusions on redirects, which cause sporadic
errors (e.g. 429) because we hit external servers in our link test.

Replace with linkinator, which is in maintenance and does the job
pretty well.

I have forked linkinator and added some patches specific to our
environment which I may push upstream later when we are confident of the
setup.

Unify the ci.yml and build.sh test steps (this will need to be tested
in CI once this merges to master).

Test-bot: skip

Author: mcdurdin

.github/workflows/ci.yml

@@ -8,6 +8,8 @@ jobs:
     runs-on: ubuntu-latest
     env:
       KEYMANHOSTS_TIER: TIER_TEST
+      CONTAINER_DESC: keyman-com-app
+      CONTAINER_PORT: 8053
 
     steps:
     - name: Checkout
@@ -22,53 +24,56 @@ jobs:
         fail-fast: true
 
     #
-    # Finally, run the tests
+    # Run tests -- these step definitions are identical across:
+    #   help.keyman.com, keyman.com, keymanweb.com, api.keyman.com
     #
+    - name: Test setup
+      shell: bash
+      run: |
+        source ./_common/tests.inc.sh
+        set -e
+        do_test_record_start_time
+        echo "TEST_START_TIME=${TEST_START_TIME}" >> "$GITHUB_ENV"
+
     - name: PHP test
+      if: ${{ !cancelled() }}
       shell: bash
       run: |
-        docker exec keyman-com-app sh -c "vendor/bin/phpunit --testdox"
+        source ./_common/tests.inc.sh
+        set -e
+        do_test_unit_tests "$CONTAINER_DESC"
 
     - name: Lint
+      if: ${{ !cancelled() }}
       shell: bash
       run: |
-        ( set +e; set +o pipefail; find . -name '*.php' | grep -v '/vendor/' | xargs -n 1 -d '\n' php -l | grep -v "No syntax errors detected"; exit ${PIPESTATUS[2]} )
+        source ./_common/tests.inc.sh
+        set -e
+        do_test_lint "$CONTAINER_DESC"
 
     - name: Check broken links
+      if: ${{ !cancelled() }}
       shell: bash
-      continue-on-error: false
       run: |
-        set +e
-        set +o pipefail
-        readarray -t ignoresArray <<< $(find ./_includes/locale/strings/keyboards/ -maxdepth 1 -name '*.php' ! -name "en.php" \
-          -execdir basename  {} .php ';')
-        baseURL="http://localhost:8053"
-        ignoreStr=("  --exclude ${baseURL}*downloads/releases/*")
-        for locale in "${ignoresArray[@]}"; do
-          ignoreStr+=" --exclude ${baseURL}/${locale}/*"
-        done
-        echo "ignoreStr: ${ignoreStr[@]}"
-        npx broken-link-checker ${baseURL}/_test --recursive --ordered ---host-requests 50 -e --filter-level 3 ${ignoreStr} | tee blc.log
-        echo "BLC_RESULT=${PIPESTATUS[0]}" >> "$GITHUB_ENV"
+        source ./_common/tests.inc.sh
+        set -e
+        do_test_links "http://localhost:${CONTAINER_PORT}"
+
+      # We split the reporting of broken links into a separate step for ease of
+      # viewing because the broken links are otherwise hidden in a sea of good
+      # links in a very long report
 
     - name: Report on broken links
+      if: ${{ !cancelled() }}
       run: |
-        set +e
-        set +o pipefail
-        cat blc.log | \
-          grep -E "BROKEN|Getting links from" | \
-          grep -B 1 "BROKEN";
-        exit "${BLC_RESULT}"
+        source ./_common/tests.inc.sh
+        set -e
+        do_test_print_link_report
 
     - name: Check PHP errors
+      if: ${{ !cancelled() }}
       shell: bash
       run: |
-        CONTAINER=`docker container ls -l -q`
-        if docker container logs $CONTAINER 2>&1 | grep -q 'php7'; then
-          echo 'PHP reported errors or warnings:'
-          docker container logs $CONTAINER 2>&1 | grep 'php7'
-          exit 1
-        else
-          echo 'No PHP errors found'
-          exit 0
-        fi
+        source ./_common/tests.inc.sh
+        set -e
+        do_test_print_container_error_logs "$CONTAINER_DESC"

.gitignore

@@ -5,7 +5,7 @@ cdn/deploy/
 tier.txt
 
 # unit test artifacts
-blc.log
+linkinator-results.json
 .phpunit.result.cache
 
 vendor*
@@ -21,4 +21,4 @@ _common/
 _control/debug
 
 # .htaccess is generated from .htaccess.in during build.sh build
-.htaccess
+.htaccess

build.sh

@@ -2,7 +2,7 @@
 ## START STANDARD SITE BUILD SCRIPT INCLUDE
 readonly THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")"
 readonly BOOTSTRAP="$(dirname "$THIS_SCRIPT")/resources/bootstrap.inc.sh"
-readonly BOOTSTRAP_VERSION=v1.0.13
+readonly BOOTSTRAP_VERSION=feat/linkinator-and-central-test-script
 if ! [ -f "$BOOTSTRAP" ] || ! source "$BOOTSTRAP"; then
   curl -H "Cache-Control: no-cache" --fail --silent --show-error -w "curl: Finished attempt to download %{url}" "https://raw.githubusercontent.com/keymanapp/shared-sites/$BOOTSTRAP_VERSION/bootstrap.inc.sh" -o "$BOOTSTRAP.tmp" || exit 1
   source "$BOOTSTRAP.tmp"
@@ -28,6 +28,9 @@ builder_describe \
   start \
   stop \
   test \
+  "--no-unit-test" \
+  "--no-lint" \
+  "--no-link-check" \
   htaccess "rebuild .htaccess from .htaccess.in (for development)"
 
 builder_parse "$@"
@@ -52,51 +55,22 @@ function do_start() {
   start_docker_container $KEYMAN_IMAGE_NAME $KEYMAN_CONTAINER_NAME $KEYMAN_CONTAINER_DESC $HOST_KEYMAN_COM $PORT_KEYMAN_COM $BUILDER_CONFIGURATION
 }
 
-function test_docker_container() {
-  # Note: ci.yml replicates these
-
-  echo "TIER_TEST" > tier.txt
-  set +e;
-  set +o pipefail;
-
-  builder_echo blue "---- PHP unit tests"
-  docker exec $KEYMAN_CONTAINER_DESC sh -c "vendor/bin/phpunit --testdox"
-
-  # Lint .php files for obvious errors
-  builder_echo blue "---- Lint PHP files"
-  docker exec $KEYMAN_CONTAINER_DESC sh -c "find . -name '*.php' | grep -v '/vendor/' | xargs -n 1 -d '\\n' php -l"
-
+function do_test_docker_container() {
   # NOTE: link checker runs on host rather than in docker image
-  builder_echo blue "---- Testing links"
+  local ignoreLocales
 
-  # determine non-en locales to ignore along with /downloads/releases
-  readarray -t ignoresArray <<< $(find ./_includes/locale/strings/keyboards/ -maxdepth 1 -name '*.php' ! -name "en.php" \
-    -execdir basename  {} .php ';')
-  local baseURL="http://localhost:8053"
-  local ignoreStr=("  --exclude ${baseURL}*/downloads/releases/*")
-  for locale in "${ignoresArray[@]}"; do
-    ignoreStr+=" --exclude ${baseURL}/${locale}/*"
-  done
-  echo "ignoreStr: ${ignoreStr[@]}"
-  npx broken-link-checker ${baseURL}/_test --recursive --ordered ---host-requests 50 -e --filter-level 3 ${ignoreStr} | tee blc.log
-  local BLC_RESULT=${PIPESTATUS[0]}
-  echo ----------------------------------------------------------------------
-  echo Link check summary
-  echo ----------------------------------------------------------------------
-  cat blc.log | \
-    grep -E "BROKEN|Getting links from" | \
-    grep -B 1 "BROKEN";
+  # determine non-en locales to ignore
+  ignoreLocales="$(jq -r 'keys | map(select(. != "en")) | join("|")' ./_includes/locale/locales.json)"
 
-  builder_echo blue "Done checking links"
-  rm tier.txt
-  return "${BLC_RESULT}"
+  test_docker_container $KEYMAN_CONTAINER_DESC $PORT_KEYMAN_COM "/_test" "/(${ignoreLocales})"  "/en/downloads/releases/"
 }
 
+
 builder_run_action configure  do_configure
 builder_run_action clean      clean_docker_container $KEYMAN_IMAGE_NAME $KEYMAN_CONTAINER_NAME
 builder_run_action stop       stop_docker_container  $KEYMAN_IMAGE_NAME $KEYMAN_CONTAINER_NAME
 builder_run_action build      build_docker_container $KEYMAN_IMAGE_NAME $KEYMAN_CONTAINER_NAME $BUILDER_CONFIGURATION
 builder_run_action start      do_start
 
-builder_run_action test       test_docker_container
+builder_run_action test       do_test_docker_container
 builder_run_action htaccess   preprocess_htaccess