Skip to content

Commit a62ceda

Browse files
mcdurdinmcdurdin
committed
chore: replace broken-link-checker with linkinator
Broken-link-checker (BLC) is basically dead -- no releases for 8 years, no updates to master for 5. The trigger for this change is, however, that BLC was not honoring exclusions on redirects, which cause sporadic errors (e.g. 429) because we hit external servers in our link test. Replace with linkinator, which is in maintenance and does the job pretty well. I have forked linkinator and added some patches specific to our environment which I may push upstream later when we are confident of the setup. Unify the ci.yml and build.sh test steps (this will need to be tested in CI once this merges to master). Test-bot: skip
1 parent ece6d4f commit a62ceda

3 files changed

Lines changed: 58 additions & 70 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 45 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ jobs:
88
runs-on: ubuntu-latest
99
env:
1010
KEYMANHOSTS_TIER: TIER_TEST
11+
CONTAINER_DESC: keyman-com-app
12+
CONTAINER_PORT: 8053
1113

1214
steps:
1315
- name: Checkout
@@ -22,53 +24,65 @@ jobs:
2224
fail-fast: true
2325

2426
#
25-
# Finally, run the tests
27+
# Run tests -- these step definitions are NEARLY identical across:
28+
# help.keyman.com, keyman.com, keymanweb.com, api.keyman.com
2629
#
30+
# Difference: testPath and skipPaths are different for do_test_links on this site
31+
#
32+
- name: Test setup
33+
shell: bash
34+
run: |
35+
source ./_common/tests.inc.sh
36+
set -e
37+
do_test_record_start_time
38+
echo "TEST_START_TIME=${TEST_START_TIME}" >> "$GITHUB_ENV"
39+
2740
- name: PHP test
41+
if: ${{ !cancelled() }}
2842
shell: bash
2943
run: |
30-
docker exec keyman-com-app sh -c "vendor/bin/phpunit --testdox"
44+
source ./_common/tests.inc.sh
45+
set -e
46+
do_test_unit_tests "$CONTAINER_DESC"
3147
3248
- name: Lint
49+
if: ${{ !cancelled() }}
3350
shell: bash
3451
run: |
35-
( set +e; set +o pipefail; find . -name '*.php' | grep -v '/vendor/' | xargs -n 1 -d '\n' php -l | grep -v "No syntax errors detected"; exit ${PIPESTATUS[2]} )
52+
source ./_common/tests.inc.sh
53+
set -e
54+
do_test_lint "$CONTAINER_DESC"
3655
3756
- name: Check broken links
57+
if: ${{ !cancelled() }}
3858
shell: bash
39-
continue-on-error: false
4059
run: |
41-
set +e
42-
set +o pipefail
43-
readarray -t ignoresArray <<< $(find ./_includes/locale/strings/keyboards/ -maxdepth 1 -name '*.php' ! -name "en.php" \
44-
-execdir basename {} .php ';')
45-
baseURL="http://localhost:8053"
46-
ignoreStr=(" --exclude ${baseURL}*downloads/releases/*")
47-
for locale in "${ignoresArray[@]}"; do
48-
ignoreStr+=" --exclude ${baseURL}/${locale}/*"
49-
done
50-
echo "ignoreStr: ${ignoreStr[@]}"
51-
npx broken-link-checker ${baseURL}/_test --recursive --ordered ---host-requests 50 -e --filter-level 3 ${ignoreStr} | tee blc.log
52-
echo "BLC_RESULT=${PIPESTATUS[0]}" >> "$GITHUB_ENV"
60+
source ./_common/tests.inc.sh
61+
set -e
62+
63+
# NOTE: link checker runs on host rather than in docker image
64+
local ignoreLocales
65+
66+
# determine non-en locales to ignore
67+
ignoreLocales="$(jq -r 'keys | map(select(. != "en")) | join("|")' ./_includes/locale/locales.json)"
68+
69+
do_test_links "http://localhost:${CONTAINER_PORT}" "/_test" "/(${ignoreLocales})" "/en/downloads/releases/" || LINK_RESULT=$?
70+
71+
# We split the reporting of broken links into a separate step for ease of
72+
# viewing because the broken links are otherwise hidden in a sea of good
73+
# links in a very long report
5374

5475
- name: Report on broken links
76+
if: ${{ !cancelled() }}
5577
run: |
56-
set +e
57-
set +o pipefail
58-
cat blc.log | \
59-
grep -E "BROKEN|Getting links from" | \
60-
grep -B 1 "BROKEN";
61-
exit "${BLC_RESULT}"
78+
source ./_common/tests.inc.sh
79+
set -e
80+
do_test_print_link_report
6281
6382
- name: Check PHP errors
83+
if: ${{ !cancelled() }}
6484
shell: bash
6585
run: |
66-
CONTAINER=`docker container ls -l -q`
67-
if docker container logs $CONTAINER 2>&1 | grep -q 'php7'; then
68-
echo 'PHP reported errors or warnings:'
69-
docker container logs $CONTAINER 2>&1 | grep 'php7'
70-
exit 1
71-
else
72-
echo 'No PHP errors found'
73-
exit 0
74-
fi
86+
source ./_common/tests.inc.sh
87+
set -e
88+
do_test_print_container_error_logs "$CONTAINER_DESC"

‎.gitignore‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ cdn/deploy/
55
tier.txt
66

77
# unit test artifacts
8-
blc.log
8+
linkinator-results.json
99
.phpunit.result.cache
1010

1111
vendor*
@@ -21,4 +21,4 @@ _common/
2121
_control/debug
2222

2323
# .htaccess is generated from .htaccess.in during build.sh build
24-
.htaccess
24+
.htaccess

‎build.sh‎

Lines changed: 11 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
## START STANDARD SITE BUILD SCRIPT INCLUDE
33
readonly THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")"
44
readonly BOOTSTRAP="$(dirname "$THIS_SCRIPT")/resources/bootstrap.inc.sh"
5-
readonly BOOTSTRAP_VERSION=v1.0.13
5+
readonly BOOTSTRAP_VERSION=feat/linkinator-and-central-test-script
66
if ! [ -f "$BOOTSTRAP" ] || ! source "$BOOTSTRAP"; then
77
curl -H "Cache-Control: no-cache" --fail --silent --show-error -w "curl: Finished attempt to download %{url}" "https://raw.githubusercontent.com/keymanapp/shared-sites/$BOOTSTRAP_VERSION/bootstrap.inc.sh" -o "$BOOTSTRAP.tmp" || exit 1
88
source "$BOOTSTRAP.tmp"
@@ -28,6 +28,9 @@ builder_describe \
2828
start \
2929
stop \
3030
test \
31+
"--no-unit-test" \
32+
"--no-lint" \
33+
"--no-link-check" \
3134
htaccess "rebuild .htaccess from .htaccess.in (for development)"
3235

3336
builder_parse "$@"
@@ -52,51 +55,22 @@ function do_start() {
5255
start_docker_container $KEYMAN_IMAGE_NAME $KEYMAN_CONTAINER_NAME $KEYMAN_CONTAINER_DESC $HOST_KEYMAN_COM $PORT_KEYMAN_COM $BUILDER_CONFIGURATION
5356
}
5457

55-
function test_docker_container() {
56-
# Note: ci.yml replicates these
57-
58-
echo "TIER_TEST" > tier.txt
59-
set +e;
60-
set +o pipefail;
61-
62-
builder_echo blue "---- PHP unit tests"
63-
docker exec $KEYMAN_CONTAINER_DESC sh -c "vendor/bin/phpunit --testdox"
64-
65-
# Lint .php files for obvious errors
66-
builder_echo blue "---- Lint PHP files"
67-
docker exec $KEYMAN_CONTAINER_DESC sh -c "find . -name '*.php' | grep -v '/vendor/' | xargs -n 1 -d '\\n' php -l"
68-
58+
function do_test_docker_container() {
6959
# NOTE: link checker runs on host rather than in docker image
70-
builder_echo blue "---- Testing links"
60+
local ignoreLocales
7161

72-
# determine non-en locales to ignore along with /downloads/releases
73-
readarray -t ignoresArray <<< $(find ./_includes/locale/strings/keyboards/ -maxdepth 1 -name '*.php' ! -name "en.php" \
74-
-execdir basename {} .php ';')
75-
local baseURL="http://localhost:8053"
76-
local ignoreStr=(" --exclude ${baseURL}*/downloads/releases/*")
77-
for locale in "${ignoresArray[@]}"; do
78-
ignoreStr+=" --exclude ${baseURL}/${locale}/*"
79-
done
80-
echo "ignoreStr: ${ignoreStr[@]}"
81-
npx broken-link-checker ${baseURL}/_test --recursive --ordered ---host-requests 50 -e --filter-level 3 ${ignoreStr} | tee blc.log
82-
local BLC_RESULT=${PIPESTATUS[0]}
83-
echo ----------------------------------------------------------------------
84-
echo Link check summary
85-
echo ----------------------------------------------------------------------
86-
cat blc.log | \
87-
grep -E "BROKEN|Getting links from" | \
88-
grep -B 1 "BROKEN";
62+
# determine non-en locales to ignore
63+
ignoreLocales="$(jq -r 'keys | map(select(. != "en")) | join("|")' ./_includes/locale/locales.json)"
8964

90-
builder_echo blue "Done checking links"
91-
rm tier.txt
92-
return "${BLC_RESULT}"
65+
test_docker_container $KEYMAN_CONTAINER_DESC $PORT_KEYMAN_COM "/_test" "/(${ignoreLocales})" "/en/downloads/releases/"
9366
}
9467

68+
9569
builder_run_action configure do_configure
9670
builder_run_action clean clean_docker_container $KEYMAN_IMAGE_NAME $KEYMAN_CONTAINER_NAME
9771
builder_run_action stop stop_docker_container $KEYMAN_IMAGE_NAME $KEYMAN_CONTAINER_NAME
9872
builder_run_action build build_docker_container $KEYMAN_IMAGE_NAME $KEYMAN_CONTAINER_NAME $BUILDER_CONFIGURATION
9973
builder_run_action start do_start
10074

101-
builder_run_action test test_docker_container
75+
builder_run_action test do_test_docker_container
10276
builder_run_action htaccess preprocess_htaccess

0 commit comments

Comments
 (0)