Merge remote-tracking branch 'upstream/main' into update-lazy-trt-compile

Author: borisfom

.github/workflows/cicd-main.yml

@@ -35,6 +35,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.ref }}
           # custom token is required to trigger actions after reformatting + pushing
           token: ${{ secrets.NEMO_REFORMAT_TOKEN }}
+          fetch-depth: 0
 
       - name: Get changed files
         id: changed-files

.github/workflows/code-formatting.yml

@@ -8,6 +8,9 @@ on:
 # Set the access for individual scopes
 permissions: write-all
 
+env:
+  PYTHON_VERSION: "3.11"
+
 jobs:
   deploy:
     runs-on: ubuntu-latest
@@ -16,7 +19,7 @@ jobs:
       image: squidfunk/mkdocs-material
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         if: github.event.repository.fork == false
         with:
           ref: gh-pages-src
@@ -36,3 +39,43 @@ jobs:
         continue-on-error: true
         run:  mkdocs gh-deploy --force
 
+  linkcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Get changed files
+        id: changed-files
+        uses: step-security/changed-files@v45.0.1
+        with:
+          files: docs/**
+          files_separator: ","
+          separator: " "
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        if: steps.changed-files.outputs.any_changed == 'true'
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install Sphinx dependencies
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: python3 -m pip install -r requirements/requirements_docs.txt
+
+      - name: Linkcheck docs build
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: make -C docs linkcheck || true
+
+      - name: Eliminate false positives
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: ./docs/check_for_broken_links.sh
+
+      - name: Upload linkcheck output
+        if: steps.changed-files.outputs.any_changed == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: linkcheck-artifact
+          path: docs/build/linkcheck
+          if-no-files-found: error
+          retention-days: 7

.github/workflows/gh-docs.yml

@@ -0,0 +1,63 @@
+# Documentation Process for NeMo
+
+## Building the Documentation
+
+1. Create and activate a virtual environment.
+
+1. Install the documentation dependencies:
+
+   ```console
+   $ python3 -m pip install -r requirements/requirements_docs.txt
+   ```
+
+1. Build the documentation:
+
+   ```console
+   $ make -C docs html
+   ```
+
+## Checking for Broken Links
+
+1. Build the documentation, as described in the preceding section, but use the following command:
+
+   ```shell
+   make -C docs clean linkcheck
+   ```
+
+1. Run the link-checking script:
+
+   ```shell
+   ./docs/check_for_broken_links.sh
+   ```
+
+If there are no broken links, then the script exits with `0`.
+
+If the script produces any output, cut and paste the `uri` value into your browser to confirm
+that the link is broken.
+
+```json
+{
+  "filename": "nlp/text_normalization/nn_text_normalization.rst",
+  "lineno": 247,
+  "status": "broken",
+  "code": 0,
+  "uri": "https://research.fb.com/wp-content/uploads/2019/03/Neural-Models-of-Text-Normalization-for-Speech-Applications.pdf",
+  "info": "400 Client Error: Bad Request for url: https://research.facebook.com/wp-content/uploads/2019/03/Neural-Models-of-Text-Normalization-for-Speech-Applications.pdf"
+}
+```
+
+If the link is OK, and this is the case with many URLs that reference GitHub repository file headings,
+then cut and paste the JSON output and add it to `docs/false_positives.json`.
+Run the script again to confirm that the URL is no longer reported as a broken link.
+
+There may be false positives due to Sphinx not being able to detect links from built html files.
+Instead of adding those to the `docs/false_positives.json` file, it would be best to rewrite the
+reference using a [:ref:](https://www.sphinx-doc.org/en/master/usage/referencing.html#role-ref).
+
+For example, instead of writing `Modules <../api.html#modules>` to link to the modules section of
+a `api.rst` file, write it as ``:ref:`Modules <asr-api-modules>` ``. And in the `api.rst` file, add
+this label before the section being linked to:
+
+```
+.. _asr-api-modules:
+```

docs/README.md

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+DOCS_DIR=$(dirname "${BASH_SOURCE[0]}")
+FALSE_POSITIVES_JSON="${DOCS_DIR}/false_positives.json"
+NEEDS_REVIEW_JSON="${DOCS_DIR}/links_needing_review.json"
+LINKCHECK_JSON="${DOCS_DIR}/build/linkcheck/output.json"
+
+function check_environment {
+  local err=0
+  if ! [ -x "$(command -v jq)" ]; then
+    >&2 echo "jq is required but is not found."
+    ((err++))
+  fi
+  if [ ! -f "${FALSE_POSITIVES_JSON}" ]; then
+    >&2 echo "A JSON file with false positives is required: ${FALSE_POSITIVES_JSON}"
+    ((err++))
+  fi
+  if [ ! -f "${LINKCHECK_JSON}" ]; then
+    >&2 echo "Did not find linkcheck output JSON file: ${LINKCHECK_JSON}."
+    >&2 echo "Run Sphinx with the linkcheck arg: make -C docs clean linkcheck"
+    ((err++))
+  fi
+  if [ "${err}" -gt 0 ]; then
+    exit 2
+  fi
+}
+
+function check_links {
+  local err=0
+  # If you know how to prevent the hack with using jq twice, lmk.
+  broken=$(jq 'select(.status == "broken")' "${LINKCHECK_JSON}" | jq -s)
+  count=$(echo "${broken}" | jq 'length')
+  for i in $(seq 0 $(($count - 1)))
+  do
+    entry=$(echo "${broken}" | jq ".[${i}]")
+    link=$(echo "${entry}" | jq -r '.uri')
+    [ -n "${DEBUG}" ] && {
+     echo >&2 "Checking for false positive: ${link}"
+    }
+    local false_positive_resp; false_positive_resp=$(jq --arg check "${link}" -s 'any(.uri == $check)' < "${FALSE_POSITIVES_JSON}")
+    local needs_review_resp; needs_review_resp=$(jq --arg check "${link}" -s 'any(.uri == $check)' < "${NEEDS_REVIEW_JSON}")
+    # "false" indicates that the URL did not match any of the URIs in the false positive file.
+    if [[ "false" = "${false_positive_resp}" && "false" = "${needs_review_resp}" ]]; then
+      ((err++))
+      echo "${entry}"
+    fi
+  done
+  exit "${err}"
+}
+
+check_environment
+check_links

docs/check_for_broken_links.sh

@@ -0,0 +1 @@
+[]

docs/combined.json

@@ -0,0 +1,8 @@
+{
+  "filename": "nlp/text_normalization/nn_text_normalization.rst",
+  "lineno": 247,
+  "status": "broken",
+  "code": 0,
+  "uri": "https://research.fb.com/wp-content/uploads/2019/03/Neural-Models-of-Text-Normalization-for-Speech-Applications.pdf",
+  "info": "400 Client Error: Bad Request for url: https://research.facebook.com/wp-content/uploads/2019/03/Neural-Models-of-Text-Normalization-for-Speech-Applications.pdf"
+}