Address comments

Author: forsyth2

README.md

@@ -8,7 +8,7 @@ This package is for implementing the software needs of the E3SM Communications t
 
 `e3sm-comms-e3sm-org-reviewer`
 - input:
-  - From WordPress under Tools > Export: xml file of WordPress pages, xml file of WordPress posts. Note: your export will go to your local machine. Example transfer command: `scp local_path/wordpress_posts.xml forsyth@perlmutter.nersc.gov:perlmutter_path/wordpress_posts.xml`.
+  - From WordPress under Tools > Export: xml file of WordPress pages, xml file of WordPress posts. Note: your export will go to your local machine. Example transfer command: `scp local_path/wordpress_posts.xml user@host:/remote/path/wordpress_posts.xml`.
   - From output of `e3sm-comms-website-reviewer`: txt file of hierarchical outline of Confluence pages, txt file of sensitive terms found on Confluence pages
   - Other: txt file of whitelisted e3sm.org pages, txt file of e3sm.org pages expected to be archived, txt file of sensitive terms, txt file of known-ok e3sm.org pages (that is, script is picking up errors we don't care about), txt file of keep-unchanged e3sm.org pages (that is, pages we don't want to change)
 - output: 3 Markdown summary reports: (1) An analysis of the e3sm.org paths, (2) An analysis of the sensitive terms found, (3) the key action items
@@ -31,7 +31,7 @@ This package is for implementing the software needs of the E3SM Communications t
   - From WordPress under Tools > Export: xml file of WordPress pages, xml file of WordPress posts
   - From output of `e3sm-comms-website-reviewer`: txt file of hierarchical outline of Confluence pages
   - Other: txt file of sensitive terms, txt file of whitelisted e3sm.org pages
-- output: Markdown summary report of sensitive terms found in exported WordPress data
+ - output: 5 files under `output/exported_xml_reviewer/`: (1) `wordpress_sensitive_terms_report.md`, (2) `wordpress_hierarchical_outline.txt`, (3) `wordpress_navigation_issues_report.md`, (4) `wordpress_invalid_internal_links_report.md`, (5) `wordpress_published_pages_link_report.md`
 
 ### Confluence API commands (require Confluence token)
 

e3sm_comms/exported_xml_reviewer/main.py

@@ -18,14 +18,24 @@
 # Configuration
 # -----------------------------------------------------------------------------
 
+# Required inputs:
 INPUT_XML_PAGES: str = f"{IO_DIR}/input/exported_xml_reviewer/wordpress_pages.xml"
 INPUT_XML_POSTS: str = f"{IO_DIR}/input/exported_xml_reviewer/wordpress_posts.xml"
-INPUT_CONFLUENCE_HIERARCHY: str = ""
 INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt"
-INPUT_WHITELIST: str = ""
 INPUT_REQUESTED_LINKS: str = f"{IO_DIR}/input/exported_xml_reviewer/requested_links.csv"
 INPUT_KNOWN_OK_LINKS: str = f"{IO_DIR}/input/exported_xml_reviewer/known_ok_links.txt"
 
+# Optional inputs:
+INPUT_CONFLUENCE_HIERARCHY: str = ""
+# INPUT_CONFLUENCE_HIERARCHY: str = (
+#     f"{IO_DIR}/input/exported_xml_reviewer/hierarchical_outline.txt"
+# )
+INPUT_WHITELIST: str = ""
+# INPUT_WHITELIST: str = (
+#     f"{IO_DIR}/input/exported_xml_reviewer/whitelisted_web_pages.txt"
+# )
+
+# Outputs:
 OUTPUT_MARKDOWN_REPORT: str = (
     f"{IO_DIR}/output/exported_xml_reviewer/wordpress_sensitive_terms_report.md"
 )
@@ -1357,7 +1367,7 @@ def write_section(
             f.write(
                 "| Published item | known archived links | published item, wrong URL, but redirection working | link does not work | valid e3sm.org links |\n"
             )
-            f.write("| --- | --- | --- | --- | ---: |\n")
+            f.write("| --- | --- | --- | --- | --- |\n")
 
             archived_total = 0
             redirected_total = 0

e3sm_comms/page_reviewer/utils_base.py

@@ -354,20 +354,20 @@ def get_e3sm_url_status(e3sm_url: str) -> str:
     try:
         response = requests.get(e3sm_url, timeout=10)
         response.raise_for_status()  # Raises HTTPError for 4xx/5xx responses
-        e3sm_url_status = "link works not logged-in"
+        return "link works not logged-in"
     except requests.exceptions.Timeout:
-        e3sm_url_status = "link times out"
-    except requests.exceptions.RequestException as e:
-        error_message: str = f"{e}"
-        if error_message.startswith(
-            "503 Server Error: Service Temporarily Unavailable for url: https://e3sm.org"
-        ):
-            e3sm_url_status = "link not whitelisted"
-        else:
-            e3sm_url_status = "link raises RequestException"
+        return "link times out"
+    except requests.exceptions.HTTPError as e:
+        status_code = e.response.status_code if e.response is not None else None
+        if status_code == 503 and (
+            e.response.url if e.response is not None else ""
+        ).startswith("https://e3sm.org"):
+            return "link not whitelisted"
+        return "link raises RequestException"
+    except requests.exceptions.RequestException:
+        return "link raises RequestException"
     except Exception:
-        e3sm_url_status = "link raises Exception"
-    return e3sm_url_status
+        return "link raises Exception"
 
 
 # Debugging ###################################################################

e3sm_comms/page_reviewer/utils_website_reviewer.py

@@ -26,7 +26,7 @@ def write_results(config: Config, page: ConfluencePage):
         with open(
             f"{config.output_dir}hierarchical_outline.txt", "a", encoding="utf-8"
         ) as f:
-            f.write(f"{page.depth * "  "}{line_id}\n")
+            f.write(f"{page.depth * '  '}{line_id}\n")
 
     if "sensitive_terms" in config.requested_output:
         # Append if sensitive terms were found.

examples/review_terms.bash

@@ -1,13 +1,15 @@
 # Before running:
-# e3sm.org > CMP Settings > CMP Advanced Setup: copy the list of pages to /global/homes/f/forsyth/ez/e3sm-comms-io/input/e3sm_org_reviewer/web_pages.txt
+# WordPress: Tools > Export > export pages (wordpress_pages.xml) and posts (wordpress_posts.xml)
+# Copy those XMLs into /global/homes/f/forsyth/ez/e3sm-comms-io/input/e3sm_org_reviewer/
+# e3sm.org > CMP Settings > CMP Advanced Setup: copy the list of pages to /global/homes/f/forsyth/ez/e3sm-comms-io/input/e3sm_org_reviewer/whitelisted_web_pages.txt
 # Also confirm confluence_top_levels_partial.txt is the list of top levels you want to use, otherwise switch it out.
 
 IO_DIR=/global/homes/f/forsyth/ez/e3sm-comms-io
 
 echo "Count of top-level Confluence pages:"
 wc -l ${IO_DIR}/input/website_reviewer/confluence_top_levels_partial.txt # Excludes MODEL, RESEARCH, DATA
-echo "Count of whitelisted e3sm.org pages":
-wc -l ${IO_DIR}/input/e3sm_org_reviewer/web_pages.txt
+ echo "Count of whitelisted e3sm.org pages:"
+ wc -l ${IO_DIR}/input/e3sm_org_reviewer/whitelisted_web_pages.txt
 
 echo "Step 1. Review Confluence"
 echo "Note: this will require a Confluence login"
@@ -20,4 +22,4 @@ e3sm-comms-e3sm-org-reviewer
 echo "Output reports:"
 echo "1. ${IO_DIR}/output/e3sm_org_reviewer/path_report.md"
 echo "2. ${IO_DIR}/output/e3sm_org_reviewer/sensitive_terms.md"
-echo "3. ${IO_DIR}/output/e3sm_org_reviewer/action_items.md
+echo "3. ${IO_DIR}/output/e3sm_org_reviewer/action_items.md"

pyproject.toml

@@ -19,6 +19,7 @@ classifiers = [
 
 dependencies = [
     "beautifulsoup4",
+    "requests",
 ]
 
 [project.optional-dependencies]