diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml new file mode 100644 index 0000000000..743c3e68bb --- /dev/null +++ b/.github/workflows/test-python.yml @@ -0,0 +1,27 @@ +name: Python tests + +on: + pull_request: + branches: + - master + +permissions: {} + +jobs: + pytest: + permissions: + contents: read + runs-on: ubuntu-24.04 + env: + CI: true + steps: + - name: Setup Action + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: Setup Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.12' + - name: Install dev dependencies + run: python -m pip install --user -r requirements-dev.txt + - name: Run pytest + run: python -m pytest diff --git a/.gitignore b/.gitignore index ed549d5b13..63adaeab80 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ venv .claude/settings.local.json .claude/settings.json .claude/worktrees/ +# Python bytecode +__pycache__/ +*.pyc \ No newline at end of file diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000000..869558c790 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,10 @@ +[pytest] +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* +addopts = -v --tb=short --strict-markers +filterwarnings = + error + # The repo's scripts predate py3.10; tolerate missing annotations. + ignore::DeprecationWarning diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000000..f2873c2c16 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,6 @@ +# Test-only dependencies. Install with: +# pip install -r requirements-dev.txt +# +# Kept separate from requirements.txt so the runtime image for mkdocs/feedgen +# does not pull in pytest. +pytest>=7.0 diff --git a/scripts/Update_CheatSheets_Index.py b/scripts/Update_CheatSheets_Index.py index 5b837528a0..b8bac3b201 100644 --- a/scripts/Update_CheatSheets_Index.py +++ b/scripts/Update_CheatSheets_Index.py @@ -8,21 +8,69 @@ and is named "Index.md". """ import os +import sys from collections import OrderedDict +from typing import Dict, Iterable, List # Define utility functions -def extract_languages_snippet_provided(cheatsheet): - languages = [] - markers = ["javascript", "java", "csharp", "c", "cpp", "html", "xml", "python", - "ruby", "php", "json", "sql", "bash", "shell", "coldfusion", "perl", - "vbnet"] - with open("../cheatsheets/" + cheatsheet, encoding="utf8") as cs_file: - cs_content = cs_file.read().lower().replace(" ","") - for marker in markers: +_LANGUAGE_MARKERS = [ + "javascript", "java", "csharp", "c", "cpp", "html", "xml", "python", + "ruby", "php", "json", "sql", "bash", "shell", "coldfusion", "perl", + "vbnet", +] + + +def extract_languages_snippet_provided( + cheatsheet: str, + cheatsheets_dir: str = "../cheatsheets", +) -> List[str]: + """Detect the languages of code snippets in the given cheatsheet. + + Looks for fenced code blocks (```` ```language ````) whose language + tag is in the recognized list. The file is read in lowercase and with + spaces stripped so detection is case- and spacing-insensitive. + + Args: + cheatsheet: Filename of the cheatsheet within ``cheatsheets_dir``. + cheatsheets_dir: Directory containing the cheatsheet file. + + Returns: + A list of recognized language names with their first letter + capitalized, in the order they were detected. + """ + languages: List[str] = [] + with open( + os.path.join(cheatsheets_dir, cheatsheet), encoding="utf8" + ) as cs_file: + cs_content = cs_file.read().lower().replace(" ", "") + for marker in _LANGUAGE_MARKERS: if "```" + marker + "\n" in cs_content: languages.append(marker.capitalize()) return languages + +def group_by_letter(cheatsheets: Iterable[str]) -> "OrderedDict[str, List[str]]": + """Group cheatsheet filenames by their first letter (uppercased). + + Filenames are grouped by the uppercase form of their first character. + The result is an :class:`OrderedDict` sorted by letter, preserving + the input order of filenames within each letter group. + """ + index: Dict[str, List[str]] = {} + for cheatsheet in cheatsheets: + letter = cheatsheet[0].upper() + index.setdefault(letter, []).append(cheatsheet) + return OrderedDict(sorted(index.items())) + + +def clean_trailing_whitespace(file_path: str) -> None: + """Strip trailing whitespace from each line in the file (in place).""" + with open(file_path, "r", encoding="utf-8") as file: + cleaned_lines = [line.rstrip() + "\n" for line in file] + with open(file_path, "w", encoding="utf-8") as file: + file.writelines(cleaned_lines) + + # Define templates cs_md_link_template = "[%s](cheatsheets/%s)" language_md_link_template = "![%s](assets/Index_%s.svg)" @@ -31,59 +79,63 @@ def extract_languages_snippet_provided(cheatsheet): cs_count_template = "**%s** cheat sheets available." cs_index_title_template = "# Index Alphabetical\n\n" -# Scan all CS files -index = {} -cs_count = 0 -cheatsheets = [f.name for f in os.scandir("../cheatsheets") if f.is_file()] -for cheatsheet in cheatsheets: - letter = cheatsheet[0].upper() - if letter not in index: - index[letter] = [cheatsheet] - else: - index[letter].append(cheatsheet) - cs_count += 1 -index = OrderedDict(sorted(index.items())) - -# Generate the index file -with open("../Index.md", "w", encoding="utf-8") as index_file: - index_file.write(cs_index_title_template) - index_count = len(index) - index_file.write(cs_count_template % cs_count) - index_file.write("\n\n*Icons beside the cheat sheet name indicate in which language(s) code snippet(s) are provided.*") - index_file.write("\n\n") - # Generate the top menu - for letter in index: - index_file.write(top_menu_template % (letter, letter.lower())) - index_file.write(" ") - index_file.write("\n\n") - # Generate letter sections - j = 0 - for letter in index: - cs_count = len(index[letter]) - index_file.write(header_template % letter) - i = 0 - for cs_file in index[letter]: - cs_name = cs_file.replace("_", " ").replace(".md", "").strip() - index_file.write(cs_md_link_template % (cs_name, cs_file)) - languages = extract_languages_snippet_provided(cs_file) - if len(languages) > 0: - index_file.write(" ") - for language in languages: - index_file.write(language_md_link_template % (language, language)) + +def main( + cheatsheets_dir: str = "../cheatsheets", + output_file: str = "../Index.md", +) -> int: + """Regenerate the alphabetical index from the cheatsheets directory. + + Scans ``cheatsheets_dir`` for files, groups them by first letter, + detects code-snippet languages, and writes the index to + ``output_file``. Returns 0 on success. + """ + cheatsheets = [f.name for f in os.scandir(cheatsheets_dir) if f.is_file()] + index = group_by_letter(cheatsheets) + cs_count = len(cheatsheets) + + with open(output_file, "w", encoding="utf-8") as index_file: + index_file.write(cs_index_title_template) + index_file.write(cs_count_template % cs_count) + index_file.write( + "\n\n*Icons beside the cheat sheet name indicate in which " + "language(s) code snippet(s) are provided.*" + ) + index_file.write("\n\n") + # Generate the top menu + for letter in index: + index_file.write(top_menu_template % (letter, letter.lower())) + index_file.write(" ") + index_file.write("\n\n") + # Generate letter sections + index_count = len(index) + for j, letter in enumerate(index): + group = index[letter] + group_count = len(group) + index_file.write(header_template % letter) + for i, cs_file in enumerate(group): + cs_name = cs_file.replace("_", " ").replace(".md", "").strip() + index_file.write(cs_md_link_template % (cs_name, cs_file)) + languages = extract_languages_snippet_provided( + cs_file, cheatsheets_dir=cheatsheets_dir + ) + if languages: index_file.write(" ") - i += 1 - index_file.write("\n") - if i != cs_count: + for language in languages: + index_file.write( + language_md_link_template % (language, language) + ) + index_file.write(" ") + index_file.write("\n") + if i + 1 != group_count: + index_file.write("\n") + if j + 1 != index_count: index_file.write("\n") - j += 1 - if j != index_count: - index_file.write("\n") -# Clean trailing whitespaces -with open("../Index.md", "r", encoding="utf-8") as file: - cleaned_lines = [line.rstrip() + "\n" for line in file] + clean_trailing_whitespace(output_file) + print("Index updated.") + return 0 -with open("../Index.md", "w", encoding="utf-8") as file: - file.writelines(cleaned_lines) -print("Index updated.") +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000000..4e9f855aaf --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,32 @@ +"""Shared pytest fixtures for the scripts/ test suite.""" +from __future__ import annotations + +from pathlib import Path + +import pytest + + +@pytest.fixture +def cheatsheets_dir(tmp_path: Path) -> Path: + """Return a fresh, empty cheatsheets/ directory inside a temp path. + + Tests that need sample cheatsheet files should write them into this + directory before invoking the script under test. + """ + d = tmp_path / "cheatsheets" + d.mkdir() + return d + + +@pytest.fixture +def write_cheatsheet(cheatsheets_dir: Path): + """Return a callable that writes a cheatsheet file with optional content. + + Usage: + write_cheatsheet("Foo.md", "# Foo\\n\\n```python\\nprint('x')\\n```") + """ + def _write(name: str, content: str = "") -> Path: + path = cheatsheets_dir / name + path.write_text(content, encoding="utf-8") + return path + return _write diff --git a/tests/test_update_cheatsheets_index.py b/tests/test_update_cheatsheets_index.py new file mode 100644 index 0000000000..b8cc30a482 --- /dev/null +++ b/tests/test_update_cheatsheets_index.py @@ -0,0 +1,193 @@ +"""Tests for scripts/Update_CheatSheets_Index.py.""" +from __future__ import annotations + +import sys +from collections import OrderedDict +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import Update_CheatSheets_Index as idx # noqa: E402 + + +class TestExtractLanguagesSnippetProvided: + def test_detects_javascript(self, cheatsheets_dir, write_cheatsheet): + write_cheatsheet( + "XSS_Cheat_Sheet.md", + "# XSS\n\n```javascript\nalert(1);\n```\n", + ) + assert idx.extract_languages_snippet_provided( + "XSS_Cheat_Sheet.md", cheatsheets_dir=str(cheatsheets_dir) + ) == ["Javascript"] + + def test_detects_multiple_languages_in_order_of_marker_list( + self, cheatsheets_dir, write_cheatsheet + ): + # Marker list order is: javascript, java, csharp, c, ... + # SQL and Python appear later; verify they are returned after + # the earlier markers in the list. + write_cheatsheet( + "Multi.md", + "```sql\nSELECT 1;\n```\n```python\nprint(1)\n```\n", + ) + result = idx.extract_languages_snippet_provided( + "Multi.md", cheatsheets_dir=str(cheatsheets_dir) + ) + assert result == ["Python", "Sql"] + + def test_returns_empty_list_for_file_without_code_blocks( + self, cheatsheets_dir, write_cheatsheet + ): + write_cheatsheet("Plain.md", "# Just headings\n\nNo code here.\n") + assert idx.extract_languages_snippet_provided( + "Plain.md", cheatsheets_dir=str(cheatsheets_dir) + ) == [] + + def test_ignores_unrecognized_languages( + self, cheatsheets_dir, write_cheatsheet + ): + # ``rust`` is not in the marker list — must not be detected. + write_cheatsheet( + "Unrecognized.md", "```rust\nfn main() {}\n```\n```python\nprint(1)\n```\n" + ) + result = idx.extract_languages_snippet_provided( + "Unrecognized.md", cheatsheets_dir=str(cheatsheets_dir) + ) + assert "Rust" not in result + assert "Python" in result + + def test_detection_is_case_and_space_insensitive( + self, cheatsheets_dir, write_cheatsheet + ): + # The implementation lowercases content and strips spaces, so + # `` ```JavaScript\n `` (no space) is detected the same as + # `` ```Java Script\n `` (with space). Verify both work. + write_cheatsheet("A.md", "```JavaScript\nx\n```\n") + write_cheatsheet("B.md", "```Java Script\nx\n```\n") + assert idx.extract_languages_snippet_provided( + "A.md", cheatsheets_dir=str(cheatsheets_dir) + ) == ["Javascript"] + assert idx.extract_languages_snippet_provided( + "B.md", cheatsheets_dir=str(cheatsheets_dir) + ) == ["Javascript"] + + +class TestGroupByLetter: + def test_groups_by_uppercased_first_letter(self): + result = idx.group_by_letter(["alpha.md", "beta.md", "Alpha2.md"]) + assert "A" in result + assert "B" in result + assert result["A"] == ["alpha.md", "Alpha2.md"] + assert result["B"] == ["beta.md"] + + def test_returns_ordered_dict_sorted_by_letter(self): + result = idx.group_by_letter(["zebra.md", "apple.md", "mango.md"]) + assert isinstance(result, OrderedDict) + assert list(result.keys()) == ["A", "M", "Z"] + + def test_preserves_input_order_within_a_letter_group(self): + files = ["b2.md", "b1.md", "b3.md"] + result = idx.group_by_letter(files) + assert result["B"] == files + + def test_empty_input_returns_empty_ordered_dict(self): + result = idx.group_by_letter([]) + assert list(result.keys()) == [] + + +class TestCleanTrailingWhitespace: + def test_strips_trailing_whitespace_from_each_line(self, tmp_path): + f = tmp_path / "with_trailing.md" + f.write_text("line 1 \nline 2\t\nline 3\n", encoding="utf-8") + idx.clean_trailing_whitespace(str(f)) + # After rstrip+"\n", trailing whitespace is gone but the + # newline itself is preserved on every line. + assert f.read_text(encoding="utf-8") == "line 1\nline 2\nline 3\n" + + def test_handles_file_with_no_trailing_whitespace(self, tmp_path): + f = tmp_path / "clean.md" + f.write_text("a\nb\nc\n", encoding="utf-8") + idx.clean_trailing_whitespace(str(f)) + assert f.read_text(encoding="utf-8") == "a\nb\nc\n" + + +class TestMain: + def test_creates_index_file_with_title_and_count( + self, cheatsheets_dir: Path, tmp_path: Path, write_cheatsheet + ): + write_cheatsheet("Authentication_Cheat_Sheet.md") + write_cheatsheet("XSS_Prevention_Cheat_Sheet.md") + write_cheatsheet("Docker_Security.md") + + output = tmp_path / "Index.md" + rc = idx.main( + cheatsheets_dir=str(cheatsheets_dir), + output_file=str(output), + ) + assert rc == 0 + content = output.read_text(encoding="utf-8") + assert content.startswith("# Index Alphabetical\n\n") + assert "**3** cheat sheets available." in content + + def test_groups_cheatsheets_by_letter_section( + self, cheatsheets_dir: Path, tmp_path: Path, write_cheatsheet + ): + write_cheatsheet("Authentication_Cheat_Sheet.md") + write_cheatsheet("XSS_Prevention_Cheat_Sheet.md") + write_cheatsheet("Docker_Security.md") + + output = tmp_path / "Index.md" + idx.main( + cheatsheets_dir=str(cheatsheets_dir), + output_file=str(output), + ) + content = output.read_text(encoding="utf-8") + assert "## A\n" in content + assert "## D\n" in content + assert "## X\n" in content + + def test_includes_language_icons_when_code_blocks_present( + self, cheatsheets_dir: Path, tmp_path: Path, write_cheatsheet + ): + write_cheatsheet( + "XSS_Prevention_Cheat_Sheet.md", + "# XSS\n\n```javascript\nalert(1);\n```\n", + ) + + output = tmp_path / "Index.md" + idx.main( + cheatsheets_dir=str(cheatsheets_dir), + output_file=str(output), + ) + content = output.read_text(encoding="utf-8") + assert "![Javascript](assets/Index_Javascript.svg)" in content + + def test_omits_language_icons_when_no_code_blocks( + self, cheatsheets_dir: Path, tmp_path: Path, write_cheatsheet + ): + write_cheatsheet("Plain_Cheat_Sheet.md", "# Plain\n\nNo code.\n") + + output = tmp_path / "Index.md" + idx.main( + cheatsheets_dir=str(cheatsheets_dir), + output_file=str(output), + ) + content = output.read_text(encoding="utf-8") + assert "assets/Index_" not in content + + def test_output_has_no_trailing_whitespace( + self, cheatsheets_dir: Path, tmp_path: Path, write_cheatsheet + ): + # The original script ends with a clean_trailing_whitespace + # step; verify that step is still executed in main(). + write_cheatsheet("Foo.md", "# Foo\n") + + output = tmp_path / "Index.md" + idx.main( + cheatsheets_dir=str(cheatsheets_dir), + output_file=str(output), + ) + for line in output.read_text(encoding="utf-8").splitlines(): + assert line == line.rstrip(), f"line has trailing whitespace: {line!r}"