test_quality_checker.py

#!/usr/bin/env python3
"""
Tests for cli/quality_checker.py functionality
"""

import tempfile
import unittest
from pathlib import Path

from skill_seekers.cli.quality_checker import QualityReport, SkillQualityChecker


class TestQualityChecker(unittest.TestCase):
    """Test quality checker functionality"""

    def create_test_skill(self, tmpdir, skill_md_content, create_references=True):
        """Helper to create a test skill directory"""
        skill_dir = Path(tmpdir) / "test-skill"
        skill_dir.mkdir()

        # Create SKILL.md
        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text(skill_md_content, encoding="utf-8")

        # Create references directory
        if create_references:
            refs_dir = skill_dir / "references"
            refs_dir.mkdir()
            (refs_dir / "index.md").write_text("# Index\n\nTest reference.", encoding="utf-8")
            (refs_dir / "getting_started.md").write_text(
                "# Getting Started\n\nHow to start.", encoding="utf-8"
            )

        return skill_dir

    def test_checker_detects_missing_skill_md(self):
        """Test that checker detects missing SKILL.md"""
        with tempfile.TemporaryDirectory() as tmpdir:
            skill_dir = Path(tmpdir) / "test-skill"
            skill_dir.mkdir()

            checker = SkillQualityChecker(skill_dir)
            report = checker.check_all()

            # Should have error about missing SKILL.md
            self.assertTrue(report.has_errors)
            self.assertTrue(any("SKILL.md" in issue.message for issue in report.errors))

    def test_checker_detects_missing_references(self):
        """Test that checker warns about missing references"""
        with tempfile.TemporaryDirectory() as tmpdir:
            skill_md = """---
name: test
---

# Test Skill

This is a test.
"""
            skill_dir = self.create_test_skill(tmpdir, skill_md, create_references=False)

            checker = SkillQualityChecker(skill_dir)
            report = checker.check_all()

            # Should have warning about missing references
            self.assertTrue(report.has_warnings)
            self.assertTrue(any("references" in issue.message.lower() for issue in report.warnings))

    def test_checker_detects_invalid_frontmatter(self):
        """Test that checker detects invalid YAML frontmatter"""
        with tempfile.TemporaryDirectory() as tmpdir:
            skill_md = """# Test Skill

No frontmatter here!
"""
            skill_dir = self.create_test_skill(tmpdir, skill_md)

            checker = SkillQualityChecker(skill_dir)
            report = checker.check_all()

            # Should have error about missing frontmatter
            self.assertTrue(report.has_errors)
            self.assertTrue(any("frontmatter" in issue.message.lower() for issue in report.errors))

    def test_checker_detects_missing_name_field(self):
        """Test that checker detects missing name field in frontmatter"""
        with tempfile.TemporaryDirectory() as tmpdir:
            skill_md = """---
description: test
---

# Test Skill
"""
            skill_dir = self.create_test_skill(tmpdir, skill_md)

            checker = SkillQualityChecker(skill_dir)
            report = checker.check_all()

            # Should have error about missing name field
            self.assertTrue(report.has_errors)
            self.assertTrue(any("name" in issue.message.lower() for issue in report.errors))

    def test_checker_detects_code_without_language(self):
        """Test that checker warns about code blocks without language tags"""
        with tempfile.TemporaryDirectory() as tmpdir:
            skill_md = """---
name: test
---

# Test Skill

Here's some code:

```
print("hello")
```
"""
            skill_dir = self.create_test_skill(tmpdir, skill_md)

            checker = SkillQualityChecker(skill_dir)
            report = checker.check_all()

            # Should have warning about code without language
            self.assertTrue(report.has_warnings)
            self.assertTrue(any("language" in issue.message.lower() for issue in report.warnings))

    def test_checker_approves_good_skill(self):
        """Test that checker gives high score to well-formed skill"""
        with tempfile.TemporaryDirectory() as tmpdir:
            skill_md = """---
name: test
description: A test skill
---

# Test Skill

## When to Use This Skill

Use this when you need to test.

## Quick Reference

Here are some examples:

```python
def hello():
    print("hello")
```

```javascript
console.log("hello");
```

## Example: Basic Usage

This shows how to use it.

## Reference Files

See the references directory for more:
- [Getting Started](references/getting_started.md)
- [Index](references/index.md)
"""
            skill_dir = self.create_test_skill(tmpdir, skill_md)

            checker = SkillQualityChecker(skill_dir)
            report = checker.check_all()

            # Should have no errors
            self.assertFalse(report.has_errors)

            # Quality score should be high
            self.assertGreaterEqual(report.quality_score, 80.0)

    def test_checker_detects_broken_links(self):
        """Test that checker detects broken internal links"""
        with tempfile.TemporaryDirectory() as tmpdir:
            skill_md = """---
name: test
---

# Test Skill

See [this file](nonexistent.md) for more info.
"""
            skill_dir = self.create_test_skill(tmpdir, skill_md)

            checker = SkillQualityChecker(skill_dir)
            report = checker.check_all()

            # Should have warning about broken link
            self.assertTrue(report.has_warnings)
            self.assertTrue(
                any("broken link" in issue.message.lower() for issue in report.warnings)
            )

    def test_quality_score_calculation(self):
        """Test that quality score is calculated correctly"""
        with tempfile.TemporaryDirectory() as tmpdir:
            report = QualityReport("test", Path(tmpdir))

            # Perfect score to start
            self.assertEqual(report.quality_score, 100.0)

            # Add an error (should deduct 15 points)
            report.add_error("test", "Test error")
            self.assertEqual(report.quality_score, 85.0)

            # Add a warning (should deduct 5 points)
            report.add_warning("test", "Test warning")
            self.assertEqual(report.quality_score, 80.0)

            # Add more errors
            report.add_error("test", "Another error")
            report.add_error("test", "Yet another error")
            self.assertEqual(report.quality_score, 50.0)

    def test_quality_grade_calculation(self):
        """Test that quality grades are assigned correctly"""
        with tempfile.TemporaryDirectory() as tmpdir:
            report = QualityReport("test", Path(tmpdir))

            # Grade A (90-100)
            self.assertEqual(report.quality_grade, "A")

            # Grade B (80-89)
            report.add_error("test", "Error 1")
            self.assertEqual(report.quality_grade, "B")

            # Grade C (70-79)
            report.add_warning("test", "Warning 1")
            report.add_warning("test", "Warning 2")
            self.assertEqual(report.quality_grade, "C")

            # Grade D (60-69)
            report.add_warning("test", "Warning 3")
            report.add_warning("test", "Warning 4")
            self.assertEqual(report.quality_grade, "D")

            # Grade F (below 60)
            report.add_error("test", "Error 2")
            report.add_error("test", "Error 3")
            self.assertEqual(report.quality_grade, "F")

    def test_is_excellent_property(self):
        """Test is_excellent property"""
        with tempfile.TemporaryDirectory() as tmpdir:
            report = QualityReport("test", Path(tmpdir))

            # Should be excellent with no issues
            self.assertTrue(report.is_excellent)

            # Adding an error should make it not excellent
            report.add_error("test", "Test error")
            self.assertFalse(report.is_excellent)

            # Clean report
            report2 = QualityReport("test", Path(tmpdir))
            # Adding a warning should also make it not excellent
            report2.add_warning("test", "Test warning")
            self.assertFalse(report2.is_excellent)


class TestCompletenessChecks(unittest.TestCase):
    """Test completeness check functionality"""

    def create_test_skill(self, tmpdir, skill_md_content):
        """Helper to create a test skill directory"""
        skill_dir = Path(tmpdir) / "test-skill"
        skill_dir.mkdir()

        # Create SKILL.md
        skill_md = skill_dir / "SKILL.md"
        skill_md.write_text(skill_md_content, encoding="utf-8")

        # Create references directory
        refs_dir = skill_dir / "references"
        refs_dir.mkdir()
        (refs_dir / "index.md").write_text("# Index\n", encoding="utf-8")

        return skill_dir

    def test_checker_detects_prerequisites_section(self):
        """Test that checker detects prerequisites section"""
        with tempfile.TemporaryDirectory() as tmpdir:
            skill_md = """---
name: test
---

# Test Skill

## Prerequisites

Make sure you have:
- Python 3.10+
- pip installed

## Usage

Run the command.
"""
            skill_dir = self.create_test_skill(tmpdir, skill_md)

            checker = SkillQualityChecker(skill_dir)
            report = checker.check_all()

            # Should have info about found prerequisites
            completeness_infos = [i for i in report.info if i.category == "completeness"]
            self.assertTrue(
                any(
                    "prerequisites" in i.message.lower() or "verification" in i.message.lower()
                    for i in completeness_infos
                )
            )

    def test_checker_detects_troubleshooting_section(self):
        """Test that checker detects troubleshooting section"""
        with tempfile.TemporaryDirectory() as tmpdir:
            skill_md = """---
name: test
---

# Test Skill

## Usage

Run the command.

## Troubleshooting

### Common Issues

If the command fails, check your permissions.
"""
            skill_dir = self.create_test_skill(tmpdir, skill_md)

            checker = SkillQualityChecker(skill_dir)
            report = checker.check_all()

            # Should have info about found troubleshooting
            completeness_infos = [i for i in report.info if i.category == "completeness"]
            self.assertTrue(
                any(
                    "troubleshoot" in i.message.lower() or "error handling" in i.message.lower()
                    for i in completeness_infos
                )
            )

    def test_checker_detects_workflow_steps(self):
        """Test that checker detects workflow steps"""
        with tempfile.TemporaryDirectory() as tmpdir:
            skill_md = """---
name: test
---

# Test Skill

## Getting Started

First, install the dependencies.

Then, configure your environment.

Next, run the setup script.

Finally, verify the installation.
"""
            skill_dir = self.create_test_skill(tmpdir, skill_md)

            checker = SkillQualityChecker(skill_dir)
            report = checker.check_all()

            # Should have info about found workflow steps
            completeness_infos = [i for i in report.info if i.category == "completeness"]
            self.assertTrue(
                any(
                    "workflow" in i.message.lower() or "step" in i.message.lower()
                    for i in completeness_infos
                )
            )

    def test_checker_suggests_adding_prerequisites(self):
        """Test that checker suggests adding prerequisites when missing"""
        with tempfile.TemporaryDirectory() as tmpdir:
            skill_md = """---
name: test
---

# Test Skill

## Usage

Just run the command.
"""
            skill_dir = self.create_test_skill(tmpdir, skill_md)

            checker = SkillQualityChecker(skill_dir)
            report = checker.check_all()

            # Should have info suggesting prerequisites
            completeness_infos = [i for i in report.info if i.category == "completeness"]
            self.assertTrue(
                any(
                    "consider" in i.message.lower() and "prerequisites" in i.message.lower()
                    for i in completeness_infos
                )
            )


class TestQualityCheckerCLI(unittest.TestCase):
    """Test quality checker CLI"""

    def test_cli_help_output(self):
        """Test that CLI help works"""
        import subprocess

        try:
            result = subprocess.run(
                ["python3", "-m", "skill_seekers.cli.quality_checker", "--help"],
                capture_output=True,
                text=True,
                timeout=5,
            )

            # Should include usage info
            output = result.stdout + result.stderr
            self.assertTrue("usage:" in output.lower() or "quality" in output.lower())
        except FileNotFoundError:
            self.skipTest("Module not installed")

    def test_cli_with_nonexistent_directory(self):
        """Test CLI behavior with nonexistent directory"""
        import subprocess

        result = subprocess.run(
            ["python3", "-m", "skill_seekers.cli.quality_checker", "/nonexistent/path"],
            capture_output=True,
            text=True,
        )

        # Should fail
        self.assertNotEqual(result.returncode, 0)


if __name__ == "__main__":
    unittest.main()