diff --git a/application/tests/harvester_test/__init__.py b/application/tests/harvester_test/__init__.py new file mode 100644 index 000000000..06b0f7440 --- /dev/null +++ b/application/tests/harvester_test/__init__.py @@ -0,0 +1,3 @@ +""" +tests for Module A configuration layer (empty for now) +""" diff --git a/application/tests/harvester_test/config_loader_test.py b/application/tests/harvester_test/config_loader_test.py new file mode 100644 index 000000000..730f4d708 --- /dev/null +++ b/application/tests/harvester_test/config_loader_test.py @@ -0,0 +1,122 @@ +from pathlib import Path +import unittest + +from application.utils.harvester.config_loader import ( + ConfigLoaderError, + ConfigFileNotFoundError, + load_repo_config, +) + +FIXTURES_DIR = Path(__file__).parent / "fixtures" + + +class ConfigLoaderTests(unittest.TestCase): + def test_load_valid_config(self): + config_path = FIXTURES_DIR / "valid_repos.yaml" + + config = load_repo_config(config_path) + + self.assertEqual(len(config.repositories), 1) + + repo = config.repositories[0] + + self.assertEqual(repo.id, "owasp-asvs") + self.assertEqual(repo.owner, "OWASP") + self.assertEqual(repo.repo, "ASVS") + + def test_missing_repository_id(self): + config_path = FIXTURES_DIR / "invalid_missing_id.yaml" + + with self.assertRaises(ConfigLoaderError): + load_repo_config(config_path) + + def test_invalid_chunk_size(self): + config_path = FIXTURES_DIR / "invalid_chunk_size.yaml" + + with self.assertRaisesRegex(ConfigLoaderError, "max_tokens"): + load_repo_config(config_path) + + def test_invalid_yaml_syntax(self): + config_path = FIXTURES_DIR / "invalid_yaml.yaml" + + with self.assertRaises(ConfigLoaderError): + load_repo_config(config_path) + + def test_missing_config_file(self): + with self.assertRaises(ConfigFileNotFoundError): + load_repo_config("does_not_exist.yaml") + + def test_invalid_polling_interval(self): + config_path = FIXTURES_DIR / "invalid_polling_interval.yaml" + + with self.assertRaises(ConfigLoaderError): + load_repo_config(config_path) + + def test_empty_include_paths(self): + config_path = FIXTURES_DIR / "empty_include_paths.yaml" + + with self.assertRaises(ConfigLoaderError): + load_repo_config(config_path) + + def test_whitespace_only_repository_id(self): + config_path = FIXTURES_DIR / "whitespace_id.yaml" + + with self.assertRaises(ConfigLoaderError): + load_repo_config(config_path) + + def test_whitespace_only_owner(self): + config_path = FIXTURES_DIR / "whitespace_owner.yaml" + + with self.assertRaises(ConfigLoaderError): + load_repo_config(config_path) + + def test_whitespace_only_repo(self): + config_path = FIXTURES_DIR / "whitespace_repo.yaml" + + with self.assertRaises(ConfigLoaderError): + load_repo_config(config_path) + + def test_whitespace_only_branch(self): + config_path = FIXTURES_DIR / "whitespace_branch.yaml" + + with self.assertRaises(ConfigLoaderError): + load_repo_config(config_path) + + def test_overlap_tokens_must_be_less_than_max_tokens(self): + config_path = FIXTURES_DIR / "invalid_overlap_tokens.yaml" + + with self.assertRaisesRegex(ConfigLoaderError, "overlap_tokens"): + load_repo_config(config_path) + + def test_load_packaged_repos_yaml(self): + config_path = ( + Path(__file__).resolve().parents[2] / "utils" / "harvester" / "repos.yaml" + ) + + config = load_repo_config(config_path) + + self.assertEqual(len(config.repositories), 2) + self.assertEqual(config.repositories[0].id, "owasp-asvs") + self.assertEqual(config.repositories[1].id, "owasp-cheatsheets") + + def test_empty_owner(self): + config_path = FIXTURES_DIR / "empty_owner.yaml" + + with self.assertRaises(ConfigLoaderError): + load_repo_config(config_path) + + def test_invalid_chunking_strategy(self): + config_path = FIXTURES_DIR / "invalid_chunking_strategy.yaml" + + with self.assertRaises(ConfigLoaderError): + load_repo_config(config_path) + + def test_invalid_polling_mode(self): + config_path = FIXTURES_DIR / "invalid_polling_mode.yaml" + + with self.assertRaises(ConfigLoaderError): + load_repo_config(config_path) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/fixtures/duplicate_include_paths.yaml b/application/tests/harvester_test/fixtures/duplicate_include_paths.yaml new file mode 100644 index 000000000..5db642199 --- /dev/null +++ b/application/tests/harvester_test/fixtures/duplicate_include_paths.yaml @@ -0,0 +1,22 @@ +repositories: + - id: duplicate-includes + type: github + enabled: true + + owner: OWASP + repo: ASVS + branch: master + + paths: + include: + - "docs/**/*.md" + - "docs/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + overlap_tokens: 100 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/duplicate_repo_ids.yaml b/application/tests/harvester_test/fixtures/duplicate_repo_ids.yaml new file mode 100644 index 000000000..0cc4c1c1e --- /dev/null +++ b/application/tests/harvester_test/fixtures/duplicate_repo_ids.yaml @@ -0,0 +1,34 @@ +repositories: + - id: asvs + type: github + owner: OWASP + repo: ASVS + + paths: + include: + - "4.0/en/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 60 + + - id: asvs + type: github + owner: OWASP + repo: CheatSheetSeries + + paths: + include: + - "cheatsheets/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/duplicate_repo_ids_case.yaml b/application/tests/harvester_test/fixtures/duplicate_repo_ids_case.yaml new file mode 100644 index 000000000..10df90943 --- /dev/null +++ b/application/tests/harvester_test/fixtures/duplicate_repo_ids_case.yaml @@ -0,0 +1,34 @@ +repositories: + - id: asvs + type: github + owner: OWASP + repo: ASVS + + paths: + include: + - "4.0/en/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 60 + + - id: ASVS + type: github + owner: OWASP + repo: CheatSheetSeries + + paths: + include: + - "cheatsheets/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/duplicate_repositories.yaml b/application/tests/harvester_test/fixtures/duplicate_repositories.yaml new file mode 100644 index 000000000..41614d39c --- /dev/null +++ b/application/tests/harvester_test/fixtures/duplicate_repositories.yaml @@ -0,0 +1,34 @@ +repositories: + - id: asvs-1 + type: github + owner: OWASP + repo: ASVS + + paths: + include: + - "4.0/en/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 60 + + - id: asvs-2 + type: github + owner: OWASP + repo: ASVS + + paths: + include: + - "4.0/en/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/duplicate_repositories_case.yaml b/application/tests/harvester_test/fixtures/duplicate_repositories_case.yaml new file mode 100644 index 000000000..ed9dd64d3 --- /dev/null +++ b/application/tests/harvester_test/fixtures/duplicate_repositories_case.yaml @@ -0,0 +1,34 @@ +repositories: + - id: asvs-1 + type: github + owner: OWASP + repo: ASVS + + paths: + include: + - "4.0/en/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 60 + + - id: asvs-2 + type: github + owner: owasp + repo: asvs + + paths: + include: + - "4.0/en/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/empty_include_paths.yaml b/application/tests/harvester_test/fixtures/empty_include_paths.yaml new file mode 100644 index 000000000..abe0bab41 --- /dev/null +++ b/application/tests/harvester_test/fixtures/empty_include_paths.yaml @@ -0,0 +1,16 @@ +repositories: + - id: asvs + type: github + owner: OWASP + repo: ASVS + + paths: + include: [] + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/empty_owner.yaml b/application/tests/harvester_test/fixtures/empty_owner.yaml new file mode 100644 index 000000000..8063a3f1b --- /dev/null +++ b/application/tests/harvester_test/fixtures/empty_owner.yaml @@ -0,0 +1,21 @@ +repositories: + - id: empty-owner + type: github + enabled: true + + owner: "" + repo: ASVS + branch: master + + paths: + include: + - "docs/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + overlap_tokens: 100 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/invalid_chunk_size.yaml b/application/tests/harvester_test/fixtures/invalid_chunk_size.yaml new file mode 100644 index 000000000..79d4cc8f0 --- /dev/null +++ b/application/tests/harvester_test/fixtures/invalid_chunk_size.yaml @@ -0,0 +1,18 @@ +repositories: + - id: owasp-asvs + + type: github + owner: OWASP + repo: ASVS + + paths: + include: + - "4.0/en/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 0 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/invalid_chunking_strategy.yaml b/application/tests/harvester_test/fixtures/invalid_chunking_strategy.yaml new file mode 100644 index 000000000..c4cb10905 --- /dev/null +++ b/application/tests/harvester_test/fixtures/invalid_chunking_strategy.yaml @@ -0,0 +1,21 @@ +repositories: + - id: invalid-strategy + type: github + enabled: true + + owner: OWASP + repo: ASVS + branch: master + + paths: + include: + - "docs/**/*.md" + + chunking: + strategy: invalid_strategy + max_tokens: 1200 + overlap_tokens: 100 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/invalid_missing_id.yaml b/application/tests/harvester_test/fixtures/invalid_missing_id.yaml new file mode 100644 index 000000000..102e675b9 --- /dev/null +++ b/application/tests/harvester_test/fixtures/invalid_missing_id.yaml @@ -0,0 +1,17 @@ +repositories: + - type: github + + owner: OWASP + repo: ASVS + + paths: + include: + - "4.0/en/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/invalid_overlap_tokens.yaml b/application/tests/harvester_test/fixtures/invalid_overlap_tokens.yaml new file mode 100644 index 000000000..36d64b11c --- /dev/null +++ b/application/tests/harvester_test/fixtures/invalid_overlap_tokens.yaml @@ -0,0 +1,16 @@ +repositories: + - id: invalid-overlap + type: github + owner: OWASP + repo: ASVS + branch: master + paths: + include: + - "docs/**/*.md" + chunking: + strategy: markdown_heading + max_tokens: 100 + overlap_tokens: 100 + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/invalid_polling_interval.yaml b/application/tests/harvester_test/fixtures/invalid_polling_interval.yaml new file mode 100644 index 000000000..3f34f8baa --- /dev/null +++ b/application/tests/harvester_test/fixtures/invalid_polling_interval.yaml @@ -0,0 +1,17 @@ +repositories: + - id: asvs + type: github + owner: OWASP + repo: ASVS + + paths: + include: + - "4.0/en/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 0 diff --git a/application/tests/harvester_test/fixtures/invalid_polling_mode.yaml b/application/tests/harvester_test/fixtures/invalid_polling_mode.yaml new file mode 100644 index 000000000..0aead47ad --- /dev/null +++ b/application/tests/harvester_test/fixtures/invalid_polling_mode.yaml @@ -0,0 +1,21 @@ +repositories: + - id: invalid-polling + type: github + enabled: true + + owner: OWASP + repo: ASVS + branch: master + + paths: + include: + - "docs/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + overlap_tokens: 100 + + polling: + mode: realtime + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/invalid_yaml.yaml b/application/tests/harvester_test/fixtures/invalid_yaml.yaml new file mode 100644 index 000000000..c60d3e23b --- /dev/null +++ b/application/tests/harvester_test/fixtures/invalid_yaml.yaml @@ -0,0 +1,4 @@ +repositories: + - id: broken + paths: + include: [unclosed diff --git a/application/tests/harvester_test/fixtures/valid_repos.yaml b/application/tests/harvester_test/fixtures/valid_repos.yaml new file mode 100644 index 000000000..fe52c8430 --- /dev/null +++ b/application/tests/harvester_test/fixtures/valid_repos.yaml @@ -0,0 +1,16 @@ +repositories: + - id: owasp-asvs + type: github + owner: OWASP + repo: ASVS + paths: + include: + - "4.0/en/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/whitespace_branch.yaml b/application/tests/harvester_test/fixtures/whitespace_branch.yaml new file mode 100644 index 000000000..35dbdf578 --- /dev/null +++ b/application/tests/harvester_test/fixtures/whitespace_branch.yaml @@ -0,0 +1,16 @@ +repositories: + - id: whitespace-branch + type: github + owner: OWASP + repo: ASVS + branch: " " + paths: + include: + - "docs/**/*.md" + chunking: + strategy: markdown_heading + max_tokens: 1200 + overlap_tokens: 100 + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/whitespace_id.yaml b/application/tests/harvester_test/fixtures/whitespace_id.yaml new file mode 100644 index 000000000..0a5d00175 --- /dev/null +++ b/application/tests/harvester_test/fixtures/whitespace_id.yaml @@ -0,0 +1,16 @@ +repositories: + - id: " " + type: github + owner: OWASP + repo: ASVS + branch: master + paths: + include: + - "docs/**/*.md" + chunking: + strategy: markdown_heading + max_tokens: 1200 + overlap_tokens: 100 + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/whitespace_owner.yaml b/application/tests/harvester_test/fixtures/whitespace_owner.yaml new file mode 100644 index 000000000..112ef72e8 --- /dev/null +++ b/application/tests/harvester_test/fixtures/whitespace_owner.yaml @@ -0,0 +1,16 @@ +repositories: + - id: whitespace-owner + type: github + owner: " " + repo: ASVS + branch: master + paths: + include: + - "docs/**/*.md" + chunking: + strategy: markdown_heading + max_tokens: 1200 + overlap_tokens: 100 + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/whitespace_repo.yaml b/application/tests/harvester_test/fixtures/whitespace_repo.yaml new file mode 100644 index 000000000..772da966c --- /dev/null +++ b/application/tests/harvester_test/fixtures/whitespace_repo.yaml @@ -0,0 +1,16 @@ +repositories: + - id: whitespace-repo + type: github + owner: OWASP + repo: " " + branch: master + paths: + include: + - "docs/**/*.md" + chunking: + strategy: markdown_heading + max_tokens: 1200 + overlap_tokens: 100 + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/git_repository_client_integration_test.py b/application/tests/harvester_test/git_repository_client_integration_test.py new file mode 100644 index 000000000..4803e1d67 --- /dev/null +++ b/application/tests/harvester_test/git_repository_client_integration_test.py @@ -0,0 +1,206 @@ +import subprocess +import tempfile +import unittest +from pathlib import Path +import threading + +from application.utils.harvester.git_repository_client import ( + GitRepositoryClient, +) + + +class IntegrationGitRepositoryClient(GitRepositoryClient): + def __init__(self, *args, repository_url: str, **kwargs): + super().__init__(*args, **kwargs) + self._repository_url = repository_url + + @property + def repository_url(self) -> str: + return self._repository_url + + +def git(*args, cwd=None): + subprocess.run( + ["git", *args], + cwd=cwd, + check=True, + capture_output=True, + text=True, + ) + + +def git_output(*args, cwd=None): + return subprocess.run( + ["git", *args], + cwd=cwd, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +class GitRepositoryClientIntegrationTests(unittest.TestCase): + def setUp(self): + self.tempdir = tempfile.TemporaryDirectory() + self.root = Path(self.tempdir.name) + + self.remote = self.root / "remote.git" + self.work = self.root / "work" + self.cache = self.root / "cache" + + git("init", "--bare", self.remote) + + git("clone", self.remote, self.work) + + git("config", "user.name", "Test User", cwd=self.work) + git("config", "user.email", "test@example.com", cwd=self.work) + git("checkout", "-b", "main", cwd=self.work) + + (self.work / "test.txt").write_text("v1") + + git("add", ".", cwd=self.work) + git("commit", "-m", "initial", cwd=self.work) + git("push", "origin", "main", cwd=self.work) + + def tearDown(self): + self.tempdir.cleanup() + + def create_client(self): + return IntegrationGitRepositoryClient( + owner="OWASP", + repository="ASVS", + local_path=self.cache, + repository_url=str(self.remote), + ) + + def test_fetch_updates_worktree_and_commit(self): + client = self.create_client() + + client.clone() + + sha1 = client.get_current_commit_sha() + + self.assertEqual( + (client.get_local_path() / "test.txt").read_text(), + "v1", + ) + + (self.work / "test.txt").write_text("v2") + + git("add", ".", cwd=self.work) + git("commit", "-m", "update", cwd=self.work) + git("push", "origin", "main", cwd=self.work) + + expected_sha = git_output( + "rev-parse", + "HEAD", + cwd=self.work, + ) + + client.fetch() + + self.assertEqual( + client.get_current_commit_sha(), + expected_sha, + ) + + self.assertNotEqual( + sha1, + expected_sha, + ) + + self.assertEqual( + (client.get_local_path() / "test.txt").read_text(), + "v2", + ) + + def test_verify_repository_integrity_rejects_fake_git_directory(self): + fake = self.root / "fake" + + fake.mkdir() + (fake / ".git").mkdir() + + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + local_path=fake, + ) + + self.assertFalse( + client.verify_repository_integrity(), + ) + + def test_verify_repository_integrity_rejects_wrong_origin(self): + other_remote = self.root / "other.git" + + git("init", "--bare", other_remote) + + client = self.create_client() + + client.clone() + + git( + "remote", + "set-url", + "origin", + other_remote, + cwd=client.get_local_path(), + ) + + self.assertFalse( + client.verify_repository_integrity(), + ) + + def test_verify_repository_integrity_rejects_missing_branch(self): + client = IntegrationGitRepositoryClient( + owner="OWASP", + repository="ASVS", + branch="dev", + local_path=self.cache, + repository_url=str(self.remote), + ) + + git("clone", self.remote, self.cache) + + self.assertFalse( + client.verify_repository_integrity(), + ) + + def test_sync_serializes_clone_operations(self): + client1 = self.create_client() + client2 = self.create_client() + + exceptions = [] + + def run_sync(client): + try: + client.sync() + except Exception as exc: + exceptions.append(exc) + + t1 = threading.Thread(target=run_sync, args=(client1,)) + t2 = threading.Thread(target=run_sync, args=(client2,)) + + t1.start() + t2.start() + + t1.join() + t2.join() + + self.assertFalse(exceptions, f"Unexpected exceptions: {exceptions}") + + self.assertTrue(client1.verify_repository_integrity()) + self.assertTrue(client2.verify_repository_integrity()) + + self.assertTrue((self.cache / ".git").exists()) + + self.assertEqual( + client1.get_current_commit_sha(), + client2.get_current_commit_sha(), + ) + + self.assertEqual((self.cache / "test.txt").read_text(), "v1") + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/git_repository_client_test.py b/application/tests/harvester_test/git_repository_client_test.py new file mode 100644 index 000000000..774715617 --- /dev/null +++ b/application/tests/harvester_test/git_repository_client_test.py @@ -0,0 +1,156 @@ +import unittest +from unittest.mock import patch + +import tempfile +from pathlib import Path + +from application.utils.harvester.git_repository_client import ( + GitRepositoryClient, +) + + +class GitRepositoryClientTests(unittest.TestCase): + def test_repository_url_generation(self): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + self.assertEqual( + client.repository_url, + "https://github.com/OWASP/ASVS.git", + ) + + def test_local_repository_path(self): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + self.assertEqual( + str(client.get_local_path()), + ".harvester_cache/owasp/asvs/main", + ) + + def test_repository_exists_locally_false(self): + with tempfile.TemporaryDirectory() as tmpdir: + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + local_path=Path(tmpdir) / "repo", + ) + + self.assertFalse(client.exists_locally()) + + def test_verify_repository_integrity_false(self): + with tempfile.TemporaryDirectory() as tmpdir: + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + local_path=Path(tmpdir) / "repo", + ) + + self.assertFalse(client.verify_repository_integrity()) + + def test_sync_clones_when_repository_missing(self): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + with ( + patch.object( + client, + "verify_repository_integrity", + return_value=False, + ), + patch.object(client, "clone") as mock_clone, + ): + client.sync() + + mock_clone.assert_called_once() + + def test_sync_fetches_when_repository_exists(self): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + with ( + patch.object( + client, + "verify_repository_integrity", + return_value=True, + ), + patch.object(client, "fetch") as mock_fetch, + ): + client.sync() + + mock_fetch.assert_called_once() + + @patch("application.utils.harvester.git_repository_client.subprocess.run") + def test_fetch_runs_git_command(self, mock_run): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + client.fetch() + + self.assertEqual(mock_run.call_count, 2) + + @patch("application.utils.harvester.git_repository_client.subprocess.run") + def test_checkout_runs_git_command(self, mock_run): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + client.checkout("main") + + mock_run.assert_called_once_with( + [ + "git", + "-C", + str(client.get_local_path()), + "checkout", + "main", + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + + @patch("application.utils.harvester.git_repository_client.subprocess.run") + def test_get_current_commit_sha_runs_git_command(self, mock_run): + mock_run.return_value.stdout = "abc123\n" + + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + sha = client.get_current_commit_sha() + + self.assertEqual(sha, "abc123") + mock_run.assert_called_once() + + @patch("application.utils.harvester.git_repository_client.subprocess.run") + def test_clone_runs_git_command(self, mock_run): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + with ( + patch.object(client, "verify_repository_integrity", return_value=False), + patch.object(client, "is_valid_repository", return_value=True), + ): + client.clone() + + mock_run.assert_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/repos_validator_test.py b/application/tests/harvester_test/repos_validator_test.py new file mode 100644 index 000000000..278018e26 --- /dev/null +++ b/application/tests/harvester_test/repos_validator_test.py @@ -0,0 +1,80 @@ +from pathlib import Path +import unittest + +from application.utils.harvester.config_loader import ( + load_repo_config, +) +from application.utils.harvester.repos_validator import ( + RepositoryValidationError, + validate_repositories, +) + +FIXTURES_DIR = Path(__file__).parent / "fixtures" + + +class ReposValidatorTests(unittest.TestCase): + def test_duplicate_repository_ids(self): + config_path = FIXTURES_DIR / "duplicate_repo_ids.yaml" + + config = load_repo_config(config_path) + + with self.assertRaisesRegex( + RepositoryValidationError, + "Duplicate repository id", + ): + validate_repositories(config) + + def test_duplicate_repositories(self): + config_path = FIXTURES_DIR / "duplicate_repositories.yaml" + + config = load_repo_config(config_path) + + with self.assertRaisesRegex( + RepositoryValidationError, + "Duplicate repository detected", + ): + validate_repositories(config) + + def test_duplicate_include_paths(self): + config_path = FIXTURES_DIR / "duplicate_include_paths.yaml" + + config = load_repo_config(config_path) + + with self.assertRaisesRegex( + RepositoryValidationError, + "duplicate include paths", + ): + validate_repositories(config) + + def test_validate_valid_repositories(self): + config_path = FIXTURES_DIR / "valid_repos.yaml" + + config = load_repo_config(config_path) + + validate_repositories(config) + + def test_duplicate_repository_ids_case_insensitive(self): + config_path = FIXTURES_DIR / "duplicate_repo_ids_case.yaml" + + config = load_repo_config(config_path) + + with self.assertRaisesRegex( + RepositoryValidationError, + "Duplicate repository id", + ): + validate_repositories(config) + + def test_duplicate_repositories_case_insensitive(self): + config_path = FIXTURES_DIR / "duplicate_repositories_case.yaml" + + config = load_repo_config(config_path) + + with self.assertRaisesRegex( + RepositoryValidationError, + "Duplicate repository detected", + ): + validate_repositories(config) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/repository_cache_test.py b/application/tests/harvester_test/repository_cache_test.py new file mode 100644 index 000000000..d3193d454 --- /dev/null +++ b/application/tests/harvester_test/repository_cache_test.py @@ -0,0 +1,84 @@ +import unittest +from pathlib import Path + +from application.utils.harvester.repository_cache import ( + build_repository_cache_path, +) + + +class RepositoryCacheTests(unittest.TestCase): + def test_build_repository_cache_path(self): + path = build_repository_cache_path( + "OWASP", + "ASVS", + ) + + self.assertEqual( + path, + Path(".harvester_cache/owasp/asvs/main"), + ) + + def test_different_branches_have_different_cache_paths(self): + main_path = build_repository_cache_path( + owner="OWASP", + repository="ASVS", + branch="main", + ) + + dev_path = build_repository_cache_path( + owner="OWASP", + repository="ASVS", + branch="dev", + ) + + self.assertNotEqual(main_path, dev_path) + + def test_case_sensitive_branches_have_different_cache_paths(self): + release_path = build_repository_cache_path( + owner="OWASP", + repository="ASVS", + branch="Release", + ) + + release_lower_path = build_repository_cache_path( + owner="OWASP", + repository="ASVS", + branch="release", + ) + + self.assertNotEqual(release_path, release_lower_path) + + def test_path_traversal_owner_rejected(self): + with self.assertRaises(ValueError): + build_repository_cache_path( + owner="../../tmp", + repository="ASVS", + ) + + def test_absolute_owner_rejected(self): + with self.assertRaises(ValueError): + build_repository_cache_path( + owner="/tmp", + repository="ASVS", + ) + + def test_invalid_repository_name_rejected(self): + with self.assertRaises(ValueError): + build_repository_cache_path( + owner="OWASP", + repository="../ASVS", + ) + + def test_branch_path_is_encoded(self): + path = build_repository_cache_path( + owner="OWASP", + repository="ASVS", + branch="feature/test", + ) + + self.assertNotIn("feature/test", str(path)) + self.assertIn("feature%2Ftest", str(path)) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/utils/harvester/__init__.py b/application/utils/harvester/__init__.py new file mode 100644 index 000000000..2ac608b3e --- /dev/null +++ b/application/utils/harvester/__init__.py @@ -0,0 +1,34 @@ +from .config_loader import ( + ConfigLoaderError, + load_repo_config, +) +from .repos_validator import ( + RepositoryValidationError, + validate_repositories, +) +from .schemas import ( + ChunkingConfig, + PathRules, + PollingConfig, + RepositoryConfig, + ReposFile, +) + +from .git_repository_client import GitRepositoryClient +from .repository_client import RepositoryClient +from .repository_cache import build_repository_cache_path + +__all__ = [ + "build_repository_cache_path", + "ChunkingConfig", + "ConfigLoaderError", + "GitRepositoryClient", + "PathRules", + "PollingConfig", + "RepositoryClient", + "RepositoryConfig", + "RepositoryValidationError", + "ReposFile", + "load_repo_config", + "validate_repositories", +] diff --git a/application/utils/harvester/config_loader.py b/application/utils/harvester/config_loader.py new file mode 100644 index 000000000..db0431e11 --- /dev/null +++ b/application/utils/harvester/config_loader.py @@ -0,0 +1,32 @@ +from pathlib import Path +import yaml +from pydantic import ValidationError +from .schemas import ReposFile + + +class ConfigLoaderError(Exception): + """Base class for configuration loading errors.""" + + +class ConfigFileNotFoundError(ConfigLoaderError): + """Raised when the configuration file cannot be found.""" + + +def load_repo_config(path: str | Path) -> ReposFile: + config_path = Path(path) + + if not config_path.is_file(): + raise ConfigFileNotFoundError(f"Configuration file not found: {config_path}") + + try: + with config_path.open("r", encoding="utf-8") as file: + raw_config = yaml.safe_load(file) + except yaml.YAMLError as exc: + raise ConfigLoaderError(f"Invalid YAML syntax in {config_path}") from exc + + try: + return ReposFile.model_validate(raw_config) + except ValidationError as exc: + raise ConfigLoaderError( + f"Schema validation failed for {config_path}: {exc}" + ) from exc diff --git a/application/utils/harvester/exclude_patterns.txt b/application/utils/harvester/exclude_patterns.txt new file mode 100644 index 000000000..499850ae8 --- /dev/null +++ b/application/utils/harvester/exclude_patterns.txt @@ -0,0 +1,18 @@ +# Placeholder for repository-level exclude patterns. + +# This file will be consumed by the Week 4 noise-reduction pipeline + +# to filter non-documentation files during harvesting. + +**/.git/* +**/node_modules/** +**/__pycache__/** +**/.claude/** +**/.cursor/** +**/*.png +**/*.jpg +**/*.jpeg +**/*.svg +**/*.gif +**/*.pdf +**/archive/** diff --git a/application/utils/harvester/git_repository_client.py b/application/utils/harvester/git_repository_client.py new file mode 100644 index 000000000..1390e6606 --- /dev/null +++ b/application/utils/harvester/git_repository_client.py @@ -0,0 +1,283 @@ +import subprocess +from pathlib import Path + +from .repository_cache import build_repository_cache_path +from .repository_client import RepositoryClient +import logging +from .repository_lock import repository_lock +import os +import shutil +import tempfile + +logger = logging.getLogger(__name__) + + +class GitRepositoryClient(RepositoryClient): + def __init__( + self, + owner: str, + repository: str, + branch: str = "main", + local_path: Path | None = None, + ) -> None: + self.owner = owner + self.repository = repository + self.branch = branch + + self.local_path = ( + local_path + if local_path is not None + else build_repository_cache_path(owner, repository, branch) + ) + + @property + def repository_url(self) -> str: + return f"https://github.com/{self.owner}/{self.repository}.git" + + def clone(self) -> None: + + logger.info( + "Cloning repository %s/%s", + self.owner, + self.repository, + ) + + self.local_path.parent.mkdir( + parents=True, + exist_ok=True, + ) + + self._clone_atomically() + + def _clone_atomically(self) -> None: + temp_path = Path( + tempfile.mkdtemp( + prefix=f"{self.repository}-", + dir=self.local_path.parent, + ) + ) + + try: + subprocess.run( + [ + "git", + "clone", + "--branch", + self.branch, + self.repository_url, + str(temp_path), + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + + if not self.is_valid_repository(temp_path): + raise RuntimeError("Temporary clone failed integrity verification") + + if self.local_path.exists(): + shutil.rmtree(temp_path) + return + + os.replace(temp_path, self.local_path) + + except subprocess.CalledProcessError as exc: + logger.error( + "Failed to clone repository %s/%s: %s", + self.owner, + self.repository, + exc.stderr, + ) + raise + + finally: + if temp_path.exists(): + shutil.rmtree(temp_path, ignore_errors=True) + + def fetch(self) -> None: + logger.info( + "Fetching repository %s/%s", + self.owner, + self.repository, + ) + + try: + subprocess.run( + [ + "git", + "-C", + str(self.local_path), + "fetch", + "origin", + self.branch, + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + + subprocess.run( + [ + "git", + "-C", + str(self.local_path), + "reset", + "--hard", + f"origin/{self.branch}", + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + + except subprocess.CalledProcessError as exc: + logger.error( + "Failed to fetch repository %s/%s: %s", + self.owner, + self.repository, + exc.stderr, + ) + raise + + def checkout(self, reference: str) -> None: + if reference.startswith("-"): + raise ValueError("Invalid git reference") + + logger.info( + "Checking out %s in %s/%s", + reference, + self.owner, + self.repository, + ) + + try: + subprocess.run( + [ + "git", + "-C", + str(self.local_path), + "checkout", + reference, + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + except subprocess.CalledProcessError as exc: + logger.error( + "Failed to checkout %s in %s/%s: %s", + reference, + self.owner, + self.repository, + exc.stderr, + ) + raise + + def get_local_path(self) -> Path: + return self.local_path + + def exists_locally(self) -> bool: + return self.local_path.exists() + + def sync(self) -> None: + logger.info( + "Synchronizing repository %s/%s", + self.owner, + self.repository, + ) + + with repository_lock(self.local_path): + if self.verify_repository_integrity(): + self.fetch() + else: + self.clone() + + def get_current_commit_sha(self) -> str: + try: + result = subprocess.run( + [ + "git", + "-C", + str(self.local_path), + "rev-parse", + "HEAD", + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + except subprocess.CalledProcessError as exc: + logger.error( + "Failed to retrieve commit SHA for %s/%s: %s", + self.owner, + self.repository, + exc.stderr, + ) + raise + + return result.stdout.strip() + + def is_valid_repository(self, repository_path: Path) -> bool: + if not (repository_path.exists() and repository_path.is_dir()): + return False + + try: + subprocess.run( + [ + "git", + "-C", + str(repository_path), + "rev-parse", + "--is-inside-work-tree", + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + + remote = subprocess.run( + [ + "git", + "-C", + str(repository_path), + "config", + "--get", + "remote.origin.url", + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ).stdout.strip() + + if remote.rstrip("/") != self.repository_url.rstrip("/"): + return False + + subprocess.run( + [ + "git", + "-C", + str(repository_path), + "show-ref", + "--verify", + f"refs/remotes/origin/{self.branch}", + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + + return True + + except subprocess.CalledProcessError: + return False + + def verify_repository_integrity(self) -> bool: + return self.is_valid_repository(self.local_path) diff --git a/application/utils/harvester/repos.yaml b/application/utils/harvester/repos.yaml new file mode 100644 index 000000000..c448e5111 --- /dev/null +++ b/application/utils/harvester/repos.yaml @@ -0,0 +1,43 @@ +repositories: + - id: owasp-asvs + type: github + enabled: true + owner: OWASP + repo: ASVS + branch: master + paths: + include: + - "4.0/en/**/*.md" + + exclude: + - "**/archive/**" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + overlap_tokens: 100 + + polling: + mode: incremental + interval_minutes: 60 + + - id: owasp-cheatsheets + type: github + enabled: true + + owner: OWASP + repo: CheatSheetSeries + branch: master + + paths: + include: + - "cheatsheets/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1000 + overlap_tokens: 100 + + polling: + mode: incremental + interval_minutes: 120 diff --git a/application/utils/harvester/repos_validator.py b/application/utils/harvester/repos_validator.py new file mode 100644 index 000000000..81cdc8273 --- /dev/null +++ b/application/utils/harvester/repos_validator.py @@ -0,0 +1,33 @@ +# as the name suggests +from .schemas import ReposFile + + +class RepositoryValidationError(Exception): + """Raised when repository configuration fails semantic validation.""" + + +def validate_repositories(config: ReposFile) -> None: + seen_ids: set[str] = set() + seen_repositories: set[tuple[str, str]] = set() + + for repository in config.repositories: + repo_id_key = repository.id.casefold() + if repo_id_key in seen_ids: + raise RepositoryValidationError( + f"Duplicate repository id found: {repository.id}" + ) + seen_ids.add(repo_id_key) + repository_key = ( + repository.owner.casefold(), + repository.repo.casefold(), + ) + if repository_key in seen_repositories: + raise RepositoryValidationError( + f"Duplicate repository detected: {repository.owner}/{repository.repo}" + ) + seen_repositories.add(repository_key) + include_patterns = set(repository.paths.include) + if len(include_patterns) != len(repository.paths.include): + raise RepositoryValidationError( + f"Repository '{repository.id}' has duplicate include paths" + ) diff --git a/application/utils/harvester/repository_cache.py b/application/utils/harvester/repository_cache.py new file mode 100644 index 000000000..94b721a60 --- /dev/null +++ b/application/utils/harvester/repository_cache.py @@ -0,0 +1,26 @@ +import os +import re + +from pathlib import Path +from urllib.parse import quote + +CACHE_ROOT = Path(os.getenv("HARVESTER_CACHE_DIR", ".harvester_cache")) + +_VALID_COMPONENT = re.compile(r"^[A-Za-z0-9_.-]+$") + + +def build_repository_cache_path( + owner: str, repository: str, branch: str = "main" +) -> Path: + if not _VALID_COMPONENT.fullmatch(owner): + raise ValueError(f"Invalid repository owner: {owner}") + + if not _VALID_COMPONENT.fullmatch(repository): + raise ValueError(f"Invalid repository name: {repository}") + + encoded_branch = quote(branch, safe="") + candidate = CACHE_ROOT / owner.casefold() / repository.casefold() / encoded_branch + + candidate.resolve().relative_to(CACHE_ROOT.resolve()) + + return candidate diff --git a/application/utils/harvester/repository_client.py b/application/utils/harvester/repository_client.py new file mode 100644 index 000000000..c545cce42 --- /dev/null +++ b/application/utils/harvester/repository_client.py @@ -0,0 +1,36 @@ +from abc import ABC, abstractmethod +from pathlib import Path + + +class RepositoryClient(ABC): + @abstractmethod + def clone(self) -> None: + """Clone repository locally.""" + + @abstractmethod + def fetch(self) -> None: + """Fetch latest repository changes.""" + + @abstractmethod + def checkout(self, reference: str) -> None: + """Checkout a branch, tag, or commit.""" + + @abstractmethod + def get_local_path(self) -> Path: + """Return local repository path.""" + + @abstractmethod + def exists_locally(self) -> bool: + """Return whether repository exists locally.""" + + @abstractmethod + def sync(self) -> None: + """Clone if missing, otherwise fetch latest changes.""" + + @abstractmethod + def get_current_commit_sha(self) -> str: + """Return HEAD commit SHA.""" + + @abstractmethod + def verify_repository_integrity(self) -> bool: + """Verify local repository integrity.""" diff --git a/application/utils/harvester/repository_lock.py b/application/utils/harvester/repository_lock.py new file mode 100644 index 000000000..9890e6e2e --- /dev/null +++ b/application/utils/harvester/repository_lock.py @@ -0,0 +1,32 @@ +import contextlib +import os +from pathlib import Path + +if os.name == "nt": + import msvcrt +else: + import fcntl + + +@contextlib.contextmanager +def repository_lock(repository_path: Path): + """ + Acquire an exclusive inter-process lock for a repository cache path. + """ + + lock_path = repository_path.with_suffix(".lock") + lock_path.parent.mkdir(parents=True, exist_ok=True) + + with lock_path.open("w") as lock_file: + if os.name == "nt": + msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1) + else: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + + try: + yield + finally: + if os.name == "nt": + msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1) + else: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) diff --git a/application/utils/harvester/schemas.py b/application/utils/harvester/schemas.py new file mode 100644 index 000000000..4589f3d69 --- /dev/null +++ b/application/utils/harvester/schemas.py @@ -0,0 +1,104 @@ +from typing import Literal +from pydantic import BaseModel, Field, ConfigDict, model_validator + + +# this will control which repo paths are included and excluded during ingestions +class PathRules(BaseModel): + model_config = ConfigDict(extra="forbid") + + include: list[str] = Field( + ..., + min_length=1, + description="Glob patterns to include during ingestions", + ) + exclude: list[str] = Field( + default_factory=list, description="Glob patterns to exclude during ingestions" + ) + + +# this will define how the harvested data should be chunked before downstream +class ChunkingConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + strategy: Literal["markdown_heading", "html_readability", "fixed_size"] = Field( + ..., + description="Chunking strategy used for text segmentation", + ) + + max_tokens: int = Field(..., gt=0, description="max token size per chunk") + + overlap_tokens: int = Field( + ge=0, + default=20, + description="token overlap between adjacent chunks", + ) + + @model_validator(mode="after") + def overlap_must_be_less_than_max(self) -> "ChunkingConfig": + if self.overlap_tokens >= self.max_tokens: + raise ValueError( + f"overlap_tokens ({self.overlap_tokens}) must be less than " + f"max_tokens ({self.max_tokens})" + ) + return self + + +# this one defines repository synchronize behaviour +class PollingConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + mode: Literal["full", "incremental"] = Field( + ..., description="repository sync mode" + ) + + interval_minutes: int = Field(..., gt=0, description="polling interval in minutes") + + +# top level repository ingestion configuration +class RepositoryConfig(BaseModel): + model_config = ConfigDict(extra="forbid", str_strip_whitespace=True) + + id: str = Field( + ..., + min_length=1, + description="unique repository identifier.", + ) + + type: Literal["github"] = Field( + ..., + description="repository source type.", + ) + enabled: bool = Field( + default=True, + description="whether ingestion is enabled for this repository.", + ) + owner: str = Field( + ..., + min_length=1, + description="repository organization.", + ) + repo: str = Field( + ..., + min_length=1, + description="repository name.", + ) + branch: str = Field( + default="main", + min_length=1, + description="Repository branch to ingest.", + ) + + paths: PathRules + chunking: ChunkingConfig + polling: PollingConfig + + +# Root configuration object loaded from repos.yaml. +class ReposFile(BaseModel): + model_config = ConfigDict(extra="forbid") + + repositories: list[RepositoryConfig] = Field( + ..., + min_length=1, + description="List of repositories configured for ingestion.", + )