Skip to content

Commit 867055d

Browse files
authored
CM-60869 SCA add restored files cleanup mechanism (#409)
1 parent 3906f27 commit 867055d

21 files changed

+1046
-17
lines changed

cycode/cli/files_collector/sca/base_restore_dependencies.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,9 @@ def try_restore_dependencies(self, document: Document) -> Optional[Document]:
9292
)
9393
if output is None: # one of the commands failed
9494
return None
95+
file_was_generated = True
9596
else:
97+
file_was_generated = False
9698
logger.debug(
9799
'Lock file already exists, skipping restore commands, %s',
98100
{'restore_file_path': restore_file_path},
@@ -107,6 +109,14 @@ def try_restore_dependencies(self, document: Document) -> Optional[Document]:
107109
'content_empty': not restore_file_content,
108110
},
109111
)
112+
113+
if file_was_generated:
114+
try:
115+
Path(restore_file_path).unlink(missing_ok=True)
116+
logger.debug('Cleaned up generated restore file, %s', {'restore_file_path': restore_file_path})
117+
except Exception as e:
118+
logger.debug('Failed to clean up generated restore file', exc_info=e)
119+
110120
return Document(relative_restore_file_path, restore_file_content, self.is_git_diff)
111121

112122
def get_manifest_dir(self, document: Document) -> Optional[str]:

cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
from os import path
2+
from pathlib import Path
23
from typing import Optional
34

45
import typer
@@ -9,7 +10,10 @@
910
execute_commands,
1011
)
1112
from cycode.cli.models import Document
12-
from cycode.cli.utils.path_utils import get_file_content, get_file_dir, join_paths
13+
from cycode.cli.utils.path_utils import get_file_content, join_paths
14+
from cycode.logger import get_logger
15+
16+
logger = get_logger('Maven Restore Dependencies')
1317

1418
BUILD_MAVEN_FILE_NAME = 'pom.xml'
1519
MAVEN_CYCLONE_DEP_TREE_FILE_NAME = 'bom.json'
@@ -42,15 +46,8 @@ def try_restore_dependencies(self, document: Document) -> Optional[Document]:
4246
if document.content is None:
4347
return self.restore_from_secondary_command(document, manifest_file_path)
4448

45-
restore_dependencies_document = super().try_restore_dependencies(document)
46-
if restore_dependencies_document is None:
47-
return None
48-
49-
restore_dependencies_document.content = get_file_content(
50-
join_paths(get_file_dir(manifest_file_path), self.get_lock_file_name())
51-
)
52-
53-
return restore_dependencies_document
49+
# super() reads the content and cleans up any generated file; no re-read needed
50+
return super().try_restore_dependencies(document)
5451

5552
def restore_from_secondary_command(self, document: Document, manifest_file_path: str) -> Optional[Document]:
5653
restore_content = execute_commands(
@@ -62,11 +59,17 @@ def restore_from_secondary_command(self, document: Document, manifest_file_path:
6259
return None
6360

6461
restore_file_path = build_dep_tree_path(document.absolute_path, MAVEN_DEP_TREE_FILE_NAME)
62+
content = get_file_content(restore_file_path)
63+
64+
try:
65+
Path(restore_file_path).unlink(missing_ok=True)
66+
except Exception as e:
67+
logger.debug('Failed to clean up generated maven dep tree file', exc_info=e)
68+
6569
return Document(
6670
path=build_dep_tree_path(document.path, MAVEN_DEP_TREE_FILE_NAME),
67-
content=get_file_content(restore_file_path),
71+
content=content,
6872
is_git_diff_format=self.is_git_diff,
69-
absolute_path=restore_file_path,
7073
)
7174

7275
def create_secondary_restore_commands(self, manifest_file_path: str) -> list[list[str]]:

tests/cli/files_collector/sca/__init__.py

Whitespace-only changes.

tests/cli/files_collector/sca/go/__init__.py

Whitespace-only changes.
Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
from pathlib import Path
2+
from typing import Optional
3+
from unittest.mock import MagicMock, patch
4+
5+
import pytest
6+
import typer
7+
8+
from cycode.cli.files_collector.sca.go.restore_go_dependencies import (
9+
GO_RESTORE_FILE_NAME,
10+
RestoreGoDependencies,
11+
)
12+
from cycode.cli.models import Document
13+
14+
_BASE_MODULE = 'cycode.cli.files_collector.sca.base_restore_dependencies'
15+
16+
17+
@pytest.fixture
18+
def mock_ctx(tmp_path: Path) -> typer.Context:
19+
ctx = MagicMock(spec=typer.Context)
20+
ctx.obj = {'monitor': False}
21+
ctx.params = {'path': str(tmp_path)}
22+
return ctx
23+
24+
25+
@pytest.fixture
26+
def restore_go(mock_ctx: typer.Context) -> RestoreGoDependencies:
27+
return RestoreGoDependencies(mock_ctx, is_git_diff=False, command_timeout=30)
28+
29+
30+
class TestIsProject:
31+
def test_go_mod_matches(self, restore_go: RestoreGoDependencies) -> None:
32+
doc = Document('go.mod', 'module example.com/mymod\ngo 1.21\n')
33+
assert restore_go.is_project(doc) is True
34+
35+
def test_go_sum_matches(self, restore_go: RestoreGoDependencies) -> None:
36+
doc = Document('go.sum', 'github.com/pkg/errors v0.9.1 h1:...\n')
37+
assert restore_go.is_project(doc) is True
38+
39+
def test_go_in_subdir_matches(self, restore_go: RestoreGoDependencies) -> None:
40+
doc = Document('myapp/go.mod', 'module example.com/mymod\n')
41+
assert restore_go.is_project(doc) is True
42+
43+
def test_pom_xml_does_not_match(self, restore_go: RestoreGoDependencies) -> None:
44+
doc = Document('pom.xml', '<project/>')
45+
assert restore_go.is_project(doc) is False
46+
47+
48+
class TestCleanup:
49+
def test_generated_output_file_is_deleted_after_restore(
50+
self, restore_go: RestoreGoDependencies, tmp_path: Path
51+
) -> None:
52+
# Go handler requires both go.mod and go.sum to be present
53+
(tmp_path / 'go.mod').write_text('module example.com/test\ngo 1.21\n')
54+
(tmp_path / 'go.sum').write_text('github.com/pkg/errors v0.9.1 h1:abc\n')
55+
doc = Document(
56+
str(tmp_path / 'go.mod'),
57+
'module example.com/test\ngo 1.21\n',
58+
absolute_path=str(tmp_path / 'go.mod'),
59+
)
60+
output_path = tmp_path / GO_RESTORE_FILE_NAME
61+
62+
def side_effect(
63+
commands: list,
64+
timeout: int,
65+
output_file_path: Optional[str] = None,
66+
working_directory: Optional[str] = None,
67+
) -> str:
68+
# Go uses create_output_file_manually=True; output_file_path is provided
69+
target = output_file_path or str(output_path)
70+
Path(target).write_text('example.com/test github.com/pkg/errors@v0.9.1\n')
71+
return 'graph output'
72+
73+
with patch(f'{_BASE_MODULE}.execute_commands', side_effect=side_effect):
74+
result = restore_go.try_restore_dependencies(doc)
75+
76+
assert result is not None
77+
assert not output_path.exists(), f'{GO_RESTORE_FILE_NAME} must be deleted after restore'
78+
79+
def test_missing_go_sum_returns_none(self, restore_go: RestoreGoDependencies, tmp_path: Path) -> None:
80+
(tmp_path / 'go.mod').write_text('module example.com/test\ngo 1.21\n')
81+
# go.sum intentionally absent
82+
doc = Document(
83+
str(tmp_path / 'go.mod'),
84+
'module example.com/test\ngo 1.21\n',
85+
absolute_path=str(tmp_path / 'go.mod'),
86+
)
87+
88+
result = restore_go.try_restore_dependencies(doc)
89+
90+
assert result is None

tests/cli/files_collector/sca/maven/__init__.py

Whitespace-only changes.
Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
from pathlib import Path
2+
from typing import Optional
3+
from unittest.mock import MagicMock, patch
4+
5+
import pytest
6+
import typer
7+
8+
from cycode.cli.files_collector.sca.maven.restore_gradle_dependencies import (
9+
BUILD_GRADLE_DEP_TREE_FILE_NAME,
10+
BUILD_GRADLE_FILE_NAME,
11+
BUILD_GRADLE_KTS_FILE_NAME,
12+
RestoreGradleDependencies,
13+
)
14+
from cycode.cli.models import Document
15+
16+
_BASE_MODULE = 'cycode.cli.files_collector.sca.base_restore_dependencies'
17+
18+
19+
@pytest.fixture
20+
def mock_ctx(tmp_path: Path) -> typer.Context:
21+
ctx = MagicMock(spec=typer.Context)
22+
ctx.obj = {'monitor': False, 'gradle_all_sub_projects': False}
23+
ctx.params = {'path': str(tmp_path)}
24+
return ctx
25+
26+
27+
@pytest.fixture
28+
def restore_gradle(mock_ctx: typer.Context) -> RestoreGradleDependencies:
29+
return RestoreGradleDependencies(mock_ctx, is_git_diff=False, command_timeout=30)
30+
31+
32+
class TestIsProject:
33+
def test_build_gradle_matches(self, restore_gradle: RestoreGradleDependencies) -> None:
34+
doc = Document('build.gradle', 'apply plugin: "java"\n')
35+
assert restore_gradle.is_project(doc) is True
36+
37+
def test_build_gradle_kts_matches(self, restore_gradle: RestoreGradleDependencies) -> None:
38+
doc = Document('build.gradle.kts', 'plugins { java }\n')
39+
assert restore_gradle.is_project(doc) is True
40+
41+
def test_pom_xml_does_not_match(self, restore_gradle: RestoreGradleDependencies) -> None:
42+
doc = Document('pom.xml', '<project/>')
43+
assert restore_gradle.is_project(doc) is False
44+
45+
def test_settings_gradle_does_not_match(self, restore_gradle: RestoreGradleDependencies) -> None:
46+
doc = Document('settings.gradle', 'rootProject.name = "test"')
47+
assert restore_gradle.is_project(doc) is False
48+
49+
50+
class TestCleanup:
51+
def test_generated_dep_tree_file_is_deleted_after_restore(
52+
self, restore_gradle: RestoreGradleDependencies, tmp_path: Path
53+
) -> None:
54+
(tmp_path / BUILD_GRADLE_FILE_NAME).write_text('apply plugin: "java"\n')
55+
doc = Document(
56+
str(tmp_path / BUILD_GRADLE_FILE_NAME),
57+
'apply plugin: "java"\n',
58+
absolute_path=str(tmp_path / BUILD_GRADLE_FILE_NAME),
59+
)
60+
output_path = tmp_path / BUILD_GRADLE_DEP_TREE_FILE_NAME
61+
62+
def side_effect(
63+
commands: list,
64+
timeout: int,
65+
output_file_path: Optional[str] = None,
66+
working_directory: Optional[str] = None,
67+
) -> str:
68+
# Gradle uses create_output_file_manually=True; output_file_path is provided
69+
target = output_file_path or str(output_path)
70+
Path(target).write_text('compileClasspath - Compile classpath:\n\\--- org.example:lib:1.0\n')
71+
return 'dep tree output'
72+
73+
with patch(f'{_BASE_MODULE}.execute_commands', side_effect=side_effect):
74+
result = restore_gradle.try_restore_dependencies(doc)
75+
76+
assert result is not None
77+
assert not output_path.exists(), f'{BUILD_GRADLE_DEP_TREE_FILE_NAME} must be deleted after restore'
78+
79+
def test_preexisting_dep_tree_file_is_not_deleted(
80+
self, restore_gradle: RestoreGradleDependencies, tmp_path: Path
81+
) -> None:
82+
dep_tree_content = 'compileClasspath - Compile classpath:\n\\--- org.example:lib:1.0\n'
83+
(tmp_path / BUILD_GRADLE_FILE_NAME).write_text('apply plugin: "java"\n')
84+
output_path = tmp_path / BUILD_GRADLE_DEP_TREE_FILE_NAME
85+
output_path.write_text(dep_tree_content)
86+
doc = Document(
87+
str(tmp_path / BUILD_GRADLE_FILE_NAME),
88+
'apply plugin: "java"\n',
89+
absolute_path=str(tmp_path / BUILD_GRADLE_FILE_NAME),
90+
)
91+
92+
result = restore_gradle.try_restore_dependencies(doc)
93+
94+
assert result is not None
95+
assert output_path.exists(), f'Pre-existing {BUILD_GRADLE_DEP_TREE_FILE_NAME} must not be deleted'
96+
97+
def test_kts_build_file_also_cleaned_up(self, restore_gradle: RestoreGradleDependencies, tmp_path: Path) -> None:
98+
(tmp_path / BUILD_GRADLE_KTS_FILE_NAME).write_text('plugins { java }\n')
99+
doc = Document(
100+
str(tmp_path / BUILD_GRADLE_KTS_FILE_NAME),
101+
'plugins { java }\n',
102+
absolute_path=str(tmp_path / BUILD_GRADLE_KTS_FILE_NAME),
103+
)
104+
output_path = tmp_path / BUILD_GRADLE_DEP_TREE_FILE_NAME
105+
106+
def side_effect(
107+
commands: list,
108+
timeout: int,
109+
output_file_path: Optional[str] = None,
110+
working_directory: Optional[str] = None,
111+
) -> str:
112+
target = output_file_path or str(output_path)
113+
Path(target).write_text('compileClasspath\n')
114+
return 'output'
115+
116+
with patch(f'{_BASE_MODULE}.execute_commands', side_effect=side_effect):
117+
result = restore_gradle.try_restore_dependencies(doc)
118+
119+
assert result is not None
120+
assert not output_path.exists(), f'{BUILD_GRADLE_DEP_TREE_FILE_NAME} must be deleted after restore'

0 commit comments

Comments
 (0)