CM-17814 - SCA commit range (#50)

Author: avishaiamiel

cli/code_scanner.py

@@ -85,24 +85,26 @@ def scan_repository_commit_history(context: click.Context, path: str, commit_ran
 
 def scan_commit_range(context: click.Context, path: str, commit_range: str):
     scan_type = context.obj["scan_type"]
-
-    if scan_type != SECRET_SCAN_TYPE:
+    if scan_type not in COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES:
         raise click.ClickException(f"Commit range scanning for {str.upper(scan_type)} is not supported")
 
-    documents_to_scan = []
-    for commit in Repo(path).iter_commits(rev=commit_range):
-        commit_id = commit.hexsha
-        parent = commit.parents[0] if commit.parents else NULL_TREE
-        diff = commit.diff(parent, create_patch=True, R=True)
-        for blob in diff:
-            doc = Document(get_path_by_os(os.path.join(path, get_diff_file_path(blob))),
-                           blob.diff.decode('utf-8', errors='replace'), True, commit_id)
-            documents_to_scan.append(doc)
-
-            documents_to_scan = exclude_irrelevant_documents_to_scan(context, documents_to_scan)
-            logger.debug('Found all relevant files in commit %s',
-                         {'path': path, 'commit_range': commit_range, 'commit_id': commit_id})
-    return scan_documents(context, documents_to_scan, is_git_diff=True, is_commit_range=True)
+    if scan_type == SCA_SCAN_TYPE:
+        return scan_sca_commit_range(context, path, commit_range)
+    else:
+        documents_to_scan = []
+        for commit in Repo(path).iter_commits(rev=commit_range):
+            commit_id = commit.hexsha
+            parent = commit.parents[0] if commit.parents else NULL_TREE
+            diff = commit.diff(parent, create_patch=True, R=True)
+            for blob in diff:
+                doc = Document(get_path_by_os(os.path.join(path, get_diff_file_path(blob))),
+                               blob.diff.decode('utf-8', errors='replace'), True, commit_id)
+                documents_to_scan.append(doc)
+
+                documents_to_scan = exclude_irrelevant_documents_to_scan(context, documents_to_scan)
+                logger.debug('Found all relevant files in commit %s',
+                             {'path': path, 'commit_range': commit_range, 'commit_id': commit_id})
+        return scan_documents(context, documents_to_scan, is_git_diff=True, is_commit_range=True)
 
 
 @click.command()
@@ -137,6 +139,17 @@ def pre_commit_scan(context: click.Context, ignored_args: List[str]):
     return scan_documents(context, documents_to_scan, is_git_diff=True)
 
 
+def scan_sca_commit_range(context: click.Context, path: str, commit_range: str):
+    from_commit_rev, to_commit_rev = parse_commit_range(commit_range, path)
+    from_commit_documents, to_commit_documents = \
+        get_commit_range_modified_documents(path, from_commit_rev, to_commit_rev)
+    exclude_irrelevant_documents_to_scan(context, from_commit_documents)
+    exclude_irrelevant_documents_to_scan(context, to_commit_documents)
+    sca_code_scanner.perform_pre_commit_range_scan_actions(path, from_commit_documents, from_commit_rev,
+                                                           to_commit_documents, to_commit_rev)
+    return scan_commit_range_documents(context, from_commit_documents, to_commit_documents)
+
+
 def scan_disk_files(context: click.Context, paths: List[str]):
     scan_type = context.obj['scan_type']
     is_git_diff = False
@@ -150,8 +163,8 @@ def scan_disk_files(context: click.Context, paths: List[str]):
     return scan_documents(context, documents, is_git_diff=is_git_diff)
 
 
-def scan_documents(context: click.Context, documents_to_scan: List[Document],
-                   is_git_diff: bool = False, is_commit_range: bool = False, scan_parameters: dict = None):
+def scan_documents(context: click.Context, documents_to_scan: List[Document], is_git_diff: bool = False,
+                   is_commit_range: bool = False, scan_parameters: dict = None):
     cycode_client = context.obj["client"]
     scan_type = context.obj["scan_type"]
     severity_threshold = context.obj["severity_threshold"]
@@ -166,17 +179,9 @@ def scan_documents(context: click.Context, documents_to_scan: List[Document],
         zipped_documents = zip_documents_to_scan(scan_type, zipped_documents, documents_to_scan)
         scan_result = perform_scan(cycode_client, zipped_documents, scan_type, scan_id, is_git_diff, is_commit_range,
                                    scan_parameters)
-        document_detections_list = enrich_scan_result(scan_result, documents_to_scan)
-        relevant_document_detections_list = exclude_irrelevant_scan_results(document_detections_list, scan_type,
-                                                                            scan_command_type, severity_threshold)
-        context.obj['report_url'] = scan_result.report_url
-        print_results(context, relevant_document_detections_list)
-
-        context.obj['issue_detected'] = len(relevant_document_detections_list) > 0
-        all_detections_count = sum(
-            [len(document_detections.detections) for document_detections in document_detections_list])
-        output_detections_count = sum(
-            [len(document_detections.detections) for document_detections in relevant_document_detections_list])
+        all_detections_count, output_detections_count = \
+            handle_scan_result(context, scan_result, scan_command_type, scan_type, severity_threshold,
+                               documents_to_scan)
         scan_completed = True
     except Exception as e:
         _handle_exception(context, e)
@@ -191,10 +196,62 @@ def scan_documents(context: click.Context, documents_to_scan: List[Document],
                         all_detections_count, len(documents_to_scan), zip_file_size, scan_command_type, error_message)
 
 
+def scan_commit_range_documents(context: click.Context, from_documents_to_scan: List[Document],
+                                to_documents_to_scan: List[Document], scan_parameters: dict = None):
+    cycode_client = context.obj["client"]
+    scan_type = context.obj["scan_type"]
+    severity_threshold = context.obj["severity_threshold"]
+    scan_command_type = context.info_name
+    error_message = None
+    all_detections_count = 0
+    output_detections_count = 0
+    scan_id = _get_scan_id(context)
+    from_commit_zipped_documents = InMemoryZip()
+    to_commit_zipped_documents = InMemoryZip()
+
+    try:
+        from_commit_zipped_documents = zip_documents_to_scan(scan_type, from_commit_zipped_documents,
+                                                             from_documents_to_scan)
+        to_commit_zipped_documents = zip_documents_to_scan(scan_type, to_commit_zipped_documents, to_documents_to_scan)
+        scan_result = perform_commit_range_scan_async(cycode_client, from_commit_zipped_documents,
+                                                      to_commit_zipped_documents, scan_type, scan_parameters)
+        all_detections_count, output_detections_count = \
+            handle_scan_result(context, scan_result, scan_command_type, scan_type, severity_threshold,
+                               to_documents_to_scan)
+        scan_completed = True
+    except Exception as e:
+        _handle_exception(context, e)
+        error_message = str(e)
+        scan_completed = False
+
+    zip_file_size = getsizeof(from_commit_zipped_documents.in_memory_zip) + getsizeof(
+        to_commit_zipped_documents.in_memory_zip)
+    logger.debug('Finished scan process, %s',
+                 {'all_violations_count': all_detections_count, 'relevant_violations_count': output_detections_count,
+                  'scan_id': str(scan_id), 'zip_file_size': zip_file_size})
+    _report_scan_status(context, scan_type, str(scan_id), scan_completed, output_detections_count,
+                        all_detections_count, len(to_documents_to_scan), zip_file_size, scan_command_type,
+                        error_message)
+
+
+def handle_scan_result(context, scan_result, scan_command_type, scan_type, severity_threshold, to_documents_to_scan):
+    document_detections_list = enrich_scan_result(scan_result, to_documents_to_scan)
+    relevant_document_detections_list = exclude_irrelevant_scan_results(document_detections_list, scan_type,
+                                                                        scan_command_type, severity_threshold)
+    context.obj['report_url'] = scan_result.report_url
+    print_results(context, relevant_document_detections_list)
+    context.obj['issue_detected'] = len(relevant_document_detections_list) > 0
+    all_detections_count = sum(
+        [len(document_detections.detections) for document_detections in document_detections_list])
+    output_detections_count = sum(
+        [len(document_detections.detections) for document_detections in relevant_document_detections_list])
+    return all_detections_count, output_detections_count
+
+
 def perform_pre_scan_documents_actions(context: click.Context, scan_type: str, documents_to_scan: List[Document],
                                        is_git_diff: bool = False):
     if scan_type == SCA_SCAN_TYPE:
-        sca_code_scanner.run_pre_scan_actions(context, documents_to_scan, is_git_diff)
+        sca_code_scanner.add_dependencies_tree_document(context, documents_to_scan, is_git_diff)
 
 
 def zip_documents_to_scan(scan_type: str, zip: InMemoryZip, documents: List[Document]):
@@ -238,14 +295,28 @@ def perform_scan_async(cycode_client, zipped_documents: InMemoryZip, scan_type:
                        scan_parameters: dict) -> ZippedFileScanResult:
     scan_async_result = cycode_client.zipped_file_scan_async(zipped_documents, scan_type, scan_parameters)
     logger.debug("scan request has been triggered successfully, scan id: %s", scan_async_result.scan_id)
-    polling_timeout = configuration_manager.get_scan_polling_timeout_in_seconds()
 
+    return poll_scan_results(cycode_client, scan_async_result.scan_id)
+
+
+def perform_commit_range_scan_async(cycode_client, from_commit_zipped_documents: InMemoryZip,
+                                    to_commit_zipped_documents: InMemoryZip, scan_type: str,
+                                    scan_parameters: dict) -> ZippedFileScanResult:
+    scan_async_result = \
+        cycode_client.multiple_zipped_file_scan_async(from_commit_zipped_documents, to_commit_zipped_documents,
+                                                      scan_type, scan_parameters)
+    logger.debug("scan request has been triggered successfully, scan id: %s", scan_async_result.scan_id)
+    return poll_scan_results(cycode_client, scan_async_result.scan_id)
+
+
+def poll_scan_results(cycode_client, scan_id: str):
+    polling_timeout = configuration_manager.get_scan_polling_timeout_in_seconds()
     end_polling_time = time.time() + polling_timeout
     while time.time() < end_polling_time:
         logger.debug("scan in progress")
-        scan_details = cycode_client.get_scan_details(scan_async_result.scan_id)
+        scan_details = cycode_client.get_scan_details(scan_id)
         if scan_details.scan_status == SCAN_STATUS_COMPLETED:
-            return _get_scan_result(cycode_client, scan_async_result, scan_details)
+            return _get_scan_result(cycode_client, scan_id, scan_details)
         if scan_details.scan_status == SCAN_STATUS_ERROR:
             raise ScanAsyncError(f'error occurred while trying to scan zip file. {scan_details.message}')
         time.sleep(SCAN_POLLING_WAIT_INTERVAL_IN_SECONDS)
@@ -367,6 +438,26 @@ def exclude_detections_by_exclusions_configuration(scan_type: str, detections) -
     return [detection for detection in detections if not _should_exclude_detection(detection, exclusions)]
 
 
+def get_commit_range_modified_documents(path: str, from_commit_rev: str, to_commit_rev: str) -> (
+        List[Document], List[Document]):
+    from_commit_documents = []
+    to_commit_documents = []
+    repo = Repo(path)
+    diff = repo.commit(from_commit_rev).diff(to_commit_rev)
+    modified_files_diff = [change for change in diff if change.change_type != COMMIT_DIFF_DELETED_FILE_CHANGE_TYPE]
+    for blob in modified_files_diff:
+        diff_file_path = get_diff_file_path(blob)
+        file_path = get_path_by_os(diff_file_path)
+
+        file_content = get_file_content_from_commit(repo, from_commit_rev, diff_file_path)
+        from_commit_documents.append(Document(file_path, file_content))
+
+        file_content = get_file_content_from_commit(repo, to_commit_rev, diff_file_path)
+        to_commit_documents.append(Document(file_path, file_content))
+
+    return from_commit_documents, to_commit_documents
+
+
 def _should_exclude_detection(detection, exclusions: Dict) -> bool:
     exclusions_by_value = exclusions.get(EXCLUSIONS_BY_VALUE_SECTION_NAME, [])
     if _is_detection_sha_configured_in_exclusions(detection, exclusions_by_value):
@@ -577,16 +668,15 @@ def _does_severity_match_severity_threshold(severity: str, severity_threshold: s
     return detection_severity_value >= Severity.try_get_value(severity_threshold)
 
 
-def _get_scan_result(cycode_client, scan_async_result: ScanInitializationResponse,
-                     scan_details: ScanDetailsResponse) -> ZippedFileScanResult:
+def _get_scan_result(cycode_client, scan_id: str, scan_details: ScanDetailsResponse) -> ZippedFileScanResult:
     scan_result = ZippedFileScanResult(did_detect=False, detections_per_file=[],
-                                       scan_id=scan_async_result.scan_id,
+                                       scan_id=scan_id,
                                        report_url=_try_get_report_url(scan_details.metadata))
     if not scan_details.detections_count:
         return scan_result
 
-    wait_for_detections_creation(cycode_client, scan_async_result.scan_id, scan_details.detections_count)
-    scan_detections = cycode_client.get_scan_detections(scan_async_result.scan_id)
+    wait_for_detections_creation(cycode_client, scan_id, scan_details.detections_count)
+    scan_detections = cycode_client.get_scan_detections(scan_id)
     scan_result.detections_per_file = _map_detections_per_file(scan_detections)
     scan_result.did_detect = True
     return scan_result
@@ -633,3 +723,18 @@ def _map_detections_per_file(detections) -> List[DetectionsPerFile]:
 
     return [DetectionsPerFile(file_name=file_name, detections=file_detections)
             for file_name, file_detections in detections_per_files.items()]
+
+
+def parse_commit_range(commit_range: str, path: str) -> (str, str):
+    from_commit_rev = None
+    to_commit_rev = None
+    for commit in Repo(path).iter_commits(rev=commit_range):
+        if not to_commit_rev:
+            to_commit_rev = commit.hexsha
+        from_commit_rev = commit.hexsha
+
+    return from_commit_rev, to_commit_rev
+
+
+def get_file_content_from_commit(repo: Repo, commit: str, file_path: str) -> str:
+    return repo.git.show(f'{commit}:{file_path}')

cli/consts.py

@@ -27,6 +27,24 @@
     'pipfile', 'pipfile.lock', 'requirements.txt', 'setup.py'
 ]
 
+PROJECT_FILES_BY_ECOSYSTEM_MAP = {
+    "crates": ["Cargo.lock", "Cargo.toml"],
+    "composer": ["composer.json", "composer.lock"],
+    "go": ["go.sum", "go.mod", "Gopkg.lock"],
+    "maven_pom": ["pom.xml"],
+    "maven_gradle": ["build.gradle", "build.gradle.kts", "gradle.lockfile"],
+    "npm": ["package.json", "package-lock.json", "yarn.lock", "npm-shrinkwrap.json", ".npmrc"],
+    "nuget": ["packages.config", "project.assets.json", "packages.lock.json", "nuget.config"],
+    "ruby_gems": ["Gemfile", "Gemfile.lock"],
+    "sbt": ["build.sbt", "build.scala", "build.sbt.lock"],
+    "pypi_poetry": ["pyproject.toml", "poetry.lock"],
+    "pypi_pipenv": ["Pipfile", "Pipfile.lock"],
+    "pypi_requirements": ["requirements.txt"],
+    "pypi_setup": ["setup.py"]
+}
+
+COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES = [SECRET_SCAN_TYPE, SCA_SCAN_TYPE]
+
 DEFAULT_CYCODE_API_URL = "https://api.cycode.com"
 DEFAULT_CYCODE_APP_URL = "https://app.cycode.com"
 
@@ -68,3 +86,5 @@
 # scan statuses
 SCAN_STATUS_COMPLETED = 'Completed'
 SCAN_STATUS_ERROR = 'Error'
+
+COMMIT_DIFF_DELETED_FILE_CHANGE_TYPE = 'D'

cli/helpers/sca_code_scanner.py

@@ -1,17 +1,65 @@
 import click
 from typing import List, Optional
+from git import Repo, GitCommandError
 from cli.utils.shell_executor import shell
 from cli.models import Document
 from cli.utils.path_utils import get_file_dir, join_paths
 from cyclient import logger
+from cli.consts import *
 
 BUILD_GRADLE_FILE_NAME = 'build.gradle'
 BUILD_GRADLE_KTS_FILE_NAME = 'build.gradle.kts'
 BUILD_GRADLE_DEP_TREE_FILE_NAME = 'gradle-dependencies-generated.txt'
 BUILD_GRADLE_DEP_TREE_TIMEOUT = 180
 
 
-def run_pre_scan_actions(context: click.Context, documents_to_scan: List[Document], is_git_diff: bool = False):
+def perform_pre_commit_range_scan_actions(path: str, from_commit_documents: List[Document],
+                                          from_commit_rev: str, to_commit_documents: List[Document],
+                                          to_commit_rev: str) -> None:
+    repo = Repo(path)
+    add_ecosystem_related_files_if_exists(from_commit_documents, repo, from_commit_rev)
+    add_ecosystem_related_files_if_exists(to_commit_documents, repo, to_commit_rev)
+
+
+def add_ecosystem_related_files_if_exists(documents: List[Document], repo: Repo, commit_rev: str):
+    documents_to_add: List[Document] = []
+    for doc in documents:
+        ecosystem = get_project_file_ecosystem(doc)
+        if ecosystem is None:
+            logger.debug("failed to resolve project file ecosystem: %s", doc.path)
+            continue
+        documents_to_add = get_ecosystem_project_files_if_exists(documents, commit_rev, doc, ecosystem, repo)
+
+    documents.extend(documents_to_add)
+
+
+def get_ecosystem_project_files_if_exists(documents, commit_rev, doc, ecosystem, repo):
+    documents_to_add: List[Document] = []
+    for ecosystem_project_file in PROJECT_FILES_BY_ECOSYSTEM_MAP.get(ecosystem):
+        file_to_search = join_paths(get_file_dir(doc.path), ecosystem_project_file)
+        if not is_project_file_exists_in_documents(documents, file_to_search):
+            try:
+                file_content = repo.git.show(f'{commit_rev}:{file_to_search}')
+            except GitCommandError:
+                continue
+            documents_to_add.append(Document(file_to_search, file_content))
+    return documents_to_add
+
+
+def is_project_file_exists_in_documents(documents: List[Document], file: str):
+    return any(doc for doc in documents if file == doc.path)
+
+
+def get_project_file_ecosystem(document: Document):
+    for ecosystem, project_files in PROJECT_FILES_BY_ECOSYSTEM_MAP.items():
+        for project_file in project_files:
+            if document.path.endswith(project_file):
+                return ecosystem
+    return None
+
+
+def add_dependencies_tree_document(context: click.Context, documents_to_scan: List[Document],
+                                   is_git_diff: bool = False) -> None:
     is_monitor_action = context.obj.get('monitor')
     project_path = context.params.get('path')
     documents_to_add: List[Document] = []
@@ -23,10 +71,11 @@ def run_pre_scan_actions(context: click.Context, documents_to_scan: List[Documen
                 logger.warning('Error occurred while trying to generate gradle dependencies tree. %s',
                                {'filename': document.path})
                 documents_to_add.append(
-                    Document(build_dep_tree_path(document.path), '', is_git_diff))
+                    Document(build_dep_tree_path(document.path, BUILD_GRADLE_DEP_TREE_FILE_NAME), '', is_git_diff))
             else:
                 documents_to_add.append(
-                    Document(build_dep_tree_path(document.path), gradle_dependencies_tree, is_git_diff))
+                    Document(build_dep_tree_path(document.path, BUILD_GRADLE_DEP_TREE_FILE_NAME),
+                             gradle_dependencies_tree, is_git_diff))
 
     documents_to_scan.extend(documents_to_add)
 
@@ -47,8 +96,8 @@ def try_generate_dependencies_tree(filename: str) -> Optional[str]:
     return gradle_dependencies
 
 
-def build_dep_tree_path(path):
-    return join_paths(get_file_dir(path), BUILD_GRADLE_DEP_TREE_FILE_NAME)
+def build_dep_tree_path(path: str, generated_file_name: str) -> str:
+    return join_paths(get_file_dir(path), generated_file_name)
 
 
 def is_gradle_project(document: Document) -> bool: