CM-17604 - Support SCA pre-commit hook (#51)

* Support SCA pre-commit hook

* Avoid failures in case file not exists in specific commit

* Avoid calling BE if no documents to scan

* Avoid failures in case file not exists in file system

* Refactoring and Typing

* use --scan-type instead of -t

* Refactoring

Author: avishaiamiel

.pre-commit-hooks.yaml

@@ -2,4 +2,9 @@
   name: Cycode pre commit defender
   language: python
   entry: cycode
-  args: ['scan', 'pre_commit']
+  args: [ 'scan', 'pre_commit' ]
+- id: cycode-sca
+  name: Cycode SCA pre commit defender
+  language: python
+  entry: cycode
+  args: [ 'scan', '--scan_type', 'sca', 'pre_commit' ]

cli/code_scanner.py

@@ -5,14 +5,15 @@
 from platform import platform
 from uuid import uuid4, UUID
 from typing import Optional
-from git import Repo, NULL_TREE, InvalidGitRepositoryError
+from git import Repo, NULL_TREE, InvalidGitRepositoryError, GitCommandError
 from sys import getsizeof
 from cli.printers import ResultsPrinter
 from cli.models import Document, DocumentDetections, Severity
 from cli.ci_integrations import get_commit_range
 from cli.consts import *
 from cli.config import configuration_manager
-from cli.utils.path_utils import is_sub_path, is_binary_file, get_file_size, get_relevant_files_in_path, get_path_by_os
+from cli.utils.path_utils import is_sub_path, is_binary_file, get_file_size, get_relevant_files_in_path, \
+    get_path_by_os, get_file_content
 from cli.utils.string_utils import get_content_size, is_binary_content
 from cli.user_settings.config_file_manager import ConfigFileManager
 from cli.zip_file import InMemoryZip
@@ -132,19 +133,31 @@ def scan_path(context: click.Context, path):
 @click.pass_context
 def pre_commit_scan(context: click.Context, ignored_args: List[str]):
     """ Use this command to scan the content that was not committed yet """
+    scan_type = context.obj['scan_type']
+    if scan_type == SCA_SCAN_TYPE:
+        return scan_sca_pre_commit(context)
+
     diff_files = Repo(os.getcwd()).index.diff("HEAD", create_patch=True, R=True)
     documents_to_scan = [Document(get_path_by_os(get_diff_file_path(file)), get_diff_file_content(file))
                          for file in diff_files]
     documents_to_scan = exclude_irrelevant_documents_to_scan(context, documents_to_scan)
     return scan_documents(context, documents_to_scan, is_git_diff=True)
 
 
+def scan_sca_pre_commit(context):
+    git_head_documents, pre_committed_documents = get_pre_commit_modified_documents()
+    git_head_documents = exclude_irrelevant_documents_to_scan(context, git_head_documents)
+    pre_committed_documents = exclude_irrelevant_documents_to_scan(context, pre_committed_documents)
+    sca_code_scanner.perform_pre_hook_range_scan_actions(git_head_documents, pre_committed_documents)
+    return scan_commit_range_documents(context, git_head_documents, pre_committed_documents)
+
+
 def scan_sca_commit_range(context: click.Context, path: str, commit_range: str):
     from_commit_rev, to_commit_rev = parse_commit_range(commit_range, path)
     from_commit_documents, to_commit_documents = \
         get_commit_range_modified_documents(path, from_commit_rev, to_commit_rev)
-    exclude_irrelevant_documents_to_scan(context, from_commit_documents)
-    exclude_irrelevant_documents_to_scan(context, to_commit_documents)
+    from_commit_documents = exclude_irrelevant_documents_to_scan(context, from_commit_documents)
+    to_commit_documents = exclude_irrelevant_documents_to_scan(context, to_commit_documents)
     sca_code_scanner.perform_pre_commit_range_scan_actions(path, from_commit_documents, from_commit_rev,
                                                            to_commit_documents, to_commit_rev)
     return scan_commit_range_documents(context, from_commit_documents, to_commit_documents)
@@ -210,11 +223,14 @@ def scan_commit_range_documents(context: click.Context, from_documents_to_scan:
     to_commit_zipped_documents = InMemoryZip()
 
     try:
-        from_commit_zipped_documents = zip_documents_to_scan(scan_type, from_commit_zipped_documents,
-                                                             from_documents_to_scan)
-        to_commit_zipped_documents = zip_documents_to_scan(scan_type, to_commit_zipped_documents, to_documents_to_scan)
-        scan_result = perform_commit_range_scan_async(cycode_client, from_commit_zipped_documents,
-                                                      to_commit_zipped_documents, scan_type, scan_parameters)
+        scan_result = init_default_scan_result(str(scan_id))
+        if should_scan_documents(from_documents_to_scan, to_documents_to_scan):
+            from_commit_zipped_documents = zip_documents_to_scan(scan_type, from_commit_zipped_documents,
+                                                                 from_documents_to_scan)
+            to_commit_zipped_documents = zip_documents_to_scan(scan_type, to_commit_zipped_documents,
+                                                               to_documents_to_scan)
+            scan_result = perform_commit_range_scan_async(cycode_client, from_commit_zipped_documents,
+                                                          to_commit_zipped_documents, scan_type, scan_parameters)
         all_detections_count, output_detections_count = \
             handle_scan_result(context, scan_result, scan_command_type, scan_type, severity_threshold,
                                to_documents_to_scan)
@@ -234,6 +250,10 @@ def scan_commit_range_documents(context: click.Context, from_documents_to_scan:
                         error_message)
 
 
+def should_scan_documents(from_documents_to_scan: List[Document], to_documents_to_scan: List[Document]) -> bool:
+    return len(from_documents_to_scan) > 0 or len(to_documents_to_scan) > 0
+
+
 def handle_scan_result(context, scan_result, scan_command_type, scan_type, severity_threshold, to_documents_to_scan):
     document_detections_list = enrich_scan_result(scan_result, to_documents_to_scan)
     relevant_document_detections_list = exclude_irrelevant_scan_results(document_detections_list, scan_type,
@@ -438,6 +458,26 @@ def exclude_detections_by_exclusions_configuration(scan_type: str, detections) -
     return [detection for detection in detections if not _should_exclude_detection(detection, exclusions)]
 
 
+def get_pre_commit_modified_documents():
+    repo = Repo(os.getcwd())
+    diff_files = repo.index.diff(GIT_HEAD_COMMIT_REV, create_patch=True, R=True)
+    git_head_documents = []
+    pre_committed_documents = []
+    for file in diff_files:
+        diff_file_path = get_diff_file_path(file)
+        file_path = get_path_by_os(diff_file_path)
+
+        file_content = sca_code_scanner.get_file_content_from_commit(repo, GIT_HEAD_COMMIT_REV, diff_file_path)
+        if file_content is not None:
+            git_head_documents.append(Document(file_path, file_content))
+
+        if os.path.exists(file_path):
+            file_content = get_file_content(file_path)
+            pre_committed_documents.append(Document(file_path, file_content))
+
+    return git_head_documents, pre_committed_documents
+
+
 def get_commit_range_modified_documents(path: str, from_commit_rev: str, to_commit_rev: str) -> (
         List[Document], List[Document]):
     from_commit_documents = []
@@ -449,11 +489,13 @@ def get_commit_range_modified_documents(path: str, from_commit_rev: str, to_comm
         diff_file_path = get_diff_file_path(blob)
         file_path = get_path_by_os(diff_file_path)
 
-        file_content = get_file_content_from_commit(repo, from_commit_rev, diff_file_path)
-        from_commit_documents.append(Document(file_path, file_content))
+        file_content = sca_code_scanner.get_file_content_from_commit(repo, from_commit_rev, diff_file_path)
+        if file_content is not None:
+            from_commit_documents.append(Document(file_path, file_content))
 
-        file_content = get_file_content_from_commit(repo, to_commit_rev, diff_file_path)
-        to_commit_documents.append(Document(file_path, file_content))
+        file_content = sca_code_scanner.get_file_content_from_commit(repo, to_commit_rev, diff_file_path)
+        if file_content is not None:
+            to_commit_documents.append(Document(file_path, file_content))
 
     return from_commit_documents, to_commit_documents
 
@@ -669,9 +711,7 @@ def _does_severity_match_severity_threshold(severity: str, severity_threshold: s
 
 
 def _get_scan_result(cycode_client, scan_id: str, scan_details: ScanDetailsResponse) -> ZippedFileScanResult:
-    scan_result = ZippedFileScanResult(did_detect=False, detections_per_file=[],
-                                       scan_id=scan_id,
-                                       report_url=_try_get_report_url(scan_details.metadata))
+    scan_result = init_default_scan_result(scan_id, scan_details.metadata)
     if not scan_details.detections_count:
         return scan_result
 
@@ -682,6 +722,12 @@ def _get_scan_result(cycode_client, scan_id: str, scan_details: ScanDetailsRespo
     return scan_result
 
 
+def init_default_scan_result(scan_id: str, scan_metadata: str = None) -> ZippedFileScanResult:
+    return ZippedFileScanResult(did_detect=False, detections_per_file=[],
+                                scan_id=scan_id,
+                                report_url=_try_get_report_url(scan_metadata))
+
+
 def _try_get_report_url(metadata: str) -> Optional[str]:
     if metadata is None:
         return None
@@ -734,7 +780,3 @@ def parse_commit_range(commit_range: str, path: str) -> (str, str):
         from_commit_rev = commit.hexsha
 
     return from_commit_rev, to_commit_rev
-
-
-def get_file_content_from_commit(repo: Repo, commit: str, file_path: str) -> str:
-    return repo.git.show(f'{commit}:{file_path}')

cli/consts.py

@@ -87,4 +87,6 @@
 SCAN_STATUS_COMPLETED = 'Completed'
 SCAN_STATUS_ERROR = 'Error'
 
+# git consts
 COMMIT_DIFF_DELETED_FILE_CHANGE_TYPE = 'D'
+GIT_HEAD_COMMIT_REV = 'HEAD'

cli/helpers/sca_code_scanner.py

@@ -1,9 +1,10 @@
+import os
 import click
 from typing import List, Optional
 from git import Repo, GitCommandError
 from cli.utils.shell_executor import shell
 from cli.models import Document
-from cli.utils.path_utils import get_file_dir, join_paths
+from cli.utils.path_utils import get_file_dir, join_paths, get_file_content
 from cyclient import logger
 from cli.consts import *
 
@@ -21,36 +22,44 @@ def perform_pre_commit_range_scan_actions(path: str, from_commit_documents: List
     add_ecosystem_related_files_if_exists(to_commit_documents, repo, to_commit_rev)
 
 
-def add_ecosystem_related_files_if_exists(documents: List[Document], repo: Repo, commit_rev: str):
-    documents_to_add: List[Document] = []
+def perform_pre_hook_range_scan_actions(git_head_documents: List[Document],
+                                        pre_committed_documents: List[Document]) -> None:
+    repo = Repo(os.getcwd())
+    add_ecosystem_related_files_if_exists(git_head_documents, repo, GIT_HEAD_COMMIT_REV)
+    add_ecosystem_related_files_if_exists(pre_committed_documents)
+
+
+def add_ecosystem_related_files_if_exists(documents: List[Document], repo: Optional[Repo] = None,
+                                          commit_rev: Optional[str] = None):
     for doc in documents:
         ecosystem = get_project_file_ecosystem(doc)
         if ecosystem is None:
             logger.debug("failed to resolve project file ecosystem: %s", doc.path)
             continue
-        documents_to_add = get_ecosystem_project_files_if_exists(documents, commit_rev, doc, ecosystem, repo)
+        documents_to_add = get_doc_ecosystem_related_project_files(doc, documents, ecosystem, commit_rev, repo)
+        documents.extend(documents_to_add)
 
-    documents.extend(documents_to_add)
 
-
-def get_ecosystem_project_files_if_exists(documents, commit_rev, doc, ecosystem, repo):
+def get_doc_ecosystem_related_project_files(doc: Document, documents: List[Document], ecosystem: str,
+                                            commit_rev: Optional[str], repo: Optional[Repo]) -> List[Document]:
     documents_to_add: List[Document] = []
     for ecosystem_project_file in PROJECT_FILES_BY_ECOSYSTEM_MAP.get(ecosystem):
         file_to_search = join_paths(get_file_dir(doc.path), ecosystem_project_file)
         if not is_project_file_exists_in_documents(documents, file_to_search):
-            try:
-                file_content = repo.git.show(f'{commit_rev}:{file_to_search}')
-            except GitCommandError:
-                continue
-            documents_to_add.append(Document(file_to_search, file_content))
+            file_content = get_file_content_from_commit(repo, commit_rev, file_to_search) if repo \
+                else get_file_content(file_to_search)
+
+            if file_content is not None:
+                documents_to_add.append(Document(file_to_search, file_content))
+
     return documents_to_add
 
 
-def is_project_file_exists_in_documents(documents: List[Document], file: str):
+def is_project_file_exists_in_documents(documents: List[Document], file: str) -> bool:
     return any(doc for doc in documents if file == doc.path)
 
 
-def get_project_file_ecosystem(document: Document):
+def get_project_file_ecosystem(document: Document) -> Optional[str]:
     for ecosystem, project_files in PROJECT_FILES_BY_ECOSYSTEM_MAP.items():
         for project_file in project_files:
             if document.path.endswith(project_file):
@@ -80,7 +89,7 @@ def add_dependencies_tree_document(context: click.Context, documents_to_scan: Li
     documents_to_scan.extend(documents_to_add)
 
 
-def get_manifest_file_path(document, is_monitor_action, project_path):
+def get_manifest_file_path(document: Document, is_monitor_action: bool, project_path: str) -> str:
     return join_paths(project_path, document.path) if is_monitor_action else document.path
 
 
@@ -102,3 +111,10 @@ def build_dep_tree_path(path: str, generated_file_name: str) -> str:
 
 def is_gradle_project(document: Document) -> bool:
     return document.path.endswith(BUILD_GRADLE_FILE_NAME) or document.path.endswith(BUILD_GRADLE_KTS_FILE_NAME)
+
+
+def get_file_content_from_commit(repo: Repo, commit: str, file_path: str) -> Optional[str]:
+    try:
+        return repo.git.show(f'{commit}:{file_path}')
+    except GitCommandError:
+        return None

cli/utils/path_utils.py

@@ -1,4 +1,4 @@
-from typing import Iterable, List
+from typing import Iterable, List, Optional
 import pathspec
 import os
 from pathlib import Path
@@ -59,3 +59,12 @@ def get_file_dir(path: str) -> str:
 
 def join_paths(path: str, filename: str) -> str:
     return os.path.join(path, filename)
+
+
+def get_file_content(file_path: str) -> Optional[str]:
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            content = f.read()
+        return content
+    except FileNotFoundError:
+        return None