Merge pull request #32 from alexrhein/ts/33505_ignore_submodules

TS-33505 allow ignoring submodules in precommit

Author: alexrhein

teamscale_precommit_client/git_utils.py

@@ -6,7 +6,7 @@
 import os
 from io import open
 
-from git import Repo, InvalidGitRepositoryError
+from git import Repo, InvalidGitRepositoryError, Diffable
 
 # [M]odified, [A]dded, [C]opied, [T]ype changed, [R]enamed (R092 should be R according to
 # https://gitpython.readthedocs.io/en/stable/reference.html#git.diff.DiffIndex, but testing it locally gave R092)
@@ -74,6 +74,9 @@ def filter_changed_files(changed_files, path_to_repository, file_encoding):
     filtered_files = []
     for changed_file in changed_files:
         file_is_valid = True
+        if (os.path.isdir(os.path.join(path_to_repository, changed_file))):
+            #ignore directories (e.g., git submodule folders)
+            continue
         if os.path.getsize(os.path.join(path_to_repository, changed_file)) > 1 * 1024 * 1024:
             print('File too large for precommit analysis. Ignoring: %s' % changed_file)
             file_is_valid = False
@@ -97,7 +100,7 @@ def filter_changed_files(changed_files, path_to_repository, file_encoding):
     return filtered_files
 
 
-def get_changed_files_and_content(path_to_repository, file_encoding):
+def get_changed_files_and_content(path_to_repository, file_encoding, ignore_subrepositories):
     """Utility method for getting the currently changed files from a Git repository.
 
     Filters the changed files using `filter_changed_files`.
@@ -108,13 +111,15 @@ def get_changed_files_and_content(path_to_repository, file_encoding):
 
         Returns:
             dict: Mapping of filename to file content for all changed files in the provided repository.
+            :param ignore_subrepositories:
     """
-    changed_files = filter_changed_files(get_changed_files(path_to_repository), path_to_repository, file_encoding)
+    changed_files = filter_changed_files(get_changed_files(path_to_repository, ignore_subrepositories),
+                                         path_to_repository, file_encoding)
     return {filename: open(os.path.join(path_to_repository, filename), encoding=file_encoding).read() for filename in
             changed_files}
 
 
-def get_changed_files(path_to_repository):
+def get_changed_files(path_to_repository, ignore_subrepositories):
     """Utility method for getting the currently changed files from a Git repository.
 
         Args:
@@ -123,11 +128,11 @@ def get_changed_files(path_to_repository):
         Returns:
             List(str): List of filenames of all changed files in the provided repository.
     """
-    diff = _get_diff_to_last_commit(path_to_repository)
+    diff = _get_diff_to_last_commit(path_to_repository, ignore_subrepositories)
     return [item.b_path for item in diff if item.change_type in _CHANGE_TYPES_CONSIDERED_FOR_PRECOMMIT]
 
 
-def get_deleted_files(path_to_repository):
+def get_deleted_files(path_to_repository, ignore_subrepositories):
     """Utility method for getting the deleted files from a Git repository.
 
         Args:
@@ -136,11 +141,11 @@ def get_deleted_files(path_to_repository):
         Returns:
             List(str): List of filenames of all deleted files in the provided repository.
     """
-    diff = _get_diff_to_last_commit(path_to_repository)
+    diff = _get_diff_to_last_commit(path_to_repository, ignore_subrepositories)
     return [item.b_path for item in diff if item.change_type == _CHANGE_TYPE_DELETED]
 
 
-def _get_diff_to_last_commit(path_to_repository):
+def _get_diff_to_last_commit(path_to_repository, ignore_subrepositories):
     """ Utility method for getting a diff between the working copy and the HEAD commit
 
         Args:
@@ -150,6 +155,11 @@ def _get_diff_to_last_commit(path_to_repository):
             List(git.diff.Diff): List of Diff objects for every file
     """
     repo = Repo(path_to_repository)
-    unstaged_diff = repo.index.diff(None)
-    staged_diff = repo.head.commit.diff()
+    if ignore_subrepositories==True:
+        unstaged_diff = repo.index.diff(other=None, paths=None, create_patch=False, ignore_submodules="all")
+        staged_diff = repo.head.commit.diff(other=Diffable.Index, paths=None, create_patch=False, ignore_submodules="all")
+    else:
+        unstaged_diff = repo.index.diff(other=None, paths=None, create_patch=False)
+        staged_diff = repo.head.commit.diff(other=Diffable.Index, paths=None, create_patch=False)
+
     return unstaged_diff + staged_diff

teamscale_precommit_client/precommit_client.py

@@ -32,7 +32,8 @@ class PrecommitClient:
     def __init__(self, teamscale_config, repository_path, path_prefix=DEFAULT_PATH_PREFIX, project_subpath='',
                  analyzed_file=None, verify=True, omit_links_to_findings=False, exclude_findings_in_changed_code=False,
                  fetch_existing_findings=False, fetch_all_findings=False, fetch_existing_findings_in_changes=False,
-                 fail_on_red_findings=False, log_to_stderr=False, file_encoding=DEFAULT_FILE_ENCODING):
+                 fail_on_red_findings=False, log_to_stderr=False, file_encoding=DEFAULT_FILE_ENCODING,
+                 ignore_subrepositories=False):
         """Constructor"""
         self.teamscale_client = TeamscaleClient(teamscale_config.url, teamscale_config.username,
                                                 teamscale_config.access_token, teamscale_config.project_id, verify)
@@ -59,6 +60,7 @@ def __init__(self, teamscale_config, repository_path, path_prefix=DEFAULT_PATH_P
         self.current_branch = ''
         self.parent_commit_timestamp = 0
         self.file_encoding = file_encoding
+        self.ignore_subrepositories = ignore_subrepositories
 
     def run(self):
         """Performs the precommit analysis. Depending on the modifications made and the flags provided to the client,
@@ -89,8 +91,9 @@ def _calculate_modifications(self):
         if not self.repository_path or not os.path.exists(self.repository_path) or not os.path.isdir(
                 self.repository_path):
             raise RuntimeError('Invalid path to file in repository: %s' % self.repository_path)
-        self.changed_files = get_changed_files_and_content(self.repository_path, self.file_encoding)
-        self.deleted_files = get_deleted_files(self.repository_path)
+        self.changed_files = get_changed_files_and_content(self.repository_path, self.file_encoding,
+                                                           self.ignore_subrepositories)
+        self.deleted_files = get_deleted_files(self.repository_path, self.ignore_subrepositories)
 
     def _retrieve_current_branch(self):
         """Retrieves the current branch from the repository."""
@@ -315,8 +318,12 @@ def _parse_args():
     parser.add_argument('--file-encoding', metavar='FILE_ENCODING', type=str,
                         help='The encoding of your files '
                              '(c.f. https://docs.python.org/3/library/codecs.html#standard-encodings).'
-                             ' By default, the system encoding will be used.',
-                        default=DEFAULT_FILE_ENCODING)
+                             ' By default, the system encoding will be used.', default=DEFAULT_FILE_ENCODING)
+    parser.add_argument('--ignore-subrepositories', dest='ignore_subrepositories', action='store_const', const=True, default=False,
+                        help='When this option is set, then we ignore subrepositories '
+                             '(git submodules) in the current repository when determining which files changed. This '
+                             'affects the files considered for precommit analysis. It does not affect the retrieval '
+                             'of "existing" findings from the Teamscale server.')
     return parser.parse_args()
 
 
@@ -344,7 +351,8 @@ def _configure_precommit_client(parsed_args):
                            fetch_existing_findings_in_changes=parsed_args.fetch_existing_findings_in_changes,
                            fail_on_red_findings=parsed_args.fail_on_red_findings,
                            log_to_stderr=parsed_args.log_to_stderr,
-                           file_encoding=parsed_args.file_encoding)
+                           file_encoding=parsed_args.file_encoding,
+                           ignore_subrepositories=parsed_args.ignore_subrepositories)
 
 
 def run():