Fixed #583 - ability to exclude in check and inventory

 * Add `--exclude` to check and inventory
 * Updated docs and tests
 * Updated changelog and version

Signed-off-by: Chin Yeung Li <tli@nexb.com>

Author: chinyeungli

CHANGELOG.rst

@@ -2,10 +2,10 @@
 Changelog
 
 xxxx-xx-xx
-    Release xx.x.x
+    Release 11.1.0
 
     * Drop support for python version earlier than 3.9
-
+    * Add ability to "exclude" path in the check and inventory #583
 
 2024-09-16
     Release 11.0.2

about.ABOUT

@@ -1,6 +1,6 @@
 about_resource: .
 name: AboutCode-toolkit
-version: 11.0.2
+version: 11.1.0
 author: Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez
 copyright: Copyright (c) nexB Inc.
 description: |

docs/source/reference.rst

@@ -187,6 +187,8 @@ Options
 
         ..  code-block:: none
 
+                --exclude PATTERN      Exclude the processing of the specified input pattern
+                                        (e.g. *tests* or test/).
                 --license              Validate the license_expression value in the input.
                 --djc api_url api_key  Validate license_expression from a DejaCode License
                                        Library API URL using the API KEY.
@@ -204,6 +206,14 @@ Details
 
         ..  code-block:: none
 
+                --exclude
+                    Exclude the processing of the specified input pattern
+
+                    It takes a pattern or an exact directory as an argument.
+                    Multiple `--exclude` can be used.
+
+                $ about check --exclude ./tests/ --exclude "*sample*" /home/project/about_files/
+
                 --license
                     Validate the license_expression value in the input.
 
@@ -240,9 +250,25 @@ Details
 
 Special Notes
 -------------
+`--djc`
+^^^^^^^
+
 If no `--djc` option is set, the tool will default to check license_expression from
 ScanCode LicenseDB.
 
+`--exclude`
+^^^^^^^^^^^
+
+As the `--exclude` option accepts patterns that may include wildcards, the running shell could
+expand these patterns into filenames, potentially causing errors. To avoid this,
+ensure the pattern is wrapped in quotes.
+
+On Windows, users can either use `^` to escape the `*` or use `--%` before `--exclude` to prevent shell globbing.
+
+$ about check /home/project/about_files/ --exclude "tests^*"
+
+$ about check /home/project/about_files/ --% --exclude "*tests*"
+
 collect_redist_src
 ==================
 
@@ -566,6 +592,8 @@ Options
 
         ..  code-block:: none
 
+                --exclude PATTERN               Exclude the processing of the specified input pattern
+                                                (e.g. *tests* or test/).
                 -f, --format [json|csv|excel]   Set OUTPUT file format.  [default: csv]
                 -q, --quiet                     Do not print any error/warning.
                 --verbose                       Show all the errors and warning.
@@ -647,6 +675,19 @@ To support multiple license file for a license, the correct format is to separat
 
 Note that if license_name is not provided, the license key will be used as the license name.
 
+`--exclude`
+^^^^^^^^^^^
+
+As the `--exclude` option accepts patterns that may include wildcards, the running shell could
+expand these patterns into filenames, potentially causing errors. To avoid this,
+ensure the pattern is wrapped in quotes.
+
+On Windows, users can either use `^` to escape the `*` or use `--%` before `--exclude` to prevent shell globbing.
+
+$ about check /home/project/about_files/ --exclude "tests^*"
+
+$ about check /home/project/about_files/ --% --exclude "*tests*"
+
 transform
 =========
 

src/attributecode/cmd.py

@@ -153,6 +153,10 @@ def validate_extensions(ctx, param, value, extensions=tuple(('.csv', '.json',)))
                 required=True,
                 metavar='OUTPUT',
                 type=click.Path(exists=False, dir_okay=False, writable=True, resolve_path=True))
+@click.option('--exclude',
+              multiple=True,
+              metavar='PATTERN',
+              help='Exclude the processing of the specified input pattern (e.g. *tests* or test/).')
 @click.option('-f', '--format',
               is_flag=False,
               default='csv',
@@ -166,7 +170,7 @@ def validate_extensions(ctx, param, value, extensions=tuple(('.csv', '.json',)))
               is_flag=True,
               help='Show all error and warning messages.')
 @click.help_option('-h', '--help')
-def inventory(location, output, format, quiet, verbose):  # NOQA
+def inventory(location, output, exclude, format, quiet, verbose):  # NOQA
     """
 Collect the inventory of .ABOUT files to a CSV/JSON/XLSX file.
 
@@ -181,7 +185,7 @@ def inventory(location, output, format, quiet, verbose):  # NOQA
     if location.lower().endswith('.zip'):
         # accept zipped ABOUT files as input
         location = extract_zip(location)
-    errors, abouts = collect_inventory(location)
+    errors, abouts = collect_inventory(location, exclude)
     write_output(abouts=abouts, location=output, format=format)
 
     errors_count = report_errors(
@@ -675,7 +679,6 @@ def collect_redist_src(location, output, from_inventory, with_structures, zip, q
 
 # FIXME: This is really only a dupe of the Inventory command
 
-
 @about.command(cls=AboutCommand,
                short_help='Validate that the format of .ABOUT files is correct and report '
                'errors and warnings.')
@@ -684,6 +687,10 @@ def collect_redist_src(location, output, from_inventory, with_structures, zip, q
                 metavar='LOCATION',
                 type=click.Path(
                     exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True))
+@click.option('--exclude',
+              multiple=True,
+              metavar='PATTERN',
+              help='Exclude the processing of the specified input pattern (e.g. *tests* or test/).')
 @click.option('--license',
               is_flag=True,
               help='Validate the license_expression value in the input.')
@@ -701,7 +708,7 @@ def collect_redist_src(location, output, from_inventory, with_structures, zip, q
               is_flag=True,
               help='Show all error and warning messages.')
 @click.help_option('-h', '--help')
-def check(location, license, djc, log, verbose):
+def check(location, exclude, license, djc, log, verbose):
     """
 Check .ABOUT file(s) at LOCATION for validity and print error messages.
 
@@ -722,7 +729,8 @@ def check(location, license, djc, log, verbose):
         api_url = djc[0].strip("'").strip('"')
         api_key = djc[1].strip("'").strip('"')
     click.echo('Checking ABOUT files...')
-    errors, abouts = collect_inventory(location)
+
+    errors, abouts = collect_inventory(location, exclude)
 
     # Validate license_expression
     if license:

src/attributecode/model.py

@@ -1553,14 +1553,14 @@ def dump_lic(self, location, license_dict):
         return license_key_name_context_url
 
 
-def collect_inventory(location):
+def collect_inventory(location, exclude=None):
     """
     Collect ABOUT files at location and return a list of errors and a list of
     About objects.
     """
     errors = []
     input_location = util.get_absolute(location)
-    about_locations = list(util.get_about_locations(input_location))
+    about_locations = list(util.get_about_locations(input_location, exclude))
 
     name_errors = util.check_file_names(about_locations)
     errors.extend(name_errors)
@@ -1572,7 +1572,7 @@ def collect_inventory(location):
         for severity, message in about.errors:
             if 'Custom Field' in message:
                 field_name = message.replace('Custom Field: ', '').strip()
-                if not field_name in custom_fields_list:
+                if field_name not in custom_fields_list:
                     custom_fields_list.append(field_name)
             else:
                 msg = (about_file_path + ": " + message)