Add safety filters and more name variants

Author: Kaushik-Kumar-CEG

etc/scripts/dataset_pipeline/annotate_composites.py

@@ -11,42 +11,44 @@
 MARKER_RE = re.compile(r'\{\{([^}]*)\}\}', re.DOTALL)
 VERSION_SUFFIX_RE = re.compile(r'\s+v?\d[\d.]*(?:\s*(?:only|or[ -]later|\+))?$', re.IGNORECASE)
 
-# extra short forms used in rule text that the license index doesnt have
+# short forms that appear in rule text but arent in the license index
 EXTRA_NAMES = {
-    'gpl-2.0': ['GPL-2.0', 'GPLv2', 'GPL 2.0', 'GPL version 2'],
+    'gpl-2.0': ['GPL-2.0', 'GPLv2', 'GPL 2.0', 'GPL version 2', 'GPL Version 2'],
     'gpl-2.0-plus': ['GPL-2.0+', 'GPLv2+', 'GPL 2.0 or later'],
     'gpl-3.0': ['GPL-3.0', 'GPLv3', 'GPL 3.0', 'GPL version 3'],
     'gpl-3.0-plus': ['GPL-3.0+', 'GPLv3+', 'GPL 3.0 or later'],
+    'lgpl-2.0-plus': ['LGPL-2.0', 'LGPL-2.0+', 'LGPL 2.0'],
     'lgpl-2.1': ['LGPL-2.1', 'LGPLv2.1', 'LGPL 2.1'],
-    'lgpl-2.1-plus': ['LGPL-2.1+', 'LGPLv2.1+'],
+    'lgpl-2.1-plus': ['LGPL-2.1+', 'LGPLv2.1+', 'LGPL 2.1 or later'],
     'lgpl-3.0': ['LGPL-3.0', 'LGPLv3', 'LGPL 3.0'],
     'lgpl-3.0-plus': ['LGPL-3.0+', 'LGPLv3+'],
     'agpl-3.0': ['AGPL-3.0', 'AGPLv3', 'AGPL 3.0'],
     'agpl-3.0-plus': ['AGPL-3.0+', 'AGPLv3+'],
+    'mpl-1.1': ['MPL-1.1', 'MPL 1.1'],
     'mpl-2.0': ['MPL-2.0', 'MPL 2.0'],
     'apache-2.0': ['Apache-2.0', 'Apache 2.0'],
-    'bsd-new': ['BSD-3-Clause', 'BSD 3-Clause'],
-    'bsd-simplified': ['BSD-2-Clause', 'BSD 2-Clause'],
+    'bsd-new': ['BSD-3-Clause', 'BSD 3-Clause', 'BSD-3-clause'],
+    'bsd-simplified': ['BSD-2-Clause', 'BSD 2-Clause', 'BSD 2-clause'],
     'mit': ['MIT License', 'MIT license', 'MIT'],
     'isc': ['ISC License', 'ISC license', 'ISC'],
     'artistic-2.0': ['Artistic-2.0', 'Artistic 2.0'],
     'epl-1.0': ['EPL-1.0', 'EPL 1.0'],
     'epl-2.0': ['EPL-2.0', 'EPL 2.0'],
     'cc-by-4.0': ['CC-BY-4.0', 'CC BY 4.0'],
-    'unlicense': ['Unlicense'],
+    'unlicense': ['Unlicense', 'UNLICENSE'],
 }
 
 
 def strip_version_suffix(name):
-    """removing trailing version from a license name"""
+    """remove trailing version number from a license name"""
     result = VERSION_SUFFIX_RE.sub('', name).strip()
     if len(result) < 10 or result == name:
         return None
     return result
 
 
 def get_candidate_names(lic):
-    """collect names to search for.longest first"""
+    """collect names to search for, longest first"""
     names = []
     if lic.name:
         names.append(lic.name)
@@ -67,7 +69,7 @@ def get_candidate_names(lic):
 
 
 def find_in_text(text, candidates):
-    """case insensitive search.returns matched span having original case"""
+    """case insensitive search, returns the matched text in original case"""
     text_lower = text.lower()
     for name in candidates:
         if not name or len(name) < 3:
@@ -85,7 +87,7 @@ def find_in_text(text, candidates):
 @click.option('--limit', type=int, default=None)
 @click.option('--dry-run', is_flag=True)
 def main(rules_dir, expression_filter, limit, dry_run):
-    """annotate rules with required phrase markers"""
+    """Annotate composite license rules with required phrase markers"""
     if not rules_dir:
         repo_rules = Path(__file__).resolve().parents[3] / 'src' / 'licensedcode' / 'data' / 'rules'
         rules_dir = str(repo_rules) if repo_rules.is_dir() else default_rules_data_dir
@@ -111,6 +113,10 @@ def main(rules_dir, expression_filter, limit, dry_run):
             continue
         if getattr(rule, 'is_required_phrase', False):
             continue
+        if getattr(rule, 'skip_for_required_phrase_generation', False):
+            continue
+        if not getattr(rule, 'is_approx_matchable', True):
+            continue
         text = rule.text or ''
         if MARKER_RE.search(text):
             continue
@@ -157,8 +163,3 @@ def main(rules_dir, expression_filter, limit, dry_run):
 
 if __name__ == '__main__':
     main()
-
-# commands:
-# python etc/scripts/dataset_pipeline/annotate_composites.py --dry-run
-# python etc/scripts/dataset_pipeline/annotate_composites.py --expression-filter apache --limit 20
-# python etc/scripts/dataset_pipeline/annotate_composites.py