Remove unused normalize_phrase and required_phrases field from dataset output

Author: Kaushik-Kumar-CEG

etc/scripts/dataset_pipeline/build_dataset.py

@@ -2,7 +2,6 @@
 # outputs a JSONL dataset for NER model training
 import hashlib
 import json
-import re
 import unicodedata
 from collections import Counter
 from pathlib import Path
@@ -14,25 +13,8 @@
 from licensedcode.tokenize import required_phrase_splitter
 
 
-def normalize_phrase(phrase):
-    """Clean raw marker phrase for training"""
-    result = phrase
-    # replace html entities
-    result = result.replace('"', '"').replace('&', '&')
-    result = result.replace('&lt;', '<').replace('&gt;', '>')
-    # strip xml tags like <name>,</license> but keep urls in angle brackets
-    result = re.sub(r'<(?![a-zA-Z]+://)[^>]+>', '', result)
-    # remove markdown backticks
-    result = result.replace('`', '')
-    # collapse whitespace and trim
-    result = re.sub(r'\s+', ' ', result).strip()
-    # strip trailing/leading punct thats not meaningful
-    result = result.strip('.,;:<>')
-    return result
-
-
 def get_rule_type(rule):
-    """is_* flag set on the rule"""
+    """Return the is_* flag set on the rule"""
     for flag in ('is_license_text', 'is_license_notice', 'is_license_reference',
                  'is_license_tag', 'is_license_intro', 'is_license_clue',
                  'is_false_positive'):
@@ -42,11 +24,11 @@ def get_rule_type(rule):
 
 
 def tag_tokens(text):
-    """Tag each word token with a BIOES label"""
+    """Tag each word token with a BIOES label based on {{ }} markers"""
     tokens = []
     labels = []
     in_phrase = False
-    count = 0  # word tokens seen since the last {{
+    count = 0
 
     for tok in required_phrase_splitter(text):
         if tok == '{{':
@@ -65,16 +47,18 @@ def tag_tokens(text):
             count += 1
         else:
             labels.append('O')
+
+    assert len(tokens) == len(labels), f'token/label mismatch: {len(tokens)} vs {len(labels)}'
     return tokens, labels
 
 
 def assign_splits(results, threshold=50):
-    """80/10/10 split. common expressions (>= threshold rules) get split per-rule,
+    """80/10/10 split by license expression to prevent data leakage.
+    Expressions with >= threshold rules get split per-rule via hash,
     rare ones stay together in one split"""
     expr_counts = Counter(e['license_expression'] for e in results)
     heavy = {e for e, c in expr_counts.items() if c >= threshold}
 
-    # rare expressions: assign each to the split that needs more rules
     light_exprs = sorted((e for e in expr_counts if e not in heavy),
                          key=lambda x: (-expr_counts[x], x))
     total = sum(expr_counts[e] for e in light_exprs)
@@ -106,18 +90,17 @@ def main(rules_dir, output_dir):
 
     total_rules = 0
     annotated = 0
-    total_phrases = 0
     results = []
 
     click.echo(f'scanning rules from: {rules_path}')
     for rf in sorted(rules_path.glob('*.RULE')):
         try:
             rule = Rule.from_file(rule_file=str(rf))
-        except Exception:
+        except Exception as e:
+            click.echo(f'  skipping {rf.name}: {e}', err=True)
             continue
         total_rules += 1
 
-        # is_required_phrase rules don't need {{ }}.the flag covers them
         if getattr(rule, 'is_required_phrase', False):
             continue
 
@@ -133,27 +116,19 @@ def main(rules_dir, output_dir):
         if not phrases:
             continue
 
-        # word tokens + BIOES labels (computed before stripping markers)
         tokens, bioes_labels = tag_tokens(text)
 
-        # strip out the {{ }} markers
-        text = text.replace('{{', '').replace('}}', '')
-
-        valid_phrases = [
-            {'phrase': p, 'phrase_normalized': normalize_phrase(p)}
-            for p in phrases
-        ]
+        # strip markers for the clean text field
+        clean_text = text.replace('{{', '').replace('}}', '')
 
         annotated += 1
-        total_phrases += len(valid_phrases)
         results.append({
             'identifier': rule.identifier,
             'license_expression': rule.license_expression or '',
             'rule_type': get_rule_type(rule),
-            'text': text,
+            'text': clean_text,
             'tokens': tokens,
             'bioes_labels': bioes_labels,
-            'required_phrases': valid_phrases,
         })
 
     # split by license expression and write
@@ -162,7 +137,6 @@ def main(rules_dir, output_dir):
     for entry in results:
         expr = entry['license_expression']
         if expr in heavy:
-            # common expressions: hash rule name for 80/10/10
             bucket = int(hashlib.md5(entry['identifier'].encode('utf-8')).hexdigest(), 16) % 100
             if bucket < 80:
                 splits['train'].append(entry)
@@ -182,7 +156,6 @@ def main(rules_dir, output_dir):
     click.echo('\ndone')
     click.echo(f'  rules scanned: {total_rules}')
     click.echo(f'  annotated: {annotated}')
-    click.echo(f'  phrases extracted: {total_phrases}')
     click.echo(f'  train: {len(splits["train"])}  val: {len(splits["val"])}  test: {len(splits["test"])}')
     click.echo(f'  output: {out_dir}')
 

etc/scripts/dataset_pipeline/test_build_dataset.py

@@ -3,37 +3,7 @@
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).parent))
-from build_dataset import normalize_phrase, tag_tokens, assign_splits
-
-
-class TestNormalizePhrase:
-
-    def test_html_entities(self):
-        assert normalize_phrase('<b>MIT</b>') == 'MIT'
-        assert normalize_phrase('the "License"') == 'the "License"'
-        assert normalize_phrase('foo & bar') == 'foo & bar'
-
-    def test_preserves_urls_in_angle_brackets(self):
-        result = normalize_phrase('<http://example.com/LICENSE>')
-        assert result == 'http://example.com/LICENSE'
-
-    def test_strips_xml_tags(self):
-        assert normalize_phrase('<name>Apache 2.0</name>') == 'Apache 2.0'
-        assert normalize_phrase('<license>MIT</license>') == 'MIT'
-
-    def test_strips_backticks(self):
-        assert normalize_phrase('`MIT License`') == 'MIT License'
-
-    def test_collapses_whitespace(self):
-        assert normalize_phrase('GNU  General\n Public   License') == 'GNU General Public License'
-
-    def test_strips_trailing_punct(self):
-        assert normalize_phrase('Apache 2.0.') == 'Apache 2.0'
-        assert normalize_phrase(',MIT,') == 'MIT'
-
-    def test_empty_after_strip(self):
-        assert normalize_phrase('<foo>') == ''
-        assert normalize_phrase('...') == ''
+from build_dataset import tag_tokens, assign_splits
 
 
 class TestTagTokens: