improve data processing

Author: ckrause

loda/llm/data_preprocessing.py

@@ -106,14 +106,22 @@ def clean_loda_code(self, program_text: str) -> str:
         code_lines = []
 
         for line in lines:
-            # Skip comment lines
+            # Skip comment lines (lines that start with ;)
             if line.strip().startswith(';'):
                 continue
             # Skip empty lines
             if not line.strip():
                 continue
-            # Add the code line
-            code_lines.append(line.strip())
+            
+            # Remove inline comments (everything after ; on the same line)
+            if ';' in line:
+                code_part = line.split(';')[0].strip()
+            else:
+                code_part = line.strip()
+            
+            # Only add non-empty code lines
+            if code_part:
+                code_lines.append(code_part)
 
         return '\n'.join(code_lines)
 

tests/test_llm.py

@@ -97,6 +97,20 @@ def test_clean_loda_code(self):
         clean_code = preprocessor.clean_loda_code(dirty_code)
         expected = "mov $1,$0\npow $1,2\nmov $0,$1"
         self.assertEqual(clean_code, expected)
+        
+        # Test inline comment removal
+        code_with_inline_comments = (
+            "; Header comment\n"
+            "add $0,1\n"
+            "sub $0,2 ; inline comment here\n"
+            "mul $1,3   ; another inline comment\n"
+            "; Full line comment\n"
+            "div $2,4\n"
+        )
+        
+        clean_inline = preprocessor.clean_loda_code(code_with_inline_comments)
+        expected_inline = "add $0,1\nsub $0,2\nmul $1,3\ndiv $2,4"
+        self.assertEqual(clean_inline, expected_inline)
 
     def test_training_example_creation(self):
         """Test TrainingExample creation."""