move llm

Author: ckrause

.github/copilot-instructions.md

@@ -96,15 +96,6 @@ program_cache = ProgramCache("path/to/programs")
 program = program_cache.get_program(sequence_id)
 ```
 
-### Token Conversion Pattern:
-```python
-# Token conversion utilities
-from loda.ml.util import program_to_tokens, tokens_to_program
-
-tokens, vocab = program_to_tokens(program)
-reconstructed = tokens_to_program(tokens)
-```
-
 ## Testing Conventions
 
 - Use CSV files in `tests/operations/` for operation test cases

loda/ml/llm/README.md loda/llm/README.mdloda/ml/llm/README.md renamed to loda/llm/README.md

@@ -67,7 +67,7 @@ pip install -r requirements.txt
 ### 1. Prepare Training Data
 
 ```python
-from loda.ml.llm.data_preprocessing import create_dataset
+from loda.llm.data_preprocessing import create_dataset
 
 # Create training dataset from OEIS programs
 dataset = create_dataset(
@@ -81,7 +81,7 @@ dataset = create_dataset(
 ### 2. Train the Model
 
 ```python
-from loda.ml.llm.trainer import train_loda_llm
+from loda.llm.trainer import train_loda_llm
 
 # Train the model
 model = train_loda_llm(
@@ -96,7 +96,7 @@ model = train_loda_llm(
 
 Command line training:
 ```bash
-python -m loda.ml.llm.trainer \
+python -m loda.llm.trainer \
     --programs_dir programs/oeis \
     --output_dir trained_model \
     --max_examples 10000 \
@@ -106,7 +106,7 @@ python -m loda.ml.llm.trainer \
 ### 3. Generate Code
 
 ```python
-from loda.ml.llm.inference import load_model_for_inference
+from loda.llm.inference import load_model_for_inference
 
 # Load trained model
 generator = load_model_for_inference("trained_model")
@@ -122,13 +122,13 @@ for result in results:
 
 Interactive mode:
 ```bash
-python -m loda.ml.llm.inference --mode interactive --model_path trained_model
+python -m loda.llm.inference --mode interactive --model_path trained_model
 ```
 
 ### 4. Evaluate Performance
 
 ```python
-from loda.ml.llm.inference import evaluate_model
+from loda.llm.inference import evaluate_model
 
 # Evaluate on test set
 metrics, results = evaluate_model("trained_model", "test_data.json")
@@ -276,7 +276,7 @@ Choose model size based on your requirements:
 Add new training examples:
 
 ```python
-from loda.ml.llm.data_preprocessing import TrainingExample
+from loda.llm.data_preprocessing import TrainingExample
 
 custom_example = TrainingExample(
     sequence_id="custom_001",

loda/ml/llm/__init__.py loda/llm/__init__.pyloda/ml/llm/__init__.py renamed to loda/llm/__init__.py

@@ -13,7 +13,7 @@
 - Evaluation metrics for generated programs
 
 Example usage:
->>> from loda.ml.llm import LodaT5Model, LodaGenerator, train_loda_llm
+>>> from loda.llm import LodaT5Model, LodaGenerator, train_loda_llm
 >>> 
 >>> # Train a model
 >>> model = train_loda_llm("programs/oeis", "trained_model")

loda/ml/llm/data_preprocessing.py loda/llm/data_preprocessing.pyloda/ml/llm/data_preprocessing.py renamed to loda/llm/data_preprocessing.py

@@ -14,7 +14,7 @@
 import os
 import sys
 import tempfile
-from loda.ml.llm import (
+from loda.llm import (
     create_dataset, 
     train_loda_llm, 
     LodaGenerator,
@@ -123,7 +123,7 @@ def main():
             evaluator = LodaEvaluator(model)
 
             # Use a subset of the training data as test data for demo
-            from loda.ml.llm.data_preprocessing import DataPreprocessor
+            from loda.llm.data_preprocessing import DataPreprocessor
             preprocessor = DataPreprocessor(programs_dir)
             test_examples = preprocessor.create_training_examples(max_examples=10)
 
@@ -142,10 +142,13 @@ def main():
 
     print("\n" + "=" * 50)
     print("Example completed!")
-    print("\nTo use the LLM in your own code:")
+    print("To use the LLM in your own code:")
     print("1. Train a model: train_loda_llm('programs/oeis', 'my_model')")
     print("2. Load for inference: generator = LodaGenerator.load_model('my_model')")
     print("3. Generate code: results = generator.generate('your description')")
+    print("\nCommand line usage:")
+    print("- Train: python -m loda.llm.trainer --programs_dir programs/oeis")
+    print("- Interactive: python -m loda.llm.inference --mode interactive --model_path my_model")
 
     return 0