Skip to content

Commit 5a27999

Browse files
Jonathan Harrisonclaude
Jonathan Harrison
and
claude
committed
Submit constraint-tracker LoRA training to HF Jobs with uv
- Created training wrapper (train_hf_job_with_deps.py) with dependency installation - Created pyproject.toml specifying all required packages - Implemented uv-based HF Jobs submission (submit_hf_job_uv.py) - Job ID: 6a0fed3cb33ece92698c00ff - Status: RUNNING on HF Jobs infrastructure - Base: Raiff1982/codette-llama-3.1-8b-merged - Dataset: constraint_tracking.jsonl (14 examples) - Adapter: constraint_tracker (3 epochs, lr=1e-4) - Expected: continuity_anchor_recall 0.70+ (from 0.200) - ETA: 4-6 hours on A10G GPU Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent ac87052 commit 5a27999

11 files changed

Lines changed: 682 additions & 399 deletions

‎.claude/settings.local.json‎

Lines changed: 0 additions & 376 deletions
This file was deleted.

‎cocoons/behavior_memory.json‎

Lines changed: 21 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,5 @@
11
{
22
"lessons": [
3-
{
4-
"type": "success",
5-
"timestamp": 1774212644.589494,
6-
"adapter": "empathy",
7-
"constraints": {
8-
"max_words": 10,
9-
"max_sentences": 1
10-
},
11-
"word_count": 3,
12-
"query_preview": "What is 2+2? One sentence under 10 words."
13-
},
14-
{
15-
"type": "success",
16-
"timestamp": 1774212667.692114,
17-
"adapter": "quantum",
18-
"constraints": {
19-
"max_words": 12,
20-
"max_sentences": 1
21-
},
22-
"word_count": 10,
23-
"query_preview": "Why do we dream? One sentence, under 12 words, include uncertainty."
24-
},
253
{
264
"type": "success",
275
"timestamp": 1774212684.675397,
@@ -508,7 +486,27 @@
508486
},
509487
"word_count": 14,
510488
"query_preview": "For this session, keep answers under 15 words and remember the phrase cobalt anc"
489+
},
490+
{
491+
"type": "success",
492+
"timestamp": 1779252654.7717123,
493+
"adapter": "philosophy",
494+
"constraints": {
495+
"max_words": 15
496+
},
497+
"word_count": 12,
498+
"query_preview": "For this session, keep answers under 15 words and remember the phrase cobalt anc"
499+
},
500+
{
501+
"type": "success",
502+
"timestamp": 1779252698.3913093,
503+
"adapter": "systems_architecture",
504+
"constraints": {
505+
"max_words": 15
506+
},
507+
"word_count": 9,
508+
"query_preview": "For this session, keep answers under 15 words and remember the phrase cobalt anc"
511509
}
512510
],
513-
"updated_at": 1777649279.5417144
511+
"updated_at": 1779252698.391664
514512
}

‎hf_job_info.json‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
"job_id": "6a0fed3cb33ece92698c00ff",
3+
"status": "JobStatus(stage='RUNNING', message=None)",
4+
"url": "https://huggingface.co/jobs/Raiff1982/6a0fed3cb33ece92698c00ff",
5+
"repo": "Raiff1982/codette-lora-adapters",
6+
"base_model": "Raiff1982/codette-llama-3.1-8b-merged",
7+
"dataset": "constraint_tracking.jsonl",
8+
"adapter": "constraint_tracker",
9+
"epochs": 3,
10+
"learning_rate": "1e-4",
11+
"method": "uv",
12+
"dependencies": [
13+
"torch>=2.0.0",
14+
"transformers>=4.36.0",
15+
"datasets>=2.14.0",
16+
"peft>=0.7.0",
17+
"trl>=0.7.0",
18+
"huggingface-hub>=0.19.0",
19+
"bitsandbytes>=0.41.0"
20+
]
21+
}

‎pyproject.toml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
[build-system]
2+
requires = ["setuptools>=68.0", "wheel"]
3+
build-backend = "setuptools.build_meta"
4+
5+
[project]
6+
name = "codette-training"
7+
version = "2.2.0"
8+
description = "Codette LoRA training on HuggingFace Jobs"
9+
requires-python = ">=3.10"
10+
11+
dependencies = [
12+
"torch>=2.0.0",
13+
"transformers>=4.36.0",
14+
"datasets>=2.14.0",
15+
"peft>=0.7.0",
16+
"trl>=0.7.0",
17+
"huggingface-hub>=0.19.0",
18+
"bitsandbytes>=0.41.0",
19+
]

‎submit_hf_job_api.py‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
#!/usr/bin/env python3
2+
"""
3+
Submit constraint-tracker LoRA training job to HF Jobs using the Python API.
4+
Uses uv for dependency management in the job environment.
5+
"""
6+
7+
import os
8+
import json
9+
from pathlib import Path
10+
from huggingface_hub import HfApi, submit_job
11+
12+
HF_TOKEN = os.environ.get("HF_TOKEN")
13+
if not HF_TOKEN:
14+
print("ERROR: HF_TOKEN environment variable not set")
15+
exit(1)
16+
17+
print("=" * 70)
18+
print("Codette Constraint-Tracker LoRA - HF Jobs Submission (Python API)")
19+
print("=" * 70)
20+
21+
# Configuration
22+
MODEL_REPO = "Raiff1982/codette-llama-3.1-8b-merged"
23+
DATASET_REPO = "Raiff1982/codette-training-data"
24+
OUTPUT_REPO = "Raiff1982/codette-lora-adapters"
25+
26+
print(f"\nJob Configuration:")
27+
print(f" Base Model: {MODEL_REPO}")
28+
print(f" Dataset: constraint_tracking.jsonl")
29+
print(f" Adapter: constraint_tracker")
30+
print(f" Epochs: 3")
31+
print(f" Learning Rate: 1e-4")
32+
print(f" Output Repo: {OUTPUT_REPO}")
33+
print(f" Environment: Python 3.10 with torch, transformers, peft, trl")
34+
35+
# Create the training command
36+
# This will run the training script with dependency installation
37+
training_command = """
38+
python -m pip install -q torch transformers datasets peft trl huggingface-hub bitsandbytes numpy &&
39+
cd /workspace &&
40+
python training/train_hf_job_with_deps.py
41+
"""
42+
43+
print(f"\n" + "=" * 70)
44+
print("Submitting job to HF Jobs...")
45+
print("=" * 70)
46+
47+
api = HfApi(token=HF_TOKEN)
48+
49+
try:
50+
# Submit the job using HF's job submission API
51+
job = submit_job(
52+
command=training_command,
53+
repo_type="model",
54+
repo_id=OUTPUT_REPO,
55+
private=True,
56+
token=HF_TOKEN,
57+
)
58+
59+
print(f"\n{'=' * 70}")
60+
print("✓ Job submitted successfully!")
61+
print(f"{'=' * 70}")
62+
print(f"\nJob Details:")
63+
print(f" Job ID: {job.job_id}")
64+
print(f" Status: {job.status}")
65+
print(f" Repo: {OUTPUT_REPO}")
66+
67+
# Save job info
68+
job_info = {
69+
"job_id": job.job_id,
70+
"status": job.status,
71+
"repo": OUTPUT_REPO,
72+
"base_model": MODEL_REPO,
73+
"dataset": "constraint_tracking.jsonl",
74+
"adapter": "constraint_tracker",
75+
"epochs": 3,
76+
"learning_rate": "1e-4",
77+
"submission_time": str(Path.cwd()),
78+
}
79+
80+
job_info_path = Path("/tmp/hf_job_info.json")
81+
with open(job_info_path, "w") as f:
82+
json.dump(job_info, f, indent=2)
83+
84+
print(f"\nJob info saved to: {job_info_path}")
85+
print(f"\nMonitor your training at:")
86+
print(f" https://huggingface.co/{OUTPUT_REPO}")
87+
print(f"\nThe job will:")
88+
print(f" 1. Install dependencies (torch, transformers, peft, trl, etc.)")
89+
print(f" 2. Download base model: {MODEL_REPO}")
90+
print(f" 3. Download dataset: constraint_tracking.jsonl from {DATASET_REPO}")
91+
print(f" 4. Train constraint_tracker LoRA adapter (3 epochs, lr=1e-4)")
92+
print(f" 5. Upload results to {OUTPUT_REPO}")
93+
print(f"\nEstimated time: 4-6 hours on A10G GPU")
94+
print(f"\nNext steps:")
95+
print(f" 1. Monitor job status at the URL above")
96+
print(f" 2. Once training completes, run:")
97+
print(f" python benchmarks/codette_runtime_benchmark.py")
98+
print(f" 3. Verify continuity_anchor_recall score ≥ 0.70")
99+
100+
except Exception as e:
101+
print(f"\n{'=' * 70}")
102+
print("✗ Job submission failed")
103+
print(f"{'=' * 70}")
104+
print(f"\nError: {e}")
105+
print(f"\nTroubleshooting:")
106+
print(f" 1. Verify HF_TOKEN is correct")
107+
print(f" 2. Check that {OUTPUT_REPO} exists and you have write access")
108+
print(f" 3. Verify internet connection")
109+
print(f" 4. Check HF API status: https://status.huggingface.co/")
110+
print(f"\nTo retry, run:")
111+
print(f" export HF_TOKEN='your-token'")
112+
print(f" python submit_hf_job_api.py")
113+
exit(1)
114+
115+
print(f"\n{'=' * 70}")
116+
print("Submission complete!")
117+
print(f"{'=' * 70}\n")

‎submit_hf_job_uv.py‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
#!/usr/bin/env python3
2+
"""
3+
Submit constraint-tracker LoRA training job to HF Jobs using uv for dependency management.
4+
Uses the HuggingFace Hub API with run_uv_job for optimal environment setup.
5+
"""
6+
7+
import os
8+
import json
9+
from pathlib import Path
10+
from huggingface_hub import HfApi
11+
12+
HF_TOKEN = os.environ.get("HF_TOKEN")
13+
if not HF_TOKEN:
14+
print("ERROR: HF_TOKEN environment variable not set")
15+
exit(1)
16+
17+
print("=" * 70)
18+
print("Codette Constraint-Tracker LoRA - HF Jobs Submission (uv mode)")
19+
print("=" * 70)
20+
21+
# Configuration
22+
MODEL_REPO = "Raiff1982/codette-llama-3.1-8b-merged"
23+
DATASET_REPO = "Raiff1982/codette-training-data"
24+
OUTPUT_REPO = "Raiff1982/codette-lora-adapters"
25+
26+
print(f"\nJob Configuration:")
27+
print(f" Base Model: {MODEL_REPO}")
28+
print(f" Dataset: constraint_tracking.jsonl")
29+
print(f" Adapter: constraint_tracker")
30+
print(f" Epochs: 3")
31+
print(f" Learning Rate: 1e-4")
32+
print(f" Output Repo: {OUTPUT_REPO}")
33+
print(f" Environment: Python 3.10 with uv dependency management")
34+
35+
# Dependencies to install
36+
dependencies = [
37+
"torch>=2.0.0",
38+
"transformers>=4.36.0",
39+
"datasets>=2.14.0",
40+
"peft>=0.7.0",
41+
"trl>=0.7.0",
42+
"huggingface-hub>=0.19.0",
43+
"bitsandbytes>=0.41.0",
44+
]
45+
46+
print(f"\nDependencies to install:")
47+
for dep in dependencies:
48+
print(f" - {dep}")
49+
50+
print(f"\n" + "=" * 70)
51+
print("Submitting job to HF Jobs (using uv for dependencies)...")
52+
print("=" * 70)
53+
54+
api = HfApi(token=HF_TOKEN)
55+
56+
try:
57+
# Use run_uv_job to submit with uv for dependency management
58+
job = api.run_uv_job(
59+
script="training/train_hf_job_with_deps.py",
60+
dependencies=dependencies,
61+
env={
62+
"HF_TOKEN": HF_TOKEN,
63+
},
64+
token=HF_TOKEN,
65+
)
66+
67+
print(f"\n{'=' * 70}")
68+
print("[SUCCESS] Job submitted successfully!")
69+
print(f"{'=' * 70}")
70+
71+
print(f"\nJob Details:")
72+
print(f" Job ID: {job.id}")
73+
print(f" Status: {job.status}")
74+
print(f" URL: {job.url}")
75+
print(f" Repo: {OUTPUT_REPO}")
76+
77+
# Save job info (convert status to string)
78+
job_info = {
79+
"job_id": job.id,
80+
"status": str(job.status),
81+
"url": job.url,
82+
"repo": OUTPUT_REPO,
83+
"base_model": MODEL_REPO,
84+
"dataset": "constraint_tracking.jsonl",
85+
"adapter": "constraint_tracker",
86+
"epochs": 3,
87+
"learning_rate": "1e-4",
88+
"method": "uv",
89+
"dependencies": dependencies,
90+
}
91+
92+
job_info_path = Path("hf_job_info.json")
93+
with open(job_info_path, "w") as f:
94+
json.dump(job_info, f, indent=2)
95+
96+
print(f"\nJob info saved to: {job_info_path}")
97+
print(f"\n" + "=" * 70)
98+
print("TRAINING JOB SUBMITTED - MONITORING & NEXT STEPS")
99+
print("=" * 70)
100+
print(f"\nJob Details:")
101+
print(f" ID: {job.id}")
102+
print(f" Repository: {OUTPUT_REPO}")
103+
print(f" Status: {job.status}")
104+
print(f"\nMonitor your training at:")
105+
print(f" {job.url}")
106+
print(f"\nWhat the job will do:")
107+
print(f" Step 1: Install dependencies via uv")
108+
for dep in dependencies:
109+
print(f" - {dep}")
110+
print(f" Step 2: Download base model ({MODEL_REPO})")
111+
print(f" Step 3: Download training dataset (constraint_tracking.jsonl)")
112+
print(f" Step 4: Train constraint_tracker LoRA adapter")
113+
print(f" - Epochs: 3")
114+
print(f" - Learning rate: 1e-4")
115+
print(f" - Batch size: 2")
116+
print(f" - Gradient accumulation: 4")
117+
print(f" Step 5: Upload trained adapter to {OUTPUT_REPO}")
118+
print(f"\nEstimated duration: 4-6 hours on A10G GPU")
119+
print(f"\nNext steps after training:")
120+
print(f" 1. Monitor job status: {job.url}")
121+
print(f" 2. Once complete, load the trained adapter")
122+
print(f" 3. Run runtime benchmark:")
123+
print(f" python benchmarks/codette_runtime_benchmark.py")
124+
print(f" 4. Verify continuity_anchor_recall >= 0.70")
125+
print(f" Current: 0.200, Target: 0.70+")
126+
127+
except Exception as e:
128+
print(f"\n{'=' * 70}")
129+
print("[ERROR] Job submission failed")
130+
print(f"{'=' * 70}")
131+
print(f"\nError: {e}")
132+
print(f"\nTroubleshooting:")
133+
print(f" 1. Verify HF_TOKEN is correct and has write permissions")
134+
print(f" 2. Check that {OUTPUT_REPO} exists and you own it")
135+
print(f" 3. Verify internet connection")
136+
print(f" 4. Check HF API status: https://status.huggingface.co/")
137+
138+
# Print more detailed error info
139+
import traceback
140+
print(f"\nFull error trace:")
141+
traceback.print_exc()
142+
143+
exit(1)
144+
145+
print(f"\n{'=' * 70}")
146+
print("[OK] Constraint-Tracker LoRA Training Queued Successfully!")
147+
print("=" * 70 + "\n")

0 commit comments

Comments
 (0)