|
| 1 | +# ------- AI GENERATED -------- |
| 2 | +# ------- [Read tutorial/opencode_build_countdown_agent.prompt.md] -------- |
| 3 | + |
| 4 | +""" |
| 5 | +CountDown Agent Training Script (Swarm Client) |
| 6 | +
|
| 7 | +This script connects to the AgentJet Swarm server and trains the countdown agent. |
| 8 | +
|
| 9 | +Usage: |
| 10 | + python -m tutorial.countdown_agent.agent_roll |
| 11 | +
|
| 12 | +Before running: |
| 13 | + 1. Start the swarm server: ajet-swarm start |
| 14 | + 2. Ensure the dataset is generated: python tutorial/countdown_agent/generate_countdown_dataset.py |
| 15 | + 3. Update the configuration variables below to match your setup |
| 16 | +""" |
| 17 | + |
| 18 | +from ajet.copilot.job import AgentJetJob |
| 19 | +from ajet.tuner_lib.experimental.as_swarm_client import ( |
| 20 | + SwarmClient, |
| 21 | + run_episodes_until_all_complete, |
| 22 | +) |
| 23 | +from ajet.default_config.ajet_default import ( |
| 24 | + AjetTaskReader, |
| 25 | + JsonlDatasetFile, |
| 26 | + JsonlTrainingFp, |
| 27 | +) |
| 28 | +from ajet.task_reader import RouterTaskReader |
| 29 | +from .agent_run import run_agent_and_compute_reward |
| 30 | + |
| 31 | + |
| 32 | +# --------- Configurations that take effect locally ------------- |
| 33 | +LOCAL_GRPO_N = 4 # GRPO group size (number of rollouts per task) |
| 34 | +LOCAL_NUM_EPOCH = 100 # Number of training epochs |
| 35 | +LOCAL_DATASET_PATH = "./tutorial/countdown_agent/countdown_dataset/train.jsonl" |
| 36 | +REMOTE_SWARM_URL = "http://localhost:10086" # Swarm server URL |
| 37 | + |
| 38 | +# --------- Configurations that take effect remotely (on swarm server) ------------- |
| 39 | +REMOTE_BATCH_SIZE = 16 # Batch size for training (as specified by user) |
| 40 | +REMOTE_ALLOCATE_GPU_PER_NODE = 8 # Number of GPUs to use (as specified by user) |
| 41 | +REMOTE_TRAIN_MODEL = ( |
| 42 | + "/mnt/data_cpfs/model_cache/modelscope/hub/Qwen/Qwen/Qwen2.5-7B-Instruct" |
| 43 | +) |
| 44 | + |
| 45 | + |
| 46 | +def main(): |
| 47 | + """ |
| 48 | + Main training loop for CountDown agent. |
| 49 | + """ |
| 50 | + |
| 51 | + # Load the CountDown dataset |
| 52 | + print(f"Loading dataset from: {LOCAL_DATASET_PATH}") |
| 53 | + dataset = RouterTaskReader( |
| 54 | + reader_type="jsonl_dataset_file", |
| 55 | + reader_config=AjetTaskReader( |
| 56 | + jsonl_dataset_file=JsonlDatasetFile( |
| 57 | + training=JsonlTrainingFp(file_path=LOCAL_DATASET_PATH) |
| 58 | + ) |
| 59 | + ), |
| 60 | + ) |
| 61 | + |
| 62 | + # Connect to swarm server and configure training |
| 63 | + print(f"Connecting to swarm server at: {REMOTE_SWARM_URL}") |
| 64 | + swarm_worker = SwarmClient(REMOTE_SWARM_URL) |
| 65 | + |
| 66 | + # Configure and start the training engine |
| 67 | + print("Configuring training parameters...") |
| 68 | + yaml_job = AgentJetJob( |
| 69 | + algorithm="grpo", # Using GRPO (Group Relative Policy Optimization) |
| 70 | + project_name="countdown-agent", |
| 71 | + experiment_name="countdown_solver_7b", |
| 72 | + n_gpu=REMOTE_ALLOCATE_GPU_PER_NODE, |
| 73 | + model=REMOTE_TRAIN_MODEL, |
| 74 | + batch_size=REMOTE_BATCH_SIZE, |
| 75 | + num_repeat=LOCAL_GRPO_N, |
| 76 | + ) |
| 77 | + |
| 78 | + print("Starting swarm engine...") |
| 79 | + swarm_worker.auto_sync_train_config_and_start_engine(yaml_job) |
| 80 | + |
| 81 | + print("\n" + "=" * 80) |
| 82 | + print("Training started!") |
| 83 | + print(f"Model: {REMOTE_TRAIN_MODEL}") |
| 84 | + print(f"GPUs: {REMOTE_ALLOCATE_GPU_PER_NODE}") |
| 85 | + print(f"Batch size: {REMOTE_BATCH_SIZE}") |
| 86 | + print(f"GRPO group size: {LOCAL_GRPO_N}") |
| 87 | + print(f"Epochs: {LOCAL_NUM_EPOCH}") |
| 88 | + print("=" * 80 + "\n") |
| 89 | + |
| 90 | + def rollout(task): |
| 91 | + """ |
| 92 | + Execute a single episode (rollout) of the agent. |
| 93 | +
|
| 94 | + Args: |
| 95 | + task: The countdown problem to solve |
| 96 | +
|
| 97 | + Returns: |
| 98 | + The reward obtained (or None on failure) |
| 99 | + """ |
| 100 | + try: |
| 101 | + # Begin episode and get API credentials |
| 102 | + episode_uuid, api_baseurl_key = swarm_worker.begin_episode() |
| 103 | + |
| 104 | + # Execute agent and compute reward |
| 105 | + workflow_output = run_agent_and_compute_reward( |
| 106 | + task, api_baseurl_key.base_url, api_baseurl_key.api_key |
| 107 | + ) |
| 108 | + |
| 109 | + # Report results back to swarm server |
| 110 | + swarm_worker.end_episode(task, episode_uuid, workflow_output) |
| 111 | + |
| 112 | + # Print rollout statistics |
| 113 | + swarm_worker.print_rollout_stat() |
| 114 | + |
| 115 | + return workflow_output.reward |
| 116 | + |
| 117 | + except Exception as e: |
| 118 | + print(f"Error during rollout: {e}") |
| 119 | + return None |
| 120 | + |
| 121 | + # Training loop |
| 122 | + next_batch = [] |
| 123 | + total_episodes = 0 |
| 124 | + |
| 125 | + for epoch in range(LOCAL_NUM_EPOCH): |
| 126 | + print(f"\n{'=' * 80}") |
| 127 | + print(f"Epoch {epoch + 1}/{LOCAL_NUM_EPOCH}") |
| 128 | + print(f"{'=' * 80}\n") |
| 129 | + |
| 130 | + for task_idx, task in enumerate(dataset.generate_training_tasks()): |
| 131 | + # For each task, perform LOCAL_GRPO_N rollouts (GRPO group) |
| 132 | + for _ in range(LOCAL_GRPO_N): |
| 133 | + next_batch.append(task) |
| 134 | + |
| 135 | + # When batch is full, execute all episodes |
| 136 | + if len(next_batch) >= (REMOTE_BATCH_SIZE * LOCAL_GRPO_N): |
| 137 | + print(f"\nExecuting batch of {len(next_batch)} episodes...") |
| 138 | + |
| 139 | + # Execute episodes with automatic retry on failure |
| 140 | + episode_results = run_episodes_until_all_complete( |
| 141 | + next_batch, func=rollout, auto_retry=True |
| 142 | + ) |
| 143 | + |
| 144 | + total_episodes += len(next_batch) |
| 145 | + |
| 146 | + # Print batch results |
| 147 | + successful = sum( |
| 148 | + 1 for r in episode_results if r is not None and r > 0 |
| 149 | + ) |
| 150 | + avg_reward = ( |
| 151 | + sum(r for r in episode_results if r is not None) |
| 152 | + / len(episode_results) |
| 153 | + if episode_results |
| 154 | + else 0 |
| 155 | + ) |
| 156 | + |
| 157 | + print(f"\nBatch completed:") |
| 158 | + print(f" Total episodes: {len(next_batch)}") |
| 159 | + print(f" Successful: {successful}") |
| 160 | + print(f" Average reward: {avg_reward:.3f}") |
| 161 | + print(f" Total episodes so far: {total_episodes}") |
| 162 | + |
| 163 | + next_batch.clear() |
| 164 | + |
| 165 | + print(f"\nEpoch {epoch + 1} completed!") |
| 166 | + |
| 167 | + print("\n" + "=" * 80) |
| 168 | + print("Training completed!") |
| 169 | + print(f"Total episodes executed: {total_episodes}") |
| 170 | + print("=" * 80) |
| 171 | + |
| 172 | + return None |
| 173 | + |
| 174 | + |
| 175 | +if __name__ == "__main__": |
| 176 | + main() |
0 commit comments