messy self play tourney

Author: jsuarez5341

config/default.ini

@@ -37,6 +37,9 @@ min_games = 2048
 elo_init = 0.0
 elo_k = 16.0
 seed = 42
+# Optional league-state JSON to use as a frozen external opponent pool instead
+# of the current run's snapshot pool. Empty keeps the default snapshot behavior.
+external_opponent_state_path =
 # Add a snapshot to the pool every snapshot_interval global steps, independent
 # of swap. 0 disables interval snapshotting (pool stays at bootstrap).
 snapshot_interval = 1_000_000_000
@@ -122,17 +125,16 @@ bot_eval_burnin_episodes = 0
 bot_eval_policy = -1
 bot_eval_max_ticks = 0
 
-# League sweep mode for Robocode: train on all but one GPU while a long-lived
-# match worker rates completed final checkpoints. league_train_gpus = 0 means
-# auto: use sweep.gpus - league_match_gpus. Ignored unless league = True.
+# League sweep mode for Robocode: train independent historical-selfplay trials
+# on all but one GPU while a long-lived match worker rates completed final
+# checkpoints. league_train_gpus = 0 means auto: use sweep.gpus -
+# league_match_gpus. Ignored unless league = True.
 league = False
 league_train_gpus = 0
 league_match_gpus = 1
 league_match_games = 4096
 league_match_eval_agents = 8192
 league_anchor_prob = 0.12
-league_opponent_frac = 0.20
-league_opponent_swap_steps = 100_000_000
 league_state_path = ''
 
 [sweep.train.total_timesteps]

config/robocode.ini

@@ -27,7 +27,10 @@ width = 800
 height = 600
 reward_damage = 0.0
 reward_spot = 0.0
-dr = 0.181647854498119
+reward_melee_damage_inflicted = 0.0
+reward_damage_taken = 0.0
+reward_range_damage_inflicted = 0.0
+dr = 0.26211986603159293
 bot_policy = 1
 max_ticks = 3000
 
@@ -38,54 +41,79 @@ num_layers = 3
 [train]
 gpus = 1
 seed = 42
+#total_timesteps = 4_866_912_352
 total_timesteps = 2_000_000_000
-learning_rate = 0.01
+learning_rate = 0.0009396226782705388
 anneal_lr = 1
 min_lr_ratio = 0
-gamma = 0.9998480946489037
-gae_lambda = 0.9786838910227905
-horizon = 256
+gamma = 0.9999
+gae_lambda = 0.9806795173706343
+replay_ratio = 0.25
+clip_coef = 0.376885220359416
+vf_coef = 1.7165574392111924
+vf_clip_coef = 0.1043072795031198
+max_grad_norm = 0.5950188529406295
+ent_coef = 1.1745624395236018e-05
+anneal_ent_coef = 0
+min_ent_coef_ratio = 0.1
+beta1 = 0.807491153888714
+beta2 = 0.9995753999950486
+eps = 1e-14
+minibatch_size = 8192
+horizon = 64
+vtrace_rho_clip = 1.1533403498692292
+vtrace_c_clip = 1.5182455166896063
+prio_alpha = 0.6808270261043561
+prio_beta0 = 0.4740573494990283
 
 [sweep]
 league = True
-match_enemy_model_path = 'resources/robocode/best_robo.bin'
-match_num_games = 4096
-match_max_ticks = 4096
-match_enemy_hidden_size = 1024
-match_enemy_num_layers = 2.69591
-bot_eval = True
-bot_eval_episodes = 32768
-bot_eval_envs = 4096
-bot_eval_burnin_episodes = 4096
-bot_eval_policy = -1
-bot_eval_max_ticks = 0
 downsample = 1
-metric = bot_perf
+metric = elo
 
 [sweep.train.total_timesteps]
 distribution = log_normal
-min = 1e8
-max = 1e9
+min = 5e8
+max = 1e11
 mean = 5e8
 scale = auto
 
-#[sweep.env.reward_damage]
-#distribution = uniform
-#min = 0.0
-#max = 0.1
-#mean = 0.01
-#scale = auto
+[sweep.policy.hidden_size]
+distribution = uniform_pow2
+min = 32
+max = 1024
+scale = auto
 
-#[sweep.env.reward_spot]
-#distribution = uniform
-#min = 0.0
-#max = 0.01
-#mean = 0.001
-#scale = auto
+[sweep.policy.num_layers]
+distribution = uniform
+min = 1
+max = 8
+scale = auto
 
 [sweep.env.dr]
 distribution = uniform
 min = 0.0
 max = 0.6
 mean = 0.3
 scale = auto
+
+[sweep.env.reward_melee_damage_inflicted]
+distribution = uniform
+min = 0.0
+max = 0.02
+mean = 0.005
+scale = auto
+
+[sweep.env.reward_range_damage_inflicted]
+distribution = uniform
+min = 0.0
+max = 0.02
+mean = 0.005
+scale = auto
+
+[sweep.env.reward_damage_taken]
+distribution = uniform
+min = -0.02
+max = 0.0
+mean = -0.005
+scale = auto

ocean/robocode/binding.c

@@ -23,14 +23,35 @@ void my_setup_perm(StaticVec* vec, Env* env, int slot_base) {
     }
 }
 
+
+static inline float dict_get_float_default(Dict* kwargs, const char* key, float default_value) {
+    DictItem* item = dict_get_unsafe(kwargs, key);
+    return item ? (float)item->value : default_value;
+}
+
 void my_init(Env* env, Dict* kwargs) {
     env->width = dict_get(kwargs, "width")->value;
     env->height = dict_get(kwargs, "height")->value;
     env->num_agents = dict_get(kwargs, "num_agents")->value;
     env->num_bots = dict_get(kwargs, "num_bots")->value;
     env->max_ticks = (int)dict_get(kwargs, "max_ticks")->value;
-    env->reward_damage = dict_get(kwargs, "reward_damage")->value;
-    env->reward_spot = dict_get(kwargs, "reward_spot")->value;
+    env->reward_damage = dict_get_float_default(kwargs, "reward_damage", 0.0f);
+    env->reward_spot = dict_get_float_default(kwargs, "reward_spot", 0.0f);
+    env->reward_melee_damage_inflicted = dict_get_float_default(kwargs, "reward_melee_damage_inflicted", 0.0f);
+    env->reward_damage_taken = dict_get_float_default(kwargs, "reward_damage_taken", 0.0f);
+    env->reward_range_damage_inflicted = dict_get_float_default(kwargs, "reward_range_damage_inflicted", 0.0f);
+    env->reward_melee_damage_inflicted_slot_0 = dict_get_float_default(kwargs,
+        "reward_melee_damage_inflicted_slot_0", env->reward_melee_damage_inflicted);
+    env->reward_damage_taken_slot_0 = dict_get_float_default(kwargs,
+        "reward_damage_taken_slot_0", env->reward_damage_taken);
+    env->reward_range_damage_inflicted_slot_0 = dict_get_float_default(kwargs,
+        "reward_range_damage_inflicted_slot_0", env->reward_range_damage_inflicted);
+    env->reward_melee_damage_inflicted_slot_1 = dict_get_float_default(kwargs,
+        "reward_melee_damage_inflicted_slot_1", env->reward_melee_damage_inflicted);
+    env->reward_damage_taken_slot_1 = dict_get_float_default(kwargs,
+        "reward_damage_taken_slot_1", env->reward_damage_taken);
+    env->reward_range_damage_inflicted_slot_1 = dict_get_float_default(kwargs,
+        "reward_range_damage_inflicted_slot_1", env->reward_range_damage_inflicted);
     DictItem* dr_item = dict_get_unsafe(kwargs, "dr");
     env->dr = dr_item ? (float)dr_item->value : 0.0f;
     env->bot_policy = dict_get(kwargs, "bot_policy")->value;
@@ -41,6 +62,9 @@ void my_log(Log* log, Dict* out) {
     dict_set(out, "perf", log->perf);
     dict_set(out, "score", log->score);
     dict_set(out, "damage_received", log->damage_received);
+    dict_set(out, "melee_damage_inflicted", log->melee_damage_inflicted);
+    dict_set(out, "damage_taken", log->damage_taken);
+    dict_set(out, "range_damage_inflicted", log->range_damage_inflicted);
     dict_set(out, "episode_return", log->episode_return);
     dict_set(out, "episode_length", log->episode_length);
     // Historical-pool stats. selfplay.py reads hist_score_bank_<b> /