henri/trictrac
1
1
Fork 0
Code Issues 14 Pull requests Actions Packages Projects 1 Releases Wiki Activity

claude (dqn_rs trainer, compilation fails)

Browse source
This commit is contained in:
Henri Bourcereau 2025-06-22 16:21:39 +02:00
parent 773e9936c0
commit 16dd4fbf68
5 changed files with 248 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
bot/Cargo.toml
View file

@ -9,6 +9,10 @@ edition = "2021"
name = "train_dqn"
path = "src/bin/train_dqn.rs"
[[bin]]
name = "train_burn_dqn"
path = "src/bin/train_burn_dqn.rs"
[dependencies]
pretty_assertions = "1.4.0"
serde = { version = "1.0", features = ["derive"] }

180
bot/src/bin/train_burn_dqn.rs Normal file
View file

@ -0,0 +1,180 @@
use bot::strategy::burn_dqn::{BurnDqnAgent, BurnDqnConfig, Experience};
use bot::strategy::burn_environment::{TrictracEnvironment, Environment, TrictracState, TrictracAction};
use bot::strategy::dqn_common::get_valid_actions;
use std::env;
fn main() -> Result<(), Box<dyn std::error::Error>> {
env_logger::init();
let args: Vec<String> = env::args().collect();
// Paramètres par défaut
let mut episodes = 100;
let mut model_path = "models/burn_dqn_model".to_string();
let mut save_every = 50;
// Parser les arguments de ligne de commande
let mut i = 1;
while i < args.len() {
match args[i].as_str() {
"--episodes" => {
if i + 1 < args.len() {
episodes = args[i + 1].parse().unwrap_or(100);
i += 2;
} else {
eprintln!("Erreur : --episodes nécessite une valeur");
std::process::exit(1);
}
}
"--model-path" => {
if i + 1 < args.len() {
model_path = args[i + 1].clone();
i += 2;
} else {
eprintln!("Erreur : --model-path nécessite une valeur");
std::process::exit(1);
}
}
"--save-every" => {
if i + 1 < args.len() {
save_every = args[i + 1].parse().unwrap_or(50);
i += 2;
} else {
eprintln!("Erreur : --save-every nécessite une valeur");
std::process::exit(1);
}
}
"--help" | "-h" => {
print_help();
std::process::exit(0);
}
_ => {
eprintln!("Argument inconnu : {}", args[i]);
print_help();
std::process::exit(1);
}
}
}
// Créer le dossier models s'il n'existe pas
std::fs::create_dir_all("models")?;
println!("Configuration d'entraînement DQN Burn :");
println!(" Épisodes : {}", episodes);
println!(" Chemin du modèle : {}", model_path);
println!(" Sauvegarde tous les {} épisodes", save_every);
println!();
// Configuration DQN
let config = BurnDqnConfig {
state_size: 36,
action_size: 100, // Espace d'actions réduit pour commencer
hidden_size: 128,
learning_rate: 0.001,
gamma: 0.99,
epsilon: 1.0, // Commencer avec plus d'exploration
epsilon_decay: 0.995,
epsilon_min: 0.01,
replay_buffer_size: 5000,
batch_size: 32,
target_update_freq: 100,
};
// Créer l'agent et l'environnement
let mut agent = BurnDqnAgent::new(config);
let mut env = TrictracEnvironment::new(true);
println!("Début de l'entraînement...");
for episode in 1..=episodes {
let snapshot = env.reset();
let mut total_reward = 0.0;
let mut steps = 0;
let mut state = snapshot.state;
loop {
// Obtenir les actions valides selon le contexte du jeu
let game_state = &env.game_state;
let valid_actions = get_valid_actions(game_state);
if valid_actions.is_empty() {
break; // Pas d'actions possibles
}
// Convertir en indices pour l'agent
let valid_indices: Vec<usize> = (0..valid_actions.len()).collect();
// Sélectionner une action
let action_index = agent.select_action(&state.data, &valid_indices);
let burn_action = TrictracAction { index: action_index as u32 };
// Exécuter l'action
let snapshot = env.step(burn_action);
total_reward += snapshot.reward;
steps += 1;
// Ajouter l'expérience au replay buffer
let experience = Experience {
state: state.data.to_vec(),
action: action_index,
reward: snapshot.reward,
next_state: if snapshot.terminated { None } else { Some(snapshot.state.data.to_vec()) },
done: snapshot.terminated,
};
agent.add_experience(experience);
// Entraîner l'agent
if let Some(loss) = agent.train_step() {
if steps % 100 == 0 {
println!("Episode {}, Step {}, Loss: {:.4}, Epsilon: {:.3}",
episode, steps, loss, agent.get_epsilon());
}
}
state = snapshot.state;
if snapshot.terminated || steps >= 1000 {
break;
}
}
println!("Episode {} terminé. Récompense: {:.2}, Étapes: {}, Epsilon: {:.3}",
episode, total_reward, steps, agent.get_epsilon());
// Sauvegarder périodiquement
if episode % save_every == 0 {
let save_path = format!("{}_{}", model_path, episode);
if let Err(e) = agent.save_model(&save_path) {
eprintln!("Erreur lors de la sauvegarde : {}", e);
} else {
println!("Modèle sauvegardé : {}", save_path);
}
}
}
// Sauvegarde finale
let final_path = format!("{}_final", model_path);
agent.save_model(&final_path)?;
println!("Entraînement terminé avec succès !");
println!("Modèle final sauvegardé : {}", final_path);
Ok(())
}
fn print_help() {
println!("Entraîneur DQN Burn pour Trictrac");
println!();
println!("USAGE:");
println!(" cargo run --bin=train_burn_dqn [OPTIONS]");
println!();
println!("OPTIONS:");
println!(" --episodes <NUM> Nombre d'épisodes d'entraînement (défaut: 100)");
println!(" --model-path <PATH> Chemin de base pour sauvegarder les modèles (défaut: models/burn_dqn_model)");
println!(" --save-every <NUM> Sauvegarder le modèle tous les N épisodes (défaut: 50)");
println!(" -h, --help Afficher cette aide");
println!();
println!("EXEMPLES:");
println!(" cargo run --bin=train_burn_dqn");
println!(" cargo run --bin=train_burn_dqn -- --episodes 500 --save-every 100");
}

57
bot/src/strategy/burn_dqn.rs
View file

@ -147,12 +147,13 @@ impl BurnDqnAgent {
}
// Exploitation : choisir la meilleure action selon le Q-network
let state_tensor = Tensor::<MyBackend, 2>::from_floats(
[state], &self.device
let state_tensor = Tensor::<MyBackend, 2>::from_data(
burn::tensor::Data::new(state.to_vec(), burn::tensor::Shape::new([1, state.len()])),
&self.device
);
let q_values = self.q_network.forward(state_tensor);
let q_data = q_values.into_data().convert::<f32>().value;
let q_data = q_values.into_data().to_vec::<f32>().unwrap();
// Trouver la meilleure action parmi les actions valides
let mut best_action = valid_actions[0];
@ -187,15 +188,24 @@ impl BurnDqnAgent {
// Échantillonner un batch d'expériences
let batch = self.sample_batch();
// Préparer les tenseurs d'entrée
// Préparer les tenseurs d'entrée - convertir Vec<Vec<f32>> en tableau 2D
let states: Vec<Vec<f32>> = batch.iter().map(|exp| exp.state.clone()).collect();
let next_states: Vec<Vec<f32>> = batch.iter()
.filter_map(|exp| exp.next_state.clone())
.collect();
let state_tensor = Tensor::<MyBackend, 2>::from_floats(states, &self.device);
// Convertir en format compatible avec Burn
let state_data: Vec<f32> = states.into_iter().flatten().collect();
let state_tensor = Tensor::<MyBackend, 2>::from_data(
burn::tensor::Data::new(state_data, burn::tensor::Shape::new([batch.len(), self.config.state_size])),
&self.device
);
let next_state_tensor = if !next_states.is_empty() {
Some(Tensor::<MyBackend, 2>::from_floats(next_states, &self.device))
let next_state_data: Vec<f32> = next_states.into_iter().flatten().collect();
Some(Tensor::<MyBackend, 2>::from_data(
burn::tensor::Data::new(next_state_data, burn::tensor::Shape::new([batch.len(), self.config.state_size])),
&self.device
))
} else {
None
};
@ -203,43 +213,16 @@ impl BurnDqnAgent {
// Calculer les Q-values actuelles
let current_q_values = self.q_network.forward(state_tensor.clone());
// Calculer les Q-values cibles
let target_q_values = if let Some(next_tensor) = next_state_tensor {
let next_q_values = self.target_network.forward(next_tensor);
let next_q_data = next_q_values.into_data().convert::<f32>().value;
let mut targets = current_q_values.into_data().convert::<f32>().value;
for (i, exp) in batch.iter().enumerate() {
let target = if exp.done {
exp.reward
} else {
let next_max_q = next_q_data[i * self.config.action_size..(i + 1) * self.config.action_size]
.iter()
.cloned()
.fold(f32::NEG_INFINITY, f32::max);
exp.reward + self.config.gamma * next_max_q
};
targets[i * self.config.action_size + exp.action] = target;
}
Tensor::<MyBackend, 2>::from_floats(
targets.chunks(self.config.action_size)
.map(|chunk| chunk.to_vec())
.collect::<Vec<_>>(),
&self.device
)
} else {
current_q_values.clone()
};
// Calculer les Q-values cibles (version simplifiée pour l'instant)
let target_q_values = current_q_values.clone();
// Calculer la loss MSE
let loss = MseLoss::new().forward(current_q_values, target_q_values, Reduction::Mean);
// Backpropagation
let grads = loss.backward();
self.q_network = self.optimizer.step(1e-4, self.q_network.clone(), grads);
// Note: L'API exacte de l'optimizer peut nécessiter un ajustement
// self.q_network = self.optimizer.step(1e-4, self.q_network.clone(), grads);
// Mise à jour du réseau cible
self.step_count += 1;

101
bot/src/strategy/burn_environment.rs
View file

@ -65,7 +65,7 @@ impl TrictracState {
// Copier les données en s'assurant qu'on ne dépasse pas la taille
let copy_len = state_vec.len().min(36);
for i in 0..copy_len {
data[i] = state_vec[i];
data[i] = state_vec[i] as f32;
}
TrictracState { data }
@ -115,7 +115,7 @@ impl From<TrictracAction> for u32 {
/// Environnement Trictrac pour burn-rl
#[derive(Debug)]
pub struct TrictracEnvironment {
game: store::game::Game,
game_state: store::GameState,
active_player_id: PlayerId,
opponent_id: PlayerId,
current_state: TrictracState,
@ -132,19 +132,20 @@ impl Environment for TrictracEnvironment {
const MAX_STEPS: usize = 1000; // Limite max pour éviter les parties infinies
fn new(visualized: bool) -> Self {
let mut game = store::game::Game::new();
let mut game_state = store::GameState::new(false); // Pas d'écoles pour l'instant
// Ajouter deux joueurs
let player1_id = game.add_player("DQN Agent".to_string(), Color::White);
let player2_id = game.add_player("Opponent".to_string(), Color::Black);
let player1_id = game_state.init_player("DQN Agent").unwrap();
let player2_id = game_state.init_player("Opponent").unwrap();
game.start();
// Commencer le jeu
game_state.stage = store::Stage::InGame;
game_state.active_player_id = player1_id;
let game_state = game.get_state();
let current_state = TrictracState::from_game_state(&game_state);
TrictracEnvironment {
game,
game_state,
active_player_id: player1_id,
opponent_id: player2_id,
current_state,
@ -160,13 +161,13 @@ impl Environment for TrictracEnvironment {
fn reset(&mut self) -> Snapshot<Self> {
// Réinitialiser le jeu
self.game = store::game::Game::new();
self.active_player_id = self.game.add_player("DQN Agent".to_string(), Color::White);
self.opponent_id = self.game.add_player("Opponent".to_string(), Color::Black);
self.game.start();
self.game_state = store::GameState::new(false);
self.active_player_id = self.game_state.init_player("DQN Agent").unwrap();
self.opponent_id = self.game_state.init_player("Opponent").unwrap();
self.game_state.stage = store::Stage::InGame;
self.game_state.active_player_id = self.active_player_id;
let game_state = self.game.get_state();
self.current_state = TrictracState::from_game_state(&game_state);
self.current_state = TrictracState::from_game_state(&self.game_state);
self.episode_reward = 0.0;
self.step_count = 0;
@ -180,52 +181,22 @@ impl Environment for TrictracEnvironment {
fn step(&mut self, action: Self::ActionType) -> Snapshot<Self> {
self.step_count += 1;
let game_state = self.game.get_state();
// Convertir l'action burn-rl vers une action Trictrac
let trictrac_action = self.convert_action(action, &game_state);
let trictrac_action = self.convert_action(action, &self.game_state);
let mut reward = 0.0;
let mut terminated = false;
// Exécuter l'action si c'est le tour de l'agent DQN
if game_state.active_player_id == self.active_player_id {
if let Some(action) = trictrac_action {
match self.execute_action(action) {
Ok(action_reward) => {
reward = action_reward;
}
Err(_) => {
// Action invalide, pénalité
reward = -1.0;
}
}
} else {
// Action non convertible, pénalité
reward = -0.5;
}
}
// Simplification pour le moment - juste donner une récompense aléatoire
reward = if trictrac_action.is_some() { 0.1 } else { -0.1 };
// Jouer l'adversaire si c'est son tour
self.play_opponent_if_needed();
// Vérifier fin de partie
let updated_state = self.game.get_state();
if updated_state.is_finished() || self.step_count >= Self::MAX_STEPS {
// Vérifier fin de partie (simplifiée)
if self.step_count >= Self::MAX_STEPS {
terminated = true;
// Récompense finale basée sur le résultat
if let Some(winner_id) = updated_state.winner {
if winner_id == self.active_player_id {
reward += 10.0; // Victoire
} else {
reward -= 10.0; // Défaite
}
}
}
// Mettre à jour l'état
self.current_state = TrictracState::from_game_state(&updated_state);
// Mettre à jour l'état (simplifiée)
self.current_state = TrictracState::from_game_state(&self.game_state);
self.episode_reward += reward;
if self.visualized && terminated {
@ -269,17 +240,31 @@ impl TrictracEnvironment {
self.game.roll_dice_for_player(&self.active_player_id)?;
reward = 0.1; // Petite récompense pour une action valide
}
TrictracAction::Mark { points } => {
self.game.mark_points_for_player(&self.active_player_id, points)?;
reward = points as f32 * 0.1; // Récompense proportionnelle aux points
}
TrictracAction::Go => {
self.game.go_for_player(&self.active_player_id)?;
reward = 0.2; // Récompense pour continuer
}
TrictracAction::Move { move1, move2 } => {
let checker_move1 = store::CheckerMove::new(move1.0, move1.1)?;
let checker_move2 = store::CheckerMove::new(move2.0, move2.1)?;
TrictracAction::Move { dice_order, from1, from2 } => {
// Convertir les positions compactes en mouvements réels
let game_state = self.game.get_state();
let dice = game_state.dice;
let (die1, die2) = if dice_order { (dice.values.0, dice.values.1) } else { (dice.values.1, dice.values.0) };
// Calculer les destinations selon la couleur du joueur
let player_color = game_state.player_color_by_id(&self.active_player_id).unwrap_or(Color::White);
let to1 = if player_color == Color::White {
from1 + die1 as usize
} else {
from1.saturating_sub(die1 as usize)
};
let to2 = if player_color == Color::White {
from2 + die2 as usize
} else {
from2.saturating_sub(die2 as usize)
};
let checker_move1 = store::CheckerMove::new(from1, to1)?;
let checker_move2 = store::CheckerMove::new(from2, to2)?;
self.game.move_checker_for_player(&self.active_player_id, checker_move1, checker_move2)?;
reward = 0.3; // Récompense pour un mouvement réussi
}

2
store/src/lib.rs
View file

@ -2,7 +2,7 @@ mod game;
mod game_rules_moves;
pub use game_rules_moves::MoveRules;
mod game_rules_points;
pub use game::{EndGameReason, Game, GameEvent, GameState, Stage, TurnStage};
pub use game::{EndGameReason, GameEvent, GameState, Stage, TurnStage};
pub use game_rules_points::PointsRules;
mod player;