refact:remove server & bevy client ; remove _big bot algs
This commit is contained in:
parent
1e773671d9
commit
74f692d7ba
38 changed files with 24 additions and 3281 deletions
|
|
@ -1,194 +0,0 @@
|
|||
use crate::burnrl::environment_big::TrictracEnvironment;
|
||||
use crate::burnrl::utils::{soft_update_linear, Config};
|
||||
use burn::backend::{ndarray::NdArrayDevice, NdArray};
|
||||
use burn::module::Module;
|
||||
use burn::nn::{Linear, LinearConfig};
|
||||
use burn::optim::AdamWConfig;
|
||||
use burn::record::{CompactRecorder, Recorder};
|
||||
use burn::tensor::activation::relu;
|
||||
use burn::tensor::backend::{AutodiffBackend, Backend};
|
||||
use burn::tensor::Tensor;
|
||||
use burn_rl::agent::DQN;
|
||||
use burn_rl::agent::{DQNModel, DQNTrainingConfig};
|
||||
use burn_rl::base::{Action, Agent, ElemType, Environment, Memory, Model, State};
|
||||
use std::time::SystemTime;
|
||||
|
||||
#[derive(Module, Debug)]
|
||||
pub struct Net<B: Backend> {
|
||||
linear_0: Linear<B>,
|
||||
linear_1: Linear<B>,
|
||||
linear_2: Linear<B>,
|
||||
}
|
||||
|
||||
impl<B: Backend> Net<B> {
|
||||
#[allow(unused)]
|
||||
pub fn new(input_size: usize, dense_size: usize, output_size: usize) -> Self {
|
||||
Self {
|
||||
linear_0: LinearConfig::new(input_size, dense_size).init(&Default::default()),
|
||||
linear_1: LinearConfig::new(dense_size, dense_size).init(&Default::default()),
|
||||
linear_2: LinearConfig::new(dense_size, output_size).init(&Default::default()),
|
||||
}
|
||||
}
|
||||
|
||||
fn consume(self) -> (Linear<B>, Linear<B>, Linear<B>) {
|
||||
(self.linear_0, self.linear_1, self.linear_2)
|
||||
}
|
||||
}
|
||||
|
||||
impl<B: Backend> Model<B, Tensor<B, 2>, Tensor<B, 2>> for Net<B> {
|
||||
fn forward(&self, input: Tensor<B, 2>) -> Tensor<B, 2> {
|
||||
let layer_0_output = relu(self.linear_0.forward(input));
|
||||
let layer_1_output = relu(self.linear_1.forward(layer_0_output));
|
||||
|
||||
relu(self.linear_2.forward(layer_1_output))
|
||||
}
|
||||
|
||||
fn infer(&self, input: Tensor<B, 2>) -> Tensor<B, 2> {
|
||||
self.forward(input)
|
||||
}
|
||||
}
|
||||
|
||||
impl<B: Backend> DQNModel<B> for Net<B> {
|
||||
fn soft_update(this: Self, that: &Self, tau: ElemType) -> Self {
|
||||
let (linear_0, linear_1, linear_2) = this.consume();
|
||||
|
||||
Self {
|
||||
linear_0: soft_update_linear(linear_0, &that.linear_0, tau),
|
||||
linear_1: soft_update_linear(linear_1, &that.linear_1, tau),
|
||||
linear_2: soft_update_linear(linear_2, &that.linear_2, tau),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
const MEMORY_SIZE: usize = 8192;
|
||||
|
||||
type MyAgent<E, B> = DQN<E, B, Net<B>>;
|
||||
|
||||
#[allow(unused)]
|
||||
// pub fn run<E: Environment + AsMut<TrictracEnvironment>, B: AutodiffBackend>(
|
||||
pub fn run<
|
||||
E: Environment + AsMut<TrictracEnvironment>,
|
||||
B: AutodiffBackend<InnerBackend = NdArray>,
|
||||
>(
|
||||
conf: &Config,
|
||||
visualized: bool,
|
||||
// ) -> DQN<E, B, Net<B>> {
|
||||
) -> impl Agent<E> {
|
||||
let mut env = E::new(visualized);
|
||||
env.as_mut().max_steps = conf.max_steps;
|
||||
|
||||
let model = Net::<B>::new(
|
||||
<<E as Environment>::StateType as State>::size(),
|
||||
conf.dense_size,
|
||||
<<E as Environment>::ActionType as Action>::size(),
|
||||
);
|
||||
|
||||
let mut agent = MyAgent::new(model);
|
||||
|
||||
// let config = DQNTrainingConfig::default();
|
||||
let config = DQNTrainingConfig {
|
||||
gamma: conf.gamma,
|
||||
tau: conf.tau,
|
||||
learning_rate: conf.learning_rate,
|
||||
batch_size: conf.batch_size,
|
||||
clip_grad: Some(burn::grad_clipping::GradientClippingConfig::Value(
|
||||
conf.clip_grad,
|
||||
)),
|
||||
};
|
||||
|
||||
let mut memory = Memory::<E, B, MEMORY_SIZE>::default();
|
||||
|
||||
let mut optimizer = AdamWConfig::new()
|
||||
.with_grad_clipping(config.clip_grad.clone())
|
||||
.init();
|
||||
|
||||
let mut policy_net = agent.model().as_ref().unwrap().clone();
|
||||
|
||||
let mut step = 0_usize;
|
||||
|
||||
for episode in 0..conf.num_episodes {
|
||||
let mut episode_done = false;
|
||||
let mut episode_reward: ElemType = 0.0;
|
||||
let mut episode_duration = 0_usize;
|
||||
let mut state = env.state();
|
||||
let mut now = SystemTime::now();
|
||||
|
||||
while !episode_done {
|
||||
let eps_threshold = conf.eps_end
|
||||
+ (conf.eps_start - conf.eps_end) * f64::exp(-(step as f64) / conf.eps_decay);
|
||||
let action =
|
||||
DQN::<E, B, Net<B>>::react_with_exploration(&policy_net, state, eps_threshold);
|
||||
let snapshot = env.step(action);
|
||||
|
||||
episode_reward +=
|
||||
<<E as Environment>::RewardType as Into<ElemType>>::into(snapshot.reward().clone());
|
||||
|
||||
memory.push(
|
||||
state,
|
||||
*snapshot.state(),
|
||||
action,
|
||||
snapshot.reward().clone(),
|
||||
snapshot.done(),
|
||||
);
|
||||
|
||||
if config.batch_size < memory.len() {
|
||||
policy_net =
|
||||
agent.train::<MEMORY_SIZE>(policy_net, &memory, &mut optimizer, &config);
|
||||
}
|
||||
|
||||
step += 1;
|
||||
episode_duration += 1;
|
||||
|
||||
if snapshot.done() || episode_duration >= conf.max_steps {
|
||||
let envmut = env.as_mut();
|
||||
let goodmoves_ratio = ((envmut.goodmoves_count as f32 / episode_duration as f32)
|
||||
* 100.0)
|
||||
.round() as u32;
|
||||
println!(
|
||||
"{{\"episode\": {episode}, \"reward\": {episode_reward:.4}, \"steps count\": {episode_duration}, \"epsilon\": {eps_threshold:.3}, \"goodmoves\": {}, \"ratio\": {}%, \"rollpoints\":{}, \"duration\": {}}}",
|
||||
envmut.goodmoves_count,
|
||||
goodmoves_ratio,
|
||||
envmut.pointrolls_count,
|
||||
now.elapsed().unwrap().as_secs(),
|
||||
);
|
||||
env.reset();
|
||||
episode_done = true;
|
||||
now = SystemTime::now();
|
||||
} else {
|
||||
state = *snapshot.state();
|
||||
}
|
||||
}
|
||||
}
|
||||
let valid_agent = agent.valid();
|
||||
if let Some(path) = &conf.save_path {
|
||||
save_model(valid_agent.model().as_ref().unwrap(), path);
|
||||
}
|
||||
valid_agent
|
||||
}
|
||||
|
||||
pub fn save_model(model: &Net<NdArray<ElemType>>, path: &String) {
|
||||
let recorder = CompactRecorder::new();
|
||||
let model_path = format!("{path}.mpk");
|
||||
println!("info: Modèle de validation sauvegardé : {model_path}");
|
||||
recorder
|
||||
.record(model.clone().into_record(), model_path.into())
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
pub fn load_model(dense_size: usize, path: &String) -> Option<Net<NdArray<ElemType>>> {
|
||||
let model_path = format!("{path}.mpk");
|
||||
// println!("Chargement du modèle depuis : {model_path}");
|
||||
|
||||
CompactRecorder::new()
|
||||
.load(model_path.into(), &NdArrayDevice::default())
|
||||
.map(|record| {
|
||||
Net::new(
|
||||
<TrictracEnvironment as Environment>::StateType::size(),
|
||||
dense_size,
|
||||
<TrictracEnvironment as Environment>::ActionType::size(),
|
||||
)
|
||||
.load_record(record)
|
||||
})
|
||||
.ok()
|
||||
}
|
||||
|
|
@ -1,9 +1,6 @@
|
|||
pub mod dqn;
|
||||
pub mod dqn_big;
|
||||
pub mod dqn_valid;
|
||||
pub mod ppo;
|
||||
pub mod ppo_big;
|
||||
pub mod ppo_valid;
|
||||
pub mod sac;
|
||||
pub mod sac_big;
|
||||
pub mod sac_valid;
|
||||
|
|
|
|||
|
|
@ -1,191 +0,0 @@
|
|||
use crate::burnrl::environment_big::TrictracEnvironment;
|
||||
use crate::burnrl::utils::Config;
|
||||
use burn::backend::{ndarray::NdArrayDevice, NdArray};
|
||||
use burn::module::Module;
|
||||
use burn::nn::{Initializer, Linear, LinearConfig};
|
||||
use burn::optim::AdamWConfig;
|
||||
use burn::record::{CompactRecorder, Recorder};
|
||||
use burn::tensor::activation::{relu, softmax};
|
||||
use burn::tensor::backend::{AutodiffBackend, Backend};
|
||||
use burn::tensor::Tensor;
|
||||
use burn_rl::agent::{PPOModel, PPOOutput, PPOTrainingConfig, PPO};
|
||||
use burn_rl::base::{Action, Agent, ElemType, Environment, Memory, Model, State};
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::time::SystemTime;
|
||||
|
||||
#[derive(Module, Debug)]
|
||||
pub struct Net<B: Backend> {
|
||||
linear: Linear<B>,
|
||||
linear_actor: Linear<B>,
|
||||
linear_critic: Linear<B>,
|
||||
}
|
||||
|
||||
impl<B: Backend> Net<B> {
|
||||
#[allow(unused)]
|
||||
pub fn new(input_size: usize, dense_size: usize, output_size: usize) -> Self {
|
||||
let initializer = Initializer::XavierUniform { gain: 1.0 };
|
||||
Self {
|
||||
linear: LinearConfig::new(input_size, dense_size)
|
||||
.with_initializer(initializer.clone())
|
||||
.init(&Default::default()),
|
||||
linear_actor: LinearConfig::new(dense_size, output_size)
|
||||
.with_initializer(initializer.clone())
|
||||
.init(&Default::default()),
|
||||
linear_critic: LinearConfig::new(dense_size, 1)
|
||||
.with_initializer(initializer)
|
||||
.init(&Default::default()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<B: Backend> Model<B, Tensor<B, 2>, PPOOutput<B>, Tensor<B, 2>> for Net<B> {
|
||||
fn forward(&self, input: Tensor<B, 2>) -> PPOOutput<B> {
|
||||
let layer_0_output = relu(self.linear.forward(input));
|
||||
let policies = softmax(self.linear_actor.forward(layer_0_output.clone()), 1);
|
||||
let values = self.linear_critic.forward(layer_0_output);
|
||||
|
||||
PPOOutput::<B>::new(policies, values)
|
||||
}
|
||||
|
||||
fn infer(&self, input: Tensor<B, 2>) -> Tensor<B, 2> {
|
||||
let layer_0_output = relu(self.linear.forward(input));
|
||||
softmax(self.linear_actor.forward(layer_0_output.clone()), 1)
|
||||
}
|
||||
}
|
||||
|
||||
impl<B: Backend> PPOModel<B> for Net<B> {}
|
||||
#[allow(unused)]
|
||||
const MEMORY_SIZE: usize = 512;
|
||||
|
||||
type MyAgent<E, B> = PPO<E, B, Net<B>>;
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn run<
|
||||
E: Environment + AsMut<TrictracEnvironment>,
|
||||
B: AutodiffBackend<InnerBackend = NdArray>,
|
||||
>(
|
||||
conf: &Config,
|
||||
visualized: bool,
|
||||
// ) -> PPO<E, B, Net<B>> {
|
||||
) -> impl Agent<E> {
|
||||
let mut env = E::new(visualized);
|
||||
env.as_mut().max_steps = conf.max_steps;
|
||||
|
||||
let mut model = Net::<B>::new(
|
||||
<<E as Environment>::StateType as State>::size(),
|
||||
conf.dense_size,
|
||||
<<E as Environment>::ActionType as Action>::size(),
|
||||
);
|
||||
let agent = MyAgent::default();
|
||||
let config = PPOTrainingConfig {
|
||||
gamma: conf.gamma,
|
||||
lambda: conf.lambda,
|
||||
epsilon_clip: conf.epsilon_clip,
|
||||
critic_weight: conf.critic_weight,
|
||||
entropy_weight: conf.entropy_weight,
|
||||
learning_rate: conf.learning_rate,
|
||||
epochs: conf.epochs,
|
||||
batch_size: conf.batch_size,
|
||||
clip_grad: Some(burn::grad_clipping::GradientClippingConfig::Value(
|
||||
conf.clip_grad,
|
||||
)),
|
||||
};
|
||||
|
||||
let mut optimizer = AdamWConfig::new()
|
||||
.with_grad_clipping(config.clip_grad.clone())
|
||||
.init();
|
||||
let mut memory = Memory::<E, B, MEMORY_SIZE>::default();
|
||||
for episode in 0..conf.num_episodes {
|
||||
let mut episode_done = false;
|
||||
let mut episode_reward = 0.0;
|
||||
let mut episode_duration = 0_usize;
|
||||
let mut now = SystemTime::now();
|
||||
|
||||
env.reset();
|
||||
while !episode_done {
|
||||
let state = env.state();
|
||||
if let Some(action) = MyAgent::<E, _>::react_with_model(&state, &model) {
|
||||
let snapshot = env.step(action);
|
||||
episode_reward += <<E as Environment>::RewardType as Into<ElemType>>::into(
|
||||
snapshot.reward().clone(),
|
||||
);
|
||||
|
||||
memory.push(
|
||||
state,
|
||||
*snapshot.state(),
|
||||
action,
|
||||
snapshot.reward().clone(),
|
||||
snapshot.done(),
|
||||
);
|
||||
|
||||
episode_duration += 1;
|
||||
episode_done = snapshot.done() || episode_duration >= conf.max_steps;
|
||||
}
|
||||
}
|
||||
println!(
|
||||
"{{\"episode\": {episode}, \"reward\": {episode_reward:.4}, \"steps count\": {episode_duration}, \"duration\": {}}}",
|
||||
now.elapsed().unwrap().as_secs(),
|
||||
);
|
||||
|
||||
now = SystemTime::now();
|
||||
model = MyAgent::train::<MEMORY_SIZE>(model, &memory, &mut optimizer, &config);
|
||||
memory.clear();
|
||||
}
|
||||
|
||||
if let Some(path) = &conf.save_path {
|
||||
let device = NdArrayDevice::default();
|
||||
let recorder = CompactRecorder::new();
|
||||
let tmp_path = env::temp_dir().join("tmp_model.mpk");
|
||||
|
||||
// Save the trained model (backend B) to a temporary file
|
||||
recorder
|
||||
.record(model.clone().into_record(), tmp_path.clone())
|
||||
.expect("Failed to save temporary model");
|
||||
|
||||
// Create a new model instance with the target backend (NdArray)
|
||||
let model_to_save: Net<NdArray<ElemType>> = Net::new(
|
||||
<<E as Environment>::StateType as State>::size(),
|
||||
conf.dense_size,
|
||||
<<E as Environment>::ActionType as Action>::size(),
|
||||
);
|
||||
|
||||
// Load the record from the temporary file into the new model
|
||||
let record = recorder
|
||||
.load(tmp_path.clone(), &device)
|
||||
.expect("Failed to load temporary model");
|
||||
let model_with_loaded_weights = model_to_save.load_record(record);
|
||||
|
||||
// Clean up the temporary file
|
||||
fs::remove_file(tmp_path).expect("Failed to remove temporary model file");
|
||||
|
||||
save_model(&model_with_loaded_weights, path);
|
||||
}
|
||||
agent.valid(model)
|
||||
}
|
||||
|
||||
pub fn save_model(model: &Net<NdArray<ElemType>>, path: &String) {
|
||||
let recorder = CompactRecorder::new();
|
||||
let model_path = format!("{path}.mpk");
|
||||
println!("info: Modèle de validation sauvegardé : {model_path}");
|
||||
recorder
|
||||
.record(model.clone().into_record(), model_path.into())
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
pub fn load_model(dense_size: usize, path: &String) -> Option<Net<NdArray<ElemType>>> {
|
||||
let model_path = format!("{path}.mpk");
|
||||
// println!("Chargement du modèle depuis : {model_path}");
|
||||
|
||||
CompactRecorder::new()
|
||||
.load(model_path.into(), &NdArrayDevice::default())
|
||||
.map(|record| {
|
||||
Net::new(
|
||||
<TrictracEnvironment as Environment>::StateType::size(),
|
||||
dense_size,
|
||||
<TrictracEnvironment as Environment>::ActionType::size(),
|
||||
)
|
||||
.load_record(record)
|
||||
})
|
||||
.ok()
|
||||
}
|
||||
|
|
@ -1,222 +0,0 @@
|
|||
use crate::burnrl::environment_big::TrictracEnvironment;
|
||||
use crate::burnrl::utils::{soft_update_linear, Config};
|
||||
use burn::backend::{ndarray::NdArrayDevice, NdArray};
|
||||
use burn::module::Module;
|
||||
use burn::nn::{Linear, LinearConfig};
|
||||
use burn::optim::AdamWConfig;
|
||||
use burn::record::{CompactRecorder, Recorder};
|
||||
use burn::tensor::activation::{relu, softmax};
|
||||
use burn::tensor::backend::{AutodiffBackend, Backend};
|
||||
use burn::tensor::Tensor;
|
||||
use burn_rl::agent::{SACActor, SACCritic, SACNets, SACOptimizer, SACTrainingConfig, SAC};
|
||||
use burn_rl::base::{Action, Agent, ElemType, Environment, Memory, Model, State};
|
||||
use std::time::SystemTime;
|
||||
|
||||
#[derive(Module, Debug)]
|
||||
pub struct Actor<B: Backend> {
|
||||
linear_0: Linear<B>,
|
||||
linear_1: Linear<B>,
|
||||
linear_2: Linear<B>,
|
||||
}
|
||||
|
||||
impl<B: Backend> Actor<B> {
|
||||
pub fn new(input_size: usize, dense_size: usize, output_size: usize) -> Self {
|
||||
Self {
|
||||
linear_0: LinearConfig::new(input_size, dense_size).init(&Default::default()),
|
||||
linear_1: LinearConfig::new(dense_size, dense_size).init(&Default::default()),
|
||||
linear_2: LinearConfig::new(dense_size, output_size).init(&Default::default()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<B: Backend> Model<B, Tensor<B, 2>, Tensor<B, 2>> for Actor<B> {
|
||||
fn forward(&self, input: Tensor<B, 2>) -> Tensor<B, 2> {
|
||||
let layer_0_output = relu(self.linear_0.forward(input));
|
||||
let layer_1_output = relu(self.linear_1.forward(layer_0_output));
|
||||
|
||||
softmax(self.linear_2.forward(layer_1_output), 1)
|
||||
}
|
||||
|
||||
fn infer(&self, input: Tensor<B, 2>) -> Tensor<B, 2> {
|
||||
self.forward(input)
|
||||
}
|
||||
}
|
||||
|
||||
impl<B: Backend> SACActor<B> for Actor<B> {}
|
||||
|
||||
#[derive(Module, Debug)]
|
||||
pub struct Critic<B: Backend> {
|
||||
linear_0: Linear<B>,
|
||||
linear_1: Linear<B>,
|
||||
linear_2: Linear<B>,
|
||||
}
|
||||
|
||||
impl<B: Backend> Critic<B> {
|
||||
pub fn new(input_size: usize, dense_size: usize, output_size: usize) -> Self {
|
||||
Self {
|
||||
linear_0: LinearConfig::new(input_size, dense_size).init(&Default::default()),
|
||||
linear_1: LinearConfig::new(dense_size, dense_size).init(&Default::default()),
|
||||
linear_2: LinearConfig::new(dense_size, output_size).init(&Default::default()),
|
||||
}
|
||||
}
|
||||
|
||||
fn consume(self) -> (Linear<B>, Linear<B>, Linear<B>) {
|
||||
(self.linear_0, self.linear_1, self.linear_2)
|
||||
}
|
||||
}
|
||||
|
||||
impl<B: Backend> Model<B, Tensor<B, 2>, Tensor<B, 2>> for Critic<B> {
|
||||
fn forward(&self, input: Tensor<B, 2>) -> Tensor<B, 2> {
|
||||
let layer_0_output = relu(self.linear_0.forward(input));
|
||||
let layer_1_output = relu(self.linear_1.forward(layer_0_output));
|
||||
|
||||
self.linear_2.forward(layer_1_output)
|
||||
}
|
||||
|
||||
fn infer(&self, input: Tensor<B, 2>) -> Tensor<B, 2> {
|
||||
self.forward(input)
|
||||
}
|
||||
}
|
||||
|
||||
impl<B: Backend> SACCritic<B> for Critic<B> {
|
||||
fn soft_update(this: Self, that: &Self, tau: ElemType) -> Self {
|
||||
let (linear_0, linear_1, linear_2) = this.consume();
|
||||
|
||||
Self {
|
||||
linear_0: soft_update_linear(linear_0, &that.linear_0, tau),
|
||||
linear_1: soft_update_linear(linear_1, &that.linear_1, tau),
|
||||
linear_2: soft_update_linear(linear_2, &that.linear_2, tau),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
const MEMORY_SIZE: usize = 4096;
|
||||
|
||||
type MyAgent<E, B> = SAC<E, B, Actor<B>>;
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn run<
|
||||
E: Environment + AsMut<TrictracEnvironment>,
|
||||
B: AutodiffBackend<InnerBackend = NdArray>,
|
||||
>(
|
||||
conf: &Config,
|
||||
visualized: bool,
|
||||
) -> impl Agent<E> {
|
||||
let mut env = E::new(visualized);
|
||||
env.as_mut().max_steps = conf.max_steps;
|
||||
let state_dim = <<E as Environment>::StateType as State>::size();
|
||||
let action_dim = <<E as Environment>::ActionType as Action>::size();
|
||||
|
||||
let actor = Actor::<B>::new(state_dim, conf.dense_size, action_dim);
|
||||
let critic_1 = Critic::<B>::new(state_dim, conf.dense_size, action_dim);
|
||||
let critic_2 = Critic::<B>::new(state_dim, conf.dense_size, action_dim);
|
||||
let mut nets = SACNets::<B, Actor<B>, Critic<B>>::new(actor, critic_1, critic_2);
|
||||
|
||||
let mut agent = MyAgent::default();
|
||||
|
||||
let config = SACTrainingConfig {
|
||||
gamma: conf.gamma,
|
||||
tau: conf.tau,
|
||||
learning_rate: conf.learning_rate,
|
||||
min_probability: conf.min_probability,
|
||||
batch_size: conf.batch_size,
|
||||
clip_grad: Some(burn::grad_clipping::GradientClippingConfig::Value(
|
||||
conf.clip_grad,
|
||||
)),
|
||||
};
|
||||
|
||||
let mut memory = Memory::<E, B, MEMORY_SIZE>::default();
|
||||
|
||||
let optimizer_config = AdamWConfig::new().with_grad_clipping(config.clip_grad.clone());
|
||||
|
||||
let mut optimizer = SACOptimizer::new(
|
||||
optimizer_config.clone().init(),
|
||||
optimizer_config.clone().init(),
|
||||
optimizer_config.clone().init(),
|
||||
optimizer_config.init(),
|
||||
);
|
||||
|
||||
let mut step = 0_usize;
|
||||
|
||||
for episode in 0..conf.num_episodes {
|
||||
let mut episode_done = false;
|
||||
let mut episode_reward = 0.0;
|
||||
let mut episode_duration = 0_usize;
|
||||
let mut state = env.state();
|
||||
let mut now = SystemTime::now();
|
||||
|
||||
while !episode_done {
|
||||
if let Some(action) = MyAgent::<E, _>::react_with_model(&state, &nets.actor) {
|
||||
let snapshot = env.step(action);
|
||||
|
||||
episode_reward += <<E as Environment>::RewardType as Into<ElemType>>::into(
|
||||
snapshot.reward().clone(),
|
||||
);
|
||||
|
||||
memory.push(
|
||||
state,
|
||||
*snapshot.state(),
|
||||
action,
|
||||
snapshot.reward().clone(),
|
||||
snapshot.done(),
|
||||
);
|
||||
|
||||
if config.batch_size < memory.len() {
|
||||
nets = agent.train::<MEMORY_SIZE, _>(nets, &memory, &mut optimizer, &config);
|
||||
}
|
||||
|
||||
step += 1;
|
||||
episode_duration += 1;
|
||||
|
||||
if snapshot.done() || episode_duration >= conf.max_steps {
|
||||
env.reset();
|
||||
episode_done = true;
|
||||
|
||||
println!(
|
||||
"{{\"episode\": {episode}, \"reward\": {episode_reward:.4}, \"steps count\": {episode_duration}, \"duration\": {}}}",
|
||||
now.elapsed().unwrap().as_secs()
|
||||
);
|
||||
now = SystemTime::now();
|
||||
} else {
|
||||
state = *snapshot.state();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let valid_agent = agent.valid(nets.actor);
|
||||
if let Some(path) = &conf.save_path {
|
||||
if let Some(model) = valid_agent.model() {
|
||||
save_model(model, path);
|
||||
}
|
||||
}
|
||||
valid_agent
|
||||
}
|
||||
|
||||
pub fn save_model(model: &Actor<NdArray<ElemType>>, path: &String) {
|
||||
let recorder = CompactRecorder::new();
|
||||
let model_path = format!("{path}.mpk");
|
||||
println!("info: Modèle de validation sauvegardé : {model_path}");
|
||||
recorder
|
||||
.record(model.clone().into_record(), model_path.into())
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
pub fn load_model(dense_size: usize, path: &String) -> Option<Actor<NdArray<ElemType>>> {
|
||||
let model_path = format!("{path}.mpk");
|
||||
// println!("Chargement du modèle depuis : {model_path}");
|
||||
|
||||
CompactRecorder::new()
|
||||
.load(model_path.into(), &NdArrayDevice::default())
|
||||
.map(|record| {
|
||||
Actor::new(
|
||||
<TrictracEnvironment as Environment>::StateType::size(),
|
||||
dense_size,
|
||||
<TrictracEnvironment as Environment>::ActionType::size(),
|
||||
)
|
||||
.load_record(record)
|
||||
})
|
||||
.ok()
|
||||
}
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue