From b91908bf7a9fd73a94c31a7e03f345356fa8bce1 Mon Sep 17 00:00:00 2001 From: Jona Reynaert Date: Thu, 7 May 2026 13:45:03 +0200 Subject: [PATCH] feat: abstracted checkpoint eval configs --- configs/evaluation/default.yaml | 8 ++++++ configs/logging/default.yaml | 5 +--- configs/main_config.yaml | 1 + .../configs/config_evaluation.py | 24 ++++++++++++++++++ .../configs/main_config.py | 2 ++ .../configs/register_configs.py | 2 ++ .../trainers/PPOTrainer.py | 18 ++++++++++--- src/experiment_logger/config_logger.py | 25 ++----------------- 8 files changed, 55 insertions(+), 30 deletions(-) create mode 100644 configs/evaluation/default.yaml create mode 100644 src/brittle_star_project/configs/config_evaluation.py diff --git a/configs/evaluation/default.yaml b/configs/evaluation/default.yaml new file mode 100644 index 0000000..ac49e46 --- /dev/null +++ b/configs/evaluation/default.yaml @@ -0,0 +1,8 @@ +# Default Evaluation Configuration +# Settings used for checkpoint evaluation during training. + +evaluate_checkpoints: false +# Max number of control steps during evaluation rollout. +eval_max_steps: 5000 +# Seed for deterministic evaluation reset. +eval_seed: 0 diff --git a/configs/logging/default.yaml b/configs/logging/default.yaml index af9bc6d..2f7de8c 100644 --- a/configs/logging/default.yaml +++ b/configs/logging/default.yaml @@ -10,7 +10,4 @@ save_checkpoints: true checkpoint_frequency: 100 upload_final_model: false upload_checkpoints: false -hf_entity: "" -evaluate_checkpoints: false -eval_max_steps: 5000 -eval_seed: 0 \ No newline at end of file +hf_entity: "" \ No newline at end of file diff --git a/configs/main_config.yaml b/configs/main_config.yaml index b7cb620..44451f4 100644 --- a/configs/main_config.yaml +++ b/configs/main_config.yaml @@ -6,6 +6,7 @@ defaults: - brittle_star_config - experiment: base - logging: default + - evaluation: default - ppo: default - architecture: centralized - morphology: 5_arms_full diff --git a/src/brittle_star_project/configs/config_evaluation.py b/src/brittle_star_project/configs/config_evaluation.py new file mode 100644 index 0000000..4eeadbc --- /dev/null +++ b/src/brittle_star_project/configs/config_evaluation.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass +class EvaluationConfig: + """Evaluation settings. + + Currently used for synchronous checkpoint evaluation during training. + """ + + # When enabled, each saved checkpoint is evaluated headlessly and the results + # are appended to a CSV in the run's metrics/ folder. + evaluate_checkpoints: bool = False + eval_max_steps: int = 5000 + eval_seed: int = 0 + + def __post_init__(self) -> None: + if self.evaluate_checkpoints and self.eval_max_steps <= 0: + raise ValueError( + "Configuration Error: 'eval_max_steps' must be > 0 when " + "'evaluate_checkpoints' is enabled." + ) diff --git a/src/brittle_star_project/configs/main_config.py b/src/brittle_star_project/configs/main_config.py index 10fd22e..7caa1aa 100644 --- a/src/brittle_star_project/configs/main_config.py +++ b/src/brittle_star_project/configs/main_config.py @@ -2,6 +2,7 @@ from dataclasses import dataclass, field from experiment_logger.config_logger import LoggingConfig from brittle_star_project.configs.config_experiment import ExperimentConfig +from brittle_star_project.configs.config_evaluation import EvaluationConfig from brittle_star_project.configs.config_ppo import PPOConfig from brittle_star_project.configs.config_architecture import ArchitectureConfig from brittle_star_project.configs.config_simulation import SimulationSettings @@ -23,6 +24,7 @@ class BrittleStarConfig: experiment: ExperimentConfig = field(default_factory=ExperimentConfig) logging: LoggingConfig = field(default_factory=LoggingConfig) + evaluation: EvaluationConfig = field(default_factory=EvaluationConfig) ppo: PPOConfig = field(default_factory=PPOConfig) # This field is polymorphic; defaults to the base class to allow subclasses # (CentralizedConfig, DecentralizedConfig) to be merged in via Hydra. diff --git a/src/brittle_star_project/configs/register_configs.py b/src/brittle_star_project/configs/register_configs.py index 147da10..9522212 100644 --- a/src/brittle_star_project/configs/register_configs.py +++ b/src/brittle_star_project/configs/register_configs.py @@ -2,6 +2,7 @@ from hydra.core.config_store import ConfigStore from experiment_logger.config_logger import LoggingConfig from brittle_star_project.configs.config_experiment import ExperimentConfig +from brittle_star_project.configs.config_evaluation import EvaluationConfig from brittle_star_project.configs.config_ppo import PPOConfig from brittle_star_project.configs.config_architecture import ( CentralizedConfig, @@ -32,6 +33,7 @@ def register_configs() -> None: # Sub-config groups — each group corresponds to a configs/ subdirectory. cs.store(group="experiment", name="base_experiment", node=ExperimentConfig) cs.store(group="logging", name="base_logging", node=LoggingConfig) + cs.store(group="evaluation", name="base_evaluation", node=EvaluationConfig) cs.store(group="ppo", name="base_ppo", node=PPOConfig) # Architecture variants — swap via CLI: architecture=decentralized diff --git a/src/brittle_star_project/trainers/PPOTrainer.py b/src/brittle_star_project/trainers/PPOTrainer.py index 9629e60..5ae20ec 100644 --- a/src/brittle_star_project/trainers/PPOTrainer.py +++ b/src/brittle_star_project/trainers/PPOTrainer.py @@ -263,6 +263,7 @@ class PPOTrainer: self.ppo = cfg.ppo self.experiment = cfg.experiment self.logging_cfg = cfg.logging + self.evaluation_cfg = cfg.evaluation self.env = env self.run_dir = run_dir self.run_name = run_name @@ -710,11 +711,22 @@ class PPOTrainer: return csv_path def _evaluate_checkpoint(self, iteration: int, *, trained_timesteps: int) -> None: - if not self.logging_cfg.evaluate_checkpoints: + if not self.evaluation_cfg.evaluate_checkpoints: return - max_steps = int(self.logging_cfg.eval_max_steps) - seed = int(self.logging_cfg.eval_seed) + max_steps = int(self.evaluation_cfg.eval_max_steps) + seed = int(self.evaluation_cfg.eval_seed) + + if max_steps <= 0: + self.logger.warning("[EVAL]: eval_max_steps must be > 0; skipping evaluation") + return + + if not self.logging_cfg.save_checkpoints or self.logging_cfg.checkpoint_frequency <= 0: + self.logger.warning( + "[EVAL]: evaluate_checkpoints is enabled but checkpoint saving is disabled; " + "skipping evaluation" + ) + return # Run evaluation best-effort; never fail training because evaluation failed. try: diff --git a/src/experiment_logger/config_logger.py b/src/experiment_logger/config_logger.py index d158e84..fd77a28 100644 --- a/src/experiment_logger/config_logger.py +++ b/src/experiment_logger/config_logger.py @@ -18,13 +18,6 @@ class LoggingConfig: upload_final_model: bool = False upload_checkpoints: bool = False - # Checkpoint evaluation (synchronous, in-process) - # When enabled, each saved checkpoint is evaluated headlessly and the results - # are appended to a CSV in the run's metrics/ folder. - evaluate_checkpoints: bool = False - eval_max_steps: int = 5000 - eval_seed: int = 0 - hf_entity: str = "" def __post_init__(self): @@ -39,19 +32,5 @@ class LoggingConfig: "both 'track' and 'save_checkpoints' to also be True." ) - if self.evaluate_checkpoints: - if not self.save_checkpoints: - raise ValueError( - "Configuration Error: 'evaluate_checkpoints' is True, but it requires " - "'save_checkpoints' to also be True." - ) - if self.checkpoint_frequency <= 0: - raise ValueError( - "Configuration Error: 'evaluate_checkpoints' is True, but it requires " - "'checkpoint_frequency' to be > 0." - ) - if self.eval_max_steps <= 0: - raise ValueError( - "Configuration Error: 'eval_max_steps' must be > 0 when " - "'evaluate_checkpoints' is enabled." - ) + # NOTE: Checkpoint evaluation settings live under the project's + # `evaluation` config group (see brittle_star_project.configs).