From 90c2c3d1f0e6d3d3633757744a2392f39b99c521 Mon Sep 17 00:00:00 2001
From: Jona Reynaert <jona.reynaert@ugent.be>
Date: Sun, 3 May 2026 15:53:26 +0200
Subject: [PATCH] feat: added flags for checkpoint evaluation

---
 configs/logging/default.yaml           |  3 +++
 src/experiment_logger/config_logger.py | 24 ++++++++++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/configs/logging/default.yaml b/configs/logging/default.yaml
index 1ecf7fd..af9bc6d 100644
--- a/configs/logging/default.yaml
+++ b/configs/logging/default.yaml
@@ -11,3 +11,6 @@ checkpoint_frequency: 100
 upload_final_model: false
 upload_checkpoints: false
 hf_entity: ""
+evaluate_checkpoints: false
+eval_max_steps: 5000
+eval_seed: 0
\ No newline at end of file
diff --git a/src/experiment_logger/config_logger.py b/src/experiment_logger/config_logger.py
index c34c162..d158e84 100644
--- a/src/experiment_logger/config_logger.py
+++ b/src/experiment_logger/config_logger.py
@@ -18,6 +18,13 @@ class LoggingConfig:
     upload_final_model: bool = False
     upload_checkpoints: bool = False
 
+    # Checkpoint evaluation (synchronous, in-process)
+    # When enabled, each saved checkpoint is evaluated headlessly and the results
+    # are appended to a CSV in the run's metrics/ folder.
+    evaluate_checkpoints: bool = False
+    eval_max_steps: int = 5000
+    eval_seed: int = 0
+
     hf_entity: str = ""
 
     def __post_init__(self):
@@ -31,3 +38,20 @@ class LoggingConfig:
                 "Configuration Error: 'upload_checkpoints' is True, but it requires "
                 "both 'track' and 'save_checkpoints' to also be True."
             )
+
+        if self.evaluate_checkpoints:
+            if not self.save_checkpoints:
+                raise ValueError(
+                    "Configuration Error: 'evaluate_checkpoints' is True, but it requires "
+                    "'save_checkpoints' to also be True."
+                )
+            if self.checkpoint_frequency <= 0:
+                raise ValueError(
+                    "Configuration Error: 'evaluate_checkpoints' is True, but it requires "
+                    "'checkpoint_frequency' to be > 0."
+                )
+            if self.eval_max_steps <= 0:
+                raise ValueError(
+                    "Configuration Error: 'eval_max_steps' must be > 0 when "
+                    "'evaluate_checkpoints' is enabled."
+                )