diff --git a/README.md b/README.md index e9cabc6..45bdf63 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,9 @@ For detailed instructions on how to use the project, please refer to the **[API 3. **Simulate a trained model:** See [Simulation & Evaluation](docs/api/simulation.md). +4. **Compare fault tolerance of models:** + See [Checkpoint & Model Evaluation](docs/api/evaluation.md) + ## HPC See **[docs/HPC.md](docs/HPC.md)** for the full guide, including environment setup, cluster selection, interactive debugging, and job submission. diff --git a/docs/README.md b/docs/README.md index 3c76684..521cb28 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ # Documentation -## Design & architecture ([`/design`](./design/)) +## Design & architecture (`/design`) If you are interested in the "why did you do it like this?" @@ -11,7 +11,7 @@ If you are interested in the "why did you do it like this?" - [Learning algorithm](./design/learning_algorithm.md): RL techniques, i.e. PPO. - [Reward function](./design/learning_algorithm.md): Goals, fitness tracking, and reward structures. -## API reference ([`/api`](./api/)) +## API reference (`/api`) If you are interested in the "how do I use it?" @@ -19,3 +19,5 @@ If you are interested in the "how do I use it?" - [Tracking & Monitoring](./api/tracking.md): Setting up WandB and TensorBoard to monitor runs. - [Simulation](./api/simulation.md): Visualizing and evaluating models. - [Environment](./api/environment.md): MuJoCo environment interaction and configuration. +- [Analysis](./api/analysis.md): Comparing checkpoints and generating plots. +- [Evaluation](./api/evaluation.md): Evaluating checkpoints and comparing fault tolerance. \ No newline at end of file diff --git a/docs/api/evaluation.md b/docs/api/evaluation.md index e5f91d6..45b880e 100644 --- a/docs/api/evaluation.md +++ b/docs/api/evaluation.md @@ -15,7 +15,7 @@ python scripts/train.py evaluation.evaluate_checkpoints=true evaluation.eval_max Results are saved to `runs//metrics/checkpoint_evaluation.csv` and synced to Weights & Biases if enabled. -## Cross-Model & Defect Tolerance Analysis +## Cross-Model & Fault Tolerance Analysis To measure how well different controllers handle damage (amputations), use `scripts/compare_models.py`. This script performs a grid search over models x morphologies. diff --git a/scripts/plots/analyze_convergence.py b/scripts/plots/analyze_convergence.py index df9826c..3bb078d 100644 --- a/scripts/plots/analyze_convergence.py +++ b/scripts/plots/analyze_convergence.py @@ -58,11 +58,10 @@ class Columns(str, Enum): # Maps architecture display names to the path of their evaluation CSV. # Update these paths once real evaluation data is available. FILE_MAPPING: dict[str, str] = { - "centralized 2 arms": "runs/dummy/dummy_centralized_2_arms.csv", - "centralized 5 arms": "runs/dummy/dummy_centralized_5_arms.csv", - "decentralized fully connected": "runs/dummy/dummy_decentralized_fully_connected.csv", - "decentralized ring-level": "runs/dummy/dummy_decentralized_ring-level.csv", - "decentralized segment-level": "runs/dummy/dummy_decentralized_segment-level.csv", + # "centralized 2 arms": "runs/dummy/dummy_centralized_2_arms.csv", + "centralized 5 arms": "runs/final-v2-centralized/checkpoint_evaluation.csv", + "decentralized fully connected": "runs/final-v2-fully-conn/checkpoint_evaluation.csv", + "decentralized ring-level": "runs/final-v2-ring/checkpoint_evaluation.csv", } # Architecture profiles for dummy data generation: (max_reward, max_velocity, sigmoid_speed) @@ -137,6 +136,10 @@ def load_metrics(file_mapping: dict[str, str]) -> pd.DataFrame: df = df[required].copy() df[Columns.ARCH] = arch_name + df[Columns.VELOCITY] = (df[Columns.INITIAL_XY_DIST] - df[Columns.FINAL_XY_DIST]) / df[ + Columns.EVAL_STEPS + ] + dfs.append(df) return pd.concat(dfs, ignore_index=True) if dfs else pd.DataFrame() @@ -333,6 +336,7 @@ def plot_results(df: pd.DataFrame, results: pd.DataFrame, output_dir: str, **kwa def obtain_data() -> pd.DataFrame: """Resolves the file mapping, falling back to generated dummy CSVs if needed.""" global USING_DUMMY_DATA + if not any(os.path.exists(p) for p in FILE_MAPPING.values()): logger.info("No real evaluation files found. Generating dummy CSVs at expected locations.") generate_dummy_csvs(FILE_MAPPING) diff --git a/scripts/tools/download_wandb_project.py b/scripts/tools/download_wandb_project.py new file mode 100644 index 0000000..bf2fb11 --- /dev/null +++ b/scripts/tools/download_wandb_project.py @@ -0,0 +1,142 @@ +from pathlib import Path +from concurrent.futures import ThreadPoolExecutor, as_completed +import threading +import wandb +import argparse + +# tune these depending on network / W&B limits +MAX_RUN_WORKERS = 8 +MAX_FILE_WORKERS = 16 +MAX_ARTIFACT_WORKERS = 8 + +api = wandb.Api() + +print_lock = threading.Lock() + + +def safe_print(*args, **kwargs): + with print_lock: + print(*args, **kwargs) + + +def download_file(file, run_dir): + target = run_dir / file.name + + try: + # skip existing files + if target.exists(): + return f"SKIP FILE {target}" + + target.parent.mkdir(parents=True, exist_ok=True) + + file.download(root=run_dir, replace=False) + + return f"DONE FILE {target}" + + except Exception as e: + return f"FAIL FILE {target}: {e}" + + +def sanitize_artifact_name(name: str): + return name.replace(":", "_") + + +def download_artifact(artifact, artifact_root): + try: + artifact_name = sanitize_artifact_name(artifact.name) + artifact_dir = artifact_root / artifact_name + + if artifact_dir.exists() and any(artifact_dir.iterdir()): + return f"SKIP ARTIFACT {artifact.name}" + + artifact_dir.mkdir(parents=True, exist_ok=True) + + artifact.download(root=artifact_dir) + + return f"DONE ARTIFACT {artifact.name}" + + except Exception as e: + return f"FAIL ARTIFACT {artifact.name}: {e}" + + +def download_run(run, root): + run_dir = root / f"{run.name}" + run_dir.mkdir(parents=True, exist_ok=True) + + safe_print(f"\n=== {run.name} ({run.id}) ===") + + # ------------------------- + # Download regular run files + # ------------------------- + files = list(run.files()) + + with ThreadPoolExecutor(max_workers=MAX_FILE_WORKERS) as executor: + futures = [executor.submit(download_file, file, run_dir) for file in files] + + for future in as_completed(futures): + safe_print(future.result()) + + # ------------------------- + # Download logged artifacts + # ------------------------- + artifact_root = run_dir / "artifacts" + + try: + artifacts = list(run.logged_artifacts()) + safe_print(f"Found {len(artifacts)} artifacts for {run.name}") + + with ThreadPoolExecutor(max_workers=MAX_ARTIFACT_WORKERS) as executor: + futures = [ + executor.submit(download_artifact, artifact, artifact_root) + for artifact in artifacts + ] + + for future in as_completed(futures): + safe_print(future.result()) + + except Exception as e: + safe_print(f"Artifact download failed for {run.name}: {e}") + + # ------------------------- + # OPTIONAL: download used/input artifacts + # ------------------------- + # try: + # used_artifacts = list(run.used_artifacts()) + # used_root = run_dir / "used_artifacts" + # + # for artifact in used_artifacts: + # download_artifact(artifact, used_root) + # except Exception as e: + # safe_print(f"Used artifact download failed: {e}") + + safe_print(f"Finished {run.name}") + + +def main(entity: str, project: str, root: Path): + root.mkdir(exist_ok=True) + + runs = list(api.runs(f"{entity}/{project}")) + + safe_print(f"Found {len(runs)} runs") + + with ThreadPoolExecutor(max_workers=MAX_RUN_WORKERS) as executor: + futures = [executor.submit(download_run, run, root) for run in runs] + + for future in as_completed(futures): + try: + future.result() + except Exception as e: + safe_print("RUN FAILED:", e) + + safe_print("\nAll downloads complete.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--entity", type=str, default="SEL3-2026-Groep-4") + parser.add_argument("--project", type=str, required=True) + parser.add_argument("--root", type=str, default="runs") + args = parser.parse_args() + + root = Path(args.root) + main(entity=args.entity, project=args.project, root=root)