From 9dce0175259084c4c6ecab13131999d57ca00e15 Mon Sep 17 00:00:00 2001 From: Tibo De Peuter Date: Sat, 4 Apr 2026 18:11:38 +0200 Subject: [PATCH] refactor(hpc): migrate to PBS/vsc-venv, reorganise scripts under scripts/hpc/ - Replace Slurm headers with PBS (#PBS) directives - Replace manual module+venv logic with vsc-venv --activate - Cluster selection via 'module swap cluster/' before qsub - MUJOCO_GL=egl retained: required for headless physics sim and video rendering - Delete deprecated scripts/hpc_install.sh and scripts/hpc_train.slurm --- scripts/hpc/install.sh | 33 +++++++++++++++++++++++ scripts/hpc/train.pbs | 38 ++++++++++++++++++++++++++ scripts/hpc_install.sh | 59 ----------------------------------------- scripts/hpc_train.slurm | 32 ---------------------- 4 files changed, 71 insertions(+), 91 deletions(-) create mode 100644 scripts/hpc/install.sh create mode 100644 scripts/hpc/train.pbs delete mode 100644 scripts/hpc_install.sh delete mode 100644 scripts/hpc_train.slurm diff --git a/scripts/hpc/install.sh b/scripts/hpc/install.sh new file mode 100644 index 0000000..2700275 --- /dev/null +++ b/scripts/hpc/install.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# scripts/hpc/install.sh — Run once on a login node (donphan) to set up the +# project's Python virtual environment using the official vsc-venv wrapper. +# +# Usage (from project root): +# bash scripts/hpc/install.sh +# +# Prerequisites: +# - Connected to VSC via the web portal (HPC Login → Shell tmux) +# - Project cloned to $VSC_DATA or $VSC_HOME +# - Run from the project root directory + +set -euo pipefail + +echo "==> Loading vsc-venv module..." +module load vsc-venv + +echo "==> Activating virtual environment (created per-cluster in \$VSC_DATA)..." +# vsc-venv transparently creates and manages a per-cluster venv in $VSC_DATA, +# loading the modules listed in env/hpc/modules.txt and pip-installing anything +# in env/hpc/requirements.txt that is not already satisfied. +source vsc-venv --activate \ + --modules env/hpc/modules.txt \ + --requirements env/hpc/requirements.txt + +echo "==> Registering Jupyter kernel..." +python -m ipykernel install --user --name="sel3_${VSC_INSTITUTE_CLUSTER}" \ + --display-name "SEL3 (${VSC_INSTITUTE_CLUSTER})" + +echo "" +echo "✅ Environment ready. Kernel: sel3_${VSC_INSTITUTE_CLUSTER}" +echo " Activate in future sessions with:" +echo " module load vsc-venv && source vsc-venv --activate --modules env/hpc/modules.txt" diff --git a/scripts/hpc/train.pbs b/scripts/hpc/train.pbs new file mode 100644 index 0000000..cf8e2c3 --- /dev/null +++ b/scripts/hpc/train.pbs @@ -0,0 +1,38 @@ +#!/bin/bash +# scripts/hpc/train.pbs — Submit PPO training as a batch job with qsub. +# +# Usage (from project root): +# qsub scripts/hpc/train.pbs +# +# To target a specific GPU cluster (default: joltik): +# module swap cluster/accelgor && qsub scripts/hpc/train.pbs +# +# Available GPU clusters: joltik, accelgor, litleo +# Debug / interactive: doduo (CPU) or donphan (interactive GPU) + +# --------------------------------------------------------------------------- +# PBS job directives +# --------------------------------------------------------------------------- +#PBS -N brittlestar-ppo +#PBS -l nodes=1:ppn=8:gpus=1 +#PBS -l walltime=24:00:00 +#PBS -l mem=32gb +#PBS -o runs/pbs_${PBS_JOBID}.out +#PBS -e runs/pbs_${PBS_JOBID}.err +# --------------------------------------------------------------------------- + +set -euo pipefail + +cd "$PBS_O_WORKDIR" + +# Activate the project virtual environment via the official vsc-venv wrapper. +# This loads the modules listed in env/hpc/modules.txt and the per-cluster venv. +module load vsc-venv +source vsc-venv --activate --modules env/hpc/modules.txt + +# EGL is required for headless MuJoCo operation — both for the physics +# simulation loop and for rendering videos to disk. Without this, MuJoCo +# attempts to open an X11 display and fails on compute nodes. +export MUJOCO_GL=egl + +python src/train.py --config configs/production_training.yaml diff --git a/scripts/hpc_install.sh b/scripts/hpc_install.sh deleted file mode 100644 index f7dbb21..0000000 --- a/scripts/hpc_install.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -# hpc_install.sh — Run once on a login node to set up the project environment. -# Usage: bash scripts/hpc_install.sh - -set -euo pipefail - -# -- Storage: put the venv on $VSC_DATA --------------------------------------- -mkdir -p "$VSC_DATA/venvs" -ln -sf "$VSC_DATA/venvs" "$VSC_HOME/venvs" - -# -- Load built-in HPC modules to save space ---------------------------------- -# Loading these will automatically load Python 3.11.3 + CUDA + GCC 12.3.0 -ml load jax/0.4.25-gfbf-2023a-CUDA-12.1.1 -ml load Flax/0.8.4-gfbf-2023a-CUDA-12.1.1 -ml load Optax/0.2.2-gfbf-2023a-CUDA-12.1.1 -ml load wandb/0.16.1-GCC-12.3.0 -ml load matplotlib/3.7.2-gfbf-2023a -ml load PyYAML/6.0-GCCcore-12.3.0 -ml load FFmpeg/5.1.2-GCCcore-12.3.0 - -export VENV_PATH="$VSC_DATA/venvs/sel3_${VSC_INSTITUTE_CLUSTER}" - -# -- Create venv with system-site-packages so it sees the loaded modules ---- -python -m venv --system-site-packages "$VENV_PATH" -source "$VENV_PATH/bin/activate" - -# -- Extract and install missing dependencies ---------------------------------- -# Dynamically parse pyproject.toml and install only what is missing natively -MISSING_DEPS=$(python -c ' -import tomllib, importlib.util, re -with open("pyproject.toml", "rb") as f: - deps = tomllib.load(f)["project"]["dependencies"] - -missing = [] -for dep in deps: - pkg = re.split(r"[\[=><~]", dep)[0].strip() - # Map common PyPI package names to their python import names - mapping = {"pyyaml": "yaml", "pyopengl": "OpenGL", "pyopengl-accelerate": "OpenGL_accelerate"} - import_name = mapping.get(pkg.lower(), pkg.replace("-", "_")) - - # If the system module cannot find the package, add it to our pip install list - if getattr(importlib.util, "find_spec", None) is None or importlib.util.find_spec(import_name) is None: - missing.append(f"\"{dep}\"") - -print(" ".join(missing)) -') - -if [ -n "$MISSING_DEPS" ]; then - echo "Installing missing dependencies: $MISSING_DEPS" - pip install --upgrade pip - eval "pip install $MISSING_DEPS" -else - echo "All dependencies from pyproject.toml are successfully satisfied by HPC system modules." -fi - -# -- Register as Jupyter kernel ----------------------------------------------- -python -m ipykernel install --user --name="sel3_${VSC_INSTITUTE_CLUSTER}-kernel" - -echo "✅ Environment created at $VENV_PATH" diff --git a/scripts/hpc_train.slurm b/scripts/hpc_train.slurm deleted file mode 100644 index 736b652..0000000 --- a/scripts/hpc_train.slurm +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=brittle-star-ppo -#SBATCH --output=runs/slurm_%j.out -#SBATCH --error=runs/slurm_%j.err -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --cpus-per-task=8 -#SBATCH --mem=32G -#SBATCH --time=24:00:00 -#SBATCH --gpus=1 -# Adjust --partition to your cluster (check with `sinfo` on the login node) -#SBATCH --partition=gpu - -# -- Load the exact same modules as in hpc_install.sh ------------------------- -ml load jax/0.4.25-gfbf-2023a-CUDA-12.1.1 -ml load Flax/0.8.4-gfbf-2023a-CUDA-12.1.1 -ml load Optax/0.2.2-gfbf-2023a-CUDA-12.1.1 -ml load wandb/0.16.1-GCC-12.3.0 -ml load matplotlib/3.7.2-gfbf-2023a -ml load PyYAML/6.0-GCCcore-12.3.0 -ml load FFmpeg/5.1.2-GCCcore-12.3.0 - -# -- Activate the system-site-packages venv ----------------------------------- -source "$VSC_DATA/venvs/sel3_${VSC_INSTITUTE_CLUSTER}/bin/activate" - -# -- HPC-specific environment flags ------------------------------------------- -export MUJOCO_GL=egl # Headless OpenGL via EGL (no display required) - -# -- Run from the project root ------------------------------------------------- -cd "$SLURM_SUBMIT_DIR" - -python src/train.py --config configs/production_training.yaml