refactor(hpc): migrate to PBS/vsc-venv, reorganise scripts under scripts/hpc/
- Replace Slurm headers with PBS (#PBS) directives - Replace manual module+venv logic with vsc-venv --activate - Cluster selection via 'module swap cluster/<name>' before qsub - MUJOCO_GL=egl retained: required for headless physics sim and video rendering - Delete deprecated scripts/hpc_install.sh and scripts/hpc_train.slurm
This commit is contained in:
parent
14fd832283
commit
9dce017525
4 changed files with 71 additions and 91 deletions
33
scripts/hpc/install.sh
Normal file
33
scripts/hpc/install.sh
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
#!/bin/bash
|
||||
# scripts/hpc/install.sh — Run once on a login node (donphan) to set up the
|
||||
# project's Python virtual environment using the official vsc-venv wrapper.
|
||||
#
|
||||
# Usage (from project root):
|
||||
# bash scripts/hpc/install.sh
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Connected to VSC via the web portal (HPC Login → Shell tmux)
|
||||
# - Project cloned to $VSC_DATA or $VSC_HOME
|
||||
# - Run from the project root directory
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
echo "==> Loading vsc-venv module..."
|
||||
module load vsc-venv
|
||||
|
||||
echo "==> Activating virtual environment (created per-cluster in \$VSC_DATA)..."
|
||||
# vsc-venv transparently creates and manages a per-cluster venv in $VSC_DATA,
|
||||
# loading the modules listed in env/hpc/modules.txt and pip-installing anything
|
||||
# in env/hpc/requirements.txt that is not already satisfied.
|
||||
source vsc-venv --activate \
|
||||
--modules env/hpc/modules.txt \
|
||||
--requirements env/hpc/requirements.txt
|
||||
|
||||
echo "==> Registering Jupyter kernel..."
|
||||
python -m ipykernel install --user --name="sel3_${VSC_INSTITUTE_CLUSTER}" \
|
||||
--display-name "SEL3 (${VSC_INSTITUTE_CLUSTER})"
|
||||
|
||||
echo ""
|
||||
echo "✅ Environment ready. Kernel: sel3_${VSC_INSTITUTE_CLUSTER}"
|
||||
echo " Activate in future sessions with:"
|
||||
echo " module load vsc-venv && source vsc-venv --activate --modules env/hpc/modules.txt"
|
||||
38
scripts/hpc/train.pbs
Normal file
38
scripts/hpc/train.pbs
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
#!/bin/bash
|
||||
# scripts/hpc/train.pbs — Submit PPO training as a batch job with qsub.
|
||||
#
|
||||
# Usage (from project root):
|
||||
# qsub scripts/hpc/train.pbs
|
||||
#
|
||||
# To target a specific GPU cluster (default: joltik):
|
||||
# module swap cluster/accelgor && qsub scripts/hpc/train.pbs
|
||||
#
|
||||
# Available GPU clusters: joltik, accelgor, litleo
|
||||
# Debug / interactive: doduo (CPU) or donphan (interactive GPU)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PBS job directives
|
||||
# ---------------------------------------------------------------------------
|
||||
#PBS -N brittlestar-ppo
|
||||
#PBS -l nodes=1:ppn=8:gpus=1
|
||||
#PBS -l walltime=24:00:00
|
||||
#PBS -l mem=32gb
|
||||
#PBS -o runs/pbs_${PBS_JOBID}.out
|
||||
#PBS -e runs/pbs_${PBS_JOBID}.err
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
cd "$PBS_O_WORKDIR"
|
||||
|
||||
# Activate the project virtual environment via the official vsc-venv wrapper.
|
||||
# This loads the modules listed in env/hpc/modules.txt and the per-cluster venv.
|
||||
module load vsc-venv
|
||||
source vsc-venv --activate --modules env/hpc/modules.txt
|
||||
|
||||
# EGL is required for headless MuJoCo operation — both for the physics
|
||||
# simulation loop and for rendering videos to disk. Without this, MuJoCo
|
||||
# attempts to open an X11 display and fails on compute nodes.
|
||||
export MUJOCO_GL=egl
|
||||
|
||||
python src/train.py --config configs/production_training.yaml
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
#!/bin/bash
|
||||
# hpc_install.sh — Run once on a login node to set up the project environment.
|
||||
# Usage: bash scripts/hpc_install.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# -- Storage: put the venv on $VSC_DATA ---------------------------------------
|
||||
mkdir -p "$VSC_DATA/venvs"
|
||||
ln -sf "$VSC_DATA/venvs" "$VSC_HOME/venvs"
|
||||
|
||||
# -- Load built-in HPC modules to save space ----------------------------------
|
||||
# Loading these will automatically load Python 3.11.3 + CUDA + GCC 12.3.0
|
||||
ml load jax/0.4.25-gfbf-2023a-CUDA-12.1.1
|
||||
ml load Flax/0.8.4-gfbf-2023a-CUDA-12.1.1
|
||||
ml load Optax/0.2.2-gfbf-2023a-CUDA-12.1.1
|
||||
ml load wandb/0.16.1-GCC-12.3.0
|
||||
ml load matplotlib/3.7.2-gfbf-2023a
|
||||
ml load PyYAML/6.0-GCCcore-12.3.0
|
||||
ml load FFmpeg/5.1.2-GCCcore-12.3.0
|
||||
|
||||
export VENV_PATH="$VSC_DATA/venvs/sel3_${VSC_INSTITUTE_CLUSTER}"
|
||||
|
||||
# -- Create venv with system-site-packages so it sees the loaded modules ----
|
||||
python -m venv --system-site-packages "$VENV_PATH"
|
||||
source "$VENV_PATH/bin/activate"
|
||||
|
||||
# -- Extract and install missing dependencies ----------------------------------
|
||||
# Dynamically parse pyproject.toml and install only what is missing natively
|
||||
MISSING_DEPS=$(python -c '
|
||||
import tomllib, importlib.util, re
|
||||
with open("pyproject.toml", "rb") as f:
|
||||
deps = tomllib.load(f)["project"]["dependencies"]
|
||||
|
||||
missing = []
|
||||
for dep in deps:
|
||||
pkg = re.split(r"[\[=><~]", dep)[0].strip()
|
||||
# Map common PyPI package names to their python import names
|
||||
mapping = {"pyyaml": "yaml", "pyopengl": "OpenGL", "pyopengl-accelerate": "OpenGL_accelerate"}
|
||||
import_name = mapping.get(pkg.lower(), pkg.replace("-", "_"))
|
||||
|
||||
# If the system module cannot find the package, add it to our pip install list
|
||||
if getattr(importlib.util, "find_spec", None) is None or importlib.util.find_spec(import_name) is None:
|
||||
missing.append(f"\"{dep}\"")
|
||||
|
||||
print(" ".join(missing))
|
||||
')
|
||||
|
||||
if [ -n "$MISSING_DEPS" ]; then
|
||||
echo "Installing missing dependencies: $MISSING_DEPS"
|
||||
pip install --upgrade pip
|
||||
eval "pip install $MISSING_DEPS"
|
||||
else
|
||||
echo "All dependencies from pyproject.toml are successfully satisfied by HPC system modules."
|
||||
fi
|
||||
|
||||
# -- Register as Jupyter kernel -----------------------------------------------
|
||||
python -m ipykernel install --user --name="sel3_${VSC_INSTITUTE_CLUSTER}-kernel"
|
||||
|
||||
echo "✅ Environment created at $VENV_PATH"
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
#!/bin/bash
|
||||
#SBATCH --job-name=brittle-star-ppo
|
||||
#SBATCH --output=runs/slurm_%j.out
|
||||
#SBATCH --error=runs/slurm_%j.err
|
||||
#SBATCH --nodes=1
|
||||
#SBATCH --ntasks=1
|
||||
#SBATCH --cpus-per-task=8
|
||||
#SBATCH --mem=32G
|
||||
#SBATCH --time=24:00:00
|
||||
#SBATCH --gpus=1
|
||||
# Adjust --partition to your cluster (check with `sinfo` on the login node)
|
||||
#SBATCH --partition=gpu
|
||||
|
||||
# -- Load the exact same modules as in hpc_install.sh -------------------------
|
||||
ml load jax/0.4.25-gfbf-2023a-CUDA-12.1.1
|
||||
ml load Flax/0.8.4-gfbf-2023a-CUDA-12.1.1
|
||||
ml load Optax/0.2.2-gfbf-2023a-CUDA-12.1.1
|
||||
ml load wandb/0.16.1-GCC-12.3.0
|
||||
ml load matplotlib/3.7.2-gfbf-2023a
|
||||
ml load PyYAML/6.0-GCCcore-12.3.0
|
||||
ml load FFmpeg/5.1.2-GCCcore-12.3.0
|
||||
|
||||
# -- Activate the system-site-packages venv -----------------------------------
|
||||
source "$VSC_DATA/venvs/sel3_${VSC_INSTITUTE_CLUSTER}/bin/activate"
|
||||
|
||||
# -- HPC-specific environment flags -------------------------------------------
|
||||
export MUJOCO_GL=egl # Headless OpenGL via EGL (no display required)
|
||||
|
||||
# -- Run from the project root -------------------------------------------------
|
||||
cd "$SLURM_SUBMIT_DIR"
|
||||
|
||||
python src/train.py --config configs/production_training.yaml
|
||||
Reference in a new issue