From a512682eddab74b4acbcdc2c205289a9bc423593 Mon Sep 17 00:00:00 2001 From: Tibo De Peuter Date: Sat, 4 Apr 2026 20:26:09 +0200 Subject: [PATCH] fix(hpc): additional insights --- docs/HPC.md | 21 ++++++++++++--------- scripts/hpc/install.sh | 8 +++++++- scripts/hpc/train.pbs | 2 ++ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/docs/HPC.md b/docs/HPC.md index 2f23704..33c06df 100644 --- a/docs/HPC.md +++ b/docs/HPC.md @@ -17,22 +17,25 @@ Check your quota: (Usage section). ## Initial Environment Setup -Run **once** from a login shell on `donphan` after cloning the repository: +Run **once** after cloning the repository, but ensure you are logged into a **compute node** on the target cluster (e.g., `donphan` or `joltik`). The login node (`doduo`) will block the installation script to prevent architecture mismatches. ```bash -# 1. Connect via the web portal → HPC Login → Interactive Apps > Shell (tmux) -# Set cluster to "donphan (interactive/debug)" +# 1. Swap to the target cluster +module swap cluster/joltik -# 2. Clone the project into $VSC_HOME (if not done yet) -cd $VSC_HOME -git clone -cd 2026SEL3-project-BrittleStar +# 2. Start an interactive session on a compute node +qsub -I -l nodes=1:ppn=8:gpus=1 -# 3. Run the install script +# 3. Navigate to the project directory and run the install script +cd SEL3-2026-Groep-4 bash scripts/hpc/install.sh + +# 4. Exit the interactive session once finished +exit ``` -> **Note:** virtual environments are cluster-specific. Re-run the script when switching to a new cluster. +> [!IMPORTANT] +> Virtual environments are cluster-specific. If you want to switch from `joltik` to `accelgor`, you must re-run the `install.sh` script while logged into an interactive session on `accelgor`. ## Interactive Debugging on donphan diff --git a/scripts/hpc/install.sh b/scripts/hpc/install.sh index 2aa3ac5..8e2c76d 100644 --- a/scripts/hpc/install.sh +++ b/scripts/hpc/install.sh @@ -1,7 +1,10 @@ #!/bin/bash # scripts/hpc/install.sh # -# Usage (from project root): +# Usage: +# module swap cluster/donphan +# qsub -I -l nodes=1:ppn=8:gpus=1 +# (Wait for session to start, then:) # bash scripts/hpc/install.sh set -eo pipefail @@ -13,9 +16,12 @@ mkdir -p "$PIP_CACHE_DIR" "$UV_CACHE_DIR" module load vsc-venv +# Temporarily disable 'unbound variable' check for vsc-venv compatibility +set +u source vsc-venv --activate \ --modules env/hpc/modules.txt \ --requirements env/hpc/requirements.txt +set -u python -m ipykernel install --user --name="sel3_${VSC_INSTITUTE_CLUSTER}" \ --display-name "SEL3 (${VSC_INSTITUTE_CLUSTER})" diff --git a/scripts/hpc/train.pbs b/scripts/hpc/train.pbs index 1ed595e..90f4345 100644 --- a/scripts/hpc/train.pbs +++ b/scripts/hpc/train.pbs @@ -30,7 +30,9 @@ DATA_RUNDIR="$VSC_DATA/runs/$RUN_ID" mkdir -p "$PIP_CACHE_DIR" "$UV_CACHE_DIR" "$SCRATCH_RUNDIR" "$DATA_RUNDIR" runs/ module load vsc-venv +set +u source vsc-venv --activate --modules env/hpc/modules.txt +set -u export MUJOCO_GL=egl export WANDB_DIR="$SCRATCH_RUNDIR"