tdpeuter/2026SEL3-project-Brittle_Star_Controller_Modularity
tdpeuter/2026SEL3-project-Brittle_Star_Controller_Modularity
Archived
1
Fork 0
Code Issues Releases Activity

fix(hpc): streamlined install

Browse source
This commit is contained in:
Tibo De Peuter 2026-04-04 22:01:21 +02:00
parent 78aa20868e
commit 15319890a6
7 changed files with 74 additions and 118 deletions
Download patch file Download diff file Expand all files Collapse all files

39
scripts/hpc/install.sh
View file

@ -1,38 +1,31 @@
#!/bin/bash
# scripts/hpc/install.sh
#
# Usage:
# module swap cluster/donphan
# qsub -I -l nodes=1:ppn=8:gpus=1
# (Wait for session to start, then:)
# Usage (on any compute node):
# bash scripts/hpc/install.sh
set -eo pipefail
# Keep caches off $VSC_HOME (quota ~3 GB).
export PIP_CACHE_DIR="$VSC_SCRATCH/.cache/pip"
export UV_CACHE_DIR="$VSC_SCRATCH/.cache/uv"
mkdir -p "$PIP_CACHE_DIR" "$UV_CACHE_DIR"
# Ensure we are in the project root
if [ -n "$PBS_O_WORKDIR" ]; then
cd "$PBS_O_WORKDIR"
fi
# Mirror configs to $VSC_DATA to avoid home quota limits (3GB)
# vsc-venv manages environments relative to the requirements file
HPC_CONFIG_DIR="$VSC_DATA/2026SEL3-project/env/hpc"
mkdir -p "$HPC_CONFIG_DIR"
cp env/hpc/*.txt "$HPC_CONFIG_DIR/"
module load vsc-venv
echo 'Synchronizing and activating environment...'
echo ">>> Synchronizing and activating environment (vsc-venv)..."
source vsc-venv --activate \
--modules env/hpc/modules.txt \
--requirements env/hpc/requirements.txt
--modules "$HPC_CONFIG_DIR/modules.txt" \
--requirements "$HPC_CONFIG_DIR/requirements.txt"
# Step 2: Force upgrade shared system dependencies to ensure venv precedence
echo 'Applying library overlays (NumPy, Protobuf)...'
# Overlay specific NumPy/Protobuf versions to ensure venv precedence
echo ">>> Applying library overlays (NumPy, Protobuf)..."
pip install --upgrade --no-deps numpy protobuf
echo 'Installing ipykernel...'
python -m ipykernel install --user --name="sel3_${VSC_INSTITUTE_CLUSTER}" \
--display-name "SEL3 (${VSC_INSTITUTE_CLUSTER})"
echo 'Done'
echo '>>> Installing ipykernel...'
CLUSTER_ID="${VSC_INSTITUTE_CLUSTER:-generic}"
python -m ipykernel install --user --name="sel3_${CLUSTER_ID}" \
--display-name "SEL3 (${CLUSTER_ID})"
echo '>>> Done'

14
scripts/hpc/train.pbs
View file

@ -3,9 +3,6 @@
#
# Usage (from project root):
# qsub scripts/hpc/train.pbs
#
# To target a specific GPU cluster (default: joltik):
# module swap cluster/accelgor && qsub scripts/hpc/train.pbs
#PBS -N brittlestar-ppo
#PBS -l nodes=1:ppn=8:gpus=1
@ -30,18 +27,23 @@ DATA_RUNDIR="$VSC_DATA/runs/$RUN_ID"
mkdir -p "$PIP_CACHE_DIR" "$UV_CACHE_DIR" "$SCRATCH_RUNDIR" "$DATA_RUNDIR" runs/
module load vsc-venv
# Activate mirrored environment from $VSC_DATA to avoid home quota
HPC_CONFIG_DIR="$VSC_DATA/2026SEL3-project/env/hpc"
set +u
source vsc-venv --activate \
--modules env/hpc/modules.txt \
--requirements env/hpc/requirements.txt
--modules "$HPC_CONFIG_DIR/modules.txt" \
--requirements "$HPC_CONFIG_DIR/requirements.txt"
set -u
# Ensure venv versions of NumPy/Protobuf take precedence
pip install --upgrade --no-deps numpy protobuf > /dev/null 2>&1
export MUJOCO_GL=egl
export WANDB_DIR="$SCRATCH_RUNDIR"
python src/train.py \
--env-config-path configs/hpc/smoke_test.yaml \
--run-dir "$SCRATCH_RUNDIR"
# ^^^ Replace with your production config, e.g. configs/production_training.yaml
cp -r "$SCRATCH_RUNDIR/." "$DATA_RUNDIR/"