From 520d429bac04186e49a3b986148a52dd1bc49a06 Mon Sep 17 00:00:00 2001
From: Tibo De Peuter <tibo.depeuter@ugent.be>
Date: Sat, 4 Apr 2026 22:48:16 +0200
Subject: [PATCH] fix(hpc): hardening

---
 scripts/hpc/install.sh | 13 +++++++------
 scripts/hpc/train.pbs  | 42 +++++++++++++++++++++++++++---------------
 2 files changed, 34 insertions(+), 21 deletions(-)

diff --git a/scripts/hpc/install.sh b/scripts/hpc/install.sh
index 9fec7c6..a37e94a 100644
--- a/scripts/hpc/install.sh
+++ b/scripts/hpc/install.sh
@@ -1,23 +1,24 @@
-#!/bin/bash
+#!/bin/bash -l
 # scripts/hpc/install.sh
 #
-# Usage (on any compute node):
-#   bash scripts/hpc/install.sh
-
 set -eo pipefail
 
+echo ">>> Starting HPC Installation in $(hostname)..."
+
 # Mirror configs to $VSC_DATA to avoid home quota limits (3GB)
-# vsc-venv manages environments relative to the requirements file
-HPC_CONFIG_DIR="$VSC_DATA/2026SEL3-project/env/hpc"
+PROJ_NAME=$(basename "$PWD")
+HPC_CONFIG_DIR="$VSC_DATA/$PROJ_NAME/env/hpc"
 mkdir -p "$HPC_CONFIG_DIR"
 cp env/hpc/*.txt "$HPC_CONFIG_DIR/"
 
 module load vsc-venv
 
 echo ">>> Synchronizing and activating environment (vsc-venv)..."
+set +eo pipefail
 source vsc-venv --activate \
     --modules "$HPC_CONFIG_DIR/modules.txt" \
     --requirements "$HPC_CONFIG_DIR/requirements.txt"
+set -eo pipefail
 
 # Overlay specific NumPy/Protobuf versions to ensure venv precedence
 echo ">>> Applying library overlays (NumPy, Protobuf)..."
diff --git a/scripts/hpc/train.pbs b/scripts/hpc/train.pbs
index 243e337..83d6284 100644
--- a/scripts/hpc/train.pbs
+++ b/scripts/hpc/train.pbs
@@ -13,32 +13,41 @@
 
 set -euo pipefail
 
-cd "$PBS_O_WORKDIR"
+# Preliminary status echo
+echo ">>> Starting training job $PBS_JOBID on $(hostname)..."
+
+if [ -n "$PBS_O_WORKDIR" ]; then
+    cd "$PBS_O_WORKDIR"
+fi
+
+# Set up storage paths immediately
+PROJ_NAME=$(basename "$PWD")
+RUN_ID="brittlestar_${PBS_JOBID}"
+SCRATCH_RUNDIR="$VSC_SCRATCH/runs/$RUN_ID"
+DATA_RUNDIR="$VSC_DATA/runs/$RUN_ID"
+mkdir -p "$SCRATCH_RUNDIR" "$DATA_RUNDIR" runs/
 
 # Keep caches off $VSC_HOME (quota ~3 GB).
 export PIP_CACHE_DIR="$VSC_SCRATCH/.cache/pip"
 export UV_CACHE_DIR="$VSC_SCRATCH/.cache/uv"
-
-# Write run outputs to scratch during the job (fast I/O),
-# then stage out to $VSC_DATA at the end for persistence.
-RUN_ID="brittlestar_${PBS_JOBID}"
-SCRATCH_RUNDIR="$VSC_SCRATCH/runs/$RUN_ID"
-DATA_RUNDIR="$VSC_DATA/runs/$RUN_ID"
-mkdir -p "$PIP_CACHE_DIR" "$UV_CACHE_DIR" "$SCRATCH_RUNDIR" "$DATA_RUNDIR" runs/
+mkdir -p "$PIP_CACHE_DIR" "$UV_CACHE_DIR"
 
 module load vsc-venv
 
-# Activate mirrored environment from $VSC_DATA to avoid home quota
-HPC_CONFIG_DIR="$VSC_DATA/2026SEL3-project/env/hpc"
-set +u
+echo ">>> Synchronizing and activating environment (vsc-venv)..."
+HPC_CONFIG_DIR="$VSC_DATA/$PROJ_NAME/env/hpc"
+if [ ! -d "$HPC_CONFIG_DIR" ]; then
+    echo "ERROR: HPC_CONFIG_DIR ($HPC_CONFIG_DIR) does not exist. Run install.sh first."
+    exit 1
+fi
+
+set +eo pipefail
 source vsc-venv --activate \
     --modules "$HPC_CONFIG_DIR/modules.txt" \
     --requirements "$HPC_CONFIG_DIR/requirements.txt"
-set -u
-
-# Ensure venv versions of NumPy/Protobuf take precedence
-pip install --upgrade --no-deps numpy protobuf > /dev/null 2>&1
+set -euo pipefail
 
+echo ">>> Starting BrittleStar training..."
 export MUJOCO_GL=egl
 export WANDB_DIR="$SCRATCH_RUNDIR"
 
@@ -46,4 +55,7 @@ python src/train.py \
     --env-config-path configs/hpc/smoke_test.yaml \
     --run-dir "$SCRATCH_RUNDIR"
 
+echo ">>> Staging out results to $DATA_RUNDIR..."
 cp -r "$SCRATCH_RUNDIR/." "$DATA_RUNDIR/"
+
+echo ">>> Done"