Tibo De Peuter
34a887cd58
Match comment style from previous cleanup commit: - No section divider bars, no echo progress lines, no prose comments on obvious commands - Replace XDG_CACHE_HOME with explicit PIP_CACHE_DIR and UV_CACHE_DIR - Drop MPLCONFIGDIR (matplotlib cache is negligible in size)
44 lines
1.1 KiB
Bash
44 lines
1.1 KiB
Bash
#!/bin/bash
|
|
# scripts/hpc/train.pbs
|
|
#
|
|
# Usage (from project root):
|
|
# qsub scripts/hpc/train.pbs
|
|
#
|
|
# To target a specific GPU cluster (default: joltik):
|
|
# module swap cluster/accelgor && qsub scripts/hpc/train.pbs
|
|
|
|
#PBS -N brittlestar-ppo
|
|
#PBS -l nodes=1:ppn=8:gpus=1
|
|
#PBS -l walltime=24:00:00
|
|
#PBS -l mem=32gb
|
|
#PBS -o /dev/null
|
|
#PBS -e /dev/null
|
|
|
|
set -euo pipefail
|
|
|
|
cd "$PBS_O_WORKDIR"
|
|
|
|
# Keep caches off $VSC_HOME (quota ~3 GB).
|
|
export PIP_CACHE_DIR="$VSC_SCRATCH/.cache/pip"
|
|
export UV_CACHE_DIR="$VSC_SCRATCH/.cache/uv"
|
|
|
|
# Write run outputs to scratch during the job (fast I/O),
|
|
# then stage out to $VSC_DATA at the end for persistence.
|
|
RUN_ID="brittlestar_${PBS_JOBID}"
|
|
SCRATCH_RUNDIR="$VSC_SCRATCH/runs/$RUN_ID"
|
|
DATA_RUNDIR="$VSC_DATA/runs/$RUN_ID"
|
|
mkdir -p "$PIP_CACHE_DIR" "$UV_CACHE_DIR" "$SCRATCH_RUNDIR" "$DATA_RUNDIR"
|
|
|
|
exec 1> "$DATA_RUNDIR/job.out" 2> "$DATA_RUNDIR/job.err"
|
|
|
|
module load vsc-venv
|
|
source vsc-venv --activate --modules env/hpc/modules.txt
|
|
|
|
export MUJOCO_GL=egl
|
|
export WANDB_DIR="$SCRATCH_RUNDIR"
|
|
|
|
python src/train.py \
|
|
--config configs/production_training.yaml \
|
|
--run_dir "$SCRATCH_RUNDIR"
|
|
|
|
cp -r "$SCRATCH_RUNDIR/." "$DATA_RUNDIR/"
|