47 lines
1.3 KiB
Bash
47 lines
1.3 KiB
Bash
#!/bin/bash
|
|
# scripts/hpc/train.pbs
|
|
#
|
|
# Usage (from project root):
|
|
# qsub scripts/hpc/train.pbs
|
|
#
|
|
# To target a specific GPU cluster (default: joltik):
|
|
# module swap cluster/accelgor && qsub scripts/hpc/train.pbs
|
|
|
|
#PBS -N brittlestar-ppo
|
|
#PBS -l nodes=1:ppn=8:gpus=1
|
|
#PBS -l walltime=24:00:00
|
|
#PBS -l mem=32gb
|
|
#PBS -o runs/brittlestar-ppo.o$PBS_JOBID
|
|
#PBS -e runs/brittlestar-ppo.e$PBS_JOBID
|
|
|
|
set -euo pipefail
|
|
|
|
cd "$PBS_O_WORKDIR"
|
|
|
|
# Keep caches off $VSC_HOME (quota ~3 GB).
|
|
export PIP_CACHE_DIR="$VSC_SCRATCH/.cache/pip"
|
|
export UV_CACHE_DIR="$VSC_SCRATCH/.cache/uv"
|
|
|
|
# Write run outputs to scratch during the job (fast I/O),
|
|
# then stage out to $VSC_DATA at the end for persistence.
|
|
RUN_ID="brittlestar_${PBS_JOBID}"
|
|
SCRATCH_RUNDIR="$VSC_SCRATCH/runs/$RUN_ID"
|
|
DATA_RUNDIR="$VSC_DATA/runs/$RUN_ID"
|
|
mkdir -p "$PIP_CACHE_DIR" "$UV_CACHE_DIR" "$SCRATCH_RUNDIR" "$DATA_RUNDIR" runs/
|
|
|
|
module load vsc-venv
|
|
set +u
|
|
source vsc-venv --activate \
|
|
--modules env/hpc/modules.txt \
|
|
--requirements env/hpc/requirements.txt
|
|
set -u
|
|
|
|
export MUJOCO_GL=egl
|
|
export WANDB_DIR="$SCRATCH_RUNDIR"
|
|
|
|
python src/train.py \
|
|
--env-config-path configs/hpc/smoke_test.yaml \
|
|
--run-dir "$SCRATCH_RUNDIR"
|
|
# ^^^ Replace with your production config, e.g. configs/production_training.yaml
|
|
|
|
cp -r "$SCRATCH_RUNDIR/." "$DATA_RUNDIR/"
|