#!/bin/bash
# scripts/hpc/train.pbs
#
# Usage (from project root):
#   qsub scripts/hpc/train.pbs
#
# To target a specific GPU cluster (default: joltik):
#   module swap cluster/accelgor && qsub scripts/hpc/train.pbs

#PBS -N brittlestar-ppo
#PBS -l nodes=1:ppn=8:gpus=1
#PBS -l walltime=24:00:00
#PBS -l mem=32gb
#PBS -o /dev/null
#PBS -e /dev/null

set -euo pipefail

cd "$PBS_O_WORKDIR"

# Keep caches off $VSC_HOME (quota ~3 GB).
export PIP_CACHE_DIR="$VSC_SCRATCH/.cache/pip"
export UV_CACHE_DIR="$VSC_SCRATCH/.cache/uv"

# Write run outputs to scratch during the job (fast I/O),
# then stage out to $VSC_DATA at the end for persistence.
RUN_ID="brittlestar_${PBS_JOBID}"
SCRATCH_RUNDIR="$VSC_SCRATCH/runs/$RUN_ID"
DATA_RUNDIR="$VSC_DATA/runs/$RUN_ID"
mkdir -p "$PIP_CACHE_DIR" "$UV_CACHE_DIR" "$SCRATCH_RUNDIR" "$DATA_RUNDIR"

exec 1> "$DATA_RUNDIR/job.out" 2> "$DATA_RUNDIR/job.err"

module load vsc-venv
source vsc-venv --activate --modules env/hpc/modules.txt

export MUJOCO_GL=egl
export WANDB_DIR="$SCRATCH_RUNDIR"

python src/train.py \
    --config configs/production_training.yaml \
    --run_dir "$SCRATCH_RUNDIR"

cp -r "$SCRATCH_RUNDIR/." "$DATA_RUNDIR/"