49 lines
1.3 KiB
Bash
49 lines
1.3 KiB
Bash
#!/bin/bash
|
|
# scripts/hpc/train.pbs
|
|
#
|
|
# Usage (from project root):
|
|
# qsub scripts/hpc/train.pbs
|
|
|
|
#PBS -N brittlestar-ppo
|
|
#PBS -l nodes=1:ppn=8:gpus=1
|
|
#PBS -l walltime=24:00:00
|
|
#PBS -l mem=32gb
|
|
#PBS -o runs/brittlestar-ppo.o$PBS_JOBID
|
|
#PBS -e runs/brittlestar-ppo.e$PBS_JOBID
|
|
|
|
set -euo pipefail
|
|
|
|
cd "$PBS_O_WORKDIR"
|
|
|
|
# Keep caches off $VSC_HOME (quota ~3 GB).
|
|
export PIP_CACHE_DIR="$VSC_SCRATCH/.cache/pip"
|
|
export UV_CACHE_DIR="$VSC_SCRATCH/.cache/uv"
|
|
|
|
# Write run outputs to scratch during the job (fast I/O),
|
|
# then stage out to $VSC_DATA at the end for persistence.
|
|
RUN_ID="brittlestar_${PBS_JOBID}"
|
|
SCRATCH_RUNDIR="$VSC_SCRATCH/runs/$RUN_ID"
|
|
DATA_RUNDIR="$VSC_DATA/runs/$RUN_ID"
|
|
mkdir -p "$PIP_CACHE_DIR" "$UV_CACHE_DIR" "$SCRATCH_RUNDIR" "$DATA_RUNDIR" runs/
|
|
|
|
module load vsc-venv
|
|
|
|
# Activate mirrored environment from $VSC_DATA to avoid home quota
|
|
HPC_CONFIG_DIR="$VSC_DATA/2026SEL3-project/env/hpc"
|
|
set +u
|
|
source vsc-venv --activate \
|
|
--modules "$HPC_CONFIG_DIR/modules.txt" \
|
|
--requirements "$HPC_CONFIG_DIR/requirements.txt"
|
|
set -u
|
|
|
|
# Ensure venv versions of NumPy/Protobuf take precedence
|
|
pip install --upgrade --no-deps numpy protobuf > /dev/null 2>&1
|
|
|
|
export MUJOCO_GL=egl
|
|
export WANDB_DIR="$SCRATCH_RUNDIR"
|
|
|
|
python src/train.py \
|
|
--env-config-path configs/hpc/smoke_test.yaml \
|
|
--run-dir "$SCRATCH_RUNDIR"
|
|
|
|
cp -r "$SCRATCH_RUNDIR/." "$DATA_RUNDIR/"
|