From 78aa20868e84aafab21f70a3a1c30e2384388762 Mon Sep 17 00:00:00 2001
From: Tibo De Peuter <tibo.depeuter@ugent.be>
Date: Sat, 4 Apr 2026 21:53:32 +0200
Subject: [PATCH] feat(hpc): smart cluster modules

---
 env/hpc/modules.txt                |  1 -
 env/hpc/modules_gpu.txt            |  1 +
 scripts/export_hpc_requirements.py |  8 +++++---
 scripts/hpc/install.sh             | 11 ++---------
 4 files changed, 8 insertions(+), 13 deletions(-)
 create mode 100644 env/hpc/modules_gpu.txt

diff --git a/env/hpc/modules.txt b/env/hpc/modules.txt
index c780513..58ca6cd 100644
--- a/env/hpc/modules.txt
+++ b/env/hpc/modules.txt
@@ -1,6 +1,5 @@
 gfbf/2024a
 GCCcore/13.3.0
 Python/3.12.3-GCCcore-13.3.0
-PyTorch/2.7.1-foss-2024a-CUDA-12.6.0
 FFmpeg/7.0.2-GCCcore-13.3.0
 PyYAML/6.0.2-GCCcore-13.3.0
diff --git a/env/hpc/modules_gpu.txt b/env/hpc/modules_gpu.txt
new file mode 100644
index 0000000..aa9f2b2
--- /dev/null
+++ b/env/hpc/modules_gpu.txt
@@ -0,0 +1 @@
+PyTorch/2.7.1-foss-2024a-CUDA-12.6.0
diff --git a/scripts/export_hpc_requirements.py b/scripts/export_hpc_requirements.py
index 65fd222..fcc9850 100644
--- a/scripts/export_hpc_requirements.py
+++ b/scripts/export_hpc_requirements.py
@@ -37,7 +37,8 @@ def main() -> None:
         print(f"Error: {modules_path} not found.", file=sys.stderr)
         sys.exit(1)
 
-    # Read normalized module names from modules.txt
+    # Read normalized module names from base modules only
+    # Library modules (like PyTorch) are kept in requirements for portability
     module_names = [
         normalise(line.split()[0].split("/")[0])
         for line in modules_path.read_text().splitlines()
@@ -51,7 +52,7 @@ def main() -> None:
     deps: list[str] = data.get("project", {}).get("dependencies", [])
 
     final_deps: list[str] = []
-    print(f"Checking dependencies against {modules_path}...", file=sys.stderr)
+    print(f"Checking dependencies against HPC module list...", file=sys.stderr)
     for dep in deps:
         name = normalise(pkg_name(dep))
         # Smart check: if the package name is a substring of any loaded module name
@@ -63,7 +64,8 @@ def main() -> None:
         final_deps.append(dep)
         print(f"  [pip] {dep}", file=sys.stderr)
 
-    output_path = ROOT / "env" / "hpc" / "requirements.txt"
+    hpc_dir = ROOT / "env" / "hpc"
+    output_path = hpc_dir / "requirements.txt"
     output_path.parent.mkdir(parents=True, exist_ok=True)
     output_path.write_text("\n".join(final_deps) + "\n")
     print(f"\nWrote {len(final_deps)} requirement(s) to {output_path}", file=sys.stderr)
diff --git a/scripts/hpc/install.sh b/scripts/hpc/install.sh
index dea8083..93c7707 100644
--- a/scripts/hpc/install.sh
+++ b/scripts/hpc/install.sh
@@ -21,19 +21,12 @@ fi
 
 module load vsc-venv
 
-echo 'Synchronizing environment...'
-# Step 1: Sync (build/update) the environment
-vsc-venv \
-    --modules env/hpc/modules.txt \
-    --requirements env/hpc/requirements.txt
-
-echo 'Activating environment...'
-# Step 2: Activate the environment
+echo 'Synchronizing and activating environment...'
 source vsc-venv --activate \
     --modules env/hpc/modules.txt \
     --requirements env/hpc/requirements.txt
 
-# Step 3: Force upgrade shared system dependencies to ensure venv precedence
+# Step 2: Force upgrade shared system dependencies to ensure venv precedence
 echo 'Applying library overlays (NumPy, Protobuf)...'
 pip install --upgrade --no-deps numpy protobuf