1
Fork 0

feat(hpc): add structured env/hpc config and requirements export script

- env/hpc/modules.txt: curated list of HPC EasyBuild modules loaded by vsc-venv
- env/hpc/requirements.txt: pip packages not covered by HPC modules (auto-generated)
- scripts/export_hpc_requirements.py: developer utility to regenerate requirements.txt
  from pyproject.toml; to be called by CI when pyproject.toml changes
This commit is contained in:
Tibo De Peuter 2026-04-04 18:10:38 +02:00
parent f2139fb419
commit 23f0b45614
3 changed files with 92 additions and 0 deletions

7
env/hpc/modules.txt vendored Normal file
View file

@ -0,0 +1,7 @@
jax/0.4.25-gfbf-2023a-CUDA-12.1.1
Flax/0.8.4-gfbf-2023a-CUDA-12.1.1
Optax/0.2.2-gfbf-2023a-CUDA-12.1.1
wandb/0.16.1-GCC-12.3.0
matplotlib/3.7.2-gfbf-2023a
PyYAML/6.0-GCCcore-12.3.0
FFmpeg/5.1.2-GCCcore-12.3.0

9
env/hpc/requirements.txt vendored Normal file
View file

@ -0,0 +1,9 @@
biorobot==0.4.2
cleanrl>=0.4.8
evosax==0.2.0
gymnasium>=1.2.3
ipykernel==7.2.0
mediapy==1.2.6
pyopengl>=3.1.10
pyopengl-accelerate>=3.1.10
tyro>=1.0.10

View file

@ -0,0 +1,76 @@
#!/usr/bin/env python3
"""Export HPC pip requirements from pyproject.toml.
This is a LOCAL DEVELOPER UTILITY run it on your own machine before pushing
code whenever pyproject.toml dependencies change. It reads the full dependency
list from pyproject.toml and subtracts packages already provided by HPC modules
(listed in env/hpc/modules.txt), then writes the remainder to
env/hpc/requirements.txt for use by vsc-venv on the cluster.
Usage:
uv run scripts/export_hpc_requirements.py
"""
from __future__ import annotations
import importlib.util
import re
import sys
from pathlib import Path
ROOT = Path(__file__).parent.parent
# Packages provided by HPC modules (PyPI name → Python import name).
# We skip any pyproject.toml dep whose import can be found after loading
# the HPC modules. As this script runs locally (without those modules), we
# maintain an explicit exclusion list keyed by normalised PyPI package name.
EXCLUDED_BY_MODULE = {
"jax",
"flax",
"optax",
"wandb",
"matplotlib",
"pyyaml", # PyYAML module
"ffmpeg", # FFmpeg is a system tool, not a Python package
}
def normalise(name: str) -> str:
"""Normalise a PyPI package name for comparison."""
return re.sub(r"[-_.]+", "-", name).lower()
def pkg_name(dep: str) -> str:
"""Extract the bare package name from a PEP 508 dependency string."""
return re.split(r"[\[=><~!;]", dep)[0].strip()
def main() -> None:
try:
import tomllib # Python ≥ 3.11
except ImportError:
import tomli as tomllib # type: ignore[no-redef]
pyproject_path = ROOT / "pyproject.toml"
with pyproject_path.open("rb") as f:
data = tomllib.load(f)
deps: list[str] = data.get("project", {}).get("dependencies", [])
missing: list[str] = []
for dep in deps:
name = normalise(pkg_name(dep))
if name in EXCLUDED_BY_MODULE:
print(f" [skip provided by HPC module] {dep}", file=sys.stderr)
continue
missing.append(dep)
print(f" [pip] {dep}", file=sys.stderr)
output_path = ROOT / "env" / "hpc" / "requirements.txt"
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text("\n".join(missing) + "\n")
print(f"\nWrote {len(missing)} requirement(s) to {output_path}", file=sys.stderr)
if __name__ == "__main__":
main()