feat(hpc): parse missing dependencies dynamically from pyproject
This commit is contained in:
parent
0e821cf251
commit
205b284b96
1 changed files with 28 additions and 5 deletions
|
|
@ -24,11 +24,34 @@ export VENV_PATH="$VSC_DATA/venvs/sel3_${VSC_INSTITUTE_CLUSTER}"
|
||||||
python -m venv --system-site-packages "$VENV_PATH"
|
python -m venv --system-site-packages "$VENV_PATH"
|
||||||
source "$VENV_PATH/bin/activate"
|
source "$VENV_PATH/bin/activate"
|
||||||
|
|
||||||
# -- Install only the missing lightweight packages ---------------------------
|
# -- Extract and install missing dependencies ----------------------------------
|
||||||
# We bypass uv here to strictly adhere to the user's storage limits requirement.
|
# Dynamically parse pyproject.toml and install only what is missing natively
|
||||||
pip install --upgrade pip
|
MISSING_DEPS=$(python -c '
|
||||||
pip install biorobot==0.4.2 evosax==0.2.0 mediapy==1.2.6 tyro>=1.0.10 \
|
import tomllib, importlib.util, re
|
||||||
cleanrl>=0.4.8 gymnasium>=1.2.3 PyOpenGL>=3.1.10 PyOpenGL-accelerate>=3.1.10
|
with open("pyproject.toml", "rb") as f:
|
||||||
|
deps = tomllib.load(f)["project"]["dependencies"]
|
||||||
|
|
||||||
|
missing = []
|
||||||
|
for dep in deps:
|
||||||
|
pkg = re.split(r"[\[=><~]", dep)[0].strip()
|
||||||
|
# Map common PyPI package names to their python import names
|
||||||
|
mapping = {"pyyaml": "yaml", "pyopengl": "OpenGL", "pyopengl-accelerate": "OpenGL_accelerate"}
|
||||||
|
import_name = mapping.get(pkg.lower(), pkg.replace("-", "_"))
|
||||||
|
|
||||||
|
# If the system module cannot find the package, add it to our pip install list
|
||||||
|
if getattr(importlib.util, "find_spec", None) is None or importlib.util.find_spec(import_name) is None:
|
||||||
|
missing.append(f"\"{dep}\"")
|
||||||
|
|
||||||
|
print(" ".join(missing))
|
||||||
|
')
|
||||||
|
|
||||||
|
if [ -n "$MISSING_DEPS" ]; then
|
||||||
|
echo "Installing missing dependencies: $MISSING_DEPS"
|
||||||
|
pip install --upgrade pip
|
||||||
|
eval "pip install $MISSING_DEPS"
|
||||||
|
else
|
||||||
|
echo "All dependencies from pyproject.toml are successfully satisfied by HPC system modules."
|
||||||
|
fi
|
||||||
|
|
||||||
# -- Register as Jupyter kernel -----------------------------------------------
|
# -- Register as Jupyter kernel -----------------------------------------------
|
||||||
python -m ipykernel install --user --name="sel3_${VSC_INSTITUTE_CLUSTER}-kernel"
|
python -m ipykernel install --user --name="sel3_${VSC_INSTITUTE_CLUSTER}-kernel"
|
||||||
|
|
|
||||||
Reference in a new issue