From 65921d88a2a04e78fb3b6a1688264dd9613e6d59 Mon Sep 17 00:00:00 2001 From: mikibonacci Date: Wed, 22 Jan 2025 19:37:05 +0000 Subject: [PATCH 1/3] Adding the miniforge installer for conda - generate_bash_to_install_conda - CONDA_DEFAULT_PATH the script to install conda is generated in the `create_conda_env` function if `install_conda` is `True`. Moreover, now we can define the `path`: `str` k:v in the `conda` input dictionary. This is needed to define a common path where to install and then find conda. --- src/aiida_pythonjob/utils.py | 116 +++++++++++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 5 deletions(-) diff --git a/src/aiida_pythonjob/utils.py b/src/aiida_pythonjob/utils.py index 5cdaffd..abb7506 100644 --- a/src/aiida_pythonjob/utils.py +++ b/src/aiida_pythonjob/utils.py @@ -4,6 +4,7 @@ from aiida.common.exceptions import NotExistent from aiida.orm import Computer, InstalledCode, User, load_code, load_computer +CONDA_DEFAULT_PATH = "$HOME/miniforge3/" def get_required_imports(func: Callable) -> Dict[str, set]: """Retrieve type hints and the corresponding modules.""" @@ -102,11 +103,86 @@ def get_or_create_code( code.store() return code +def generate_bash_to_install_conda( + shell: str = "posix", + destination: str = CONDA_DEFAULT_PATH, + modules: Optional[list] = None, +): + """ + Args: + shell (str): The type of shell to initialize conda for (default is "posix"). + destination (str): The installation directory for Miniforge (default is "$HOME/miniforge3"). + modules (list): A list of system modules to load before running the script (default is None). + Returns: + str: A bash script as a string to install Miniforge and set up conda. + + Generates a bash script to install conda via miniforge on a local/remote computer. + The default channel (the only one) is automatically set to be conda-forge, avoiding then to + use Anaconda channels, restricted by the license. + We anyway perform a check to be sure that the installation will not use Anaconda channels. + If python_version is None, it uses the Python version from the local environment. + """ + + # Start of the script + script = "#!/bin/bash\n\n" + + # Load modules if provided + if modules: + script += "# Load specified system modules\n" + for module in modules: + script += f"module load {module}\n" + + script += f''' +# Check if conda is already installed +if command -v {destination}/bin/conda &> /dev/null; then + echo "Conda is already installed. Skipping installation." +else\n +''' + + # Getting minimum Miniforge installer as recommended here: https://github.com/conda-forge/miniforge?tab=readme-ov-file + script += "# Downloading Miniforge installer\n" + script += 'curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh\n' + + # Running the installer + script += "# Running the Miniforge installer\n" + script += f'bash Miniforge3-$(uname)-$(uname -m).sh -b -p {destination}\n' + + # Conda shell hook initialization for proper conda activation + script += "# Initialize Conda for this shell\n" + script += f'eval "$({destination}/bin/conda shell.{shell} hook)"\n' + + # Ensure the default Anaconda channel is not present in the conda configuration + script += ''' +# Check if 'conda config --show channels | grep default' returns anything +if conda config --show channels | grep -q "defaults"; then + echo "The default Anaconda channel is present in the conda configuration. We remove it." + conda config --remove channels defaults +else + echo "The default Anaconda channel is not present in the conda configuration. Good." +fi +''' + + # Ensure the conda-forge channel is present in the conda configuration + script += ''' +# Ensure conda-forge is there +if conda config --show channels | grep -q "conda-forge"; then + echo "The conda-forge channel is present in the conda configuration. Good." +else + echo "The conda-forge channel is not present in the conda configuration. We add it." + conda config --append channels conda-forge +fi +''' + + script += 'fi\n' + # End of the script + script += 'echo "Miniforge-based conda installation is complete."\n\n' + + return script def generate_bash_to_create_python_env( name: str, pip: Optional[List[str]] = None, - conda: Optional[Dict[str, list]] = None, + conda: Optional[Dict[str, list]] = {}, modules: Optional[List[str]] = None, python_version: Optional[str] = None, variables: Optional[Dict[str, str]] = None, @@ -115,13 +191,15 @@ def generate_bash_to_create_python_env( """ Generates a bash script for creating or updating a Python environment on a remote computer. If python_version is None, it uses the Python version from the local environment. - Conda is a dictionary that can include 'channels' and 'dependencies'. + Conda is a dictionary that can include 'channels' and 'dependencies' and 'path', where 'path' is the path to the + conda executable (not included in the path), and is needed only to activate the environment. """ import sys pip = pip or [] conda_channels = conda.get("channels", []) if conda else [] conda_dependencies = conda.get("dependencies", []) if conda else [] + conda_path = conda.get("path", CONDA_DEFAULT_PATH) # Determine the Python version from the local environment if not provided local_python_version = f"{sys.version_info.major}.{sys.version_info.minor}" desired_python_version = python_version if python_version is not None else local_python_version @@ -137,7 +215,7 @@ def generate_bash_to_create_python_env( # Conda shell hook initialization for proper conda activation script += "# Initialize Conda for this shell\n" - script += f'eval "$(conda shell.{shell} hook)"\n' + script += f'eval "$({conda_path}/bin/conda shell.{shell} hook)"\n' script += "# Setup the Python environment\n" script += "if ! conda info --envs | grep -q ^{name}$; then\n" @@ -177,13 +255,34 @@ def create_conda_env( computer: Union[str, Computer], name: str, pip: Optional[List[str]] = None, - conda: Optional[List[str]] = None, + conda: Optional[Dict[str, list]] = {}, modules: Optional[List[str]] = None, python_version: Optional[str] = None, variables: Optional[Dict[str, str]] = None, shell: str = "posix", + install_conda: bool = False, + ) -> Tuple[bool, str]: - """Test that there is no unexpected output from the connection.""" + """ + + Create a conda environment on a remote computer. + + Parameters: + - computer (Union[str, Computer]): The computer on which to create the environment. Can be a string (computer label) or a Computer object. + - name (str): The name of the conda environment to create. + - pip (Optional[List[str]]): List of pip packages to install in the environment. + - conda (Optional[List[str]]): List of conda packages to install in the environment. See the + `generate_bash_to_create_python_env` function for details. + - modules (Optional[List[str]]): List of modules to load before creating the environment. + - python_version (Optional[str]): The Python version to use for the environment. + - variables (Optional[Dict[str, str]]): Environment variables to set during the environment creation. + - shell (str): The shell type to use (default is "posix"). + - install_conda (bool): Whether to install conda if it is not already installed (default is False). + + Returns: + - Tuple[bool, str]: A tuple containing a boolean indicating success or failure, and a string message with details. + + Test that there is no unexpected output from the connection.""" # Execute a command that should not return any error, except ``NotImplementedError`` # since not all transport plugins implement remote command execution. from aiida.common.exceptions import NotExistent @@ -200,6 +299,13 @@ def create_conda_env( transport = authinfo.get_transport() script = generate_bash_to_create_python_env(name, pip, conda, modules, python_version, variables, shell) + + conda_path = conda.get("path", CONDA_DEFAULT_PATH) + + if install_conda: + install_conda_script = generate_bash_to_install_conda(shell, destination=conda_path, modules=modules) + script = install_conda_script + script + with transport: scheduler.set_transport(transport) try: From d7f856ae0c28b2209ecedc910ebadf094425254b Mon Sep 17 00:00:00 2001 From: mikibonacci Date: Wed, 22 Jan 2025 19:45:23 +0000 Subject: [PATCH 2/3] Fixing docstrings and pre-commit --- src/aiida_pythonjob/utils.py | 58 +++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/src/aiida_pythonjob/utils.py b/src/aiida_pythonjob/utils.py index abb7506..a827a3a 100644 --- a/src/aiida_pythonjob/utils.py +++ b/src/aiida_pythonjob/utils.py @@ -6,6 +6,7 @@ CONDA_DEFAULT_PATH = "$HOME/miniforge3/" + def get_required_imports(func: Callable) -> Dict[str, set]: """Retrieve type hints and the corresponding modules.""" type_hints = get_type_hints(func) @@ -103,6 +104,7 @@ def get_or_create_code( code.store() return code + def generate_bash_to_install_conda( shell: str = "posix", destination: str = CONDA_DEFAULT_PATH, @@ -111,18 +113,18 @@ def generate_bash_to_install_conda( """ Args: shell (str): The type of shell to initialize conda for (default is "posix"). - destination (str): The installation directory for Miniforge (default is "$HOME/miniforge3"). + destination (str): The installation directory for Miniforge (default is CONDA_DEFAULT_PATH). modules (list): A list of system modules to load before running the script (default is None). Returns: str: A bash script as a string to install Miniforge and set up conda. - + Generates a bash script to install conda via miniforge on a local/remote computer. - The default channel (the only one) is automatically set to be conda-forge, avoiding then to - use Anaconda channels, restricted by the license. + The default channel (the only one) is automatically set to be conda-forge, avoiding then to + use Anaconda channels, restricted by the license. We anyway perform a check to be sure that the installation will not use Anaconda channels. If python_version is None, it uses the Python version from the local environment. """ - + # Start of the script script = "#!/bin/bash\n\n" @@ -131,28 +133,29 @@ def generate_bash_to_install_conda( script += "# Load specified system modules\n" for module in modules: script += f"module load {module}\n" - - script += f''' + + script += f""" # Check if conda is already installed if command -v {destination}/bin/conda &> /dev/null; then echo "Conda is already installed. Skipping installation." else\n -''' - +""" + # Getting minimum Miniforge installer as recommended here: https://github.com/conda-forge/miniforge?tab=readme-ov-file script += "# Downloading Miniforge installer\n" - script += 'curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh\n' + script += "curl -L -O \ + https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh\n" # Running the installer script += "# Running the Miniforge installer\n" - script += f'bash Miniforge3-$(uname)-$(uname -m).sh -b -p {destination}\n' - + script += f"bash Miniforge3-$(uname)-$(uname -m).sh -b -p {destination}\n" + # Conda shell hook initialization for proper conda activation script += "# Initialize Conda for this shell\n" script += f'eval "$({destination}/bin/conda shell.{shell} hook)"\n' # Ensure the default Anaconda channel is not present in the conda configuration - script += ''' + script += """ # Check if 'conda config --show channels | grep default' returns anything if conda config --show channels | grep -q "defaults"; then echo "The default Anaconda channel is present in the conda configuration. We remove it." @@ -160,10 +163,10 @@ def generate_bash_to_install_conda( else echo "The default Anaconda channel is not present in the conda configuration. Good." fi -''' +""" # Ensure the conda-forge channel is present in the conda configuration - script += ''' + script += """ # Ensure conda-forge is there if conda config --show channels | grep -q "conda-forge"; then echo "The conda-forge channel is present in the conda configuration. Good." @@ -171,14 +174,15 @@ def generate_bash_to_install_conda( echo "The conda-forge channel is not present in the conda configuration. We add it." conda config --append channels conda-forge fi -''' - - script += 'fi\n' +""" + + script += "fi\n" # End of the script script += 'echo "Miniforge-based conda installation is complete."\n\n' - + return script + def generate_bash_to_create_python_env( name: str, pip: Optional[List[str]] = None, @@ -261,17 +265,17 @@ def create_conda_env( variables: Optional[Dict[str, str]] = None, shell: str = "posix", install_conda: bool = False, - ) -> Tuple[bool, str]: """ - + Create a conda environment on a remote computer. Parameters: - - computer (Union[str, Computer]): The computer on which to create the environment. Can be a string (computer label) or a Computer object. + - computer (Union[str, Computer]): The computer on which to create the environment. + Can be a string (computer label) or a Computer object. - name (str): The name of the conda environment to create. - pip (Optional[List[str]]): List of pip packages to install in the environment. - - conda (Optional[List[str]]): List of conda packages to install in the environment. See the + - conda (Optional[List[str]]): List of conda packages to install in the environment. See the `generate_bash_to_create_python_env` function for details. - modules (Optional[List[str]]): List of modules to load before creating the environment. - python_version (Optional[str]): The Python version to use for the environment. @@ -281,7 +285,7 @@ def create_conda_env( Returns: - Tuple[bool, str]: A tuple containing a boolean indicating success or failure, and a string message with details. - + Test that there is no unexpected output from the connection.""" # Execute a command that should not return any error, except ``NotImplementedError`` # since not all transport plugins implement remote command execution. @@ -299,13 +303,13 @@ def create_conda_env( transport = authinfo.get_transport() script = generate_bash_to_create_python_env(name, pip, conda, modules, python_version, variables, shell) - + conda_path = conda.get("path", CONDA_DEFAULT_PATH) - + if install_conda: install_conda_script = generate_bash_to_install_conda(shell, destination=conda_path, modules=modules) script = install_conda_script + script - + with transport: scheduler.set_transport(transport) try: From 17562dc37305cda8e98a1eb64cbefb1b621edcd4 Mon Sep 17 00:00:00 2001 From: mikibonacci Date: Tue, 28 Jan 2025 15:38:31 +0000 Subject: [PATCH 3/3] Adding doc --- docs/gallery/autogen/how_to.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/gallery/autogen/how_to.py b/docs/gallery/autogen/how_to.py index 67885eb..d95ed50 100644 --- a/docs/gallery/autogen/how_to.py +++ b/docs/gallery/autogen/how_to.py @@ -49,7 +49,7 @@ # from aiida_pythonjob.utils import create_conda_env # # create a conda environment on remote computer # create_conda_env( -# "merlin6", # Remote computer +# "merlin6", # Remote computer, already stored in the AiiDA database # "test_pythonjob", # Name of the conda environment # modules=["anaconda"], # Modules to load (e.g., Anaconda) # pip=["numpy", "matplotlib"], # Python packages to install via pip @@ -60,6 +60,33 @@ # ) # # +# If you don't have conda installed on the remote computer, or you can use Anaconda for license reasons, +# you can run the `create_conda_env` function +# with the `install_conda` parameter set to `True`. This will install conda on the remote computer via the +# miniforge installer. You can find more information about Miniforge and download the installer from the +# official [Miniforge GitHub repository](https://github.com/conda-forge/miniforge). The `conda` dictionary +# can be used to specify the desired conda environment path, adding a new key "path": "/path/to/conda"`., +# e.g.: +# +# .. code-block:: python +# +# from aiida_pythonjob.utils import create_conda_env +# # create a conda environment on remote computer +# create_conda_env( +# "merlin6", # Remote computer, already stored in the AiiDA database +# "test_pythonjob", # Name of the conda environment +# modules=["anaconda"], # Modules to load (e.g., Anaconda) +# pip=["numpy", "matplotlib"], # Python packages to install via pip +# conda={ # Conda-specific settings +# "channels": ["conda-forge"], # Channels to use +# "dependencies": ["qe"] # Conda packages to install +# "path": "$HOME/miniforge3/" # path to the (new) conda installation +# }, +# ) +# +# +# By default, the conda path will be set to `$HOME/miniforge3/`, if the `path` key is not provided. +# ###################################################################### # Default outputs