From 82d1adbcbafd570b302fe05a73b54de08a4d2d0b Mon Sep 17 00:00:00 2001 From: Timothee Mathieu Date: Wed, 28 Aug 2024 14:41:18 +0200 Subject: [PATCH 01/14] proof of concept, it works --- guix_env/cli.py | 95 +++++++++++++++++------- guix_env/template_scripts/activate.sh | 21 ++---- guix_env/template_scripts/channels.scm | 7 ++ guix_env/template_scripts/create_env.sh | 6 ++ guix_env/template_scripts/pyproject.toml | 11 +++ guix_env/template_scripts/run_script.sh | 9 ++- guix_env/template_scripts/zshrc | 10 ++- 7 files changed, 112 insertions(+), 47 deletions(-) create mode 100644 guix_env/template_scripts/channels.scm create mode 100644 guix_env/template_scripts/create_env.sh create mode 100644 guix_env/template_scripts/pyproject.toml diff --git a/guix_env/cli.py b/guix_env/cli.py index 7ea09bb..5717219 100644 --- a/guix_env/cli.py +++ b/guix_env/cli.py @@ -16,11 +16,13 @@ default_guix_packages = [ "python", "python-toolchain", + "poetry-next", # this comes from perso channel while waiting for guix to have a newer version of poetry "bash", "glibc-locales", "nss-certs", "coreutils", "diffutils", + "curl", "git", "make", "zlib", @@ -38,8 +40,6 @@ "zsh" ] -default_python = ["numpy", "matplotlib", "PyQt5", "scipy"] - @click.group() @click.pass_context def guix_env(ctx): @@ -52,14 +52,17 @@ def guix_env(ctx): @click.argument('name',required = True, type=str) @click.option('--channel-file',required = False, type=str, help="Path to a channel file to be used in the guix install") @click.option('--requirements-file',required = False, type=str, help="Path to a requirements.txt file to be used in the python install") -@click.option('--manifest-file',required = False, type=str, help="Path to a manifest file to be used in the guix install") +@click.option('--pyproject-file',required = False, type=str, help="Path to a pyproject.toml file to be used in the python install (override requirement file if both are given).") +@click.option('--poetry-lock-file',required = False, type=str, help="Path to a poetry.lock file to be used in the python install") +@click.option('--manifest-file',required = False, type=str, help="Path to a manifest file to be used in the guix install. Will replace the default manifest.") @click.option('--guix-args',required = False, type=str, default="-CFNW", help="arguments to be passed to guix") @click.pass_context -def create(ctx, name, channel_file, requirements_file, manifest_file, guix_args): +def create(ctx, name, channel_file, requirements_file, pyproject_file, poetry_lock_file, manifest_file, guix_args): """ Create an environment with name `name`. A channel file can be specified, otherwise a channel file will be automatically created. """ + assert not os.path.isdir(os.path.join(main_dir, name)), "Environment already exist" os.system('mkdir -p '+os.path.join(main_dir, name, "bin")) @@ -67,14 +70,14 @@ def create(ctx, name, channel_file, requirements_file, manifest_file, guix_args) # TBC - if channel_file is None: - channels = subprocess.run(["guix", "describe", "-f", "channels"], capture_output=True).stdout.decode() - else: - channels = subprocess.run(["cat", channel_file], capture_output=True).stdout.decode() + channels = _make_channel_file(channel_file) template = environment.get_template("activate.sh") script = template.render(name = name, guix_args = guix_args) + + + with open(os.path.join(main_dir, name, "bin", ".zshrc"), "w") as myfile: myfile.write(zshrc) @@ -83,7 +86,6 @@ def create(ctx, name, channel_file, requirements_file, manifest_file, guix_args) os.system('chmod +x '+os.path.join(main_dir, name, "bin", "activate.sh")) with open(os.path.join(main_dir, name, "bin", "run.sh"), "w") as myfile: - template_run = environment.get_template("run_script.sh") run_script = template_run.render(name = name, guix_args = guix_args) myfile.write(run_script) @@ -103,21 +105,37 @@ def create(ctx, name, channel_file, requirements_file, manifest_file, guix_args) ) else: os.system("cp "+manifest_file+" "+os.path.join(main_dir, name, "manifest.scm")) - - print("Creation of virtual environment") - os.system("guix time-machine --channels=${HOME}/.guix_env/"+name+"/channels.scm -- shell python -- python3 -m venv ~/.guix_env/guix_env_venv/"+name+"_venv") - run_file = os.path.join(main_dir, name, "bin", "run.sh") - if requirements_file is None: - print("Installing default python libs") - with open(os.path.join(main_dir, name, "requirements.txt"), "w") as myfile: - myfile.write("\n".join(default_python)) - requirements_file = os.path.join(main_dir, name, "requirements.txt") - - os.system(run_file+" python -m pip install -r "+requirements_file) - os.system(run_file+" python -m pip freeze > "+os.path.join(main_dir, name, "requirements.txt")) + guix_python_cmd = f"guix time-machine --channels=$HOME/.guix_env/{name}/channels.scm -- shell python -- python3 --version | cut -d ' ' -f 2" + + python_version = subprocess.check_output(guix_python_cmd, shell=True).decode().strip() + + if pyproject_file is None: + author = subprocess.run(["whoami"], capture_output=True).stdout.decode() + pyproject = environment.get_template("pyproject.toml").render(name = name, python_version = python_version) + else: + with open(pyproject_file, "r") as myfile: + pyproject = myfile.read() + with open(os.path.join(main_dir, name, "pyproject.toml"), "w") as myfile: + myfile.write(pyproject) + + if poetry_lock_file is not None: + os.system(f"cp {poetry_lock_file} {os.path.join(main_dir, name)}") + + + run_file = os.path.join(main_dir, name, "bin", "run.sh") + + os.system(run_file + f" poetry install --directory={os.path.join(main_dir, name)}") + if requirements_file is not None: + os.system(run_file +f" cat {requirements_file} | xargs poetry add --directory={os.path.join(main_dir, name)}") + + # make completions + os.system(run_file + " poetry completions zsh > "+os.path.join(main_dir, name,"_poetry")) + + + @guix_env.command() @click.argument('name',required = True, type=str) @click.pass_context @@ -125,11 +143,12 @@ def update(ctx, name): """ Update the channel file (and as a consequence, it will update the packages managed by guix at next shell/run). """ - channels = subprocess.run(["guix", "describe", "-f", "channels"], capture_output=True).stdout.decode() + channels = _make_channel_file(os.path.join(main_dir, name, "channels.scm")) with open(os.path.join(main_dir, name, "channels.scm"), "w") as myfile: myfile.write(channels) + @guix_env.command() @click.argument('name',required = True, type=str) @click.pass_context @@ -137,17 +156,20 @@ def rm(ctx, name): """ Remove a guix-env environment. """ - shutil.rmtree(os.path.join(main_dir, name)) - shutil.rmtree(os.path.join(main_dir, "guix_env_venv", name+"_venv")) + if os.path.isdir(os.path.join(main_dir, name)): + shutil.rmtree(os.path.join(main_dir, name)) + if os.path.isdir(os.path.join(main_dir, "guix_env_venv", name+"_venv")): + shutil.rmtree(os.path.join(main_dir, "guix_env_venv", name+"_venv")) @guix_env.command() @click.argument('name',required = True, type=str) @click.argument('pkg',required = True, type=str) @click.pass_context -def add(ctx, name, pkg): +def add_guix(ctx, name, pkg): """ - Add the package `pkg` to the environment named `name`. + Add the guix package `pkg` to the environment named `name`. + Warning: if you add a package from inside an environment, the package will not be available until you reconstruct the environment. """ with open(os.path.join(main_dir, name, "manifest.scm"), "r") as myfile: packages = myfile.read().split("(")[2].split(")")[0] @@ -161,6 +183,17 @@ def add(ctx, name, pkg): "(specifications->manifest '(\n\"" + '"\n "'.join(packages) + '"\n))' ) +@guix_env.command() +@click.argument('name',required = True, type=str) +@click.argument('pkg',required = True, type=str) +@click.pass_context +def add_python(ctx, name, pkg): + """ + Add the guix package `pkg` to the environment named `name`. + """ + run_file = os.path.join(main_dir, name, "bin", "run.sh") + os.system(run_file + f" poetry add {pkg} --directory={os.path.join(main_dir, name)}") + @guix_env.command() @click.pass_context def list(ctx): @@ -243,3 +276,13 @@ def _is_in_guix(pkg): if pkg == name.strip(): res = True return res + +def _make_channel_file(channel_file=None): + + if channel_file is None: + system_channels = subprocess.run(["guix", "describe", "-f", "channels"], capture_output=True).stdout.decode() + else: + system_channels = subprocess.run(["cat", channel_file], capture_output=True).stdout.decode() + + channels = environment.get_template("channels.scm").render(system_channels = system_channels) + return channels diff --git a/guix_env/template_scripts/activate.sh b/guix_env/template_scripts/activate.sh index 3da109c..f5fa019 100644 --- a/guix_env/template_scripts/activate.sh +++ b/guix_env/template_scripts/activate.sh @@ -4,22 +4,11 @@ export LD_LIBRARY_PATH=/lib echo 'Entering guix environment' -rm ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 -ln -s $(which python3) ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 -source ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/activate +[ -f ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 ] && rm ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 && ln -s $(which python3) ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 -pip freeze > /tmp/guix_env_{{ name }}_requirements.txt -if cmp --silent -- "/tmp/guix_env_{{ name }}_requirements.txt" "${HOME}/.guix_env/{{ name }}/requirements.txt" -then - echo -else - read -p "Requirement and installed packages are different. Reinstall from requirements.txt ? (y/n)" -n 1 -r - echo # (optional) move to a new line - if [[ $REPLY =~ ^[Yy]$ ]] - then - pip install -r ${HOME}/.guix_env/{{ name }}/requirements.txt - fi -fi +export SHELL=$(realpath $(which zsh)) +export ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin + +poetry shell --directory=${HOME}/.guix_env/{{ name }} -ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin zsh diff --git a/guix_env/template_scripts/channels.scm b/guix_env/template_scripts/channels.scm new file mode 100644 index 0000000..94ed5ec --- /dev/null +++ b/guix_env/template_scripts/channels.scm @@ -0,0 +1,7 @@ +(cons* (channel + (name 't-guix) + (url "https://github.com/TimotheeMathieu/t-guix") + (branch "main") + (commit "9af6b8eecb6110d93bdf5f263ef499d065af9500") + ) + {{ system_channels }}) diff --git a/guix_env/template_scripts/create_env.sh b/guix_env/template_scripts/create_env.sh new file mode 100644 index 0000000..4bb04aa --- /dev/null +++ b/guix_env/template_scripts/create_env.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env -S guix time-machine --channels=${HOME}/.guix_env/{{ name }}/channels.scm -- shell {{ guix_args }} --preserve='(^DISPLAY$|^XAUTHORITY$|^TERM$)' --share=${HOME} --expose=/dev/dri --expose=/sys -m ${HOME}/.guix_env/{{ name }}/manifest.scm -- bash + + +echo "Installing poetry" +curl -sSL https://install.python-poetry.org | python3 - +poetry install --no-root --directory={{ directory }} diff --git a/guix_env/template_scripts/pyproject.toml b/guix_env/template_scripts/pyproject.toml new file mode 100644 index 0000000..bb88713 --- /dev/null +++ b/guix_env/template_scripts/pyproject.toml @@ -0,0 +1,11 @@ +[tool.poetry] +name = "{{ name }}" +package-mode = false +description = "Dependency file for guix-env environment {{ name }}" + +[tool.poetry.dependencies] +python = "{{ python_version }}" +numpy = "*" +scipy = "*" +matplotlib = "*" +PyQt6 = "*" \ No newline at end of file diff --git a/guix_env/template_scripts/run_script.sh b/guix_env/template_scripts/run_script.sh index 5e67ec0..a8f5fdd 100644 --- a/guix_env/template_scripts/run_script.sh +++ b/guix_env/template_scripts/run_script.sh @@ -3,8 +3,9 @@ # Link the lib file for FHS library handling export LD_LIBRARY_PATH=/lib -rm ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 -ln -s $(which python3) ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 -source ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/activate +[ -f ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 ] && rm ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 && ln -s $(which python3) ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 -$@ +export SHELL=$(realpath $(which zsh)) +export ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin + +cd ${HOME}/.guix_env/{{ name }} && poetry run $@ diff --git a/guix_env/template_scripts/zshrc b/guix_env/template_scripts/zshrc index 5bbaed3..2d0e3bb 100644 --- a/guix_env/template_scripts/zshrc +++ b/guix_env/template_scripts/zshrc @@ -1,6 +1,8 @@ # -*- mode: shell-script -*- -autoload -U compinit promptinit +# completions +fpath+=~/.guix_env/{{ name }} +autoload -Uz compinit promptinit compinit promptinit @@ -41,5 +43,11 @@ pip() fi } +export POETRY_VIRTUALENVS_IN_PROJECT=true +export GUIX_ENV_NAME={{ name }} +export SHELL=$(realpath $(which zsh)) + PROMPT="%{$fg[green]%}%n@{{ name }}-guix-env %T %~ %{$reset_color%}%# >" + + From 5b0fb00ec86dc0d703d4d0e96d218ac66ba5219e Mon Sep 17 00:00:00 2001 From: Timothee Mathieu Date: Wed, 28 Aug 2024 15:38:13 +0200 Subject: [PATCH 02/14] add zhistory env var --- guix_env/template_scripts/zshrc | 1 + 1 file changed, 1 insertion(+) diff --git a/guix_env/template_scripts/zshrc b/guix_env/template_scripts/zshrc index e485182..b85da7a 100644 --- a/guix_env/template_scripts/zshrc +++ b/guix_env/template_scripts/zshrc @@ -47,3 +47,4 @@ pip() PROMPT="%{$fg[green]%}%n@{{ name }}-guix-env %T %~ %{$reset_color%}%# >" +HISTFILE="$HOME/.guix_env/{{ name }}/.zhistory" From 1f33a35749faac1de8707ab4e562419aa927fb67 Mon Sep 17 00:00:00 2001 From: Timothee Mathieu Date: Wed, 28 Aug 2024 16:07:31 +0200 Subject: [PATCH 03/14] pre_env and poetry working --- guix_env/cli.py | 4 ++++ guix_env/template_scripts/activate.sh | 9 ++------- guix_env/template_scripts/pre_env | 9 +++++++++ guix_env/template_scripts/run_script.sh | 6 ++---- guix_env/template_scripts/zshrc | 7 +------ 5 files changed, 18 insertions(+), 17 deletions(-) create mode 100644 guix_env/template_scripts/pre_env diff --git a/guix_env/cli.py b/guix_env/cli.py index 5717219..b3f1ba2 100644 --- a/guix_env/cli.py +++ b/guix_env/cli.py @@ -123,6 +123,10 @@ def create(ctx, name, channel_file, requirements_file, pyproject_file, poetry_lo if poetry_lock_file is not None: os.system(f"cp {poetry_lock_file} {os.path.join(main_dir, name)}") + with open(os.path.join(main_dir, name, "pre_env"), "w") as myfile: + pre_env = environment.get_template("pre_env").render(name=name) + myfile.write(pre_env) + os.system("chmod +x "+os.path.join(main_dir, name, "pre_env")) run_file = os.path.join(main_dir, name, "bin", "run.sh") diff --git a/guix_env/template_scripts/activate.sh b/guix_env/template_scripts/activate.sh index f5fa019..83a5055 100644 --- a/guix_env/template_scripts/activate.sh +++ b/guix_env/template_scripts/activate.sh @@ -1,14 +1,9 @@ #!/usr/bin/env -S guix time-machine --channels=${HOME}/.guix_env/{{ name }}/channels.scm -- shell {{ guix_args }} --preserve='(^DISPLAY$|^XAUTHORITY$|^TERM$)' --share=${HOME} --expose=/dev/dri --expose=/sys -m ${HOME}/.guix_env/{{ name }}/manifest.scm -- bash # Link the lib file for FHS library handling -export LD_LIBRARY_PATH=/lib + echo 'Entering guix environment' [ -f ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 ] && rm ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 && ln -s $(which python3) ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 -export SHELL=$(realpath $(which zsh)) -export ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin - -poetry shell --directory=${HOME}/.guix_env/{{ name }} - - +${HOME}/.guix_env/{{ name }}/pre_env zsh diff --git a/guix_env/template_scripts/pre_env b/guix_env/template_scripts/pre_env new file mode 100644 index 0000000..ce624e3 --- /dev/null +++ b/guix_env/template_scripts/pre_env @@ -0,0 +1,9 @@ +#!/bin/sh + +export POETRY_VIRTUALENVS_IN_PROJECT=true +export GUIX_ENV_NAME={{ name }} +export SHELL=$(realpath $(which zsh)) +export LD_LIBRARY_PATH=/lib +export ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin + +exec "$@" \ No newline at end of file diff --git a/guix_env/template_scripts/run_script.sh b/guix_env/template_scripts/run_script.sh index a8f5fdd..6b02d82 100644 --- a/guix_env/template_scripts/run_script.sh +++ b/guix_env/template_scripts/run_script.sh @@ -1,11 +1,9 @@ #!/usr/bin/env -S guix time-machine --channels=${HOME}/.guix_env/{{ name }}/channels.scm -- shell {{ guix_args }} --preserve='(^DISPLAY$|^XAUTHORITY$|^TERM$)' --share=${HOME} --expose=/dev/dri --expose=/sys -m ${HOME}/.guix_env/{{ name }}/manifest.scm -- bash # Link the lib file for FHS library handling -export LD_LIBRARY_PATH=/lib [ -f ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 ] && rm ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 && ln -s $(which python3) ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 -export SHELL=$(realpath $(which zsh)) -export ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin -cd ${HOME}/.guix_env/{{ name }} && poetry run $@ + +cd ${HOME}/.guix_env/{{ name }} && ${HOME}/.guix_env/{{ name }}/pre_env poetry run $@ diff --git a/guix_env/template_scripts/zshrc b/guix_env/template_scripts/zshrc index 2d0e3bb..ce85075 100644 --- a/guix_env/template_scripts/zshrc +++ b/guix_env/template_scripts/zshrc @@ -31,7 +31,6 @@ bindkey "^A" beginning-of-line autoload -U colors && colors alias ls='ls --color' - REQFILE={{ reqfile }} pip() @@ -43,11 +42,7 @@ pip() fi } -export POETRY_VIRTUALENVS_IN_PROJECT=true -export GUIX_ENV_NAME={{ name }} -export SHELL=$(realpath $(which zsh)) - PROMPT="%{$fg[green]%}%n@{{ name }}-guix-env %T %~ %{$reset_color%}%# >" - +. $HOME/.guix_env/{{ name }}/.venv/bin/activate From 321722811255d5ed5bf9da9751b518e4c63f1cca Mon Sep 17 00:00:00 2001 From: Timothee Mathieu Date: Wed, 28 Aug 2024 17:01:58 +0200 Subject: [PATCH 04/14] move poetry cache to guix-env folder, fix requirements.txt handling --- guix_env/cli.py | 4 +++- guix_env/template_scripts/pre_env | 3 ++- guix_env/template_scripts/pyproject.toml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/guix_env/cli.py b/guix_env/cli.py index 7120798..8bc6fae 100644 --- a/guix_env/cli.py +++ b/guix_env/cli.py @@ -38,6 +38,7 @@ "dbus", "xcb-util-cursor", "ncurses", + "nano", "zsh" ] @@ -134,11 +135,12 @@ def create(ctx, name, channel_file, requirements_file, pyproject_file, poetry_lo os.system(run_file + f" poetry install --directory={os.path.join(main_dir, name)}") if requirements_file is not None: + requirements_file = os.path.abspath(requirements_file) os.system(run_file +f" cat {requirements_file} | xargs poetry add --directory={os.path.join(main_dir, name)}") # make completions os.system(run_file + " poetry completions zsh > "+os.path.join(main_dir, name,"_poetry")) - + os.system(run_file + f" poetry config virtualenvs.prompt ' ' --local --directory={os.path.join(main_dir, name)}") @guix_env.command() diff --git a/guix_env/template_scripts/pre_env b/guix_env/template_scripts/pre_env index ce624e3..cb4e24a 100644 --- a/guix_env/template_scripts/pre_env +++ b/guix_env/template_scripts/pre_env @@ -1,9 +1,10 @@ #!/bin/sh +export POETRY_CACHE_DIR=${HOME}/.guix_env/poetry_cache export POETRY_VIRTUALENVS_IN_PROJECT=true export GUIX_ENV_NAME={{ name }} export SHELL=$(realpath $(which zsh)) export LD_LIBRARY_PATH=/lib export ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin -exec "$@" \ No newline at end of file +exec "$@" diff --git a/guix_env/template_scripts/pyproject.toml b/guix_env/template_scripts/pyproject.toml index bb88713..502f333 100644 --- a/guix_env/template_scripts/pyproject.toml +++ b/guix_env/template_scripts/pyproject.toml @@ -1,5 +1,5 @@ [tool.poetry] -name = "{{ name }}" +name = "guix-env-{{ name }}" package-mode = false description = "Dependency file for guix-env environment {{ name }}" From 44f370068f57bbfc36371802f7126c8e6ac99e83 Mon Sep 17 00:00:00 2001 From: Timothee Mathieu Date: Thu, 29 Aug 2024 13:35:53 +0200 Subject: [PATCH 05/14] better shell --- README.md | 15 +++----- guix_env/cli.py | 51 ++++++++++--------------- guix_env/template_scripts/activate.sh | 9 ----- guix_env/template_scripts/create_env.sh | 6 --- guix_env/template_scripts/pre_env | 3 +- guix_env/template_scripts/run_script.sh | 9 ----- guix_env/template_scripts/use_env.sh | 9 +++++ 7 files changed, 38 insertions(+), 64 deletions(-) delete mode 100644 guix_env/template_scripts/activate.sh delete mode 100644 guix_env/template_scripts/create_env.sh delete mode 100644 guix_env/template_scripts/run_script.sh create mode 100644 guix_env/template_scripts/use_env.sh diff --git a/README.md b/README.md index 4539280..41b7843 100644 --- a/README.md +++ b/README.md @@ -21,18 +21,15 @@ guix-env shell my_env_name The first run may be a bit slow because of guix downloading a bunch of packages but the second run should be faster as guix cache the packages it uses in `/gnu/store` (remark: don't forget to use `guix gc` to clear the store periodically). -Then, you are good to go and do anything you wish in your environment. You are in a python virtual environment and you can install new python packages with pip. To add new guix package, use `guix-env add my_env_name my_package_name` from outside the environment. +Then, you are good to go and do anything you wish in your environment. You are in a python virtual environment and you can install new python packages with pip. To add new guix package, use `guix-env add-guix my_env_name my_package_name` from outside the environment. + +TODO: explain poetry in guix-env -One of the qualities of guix-env is its *reproducibility*, you can use the three files `manifest.scm`, `channels.scm` and `requirements.txt` that are in `~/.guix-env/my_env_name` to reproduce the environment using the following command: -``` -guix-env create my_env_name --channel-file channels.scm --manifest-file manifest.scm --requirements-file requirements.txt -``` -Remark that it is not perfect reproductibility because the requirements.txt file is created using pip whether it would be better to use a lock file generated by `pip-tools` or `poetry`, or even better to use guix as python package manager. For now this is not implemented. ## TODO -- Better documentation -- better requirements.txt -- switch to poetry or pip-tools to handle locks? -- Tests +- Better documentation -- include explanations of how it works: poetry, what do we share, how to tinker with it, what changes are made... +- Make tests - Handle GPU ? - Feature: rollback, similar to what can be done with guix-home. +- Feature: use tmux inside the env and share tmp to make a real daemon. https://stackoverflow.com/questions/16398850/create-new-tmux-session-from-inside-a-tmux-session diff --git a/guix_env/cli.py b/guix_env/cli.py index 8bc6fae..b7c1943 100644 --- a/guix_env/cli.py +++ b/guix_env/cli.py @@ -39,6 +39,7 @@ "xcb-util-cursor", "ncurses", "nano", + "tmux", "zsh" ] @@ -74,24 +75,15 @@ def create(ctx, name, channel_file, requirements_file, pyproject_file, poetry_lo channels = _make_channel_file(channel_file) - template = environment.get_template("activate.sh") - script = template.render(name = name, guix_args = guix_args) - - + env_script = environment.get_template("use_env.sh").render(name=name, guix_args = guix_args) with open(os.path.join(main_dir, name, "bin", ".zshrc"), "w") as myfile: myfile.write(zshrc) - - with open(os.path.join(main_dir, name, "bin", "activate.sh"), "w") as myfile: - myfile.write(script) - os.system('chmod +x '+os.path.join(main_dir, name, "bin", "activate.sh")) + with open(os.path.join(main_dir, name, "bin", "use_env.sh"), "w") as myfile: + myfile.write(env_script) + os.system('chmod +x '+os.path.join(main_dir, name, "bin", "use_env.sh")) - with open(os.path.join(main_dir, name, "bin", "run.sh"), "w") as myfile: - template_run = environment.get_template("run_script.sh") - run_script = template_run.render(name = name, guix_args = guix_args) - myfile.write(run_script) - os.system('chmod +x '+os.path.join(main_dir, name, "bin", "run.sh")) if channel_file is None: with open(os.path.join(main_dir, name, "channels.scm"), "w") as myfile: @@ -130,17 +122,16 @@ def create(ctx, name, channel_file, requirements_file, pyproject_file, poetry_lo myfile.write(pre_env) os.system("chmod +x "+os.path.join(main_dir, name, "pre_env")) - run_file = os.path.join(main_dir, name, "bin", "run.sh") - - os.system(run_file + f" poetry install --directory={os.path.join(main_dir, name)}") + env_file = os.path.join(main_dir, name, "bin", "use_env.sh") + os.system(f"{env_file} poetry install --no-root --directory={os.path.join(main_dir, name)}") if requirements_file is not None: requirements_file = os.path.abspath(requirements_file) - os.system(run_file +f" cat {requirements_file} | xargs poetry add --directory={os.path.join(main_dir, name)}") + os.system(f" {env_file} cat {requirements_file} | xargs poetry add --directory={os.path.join(main_dir, name)}") # make completions - os.system(run_file + " poetry completions zsh > "+os.path.join(main_dir, name,"_poetry")) - os.system(run_file + f" poetry config virtualenvs.prompt ' ' --local --directory={os.path.join(main_dir, name)}") + os.system(f"{env_file} poetry completions zsh > "+os.path.join(main_dir, name,"_poetry")) + os.system(f"{env_file} poetry config virtualenvs.prompt ' ' --local --directory={os.path.join(main_dir, name)}") @guix_env.command() @@ -153,6 +144,7 @@ def update(ctx, name): channels = _make_channel_file(os.path.join(main_dir, name, "channels.scm")) with open(os.path.join(main_dir, name, "channels.scm"), "w") as myfile: myfile.write(channels) + # TODO: add poetry update and a capacity to roll-back @@ -196,10 +188,10 @@ def add_guix(ctx, name, pkg): @click.pass_context def add_python(ctx, name, pkg): """ - Add the guix package `pkg` to the environment named `name`. + Add the python package `pkg` to the environment named `name`. """ - run_file = os.path.join(main_dir, name, "bin", "run.sh") - os.system(run_file + f" poetry add {pkg} --directory={os.path.join(main_dir, name)}") + env_file = os.path.join(main_dir, name, "bin", "use_env.sh") + os.system(f"{env_file} poetry add {pkg} --directory={os.path.join(main_dir, name)}") @guix_env.command() @click.pass_context @@ -217,8 +209,8 @@ def info(ctx, name): Get informations on environment with name `name`. """ click.echo("Environment located in "+os.path.join(main_dir, name)) - run_file = os.path.join(main_dir, name, "bin", "run.sh") - os.system(run_file+" guix describe") + env_file = os.path.join(main_dir, name, "bin", "use_env.sh") + os.system(env_file+" guix describe") with open(os.path.join(main_dir, name, "manifest.scm"), "r") as myfile: packages = myfile.read().split("(")[2].split(")")[0] @@ -230,7 +222,7 @@ def info(ctx, name): click.echo("\n".join(packages)) click.echo("-"*10) click.echo("Installed python packages") - os.system(run_file+" pip3 freeze") + os.system(f"{env_file} poetry run pip3 freeze --directory={os.path.join(main_dir, name)}") @guix_env.command() @click.argument('name',required = True, type=str) @@ -241,8 +233,7 @@ def shell(ctx, name, tmux, cwd): """ Open a shell in the environment with name `name`. """ - - activation_file = os.path.join(main_dir, name, "bin", "activate.sh") + env_file = os.path.join(main_dir, name, "bin", "use_env.sh") if tmux: child = subprocess.run(["tmux","has-session", "-t", "guix_env_"+name],capture_output=True,text=True) @@ -257,7 +248,7 @@ def shell(ctx, name, tmux, cwd): print("done cwd") os.system("tmux attach -t guix_env_"+name) else: - os.system(activation_file) + os.system(f"{env_file} zsh ") @guix_env.command() @click.argument('name',required = True, type=str) @@ -270,8 +261,8 @@ def run(ctx, name, cmd): Example of usage is guix-env run my_env "ls $HOME/" """ - run_file = os.path.join(main_dir, name, "bin", "run.sh") - os.system(run_file+" "+cmd) + env_file = os.path.join(main_dir, name, "bin", "use_env.sh") + os.system(f"{env_file} poetry run --directory={os.path.join(main_dir, name)} {cmd}" ) def _is_in_guix(pkg): diff --git a/guix_env/template_scripts/activate.sh b/guix_env/template_scripts/activate.sh deleted file mode 100644 index 83a5055..0000000 --- a/guix_env/template_scripts/activate.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env -S guix time-machine --channels=${HOME}/.guix_env/{{ name }}/channels.scm -- shell {{ guix_args }} --preserve='(^DISPLAY$|^XAUTHORITY$|^TERM$)' --share=${HOME} --expose=/dev/dri --expose=/sys -m ${HOME}/.guix_env/{{ name }}/manifest.scm -- bash - -# Link the lib file for FHS library handling - - -echo 'Entering guix environment' -[ -f ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 ] && rm ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 && ln -s $(which python3) ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 - -${HOME}/.guix_env/{{ name }}/pre_env zsh diff --git a/guix_env/template_scripts/create_env.sh b/guix_env/template_scripts/create_env.sh deleted file mode 100644 index 4bb04aa..0000000 --- a/guix_env/template_scripts/create_env.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env -S guix time-machine --channels=${HOME}/.guix_env/{{ name }}/channels.scm -- shell {{ guix_args }} --preserve='(^DISPLAY$|^XAUTHORITY$|^TERM$)' --share=${HOME} --expose=/dev/dri --expose=/sys -m ${HOME}/.guix_env/{{ name }}/manifest.scm -- bash - - -echo "Installing poetry" -curl -sSL https://install.python-poetry.org | python3 - -poetry install --no-root --directory={{ directory }} diff --git a/guix_env/template_scripts/pre_env b/guix_env/template_scripts/pre_env index cb4e24a..5fb174c 100644 --- a/guix_env/template_scripts/pre_env +++ b/guix_env/template_scripts/pre_env @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env -S bash export POETRY_CACHE_DIR=${HOME}/.guix_env/poetry_cache export POETRY_VIRTUALENVS_IN_PROJECT=true @@ -6,5 +6,6 @@ export GUIX_ENV_NAME={{ name }} export SHELL=$(realpath $(which zsh)) export LD_LIBRARY_PATH=/lib export ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin +export TERM=ansi exec "$@" diff --git a/guix_env/template_scripts/run_script.sh b/guix_env/template_scripts/run_script.sh deleted file mode 100644 index 6b02d82..0000000 --- a/guix_env/template_scripts/run_script.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env -S guix time-machine --channels=${HOME}/.guix_env/{{ name }}/channels.scm -- shell {{ guix_args }} --preserve='(^DISPLAY$|^XAUTHORITY$|^TERM$)' --share=${HOME} --expose=/dev/dri --expose=/sys -m ${HOME}/.guix_env/{{ name }}/manifest.scm -- bash - -# Link the lib file for FHS library handling - -[ -f ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 ] && rm ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 && ln -s $(which python3) ${HOME}/.guix_env/guix_env_venv/{{ name }}_venv/bin/python3 - - - -cd ${HOME}/.guix_env/{{ name }} && ${HOME}/.guix_env/{{ name }}/pre_env poetry run $@ diff --git a/guix_env/template_scripts/use_env.sh b/guix_env/template_scripts/use_env.sh new file mode 100644 index 0000000..91c2264 --- /dev/null +++ b/guix_env/template_scripts/use_env.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# We use this script instead of having it in the shebang because the guix line is too long for a shebang. +# Found on stackoverflow https://stackoverflow.com/questions/10813538/shebang-line-limit-in-bash-and-linux-kernel +script="$1" +guix_cmd="guix time-machine --channels=${HOME}/.guix_env/{{ name }}/channels.scm -- shell {{ guix_args }} --preserve='(^DISPLAY$|^XAUTHORITY$|^TERM$)' --share=${HOME} --share=/tmp --expose=/dev/dri --expose=/sys -m ${HOME}/.guix_env/{{ name }}/manifest.scm -- " + +# now run it, passing in the remaining command line arguments +shift 1 +$guix_cmd ${HOME}/.guix_env/{{ name }}/pre_env "$script" "${@}" From b29611fb662a6710edaca1801622d0c5425eff09 Mon Sep 17 00:00:00 2001 From: Timothee Mathieu Date: Mon, 7 Oct 2024 12:09:03 +0200 Subject: [PATCH 06/14] update --- README.md | 9 +-- guix_env/cli.py | 71 ++++++++++++--------- guix_env/template_scripts/.#pre_env | 1 + guix_env/template_scripts/activate.sh | 3 + guix_env/template_scripts/channels.scm | 1 - guix_env/template_scripts/create_env.sh | 23 +++++++ guix_env/template_scripts/launch_in_guix.sh | 3 + guix_env/template_scripts/launch_shell.sh | 12 ++++ guix_env/template_scripts/long_shebang.sh | 12 ++++ guix_env/template_scripts/pre_env | 2 +- guix_env/template_scripts/run_script.sh | 13 ++++ guix_env/template_scripts/use_env.sh | 18 +++--- guix_env/template_scripts/zshrc | 3 + 13 files changed, 127 insertions(+), 44 deletions(-) create mode 120000 guix_env/template_scripts/.#pre_env create mode 100644 guix_env/template_scripts/activate.sh create mode 100644 guix_env/template_scripts/create_env.sh create mode 100644 guix_env/template_scripts/launch_in_guix.sh create mode 100644 guix_env/template_scripts/launch_shell.sh create mode 100644 guix_env/template_scripts/long_shebang.sh create mode 100644 guix_env/template_scripts/run_script.sh diff --git a/README.md b/README.md index 41b7843..0f4cbdf 100644 --- a/README.md +++ b/README.md @@ -23,13 +23,14 @@ The first run may be a bit slow because of guix downloading a bunch of packages Then, you are good to go and do anything you wish in your environment. You are in a python virtual environment and you can install new python packages with pip. To add new guix package, use `guix-env add-guix my_env_name my_package_name` from outside the environment. -TODO: explain poetry in guix-env ## TODO +TODO: explain poetry in guix-env -- Better documentation -- include explanations of how it works: poetry, what do we share, how to tinker with it, what changes are made... +- Better documentation -- include explanations of how it works: poetry, what do we share (what we do not share, e.g. .local), how to tinker with it, what changes are made... - Make tests -- Handle GPU ? +- Have an alias that install guix_env in a guix shell environment so that we can install & use guix-env in a reproducible maneer. - Feature: rollback, similar to what can be done with guix-home. -- Feature: use tmux inside the env and share tmp to make a real daemon. https://stackoverflow.com/questions/16398850/create-new-tmux-session-from-inside-a-tmux-session +- Handle GPU ? +- Feature: use tmux inside the env and share tmp to make a sort of daemon. https://stackoverflow.com/questions/16398850/create-new-tmux-session-from-inside-a-tmux-session diff --git a/guix_env/cli.py b/guix_env/cli.py index b7c1943..6814907 100644 --- a/guix_env/cli.py +++ b/guix_env/cli.py @@ -6,6 +6,7 @@ import shutil from jinja2 import Environment, FileSystemLoader +# TODO: add test that the environment exists before doing anything. main_dir=os.path.join(os.getenv("HOME"), ".guix_env") file_path = os.path.realpath(__file__) @@ -65,26 +66,24 @@ def create(ctx, name, channel_file, requirements_file, pyproject_file, poetry_lo Create an environment with name `name`. A channel file can be specified, otherwise a channel file will be automatically created. """ - assert not os.path.isdir(os.path.join(main_dir, name)), "Environment already exist" os.system('mkdir -p '+os.path.join(main_dir, name, "bin")) + os.system('mkdir -p '+os.path.join(main_dir, name, ".local")) zshrc = environment.get_template("zshrc").render(name = name, reqfile = os.path.join(main_dir, name, "requirements.txt")) # TBC channels = _make_channel_file(channel_file) - - env_script = environment.get_template("use_env.sh").render(name=name, guix_args = guix_args) - + home = os.getenv("HOME") + run_script = environment.get_template("run_script.sh").render(name=name, guix_args = guix_args, HOME=home) with open(os.path.join(main_dir, name, "bin", ".zshrc"), "w") as myfile: myfile.write(zshrc) - with open(os.path.join(main_dir, name, "bin", "use_env.sh"), "w") as myfile: - myfile.write(env_script) - os.system('chmod +x '+os.path.join(main_dir, name, "bin", "use_env.sh")) + with open(os.path.join(main_dir, name, "bin", "run_script.sh"), "w") as myfile: + myfile.write(run_script) + os.system('chmod +x '+os.path.join(main_dir, name, "bin", "run_script.sh")) - if channel_file is None: with open(os.path.join(main_dir, name, "channels.scm"), "w") as myfile: myfile.write(channels) @@ -117,22 +116,36 @@ def create(ctx, name, channel_file, requirements_file, pyproject_file, poetry_lo if poetry_lock_file is not None: os.system(f"cp {poetry_lock_file} {os.path.join(main_dir, name)}") - with open(os.path.join(main_dir, name, "pre_env"), "w") as myfile: - pre_env = environment.get_template("pre_env").render(name=name) - myfile.write(pre_env) - os.system("chmod +x "+os.path.join(main_dir, name, "pre_env")) - - env_file = os.path.join(main_dir, name, "bin", "use_env.sh") - os.system(f"{env_file} poetry install --no-root --directory={os.path.join(main_dir, name)}") + with open(os.path.join(main_dir, name, "bin", "launch_in_guix.sh"), "w") as myfile: + launcher = environment.get_template("launch_in_guix.sh").render(name=name, guix_args = guix_args,) + myfile.write(launcher) + + os.system("chmod +x "+os.path.join(main_dir, name, "bin", "launch_in_guix.sh")) + + # TODO: use one template file for the environment and add the rest with ninja + with open(os.path.join(main_dir, name, "bin", "launch_shell.sh"), "w") as myfile: + launcher = environment.get_template("launch_shell.sh").render(name=name) + myfile.write(launcher) + os.system("chmod +x "+os.path.join(main_dir, name, "bin", "launch_shell.sh")) + + if requirements_file is None: + reqfile = "" + else: + os.system("cp "+os.path.realpath(requirements_file)+ " /tmp/requirements_for_guix_env.txt") + reqfile = "/tmp/requirements_for_guix_env.txt" + create_env_file = environment.get_template("create_env.sh").render(name=name, + directory = os.path.join(main_dir, name), + requirements = reqfile) + with open(os.path.join("/tmp", "create_guix_env.sh"), "w") as myfile: + myfile.write(create_env_file) + os.system("chmod +x "+os.path.join("/tmp", "create_guix_env.sh")) + + run_script = environment.get_template("run_script.sh").render(name=name) + with open(os.path.join(main_dir, name, "bin", "run_script.sh"), "w") as myfile: + myfile.write(create_env_file) + os.system("chmod +x "+os.path.join(main_dir, name, "bin", "run_script.sh")) - if requirements_file is not None: - requirements_file = os.path.abspath(requirements_file) - os.system(f" {env_file} cat {requirements_file} | xargs poetry add --directory={os.path.join(main_dir, name)}") - - # make completions - os.system(f"{env_file} poetry completions zsh > "+os.path.join(main_dir, name,"_poetry")) - os.system(f"{env_file} poetry config virtualenvs.prompt ' ' --local --directory={os.path.join(main_dir, name)}") - + os.system(os.path.join(main_dir, name, "bin", "launch_in_guix.sh")+ " " + os.path.join("/tmp", "create_guix_env.sh")) @guix_env.command() @click.argument('name',required = True, type=str) @@ -157,8 +170,6 @@ def rm(ctx, name): """ if os.path.isdir(os.path.join(main_dir, name)): shutil.rmtree(os.path.join(main_dir, name)) - if os.path.isdir(os.path.join(main_dir, "guix_env_venv", name+"_venv")): - shutil.rmtree(os.path.join(main_dir, "guix_env_venv", name+"_venv")) @guix_env.command() @@ -233,7 +244,7 @@ def shell(ctx, name, tmux, cwd): """ Open a shell in the environment with name `name`. """ - env_file = os.path.join(main_dir, name, "bin", "use_env.sh") + # env_file = os.path.join(main_dir, name, "bin", "use_env.sh") if tmux: child = subprocess.run(["tmux","has-session", "-t", "guix_env_"+name],capture_output=True,text=True) @@ -248,7 +259,7 @@ def shell(ctx, name, tmux, cwd): print("done cwd") os.system("tmux attach -t guix_env_"+name) else: - os.system(f"{env_file} zsh ") + os.system(os.path.join(main_dir, name, "bin", "launch_in_guix.sh") + " " + os.path.join(main_dir, name, "bin", "launch_shell.sh")) @guix_env.command() @click.argument('name',required = True, type=str) @@ -261,9 +272,9 @@ def run(ctx, name, cmd): Example of usage is guix-env run my_env "ls $HOME/" """ - env_file = os.path.join(main_dir, name, "bin", "use_env.sh") - os.system(f"{env_file} poetry run --directory={os.path.join(main_dir, name)} {cmd}" ) - + + os.system(os.path.join(main_dir, name, "bin", "launch_in_guix.sh")+ " " + os.path.join(main_dir, name, "bin", "run_script.sh") + " " + cmd) + def _is_in_guix(pkg): output = subprocess.run(["guix", "search", pkg], capture_output=True).stdout.decode() diff --git a/guix_env/template_scripts/.#pre_env b/guix_env/template_scripts/.#pre_env new file mode 120000 index 0000000..8371737 --- /dev/null +++ b/guix_env/template_scripts/.#pre_env @@ -0,0 +1 @@ +dr.t@guix-T.5752 \ No newline at end of file diff --git a/guix_env/template_scripts/activate.sh b/guix_env/template_scripts/activate.sh new file mode 100644 index 0000000..efe8bc9 --- /dev/null +++ b/guix_env/template_scripts/activate.sh @@ -0,0 +1,3 @@ +echo 'Entering guix environment' + +${HOME}/.guix_env/{{ name }}bin/use_env.sh ${HOME}/.guix_env/{{ name }}/pre_env zsh diff --git a/guix_env/template_scripts/channels.scm b/guix_env/template_scripts/channels.scm index 94ed5ec..6171da9 100644 --- a/guix_env/template_scripts/channels.scm +++ b/guix_env/template_scripts/channels.scm @@ -2,6 +2,5 @@ (name 't-guix) (url "https://github.com/TimotheeMathieu/t-guix") (branch "main") - (commit "9af6b8eecb6110d93bdf5f263ef499d065af9500") ) {{ system_channels }}) diff --git a/guix_env/template_scripts/create_env.sh b/guix_env/template_scripts/create_env.sh new file mode 100644 index 0000000..ff63f73 --- /dev/null +++ b/guix_env/template_scripts/create_env.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env -S bash +# Must be run through launch_in_guix + +export POETRY_CACHE_DIR=${HOME}/.guix_env/poetry_cache +export POETRY_VIRTUALENVS_IN_PROJECT=true +export GUIX_ENV_NAME={{ name }} +export SHELL=$(realpath $(which zsh)) +export LD_LIBRARY_PATH=/lib +export ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin +export TERM=ansi + +echo "Creating the environment" + +export REQUIREMENTS_FILE={{ requirements }} + +poetry config virtualenvs.prompt ' ' --local --directory={{ directory }} +poetry install --no-root --directory={{ directory }} + +if [[ ! -z $REQUIREMENTS_FILE ]]; then + cat $REQUIREMENTS_FILE | xargs poetry add --directory={{ directory }} +fi + +poetry completions zsh > {{ directory }}/_poetry diff --git a/guix_env/template_scripts/launch_in_guix.sh b/guix_env/template_scripts/launch_in_guix.sh new file mode 100644 index 0000000..1b3533c --- /dev/null +++ b/guix_env/template_scripts/launch_in_guix.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +guix time-machine --channels=${HOME}/.guix_env/{{ name }}/channels.scm -- shell {{ guix_args }} --preserve='(^DISPLAY$|^XAUTHORITY$|^TERM$)' --share=${HOME}/.guix_env --share=${HOME}/.guix_env/{{ name }}/.local=${HOME}/.local --share=/tmp --expose=/dev/dri --expose=/sys -m ${HOME}/.guix_env/{{ name }}/manifest.scm -- "$@" diff --git a/guix_env/template_scripts/launch_shell.sh b/guix_env/template_scripts/launch_shell.sh new file mode 100644 index 0000000..87c3c21 --- /dev/null +++ b/guix_env/template_scripts/launch_shell.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env -S bash +# Must be run through launch_in_guix + +export POETRY_CACHE_DIR=${HOME}/.guix_env/poetry_cache +export POETRY_VIRTUALENVS_IN_PROJECT=true +export GUIX_ENV_NAME={{ name }} +export SHELL=$(realpath $(which zsh)) +export LD_LIBRARY_PATH=/lib +export ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin +export TERM=ansi + +zsh diff --git a/guix_env/template_scripts/long_shebang.sh b/guix_env/template_scripts/long_shebang.sh new file mode 100644 index 0000000..0cf12a2 --- /dev/null +++ b/guix_env/template_scripts/long_shebang.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env guix shell --pure --preserve='(^DISPLAY$|^XAUTHORITY$|^TERM$)' bash coreutils -- bash +# From https://stackoverflow.com/questions/10813538/shebang-line-limit-in-bash-and-linux-kernel + +script="$1" +shebang=$(head -1 "$script") + +# use an array in case a argument is there too +interp=( ${shebang#\#!} ) + +# now run it, passing in the remaining command line arguments +shift 1 +exec "${interp[@]}" "$script" "${@}" diff --git a/guix_env/template_scripts/pre_env b/guix_env/template_scripts/pre_env index 5fb174c..731c7ba 100644 --- a/guix_env/template_scripts/pre_env +++ b/guix_env/template_scripts/pre_env @@ -8,4 +8,4 @@ export LD_LIBRARY_PATH=/lib export ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin export TERM=ansi -exec "$@" +{{ cmds }} \ No newline at end of file diff --git a/guix_env/template_scripts/run_script.sh b/guix_env/template_scripts/run_script.sh new file mode 100644 index 0000000..3736802 --- /dev/null +++ b/guix_env/template_scripts/run_script.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env -S bash +# Must be run through launch_in_guix + +export POETRY_CACHE_DIR=${HOME}/.guix_env/poetry_cache +export POETRY_VIRTUALENVS_IN_PROJECT=true +export GUIX_ENV_NAME={{ name }} +export SHELL=$(realpath $(which zsh)) +export LD_LIBRARY_PATH=/lib +export ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin +export TERM=ansi + +# Link the lib file for FHS library handling +poetry run --directory=${HOME}/.guix_env/{{ name }} $@ diff --git a/guix_env/template_scripts/use_env.sh b/guix_env/template_scripts/use_env.sh index 91c2264..b490d41 100644 --- a/guix_env/template_scripts/use_env.sh +++ b/guix_env/template_scripts/use_env.sh @@ -1,9 +1,11 @@ -#!/usr/bin/env bash -# We use this script instead of having it in the shebang because the guix line is too long for a shebang. -# Found on stackoverflow https://stackoverflow.com/questions/10813538/shebang-line-limit-in-bash-and-linux-kernel -script="$1" -guix_cmd="guix time-machine --channels=${HOME}/.guix_env/{{ name }}/channels.scm -- shell {{ guix_args }} --preserve='(^DISPLAY$|^XAUTHORITY$|^TERM$)' --share=${HOME} --share=/tmp --expose=/dev/dri --expose=/sys -m ${HOME}/.guix_env/{{ name }}/manifest.scm -- " +#!/usr/bin/env -S guix time-machine --channels=${HOME}/.guix_env/{{ name }}/channels.scm -- shell {{ guix_args }} --preserve='(^DISPLAY$|^XAUTHORITY$|^TERM$)' --share=${HOME} --share=/tmp --expose=/dev/dri --expose=/sys -m ${HOME}/.guix_env/{{ name }}/manifest.scm -- sh -# now run it, passing in the remaining command line arguments -shift 1 -$guix_cmd ${HOME}/.guix_env/{{ name }}/pre_env "$script" "${@}" +export POETRY_CACHE_DIR=${HOME}/.guix_env/poetry_cache +export POETRY_VIRTUALENVS_IN_PROJECT=true +export GUIX_ENV_NAME={{ name }} +export SHELL=$(realpath $(which zsh)) +export LD_LIBRARY_PATH=/lib +export ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin +export TERM=ansi + +exec "$@" diff --git a/guix_env/template_scripts/zshrc b/guix_env/template_scripts/zshrc index 38fc005..33e6ecf 100644 --- a/guix_env/template_scripts/zshrc +++ b/guix_env/template_scripts/zshrc @@ -29,6 +29,8 @@ bindkey "\e[1~" beginning-of-line # Home bindkey "\e[4~" end-of-line # End bindkey "\e[7~" beginning-of-line # Home bindkey "\e[8~" end-of-line # End +bindkey "\eOF" end-of-line +bindkey "\eOH" beginning-of-line bindkey "^E" end-of-line bindkey "^A" beginning-of-line @@ -51,3 +53,4 @@ PROMPT="%{$fg[green]%}%n@{{ name }}-guix-env %T %~ %{$reset_color%}%# >" HISTFILE="$HOME/.guix_env/{{ name }}/.zhistory" . $HOME/.guix_env/{{ name }}/.venv/bin/activate + From 93c204e6fc11d642b000a0846c00913af8ef4181 Mon Sep 17 00:00:00 2001 From: Timothee Mathieu Date: Mon, 7 Oct 2024 12:10:31 +0200 Subject: [PATCH 07/14] clean --- guix_env/template_scripts/.#pre_env | 1 - guix_env/template_scripts/activate.sh | 3 --- guix_env/template_scripts/long_shebang.sh | 12 ------------ guix_env/template_scripts/pre_env | 11 ----------- guix_env/template_scripts/use_env.sh | 11 ----------- 5 files changed, 38 deletions(-) delete mode 120000 guix_env/template_scripts/.#pre_env delete mode 100644 guix_env/template_scripts/activate.sh delete mode 100644 guix_env/template_scripts/long_shebang.sh delete mode 100644 guix_env/template_scripts/pre_env delete mode 100644 guix_env/template_scripts/use_env.sh diff --git a/guix_env/template_scripts/.#pre_env b/guix_env/template_scripts/.#pre_env deleted file mode 120000 index 8371737..0000000 --- a/guix_env/template_scripts/.#pre_env +++ /dev/null @@ -1 +0,0 @@ -dr.t@guix-T.5752 \ No newline at end of file diff --git a/guix_env/template_scripts/activate.sh b/guix_env/template_scripts/activate.sh deleted file mode 100644 index efe8bc9..0000000 --- a/guix_env/template_scripts/activate.sh +++ /dev/null @@ -1,3 +0,0 @@ -echo 'Entering guix environment' - -${HOME}/.guix_env/{{ name }}bin/use_env.sh ${HOME}/.guix_env/{{ name }}/pre_env zsh diff --git a/guix_env/template_scripts/long_shebang.sh b/guix_env/template_scripts/long_shebang.sh deleted file mode 100644 index 0cf12a2..0000000 --- a/guix_env/template_scripts/long_shebang.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env guix shell --pure --preserve='(^DISPLAY$|^XAUTHORITY$|^TERM$)' bash coreutils -- bash -# From https://stackoverflow.com/questions/10813538/shebang-line-limit-in-bash-and-linux-kernel - -script="$1" -shebang=$(head -1 "$script") - -# use an array in case a argument is there too -interp=( ${shebang#\#!} ) - -# now run it, passing in the remaining command line arguments -shift 1 -exec "${interp[@]}" "$script" "${@}" diff --git a/guix_env/template_scripts/pre_env b/guix_env/template_scripts/pre_env deleted file mode 100644 index 731c7ba..0000000 --- a/guix_env/template_scripts/pre_env +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env -S bash - -export POETRY_CACHE_DIR=${HOME}/.guix_env/poetry_cache -export POETRY_VIRTUALENVS_IN_PROJECT=true -export GUIX_ENV_NAME={{ name }} -export SHELL=$(realpath $(which zsh)) -export LD_LIBRARY_PATH=/lib -export ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin -export TERM=ansi - -{{ cmds }} \ No newline at end of file diff --git a/guix_env/template_scripts/use_env.sh b/guix_env/template_scripts/use_env.sh deleted file mode 100644 index b490d41..0000000 --- a/guix_env/template_scripts/use_env.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env -S guix time-machine --channels=${HOME}/.guix_env/{{ name }}/channels.scm -- shell {{ guix_args }} --preserve='(^DISPLAY$|^XAUTHORITY$|^TERM$)' --share=${HOME} --share=/tmp --expose=/dev/dri --expose=/sys -m ${HOME}/.guix_env/{{ name }}/manifest.scm -- sh - -export POETRY_CACHE_DIR=${HOME}/.guix_env/poetry_cache -export POETRY_VIRTUALENVS_IN_PROJECT=true -export GUIX_ENV_NAME={{ name }} -export SHELL=$(realpath $(which zsh)) -export LD_LIBRARY_PATH=/lib -export ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin -export TERM=ansi - -exec "$@" From 810e9c7c491da7561fb00725261ae338fbfba5c6 Mon Sep 17 00:00:00 2001 From: Timothee Mathieu Date: Thu, 12 Dec 2024 11:46:21 +0100 Subject: [PATCH 08/14] update to use poetry ad to be able to insntall without python --- README.md | 4 +- guix_env/__init__.py | 0 guix_env/cli.py | 107 ++++++++++++---------- guix_env/template_scripts/launch_shell.sh | 7 ++ guix_env/template_scripts/pyproject.toml | 2 +- guix_env/template_scripts/run_script.sh | 14 ++- guix_env/template_scripts/use_env.sh | 11 +++ guix_env/template_scripts/zshrc | 16 +++- 8 files changed, 104 insertions(+), 57 deletions(-) create mode 100644 guix_env/__init__.py create mode 100644 guix_env/template_scripts/use_env.sh diff --git a/README.md b/README.md index 0f4cbdf..224629f 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,10 @@ Then, you are good to go and do anything you wish in your environment. You are i TODO: explain poetry in guix-env - Better documentation -- include explanations of how it works: poetry, what do we share (what we do not share, e.g. .local), how to tinker with it, what changes are made... +- Have internal poetry commands +- Comment more - Make tests - Have an alias that install guix_env in a guix shell environment so that we can install & use guix-env in a reproducible maneer. -- Feature: rollback, similar to what can be done with guix-home. +- Feature: rollback, at first this could be through git repo that auto-commit. - Handle GPU ? - Feature: use tmux inside the env and share tmp to make a sort of daemon. https://stackoverflow.com/questions/16398850/create-new-tmux-session-from-inside-a-tmux-session diff --git a/guix_env/__init__.py b/guix_env/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/guix_env/cli.py b/guix_env/cli.py index 6814907..b17fd31 100644 --- a/guix_env/cli.py +++ b/guix_env/cli.py @@ -14,15 +14,26 @@ environment = Environment(loader=FileSystemLoader( os.path.join(os.path.dirname(file_path),"template_scripts/"))) -default_guix_packages = [ + +guix_python_packages = [ "python", "python-toolchain", "poetry-next", # this comes from perso channel while waiting for guix to have a newer version of poetry + "xcb-util", # xcb is for matplotlib to be able to plt.show + "xcb-util-wm", + "xcb-util-image", + "xcb-util-keysyms", + "xcb-util-renderutil", + "xcb-util-cursor", + ] + +default_guix_packages = [ "bash", "glibc-locales", "nss-certs", "coreutils", "diffutils", + "findutils", "curl", "git", "make", @@ -31,13 +42,7 @@ "tcl", "gtk", "grep", - "xcb-util", # xcb/dbus is for matplotlib to be able to plt.show - "xcb-util-wm", - "xcb-util-image", - "xcb-util-keysyms", - "xcb-util-renderutil", "dbus", - "xcb-util-cursor", "ncurses", "nano", "tmux", @@ -56,27 +61,27 @@ def guix_env(ctx): @click.argument('name',required = True, type=str) @click.option('--channel-file',required = False, type=str, help="Path to a channel file to be used in the guix install") @click.option('--requirements-file',required = False, type=str, help="Path to a requirements.txt file to be used in the python install") +@click.option('--without-python', is_flag=True, help="Do an environment without python") @click.option('--pyproject-file',required = False, type=str, help="Path to a pyproject.toml file to be used in the python install (override requirement file if both are given).") @click.option('--poetry-lock-file',required = False, type=str, help="Path to a poetry.lock file to be used in the python install") @click.option('--manifest-file',required = False, type=str, help="Path to a manifest file to be used in the guix install. Will replace the default manifest.") @click.option('--guix-args',required = False, type=str, default="-CFNW", help="arguments to be passed to guix") @click.pass_context -def create(ctx, name, channel_file, requirements_file, pyproject_file, poetry_lock_file, manifest_file, guix_args): +def create(ctx, name, channel_file, without_python, requirements_file, pyproject_file, poetry_lock_file, manifest_file, guix_args): """ Create an environment with name `name`. A channel file can be specified, otherwise a channel file will be automatically created. """ + with_python = not without_python assert not os.path.isdir(os.path.join(main_dir, name)), "Environment already exist" os.system('mkdir -p '+os.path.join(main_dir, name, "bin")) os.system('mkdir -p '+os.path.join(main_dir, name, ".local")) zshrc = environment.get_template("zshrc").render(name = name, reqfile = os.path.join(main_dir, name, "requirements.txt")) - # TBC - channels = _make_channel_file(channel_file) home = os.getenv("HOME") - run_script = environment.get_template("run_script.sh").render(name=name, guix_args = guix_args, HOME=home) + run_script = environment.get_template("run_script.sh").render(name=name, guix_args = guix_args, HOME=home, with_python=with_python) with open(os.path.join(main_dir, name, "bin", ".zshrc"), "w") as myfile: myfile.write(zshrc) @@ -93,29 +98,14 @@ def create(ctx, name, channel_file, requirements_file, pyproject_file, poetry_lo if manifest_file is None: with open(os.path.join(main_dir, name, "manifest.scm"), "w") as myfile: packages = default_guix_packages + if with_python: + packages = packages + guix_python_packages myfile.write( "(specifications->manifest '(\n\"" + '"\n "'.join(packages) + '"\n))' ) else: os.system("cp "+manifest_file+" "+os.path.join(main_dir, name, "manifest.scm")) - guix_python_cmd = f"guix time-machine --channels=$HOME/.guix_env/{name}/channels.scm -- shell python -- python3 --version | cut -d ' ' -f 2" - - python_version = subprocess.check_output(guix_python_cmd, shell=True).decode().strip() - - if pyproject_file is None: - author = subprocess.run(["whoami"], capture_output=True).stdout.decode() - pyproject = environment.get_template("pyproject.toml").render(name = name, python_version = python_version) - else: - with open(pyproject_file, "r") as myfile: - pyproject = myfile.read() - - with open(os.path.join(main_dir, name, "pyproject.toml"), "w") as myfile: - myfile.write(pyproject) - - if poetry_lock_file is not None: - os.system(f"cp {poetry_lock_file} {os.path.join(main_dir, name)}") - with open(os.path.join(main_dir, name, "bin", "launch_in_guix.sh"), "w") as myfile: launcher = environment.get_template("launch_in_guix.sh").render(name=name, guix_args = guix_args,) myfile.write(launcher) @@ -124,28 +114,16 @@ def create(ctx, name, channel_file, requirements_file, pyproject_file, poetry_lo # TODO: use one template file for the environment and add the rest with ninja with open(os.path.join(main_dir, name, "bin", "launch_shell.sh"), "w") as myfile: - launcher = environment.get_template("launch_shell.sh").render(name=name) + launcher = environment.get_template("launch_shell.sh").render(name=name, with_python=with_python) myfile.write(launcher) os.system("chmod +x "+os.path.join(main_dir, name, "bin", "launch_shell.sh")) - if requirements_file is None: - reqfile = "" - else: - os.system("cp "+os.path.realpath(requirements_file)+ " /tmp/requirements_for_guix_env.txt") - reqfile = "/tmp/requirements_for_guix_env.txt" - create_env_file = environment.get_template("create_env.sh").render(name=name, - directory = os.path.join(main_dir, name), - requirements = reqfile) - with open(os.path.join("/tmp", "create_guix_env.sh"), "w") as myfile: - myfile.write(create_env_file) - os.system("chmod +x "+os.path.join("/tmp", "create_guix_env.sh")) - - run_script = environment.get_template("run_script.sh").render(name=name) - with open(os.path.join(main_dir, name, "bin", "run_script.sh"), "w") as myfile: - myfile.write(create_env_file) - os.system("chmod +x "+os.path.join(main_dir, name, "bin", "run_script.sh")) + if with_python: + ### construct a poetry environment optionally with the specified requirements + _make_python_env(main_dir, name, pyproject_file, poetry_lock_file, requirements_file) + + print(f"Guix-env environment {name} has beenn created, its files can be found in {os.path.join(main_dir, name)}") - os.system(os.path.join(main_dir, name, "bin", "launch_in_guix.sh")+ " " + os.path.join("/tmp", "create_guix_env.sh")) @guix_env.command() @click.argument('name',required = True, type=str) @@ -159,8 +137,6 @@ def update(ctx, name): myfile.write(channels) # TODO: add poetry update and a capacity to roll-back - - @guix_env.command() @click.argument('name',required = True, type=str) @click.pass_context @@ -169,6 +145,7 @@ def rm(ctx, name): Remove a guix-env environment. """ if os.path.isdir(os.path.join(main_dir, name)): + print("Removing ", os.path.join(main_dir, name)) shutil.rmtree(os.path.join(main_dir, name)) @@ -295,3 +272,37 @@ def _make_channel_file(channel_file=None): channels = environment.get_template("channels.scm").render(system_channels = system_channels) return channels + + +def _make_python_env(main_dir, name, pyproject_file, poetry_lock_file, requirements_file): + + guix_python_cmd = f"guix time-machine --channels=$HOME/.guix_env/{name}/channels.scm -- shell python -- python3 --version | cut -d ' ' -f 2" + python_version = subprocess.check_output(guix_python_cmd, shell=True).decode().strip() + + if pyproject_file is None: + author = subprocess.run(["whoami"], capture_output=True).stdout.decode() + pyproject = environment.get_template("pyproject.toml").render(name = name, python_version = python_version) + else: + with open(pyproject_file, "r") as myfile: + pyproject = myfile.read() + + with open(os.path.join(main_dir, name, "pyproject.toml"), "w") as myfile: + myfile.write(pyproject) + + if poetry_lock_file is not None: + os.system(f"cp {poetry_lock_file} {os.path.join(main_dir, name)}") + + if requirements_file is None: + reqfile = "" + else: + os.system("cp "+os.path.realpath(requirements_file)+ " /tmp/requirements_for_guix_env.txt") + reqfile = "/tmp/requirements_for_guix_env.txt" + + + create_env_file = environment.get_template("create_env.sh").render(name=name, + directory = os.path.join(main_dir, name), + requirements = reqfile) + with open(os.path.join("/tmp", "create_guix_env.sh"), "w") as myfile: + myfile.write(create_env_file) + os.system("chmod +x "+os.path.join("/tmp", "create_guix_env.sh")) + os.system(os.path.join(main_dir, name, "bin", "launch_in_guix.sh")+ " " + os.path.join("/tmp", "create_guix_env.sh")) diff --git a/guix_env/template_scripts/launch_shell.sh b/guix_env/template_scripts/launch_shell.sh index 87c3c21..1ccabee 100644 --- a/guix_env/template_scripts/launch_shell.sh +++ b/guix_env/template_scripts/launch_shell.sh @@ -1,8 +1,15 @@ #!/usr/bin/env -S bash # Must be run through launch_in_guix +{% if with_python %} export POETRY_CACHE_DIR=${HOME}/.guix_env/poetry_cache export POETRY_VIRTUALENVS_IN_PROJECT=true +export REQUIREMENTS_FILE={{ requirements }} + +{% else %} + +{% endif %} + export GUIX_ENV_NAME={{ name }} export SHELL=$(realpath $(which zsh)) export LD_LIBRARY_PATH=/lib diff --git a/guix_env/template_scripts/pyproject.toml b/guix_env/template_scripts/pyproject.toml index 502f333..e54c48a 100644 --- a/guix_env/template_scripts/pyproject.toml +++ b/guix_env/template_scripts/pyproject.toml @@ -8,4 +8,4 @@ python = "{{ python_version }}" numpy = "*" scipy = "*" matplotlib = "*" -PyQt6 = "*" \ No newline at end of file +PyQt6 = "*" diff --git a/guix_env/template_scripts/run_script.sh b/guix_env/template_scripts/run_script.sh index 3736802..a236904 100644 --- a/guix_env/template_scripts/run_script.sh +++ b/guix_env/template_scripts/run_script.sh @@ -1,13 +1,19 @@ #!/usr/bin/env -S bash # Must be run through launch_in_guix -export POETRY_CACHE_DIR=${HOME}/.guix_env/poetry_cache -export POETRY_VIRTUALENVS_IN_PROJECT=true + export GUIX_ENV_NAME={{ name }} export SHELL=$(realpath $(which zsh)) -export LD_LIBRARY_PATH=/lib +export LD_LIBRARY_PATH=/lib # Link the lib file for FHS library handling export ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin export TERM=ansi -# Link the lib file for FHS library handling +{% if with_python %} +export POETRY_CACHE_DIR=${HOME}/.guix_env/poetry_cache +export POETRY_VIRTUALENVS_IN_PROJECT=true poetry run --directory=${HOME}/.guix_env/{{ name }} $@ + +{% else %} +$@ + +{% endif %} diff --git a/guix_env/template_scripts/use_env.sh b/guix_env/template_scripts/use_env.sh new file mode 100644 index 0000000..b490d41 --- /dev/null +++ b/guix_env/template_scripts/use_env.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env -S guix time-machine --channels=${HOME}/.guix_env/{{ name }}/channels.scm -- shell {{ guix_args }} --preserve='(^DISPLAY$|^XAUTHORITY$|^TERM$)' --share=${HOME} --share=/tmp --expose=/dev/dri --expose=/sys -m ${HOME}/.guix_env/{{ name }}/manifest.scm -- sh + +export POETRY_CACHE_DIR=${HOME}/.guix_env/poetry_cache +export POETRY_VIRTUALENVS_IN_PROJECT=true +export GUIX_ENV_NAME={{ name }} +export SHELL=$(realpath $(which zsh)) +export LD_LIBRARY_PATH=/lib +export ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin +export TERM=ansi + +exec "$@" diff --git a/guix_env/template_scripts/zshrc b/guix_env/template_scripts/zshrc index 33e6ecf..a87217d 100644 --- a/guix_env/template_scripts/zshrc +++ b/guix_env/template_scripts/zshrc @@ -38,6 +38,12 @@ autoload -U colors && colors alias ls='ls --color' +PROMPT="%{$fg[green]%}%n@{{ name }}-guix-env %T %~ +%{$reset_color%}%# >" +HISTFILE="$HOME/.guix_env/{{ name }}/.zhistory" + +{% if with_python %} + REQFILE={{ reqfile }} pip() @@ -49,8 +55,12 @@ pip() fi } -PROMPT="%{$fg[green]%}%n@{{ name }}-guix-env %T %~ -%{$reset_color%}%# >" -HISTFILE="$HOME/.guix_env/{{ name }}/.zhistory" . $HOME/.guix_env/{{ name }}/.venv/bin/activate +{% else %} + +{% endif %} + + + + From 003e908fa091299bd5114485e5ceacb549e6a508 Mon Sep 17 00:00:00 2001 From: Timothee Mathieu Date: Thu, 12 Dec 2024 11:46:59 +0100 Subject: [PATCH 09/14] clean --- guix_env/template_scripts/use_env.sh | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 guix_env/template_scripts/use_env.sh diff --git a/guix_env/template_scripts/use_env.sh b/guix_env/template_scripts/use_env.sh deleted file mode 100644 index b490d41..0000000 --- a/guix_env/template_scripts/use_env.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env -S guix time-machine --channels=${HOME}/.guix_env/{{ name }}/channels.scm -- shell {{ guix_args }} --preserve='(^DISPLAY$|^XAUTHORITY$|^TERM$)' --share=${HOME} --share=/tmp --expose=/dev/dri --expose=/sys -m ${HOME}/.guix_env/{{ name }}/manifest.scm -- sh - -export POETRY_CACHE_DIR=${HOME}/.guix_env/poetry_cache -export POETRY_VIRTUALENVS_IN_PROJECT=true -export GUIX_ENV_NAME={{ name }} -export SHELL=$(realpath $(which zsh)) -export LD_LIBRARY_PATH=/lib -export ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin -export TERM=ansi - -exec "$@" From 78a345ab6152c7fd173bae78fc12ce95fe6aba0d Mon Sep 17 00:00:00 2001 From: Timothee Mathieu Date: Thu, 12 Dec 2024 14:11:27 +0100 Subject: [PATCH 10/14] internal poetry command --- README.md | 4 +--- guix_env/cli.py | 6 ++---- guix_env/template_scripts/zshrc | 14 +++++++++++++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 224629f..9293eba 100644 --- a/README.md +++ b/README.md @@ -21,15 +21,13 @@ guix-env shell my_env_name The first run may be a bit slow because of guix downloading a bunch of packages but the second run should be faster as guix cache the packages it uses in `/gnu/store` (remark: don't forget to use `guix gc` to clear the store periodically). -Then, you are good to go and do anything you wish in your environment. You are in a python virtual environment and you can install new python packages with pip. To add new guix package, use `guix-env add-guix my_env_name my_package_name` from outside the environment. - +Then, you are good to go and do anything you wish in your environment. You are in a python virtual environment that is managed with poetry for reproducibility purpose, to install new python package from inside the environment, use `gep add package_name`, `gep` stands for guix-env-poetry and is just an alias of poetry that install at the right place. To add new guix package, use `guix-env add-guix my_env_name my_package_name` from outside the environment. ## TODO TODO: explain poetry in guix-env - Better documentation -- include explanations of how it works: poetry, what do we share (what we do not share, e.g. .local), how to tinker with it, what changes are made... -- Have internal poetry commands - Comment more - Make tests - Have an alias that install guix_env in a guix shell environment so that we can install & use guix-env in a reproducible maneer. diff --git a/guix_env/cli.py b/guix_env/cli.py index b17fd31..a247940 100644 --- a/guix_env/cli.py +++ b/guix_env/cli.py @@ -14,7 +14,6 @@ environment = Environment(loader=FileSystemLoader( os.path.join(os.path.dirname(file_path),"template_scripts/"))) - guix_python_packages = [ "python", "python-toolchain", @@ -77,7 +76,7 @@ def create(ctx, name, channel_file, without_python, requirements_file, pyproject os.system('mkdir -p '+os.path.join(main_dir, name, "bin")) os.system('mkdir -p '+os.path.join(main_dir, name, ".local")) - zshrc = environment.get_template("zshrc").render(name = name, reqfile = os.path.join(main_dir, name, "requirements.txt")) + zshrc = environment.get_template("zshrc").render(name = name, reqfile = os.path.join(main_dir, name, "requirements.txt"), with_python=with_python) channels = _make_channel_file(channel_file) home = os.getenv("HOME") @@ -221,6 +220,7 @@ def shell(ctx, name, tmux, cwd): """ Open a shell in the environment with name `name`. """ + assert os.path.isdir(os.path.join(main_dir, name)), "Environment does not exist" # env_file = os.path.join(main_dir, name, "bin", "use_env.sh") if tmux: @@ -264,7 +264,6 @@ def _is_in_guix(pkg): return res def _make_channel_file(channel_file=None): - if channel_file is None: system_channels = subprocess.run(["guix", "describe", "-f", "channels"], capture_output=True).stdout.decode() else: @@ -275,7 +274,6 @@ def _make_channel_file(channel_file=None): def _make_python_env(main_dir, name, pyproject_file, poetry_lock_file, requirements_file): - guix_python_cmd = f"guix time-machine --channels=$HOME/.guix_env/{name}/channels.scm -- shell python -- python3 --version | cut -d ' ' -f 2" python_version = subprocess.check_output(guix_python_cmd, shell=True).decode().strip() diff --git a/guix_env/template_scripts/zshrc b/guix_env/template_scripts/zshrc index a87217d..e652aea 100644 --- a/guix_env/template_scripts/zshrc +++ b/guix_env/template_scripts/zshrc @@ -41,9 +41,14 @@ alias ls='ls --color' PROMPT="%{$fg[green]%}%n@{{ name }}-guix-env %T %~ %{$reset_color%}%# >" HISTFILE="$HOME/.guix_env/{{ name }}/.zhistory" +echo "Welcome to your guix-env environment: {{ name }}" {% if with_python %} +. $HOME/.guix_env/{{ name }}/.venv/bin/activate + + + REQFILE={{ reqfile }} pip() @@ -55,7 +60,14 @@ pip() fi } -. $HOME/.guix_env/{{ name }}/.venv/bin/activate +gep() # guix-env poetry inside guix-env, used to add packages to the guix-env virtual env. +{ + cd $HOME/.guix_env/{{ name }} + command poetry "$@" + cd - +} + +echo "To install python package, use 'gep add package_name'. gep is ann alias for poetry that install things at the right place." {% else %} From 76eb1aa918fc4f4db2246cb2d8a6171cc654903e Mon Sep 17 00:00:00 2001 From: Timothee Mathieu Date: Thu, 12 Dec 2024 14:22:01 +0100 Subject: [PATCH 11/14] add explanations and move zshrc --- README.md | 18 +++++++++++------- guix_env/cli.py | 2 +- guix_env/template_scripts/launch_shell.sh | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 9293eba..ceec5a0 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,6 @@ # Guix & venv environments for reproducible python development -This package give a cli tool to construct and enter environments constructed through guix (for the system-level packages) and pip/venv (for python packages). - -Remark that due to the use of pip, the resulting environment is not perfectly reproducible as it would be better to use guix all the way but this way is much simpler when using dev python packages that are not yet packaged in guix. - +This package give a cli tool to construct and enter environments constructed through guix (for the system-level packages) and optionally a poetry environment (for python packages). ## Usage Guix must be installed on the system, see [the guix manual](https://guix.gnu.org/manual/en/html_node/Binary-Installation.html) to do this. @@ -13,7 +10,7 @@ pip install git+https://github.com/TimotheeMathieu/guix-env guix-env create my_env_name ``` -This should create a directory in `~/.guix-env` containing files needed for the environment to run. Note that every file generated by guix-env will be saved into `~/.guix-env`. Then to spawn a shell in the environment, do +This should create a directory in `~/.guix-env` containing files needed for the environment to run. Note that every file generated by guix-env will be saved into `~/.guix-env` and removing the environment is as simple as removing this directory (also doable through the `guix-env rm` command). Then to spawn a shell in the environment, do ``` guix-env shell my_env_name @@ -24,13 +21,20 @@ The first run may be a bit slow because of guix downloading a bunch of packages Then, you are good to go and do anything you wish in your environment. You are in a python virtual environment that is managed with poetry for reproducibility purpose, to install new python package from inside the environment, use `gep add package_name`, `gep` stands for guix-env-poetry and is just an alias of poetry that install at the right place. To add new guix package, use `guix-env add-guix my_env_name my_package_name` from outside the environment. + +Remark that I made a few opinionated design choices: +- I included some guix and python packages that are convenient for basic shell commands and basic graphical display. +- I use zsh shell in the guix-env environments. +- Every environment has its own .local and .zshrc. They can be accessed in $HOME/.guix_env/env_name +- I do not share the home directory, by default the only directory shared are the current directory and its children (default from guix shell). +- I use the Filesystem Hierarchy Standard (FHS) emulator of guix shell to populate /bin and /lib apropriately for some python compatibility. + ## TODO -TODO: explain poetry in guix-env - Better documentation -- include explanations of how it works: poetry, what do we share (what we do not share, e.g. .local), how to tinker with it, what changes are made... - Comment more - Make tests -- Have an alias that install guix_env in a guix shell environment so that we can install & use guix-env in a reproducible maneer. +- Have an alias that install guix_env in a guix shell environment so that we can install & use guix-env in a reproducible maneer and without needing to install anything. - Feature: rollback, at first this could be through git repo that auto-commit. - Handle GPU ? - Feature: use tmux inside the env and share tmp to make a sort of daemon. https://stackoverflow.com/questions/16398850/create-new-tmux-session-from-inside-a-tmux-session diff --git a/guix_env/cli.py b/guix_env/cli.py index a247940..ce78452 100644 --- a/guix_env/cli.py +++ b/guix_env/cli.py @@ -82,7 +82,7 @@ def create(ctx, name, channel_file, without_python, requirements_file, pyproject home = os.getenv("HOME") run_script = environment.get_template("run_script.sh").render(name=name, guix_args = guix_args, HOME=home, with_python=with_python) - with open(os.path.join(main_dir, name, "bin", ".zshrc"), "w") as myfile: + with open(os.path.join(main_dir, name, ".zshrc"), "w") as myfile: myfile.write(zshrc) with open(os.path.join(main_dir, name, "bin", "run_script.sh"), "w") as myfile: myfile.write(run_script) diff --git a/guix_env/template_scripts/launch_shell.sh b/guix_env/template_scripts/launch_shell.sh index 1ccabee..4352ce4 100644 --- a/guix_env/template_scripts/launch_shell.sh +++ b/guix_env/template_scripts/launch_shell.sh @@ -13,7 +13,7 @@ export REQUIREMENTS_FILE={{ requirements }} export GUIX_ENV_NAME={{ name }} export SHELL=$(realpath $(which zsh)) export LD_LIBRARY_PATH=/lib -export ZDOTDIR=${HOME}/.guix_env/{{ name }}/bin +export ZDOTDIR=${HOME}/.guix_env/{{ name }} export TERM=ansi zsh From 371b22d9c553d66ad429186779fc393ab582d168 Mon Sep 17 00:00:00 2001 From: Timothee Mathieu Date: Thu, 12 Dec 2024 14:23:53 +0100 Subject: [PATCH 12/14] add explanations --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ceec5a0..0a7dbba 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Remark that I made a few opinionated design choices: - Every environment has its own .local and .zshrc. They can be accessed in $HOME/.guix_env/env_name - I do not share the home directory, by default the only directory shared are the current directory and its children (default from guix shell). - I use the Filesystem Hierarchy Standard (FHS) emulator of guix shell to populate /bin and /lib apropriately for some python compatibility. +- I use a poetry cache specific to guix-env environments. It is shared among the guix envs but not with the host system. ## TODO From 74a0de0280db755ffc585030a8415c1d614fef73 Mon Sep 17 00:00:00 2001 From: Timothee Mathieu Date: Thu, 12 Dec 2024 14:27:45 +0100 Subject: [PATCH 13/14] fix bug run --- guix_env/template_scripts/run_script.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/guix_env/template_scripts/run_script.sh b/guix_env/template_scripts/run_script.sh index a236904..ed7c83e 100644 --- a/guix_env/template_scripts/run_script.sh +++ b/guix_env/template_scripts/run_script.sh @@ -11,7 +11,9 @@ export TERM=ansi {% if with_python %} export POETRY_CACHE_DIR=${HOME}/.guix_env/poetry_cache export POETRY_VIRTUALENVS_IN_PROJECT=true -poetry run --directory=${HOME}/.guix_env/{{ name }} $@ +. $HOME/.guix_env/{{ name }}/.venv/bin/activate + +$@ {% else %} $@ From aedaade1e44bd8bd78881de5a93e470ab64ac914 Mon Sep 17 00:00:00 2001 From: TimotheeMathieu Date: Sat, 14 Dec 2024 09:54:58 +0100 Subject: [PATCH 14/14] fix commands and add some tests --- .github/workflows/tests.yml | 31 ++++++++++++++ guix_env/cli.py | 56 ++++++++++++------------- guix_env/template_scripts/run_script.sh | 3 +- guix_env/template_scripts/zshrc | 3 -- 4 files changed, 60 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..509bb3f --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,31 @@ +name: tests +on: + pull_request_target: + branches: + - main + types: [closed] + push: + branches: + - main + +permissions: + contents: write + +jobs: + docs: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + pip install . + pip install pytest pytest-cov + + - name: test + run: | + pytest --cov=guix_env tests \ No newline at end of file diff --git a/guix_env/cli.py b/guix_env/cli.py index ce78452..ec94299 100644 --- a/guix_env/cli.py +++ b/guix_env/cli.py @@ -75,12 +75,12 @@ def create(ctx, name, channel_file, without_python, requirements_file, pyproject assert not os.path.isdir(os.path.join(main_dir, name)), "Environment already exist" os.system('mkdir -p '+os.path.join(main_dir, name, "bin")) os.system('mkdir -p '+os.path.join(main_dir, name, ".local")) + home = os.getenv("HOME") zshrc = environment.get_template("zshrc").render(name = name, reqfile = os.path.join(main_dir, name, "requirements.txt"), with_python=with_python) + run_script = environment.get_template("run_script.sh").render(name=name, guix_args = guix_args, HOME=home, with_python=with_python) channels = _make_channel_file(channel_file) - home = os.getenv("HOME") - run_script = environment.get_template("run_script.sh").render(name=name, guix_args = guix_args, HOME=home, with_python=with_python) with open(os.path.join(main_dir, name, ".zshrc"), "w") as myfile: myfile.write(zshrc) @@ -88,12 +88,13 @@ def create(ctx, name, channel_file, without_python, requirements_file, pyproject myfile.write(run_script) os.system('chmod +x '+os.path.join(main_dir, name, "bin", "run_script.sh")) + # Guix manifest and channel files if channel_file is None: with open(os.path.join(main_dir, name, "channels.scm"), "w") as myfile: myfile.write(channels) else: os.system("cp "+channel_file+" "+os.path.join(main_dir, name, "channels.scm")) - + if manifest_file is None: with open(os.path.join(main_dir, name, "manifest.scm"), "w") as myfile: packages = default_guix_packages @@ -105,13 +106,13 @@ def create(ctx, name, channel_file, without_python, requirements_file, pyproject else: os.system("cp "+manifest_file+" "+os.path.join(main_dir, name, "manifest.scm")) + # with open(os.path.join(main_dir, name, "bin", "launch_in_guix.sh"), "w") as myfile: launcher = environment.get_template("launch_in_guix.sh").render(name=name, guix_args = guix_args,) myfile.write(launcher) os.system("chmod +x "+os.path.join(main_dir, name, "bin", "launch_in_guix.sh")) - # TODO: use one template file for the environment and add the rest with ninja with open(os.path.join(main_dir, name, "bin", "launch_shell.sh"), "w") as myfile: launcher = environment.get_template("launch_shell.sh").render(name=name, with_python=with_python) myfile.write(launcher) @@ -129,12 +130,17 @@ def create(ctx, name, channel_file, without_python, requirements_file, pyproject @click.pass_context def update(ctx, name): """ - Update the channel file (and as a consequence, it will update the packages managed by guix at next shell/run). + Update the channel file to the current guix channel file (and as a consequence, it will update the packages managed by guix at next shell/run). + """ + print("Updating channel file") channels = _make_channel_file(os.path.join(main_dir, name, "channels.scm")) with open(os.path.join(main_dir, name, "channels.scm"), "w") as myfile: myfile.write(channels) - # TODO: add poetry update and a capacity to roll-back + if os.path.isfile(os.path.join(main_dir, name, "pyproject.toml")): + print("Found python install, updating") + _launch_cmd(name, "gep update") + @guix_env.command() @click.argument('name',required = True, type=str) @@ -147,7 +153,6 @@ def rm(ctx, name): print("Removing ", os.path.join(main_dir, name)) shutil.rmtree(os.path.join(main_dir, name)) - @guix_env.command() @click.argument('name',required = True, type=str) @click.argument('pkg',required = True, type=str) @@ -168,6 +173,7 @@ def add_guix(ctx, name, pkg): myfile.write( "(specifications->manifest '(\n\"" + '"\n "'.join(packages) + '"\n))' ) + print(f"Package {name} added to the manifest") @guix_env.command() @click.argument('name',required = True, type=str) @@ -177,8 +183,7 @@ def add_python(ctx, name, pkg): """ Add the python package `pkg` to the environment named `name`. """ - env_file = os.path.join(main_dir, name, "bin", "use_env.sh") - os.system(f"{env_file} poetry add {pkg} --directory={os.path.join(main_dir, name)}") + _launch_cmd(name, f"gep add {pkg}") @guix_env.command() @click.pass_context @@ -196,8 +201,7 @@ def info(ctx, name): Get informations on environment with name `name`. """ click.echo("Environment located in "+os.path.join(main_dir, name)) - env_file = os.path.join(main_dir, name, "bin", "use_env.sh") - os.system(env_file+" guix describe") + _launch_cmd(name," guix describe") with open(os.path.join(main_dir, name, "manifest.scm"), "r") as myfile: packages = myfile.read().split("(")[2].split(")")[0] @@ -209,7 +213,7 @@ def info(ctx, name): click.echo("\n".join(packages)) click.echo("-"*10) click.echo("Installed python packages") - os.system(f"{env_file} poetry run pip3 freeze --directory={os.path.join(main_dir, name)}") + _launch_cmd(name, "gep run pip3 freeze") @guix_env.command() @click.argument('name',required = True, type=str) @@ -221,22 +225,11 @@ def shell(ctx, name, tmux, cwd): Open a shell in the environment with name `name`. """ assert os.path.isdir(os.path.join(main_dir, name)), "Environment does not exist" - # env_file = os.path.join(main_dir, name, "bin", "use_env.sh") - - if tmux: - child = subprocess.run(["tmux","has-session", "-t", "guix_env_"+name],capture_output=True,text=True) - rc = child.returncode - if rc != 0: - print("env not launched yet, launching now") - os.system("tmux new-session -d -s guix_env_"+name+" "+activation_file) - - if cwd: - wd = os.getcwd() - os.system("tmux send-keys -t guix_env_"+name+" \" cd "+wd+ " && clear\" ENTER") - print("done cwd") - os.system("tmux attach -t guix_env_"+name) - else: - os.system(os.path.join(main_dir, name, "bin", "launch_in_guix.sh") + " " + os.path.join(main_dir, name, "bin", "launch_shell.sh")) + + print(f"Welcome to your guix-env environment: {name}") + print("To install python package, use 'gep add package_name'. gep is ann alias for poetry that install things at the right place.") + + os.system(os.path.join(main_dir, name, "bin", "launch_in_guix.sh") + " " + os.path.join(main_dir, name, "bin", "launch_shell.sh")) @guix_env.command() @click.argument('name',required = True, type=str) @@ -250,10 +243,15 @@ def run(ctx, name, cmd): guix-env run my_env "ls $HOME/" """ - os.system(os.path.join(main_dir, name, "bin", "launch_in_guix.sh")+ " " + os.path.join(main_dir, name, "bin", "run_script.sh") + " " + cmd) + _launch_cmd(name, cmd) + +def _launch_cmd(name, cmd): + os.system(os.path.join(main_dir, name, "bin", "launch_in_guix.sh")+ " " + os.path.join(main_dir, name, "bin", "run_script.sh") + " " + cmd) + def _is_in_guix(pkg): + print("Checking that the package is indeed a guix package") output = subprocess.run(["guix", "search", pkg], capture_output=True).stdout.decode() output = output.split("name: ") names = [o.split("\n")[0] for o in output] diff --git a/guix_env/template_scripts/run_script.sh b/guix_env/template_scripts/run_script.sh index ed7c83e..5fb5d45 100644 --- a/guix_env/template_scripts/run_script.sh +++ b/guix_env/template_scripts/run_script.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env -S bash +#!/usr/bin/env -S zsh # Must be run through launch_in_guix @@ -12,6 +12,7 @@ export TERM=ansi export POETRY_CACHE_DIR=${HOME}/.guix_env/poetry_cache export POETRY_VIRTUALENVS_IN_PROJECT=true . $HOME/.guix_env/{{ name }}/.venv/bin/activate +. $HOME/.guix_env/{{ name }}/.zshrc $@ diff --git a/guix_env/template_scripts/zshrc b/guix_env/template_scripts/zshrc index e652aea..c0a44a1 100644 --- a/guix_env/template_scripts/zshrc +++ b/guix_env/template_scripts/zshrc @@ -41,7 +41,6 @@ alias ls='ls --color' PROMPT="%{$fg[green]%}%n@{{ name }}-guix-env %T %~ %{$reset_color%}%# >" HISTFILE="$HOME/.guix_env/{{ name }}/.zhistory" -echo "Welcome to your guix-env environment: {{ name }}" {% if with_python %} @@ -67,8 +66,6 @@ gep() # guix-env poetry inside guix-env, used to add packages to the guix-env vi cd - } -echo "To install python package, use 'gep add package_name'. gep is ann alias for poetry that install things at the right place." - {% else %} {% endif %}