add Preferences.jl-based configuration (#742)

* add Preferences.jl-based configuration

* nitpicks

* docs nitpicks

---------

Co-authored-by: Christopher Rowley <github.com/cjdoris>

Author: marius311

Project.toml

@@ -10,6 +10,7 @@ Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Preferences = "21216c6a-2e73-6563-6e65-726566657250"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 UnsafePointers = "e17b2a0c-0bdf-430a-bd0c-3a23cae4ff39"
@@ -22,6 +23,7 @@ Libdl = "1"
 MacroTools = "0.5"
 Markdown = "1"
 Pkg = "1"
+Preferences = "1"
 PyCall = "1"
 Serialization = "1"
 Tables = "1"

docs/src/compat.md

@@ -13,7 +13,7 @@ Whenever a Python exception is displayed by Julia, `sys.last_traceback` and frie
 
 Python objects can be serialised with the [`Serialization`](https://docs.julialang.org/en/v1/stdlib/Serialization/) stdlib.
 This uses [`pickle`](https://docs.python.org/3/library/pickle.html) library under the hood.
-You can opt into using [`dill`](https://pypi.org/project/dill/) instead by setting the environment variable `JULIA_PYTHONCALL_PICKLE="dill"`.
+You can opt into using [`dill`](https://pypi.org/project/dill/) instead by setting the `pickle` preference or the `JULIA_PYTHONCALL_PICKLE` environment variable to `"dill"`.
 
 ## Tabular data / Pandas
 

docs/src/pythoncall.md

@@ -280,35 +280,80 @@ By default, PythonCall uses [CondaPkg.jl](https://github.com/JuliaPy/CondaPkg.jl
 its dependencies. This will install Conda and use it to create a Conda environment specific
 to your current Julia project containing Python and any required Python packages.
 
-#### If you already have Python and required Python packages installed
+You can configure PythonCall with the preferences listed below. These can be set either as
+[Julia preferences](https://github.com/JuliaPackaging/Preferences.jl) or as environment
+variables.
+
+| Preference | Environment Variable | Description |
+| ---------- | -------------------- | ----------- |
+| `exe` | `JULIA_PYTHONCALL_EXE` | Path to the Python executable, or special values (see below). |
+| `lib` | `JULIA_PYTHONCALL_LIB` | Path to the Python library (usually inferred automatically). |
+| `pickle` | `JULIA_PYTHONCALL_PICKLE` | Pickle module to use for serialization (`pickle` or `dill`). |
+
+The easiest way to set these preferences is with the
+[`PreferenceTools`](https://github.com/cjdoris/PreferenceTools.jl)
+package. For example:
+```
+julia> using PreferenceTools
+julia> # now press ] to enter the Pkg REPL
+pkg> preference add PythonCall exe=/path/to/python
+```
+
+For CondaPkg preferences (such as `backend`, `exe`, `env`), see the
+[CondaPkg documentation](https://github.com/JuliaPy/CondaPkg.jl#preferences).
+
+### Special `exe` values
+
+The `exe` preference (or `JULIA_PYTHONCALL_EXE` environment variable) supports the following
+special values:
+- `@CondaPkg`: Use Python from CondaPkg (the default).
+- `@PyCall`: Use the same Python as PyCall. [See here](@ref faq-pycall).
+- `@venv`: Use Python from a `.venv` virtual environment in the current active project.
+
+Otherwise, the value is interpreted as:
+- An absolute path to a Python executable.
+- A relative path (containing `/` or `\`) resolved relative to the current active project.
+- A command name to search for in `PATH`.
+
+### If you already have Python and required Python packages installed
 
 ```julia
 ENV["JULIA_CONDAPKG_BACKEND"] = "Null"
 ENV["JULIA_PYTHONCALL_EXE"] = "/path/to/python"  # optional
-ENV["JULIA_PYTHONCALL_EXE"] = "@PyCall"  # optional
-ENV["JULIA_PYTHONCALL_EXE"] = "@venv" # optional
+```
+
+Or using preferences:
+```
+pkg> preference add CondaPkg backend=Null
+pkg> preference add PythonCall exe=/path/to/python  # optional
 ```
 
 By setting the CondaPkg backend to Null, it will never install any Conda packages. In this
 case, PythonCall will use whichever Python is currently installed and in your `PATH`. You
 must have already installed any Python packages that you need.
 
-If `python` is not in your `PATH`, you will also need to set `JULIA_PYTHONCALL_EXE` to its
-path. Relative paths are resolved relative to the current active project.
+If `python` is not in your `PATH`, you will also need to set `exe` (or `JULIA_PYTHONCALL_EXE`)
+to its path. Relative paths are resolved relative to the current active project.
 
-If you also use PyCall, you can set `JULIA_PYTHONCALL_EXE=@PyCall` to use the same Python
-interpreter. [See here](@ref faq-pycall).
+If you also use PyCall, you can set `exe=@PyCall` to use the same Python interpreter.
+[See here](@ref faq-pycall).
 
 If you have a Python virtual environment at `.venv` in your current active project, you
-can set `JULIA_PYTHONCALL_EXE=@venv` to use it.
+can set `exe=@venv` to use it.
 
-#### If you already have a Conda environment
+### If you already have a Conda environment
 
 ```julia
 ENV["JULIA_CONDAPKG_BACKEND"] = "Current"
 ENV["JULIA_CONDAPKG_EXE"] = "/path/to/conda"  # optional
 ```
 
+Or using preferences:
+```
+pkg> preference add CondaPkg backend=Current
+pkg> preference add CondaPkg exe=/path/to/conda  # optional
+```
+
 The Current backend to CondaPkg will use the currently activated Conda environment instead
 of creating a new one.
 
@@ -317,23 +362,29 @@ If you already have your dependencies installed and do not want the environment
 modified, then see the previous section.
 
 If `conda`, `mamba` or `micromamba` is not in your `PATH` you will also need to set
-`JULIA_CONDAPKG_EXE` to its path.
+`JULIA_CONDAPKG_EXE` (or the CondaPkg `exe` preference) to its path.
 
-#### If you already have Conda, Mamba or MicroMamba
+### If you already have Conda, Mamba, MicroMamba or Pixi
 
 ```julia
-ENV["JULIA_CONDAPKG_BACKEND"] = "System"
+ENV["JULIA_CONDAPKG_BACKEND"] = "System"  # or "SystemPixi" for Pixi
 ENV["JULIA_CONDAPKG_EXE"] = "/path/to/conda"  # optional
 ```
 
-The System backend to CondaPkg will use your preinstalled Conda implementation instead of
-downloading one.
+Or using preferences:
+```
+pkg> preference add CondaPkg backend=System
+pkg> preference add CondaPkg exe=/path/to/conda  # optional
+```
+
+The System (or SystemPixi) backend to CondaPkg will use your preinstalled Conda (or
+Pixi) implementation instead of downloading one.
 
 Note that this will still create a new Conda environment and install any required packages
 into it. If you want to use a pre-existing Conda environment, see the previous section.
 
-If `conda`, `mamba` or `micromamba` is not in your `PATH` you will also need to set
-`JULIA_CONDAPKG_EXE` to its path.
+If `conda`, `mamba`, `micromamba` or `pixi` is not in your `PATH` you will also need to
+set `JULIA_CONDAPKG_EXE` (or the CondaPkg `exe` preference) to its path.
 
 ## [Installing Python packages](@id python-deps)
 

src/C/C.jl

@@ -5,6 +5,8 @@ This module provides a direct interface to the Python C API.
 """
 module C
 
+using ..Utils
+
 using Base: @kwdef
 using UnsafePointers: UnsafePtr
 using CondaPkg: CondaPkg
@@ -15,7 +17,6 @@ using Libdl:
 import ..PythonCall:
     python_executable_path, python_library_path, python_library_handle, python_version
 
-
 include("consts.jl")
 include("pointers.jl")
 include("extras.jl")

src/C/context.jl

@@ -115,15 +115,15 @@ function init_context()
         Py_IsInitialized() == 0 && error("Python is not already initialized.")
         CTX.is_initialized = true
         CTX.which = :embedded
-        exe_path = get(ENV, "JULIA_PYTHONCALL_EXE", "")
+        exe_path = Utils.getpref_exe()
         if exe_path != ""
             CTX.exe_path = exe_path
             # this ensures PyCall uses the same Python interpreter
             get!(ENV, "PYTHON", exe_path)
         end
     else
         # Find Python executable
-        exe_path = get(ENV, "JULIA_PYTHONCALL_EXE", "")
+        exe_path = Utils.getpref_exe()
         if exe_path == "" || exe_path == "@CondaPkg"
             if CondaPkg.backend() == :Null
                 exe_path = Sys.which("python")
@@ -158,7 +158,7 @@ function init_context()
                 exe_path = abspath(exe_path, "bin", "python")::String
             end
         elseif startswith(exe_path, "@")
-            error("invalid JULIA_PYTHONCALL_EXE=$exe_path")
+            error("invalid exe: $exe_path")
         else
             # Otherwise we use the Python specified
             CTX.which = :unknown
@@ -199,7 +199,7 @@ function init_context()
         # Find and open Python library
         lib_path = something(
             CTX.lib_path === missing ? nothing : CTX.lib_path,
-            get(ENV, "JULIA_PYTHONCALL_LIB", nothing),
+            Utils.getpref_lib(),
             Some(nothing),
         )
         if lib_path !== nothing
@@ -226,7 +226,7 @@ function init_context()
             CTX.lib_path === missing && error("""
                 Could not find Python library for Python executable $(repr(CTX.exe_path)).
 
-                If you know where the library is, set environment variable 'JULIA_PYTHONCALL_LIB' to its path.
+                If you know where the library is, set the 'lib' preference or 'JULIA_PYTHONCALL_LIB' environment variable to its path.
                 """)
         end
 

src/Compat/serialization.jl

@@ -2,7 +2,7 @@
 #
 # We use pickle to serialise Python objects to bytes.
 
-_pickle_module() = pyimport(get(ENV, "JULIA_PYTHONCALL_PICKLE", "pickle"))
+_pickle_module() = pyimport(Utils.getpref_pickle())
 
 function serialize_py(s, x::Py)
     if pyisnull(x)

src/Utils/Utils.jl

@@ -1,5 +1,23 @@
 module Utils
 
+using Preferences: @load_preference
+
+function getpref(::Type{T}, prefname, envname, default = nothing) where {T}
+    ans = @load_preference(prefname, nothing)
+    ans === nothing || return checkpref(T, ans)::T
+    ans = get(ENV, envname, "")
+    isempty(ans) || return checkpref(T, ans)::T
+    return default
+end
+
+checkpref(::Type{String}, x) = error("invalid preference of type $(type(x)), expecting a string")
+checkpref(::Type{String}, x::AbstractString) = convert(String, x)
+
+# Specific preference functions
+getpref_exe() = getpref(String, "exe", "JULIA_PYTHONCALL_EXE", "")
+getpref_lib() = getpref(String, "lib", "JULIA_PYTHONCALL_LIB", nothing)
+getpref_pickle() = getpref(String, "pickle", "JULIA_PYTHONCALL_PICKLE", "pickle")
+
 function explode_union(T)
     @nospecialize T