Resolving dependency conflicts

.github/workflows/python-package.yml

@@ -69,12 +69,17 @@ jobs:
         max_attempts: 3
         timeout_minutes: 5
         retry_on: error
+        # For the discussion on why the added `pipenv lock` is necessary/warranted,
+        # see a lengthy note in Pipfile from Kohsuke dated 2024-12-24.
         command: |
           python -m pip install --upgrade pip
           pip install pipenv
+          pipenv lock
           pipenv install --dev --python ${{ matrix.python-version }}
     - name: Build
       run: |
+        # for dependency resolution problem diagnosis, report the ultimate list of packages we got
+        pip list
         pipenv run build
         pipenv run install
     - name: Type check

.gitignore

@@ -87,13 +87,6 @@ ipython_config.py
 #   intended to run in multiple environments; otherwise, check them in:
 # .python-version
 
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow
 __pypackages__/
 
@@ -137,9 +130,6 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
-# Targetting multiple Python versions
-Pipfile.lock
-
 .vscode/
 bazel-*
 

Pipfile

@@ -6,18 +6,49 @@ verify_ssl = true
 [requires]
 python_version = "3.6"
 
+# The fundamental challenges of our dependency management is that our runtime dependencies need to be ancient,
+# but we prefer to use the latest dev tools. Pipenv cannot manage two sets of Pythons for runtime and dev tools,
+# so sometimes those two opposing forces made it impossible to find a workable version for some packages
+#
+# Here's the general guidance on how to navigate this tretchrous water:
+# - Let's have some policy guideline and telemetry to figure out the Python version support window.
+#   (Relevant: https://devguide.python.org/versions/)
+# - Where newer tooling demands newer libraries that conflict with other dependencies, we need to hold back the tooling.
+#
+# In some dependency managers, like Maven, it is possible for us to explicitly specify a version of a library to
+# resolve a conflict, but it looks like Pipfile doesn't support that. See https://github.com/pypa/pipenv/issues/1921
 [dev-packages]
 flake8 = "*"
-setuptools = ">=30.3.0"
+### setuptools dependency management  (by Kohsuke 2024-12-24)
+# I run into an interesting problem that I eventually tracked down to a compatibility between 'setuptools' and 'packaging'
+# (see https://github.com/pypa/setuptools/issues/4483 for more details)
+#
+# In diagnosing this problem further, I found out that pipenv doesn't control the version of 'setuptools'
+# (see https://github.com/pypa/pipenv/issues/2364 for more details), because pipenv installs virtualenv, which in turn
+# ends up controlling the version of setuptools. However, pipenv does control the version of 'packaging'.
+# As a result, when we want to test with other Python versions, we end up with a combination of setuptools and
+# packaging that breaks the build. I decided to eventually solve this by running `pipenv lock` on these integration
+# tests with newer Python. See .github/workflows/python-package.yml
+#
+# Why? Really, it's mainly because this does happen to keep builds green, by presumably finding a good
+# combination of 'setuptools' and 'packaging' for a given Python version. That said, I can imagine that it will break
+# some other aspects later down the road, for example where we pin back some dependencies to an old version.
+#
+# Ultimately, in order to correctly test this package with newer version of Python, we want to build with
+# the oldest version of Python we support, then use it from a newer version of Python. The most natural way to
+# express that is to split out tests into a separate package, have that depend on a newer Python and the 'launchable'
+# module, and run tests from there. That way, the dependency constraint problem of the 'launchable' module need not
+# account for a wide range of Python versions we need to deal with. This is exactly what users do, too.
+#
+# That, however, is a way bigger change than I wanted to take on now, so I'm just recording that guidance for
+# the future.
 setuptools-scm = "*"
 wheel = "*"
 # The last flake8 version that supports Python 3.6 specifies "pycodestyle >=
 # 2.9.0, < 2.10.0". ref: https://github.com/PyCQA/flake8/pull/1633
 # The latest autopep8 specifies "pycodestyle >= 2.10.0". This conflict cannot be resolved. Pin the version to resolve this.
 autopep8 = "<=1.7.0"
-importlib-metadata = "<7.2"
 isort = "*"
-more_itertools = "<10.4"
 mypy = "*"
 pre-commit = "*"
 responses = "*"
@@ -26,11 +57,18 @@ types-pkg_resources = "0.1.3"
 types-python-dateutil = "*"
 types-requests = "*"
 types-tabulate = "*"
+# newer virtualenv creates a conflict with importlib-metadata. This is the latest version that seems to avoid that
+virtualenv = "==20.16.2"
 lxml = "<=5.2.2"
 unittest-xml-reporting = "*"
 
 [packages]
 launchable = {editable = true, path = "."}
+# AIU, runtime dependencies specified in setup.cfg should be automatically picked up by Pipenv.
+# But for whatever reasons, that doesn't appear to be the case for those two packages. Maybe
+# ';python_version' decorator is confusing it?
+click = "*"
+more_itertools = "*"
 
 [scripts]
 build = "python setup.py sdist bdist_wheel"