From 2f52ed25bd94e1ac39e96d13a4c2bc1ccef8b1ee Mon Sep 17 00:00:00 2001 From: Edward Brown Date: Wed, 21 May 2025 14:28:52 +0100 Subject: [PATCH 01/10] Makefile and setup.py updates for binary wheel Modified the Makefile to add `setuptools` and `injectlib` targets, to add shared library. Plus add binary/wheel packaging. --- python/Makefile | 10 ++++++++-- python/setup.py | 7 ++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/python/Makefile b/python/Makefile index e5c74f5dd..f2ca52c03 100644 --- a/python/Makefile +++ b/python/Makefile @@ -40,10 +40,16 @@ pkg-test: venv tar xzf dist/yamlscript-*.tar.gz cat yamlscript-*/PKG-INFO -dist: venv MANIFEST.in .long_description.md +setuptools: venv + pip install setuptools + +injectlib: + cp $(LIBYS_SO_FQNP) ./lib/yamlscript/ + +dist: setuptools injectlib MANIFEST.in .long_description.md ( \ $(VENV) && \ - $(PYTHON) setup.py sdist \ + $(PYTHON) setup.py sdist bdist_wheel \ ) release: publish diff --git a/python/setup.py b/python/setup.py index 59963b8e1..1ca2ed207 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,8 +1,9 @@ version = '0.1.96' -from setuptools import setup import pathlib +from setuptools import setup + root = pathlib.Path(__file__).parent.resolve() long_description = \ @@ -22,6 +23,10 @@ packages = ['yamlscript'], package_dir = {'': 'lib'}, + package_data={ + 'yamlscript': ['libyamlscript.so.0.1.96'], + }, + python_requires = '>=3.6, <4', install_requires = [ 'pyyaml', From b4f1352e2c5691e121bb4891cfb3586ed15dfe88 Mon Sep 17 00:00:00 2001 From: Edward Brown Date: Wed, 21 May 2025 19:29:47 +0100 Subject: [PATCH 02/10] Refactor library path resolution. Updated the initialization logic to prioritize locating the shared library in the bindings directory, for binary wheel distributions. --- python/Makefile | 2 +- python/lib/yamlscript/__init__.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/python/Makefile b/python/Makefile index f2ca52c03..ab960c531 100644 --- a/python/Makefile +++ b/python/Makefile @@ -8,7 +8,7 @@ VENV := source $(PYTHON_VENV)/bin/activate export PYTHONPATH := $(ROOT)/python/lib - +#export LD_LIBRARY_PATH := NOTHING #------------------------------------------------------------------------------ build:: build-doc diff --git a/python/lib/yamlscript/__init__.py b/python/lib/yamlscript/__init__.py index 7cd624be6..892f1a90d 100644 --- a/python/lib/yamlscript/__init__.py +++ b/python/lib/yamlscript/__init__.py @@ -18,9 +18,11 @@ # We currently only support binding to an exact version of libyamlscript. yamlscript_version = '0.1.96' -import os, sys import ctypes import json +import os +import sys +from pathlib import Path # Require Python 3.6 or greater: assert sys.version_info >= (3, 6), \ @@ -44,6 +46,12 @@ def find_libyamlscript_path(): libyamlscript_name = \ "libyamlscript.%s.%s" % (so, yamlscript_version) + # First check for shared library in bindings directory, in case of binary wheel distribution. + path = (Path(__file__).parent / libyamlscript_name).absolute() + if path.exists(): + # from fmtr.tools import debug; debug.trace(is_debug=True) + return path.absolute() + # Use LD_LIBRARY_PATH to find libyamlscript shared library, or default to # '/usr/local/lib' (where it is installed by default): ld_library_path = os.environ.get('LD_LIBRARY_PATH') From 661351bf058c0b759b409cac4edab03955516079 Mon Sep 17 00:00:00 2001 From: Edward Brown Date: Thu, 22 May 2025 14:16:33 +0100 Subject: [PATCH 03/10] Refactor setup.py and library handling. Reorganized setup.py for better modularity, replacing hardcoded variables with dynamically retrieved values. Introduced `get_libyamlscript_ext` for determining library extensions --- python/Makefile | 2 +- python/lib/yamlscript/__init__.py | 19 +++++++++++++------ python/setup.py | 12 +++++++----- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/python/Makefile b/python/Makefile index ab960c531..243924162 100644 --- a/python/Makefile +++ b/python/Makefile @@ -6,7 +6,7 @@ PYTHON_VENV := $(ROOT)/python/.venv VENV := source $(PYTHON_VENV)/bin/activate -export PYTHONPATH := $(ROOT)/python/lib +#export PYTHONPATH := $(ROOT)/python/lib #export LD_LIBRARY_PATH := NOTHING #------------------------------------------------------------------------------ diff --git a/python/lib/yamlscript/__init__.py b/python/lib/yamlscript/__init__.py index 892f1a90d..493b013bc 100644 --- a/python/lib/yamlscript/__init__.py +++ b/python/lib/yamlscript/__init__.py @@ -28,19 +28,24 @@ assert sys.version_info >= (3, 6), \ "Python 3.6 or greater required for 'yamlscript'." -# Find the libyamlscript shared library file path: -def find_libyamlscript_path(): + +def get_libyamlscript_ext(): # We currently only support platforms that GraalVM supports. # And Windows is not yet implemented... # Confirm platform and determine file extension: + if sys.platform == 'linux': - so = 'so' + return 'so' elif sys.platform == 'darwin': - so = 'dylib' + return 'dylib' else: raise Exception( "Unsupported platform '%s' for yamlscript." % sys.platform) +# Find the libyamlscript shared library file path: +def find_libyamlscript_path(): + so = get_libyamlscript_ext() + # We currently bind to an exact version of libyamlscript. # eg 'libyamlscript.so.0.1.96' libyamlscript_name = \ @@ -50,7 +55,7 @@ def find_libyamlscript_path(): path = (Path(__file__).parent / libyamlscript_name).absolute() if path.exists(): # from fmtr.tools import debug; debug.trace(is_debug=True) - return path.absolute() + return path # Use LD_LIBRARY_PATH to find libyamlscript shared library, or default to # '/usr/local/lib' (where it is installed by default): @@ -77,7 +82,9 @@ def find_libyamlscript_path(): return libyamlscript_path # Load libyamlscript shared library: -libyamlscript = ctypes.CDLL(find_libyamlscript_path()) +path = find_libyamlscript_path() +# print(f'Loading libyamlscript shared library: {path} ...',) +libyamlscript = ctypes.CDLL(path) # Create binding to 'load_ys_to_json' function: load_ys_to_json = libyamlscript.load_ys_to_json diff --git a/python/setup.py b/python/setup.py index 1ca2ed207..418429961 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,9 +1,11 @@ -version = '0.1.96' - import pathlib from setuptools import setup +from lib.yamlscript import yamlscript_version as version, get_libyamlscript_ext + +NAME = 'yamlscript' +ext = get_libyamlscript_ext() root = pathlib.Path(__file__).parent.resolve() long_description = \ @@ -11,7 +13,7 @@ .read_text(encoding='utf-8') setup( - name = 'yamlscript', + name=NAME, version = version, description = 'Program in YAML — Code is Data', license = 'MIT', @@ -20,11 +22,11 @@ author = 'Ingy döt Net', author_email = 'ingy@ingy.net', - packages = ['yamlscript'], + packages=[NAME], package_dir = {'': 'lib'}, package_data={ - 'yamlscript': ['libyamlscript.so.0.1.96'], + NAME: [f'lib{NAME}.{ext}.{version}'], }, python_requires = '>=3.6, <4', From a1b1dac75ffd16d4c15fb660370ed1efd05a8284 Mon Sep 17 00:00:00 2001 From: Edward Brown Date: Thu, 22 May 2025 14:27:43 +0100 Subject: [PATCH 04/10] Further setup.py/__init__.py refactoring --- python/lib/yamlscript/__init__.py | 17 ++++++++++------- python/setup.py | 7 +++---- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/python/lib/yamlscript/__init__.py b/python/lib/yamlscript/__init__.py index 493b013bc..994a23cfd 100644 --- a/python/lib/yamlscript/__init__.py +++ b/python/lib/yamlscript/__init__.py @@ -16,6 +16,7 @@ # This value is automatically updated by 'make bump'. # The version number is used to find the correct shared library file. # We currently only support binding to an exact version of libyamlscript. +NAME = 'yamlscript' yamlscript_version = '0.1.96' import ctypes @@ -29,28 +30,30 @@ "Python 3.6 or greater required for 'yamlscript'." -def get_libyamlscript_ext(): +def get_libyamlscript_name(): # We currently only support platforms that GraalVM supports. # And Windows is not yet implemented... # Confirm platform and determine file extension: if sys.platform == 'linux': - return 'so' + so = 'so' elif sys.platform == 'darwin': - return 'dylib' + so = 'dylib' else: raise Exception( "Unsupported platform '%s' for yamlscript." % sys.platform) -# Find the libyamlscript shared library file path: -def find_libyamlscript_path(): - so = get_libyamlscript_ext() - # We currently bind to an exact version of libyamlscript. # eg 'libyamlscript.so.0.1.96' libyamlscript_name = \ "libyamlscript.%s.%s" % (so, yamlscript_version) + return libyamlscript_name + +# Find the libyamlscript shared library file path: +def find_libyamlscript_path(): + libyamlscript_name = get_libyamlscript_name() + # First check for shared library in bindings directory, in case of binary wheel distribution. path = (Path(__file__).parent / libyamlscript_name).absolute() if path.exists(): diff --git a/python/setup.py b/python/setup.py index 418429961..ff94e00fd 100644 --- a/python/setup.py +++ b/python/setup.py @@ -2,10 +2,9 @@ from setuptools import setup -from lib.yamlscript import yamlscript_version as version, get_libyamlscript_ext +from lib.yamlscript import NAME, yamlscript_version as version, get_libyamlscript_name -NAME = 'yamlscript' -ext = get_libyamlscript_ext() +libyamlscript_name = get_libyamlscript_name() root = pathlib.Path(__file__).parent.resolve() long_description = \ @@ -26,7 +25,7 @@ package_dir = {'': 'lib'}, package_data={ - NAME: [f'lib{NAME}.{ext}.{version}'], + NAME: [libyamlscript_name], }, python_requires = '>=3.6, <4', From 1226e9bfd7e17ac5a0f848928c5ea8c0e0b43d22 Mon Sep 17 00:00:00 2001 From: Edward Brown Date: Fri, 23 May 2025 09:15:17 +0100 Subject: [PATCH 05/10] Add custom build_ext class for platform-specific wheel tagging Introduce `LibYAMLScriptExtensionBuilder` to ensure setuptools correctly tags wheels with platform/architecture-specific metadata. --- python/setup.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/python/setup.py b/python/setup.py index ff94e00fd..07ee35d2b 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,6 +1,7 @@ import pathlib -from setuptools import setup +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext from lib.yamlscript import NAME, yamlscript_version as version, get_libyamlscript_name @@ -11,6 +12,25 @@ (root / '.long_description.md') \ .read_text(encoding='utf-8') + +class LibYAMLScriptExtensionBuilder(build_ext): + """ + + The shared library is pre-built, but we need to provide setuptools + with a dummy extension builder, so that it knows that the wheels + aren't just pure-Python and tags them with the correct + platform/architecture-specific naming and metadata. + + """ + + def build_extensions(self): + """ + + Build nothing. + + """ + pass + setup( name=NAME, version = version, @@ -24,11 +44,16 @@ packages=[NAME], package_dir = {'': 'lib'}, + ext_modules=[ + Extension(name=NAME, sources=[]) + ], + cmdclass={ + 'build_ext': LibYAMLScriptExtensionBuilder, + }, + package_data={ NAME: [libyamlscript_name], }, - - python_requires = '>=3.6, <4', install_requires = [ 'pyyaml', ], From 24b9e12f583cf0cf35440938a31b7fd0f816d848 Mon Sep 17 00:00:00 2001 From: Edward Brown Date: Tue, 27 May 2025 15:49:42 +0100 Subject: [PATCH 06/10] Revert test changes to Makefile and refine --- python/Makefile | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/python/Makefile b/python/Makefile index 243924162..e415972ff 100644 --- a/python/Makefile +++ b/python/Makefile @@ -6,9 +6,9 @@ PYTHON_VENV := $(ROOT)/python/.venv VENV := source $(PYTHON_VENV)/bin/activate -#export PYTHONPATH := $(ROOT)/python/lib +export PYTHONPATH := $(ROOT)/python/lib + -#export LD_LIBRARY_PATH := NOTHING #------------------------------------------------------------------------------ build:: build-doc @@ -40,24 +40,20 @@ pkg-test: venv tar xzf dist/yamlscript-*.tar.gz cat yamlscript-*/PKG-INFO -setuptools: venv - pip install setuptools -injectlib: - cp $(LIBYS_SO_FQNP) ./lib/yamlscript/ -dist: setuptools injectlib MANIFEST.in .long_description.md - ( \ - $(VENV) && \ - $(PYTHON) setup.py sdist bdist_wheel \ - ) +dist: venv MANIFEST.in .long_description.md + $(VENV) && \ + $(PYTHON) setup.py sdist && \ + cp $(LIBYS_SO_FQNP) ./lib/yamlscript/ && \ + $(PYTHON) setup.py bdist_wheel release: publish publish: dist ( \ $(VENV) && \ - twine upload --verbose --repository yamlscript dist/yamlscript-*.tar.gz \ + twine upload --verbose --repository yamlscript dist/yamlscript-*.tar.gz dist/yamlscript-*.whl \ ) clean:: @@ -78,7 +74,8 @@ $(PYTHON_VENV): pip install \ pytest \ pyyaml \ - twine + twine \ + setuptools MANIFEST.in: echo 'include ReadMe.md' > $@ From c2b276e1196794ce7e79def902639fd684462b61 Mon Sep 17 00:00:00 2001 From: Edward Brown Date: Tue, 27 May 2025 15:50:53 +0100 Subject: [PATCH 07/10] Better handle shared library and package data in setup.py --- python/setup.py | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/python/setup.py b/python/setup.py index 07ee35d2b..5109f388a 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,18 +1,36 @@ -import pathlib +import sys +from pathlib import Path from setuptools import setup, Extension from setuptools.command.build_ext import build_ext -from lib.yamlscript import NAME, yamlscript_version as version, get_libyamlscript_name +version = '0.1.96' -libyamlscript_name = get_libyamlscript_name() -root = pathlib.Path(__file__).parent.resolve() +NAME = 'yamlscript' +PACKAGE_DIR = 'lib' +EXTENSIONS = dict(linux='so', darwin='dylib') +root = Path(__file__).parent.resolve() long_description = \ (root / '.long_description.md') \ .read_text(encoding='utf-8') +def get_package_data(): + """ + + Include the shared library in the package data if a) this is a supported platform, and b) the shared library exists. + + """ + so = EXTENSIONS.get(sys.platform) + + if so: + filename = f"lib{NAME}.{so}.{version}" + lib_exists = (root / PACKAGE_DIR / NAME / filename).exists() + if lib_exists: + return {NAME: [filename]} + return {} + class LibYAMLScriptExtensionBuilder(build_ext): """ @@ -31,6 +49,7 @@ def build_extensions(self): """ pass + setup( name=NAME, version = version, @@ -42,7 +61,9 @@ def build_extensions(self): author_email = 'ingy@ingy.net', packages=[NAME], - package_dir = {'': 'lib'}, + package_dir={'': PACKAGE_DIR}, + + python_requires='>=3.6, <4', ext_modules=[ Extension(name=NAME, sources=[]) @@ -51,9 +72,7 @@ def build_extensions(self): 'build_ext': LibYAMLScriptExtensionBuilder, }, - package_data={ - NAME: [libyamlscript_name], - }, + package_data=get_package_data(), install_requires = [ 'pyyaml', ], From 6ec8746de020fa2af840c1a84ba3a8a3db391705 Mon Sep 17 00:00:00 2001 From: Edward Brown Date: Wed, 28 May 2025 10:20:01 +0100 Subject: [PATCH 08/10] Avoid building platform specific package if not required --- python/setup.py | 48 +++++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/python/setup.py b/python/setup.py index 5109f388a..57d747903 100644 --- a/python/setup.py +++ b/python/setup.py @@ -9,28 +9,18 @@ NAME = 'yamlscript' PACKAGE_DIR = 'lib' EXTENSIONS = dict(linux='so', darwin='dylib') +so = EXTENSIONS.get(sys.platform) + +if not so: + raise RuntimeError(f"Unsupported platform: {sys.platform}. Should be one of {','.join(EXTENSIONS.keys())}.") root = Path(__file__).parent.resolve() +filename = f"lib{NAME}.{so}.{version}" +path_lib = root / PACKAGE_DIR / NAME / filename long_description = \ (root / '.long_description.md') \ .read_text(encoding='utf-8') - -def get_package_data(): - """ - - Include the shared library in the package data if a) this is a supported platform, and b) the shared library exists. - - """ - so = EXTENSIONS.get(sys.platform) - - if so: - filename = f"lib{NAME}.{so}.{version}" - lib_exists = (root / PACKAGE_DIR / NAME / filename).exists() - if lib_exists: - return {NAME: [filename]} - return {} - class LibYAMLScriptExtensionBuilder(build_ext): """ @@ -50,6 +40,21 @@ def build_extensions(self): pass +if path_lib.exists(): + # If the shared library exists, only then add the relevant extension builder. + # Otherwise keep the package generic. + extension_config = dict( + ext_modules=[ + Extension(name=NAME, sources=[]) + ], + cmdclass=dict( + build_ext=LibYAMLScriptExtensionBuilder, + ), + package_data={NAME: [filename]}, + ) +else: + extension_config = dict() + setup( name=NAME, version = version, @@ -64,15 +69,6 @@ def build_extensions(self): package_dir={'': PACKAGE_DIR}, python_requires='>=3.6, <4', - - ext_modules=[ - Extension(name=NAME, sources=[]) - ], - cmdclass={ - 'build_ext': LibYAMLScriptExtensionBuilder, - }, - - package_data=get_package_data(), install_requires = [ 'pyyaml', ], @@ -95,4 +91,6 @@ def build_extensions(self): long_description = long_description, long_description_content_type = 'text/markdown', + + **extension_config, ) From f30c06e4c2edb463cb7407ca0f6111dd4156ac4d Mon Sep 17 00:00:00 2001 From: Edward Brown Date: Wed, 28 May 2025 17:36:45 +0100 Subject: [PATCH 09/10] Add show_info console script --- python/lib/yamlscript/__init__.py | 26 +++++++++++++++++++++----- python/setup.py | 6 ++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/python/lib/yamlscript/__init__.py b/python/lib/yamlscript/__init__.py index 994a23cfd..fb1b8f39e 100644 --- a/python/lib/yamlscript/__init__.py +++ b/python/lib/yamlscript/__init__.py @@ -57,8 +57,7 @@ def find_libyamlscript_path(): # First check for shared library in bindings directory, in case of binary wheel distribution. path = (Path(__file__).parent / libyamlscript_name).absolute() if path.exists(): - # from fmtr.tools import debug; debug.trace(is_debug=True) - return path + return str(path) # Use LD_LIBRARY_PATH to find libyamlscript shared library, or default to # '/usr/local/lib' (where it is installed by default): @@ -85,9 +84,8 @@ def find_libyamlscript_path(): return libyamlscript_path # Load libyamlscript shared library: -path = find_libyamlscript_path() -# print(f'Loading libyamlscript shared library: {path} ...',) -libyamlscript = ctypes.CDLL(path) +libyamlscript_path = find_libyamlscript_path() +libyamlscript = ctypes.CDLL(libyamlscript_path) # Create binding to 'load_ys_to_json' function: load_ys_to_json = libyamlscript.load_ys_to_json @@ -156,3 +154,21 @@ def __del__(self): rc = libyamlscript.graal_tear_down_isolate(self.isolatethread) if rc != 0: raise Exception("Failed to tear down isolate") + + +def show_info(): + """ + + Show YAMLScript package info for debugging purposes + + """ + from textwrap import dedent + info = f""" + {yamlscript_version=} + {sys.platform=} + {libyamlscript_path=} + {YAMLScript().load("inc: 41")=} + {YAMLScript().load("!YS-v0\ninc: 41")=} + """ + info = dedent(info).strip() + print(info) diff --git a/python/setup.py b/python/setup.py index 57d747903..f5fcc55f4 100644 --- a/python/setup.py +++ b/python/setup.py @@ -92,5 +92,11 @@ def build_extensions(self): long_description = long_description, long_description_content_type = 'text/markdown', + entry_points=dict( + console_scripts=[ + f'ys-py-show-info = {NAME}:show_info', + ], + ), + **extension_config, ) From 943f53aae61518bfc41df99bce5474e9c6120e41 Mon Sep 17 00:00:00 2001 From: Edward Brown Date: Wed, 28 May 2025 19:51:56 +0100 Subject: [PATCH 10/10] Add test Dockerfile --- Dockerfile | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..084f41e1b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM python + +USER root +WORKDIR / +RUN git clone https://github.com/yaml/yamlscript +WORKDIR /yamlscript +RUN make build + +RUN pip install fmtr.tools[debug] setuptools + +COPY . . +WORKDIR /yamlscript/python + + +RUN make dist +WORKDIR /tmp + +RUN pip uninstall yamlscript -y +RUN pip install /yamlscript/python/dist/yamlscript-*.whl +RUN bash -c "ys-py-show-info > bdist_wheel.info.txt" + +RUN pip install /yamlscript/python/dist/yamlscript-0.1.96.tar.gz +RUN bash -c "export LD_LIBRARY_PATH=/yamlscript/libyamlscript/lib && ys-py-show-info > sdist.info.txt" + + +CMD sleep infinity \ No newline at end of file