diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index b5588ed7ca9..bd1218a7dd7 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -529,6 +529,13 @@ def is_poetry_pyproject_toml(location): return False +def is_uv_pyproject_toml(location): + with open(location, 'r') as file: + if "tool.uv" in file.read(): + return True + return False + + class BasePoetryPythonLayout(BaseExtractedPythonLayout): """ Base class for poetry python projects. @@ -832,6 +839,279 @@ def parse(cls, location, package_only=False): yield models.PackageData.from_data(package_data, package_only) +def parse_dependency_requirement(requirement, scope='dependencies', is_runtime=True): + """ + Parse a dependency requirement string and return a DependentPackage or None. + This now delegates to get_requires_dependencies for consistency and to avoid code duplication. + """ + if not requirement: + return None + # get_requires_dependencies expects a list of requirements + deps = get_requires_dependencies( + requires=[requirement], + default_scope=scope, + is_runtime=is_runtime, + ) + # Return the first DependentPackage if any, else None + return deps[0] if deps else None + + +class BaseUvPythonLayout(BaseExtractedPythonLayout): + + @classmethod + def assemble(cls, package_data, resource, codebase, package_adder): + if codebase.has_single_resource: + yield from models.DatafileHandler.assemble(package_data, resource, codebase, package_adder) + return + + package_resource = None + if resource.name == 'pyproject.toml': + package_resource = resource + elif resource.name == 'uv.lock': + if resource.has_parent(): + siblings = resource.siblings(codebase) + package_resource = [r for r in siblings if r.name == 'pyproject.toml'] + if package_resource: + package_resource = package_resource[0] + + if not package_resource: + yield from yield_dependencies_from_package_resource(resource) + return + + assert len(package_resource.package_data) == 1, f'Invalid pyproject.toml for {package_resource.path}' + pkg_data = package_resource.package_data[0] + pkg_data = models.PackageData.from_dict(pkg_data) + + if pkg_data.purl: + package = models.Package.from_package_data( + package_data=pkg_data, + datafile_path=package_resource.path, + ) + package_uid = package.package_uid + package.populate_license_fields() + yield package + + root = package_resource.parent(codebase) + if root: + for pypi_res in cls.walk_pypi(resource=root, codebase=codebase): + if package_uid and package_uid not in pypi_res.for_packages: + package_adder(package_uid, pypi_res, codebase) + yield pypi_res + + yield package_resource + + else: + # we have no package, so deps are not for a specific package uid + package_uid = None + + # in all cases yield possible dependencies + yield from yield_dependencies_from_package_data(pkg_data, package_resource.path, package_uid) + + # we yield this as we do not want this further processed + yield package_resource + + for lock_file in package_resource.siblings(codebase): + if lock_file.name == 'uv.lock': + yield from yield_dependencies_from_package_resource(lock_file, package_uid) + + if package_uid and package_uid not in lock_file.for_packages: + package_adder(package_uid, lock_file, codebase) + yield lock_file + + +class UvPyprojectTomlHandler(BaseUvPythonLayout): + datasource_id = 'pypi_uv_pyproject_toml' + path_patterns = ('*pyproject.toml',) + default_package_type = 'pypi' + default_primary_language = 'Python' + description = 'Python UV pyproject.toml' + documentation_url = 'https://docs.astral.sh/uv/' + + @classmethod + def is_datafile(cls, location, filetypes=tuple()): + """ + Return True if the file at location is likely a UV pyproject.toml file. + """ + if super().is_datafile(location, filetypes=filetypes) is False: + return False + return is_uv_pyproject_toml(location) + + @classmethod + def parse(cls, location, package_only=False): + """ + Parse a UV pyproject.toml file and yield a PackageData. + """ + with open(location, "rb") as fp: + pyproject_data = tomllib.load(fp) + + project = pyproject_data.get('project', {}) + tool_uv = pyproject_data.get('tool', {}).get('uv', {}) + + name = project.get('name') + version = project.get('version') + description = project.get('description') + + # Standard dependencies + dependencies = [] + for dep_requirement in project.get('dependencies', []): + dependency = parse_dependency_requirement( + requirement=dep_requirement, + scope='dependencies', + is_runtime=True, + ) + if dependency: + dependencies.append(dependency.to_dict()) + + # UV dev dependencies + dev_dependencies = tool_uv.get('dev-dependencies', []) + for dep_requirement in dev_dependencies: + dependency = parse_dependency_requirement( + requirement=dep_requirement, + scope='dev-dependencies', + is_runtime=False, + ) + if dependency: + dependencies.append(dependency.to_dict()) + + # Extra dependencies (optional dependency groups) + optional_dependencies = project.get('optional-dependencies', {}) + for group_name, group_deps in optional_dependencies.items(): + for dep_requirement in group_deps: + dependency = parse_dependency_requirement( + requirement=dep_requirement, + scope=group_name, + is_runtime=False, + ) + if dependency: + dependencies.append(dependency.to_dict()) + + extra_data = {} + if tool_uv: + extra_data['uv_config'] = tool_uv + + requires_python = project.get('requires-python') + if requires_python: + extra_data['python_version'] = requires_python + + package_data = dict( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + primary_language='Python', + name=name, + version=version, + description=description, + extra_data=extra_data if extra_data else None, + dependencies=dependencies, + ) + + yield models.PackageData.from_data(package_data, package_only) + + +class UvLockHandler(BaseUvPythonLayout): + datasource_id = 'pypi_uv_lock' + path_patterns = ('*uv.lock',) + default_package_type = 'pypi' + default_primary_language = 'Python' + description = 'Python UV lockfile' + documentation_url = 'https://docs.astral.sh/uv/' + + @classmethod + def parse(cls, location, package_only=False): + with open(location, "rb") as fp: + toml_data = tomllib.load(fp) + + packages = toml_data.get('package') + if not packages: + return + + lock_version = toml_data.get('version') + requires_python = toml_data.get('requires-python') + + dependencies = [] + for package in packages: + dependencies_for_resolved = [] + + # Handle dependencies - UV uses a different format than Poetry + deps = package.get("dependencies") or [] + for dep in deps: + if isinstance(dep, dict): + # UV format: {name: "package-name", marker: "condition"} + dep_name = dep.get('name') + marker = dep.get('marker') + purl = PackageURL( + type=cls.default_package_type, + name=dep_name, + ) + dependency = models.DependentPackage( + purl=purl.to_string(), + extracted_requirement=marker, + scope="dependencies", + is_runtime=True, + is_optional=False, + is_direct=True, + is_pinned=False, + ) + dependencies_for_resolved.append(dependency.to_dict()) + elif isinstance(dep, str): + # Simple string dependency + dependency = parse_dependency_requirement( + requirement=dep, + scope='dependencies', + is_runtime=True, + ) + if dependency: + dependencies_for_resolved.append(dependency.to_dict()) + + name = package.get('name') + version = package.get('version') + description = package.get('description') + homepage_url = package.get('homepage_url') + keywords = package.get('keywords') + parties = package.get('parties') + urls = get_pypi_urls(name, version) + + package_data = dict( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + primary_language='Python', + name=name, + version=version, + description=description, + homepage_url=homepage_url, + keywords=keywords, + parties=parties, + is_virtual=True, + dependencies=dependencies_for_resolved, + **urls, + ) + resolved_package = models.PackageData.from_data(package_data, package_only) + + dependency = models.DependentPackage( + purl=resolved_package.purl, + extracted_requirement=None, + scope=None, + is_runtime=True, + is_optional=False, + is_direct=False, + is_pinned=True, + resolved_package=resolved_package.to_dict() + ) + dependencies.append(dependency.to_dict()) + + extra_data = {} + extra_data['python_version'] = requires_python + extra_data['lock_version'] = lock_version + + package_data = dict( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + primary_language='Python', + extra_data=extra_data, + dependencies=dependencies, + ) + yield models.PackageData.from_data(package_data, package_only) + + class PipInspectDeplockHandler(models.DatafileHandler): datasource_id = 'pypi_inspect_deplock' path_patterns = ('*pip-inspect.deplock',) diff --git a/tests/packagedcode/data/pypi/uv/attrs-package-assembly-expected.json b/tests/packagedcode/data/pypi/uv/attrs-package-assembly-expected.json new file mode 100644 index 00000000000..6447986fc50 --- /dev/null +++ b/tests/packagedcode/data/pypi/uv/attrs-package-assembly-expected.json @@ -0,0 +1,335 @@ +{ + "packages": [ + { + "type": "pypi", + "namespace": null, + "name": "attrs", + "version": "0.0.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": "Classes Without Boilerplate", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Hynek Schlawack", + "email": "hs@ox.cx", + "url": null + } + ], + "keywords": [ + "class", + "attribute", + "boilerplate", + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Typing :: Typed" + ], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/python-attrs/attrs", + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": "mit", + "declared_license_expression_spdx": "MIT", + "license_detections": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "matches": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "from_file": "attrs/pyproject.toml", + "start_line": 1, + "end_line": 1, + "matcher": "1-spdx-id", + "score": 100.0, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06", + "rule_url": null, + "matched_text": "MIT" + } + ], + "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "license: MIT\n", + "notice_text": null, + "source_packages": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "Documentation": "https://www.attrs.org/", + "Changelog": "https://www.attrs.org/en/stable/changelog.html", + "Funding": "https://github.com/sponsors/hynek" + }, + "repository_homepage_url": "https://pypi.org/project/attrs", + "repository_download_url": "https://pypi.org/packages/source/a/attrs/attrs-0.0.0.tar.gz", + "api_data_url": "https://pypi.org/pypi/attrs/0.0.0/json", + "package_uid": "pkg:pypi/attrs@0.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_paths": [ + "pyproject.toml" + ], + "datasource_ids": [ + "pypi_pyproject_toml" + ], + "purl": "pkg:pypi/attrs@0.0.0" + } + ], + "dependencies": [ + { + "purl": "pkg:pypi/pytest", + "extracted_requirement": null, + "scope": "dev", + "is_runtime": true, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:pypi/pytest?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:pypi/attrs@0.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "pyproject.toml", + "datasource_id": "pypi_pyproject_toml" + }, + { + "purl": "pkg:pypi/hypothesis", + "extracted_requirement": null, + "scope": "dev", + "is_runtime": true, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:pypi/hypothesis?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:pypi/attrs@0.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "pyproject.toml", + "datasource_id": "pypi_pyproject_toml" + }, + { + "purl": "pkg:pypi/sphinx", + "extracted_requirement": null, + "scope": "docs", + "is_runtime": true, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:pypi/sphinx?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:pypi/attrs@0.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "pyproject.toml", + "datasource_id": "pypi_pyproject_toml" + }, + { + "purl": "pkg:pypi/pytest", + "extracted_requirement": null, + "scope": "dev", + "is_runtime": true, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:pypi/pytest?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:pypi/attrs@0.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "pyproject.toml", + "datasource_id": "pypi_pyproject_toml" + }, + { + "purl": "pkg:pypi/hypothesis", + "extracted_requirement": null, + "scope": "dev", + "is_runtime": true, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:pypi/hypothesis?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:pypi/attrs@0.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "pyproject.toml", + "datasource_id": "pypi_pyproject_toml" + }, + { + "purl": "pkg:pypi/sphinx", + "extracted_requirement": null, + "scope": "docs", + "is_runtime": true, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:pypi/sphinx?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:pypi/attrs@0.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "pyproject.toml", + "datasource_id": "pypi_pyproject_toml" + } + ], + "files": [ + { + "path": "pyproject.toml", + "type": "file", + "package_data": [ + { + "type": "pypi", + "namespace": null, + "name": "attrs", + "version": "0.0.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": "Classes Without Boilerplate", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Hynek Schlawack", + "email": "hs@ox.cx", + "url": null + } + ], + "keywords": [ + "class", + "attribute", + "boilerplate", + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Typing :: Typed" + ], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/python-attrs/attrs", + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": "mit", + "declared_license_expression_spdx": "MIT", + "license_detections": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "matches": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "from_file": "attrs/pyproject.toml", + "start_line": 1, + "end_line": 1, + "matcher": "1-spdx-id", + "score": 100.0, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06", + "rule_url": null, + "matched_text": "MIT" + } + ], + "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "license: MIT\n", + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "Documentation": "https://www.attrs.org/", + "Changelog": "https://www.attrs.org/en/stable/changelog.html", + "Funding": "https://github.com/sponsors/hynek" + }, + "dependencies": [ + { + "purl": "pkg:pypi/pytest", + "extracted_requirement": null, + "scope": "dev", + "is_runtime": true, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/hypothesis", + "extracted_requirement": null, + "scope": "dev", + "is_runtime": true, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/sphinx", + "extracted_requirement": null, + "scope": "docs", + "is_runtime": true, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://pypi.org/project/attrs", + "repository_download_url": "https://pypi.org/packages/source/a/attrs/attrs-0.0.0.tar.gz", + "api_data_url": "https://pypi.org/pypi/attrs/0.0.0/json", + "datasource_id": "pypi_pyproject_toml", + "purl": "pkg:pypi/attrs@0.0.0" + } + ], + "for_packages": [ + "pkg:pypi/attrs@0.0.0?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "uv.lock", + "type": "file", + "package_data": [], + "for_packages": [ + "pkg:pypi/attrs@0.0.0?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/uv/attrs/pyproject.toml b/tests/packagedcode/data/pypi/uv/attrs/pyproject.toml new file mode 100644 index 00000000000..140eaa846d9 --- /dev/null +++ b/tests/packagedcode/data/pypi/uv/attrs/pyproject.toml @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: MIT + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "attrs" +version = "0.0.0" # Use dynamic if needed, or set your version +description = "Classes Without Boilerplate" +authors = [{ name = "Hynek Schlawack", email = "hs@ox.cx" }] +license = "MIT" +license-files = ["LICENSE"] +requires-python = ">=3.9" +keywords = ["class", "attribute", "boilerplate"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Typing :: Typed", +] +dependencies = [] + +[project.urls] +Documentation = "https://www.attrs.org/" +Changelog = "https://www.attrs.org/en/stable/changelog.html" +Source = "https://github.com/python-attrs/attrs" +Funding = "https://github.com/sponsors/hynek" + +[project.optional-dependencies] +dev = ["pytest", "hypothesis"] +docs = ["sphinx"] + +[tool.uv] +dev-dependencies = ["pytest", "hypothesis"] diff --git a/tests/packagedcode/data/pypi/uv/attrs/uv.lock b/tests/packagedcode/data/pypi/uv/attrs/uv.lock new file mode 100644 index 00000000000..6fae2b002ec --- /dev/null +++ b/tests/packagedcode/data/pypi/uv/attrs/uv.lock @@ -0,0 +1,17 @@ +# Test data based on https://github.com/python-attrs/attrs (trimmed for test purposes) +version = 1 +revision = 3 +requires-python = ">=3.9" + +[[package]] +name = "attrs" +version = "23.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata", marker = "python_version < '3.8'" }, + { name = "typing-extensions", marker = "python_version < '3.8'" } +] +sdist = { url = "https://files.pythonhosted.org/packages/source/a/attrs/attrs-23.2.0.tar.gz", hash = "sha256:...", size = 12345, upload-time = "2025-01-01T00:00:00Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/.../attrs-23.2.0-py3-none-any.whl", hash = "sha256:...", size = 12345, upload-time = "2025-01-01T00:00:00Z" } +] diff --git a/tests/packagedcode/test_pypi.py b/tests/packagedcode/test_pypi.py index 3dcfa7d4268..4010038fc05 100644 --- a/tests/packagedcode/test_pypi.py +++ b/tests/packagedcode/test_pypi.py @@ -86,6 +86,13 @@ def test_package_scan_poetry_end_to_end(self): run_scan_click(['--package', '--processes', '-1', test_dir, '--json-pp', result_file]) check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES) + def test_package_scan_uv_assemble_end_to_end(self): + test_dir = self.get_test_loc('pypi/uv/attrs/') + result_file = self.get_temp_file('json') + expected_file = self.get_test_loc('pypi/uv/attrs-package-assembly-expected.json', must_exist=not REGEN_TEST_FIXTURES) + run_scan_click(['--package', '--strip-root', '--processes', '-1', test_dir, '--json', result_file]) + check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES) + class TestPyPiDevelopEggInfoPkgInfo(PackageTester): test_data_dir = os.path.join(os.path.dirname(__file__), 'data')