diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index c14fce435f9..893a0bd6f23 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -84,6 +84,8 @@ jobs: - name: Run tests run: | # FIXME: The unit tests currently only work with editable installs + # Install DIRACCommon first to ensure dependencies are resolved correctly + pip install -e ./dirac-common[testing] pip install -e .[server,testing] ${{ matrix.command }} env: @@ -135,9 +137,10 @@ jobs: sed -i.bak 's@editable = true@editable = false@g' pixi.toml rm pixi.toml.bak # Add annotations to github actions - pixi add --no-install --pypi --feature diracx-core pytest-github-actions-annotate-failures + pixi add --pypi --feature diracx-core pytest-github-actions-annotate-failures # Add the current DIRAC clone to the pixi.toml - pixi add --no-install --pypi --feature diracx-core 'DIRAC @ file://'$PWD'/../DIRAC' + pixi add --pypi --feature diracx-core 'DIRACCommon @ file://'$PWD'/../DIRAC/dirac-common' + pixi add --pypi --feature diracx-core 'DIRAC @ file://'$PWD'/../DIRAC' # Show any changes git diff - uses: prefix-dev/setup-pixi@v0.8.11 diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 605c51a9b60..f8c206020ea 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -106,8 +106,19 @@ jobs: fi fi fi - - name: Build distributions + - name: Build DIRACCommon distribution + if: steps.check-tag.outputs.create-release == 'true' run: | + cd dirac-common + python -m build + cd .. + - name: Pin DIRACCommon version and build DIRAC distribution + run: | + # If we're making a release, pin DIRACCommon to exact version + if [[ "${{ steps.check-tag.outputs.create-release }}" == "true" ]]; then + DIRACCOMMON_VERSION=$(cd dirac-common && python -m setuptools_scm | sed 's@Guessed Version @@g' | sed -E 's@(\.dev|\+g).+@@g') + python .github/workflows/pin_diraccommon_version.py "$DIRACCOMMON_VERSION" + fi python -m build - name: Make release on GitHub if: steps.check-tag.outputs.create-release == 'true' @@ -123,7 +134,14 @@ jobs: --version="${NEW_VERSION}" \ --rev="$(git rev-parse HEAD)" \ --release-notes-fn="release.notes.new" - - name: Publish package on PyPI + - name: Publish DIRACCommon to PyPI + if: steps.check-tag.outputs.create-release == 'true' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + packages-dir: dirac-common/dist/ + - name: Publish DIRAC to PyPI if: steps.check-tag.outputs.create-release == 'true' uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/dirac-common.yml b/.github/workflows/dirac-common.yml new file mode 100644 index 00000000000..561c7696e87 --- /dev/null +++ b/.github/workflows/dirac-common.yml @@ -0,0 +1,51 @@ +name: DIRACCommon Tests + +on: + push: + branches: + - integration + - rel-* + paths: + - 'dirac-common/**' + - '.github/workflows/dirac-common.yml' + pull_request: + branches: + - integration + - rel-* + paths: + - 'dirac-common/**' + - '.github/workflows/dirac-common.yml' + +jobs: + test-dirac-common: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Need full history for setuptools_scm + + - uses: prefix-dev/setup-pixi@v0.9.0 + with: + run-install: false + post-cleanup: false + + - name: Apply workarounds + run: | + # Workaround for https://github.com/prefix-dev/pixi/issues/3762 + sed -i.bak 's@editable = true@editable = false@g' dirac-common/pyproject.toml + rm dirac-common/pyproject.toml.bak + # Show any changes + git diff + + - uses: prefix-dev/setup-pixi@v0.9.0 + with: + cache: false + environments: testing + manifest-path: dirac-common/pyproject.toml + + - name: Run tests with pixi + run: | + cd dirac-common + pixi add --feature testing pytest-github-actions-annotate-failures + pixi run pytest diff --git a/.github/workflows/pin_diraccommon_version.py b/.github/workflows/pin_diraccommon_version.py new file mode 100755 index 00000000000..494adf2ecf4 --- /dev/null +++ b/.github/workflows/pin_diraccommon_version.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +""" +Pin DIRACCommon version in setup.cfg during deployment. + +This script is used during the deployment process to ensure DIRAC +depends on the exact version of DIRACCommon being released. +""" + +import re +import sys +from pathlib import Path +import subprocess + + +def get_diraccommon_version(): + """Get the current version of DIRACCommon from setuptools_scm.""" + result = subprocess.run( + ["python", "-m", "setuptools_scm"], cwd="dirac-common", capture_output=True, text=True, check=True + ) + # Extract version from output like "Guessed Version 9.0.0a65.dev7+g995f95504" + version_match = re.search(r"Guessed Version (\S+)", result.stdout) + if not version_match: + # Try direct output format + version = result.stdout.strip() + else: + version = version_match.group(1) + + # Clean up the version for release (remove dev and git hash parts) + version = re.sub(r"(\.dev|\+g).+", "", version) + return version + + +def pin_diraccommon_version(version): + """Pin DIRACCommon to exact version in setup.cfg.""" + setup_cfg = Path("setup.cfg") + content = setup_cfg.read_text() + + # Replace the DIRACCommon line with exact version pin + updated_content = re.sub(r"^(\s*)DIRACCommon\s*$", f"\\1DIRACCommon=={version}", content, flags=re.MULTILINE) + + if content == updated_content: + print(f"Warning: DIRACCommon line not found or already pinned in setup.cfg") + return False + + setup_cfg.write_text(updated_content) + print(f"Pinned DIRACCommon to version {version} in setup.cfg") + return True + + +def main(): + if len(sys.argv) > 1: + version = sys.argv[1] + else: + version = get_diraccommon_version() + + if pin_diraccommon_version(version): + print(f"Successfully pinned DIRACCommon to {version}") + sys.exit(0) + else: + print("Failed to pin DIRACCommon version") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/.gitignore b/.gitignore index 673900f3377..522461b09f7 100644 --- a/.gitignore +++ b/.gitignore @@ -86,3 +86,8 @@ docs/source/AdministratorGuide/CommandReference docs/source/UserGuide/CommandReference docs/_build docs/source/_build + +# pixi environments +.pixi +*.egg-info +pixi.lock diff --git a/dirac-common/README.md b/dirac-common/README.md new file mode 100644 index 00000000000..42a1c4e2d33 --- /dev/null +++ b/dirac-common/README.md @@ -0,0 +1,47 @@ +# DIRACCommon + +Stateless utilities extracted from DIRAC for use by DiracX and other projects without triggering DIRAC's global state initialization. + +## Purpose + +This package solves the circular dependency issue where DiracX needs DIRAC utilities but importing DIRAC triggers global state initialization. DIRACCommon contains only stateless utilities that can be safely imported without side effects. + +## Contents + +- `DIRACCommon.Utils.ReturnValues`: DIRAC's S_OK/S_ERROR return value system +- `DIRACCommon.Utils.DErrno`: DIRAC error codes and utilities + +## Installation + +```bash +pip install DIRACCommon +``` + +## Usage + +```python +from DIRACCommon.Utils.ReturnValues import S_OK, S_ERROR + +def my_function(): + if success: + return S_OK("Operation successful") + else: + return S_ERROR("Operation failed") +``` + +## Development + +This package is part of the DIRAC project and shares its version number. When DIRAC is released, DIRACCommon is also released with the same version. + +```bash +pixi install +pixi run pytest +``` + +## Guidelines for Adding Code + +Code added to DIRACCommon must: +- Be completely stateless +- Not import or use any of DIRAC's global objects (`gConfig`, `gLogger`, `gMonitor`, `Operations`) +- Not establish database connections +- Not have side effects on import diff --git a/dirac-common/pyproject.toml b/dirac-common/pyproject.toml new file mode 100644 index 00000000000..a9f64fd2840 --- /dev/null +++ b/dirac-common/pyproject.toml @@ -0,0 +1,133 @@ +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[project] +name = "DIRACCommon" +description = "Stateless utilities extracted from DIRAC for use by DiracX and other projects" +readme = "README.md" +requires-python = ">=3.11" +license = {text = "GPL-3.0-only"} +authors = [ + {name = "DIRAC Collaboration", email = "dirac-dev@cern.ch"}, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering", + "Topic :: System :: Distributed Computing", +] +dependencies = [ + "typing-extensions>=4.0.0", +] +dynamic = ["version"] + +[project.optional-dependencies] +testing = [ + "pytest>=7.0.0", + "pytest-cov>=4.0.0", +] + +[project.urls] +Homepage = "https://github.com/DIRACGrid/DIRAC" +Documentation = "https://dirac.readthedocs.io/" +"Source Code" = "https://github.com/DIRACGrid/DIRAC" + +[tool.hatch.version] +source = "vcs" + +[tool.hatch.version.raw-options] +root = ".." + +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/DIRACCommon"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = "test_*.py" +python_classes = "Test*" +python_functions = "test_*" +addopts = ["-v", "--cov=DIRACCommon", "--cov-report=term-missing"] + +[tool.coverage.run] +source = ["src/DIRACCommon"] +omit = ["*/tests/*"] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise AssertionError", + "raise NotImplementedError", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.mypy] +python_version = "3.11" +files = ["src/DIRACCommon"] +strict = true +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_unreachable = true +strict_equality = true + +[tool.ruff] +line-length = 120 +target-version = "py311" +select = [ + "E", # pycodestyle errors + "F", # pyflakes + "B", # flake8-bugbear + "I", # isort + "PLE", # pylint errors + "UP", # pyupgrade +] +ignore = [ + "B905", # zip without explicit strict parameter + "B008", # do not perform function calls in argument defaults + "B006", # do not use mutable data structures for argument defaults +] + +[tool.ruff.lint.flake8-tidy-imports.banned-api] +# This ensures DIRACCommon never imports from DIRAC +"DIRAC" = {msg = "DIRACCommon must not import from DIRAC to avoid global state initialization"} + +[tool.black] +line-length = 120 +target-version = ['py311'] + +[tool.isort] +profile = "black" +line_length = 120 + +[tool.pixi.workspace] +channels = ["conda-forge"] +platforms = ["linux-64", "linux-aarch64", "osx-arm64"] + +[tool.pixi.pypi-dependencies] +DIRACCommon = { path = ".", editable = true } + +[tool.pixi.feature.testing.tasks.pytest] +cmd = "pytest" + +[tool.pixi.environments] +default = { solve-group = "default" } +testing = { features = ["testing"], solve-group = "default" } + +[tool.pixi.tasks] diff --git a/dirac-common/src/DIRACCommon/Utils/DErrno.py b/dirac-common/src/DIRACCommon/Utils/DErrno.py new file mode 100644 index 00000000000..4b829000239 --- /dev/null +++ b/dirac-common/src/DIRACCommon/Utils/DErrno.py @@ -0,0 +1,327 @@ +""" :mod: DErrno + + ========================== + + .. module: DErrno + + :synopsis: Error list and utilities for handling errors in DIRAC + + + This module contains list of errors that can be encountered in DIRAC. + It complements the errno module of python. + + It also contains utilities to manipulate these errors. + + This is a stateless version extracted to DIRACCommon to avoid circular dependencies. + The extension loading functionality has been removed. +""" +import os + +# To avoid conflict, the error numbers should be greater than 1000 +# We decided to group the by range of 100 per system + +# 1000: Generic +# 1100: Core +# 1200: Framework +# 1300: Interfaces +# 1400: Config +# 1500: WMS + Workflow +# 1600: DMS + StorageManagement +# 1700: RMS +# 1800: Accounting + Monitoring +# 1900: TS + Production +# 2000: Resources + RSS + +# ## Generic (10XX) +# Python related: 0X +ETYPE = 1000 +EIMPERR = 1001 +ENOMETH = 1002 +ECONF = 1003 +EVALUE = 1004 +EEEXCEPTION = 1005 +# Files manipulation: 1X +ECTMPF = 1010 +EOF = 1011 +ERF = 1012 +EWF = 1013 +ESPF = 1014 + +# ## Core (11XX) +# Certificates and Proxy: 0X +EX509 = 1100 +EPROXYFIND = 1101 +EPROXYREAD = 1102 +ECERTFIND = 1103 +ECERTREAD = 1104 +ENOCERT = 1105 +ENOCHAIN = 1106 +ENOPKEY = 1107 +ENOGROUP = 1108 +# DISET: 1X +EDISET = 1110 +ENOAUTH = 1111 +# 3rd party security: 2X +E3RDPARTY = 1120 +EVOMS = 1121 +# Databases : 3X +EDB = 1130 +EMYSQL = 1131 +ESQLA = 1132 +# Message Queues: 4X +EMQUKN = 1140 +EMQNOM = 1141 +EMQCONN = 1142 +# OpenSearch +EELNOFOUND = 1146 +# Tokens +EATOKENFIND = 1150 +EATOKENREAD = 1151 +ETOKENTYPE = 1152 + +# config +ESECTION = 1400 + +# processes +EEZOMBIE = 1147 +EENOPID = 1148 + +# ## WMS/Workflow +EWMSUKN = 1500 +EWMSJDL = 1501 +EWMSRESC = 1502 +EWMSSUBM = 1503 +EWMSJMAN = 1504 +EWMSSTATUS = 1505 +EWMSNOMATCH = 1510 +EWMSPLTVER = 1511 +EWMSNOPILOT = 1550 + +# ## DMS/StorageManagement (16XX) +EFILESIZE = 1601 +EGFAL = 1602 +EBADCKS = 1603 +EFCERR = 1604 + +# ## RMS (17XX) +ERMSUKN = 1700 + +# ## TS (19XX) +ETSUKN = 1900 +ETSDATA = 1901 + +# ## Resources and RSS (20XX) +ERESGEN = 2000 +ERESUNA = 2001 +ERESUNK = 2002 + +# This translates the integer number into the name of the variable +dErrorCode = { + # ## Generic (10XX) + # 100X: Python related + 1000: "ETYPE", + 1001: "EIMPERR", + 1002: "ENOMETH", + 1003: "ECONF", + 1004: "EVALUE", + 1005: "EEEXCEPTION", + # 101X: Files manipulation + 1010: "ECTMPF", + 1011: "EOF", + 1012: "ERF", + 1013: "EWF", + 1014: "ESPF", + # ## Core + # 110X: Certificates and Proxy + 1100: "EX509", + 1101: "EPROXYFIND", + 1102: "EPROXYREAD", + 1103: "ECERTFIND", + 1104: "ECERTREAD", + 1105: "ENOCERT", + 1106: "ENOCHAIN", + 1107: "ENOPKEY", + 1108: "ENOGROUP", + # 111X: DISET + 1110: "EDISET", + 1111: "ENOAUTH", + # 112X: 3rd party security + 1120: "E3RDPARTY", + 1121: "EVOMS", + # 113X: Databases + 1130: "EDB", + 1131: "EMYSQL", + 1132: "ESQLA", + # 114X: Message Queues + 1140: "EMQUKN", + 1141: "EMQNOM", + 1142: "EMQCONN", + # OpenSearch + 1146: "EELNOFOUND", + # 115X: Tokens + 1150: "EATOKENFIND", + 1151: "EATOKENREAD", + 1152: "ETOKENTYPE", + # Config + 1400: "ESECTION", + # Processes + 1147: "EEZOMBIE", + 1148: "EENOPID", + # WMS/Workflow + 1500: "EWMSUKN", + 1501: "EWMSJDL", + 1502: "EWMSRESC", + 1503: "EWMSSUBM", + 1504: "EWMSJMAN", + 1505: "EWMSSTATUS", + 1510: "EWMSNOMATCH", + 1511: "EWMSPLTVER", + 1550: "EWMSNOPILOT", + # DMS/StorageManagement + 1601: "EFILESIZE", + 1602: "EGFAL", + 1603: "EBADCKS", + 1604: "EFCERR", + # RMS + 1700: "ERMSUKN", + # Resources and RSS + 2000: "ERESGEN", + 2001: "ERESUNA", + 2002: "ERESUNK", + # TS + 1900: "ETSUKN", + 1901: "ETSDATA", +} + + +dStrError = { # Generic (10XX) + # 100X: Python related + ETYPE: "Object Type Error", + EIMPERR: "Failed to import library", + ENOMETH: "No such method or function", + ECONF: "Configuration error", + EVALUE: "Wrong value passed", + EEEXCEPTION: "runtime general exception", + # 101X: Files manipulation + ECTMPF: "Failed to create temporary file", + EOF: "Cannot open file", + ERF: "Cannot read from file", + EWF: "Cannot write to file", + ESPF: "Cannot set permissions to file", + # ## Core + # 110X: Certificates and Proxy + EX509: "Generic Error with X509", + EPROXYFIND: "Can't find proxy", + EPROXYREAD: "Can't read proxy", + ECERTFIND: "Can't find certificate", + ECERTREAD: "Can't read certificate", + ENOCERT: "No certificate loaded", + ENOCHAIN: "No chain loaded", + ENOPKEY: "No private key loaded", + ENOGROUP: "No DIRAC group", + # 111X: DISET + EDISET: "DISET Error", + ENOAUTH: "Unauthorized query", + # 112X: 3rd party security + E3RDPARTY: "3rd party security service error", + EVOMS: "VOMS Error", + # 113X: Databases + EDB: "Database Error", + EMYSQL: "MySQL Error", + ESQLA: "SQLAlchemy Error", + # 114X: Message Queues + EMQUKN: "Unknown MQ Error", + EMQNOM: "No messages", + EMQCONN: "MQ connection failure", + # 114X OpenSearch + EELNOFOUND: "Index not found", + # 115X: Tokens + EATOKENFIND: "Can't find a bearer access token.", + EATOKENREAD: "Can't read a bearer access token.", + ETOKENTYPE: "Unsupported access token type.", + # Config + ESECTION: "Section is not found", + # processes + EEZOMBIE: "Zombie process", + EENOPID: "No PID of process", + # WMS/Workflow + EWMSUKN: "Unknown WMS error", + EWMSJDL: "Invalid job description", + EWMSRESC: "Job to reschedule", + EWMSSUBM: "Job submission error", + EWMSJMAN: "Job management error", + EWMSSTATUS: "Job status error", + EWMSNOPILOT: "No pilots found", + EWMSPLTVER: "Pilot version does not match", + EWMSNOMATCH: "No match found", + # DMS/StorageManagement + EFILESIZE: "Bad file size", + EGFAL: "Error with the gfal call", + EBADCKS: "Bad checksum", + EFCERR: "FileCatalog error", + # RMS + ERMSUKN: "Unknown RMS error", + # Resources and RSS + ERESGEN: "Unknown Resource Failure", + ERESUNA: "Resource not available", + ERESUNK: "Unknown Resource", + # TS + ETSUKN: "Unknown Transformation System Error", + ETSDATA: "Invalid Input Data definition", +} + + +def strerror(code: int) -> str: + """This method wraps up os.strerror, and behave the same way. + It completes it with the DIRAC specific errors. + """ + + if code == 0: + return "Undefined error" + + errMsg = f"Unknown error {code}" + + try: + errMsg = dStrError[code] + except KeyError: + # It is not a DIRAC specific error, try the os one + try: + errMsg = os.strerror(code) + # On some system, os.strerror raises an exception with unknown code, + # on others, it returns a message... + except ValueError: + pass + + return errMsg + + +def cmpError(inErr: str | int | dict, candidate: int) -> bool: + """This function compares an error (in its old form (a string or dictionary) or in its int form + with a candidate error code. + + :param inErr: a string, an integer, a S_ERROR dictionary + :type inErr: str or int or S_ERROR + :param int candidate: error code to compare with + + :return: True or False + + If an S_ERROR instance is passed, we compare the code with S_ERROR['Errno'] + If it is a Integer, we do a direct comparison + If it is a String, we use strerror to check the error string + """ + + if isinstance(inErr, str): # old style + # Compare error message strings + errMsg = strerror(candidate) + return errMsg in inErr + elif isinstance(inErr, dict): # if the S_ERROR structure is given + # Check if Errno defined in the dict + errorNumber = inErr.get("Errno") + if errorNumber: + return errorNumber == candidate + errMsg = strerror(candidate) + return errMsg in inErr.get("Message", "") + elif isinstance(inErr, int): + return inErr == candidate + else: + raise TypeError(f"Unknown input error type {type(inErr)}") diff --git a/dirac-common/src/DIRACCommon/Utils/ReturnValues.py b/dirac-common/src/DIRACCommon/Utils/ReturnValues.py new file mode 100644 index 00000000000..b075739e4e6 --- /dev/null +++ b/dirac-common/src/DIRACCommon/Utils/ReturnValues.py @@ -0,0 +1,255 @@ +""" + DIRAC return dictionary + + Message values are converted to string + + keys are converted to string +""" +from __future__ import annotations + +import functools +import sys +import traceback +from types import TracebackType +from typing import Any, Callable, cast, Generic, Literal, overload, Type, TypeVar, Union +from typing_extensions import TypedDict, ParamSpec, NotRequired + +from DIRACCommon.Utils.DErrno import strerror + + +T = TypeVar("T") +P = ParamSpec("P") + + +class DOKReturnType(TypedDict, Generic[T]): + """used for typing the DIRAC return structure""" + + OK: Literal[True] + Value: T + + +class DErrorReturnType(TypedDict): + """used for typing the DIRAC return structure""" + + OK: Literal[False] + Message: str + Errno: int + ExecInfo: NotRequired[tuple[type[BaseException], BaseException, TracebackType]] + CallStack: NotRequired[list[str]] + + +DReturnType = Union[DOKReturnType[T], DErrorReturnType] + + +def S_ERROR(*args: Any, **kwargs: Any) -> DErrorReturnType: + """return value on error condition + + Arguments are either Errno and ErrorMessage or just ErrorMessage fro backward compatibility + + :param int errno: Error number + :param string message: Error message + :param list callStack: Manually override the CallStack attribute better performance + """ + callStack = kwargs.pop("callStack", None) + + result: DErrorReturnType = {"OK": False, "Errno": 0, "Message": ""} + + message = "" + if args: + if isinstance(args[0], int): + result["Errno"] = args[0] + if len(args) > 1: + message = args[1] + else: + message = args[0] + + if result["Errno"]: + message = f"{strerror(result['Errno'])} ( {result['Errno']} : {message})" + result["Message"] = message + + if callStack is None: + try: + callStack = traceback.format_stack() + callStack.pop() + except Exception: + callStack = [] + + result["CallStack"] = callStack + + return result + + +# mypy doesn't understand default parameter values with generics so use overloads (python/mypy#3737) +@overload +def S_OK() -> DOKReturnType[None]: + ... + + +@overload +def S_OK(value: T) -> DOKReturnType[T]: + ... + + +def S_OK(value=None): # type: ignore + """return value on success + + :param value: value of the 'Value' + :return: dictionary { 'OK' : True, 'Value' : value } + """ + return {"OK": True, "Value": value} + + +def isReturnStructure(unk: Any) -> bool: + """Check if value is an `S_OK`/`S_ERROR` object""" + if not isinstance(unk, dict): + return False + if "OK" not in unk: + return False + if unk["OK"]: + return "Value" in unk + else: + return "Message" in unk + + +def isSError(value: Any) -> bool: + """Check if value is an `S_ERROR` object""" + if not isinstance(value, dict): + return False + if "OK" not in value: + return False + return "Message" in value + + +def reprReturnErrorStructure(struct: DErrorReturnType, full: bool = False) -> str: + errorNumber = struct.get("Errno", 0) + message = struct.get("Message", "") + if errorNumber: + reprStr = f"{strerror(errorNumber)} ( {errorNumber} : {message})" + else: + reprStr = message + + if full: + callStack = struct.get("CallStack") + if callStack: + reprStr += "\n" + "".join(callStack) + + return reprStr + + +def returnSingleResult(dictRes: DReturnType[Any]) -> DReturnType[Any]: + """Transform the S_OK{Successful/Failed} dictionary convention into + an S_OK/S_ERROR return. To be used when a single returned entity + is expected from a generally bulk call. + + :param dictRes: S_ERROR or S_OK( "Failed" : {}, "Successful" : {}) + :returns: S_ERROR or S_OK(value) + + The following rules are applied: + + - if dictRes is an S_ERROR: returns it as is + - we start by looking at the Failed directory + - if there are several items in a dictionary, we return the first one + - if both dictionaries are empty, we return S_ERROR + - For an item in Failed, we return S_ERROR + - Far an item in Successful we return S_OK + + Behavior examples (would be perfect unit test :-) ):: + + {'Message': 'Kaput', 'OK': False} -> {'Message': 'Kaput', 'OK': False} + {'OK': True, 'Value': {'Successful': {}, 'Failed': {'a': 1}}} -> {'Message': '1', 'OK': False} + {'OK': True, 'Value': {'Successful': {'b': 2}, 'Failed': {}}} -> {'OK': True, 'Value': 2} + {'OK': True, 'Value': {'Successful': {'b': 2}, 'Failed': {'a': 1}}} -> {'Message': '1', 'OK': False} + {'OK': True, 'Value': {'Successful': {'b': 2}, 'Failed': {'a': 1, 'c': 3}}} -> {'Message': '1', 'OK': False} + {'OK': True, 'Value': {'Successful': {'b': 2, 'd': 4}, 'Failed': {}}} -> {'OK': True, 'Value': 2} + {'OK': True, 'Value': {'Successful': {}, 'Failed': {}}} -> + {'Message': 'returnSingleResult: Failed and Successful dictionaries are empty', 'OK': False} + """ + # if S_ERROR was returned, we return it as well + if not dictRes["OK"]: + return dictRes + # if there is a Failed, we return the first one in an S_ERROR + if "Failed" in dictRes["Value"] and len(dictRes["Value"]["Failed"]): + errorMessage = list(dictRes["Value"]["Failed"].values())[0] + if isinstance(errorMessage, dict): + if isReturnStructure(errorMessage): + return cast(DErrorReturnType, errorMessage) + else: + return S_ERROR(str(errorMessage)) + return S_ERROR(errorMessage) + # if there is a Successful, we return the first one in an S_OK + elif "Successful" in dictRes["Value"] and len(dictRes["Value"]["Successful"]): + return S_OK(list(dictRes["Value"]["Successful"].values())[0]) + else: + return S_ERROR("returnSingleResult: Failed and Successful dictionaries are empty") + + +class SErrorException(Exception): + """Exception class for use with `convertToReturnValue`""" + + def __init__(self, result: DErrorReturnType | str, errCode: int = 0): + """Create a new exception return value + + If `result` is a `S_ERROR` return it directly else convert it to an + appropriate value using `S_ERROR(errCode, result)`. + + :param result: The error to propagate + :param errCode: the error code to propagate + """ + if not isSError(result): + result = S_ERROR(errCode, result) + self.result = cast(DErrorReturnType, result) + + +def returnValueOrRaise(result: DReturnType[T], *, errorCode: int = 0) -> T: + """Unwrap an S_OK/S_ERROR response into a value or Exception + + This method assists with using exceptions in DIRAC code by raising + :exc:`SErrorException` if `result` is an error. This can then by propagated + automatically as an `S_ERROR` by wrapping public facing functions with + `@convertToReturnValue`. + + :param result: Result of a DIRAC function which returns `S_OK`/`S_ERROR` + :returns: The value associated with the `S_OK` object + :raises: If `result["OK"]` is falsey the original exception is re-raised. + If no exception is known an :exc:`SErrorException` is raised. + """ + if not result["OK"]: + if "ExecInfo" in result: + raise result["ExecInfo"][0] + else: + raise SErrorException(result, errorCode) + return result["Value"] + + +def convertToReturnValue(func: Callable[P, T]) -> Callable[P, DReturnType[T]]: + """Decorate a function to convert return values to `S_OK`/`S_ERROR` + + If `func` returns, wrap the return value in `S_OK`. + If `func` raises :exc:`SErrorException`, return the associated `S_ERROR` + If `func` raises any other exception type, convert it to an `S_ERROR` object + + :param result: The bare result of a function call + :returns: `S_OK`/`S_ERROR` + """ + + @functools.wraps(func) + def wrapped(*args: P.args, **kwargs: P.kwargs) -> DReturnType[T]: + try: + value = func(*args, **kwargs) + except SErrorException as e: + return e.result + except Exception as e: + retval = S_ERROR(f"{repr(e)}: {e}") + # Replace CallStack with the one from the exception + # Use cast as mypy doesn't understand that sys.exc_info can't return None in an exception block + retval["ExecInfo"] = cast(tuple[type[BaseException], BaseException, TracebackType], sys.exc_info()) + exc_type, exc_value, exc_tb = retval["ExecInfo"] + retval["CallStack"] = traceback.format_tb(exc_tb) + return retval + else: + return S_OK(value) + + # functools will copy the annotations. Since we change the return type + # we have to update it + wrapped.__annotations__["return"] = DReturnType + return wrapped diff --git a/dirac-common/src/DIRACCommon/Utils/__init__.py b/dirac-common/src/DIRACCommon/Utils/__init__.py new file mode 100644 index 00000000000..fc544308069 --- /dev/null +++ b/dirac-common/src/DIRACCommon/Utils/__init__.py @@ -0,0 +1,3 @@ +""" +DIRACCommon.Utils - Stateless utility functions +""" diff --git a/dirac-common/src/DIRACCommon/__init__.py b/dirac-common/src/DIRACCommon/__init__.py new file mode 100644 index 00000000000..3beda13b6d6 --- /dev/null +++ b/dirac-common/src/DIRACCommon/__init__.py @@ -0,0 +1,21 @@ +""" +DIRACCommon - Stateless utilities for DIRAC + +This package contains stateless utilities extracted from DIRAC that can be used +by DiracX and other projects without triggering DIRAC's global state initialization. + +The utilities here should not depend on: +- gConfig (Configuration system) +- gLogger (Global logging) +- gMonitor (Monitoring) +- Database connections +- Any other global state +""" + +import importlib.metadata + +try: + __version__ = importlib.metadata.version(__name__) +except importlib.metadata.PackageNotFoundError: + # package is not installed + __version__ = "Unknown" diff --git a/dirac-common/tests/Utils/test_DErrno.py b/dirac-common/tests/Utils/test_DErrno.py new file mode 100644 index 00000000000..fdc7ec328e1 --- /dev/null +++ b/dirac-common/tests/Utils/test_DErrno.py @@ -0,0 +1,50 @@ +"""Tests for DErrno module""" + +import pytest +from DIRACCommon.Utils import DErrno + + +def test_strerror(): + """Test strerror function""" + # Test DIRAC specific errors + assert DErrno.strerror(DErrno.ETYPE) == "Object Type Error" + assert DErrno.strerror(DErrno.EIMPERR) == "Failed to import library" + assert DErrno.strerror(DErrno.EOF) == "Cannot open file" + + # Test unknown error + assert "Unknown error" in DErrno.strerror(999999) + + # Test zero error + assert DErrno.strerror(0) == "Undefined error" + + # Test OS errors (should fall back to os.strerror) + # Error code 2 is usually "No such file or directory" on Unix + import errno + + assert DErrno.strerror(errno.ENOENT) == "No such file or directory" + + +def test_cmpError(): + """Test cmpError function""" + # Test with integer + assert DErrno.cmpError(DErrno.ETYPE, DErrno.ETYPE) is True + assert DErrno.cmpError(DErrno.ETYPE, DErrno.EIMPERR) is False + + # Test with string (old style) + assert DErrno.cmpError("Object Type Error", DErrno.ETYPE) is True + assert DErrno.cmpError("Some error with Object Type Error in it", DErrno.ETYPE) is True + assert DErrno.cmpError("Different error", DErrno.ETYPE) is False + + # Test with S_ERROR dictionary + error_dict = {"OK": False, "Message": "Object Type Error", "Errno": DErrno.ETYPE} + assert DErrno.cmpError(error_dict, DErrno.ETYPE) is True + + error_dict_no_errno = {"OK": False, "Message": "Object Type Error"} + assert DErrno.cmpError(error_dict_no_errno, DErrno.ETYPE) is True + + error_dict_wrong = {"OK": False, "Message": "Different error", "Errno": DErrno.EIMPERR} + assert DErrno.cmpError(error_dict_wrong, DErrno.ETYPE) is False + + # Test with invalid type + with pytest.raises(TypeError): + DErrno.cmpError([], DErrno.ETYPE) diff --git a/dirac-common/tests/Utils/test_ReturnValues.py b/dirac-common/tests/Utils/test_ReturnValues.py new file mode 100644 index 00000000000..ef1bef8ad7b --- /dev/null +++ b/dirac-common/tests/Utils/test_ReturnValues.py @@ -0,0 +1,73 @@ +import pytest + +from DIRACCommon.Utils.ReturnValues import S_OK, S_ERROR, SErrorException, convertToReturnValue, returnValueOrRaise + + +def test_Ok(): + retVal = S_OK("Hello world") + assert retVal["OK"] is True + assert retVal["Value"] == "Hello world" + + +def test_Error(): + retVal = S_ERROR("This is bad") + assert retVal["OK"] is False + assert retVal["Message"] == "This is bad" + callStack = "".join(retVal["CallStack"]) + assert "test_ReturnValues" in callStack + assert "test_Error" in callStack + + +def test_ErrorWithCustomTraceback(): + retVal = S_ERROR("This is bad", callStack=["My callstack"]) + assert retVal["OK"] is False + assert retVal["Message"] == "This is bad" + assert retVal["CallStack"] == ["My callstack"] + + +class CustomException(Exception): + pass + + +@convertToReturnValue +def _happyFunction(): + return {"12345": "Success"} + + +@convertToReturnValue +def _sadFunction(): + raise CustomException("I am sad") + return {} + + +@convertToReturnValue +def _verySadFunction(): + raise SErrorException("I am very sad") + + +@convertToReturnValue +def _sadButPreciseFunction(): + raise SErrorException("I am sad, yet precise", errCode=123) + + +def test_convertToReturnValue(): + retVal = _happyFunction() + assert retVal["OK"] is True + assert retVal["Value"] == {"12345": "Success"} + # Make sure exceptions are captured correctly + retVal = _sadFunction() + assert retVal["OK"] is False + assert "CustomException" in retVal["Message"] + # Make sure the exception is re-raised + with pytest.raises(CustomException): + returnValueOrRaise(_sadFunction()) + + retVal = _verySadFunction() + assert retVal["OK"] is False + assert retVal["Errno"] == 0 + assert retVal["Message"] == "I am very sad" + + retVal = _sadButPreciseFunction() + assert retVal["OK"] is False + assert retVal["Errno"] == 123 + assert "I am sad, yet precise" in retVal["Message"] diff --git a/dirac-common/tests/__init__.py b/dirac-common/tests/__init__.py new file mode 100644 index 00000000000..87956b78519 --- /dev/null +++ b/dirac-common/tests/__init__.py @@ -0,0 +1 @@ +"""DIRACCommon test suite""" diff --git a/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/editingCode.rst b/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/editingCode.rst index d91517f9cd7..6311943be55 100644 --- a/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/editingCode.rst +++ b/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/editingCode.rst @@ -169,6 +169,7 @@ The locally cloned source code can be installed inside your ``conda`` or ``DIRAC .. code-block:: bash :linenos: + pip install -e ./dirac-common # Install DIRACCommon first pip install -e .[testing] This creates an *editable* installation meaning any changes you make will be automatically discovered whenever you next ``import DIRAC``. Additionally the ``testing`` extra causes ``pip`` to install useful dependencies such as ``pytest`` and ``pycodestyle``. diff --git a/environment.yml b/environment.yml index 4f062b9f643..3c25b0c3c2e 100644 --- a/environment.yml +++ b/environment.yml @@ -108,5 +108,6 @@ dependencies: # It should eventually be part of DIRACGrid - git+https://github.com/DIRACGrid/tornado_m2crypto - -e .[server] + - -e ./dirac-common/ # Add diracdoctools - -e docs/ diff --git a/integration_tests.py b/integration_tests.py index 619bf6b705c..00931ce11a2 100755 --- a/integration_tests.py +++ b/integration_tests.py @@ -39,7 +39,9 @@ "INSTALLATION_BRANCH": "", "DEBUG": "Yes", } -DEFAULT_MODULES = {"DIRAC": Path(__file__).parent.absolute()} +DEFAULT_MODULES = { + "DIRAC": Path(__file__).parent.absolute(), +} # All services that have a FutureClient, but we *explicitly* deactivate # (for example if we did not finish to develop it) DIRACX_DISABLED_SERVICES = [ diff --git a/setup.cfg b/setup.cfg index f48db258a45..9d999dfbe3a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,6 +30,7 @@ install_requires = certifi cwltool diraccfg + DIRACCommon diracx-client >=v0.0.1a18 diracx-core >=v0.0.1a18 db12 diff --git a/src/DIRAC/Core/Utilities/DErrno.py b/src/DIRAC/Core/Utilities/DErrno.py index d8e2dc69889..5ebbddd34fd 100644 --- a/src/DIRAC/Core/Utilities/DErrno.py +++ b/src/DIRAC/Core/Utilities/DErrno.py @@ -37,320 +37,16 @@ DErrno.ERRX : ['An error message for ERRX that is specific to LHCb']} """ -import os import importlib import sys -from DIRAC.Core.Utilities.Extensions import extensionsByPriority - -# To avoid conflict, the error numbers should be greater than 1000 -# We decided to group the by range of 100 per system - -# 1000: Generic -# 1100: Core -# 1200: Framework -# 1300: Interfaces -# 1400: Config -# 1500: WMS + Workflow -# 1600: DMS + StorageManagement -# 1700: RMS -# 1800: Accounting + Monitoring -# 1900: TS + Production -# 2000: Resources + RSS - -# ## Generic (10XX) -# Python related: 0X -ETYPE = 1000 -EIMPERR = 1001 -ENOMETH = 1002 -ECONF = 1003 -EVALUE = 1004 -EEEXCEPTION = 1005 -# Files manipulation: 1X -ECTMPF = 1010 -EOF = 1011 -ERF = 1012 -EWF = 1013 -ESPF = 1014 - -# ## Core (11XX) -# Certificates and Proxy: 0X -EX509 = 1100 -EPROXYFIND = 1101 -EPROXYREAD = 1102 -ECERTFIND = 1103 -ECERTREAD = 1104 -ENOCERT = 1105 -ENOCHAIN = 1106 -ENOPKEY = 1107 -ENOGROUP = 1108 -# DISET: 1X -EDISET = 1110 -ENOAUTH = 1111 -# 3rd party security: 2X -E3RDPARTY = 1120 -EVOMS = 1121 -# Databases : 3X -EDB = 1130 -EMYSQL = 1131 -ESQLA = 1132 -# Message Queues: 4X -EMQUKN = 1140 -EMQNOM = 1141 -EMQCONN = 1142 -# OpenSearch -EELNOFOUND = 1146 -# Tokens -EATOKENFIND = 1150 -EATOKENREAD = 1151 -ETOKENTYPE = 1152 - -# config -ESECTION = 1400 - -# processes -EEZOMBIE = 1147 -EENOPID = 1148 - -# ## WMS/Workflow -EWMSUKN = 1500 -EWMSJDL = 1501 -EWMSRESC = 1502 -EWMSSUBM = 1503 -EWMSJMAN = 1504 -EWMSSTATUS = 1505 -EWMSNOMATCH = 1510 -EWMSPLTVER = 1511 -EWMSNOPILOT = 1550 - -# ## DMS/StorageManagement (16XX) -EFILESIZE = 1601 -EGFAL = 1602 -EBADCKS = 1603 -EFCERR = 1604 - -# ## RMS (17XX) -ERMSUKN = 1700 - -# ## TS (19XX) -ETSUKN = 1900 -ETSDATA = 1901 - -# ## Resources and RSS (20XX) -ERESGEN = 2000 -ERESUNA = 2001 -ERESUNK = 2002 - -# This translates the integer number into the name of the variable -dErrorCode = { - # ## Generic (10XX) - # 100X: Python related - 1000: "ETYPE", - 1001: "EIMPERR", - 1002: "ENOMETH", - 1003: "ECONF", - 1004: "EVALUE", - 1005: "EEEXCEPTION", - # 101X: Files manipulation - 1010: "ECTMPF", - 1011: "EOF", - 1012: "ERF", - 1013: "EWF", - 1014: "ESPF", - # ## Core - # 110X: Certificates and Proxy - 1100: "EX509", - 1101: "EPROXYFIND", - 1102: "EPROXYREAD", - 1103: "ECERTFIND", - 1104: "ECERTREAD", - 1105: "ENOCERT", - 1106: "ENOCHAIN", - 1107: "ENOPKEY", - 1108: "ENOGROUP", - # 111X: DISET - 1110: "EDISET", - 1111: "ENOAUTH", - # 112X: 3rd party security - 1120: "E3RDPARTY", - 1121: "EVOMS", - # 113X: Databases - 1130: "EDB", - 1131: "EMYSQL", - 1132: "ESQLA", - # 114X: Message Queues - 1140: "EMQUKN", - 1141: "EMQNOM", - 1142: "EMQCONN", - # OpenSearch - 1146: "EELNOFOUND", - # 115X: Tokens - 1150: "EATOKENFIND", - 1151: "EATOKENREAD", - 1152: "ETOKENTYPE", - # Config - 1400: "ESECTION", - # Processes - 1147: "EEZOMBIE", - 1148: "EENOPID", - # WMS/Workflow - 1500: "EWMSUKN", - 1501: "EWMSJDL", - 1502: "EWMSRESC", - 1503: "EWMSSUBM", - 1504: "EWMSJMAN", - 1505: "EWMSSTATUS", - 1510: "EWMSNOMATCH", - 1511: "EWMSPLTVER", - 1550: "EWMSNOPILOT", - # DMS/StorageManagement - 1601: "EFILESIZE", - 1602: "EGFAL", - 1603: "EBADCKS", - 1604: "EFCERR", - # RMS - 1700: "ERMSUKN", - # Resources and RSS - 2000: "ERESGEN", - 2001: "ERESUNA", - 2002: "ERESUNK", - # TS - 1900: "ETSUKN", - 1901: "ETSDATA", -} - - -dStrError = { # Generic (10XX) - # 100X: Python related - ETYPE: "Object Type Error", - EIMPERR: "Failed to import library", - ENOMETH: "No such method or function", - ECONF: "Configuration error", - EVALUE: "Wrong value passed", - EEEXCEPTION: "runtime general exception", - # 101X: Files manipulation - ECTMPF: "Failed to create temporary file", - EOF: "Cannot open file", - ERF: "Cannot read from file", - EWF: "Cannot write to file", - ESPF: "Cannot set permissions to file", - # ## Core - # 110X: Certificates and Proxy - EX509: "Generic Error with X509", - EPROXYFIND: "Can't find proxy", - EPROXYREAD: "Can't read proxy", - ECERTFIND: "Can't find certificate", - ECERTREAD: "Can't read certificate", - ENOCERT: "No certificate loaded", - ENOCHAIN: "No chain loaded", - ENOPKEY: "No private key loaded", - ENOGROUP: "No DIRAC group", - # 111X: DISET - EDISET: "DISET Error", - ENOAUTH: "Unauthorized query", - # 112X: 3rd party security - E3RDPARTY: "3rd party security service error", - EVOMS: "VOMS Error", - # 113X: Databases - EDB: "Database Error", - EMYSQL: "MySQL Error", - ESQLA: "SQLAlchemy Error", - # 114X: Message Queues - EMQUKN: "Unknown MQ Error", - EMQNOM: "No messages", - EMQCONN: "MQ connection failure", - # 114X OpenSearch - EELNOFOUND: "Index not found", - # 115X: Tokens - EATOKENFIND: "Can't find a bearer access token.", - EATOKENREAD: "Can't read a bearer access token.", - ETOKENTYPE: "Unsupported access token type.", - # Config - ESECTION: "Section is not found", - # processes - EEZOMBIE: "Zombie process", - EENOPID: "No PID of process", - # WMS/Workflow - EWMSUKN: "Unknown WMS error", - EWMSJDL: "Invalid job description", - EWMSRESC: "Job to reschedule", - EWMSSUBM: "Job submission error", - EWMSJMAN: "Job management error", - EWMSSTATUS: "Job status error", - EWMSNOPILOT: "No pilots found", - EWMSPLTVER: "Pilot version does not match", - EWMSNOMATCH: "No match found", - # DMS/StorageManagement - EFILESIZE: "Bad file size", - EGFAL: "Error with the gfal call", - EBADCKS: "Bad checksum", - EFCERR: "FileCatalog error", - # RMS - ERMSUKN: "Unknown RMS error", - # Resources and RSS - ERESGEN: "Unknown Resource Failure", - ERESUNA: "Resource not available", - ERESUNK: "Unknown Resource", - # TS - ETSUKN: "Unknown Transformation System Error", - ETSDATA: "Invalid Input Data definition", -} +# Import all the stateless parts from DIRACCommon +from DIRACCommon.Utils.DErrno import * # noqa: F401, F403 +from DIRAC.Core.Utilities.Extensions import extensionsByPriority -def strerror(code: int) -> str: - """This method wraps up os.strerror, and behave the same way. - It completes it with the DIRAC specific errors. - """ - - if code == 0: - return "Undefined error" - - errMsg = f"Unknown error {code}" - - try: - errMsg = dStrError[code] - except KeyError: - # It is not a DIRAC specific error, try the os one - try: - errMsg = os.strerror(code) - # On some system, os.strerror raises an exception with unknown code, - # on others, it returns a message... - except ValueError: - pass - - return errMsg - - -def cmpError(inErr, candidate): - """This function compares an error (in its old form (a string or dictionary) or in its int form - with a candidate error code. - - :param inErr: a string, an integer, a S_ERROR dictionary - :type inErr: str or int or S_ERROR - :param int candidate: error code to compare with - - :return: True or False - - If an S_ERROR instance is passed, we compare the code with S_ERROR['Errno'] - If it is a Integer, we do a direct comparison - If it is a String, we use strerror to check the error string - """ - - if isinstance(inErr, str): # old style - # Compare error message strings - errMsg = strerror(candidate) - return errMsg in inErr - elif isinstance(inErr, dict): # if the S_ERROR structure is given - # Check if Errno defined in the dict - errorNumber = inErr.get("Errno") - if errorNumber: - return errorNumber == candidate - errMsg = strerror(candidate) - return errMsg in inErr.get("Message", "") - elif isinstance(inErr, int): - return inErr == candidate - else: - raise TypeError(f"Unknown input error type {type(inErr)}") +# compatErrorString is used by the extension mechanism but not in DIRACCommon +compatErrorString = {} def includeExtensionErrors(): diff --git a/src/DIRAC/Core/Utilities/ReturnValues.py b/src/DIRAC/Core/Utilities/ReturnValues.py index 6312df1d89a..02e083ffafc 100755 --- a/src/DIRAC/Core/Utilities/ReturnValues.py +++ b/src/DIRAC/Core/Utilities/ReturnValues.py @@ -1,255 +1,10 @@ -""" - DIRAC return dictionary +"""Backward compatibility wrapper - moved to DIRACCommon - Message values are converted to string +This module has been moved to DIRACCommon.Utils.ReturnValues to avoid +circular dependencies and allow DiracX to use these utilities without +triggering DIRAC's global state initialization. - keys are converted to string +All exports are maintained for backward compatibility. """ -from __future__ import annotations - -import functools -import sys -import traceback -from types import TracebackType -from typing import Any, Callable, cast, Generic, Literal, overload, Type, TypeVar, Union -from typing_extensions import TypedDict, ParamSpec, NotRequired - -from DIRAC.Core.Utilities.DErrno import strerror - - -T = TypeVar("T") -P = ParamSpec("P") - - -class DOKReturnType(TypedDict, Generic[T]): - """used for typing the DIRAC return structure""" - - OK: Literal[True] - Value: T - - -class DErrorReturnType(TypedDict): - """used for typing the DIRAC return structure""" - - OK: Literal[False] - Message: str - Errno: int - ExecInfo: NotRequired[tuple[type[BaseException], BaseException, TracebackType]] - CallStack: NotRequired[list[str]] - - -DReturnType = Union[DOKReturnType[T], DErrorReturnType] - - -def S_ERROR(*args: Any, **kwargs: Any) -> DErrorReturnType: - """return value on error condition - - Arguments are either Errno and ErrorMessage or just ErrorMessage fro backward compatibility - - :param int errno: Error number - :param string message: Error message - :param list callStack: Manually override the CallStack attribute better performance - """ - callStack = kwargs.pop("callStack", None) - - result: DErrorReturnType = {"OK": False, "Errno": 0, "Message": ""} - - message = "" - if args: - if isinstance(args[0], int): - result["Errno"] = args[0] - if len(args) > 1: - message = args[1] - else: - message = args[0] - - if result["Errno"]: - message = f"{strerror(result['Errno'])} ( {result['Errno']} : {message})" - result["Message"] = message - - if callStack is None: - try: - callStack = traceback.format_stack() - callStack.pop() - except Exception: - callStack = [] - - result["CallStack"] = callStack - - return result - - -# mypy doesn't understand default parameter values with generics so use overloads (python/mypy#3737) -@overload -def S_OK() -> DOKReturnType[None]: - ... - - -@overload -def S_OK(value: T) -> DOKReturnType[T]: - ... - - -def S_OK(value=None): # type: ignore - """return value on success - - :param value: value of the 'Value' - :return: dictionary { 'OK' : True, 'Value' : value } - """ - return {"OK": True, "Value": value} - - -def isReturnStructure(unk: Any) -> bool: - """Check if value is an `S_OK`/`S_ERROR` object""" - if not isinstance(unk, dict): - return False - if "OK" not in unk: - return False - if unk["OK"]: - return "Value" in unk - else: - return "Message" in unk - - -def isSError(value: Any) -> bool: - """Check if value is an `S_ERROR` object""" - if not isinstance(value, dict): - return False - if "OK" not in value: - return False - return "Message" in value - - -def reprReturnErrorStructure(struct: DErrorReturnType, full: bool = False) -> str: - errorNumber = struct.get("Errno", 0) - message = struct.get("Message", "") - if errorNumber: - reprStr = f"{strerror(errorNumber)} ( {errorNumber} : {message})" - else: - reprStr = message - - if full: - callStack = struct.get("CallStack") - if callStack: - reprStr += "\n" + "".join(callStack) - - return reprStr - - -def returnSingleResult(dictRes: DReturnType[Any]) -> DReturnType[Any]: - """Transform the S_OK{Successful/Failed} dictionary convention into - an S_OK/S_ERROR return. To be used when a single returned entity - is expected from a generally bulk call. - - :param dictRes: S_ERROR or S_OK( "Failed" : {}, "Successful" : {}) - :returns: S_ERROR or S_OK(value) - - The following rules are applied: - - - if dictRes is an S_ERROR: returns it as is - - we start by looking at the Failed directory - - if there are several items in a dictionary, we return the first one - - if both dictionaries are empty, we return S_ERROR - - For an item in Failed, we return S_ERROR - - Far an item in Successful we return S_OK - - Behavior examples (would be perfect unit test :-) ):: - - {'Message': 'Kaput', 'OK': False} -> {'Message': 'Kaput', 'OK': False} - {'OK': True, 'Value': {'Successful': {}, 'Failed': {'a': 1}}} -> {'Message': '1', 'OK': False} - {'OK': True, 'Value': {'Successful': {'b': 2}, 'Failed': {}}} -> {'OK': True, 'Value': 2} - {'OK': True, 'Value': {'Successful': {'b': 2}, 'Failed': {'a': 1}}} -> {'Message': '1', 'OK': False} - {'OK': True, 'Value': {'Successful': {'b': 2}, 'Failed': {'a': 1, 'c': 3}}} -> {'Message': '1', 'OK': False} - {'OK': True, 'Value': {'Successful': {'b': 2, 'd': 4}, 'Failed': {}}} -> {'OK': True, 'Value': 2} - {'OK': True, 'Value': {'Successful': {}, 'Failed': {}}} -> - {'Message': 'returnSingleResult: Failed and Successful dictionaries are empty', 'OK': False} - """ - # if S_ERROR was returned, we return it as well - if not dictRes["OK"]: - return dictRes - # if there is a Failed, we return the first one in an S_ERROR - if "Failed" in dictRes["Value"] and len(dictRes["Value"]["Failed"]): - errorMessage = list(dictRes["Value"]["Failed"].values())[0] - if isinstance(errorMessage, dict): - if isReturnStructure(errorMessage): - return cast(DErrorReturnType, errorMessage) - else: - return S_ERROR(str(errorMessage)) - return S_ERROR(errorMessage) - # if there is a Successful, we return the first one in an S_OK - elif "Successful" in dictRes["Value"] and len(dictRes["Value"]["Successful"]): - return S_OK(list(dictRes["Value"]["Successful"].values())[0]) - else: - return S_ERROR("returnSingleResult: Failed and Successful dictionaries are empty") - - -class SErrorException(Exception): - """Exception class for use with `convertToReturnValue`""" - - def __init__(self, result: DErrorReturnType | str, errCode: int = 0): - """Create a new exception return value - - If `result` is a `S_ERROR` return it directly else convert it to an - appropriate value using `S_ERROR(errCode, result)`. - - :param result: The error to propagate - :param errCode: the error code to propagate - """ - if not isSError(result): - result = S_ERROR(errCode, result) - self.result = cast(DErrorReturnType, result) - - -def returnValueOrRaise(result: DReturnType[T], *, errorCode: int = 0) -> T: - """Unwrap an S_OK/S_ERROR response into a value or Exception - - This method assists with using exceptions in DIRAC code by raising - :exc:`SErrorException` if `result` is an error. This can then by propagated - automatically as an `S_ERROR` by wrapping public facing functions with - `@convertToReturnValue`. - - :param result: Result of a DIRAC function which returns `S_OK`/`S_ERROR` - :returns: The value associated with the `S_OK` object - :raises: If `result["OK"]` is falsey the original exception is re-raised. - If no exception is known an :exc:`SErrorException` is raised. - """ - if not result["OK"]: - if "ExecInfo" in result: - raise result["ExecInfo"][0] - else: - raise SErrorException(result, errorCode) - return result["Value"] - - -def convertToReturnValue(func: Callable[P, T]) -> Callable[P, DReturnType[T]]: - """Decorate a function to convert return values to `S_OK`/`S_ERROR` - - If `func` returns, wrap the return value in `S_OK`. - If `func` raises :exc:`SErrorException`, return the associated `S_ERROR` - If `func` raises any other exception type, convert it to an `S_ERROR` object - - :param result: The bare result of a function call - :returns: `S_OK`/`S_ERROR` - """ - - @functools.wraps(func) - def wrapped(*args: P.args, **kwargs: P.kwargs) -> DReturnType[T]: - try: - value = func(*args, **kwargs) - except SErrorException as e: - return e.result - except Exception as e: - retval = S_ERROR(f"{repr(e)}: {e}") - # Replace CallStack with the one from the exception - # Use cast as mypy doesn't understand that sys.exc_info can't return None in an exception block - retval["ExecInfo"] = cast(tuple[type[BaseException], BaseException, TracebackType], sys.exc_info()) - exc_type, exc_value, exc_tb = retval["ExecInfo"] - retval["CallStack"] = traceback.format_tb(exc_tb) - return retval - else: - return S_OK(value) - - # functools will copy the annotations. Since we change the return type - # we have to update it - wrapped.__annotations__["return"] = DReturnType - return wrapped +# Re-export everything from DIRACCommon for backward compatibility +from DIRACCommon.Utils.ReturnValues import * # noqa: F401, F403 diff --git a/tests/.dirac-ci-config.yaml b/tests/.dirac-ci-config.yaml index 87fd088a218..84b722df09d 100644 --- a/tests/.dirac-ci-config.yaml +++ b/tests/.dirac-ci-config.yaml @@ -4,7 +4,7 @@ config: CLIENT_UPLOAD_BASE64: SSBsaWtlIHBpenphIQo= CLIENT_UPLOAD_LFN: LFN:/vo/test_lfn.txt CLIENT_UPLOAD_FILE: test_lfn.txt - PILOT_INSTALLATION_COMMAND: dirac-pilot.py --modules /home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC -M 2 -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --pilotUUID=whatever12345 --CVMFS_locations=/home/dirac/ -o diracInstallOnly --wnVO=vo --debug + PILOT_INSTALLATION_COMMAND: dirac-pilot.py --modules /home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC/dirac-common,/home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC -M 2 -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --pilotUUID=whatever12345 --CVMFS_locations=/home/dirac/ -o diracInstallOnly --wnVO=vo --debug PILOT_JSON: "{ \"timestamp\": \"2023-02-13T14:34:26.725499\", \"CEs\": { diff --git a/tests/Jenkins/dirac_ci.sh b/tests/Jenkins/dirac_ci.sh index db60c7b9f62..87e67db2b66 100644 --- a/tests/Jenkins/dirac_ci.sh +++ b/tests/Jenkins/dirac_ci.sh @@ -135,7 +135,14 @@ installSite() { ln -s "${SERVERINSTALLDIR}/diracos/etc" "${SERVERINSTALLDIR}/etc" source diracos/diracosrc for module_path in "${ALTERNATIVE_MODULES[@]}"; do - pip install ${PIP_INSTALL_EXTRA_ARGS:-} "${module_path}[server]" + # Special handling for DIRAC with DIRACCommon subdirectory + if [[ -d "${module_path}/dirac-common" ]]; then + # Install DIRACCommon first from within the DIRAC repo context to preserve git versioning + pip install ${PIP_INSTALL_EXTRA_ARGS:-} "${module_path}/dirac-common" + pip install ${PIP_INSTALL_EXTRA_ARGS:-} "${module_path}[server]" + else + pip install ${PIP_INSTALL_EXTRA_ARGS:-} "${module_path}[server]" + fi done cd - diff --git a/tests/Jenkins/utilities.sh b/tests/Jenkins/utilities.sh index a911f95dd12..f61defbba3d 100644 --- a/tests/Jenkins/utilities.sh +++ b/tests/Jenkins/utilities.sh @@ -301,7 +301,14 @@ installDIRAC() { pip install "git+https://github.com/DIRACGrid/DIRAC.git@${INSTALLATION_BRANCH}#egg=DIRAC[client]" else for module_path in "${ALTERNATIVE_MODULES[@]}"; do - pip install ${PIP_INSTALL_EXTRA_ARGS:-} "${module_path}" + # Special handling for DIRAC with DIRACCommon subdirectory + if [[ -d "${module_path}/dirac-common" ]]; then + # Install DIRACCommon first from within the DIRAC repo context to preserve git versioning + pip install ${PIP_INSTALL_EXTRA_ARGS:-} "${module_path}/dirac-common" + pip install ${PIP_INSTALL_EXTRA_ARGS:-} "${module_path}" + else + pip install ${PIP_INSTALL_EXTRA_ARGS:-} "${module_path}" + fi done fi