From a45364c8adbd26d9b1b7bb1ce7d25baeec4cefa4 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Mon, 22 Sep 2025 12:41:00 +0200 Subject: [PATCH 1/7] feat: removed python2 support --- .github/workflows/basic.yml | 3 - .pylintrc | 4 - Pilot/dirac-pilot.py | 14 +-- Pilot/pilotCommands.py | 29 ++---- Pilot/pilotTools.py | 122 ++++++-------------------- Pilot/proxyTools.py | 2 - Pilot/tests/Test_Pilot.py | 10 ++- Pilot/tests/Test_proxyTools.py | 12 +-- Pilot/tests/Test_simplePilotLogger.py | 27 ++---- Untitled | 11 +++ environment.yml | 1 - 11 files changed, 60 insertions(+), 175 deletions(-) create mode 100644 Untitled diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index e0f9221b..629d740e 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -24,7 +24,6 @@ jobs: fail-fast: false matrix: python: - - 2.7.18 - 3.6.15 - 3.9.17 @@ -55,7 +54,6 @@ jobs: fail-fast: false matrix: python: - - 2.7.18 - 3.6.15 - 3.9.17 @@ -82,7 +80,6 @@ jobs: fail-fast: false matrix: python: - - 2.7.18 - 3.6.15 - 3.9.17 diff --git a/.pylintrc b/.pylintrc index 2bae85fd..7c158f91 100644 --- a/.pylintrc +++ b/.pylintrc @@ -18,7 +18,3 @@ dummy-variables=_ disable= invalid-name, line-too-long, # would be nice to remove this one - consider-using-f-string, # python2/3 support - unspecified-encoding, # python2/3 support - super-with-arguments, # python2/3 support - redefined-builtin, # python2/3 support \ No newline at end of file diff --git a/Pilot/dirac-pilot.py b/Pilot/dirac-pilot.py index 9c434c97..a2858c42 100644 --- a/Pilot/dirac-pilot.py +++ b/Pilot/dirac-pilot.py @@ -19,19 +19,10 @@ But, as said, all the actions are actually configurable. """ -from __future__ import absolute_import, division, print_function - import os import sys import time - -############################ -# python 2 -> 3 "hacks" - -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO +from io import StringIO try: from Pilot.pilotTools import ( @@ -41,7 +32,7 @@ getCommand, pythonPathCheck, ) -except ImportError: +except ModuleNotFoundError: from pilotTools import ( Logger, PilotParams, @@ -49,6 +40,7 @@ getCommand, pythonPathCheck, ) + ############################ if __name__ == "__main__": diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index 4815f44b..3d3b460d 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -17,8 +17,6 @@ def __init__(self, pilotParams): execution. """ -from __future__ import absolute_import, division, print_function - import filecmp import os import platform @@ -28,22 +26,9 @@ def __init__(self, pilotParams): import sys import time import traceback -import subprocess from collections import Counter - -############################ -# python 2 -> 3 "hacks" -try: - # For Python 3.0 and later - from http.client import HTTPSConnection -except ImportError: - # Fall back to Python 2 - from httplib import HTTPSConnection - -try: - from shlex import quote -except ImportError: - from pipes import quote +from http.client import HTTPSConnection +from shlex import quote try: from Pilot.pilotTools import ( @@ -53,7 +38,7 @@ def __init__(self, pilotParams): safe_listdir, sendMessage, ) -except ImportError: +except ModuleNotFoundError: from pilotTools import ( CommandBase, getSubmitterInfo, @@ -61,6 +46,7 @@ def __init__(self, pilotParams): safe_listdir, sendMessage, ) + ############################ @@ -283,7 +269,7 @@ def _getPreinstalledEnvScript(self): self.pp.installEnv["DIRAC_RC_PATH"] = preinstalledEnvScript def _localInstallDIRAC(self): - """Install python3 version of DIRAC client""" + """Install DIRAC client""" self.log.info("Installing DIRAC locally") @@ -296,10 +282,7 @@ def _localInstallDIRAC(self): # 1. Get the DIRACOS installer name # curl -O -L https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/DIRACOS-Linux-$(uname -m).sh - try: - machine = os.uname().machine # py3 - except AttributeError: - machine = os.uname()[4] # py2 + machine = os.uname().machine installerName = "DIRACOS-Linux-%s.sh" % machine diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 8afe0f62..04cd78cc 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -1,9 +1,8 @@ """A set of common tools to be used in pilot commands""" -from __future__ import absolute_import, division, print_function - import fcntl import getopt +import importlib.util import json import os import re @@ -16,81 +15,26 @@ import warnings from datetime import datetime from functools import partial, wraps -from threading import RLock - -############################ -# python 2 -> 3 "hacks" -try: - from urllib.error import HTTPError, URLError - from urllib.parse import urlencode - from urllib.request import urlopen -except ImportError: - from urllib import urlencode - - from urllib2 import HTTPError, URLError, urlopen - -try: - import importlib.util - from importlib import import_module - - def load_module_from_path(module_name, path_to_module): - spec = importlib.util.spec_from_file_location(module_name, path_to_module) # pylint: disable=no-member - module = importlib.util.module_from_spec(spec) # pylint: disable=no-member - spec.loader.exec_module(module) - return module - -except ImportError: - - def import_module(module): - import imp - - impData = imp.find_module(module) - return imp.load_module(module, *impData) - - def load_module_from_path(module_name, path_to_module): - import imp - - fp, pathname, description = imp.find_module(module_name, [path_to_module]) - try: - return imp.load_module(module_name, fp, pathname, description) - finally: - if fp: - fp.close() - - -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - -try: - basestring # pylint: disable=used-before-assignment -except NameError: - basestring = str +from importlib import import_module +from io import StringIO +from threading import RLock, Timer +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import urlopen try: from Pilot.proxyTools import getVO -except ImportError: +except ModuleNotFoundError: from proxyTools import getVO -try: - FileNotFoundError # pylint: disable=used-before-assignment - # because of https://github.com/PyCQA/pylint/issues/6748 -except NameError: - FileNotFoundError = OSError - -try: - IsADirectoryError # pylint: disable=used-before-assignment -except NameError: - IsADirectoryError = IOError +# Utilities functions -# Timer 2.7 and < 3.3 versions issue where Timer is a function -if sys.version_info.major == 2 or sys.version_info.major == 3 and sys.version_info.minor < 3: - from threading import _Timer as Timer # pylint: disable=no-name-in-module -else: - from threading import Timer -# Utilities functions +def load_module_from_path(module_name, path_to_module): + spec = importlib.util.spec_from_file_location(module_name, path_to_module) # pylint: disable=no-member + module = importlib.util.module_from_spec(spec) # pylint: disable=no-member + spec.loader.exec_module(module) + return module def parseVersion(releaseVersion): @@ -399,7 +343,7 @@ def loadModule(self, modName, hideExceptions=False): def __recurseImport(self, modName, parentModule=None, hideExceptions=False): """Internal function to load modules""" - if isinstance(modName, basestring): + if isinstance(modName, str): modName = modName.split(".") try: if parentModule: @@ -713,11 +657,7 @@ def sendMessage(url, pilotUUID, wnVO, method, rawMessage): context.load_cert_chain(os.path.join(cert, "hostcert.pem"), os.path.join(cert, "hostkey.pem")) raw_data = {"method": method, "args": message, "extraCredentials": '"hosts"'} - if sys.version_info.major == 3: - data = urlencode(raw_data).encode("utf-8") # encode to bytes ! for python3 - else: - # Python2 - data = urlencode(raw_data) + data = urlencode(raw_data).encode("utf-8") # encode to bytes res = urlopen(url, data, context=context) res.close() @@ -787,17 +727,7 @@ def executeAndGetOutput(self, cmd, environDict=None): if not outChunk: continue dataWasRead = True - if sys.version_info.major == 2: - # Ensure outChunk is unicode in Python 2 - if isinstance(outChunk, str): - outChunk = outChunk.decode("utf-8") - # Strip unicode replacement characters - # Ensure correct type conversion in Python 2 - outChunk = str(outChunk.replace(u"\ufffd", "")) - # Avoid potential str() issues in Py2 - outChunk = unicode(outChunk) # pylint: disable=undefined-variable - else: - outChunk = str(outChunk.replace("\ufffd", "")) # Python 3: Ensure it's a string + outChunk = str(outChunk.replace("\ufffd", "")) # Ensure it's a string if stream == _p.stderr: sys.stderr.write(outChunk) @@ -1455,7 +1385,7 @@ def __initJSON(self): # Commands first # FIXME: pilotSynchronizer() should publish these as comma-separated lists. We are ready for that. try: - if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType], basestring): + if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType], str): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType].split(",") @@ -1466,7 +1396,7 @@ def __initJSON(self): ] except KeyError: try: - if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"], basestring): + if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"], str): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"].split(",") @@ -1477,7 +1407,7 @@ def __initJSON(self): ] except KeyError: try: - if isinstance(self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType], basestring): + if isinstance(self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType], str): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType].split(",") @@ -1488,7 +1418,7 @@ def __initJSON(self): ] except KeyError: try: - if isinstance(self.pilotJSON["Defaults"]["Commands"]["Defaults"], basestring): + if isinstance(self.pilotJSON["Defaults"]["Commands"]["Defaults"], str): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Defaults"]["Commands"]["Defaults"].split(",") ] @@ -1504,7 +1434,7 @@ def __initJSON(self): # pilotSynchronizer() can publish this as a comma separated list. We are ready for that. try: if isinstance( - self.pilotJSON["Setups"][self.setup]["CommandExtensions"], basestring + self.pilotJSON["Setups"][self.setup]["CommandExtensions"], str ): # In the specific setup? self.commandExtensions = [ str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["CommandExtensions"].split(",") @@ -1516,7 +1446,7 @@ def __initJSON(self): except KeyError: try: if isinstance( - self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"], basestring + self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"], str ): # Or in the defaults section? self.commandExtensions = [ str(pv).strip() for pv in self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"].split(",") @@ -1533,7 +1463,7 @@ def __initJSON(self): # pilotSynchronizer() can publish this as a comma separated list. We are ready for that try: if isinstance( - self.pilotJSON["ConfigurationServers"], basestring + self.pilotJSON["ConfigurationServers"], str ): # Generic, there may also be setup-specific ones self.configServer = ",".join( [str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"].split(",")] @@ -1544,7 +1474,7 @@ def __initJSON(self): pass try: # now trying to see if there is setup-specific ones if isinstance( - self.pilotJSON["Setups"][self.setup]["ConfigurationServer"], basestring + self.pilotJSON["Setups"][self.setup]["ConfigurationServer"], str ): # In the specific setup? self.configServer = ",".join( [str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["ConfigurationServer"].split(",")] @@ -1556,7 +1486,7 @@ def __initJSON(self): except KeyError: # and if it doesn't exist try: if isinstance( - self.pilotJSON["Setups"]["Defaults"]["ConfigurationServer"], basestring + self.pilotJSON["Setups"]["Defaults"]["ConfigurationServer"], str ): # Is there one in the defaults section? self.configServer = ",".join( [ diff --git a/Pilot/proxyTools.py b/Pilot/proxyTools.py index a5fa652e..8792a34a 100644 --- a/Pilot/proxyTools.py +++ b/Pilot/proxyTools.py @@ -1,7 +1,5 @@ """few functions for dealing with proxies""" -from __future__ import absolute_import, division, print_function - import re from base64 import b16decode from subprocess import PIPE, Popen diff --git a/Pilot/tests/Test_Pilot.py b/Pilot/tests/Test_Pilot.py index 8a1b75a1..db1286eb 100644 --- a/Pilot/tests/Test_Pilot.py +++ b/Pilot/tests/Test_Pilot.py @@ -1,7 +1,5 @@ """Test class for Pilot""" -from __future__ import absolute_import, division, print_function - import json import os import shutil @@ -12,8 +10,12 @@ # imports import unittest -from Pilot.pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes -from Pilot.pilotTools import PilotParams +try: + from Pilot.pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes + from Pilot.pilotTools import PilotParams +except ModuleNotFoundError: + from pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes + from pilotTools import PilotParams class PilotTestCase(unittest.TestCase): diff --git a/Pilot/tests/Test_proxyTools.py b/Pilot/tests/Test_proxyTools.py index 7a8688cb..48280277 100644 --- a/Pilot/tests/Test_proxyTools.py +++ b/Pilot/tests/Test_proxyTools.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import, division, print_function import os import shlex @@ -6,20 +5,13 @@ import subprocess import sys import unittest +from unittest.mock import patch -############################ -# python 2 -> 3 "hacks" try: from Pilot.proxyTools import getVO, parseASN1 -except ImportError: +except ModuleNotFoundError: from proxyTools import getVO, parseASN1 -try: - from unittest.mock import patch -except ImportError: - from mock import patch - - class TestProxyTools(unittest.TestCase): def test_getVO(self): vo = "unknown" diff --git a/Pilot/tests/Test_simplePilotLogger.py b/Pilot/tests/Test_simplePilotLogger.py index 1fc448ae..222eb4d2 100644 --- a/Pilot/tests/Test_simplePilotLogger.py +++ b/Pilot/tests/Test_simplePilotLogger.py @@ -1,27 +1,18 @@ #!/usr/bin/env python -from __future__ import absolute_import, division, print_function - import json import os import random import string -import sys import tempfile +import unittest +from unittest.mock import patch try: from Pilot.pilotTools import CommandBase, Logger, PilotParams -except ImportError: +except ModuleNotFoundError: from pilotTools import CommandBase, Logger, PilotParams -import unittest - -try: - from unittest.mock import patch -except ImportError: - from mock import patch - - class TestPilotParams(unittest.TestCase): @patch("sys.argv") def test_pilotParamsInit(self, argvmock): @@ -146,16 +137,10 @@ def test_executeAndGetOutput(self, popenMock, argvmock): for size in [1000, 1024, 1025, 2005]: random_str = "".join(random.choice(string.ascii_letters + "\n") for i in range(size)) - if sys.version_info.major == 3: - random_bytes = random_str.encode("UTF-8") - self.stdout_mock.write(random_bytes) - else: - self.stdout_mock.write(random_str) + random_bytes = random_str.encode("UTF-8") + self.stdout_mock.write(random_bytes) self.stdout_mock.seek(0) - if sys.version_info.major == 3: - self.stderr_mock.write("Errare humanum est!".encode("UTF-8")) - else: - self.stderr_mock.write("Errare humanum est!") + self.stderr_mock.write("Errare humanum est!".encode("UTF-8")) self.stderr_mock.seek(0) pp = PilotParams() diff --git a/Untitled b/Untitled new file mode 100644 index 00000000..10dd1dde --- /dev/null +++ b/Untitled @@ -0,0 +1,11 @@ + UPDATE sb_SandBoxes AS s + JOIN ( + SELECT + MAX(OwnerId) AS badId, + MIN(OwnerId) AS goodId + FROM sb_Owners + GROUP BY Owner, OwnerGroup, VO + HAVING COUNT(*) > 1 + ) AS xxx + ON s.OwnerId = xxx.badId + SET s.OwnerId = xxx.goodId; diff --git a/environment.yml b/environment.yml index 41e0a564..72e2765e 100644 --- a/environment.yml +++ b/environment.yml @@ -11,7 +11,6 @@ dependencies: - requests # testing and development - pycodestyle - - caniusepython3 - coverage - mock - pylint From 264587ce7287a9acd656842f29284b49958f2f9e Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Wed, 22 Oct 2025 12:51:03 +0200 Subject: [PATCH 2/7] fix: use absolute imports --- Pilot/dirac-pilot.py | 23 ++++-------- Pilot/pilotCommands.py | 11 +----- Pilot/pilotTools.py | 5 +-- Pilot/tests/Test_Pilot.py | 8 ++--- Pilot/tests/Test_proxyTools.py | 50 +++------------------------ Pilot/tests/Test_simplePilotLogger.py | 5 +-- Untitled | 11 ------ 7 files changed, 16 insertions(+), 97 deletions(-) delete mode 100644 Untitled diff --git a/Pilot/dirac-pilot.py b/Pilot/dirac-pilot.py index a2858c42..b3f0866c 100644 --- a/Pilot/dirac-pilot.py +++ b/Pilot/dirac-pilot.py @@ -24,22 +24,13 @@ import time from io import StringIO -try: - from Pilot.pilotTools import ( - Logger, - PilotParams, - RemoteLogger, - getCommand, - pythonPathCheck, - ) -except ModuleNotFoundError: - from pilotTools import ( - Logger, - PilotParams, - RemoteLogger, - getCommand, - pythonPathCheck, - ) +from .pilotTools import ( + Logger, + PilotParams, + RemoteLogger, + getCommand, + pythonPathCheck, +) ############################ diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index 3d3b460d..7411f4a5 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -30,16 +30,7 @@ def __init__(self, pilotParams): from http.client import HTTPSConnection from shlex import quote -try: - from Pilot.pilotTools import ( - CommandBase, - getSubmitterInfo, - retrieveUrlTimeout, - safe_listdir, - sendMessage, - ) -except ModuleNotFoundError: - from pilotTools import ( +from .pilotTools import ( CommandBase, getSubmitterInfo, retrieveUrlTimeout, diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 04cd78cc..7d9530ad 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -22,10 +22,7 @@ from urllib.parse import urlencode from urllib.request import urlopen -try: - from Pilot.proxyTools import getVO -except ModuleNotFoundError: - from proxyTools import getVO +from .proxyTools import getVO # Utilities functions diff --git a/Pilot/tests/Test_Pilot.py b/Pilot/tests/Test_Pilot.py index db1286eb..bf0f54c1 100644 --- a/Pilot/tests/Test_Pilot.py +++ b/Pilot/tests/Test_Pilot.py @@ -10,12 +10,8 @@ # imports import unittest -try: - from Pilot.pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes - from Pilot.pilotTools import PilotParams -except ModuleNotFoundError: - from pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes - from pilotTools import PilotParams +from ..pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes +from ..pilotTools import PilotParams class PilotTestCase(unittest.TestCase): diff --git a/Pilot/tests/Test_proxyTools.py b/Pilot/tests/Test_proxyTools.py index 48280277..dc6c3a95 100644 --- a/Pilot/tests/Test_proxyTools.py +++ b/Pilot/tests/Test_proxyTools.py @@ -7,10 +7,7 @@ import unittest from unittest.mock import patch -try: - from Pilot.proxyTools import getVO, parseASN1 -except ModuleNotFoundError: - from proxyTools import getVO, parseASN1 +from ..proxyTools import getVO, parseASN1 class TestProxyTools(unittest.TestCase): def test_getVO(self): @@ -84,46 +81,7 @@ def __createFakeProxy(self, proxyFile): """ Create a fake proxy locally. """ - basedir = os.path.dirname(__file__) - shutil.copy(basedir + "/certs/user/userkey.pem", basedir + "/certs/user/userkey400.pem") - os.chmod(basedir + "/certs/user/userkey400.pem", 0o400) - ret = self.createFakeProxy( - basedir + "/certs/user/usercert.pem", - basedir + "/certs/user/userkey400.pem", - "fakeserver.cern.ch:15000", - "fakevo", - basedir + "/certs//host/hostcert.pem", - basedir + "/certs/host/hostkey.pem", - basedir + "/certs/ca", - proxyFile, - ) - os.remove(basedir + "/certs/user/userkey400.pem") - return ret - - def createFakeProxy(self, usercert, userkey, serverURI, vo, hostcert, hostkey, CACertDir, proxyfile): - """ - voms-proxy-fake --cert usercert.pem - --key userkey.pem - -rfc - -fqan "/fakevo/Role=user/Capability=NULL" - -uri fakeserver.cern.ch:15000 - -voms fakevo - -hostcert hostcert.pem - -hostkey hostkey.pem - -certdir ca - """ - opt = ( - '--cert %s --key %s -rfc -fqan "/fakevo/Role=user/Capability=NULL" -uri %s -voms %s -hostcert %s' - " -hostkey %s -certdir %s -out %s" - % (usercert, userkey, serverURI, vo, hostcert, hostkey, CACertDir, proxyfile) - ) - proc = subprocess.Popen( - shlex.split("voms-proxy-fake " + opt), - bufsize=1, - stdout=sys.stdout, - stderr=sys.stderr, - universal_newlines=True, - ) - proc.communicate() - return proc.returncode + shutil.copy(basedir + "/certs/voms/proxy.pem", proxyFile) + return 0 + diff --git a/Pilot/tests/Test_simplePilotLogger.py b/Pilot/tests/Test_simplePilotLogger.py index 222eb4d2..aec1b191 100644 --- a/Pilot/tests/Test_simplePilotLogger.py +++ b/Pilot/tests/Test_simplePilotLogger.py @@ -8,10 +8,7 @@ import unittest from unittest.mock import patch -try: - from Pilot.pilotTools import CommandBase, Logger, PilotParams -except ModuleNotFoundError: - from pilotTools import CommandBase, Logger, PilotParams +from ..pilotTools import CommandBase, Logger, PilotParams class TestPilotParams(unittest.TestCase): @patch("sys.argv") diff --git a/Untitled b/Untitled deleted file mode 100644 index 10dd1dde..00000000 --- a/Untitled +++ /dev/null @@ -1,11 +0,0 @@ - UPDATE sb_SandBoxes AS s - JOIN ( - SELECT - MAX(OwnerId) AS badId, - MIN(OwnerId) AS goodId - FROM sb_Owners - GROUP BY Owner, OwnerGroup, VO - HAVING COUNT(*) > 1 - ) AS xxx - ON s.OwnerId = xxx.badId - SET s.OwnerId = xxx.goodId; From 388745b2d739f26c1bacc99d8799995f8f1196bd Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Fri, 24 Oct 2025 10:10:46 +0200 Subject: [PATCH 3/7] Revert "feat: removed python2 support" --- .github/workflows/basic.yml | 3 + .pylintrc | 4 + Pilot/dirac-pilot.py | 33 +++++-- Pilot/pilotCommands.py | 38 ++++++-- Pilot/pilotTools.py | 125 ++++++++++++++++++++------ Pilot/proxyTools.py | 2 + Pilot/tests/Test_Pilot.py | 6 +- Pilot/tests/Test_proxyTools.py | 60 +++++++++++-- Pilot/tests/Test_simplePilotLogger.py | 28 ++++-- environment.yml | 1 + 10 files changed, 248 insertions(+), 52 deletions(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 629d740e..e0f9221b 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -24,6 +24,7 @@ jobs: fail-fast: false matrix: python: + - 2.7.18 - 3.6.15 - 3.9.17 @@ -54,6 +55,7 @@ jobs: fail-fast: false matrix: python: + - 2.7.18 - 3.6.15 - 3.9.17 @@ -80,6 +82,7 @@ jobs: fail-fast: false matrix: python: + - 2.7.18 - 3.6.15 - 3.9.17 diff --git a/.pylintrc b/.pylintrc index 7c158f91..2bae85fd 100644 --- a/.pylintrc +++ b/.pylintrc @@ -18,3 +18,7 @@ dummy-variables=_ disable= invalid-name, line-too-long, # would be nice to remove this one + consider-using-f-string, # python2/3 support + unspecified-encoding, # python2/3 support + super-with-arguments, # python2/3 support + redefined-builtin, # python2/3 support \ No newline at end of file diff --git a/Pilot/dirac-pilot.py b/Pilot/dirac-pilot.py index b3f0866c..9c434c97 100644 --- a/Pilot/dirac-pilot.py +++ b/Pilot/dirac-pilot.py @@ -19,19 +19,36 @@ But, as said, all the actions are actually configurable. """ +from __future__ import absolute_import, division, print_function + import os import sys import time -from io import StringIO -from .pilotTools import ( - Logger, - PilotParams, - RemoteLogger, - getCommand, - pythonPathCheck, -) +############################ +# python 2 -> 3 "hacks" + +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO +try: + from Pilot.pilotTools import ( + Logger, + PilotParams, + RemoteLogger, + getCommand, + pythonPathCheck, + ) +except ImportError: + from pilotTools import ( + Logger, + PilotParams, + RemoteLogger, + getCommand, + pythonPathCheck, + ) ############################ if __name__ == "__main__": diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index 7411f4a5..4815f44b 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -17,6 +17,8 @@ def __init__(self, pilotParams): execution. """ +from __future__ import absolute_import, division, print_function + import filecmp import os import platform @@ -26,18 +28,39 @@ def __init__(self, pilotParams): import sys import time import traceback +import subprocess from collections import Counter -from http.client import HTTPSConnection -from shlex import quote -from .pilotTools import ( +############################ +# python 2 -> 3 "hacks" +try: + # For Python 3.0 and later + from http.client import HTTPSConnection +except ImportError: + # Fall back to Python 2 + from httplib import HTTPSConnection + +try: + from shlex import quote +except ImportError: + from pipes import quote + +try: + from Pilot.pilotTools import ( + CommandBase, + getSubmitterInfo, + retrieveUrlTimeout, + safe_listdir, + sendMessage, + ) +except ImportError: + from pilotTools import ( CommandBase, getSubmitterInfo, retrieveUrlTimeout, safe_listdir, sendMessage, ) - ############################ @@ -260,7 +283,7 @@ def _getPreinstalledEnvScript(self): self.pp.installEnv["DIRAC_RC_PATH"] = preinstalledEnvScript def _localInstallDIRAC(self): - """Install DIRAC client""" + """Install python3 version of DIRAC client""" self.log.info("Installing DIRAC locally") @@ -273,7 +296,10 @@ def _localInstallDIRAC(self): # 1. Get the DIRACOS installer name # curl -O -L https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/DIRACOS-Linux-$(uname -m).sh - machine = os.uname().machine + try: + machine = os.uname().machine # py3 + except AttributeError: + machine = os.uname()[4] # py2 installerName = "DIRACOS-Linux-%s.sh" % machine diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 7d9530ad..8afe0f62 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -1,8 +1,9 @@ """A set of common tools to be used in pilot commands""" +from __future__ import absolute_import, division, print_function + import fcntl import getopt -import importlib.util import json import os import re @@ -15,23 +16,81 @@ import warnings from datetime import datetime from functools import partial, wraps -from importlib import import_module -from io import StringIO -from threading import RLock, Timer -from urllib.error import HTTPError, URLError -from urllib.parse import urlencode -from urllib.request import urlopen +from threading import RLock -from .proxyTools import getVO +############################ +# python 2 -> 3 "hacks" +try: + from urllib.error import HTTPError, URLError + from urllib.parse import urlencode + from urllib.request import urlopen +except ImportError: + from urllib import urlencode -# Utilities functions + from urllib2 import HTTPError, URLError, urlopen + +try: + import importlib.util + from importlib import import_module + def load_module_from_path(module_name, path_to_module): + spec = importlib.util.spec_from_file_location(module_name, path_to_module) # pylint: disable=no-member + module = importlib.util.module_from_spec(spec) # pylint: disable=no-member + spec.loader.exec_module(module) + return module -def load_module_from_path(module_name, path_to_module): - spec = importlib.util.spec_from_file_location(module_name, path_to_module) # pylint: disable=no-member - module = importlib.util.module_from_spec(spec) # pylint: disable=no-member - spec.loader.exec_module(module) - return module +except ImportError: + + def import_module(module): + import imp + + impData = imp.find_module(module) + return imp.load_module(module, *impData) + + def load_module_from_path(module_name, path_to_module): + import imp + + fp, pathname, description = imp.find_module(module_name, [path_to_module]) + try: + return imp.load_module(module_name, fp, pathname, description) + finally: + if fp: + fp.close() + + +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + +try: + basestring # pylint: disable=used-before-assignment +except NameError: + basestring = str + +try: + from Pilot.proxyTools import getVO +except ImportError: + from proxyTools import getVO + +try: + FileNotFoundError # pylint: disable=used-before-assignment + # because of https://github.com/PyCQA/pylint/issues/6748 +except NameError: + FileNotFoundError = OSError + +try: + IsADirectoryError # pylint: disable=used-before-assignment +except NameError: + IsADirectoryError = IOError + +# Timer 2.7 and < 3.3 versions issue where Timer is a function +if sys.version_info.major == 2 or sys.version_info.major == 3 and sys.version_info.minor < 3: + from threading import _Timer as Timer # pylint: disable=no-name-in-module +else: + from threading import Timer + +# Utilities functions def parseVersion(releaseVersion): @@ -340,7 +399,7 @@ def loadModule(self, modName, hideExceptions=False): def __recurseImport(self, modName, parentModule=None, hideExceptions=False): """Internal function to load modules""" - if isinstance(modName, str): + if isinstance(modName, basestring): modName = modName.split(".") try: if parentModule: @@ -654,7 +713,11 @@ def sendMessage(url, pilotUUID, wnVO, method, rawMessage): context.load_cert_chain(os.path.join(cert, "hostcert.pem"), os.path.join(cert, "hostkey.pem")) raw_data = {"method": method, "args": message, "extraCredentials": '"hosts"'} - data = urlencode(raw_data).encode("utf-8") # encode to bytes + if sys.version_info.major == 3: + data = urlencode(raw_data).encode("utf-8") # encode to bytes ! for python3 + else: + # Python2 + data = urlencode(raw_data) res = urlopen(url, data, context=context) res.close() @@ -724,7 +787,17 @@ def executeAndGetOutput(self, cmd, environDict=None): if not outChunk: continue dataWasRead = True - outChunk = str(outChunk.replace("\ufffd", "")) # Ensure it's a string + if sys.version_info.major == 2: + # Ensure outChunk is unicode in Python 2 + if isinstance(outChunk, str): + outChunk = outChunk.decode("utf-8") + # Strip unicode replacement characters + # Ensure correct type conversion in Python 2 + outChunk = str(outChunk.replace(u"\ufffd", "")) + # Avoid potential str() issues in Py2 + outChunk = unicode(outChunk) # pylint: disable=undefined-variable + else: + outChunk = str(outChunk.replace("\ufffd", "")) # Python 3: Ensure it's a string if stream == _p.stderr: sys.stderr.write(outChunk) @@ -1382,7 +1455,7 @@ def __initJSON(self): # Commands first # FIXME: pilotSynchronizer() should publish these as comma-separated lists. We are ready for that. try: - if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType], str): + if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType], basestring): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType].split(",") @@ -1393,7 +1466,7 @@ def __initJSON(self): ] except KeyError: try: - if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"], str): + if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"], basestring): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"].split(",") @@ -1404,7 +1477,7 @@ def __initJSON(self): ] except KeyError: try: - if isinstance(self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType], str): + if isinstance(self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType], basestring): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType].split(",") @@ -1415,7 +1488,7 @@ def __initJSON(self): ] except KeyError: try: - if isinstance(self.pilotJSON["Defaults"]["Commands"]["Defaults"], str): + if isinstance(self.pilotJSON["Defaults"]["Commands"]["Defaults"], basestring): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Defaults"]["Commands"]["Defaults"].split(",") ] @@ -1431,7 +1504,7 @@ def __initJSON(self): # pilotSynchronizer() can publish this as a comma separated list. We are ready for that. try: if isinstance( - self.pilotJSON["Setups"][self.setup]["CommandExtensions"], str + self.pilotJSON["Setups"][self.setup]["CommandExtensions"], basestring ): # In the specific setup? self.commandExtensions = [ str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["CommandExtensions"].split(",") @@ -1443,7 +1516,7 @@ def __initJSON(self): except KeyError: try: if isinstance( - self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"], str + self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"], basestring ): # Or in the defaults section? self.commandExtensions = [ str(pv).strip() for pv in self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"].split(",") @@ -1460,7 +1533,7 @@ def __initJSON(self): # pilotSynchronizer() can publish this as a comma separated list. We are ready for that try: if isinstance( - self.pilotJSON["ConfigurationServers"], str + self.pilotJSON["ConfigurationServers"], basestring ): # Generic, there may also be setup-specific ones self.configServer = ",".join( [str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"].split(",")] @@ -1471,7 +1544,7 @@ def __initJSON(self): pass try: # now trying to see if there is setup-specific ones if isinstance( - self.pilotJSON["Setups"][self.setup]["ConfigurationServer"], str + self.pilotJSON["Setups"][self.setup]["ConfigurationServer"], basestring ): # In the specific setup? self.configServer = ",".join( [str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["ConfigurationServer"].split(",")] @@ -1483,7 +1556,7 @@ def __initJSON(self): except KeyError: # and if it doesn't exist try: if isinstance( - self.pilotJSON["Setups"]["Defaults"]["ConfigurationServer"], str + self.pilotJSON["Setups"]["Defaults"]["ConfigurationServer"], basestring ): # Is there one in the defaults section? self.configServer = ",".join( [ diff --git a/Pilot/proxyTools.py b/Pilot/proxyTools.py index 8792a34a..a5fa652e 100644 --- a/Pilot/proxyTools.py +++ b/Pilot/proxyTools.py @@ -1,5 +1,7 @@ """few functions for dealing with proxies""" +from __future__ import absolute_import, division, print_function + import re from base64 import b16decode from subprocess import PIPE, Popen diff --git a/Pilot/tests/Test_Pilot.py b/Pilot/tests/Test_Pilot.py index bf0f54c1..8a1b75a1 100644 --- a/Pilot/tests/Test_Pilot.py +++ b/Pilot/tests/Test_Pilot.py @@ -1,5 +1,7 @@ """Test class for Pilot""" +from __future__ import absolute_import, division, print_function + import json import os import shutil @@ -10,8 +12,8 @@ # imports import unittest -from ..pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes -from ..pilotTools import PilotParams +from Pilot.pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes +from Pilot.pilotTools import PilotParams class PilotTestCase(unittest.TestCase): diff --git a/Pilot/tests/Test_proxyTools.py b/Pilot/tests/Test_proxyTools.py index dc6c3a95..7a8688cb 100644 --- a/Pilot/tests/Test_proxyTools.py +++ b/Pilot/tests/Test_proxyTools.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, division, print_function import os import shlex @@ -5,9 +6,19 @@ import subprocess import sys import unittest -from unittest.mock import patch -from ..proxyTools import getVO, parseASN1 +############################ +# python 2 -> 3 "hacks" +try: + from Pilot.proxyTools import getVO, parseASN1 +except ImportError: + from proxyTools import getVO, parseASN1 + +try: + from unittest.mock import patch +except ImportError: + from mock import patch + class TestProxyTools(unittest.TestCase): def test_getVO(self): @@ -81,7 +92,46 @@ def __createFakeProxy(self, proxyFile): """ Create a fake proxy locally. """ - basedir = os.path.dirname(__file__) - shutil.copy(basedir + "/certs/voms/proxy.pem", proxyFile) - return 0 + basedir = os.path.dirname(__file__) + shutil.copy(basedir + "/certs/user/userkey.pem", basedir + "/certs/user/userkey400.pem") + os.chmod(basedir + "/certs/user/userkey400.pem", 0o400) + ret = self.createFakeProxy( + basedir + "/certs/user/usercert.pem", + basedir + "/certs/user/userkey400.pem", + "fakeserver.cern.ch:15000", + "fakevo", + basedir + "/certs//host/hostcert.pem", + basedir + "/certs/host/hostkey.pem", + basedir + "/certs/ca", + proxyFile, + ) + os.remove(basedir + "/certs/user/userkey400.pem") + return ret + + def createFakeProxy(self, usercert, userkey, serverURI, vo, hostcert, hostkey, CACertDir, proxyfile): + """ + voms-proxy-fake --cert usercert.pem + --key userkey.pem + -rfc + -fqan "/fakevo/Role=user/Capability=NULL" + -uri fakeserver.cern.ch:15000 + -voms fakevo + -hostcert hostcert.pem + -hostkey hostkey.pem + -certdir ca + """ + opt = ( + '--cert %s --key %s -rfc -fqan "/fakevo/Role=user/Capability=NULL" -uri %s -voms %s -hostcert %s' + " -hostkey %s -certdir %s -out %s" + % (usercert, userkey, serverURI, vo, hostcert, hostkey, CACertDir, proxyfile) + ) + proc = subprocess.Popen( + shlex.split("voms-proxy-fake " + opt), + bufsize=1, + stdout=sys.stdout, + stderr=sys.stderr, + universal_newlines=True, + ) + proc.communicate() + return proc.returncode diff --git a/Pilot/tests/Test_simplePilotLogger.py b/Pilot/tests/Test_simplePilotLogger.py index aec1b191..1fc448ae 100644 --- a/Pilot/tests/Test_simplePilotLogger.py +++ b/Pilot/tests/Test_simplePilotLogger.py @@ -1,14 +1,26 @@ #!/usr/bin/env python +from __future__ import absolute_import, division, print_function + import json import os import random import string +import sys import tempfile + +try: + from Pilot.pilotTools import CommandBase, Logger, PilotParams +except ImportError: + from pilotTools import CommandBase, Logger, PilotParams + import unittest -from unittest.mock import patch -from ..pilotTools import CommandBase, Logger, PilotParams +try: + from unittest.mock import patch +except ImportError: + from mock import patch + class TestPilotParams(unittest.TestCase): @patch("sys.argv") @@ -134,10 +146,16 @@ def test_executeAndGetOutput(self, popenMock, argvmock): for size in [1000, 1024, 1025, 2005]: random_str = "".join(random.choice(string.ascii_letters + "\n") for i in range(size)) - random_bytes = random_str.encode("UTF-8") - self.stdout_mock.write(random_bytes) + if sys.version_info.major == 3: + random_bytes = random_str.encode("UTF-8") + self.stdout_mock.write(random_bytes) + else: + self.stdout_mock.write(random_str) self.stdout_mock.seek(0) - self.stderr_mock.write("Errare humanum est!".encode("UTF-8")) + if sys.version_info.major == 3: + self.stderr_mock.write("Errare humanum est!".encode("UTF-8")) + else: + self.stderr_mock.write("Errare humanum est!") self.stderr_mock.seek(0) pp = PilotParams() diff --git a/environment.yml b/environment.yml index 72e2765e..41e0a564 100644 --- a/environment.yml +++ b/environment.yml @@ -11,6 +11,7 @@ dependencies: - requests # testing and development - pycodestyle + - caniusepython3 - coverage - mock - pylint From 56748e9f75043ddea0a63b8b036211e913562b82 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Wed, 12 Nov 2025 17:25:32 +0100 Subject: [PATCH 4/7] test: integration test fixes --- .github/workflows/integration.yml | 151 ++++++------------------------ 1 file changed, 31 insertions(+), 120 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 64d1411f..ee252cfe 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -14,7 +14,7 @@ jobs: pilot_schema: - pilot_oldSchema.json - pilot_newSchema.json - VO: + VO: - dteam - gridpp @@ -58,7 +58,7 @@ jobs: pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::${{ matrix.dirac_branch }} -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=${{ matrix.VO }} --pilotUUID="${pilotUUID}" --debug - release_prod_pre-cvmfs: + release_prod_pre-cvmfs_CEs: runs-on: ubuntu-latest strategy: @@ -68,7 +68,13 @@ jobs: - pilot_newSchema.json dirac_version: - prod - - pre + - old + ce: + - jenkins.cern.ch + - jenkins-singularity.cern.ch + - jenkins-mp.cern.ch + - jenkins-mp-pool.cern.ch + - jenkins-mp-pool-singularity.cern.ch steps: - uses: actions/checkout@v4 @@ -92,9 +98,9 @@ jobs: chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem if [ "${{ matrix.dirac_version }}" == "prod" ]; then - version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep -v 'a' | tail -n 1) + version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v9' | tail -n 1) else - version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'a' | tail -n 1) + version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v8' | tail -n 1) fi source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc pilot_proxy_file=$(mktemp) @@ -112,59 +118,7 @@ jobs: g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) - X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --architectureScript=dirac-platform --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --wnVO=dteam --pilotUUID="${pilotUUID}" --debug - - - release_prod-cvmfs_no_env_CEs: - runs-on: ubuntu-latest - - strategy: - matrix: - ce: - - jenkins.cern.ch - - jenkins-singularity.cern.ch - - jenkins-mp.cern.ch - - jenkins-mp-pool.cern.ch - - jenkins-mp-pool-singularity.cern.ch - - jenkins-revised.cern.ch - - steps: - - uses: actions/checkout@v4 - - uses: cvmfs-contrib/github-action-cvmfs@v3 - - - name: Test CernVM-FS - run: ls /cvmfs/dirac.egi.eu - - - name: Tests - env: - HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} - HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} - run: | - mkdir -p ${GITHUB_WORKSPACE}/Client/etc/grid-security - cd ${GITHUB_WORKSPACE}/Client - echo "$HOSTCERT_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem - echo "$HOSTKEY_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem - chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem - chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem - version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'a' | tail -n 1) - source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc - pilot_proxy_file=$(mktemp) - dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg - dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg - cd ${GITHUB_WORKSPACE}/Pilot - cp ../tests/CI/pilot_newSchema.json pilot.json - sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json - sed -i "s/VAR_JENKINS_CE/${{ matrix.ce }}/g" pilot.json - sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json - sed -i "s/VAR_DIRAC_VERSION/${version}/g" pilot.json - sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json - sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json - sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - g_job="testintegrationworkflow${GITHUB_JOB//-/}" - pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" - pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) - X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -S DIRAC-Certification -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --wnVO=dteam -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug - + X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -S DIRAC-Certification -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --architectureScript=dirac-platform --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --wnVO=dteam --pilotUUID="${pilotUUID}" --debug matching: runs-on: ubuntu-latest @@ -186,6 +140,9 @@ jobs: env: HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} + X509_CERT_DIR: /cvmfs/grid.cern.ch/etc/grid-security/certificates + X509_VOMS_DIR: /cvmfs/grid.cern.ch/etc/grid-security/vomsdir + DIRAC_VOMSES: /cvmfs/grid.cern.ch/etc/grid-security/vomses run: | mkdir -p ${GITHUB_WORKSPACE}/Client/etc/grid-security cd ${GITHUB_WORKSPACE}/Client @@ -193,7 +150,7 @@ jobs: echo "$HOSTKEY_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem - version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'a' | tail -n 1) + version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep -v 'a' | tail -n 1) source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc pilot_proxy_file=$(mktemp) dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg @@ -212,18 +169,17 @@ jobs: pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::integration -M 1 -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=dteam --pilotUUID="${pilotUUID}" --debug - release_prod_pre-cvmfs_matching: runs-on: ubuntu-latest strategy: matrix: - VO: + VO: - dteam - gridpp dirac_version: - prod - - pre + - old steps: - uses: actions/checkout@v4 @@ -236,6 +192,9 @@ jobs: env: HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} + X509_CERT_DIR: /cvmfs/grid.cern.ch/etc/grid-security/certificates + X509_VOMS_DIR: /cvmfs/grid.cern.ch/etc/grid-security/vomsdir + DIRAC_VOMSES: /cvmfs/grid.cern.ch/etc/grid-security/vomses run: | mkdir -p ${GITHUB_WORKSPACE}/Client/etc/grid-security cd ${GITHUB_WORKSPACE}/Client @@ -245,9 +204,9 @@ jobs: chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem cd ${GITHUB_WORKSPACE}/Pilot if [ "${{ matrix.dirac_version }}" == "prod" ]; then - version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep -v 'a' | tail -n 1) + version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v9' | tail -n 1) else - version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'a' | tail -n 1) + version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v8' | tail -n 1) fi source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc pilot_proxy_file=$(mktemp) @@ -267,7 +226,6 @@ jobs: pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --wnVO=${{ matrix.VO }} -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug - release_prod-cvmfs_matching_CEs: runs-on: ubuntu-latest @@ -279,7 +237,6 @@ jobs: - jenkins-mp-full.cern.ch - jenkins-mp-pool-full.cern.ch - jenkins-mp-pool-singularity-full.cern.ch - - jenkins-revised.cern.ch steps: - uses: actions/checkout@v4 @@ -292,6 +249,9 @@ jobs: env: HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} + X509_CERT_DIR: /cvmfs/grid.cern.ch/etc/grid-security/certificates + X509_VOMS_DIR: /cvmfs/grid.cern.ch/etc/grid-security/vomsdir + DIRAC_VOMSES: /cvmfs/grid.cern.ch/etc/grid-security/vomses run: | mkdir -p ${GITHUB_WORKSPACE}/Client/etc/grid-security cd ${GITHUB_WORKSPACE}/Client @@ -299,7 +259,7 @@ jobs: echo "$HOSTKEY_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem - version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'a' | tail -n 1) + version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep -v 'a' | tail -n 1) source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc pilot_proxy_file=$(mktemp) dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg @@ -318,7 +278,6 @@ jobs: pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --StopAfterFailedMatches=1 --wnVO=dteam -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug - ################################## ### # extensions tests ################################## @@ -380,58 +339,7 @@ jobs: pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --architectureScript=dirac-architecture --wnVO=LHCb -o lbRunOnly --pilotUUID="${pilotUUID}" --debug - ext-lhcb_integration_dirac_installer_no_env: - runs-on: ubuntu-latest - - strategy: - matrix: - pilot_schema: - - pilot_oldSchema.json - - pilot_newSchema.json - ce: - - jenkins-lhcb-d.cern.ch - - jenkins-lhcb-dd.cern.ch - - steps: - - uses: actions/checkout@v4 - - uses: cvmfs-contrib/github-action-cvmfs@v3 - - - name: Test CernVM-FS - run: ls /cvmfs/lhcb.cern.ch - - - name: Tests - env: - HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} - HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} - run: | - mkdir -p ${GITHUB_WORKSPACE}/Client/etc/grid-security - cd ${GITHUB_WORKSPACE}/Client - echo "$HOSTCERT_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem - echo "$HOSTKEY_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem - chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem - chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem - version=$(echo $(curl -s "https://gitlab.cern.ch/api/v4/projects/3588/releases") | jq -r '.[].name' | sort -V | grep -v 'a' | tail -n 1) - source /cvmfs/lhcb.cern.ch/lhcbdirac/${version}/Linux-x86_64/diracosrc - pilot_proxy_file=$(mktemp) - dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg - dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg - X509_CERT_DIR="/cvmfs/lhcb.cern.ch/etc/grid-security/certificates" X509_VOMS_DIR="/cvmfs/lhcb.cern.ch/etc/grid-security/vomsdir" DIRAC_VOMSES="/cvmfs/lhcb.cern.ch/etc/grid-security/vomses" dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg - cd ${GITHUB_WORKSPACE}/Pilot - curl https://lhcbdirac.s3.cern.ch/Pilot3/LHCbPilotCommands.py -o LHCbPilotCommands.py - cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json - sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json - sed -i "s/VAR_JENKINS_CE/${{ matrix.ce }}/g" pilot.json - sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json - sed -i "s/VAR_DIRAC_VERSION/${version}/g" pilot.json - sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json - sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json - sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - g_job="testintegrationworkflow${GITHUB_JOB//-/}" - pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" - pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) - X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --wnVO=LHCb -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug - - ext-lhcb_integration_no_env: + ext-lhcb_integration_cvmfs: runs-on: ubuntu-latest strategy: @@ -454,6 +362,9 @@ jobs: env: HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} + X509_CERT_DIR: /cvmfs/lhcb.cern.ch/etc/grid-security/certificates + X509_VOMS_DIR: /cvmfs/lhcb.cern.ch/etc/grid-security/vomsdir + DIRAC_VOMSES: /cvmfs/lhcb.cern.ch/etc/grid-security/vomses run: | mkdir -p ${GITHUB_WORKSPACE}/Client/etc/grid-security cd ${GITHUB_WORKSPACE}/Client From aab9944b538f4a654a87d69ba918d33602d3fbec Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Thu, 30 Oct 2025 15:30:45 +0100 Subject: [PATCH 5/7] style: just blacked everything --- Pilot/pilotCommands.py | 5 +++-- Pilot/pilotTools.py | 2 +- Pilot/proxyTools.py | 6 +++--- Pilot/tests/Test_simplePilotLogger.py | 1 + 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index 4815f44b..c2e6cb1e 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -834,7 +834,7 @@ def execute(self): archScript = self.pp.architectureScript if self.pp.architectureScript.split(" ")[0] == "dirac-apptainer-exec": archScript = " ".join(self.pp.architectureScript.split(" ")[1:]) - + architectureCmd = "%s %s -ddd" % (archScript, " ".join(cfg)) if self.pp.architectureScript.split(" ")[0] == "dirac-apptainer-exec": @@ -872,10 +872,12 @@ def execute(self): return localArchitecture + class ConfigureArchitectureWithoutCLI(CommandBase): """This command determines the platform. Separated from the ConfigureDIRAC command for easier extensibility. """ + def getPlatformString(self): # Modified to return our desired platform string, R. Graciani platformTuple = (platform.system(), platform.machine()) @@ -903,7 +905,6 @@ def execute(self): self.log.error("Configuration error [ERROR %s]" % str(e)) self.exitWithError(1) - cfg = ["-FDMH"] # force update, skip CA checks, skip CA download, skip VOMS if self.pp.useServerCertificate: cfg.append("--UseServerCertificate") diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 8afe0f62..b8e0f1ec 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -793,7 +793,7 @@ def executeAndGetOutput(self, cmd, environDict=None): outChunk = outChunk.decode("utf-8") # Strip unicode replacement characters # Ensure correct type conversion in Python 2 - outChunk = str(outChunk.replace(u"\ufffd", "")) + outChunk = str(outChunk.replace("\ufffd", "")) # Avoid potential str() issues in Py2 outChunk = unicode(outChunk) # pylint: disable=undefined-variable else: diff --git a/Pilot/proxyTools.py b/Pilot/proxyTools.py index a5fa652e..4aac355d 100644 --- a/Pilot/proxyTools.py +++ b/Pilot/proxyTools.py @@ -9,7 +9,7 @@ VOMS_FQANS_OID = b"1.3.6.1.4.1.8005.100.100.4" VOMS_EXTENSION_OID = b"1.3.6.1.4.1.8005.100.100.5" -RE_OPENSSL_ANS1_FORMAT = re.compile(br"^\s*\d+:d=(\d+)\s+hl=") +RE_OPENSSL_ANS1_FORMAT = re.compile(rb"^\s*\d+:d=(\d+)\s+hl=") def parseASN1(data): @@ -41,7 +41,7 @@ def getVO(proxy_data): str: A VO """ - chain = re.findall(br"-----BEGIN CERTIFICATE-----\n.+?\n-----END CERTIFICATE-----", proxy_data, flags=re.DOTALL) + chain = re.findall(rb"-----BEGIN CERTIFICATE-----\n.+?\n-----END CERTIFICATE-----", proxy_data, flags=re.DOTALL) for cert in chain: proc = Popen(["openssl", "x509", "-outform", "der"], stdin=PIPE, stdout=PIPE) out, _ = proc.communicate(cert) @@ -61,7 +61,7 @@ def getVO(proxy_data): if depth <= initial_depth: break # Look for a role, if it exists the VO is the first element - match = re.search(br"OCTET STRING\s+:/([a-zA-Z0-9]+)/Role=", line) + match = re.search(rb"OCTET STRING\s+:/([a-zA-Z0-9]+)/Role=", line) if match: return match.groups()[0].decode() raise NotImplementedError("Something went very wrong") diff --git a/Pilot/tests/Test_simplePilotLogger.py b/Pilot/tests/Test_simplePilotLogger.py index 1fc448ae..df2ac0c2 100644 --- a/Pilot/tests/Test_simplePilotLogger.py +++ b/Pilot/tests/Test_simplePilotLogger.py @@ -172,5 +172,6 @@ def test_executeAndGetOutput(self, popenMock, argvmock): self.stdout_mock.truncate() self.stderr_mock.truncate() + if __name__ == "__main__": unittest.main() From 3268f4bde04831fde85896bee2fab17f25c9d114 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Thu, 30 Oct 2025 15:52:56 +0100 Subject: [PATCH 6/7] style: json styling applied --- tests/CI/pilot_newSchema.json | 307 +++++++++++++++++----------------- tests/CI/pilot_oldSchema.json | 4 +- 2 files changed, 157 insertions(+), 154 deletions(-) diff --git a/tests/CI/pilot_newSchema.json b/tests/CI/pilot_newSchema.json index 0c8c1d32..ee793113 100644 --- a/tests/CI/pilot_newSchema.json +++ b/tests/CI/pilot_newSchema.json @@ -1,157 +1,160 @@ { - "timestamp": "2023-02-13T14:34:26.725499", - "CEs": { - "jenkins.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST" - }, - "jenkins-full.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-FULL" - }, - "jenkins-mp.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-MP" - }, - "jenkins-mp-full.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-FULL" - }, - "jenkins-singularity.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST", - "LocalCEType": "Singularity", - "InstallDIRACInContainer": "True" - }, - "jenkins-singularity-full.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-FULL", - "LocalCEType": "Singularity" - }, - "jenkins-mp-pool.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "Queue": "VAR_JENKINS_QUEUE", - "GridCEType": "TEST-MP", - "LocalCEType": "Pool" - }, - "jenkins-mp-pool-full.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-FULL", - "LocalCEType": "InProcess", - "VAR_JENKINS_QUEUE": { - "LocalCEType": "Pool" - } - }, - "jenkins-mp-pool-sudo.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-MP", - "VAR_JENKINS_QUEUE": { - "LocalCEType": "Pool/Sudo" - } - }, - "jenkins-mp-pool-sudo-full.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-FULL", - "VAR_JENKINS_QUEUE": { - "LocalCEType": "Pool/Sudo" - } - }, - "jenkins-mp-pool-singularity.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-MP", - "VAR_JENKINS_QUEUE": { - "LocalCEType": "Pool/Singularity" - } - }, - "jenkins-mp-pool-singularity-full.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-FULL", - "VAR_JENKINS_QUEUE": { - "LocalCEType": "Pool/Singularity" - } - }, - "jenkins-lhcb.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "LHCb" - }, - "jenkins-lhcb-d.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "LHCbD" - }, - "jenkins-lhcb-dd.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "LHCbDD" - }, + "timestamp": "2023-02-13T14:34:26.725499", + "CEs": { + "jenkins.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST" + }, + "jenkins-full.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-FULL" + }, + "jenkins-mp.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-MP" + }, + "jenkins-mp-full.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-FULL" + }, + "jenkins-singularity.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST", + "LocalCEType": "Singularity", + "InstallDIRACInContainer": "True" + }, + "jenkins-singularity-full.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-FULL", + "LocalCEType": "Singularity" + }, + "jenkins-mp-pool.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "Queue": "VAR_JENKINS_QUEUE", + "GridCEType": "TEST-MP", + "LocalCEType": "Pool" + }, + "jenkins-mp-pool-full.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-FULL", + "LocalCEType": "InProcess", + "VAR_JENKINS_QUEUE": { + "LocalCEType": "Pool" + } + }, + "jenkins-mp-pool-sudo.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-MP", + "VAR_JENKINS_QUEUE": { + "LocalCEType": "Pool/Sudo" + } + }, + "jenkins-mp-pool-sudo-full.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-FULL", + "VAR_JENKINS_QUEUE": { + "LocalCEType": "Pool/Sudo" + } + }, + "jenkins-mp-pool-singularity.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-MP", + "VAR_JENKINS_QUEUE": { + "LocalCEType": "Pool/Singularity" + } + }, + "jenkins-mp-pool-singularity-full.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-FULL", + "VAR_JENKINS_QUEUE": { + "LocalCEType": "Pool/Singularity" + } + }, + "jenkins-lhcb.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "LHCb" + }, + "jenkins-lhcb-d.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "LHCbD" + }, + "jenkins-lhcb-dd.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "LHCbDD" + }, "jenkins-revised.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-REVISED-COMMANDS" + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-REVISED-COMMANDS" } - }, - "Defaults": { - "Pilot": { - "RemoteLogging": "False", - "Commands": { - "TEST": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", - "TEST-MP": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", - "TEST-FULL": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements, LaunchAgent", - "LHCb": "CheckWorkerNode, LHCbInstallDIRAC, LHCbConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", - "LHCbD": "CheckWorkerNode, InstallDIRAC, LHCbConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", - "LHCbDD": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", - "TEST-REVISED-COMMANDS": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitectureWithoutCLI, ConfigureCPURequirements" + }, + "Defaults": { + "Pilot": { + "RemoteLogging": "False", + "Commands": { + "TEST": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", + "TEST-MP": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", + "TEST-FULL": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements, LaunchAgent", + "LHCb": "CheckWorkerNode, LHCbInstallDIRAC, LHCbConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", + "LHCbD": "CheckWorkerNode, InstallDIRAC, LHCbConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", + "LHCbDD": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", + "TEST-REVISED-COMMANDS": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitectureWithoutCLI, ConfigureCPURequirements" } - } - }, - "gridpp": { - "Pilot": { - "Version": "VAR_DIRAC_VERSION", - "Extensions": "None", - "CheckVersion": "False", - "pilotFileServer": "lbcertifdirac70.cern.ch:8443", - "pilotRepoBranch": "does_not_matter", - "pilotRepo": "https://github.com/does_not_matter/Pilot.git", - "GenericPilotGroup": "gridpp_pilot", - "GenericPilotDN": "VAR_USERDN_GRIDPP", - "RemoteLogging": "False", - "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", - "UploadSE": "UKI-LT2-IC-HEP-disk", - "UploadPath": "/gridpp/pilotlogs/", - "RemoteLoggerTimerInterval": 0, - "LoggingShifterName": "GridPPLogManager", - "PilotLogLevel": "DEBUG", - "CVMFS_locations": "/cvmfs/grid.cern.ch, /cvmfs/dirac.egi.eu" - } - }, - "dteam": { - "Pilot": { - "Version": "VAR_DIRAC_VERSION", - "CheckVersion": "False", - "pilotFileServer": "lbcertifdirac70.cern.ch:8443", - "pilotRepoBranch": "should_not_matter", - "pilotRepo": "https://github.com/should_not_matter/Pilot.git", - "GenericPilotGroup": "dteam_pilot", - "GenericPilotDN": "VAR_USERDN", - "RemoteLogging": "True", - "RemoteLoggerCEsWhiteList": "jenkins.cern.ch, jenkins-mp-pool.cern.ch", - "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", - "RemoteLoggerTimerInterval": 0, - "PilotLogLevel": "DEBUG", - "CVMFS_locations": "/cvmfs/grid.cern.ch, /cvmfs/dirac.egi.eu" - } - }, - "LHCb": { - "Pilot": { - "Version": "VAR_DIRAC_VERSION", - "CheckVersion": "False", - "pilotFileServer": "lbcertifdirac70.cern.ch:8443", - "GenericPilotGroup": "dteam_pilot", - "GenericPilotDN": "VAR_USERDN", - "RemoteLogging": "False", - "RemoteLoggerTimerInterval": 0, - "PilotLogLevel": "DEBUG", - "CVMFS_locations": "/cvmfs/lhcb.cern.ch/, $VO_LHCB_SW_DIR" - } - }, - "ConfigurationServers": [ - "VAR_CS" - ] + } + }, + "gridpp": { + "Pilot": { + "Version": "VAR_DIRAC_VERSION", + "Extensions": "None", + "CheckVersion": "False", + "pilotFileServer": "lbcertifdirac70.cern.ch:8443", + "pilotRepoBranch": "does_not_matter", + "pilotRepo": "https://github.com/does_not_matter/Pilot.git", + "GenericPilotGroup": "gridpp_pilot", + "GenericPilotDN": "VAR_USERDN_GRIDPP", + "RemoteLogging": "False", + "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", + "UploadSE": "UKI-LT2-IC-HEP-disk", + "UploadPath": "/gridpp/pilotlogs/", + "RemoteLoggerTimerInterval": 0, + "LoggingShifterName": "GridPPLogManager", + "PilotLogLevel": "DEBUG", + "CVMFS_locations": "/cvmfs/grid.cern.ch, /cvmfs/dirac.egi.eu" + } + }, + "dteam": { + "Pilot": { + "Version": "VAR_DIRAC_VERSION", + "CheckVersion": "False", + "pilotFileServer": "lbcertifdirac70.cern.ch:8443", + "pilotRepoBranch": "should_not_matter", + "pilotRepo": "https://github.com/should_not_matter/Pilot.git", + "GenericPilotGroup": "dteam_pilot", + "GenericPilotDN": "VAR_USERDN", + "RemoteLogging": "True", + "RemoteLoggerCEsWhiteList": "jenkins.cern.ch, jenkins-mp-pool.cern.ch", + "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", + "RemoteLoggerTimerInterval": 0, + "PilotLogLevel": "DEBUG", + "CVMFS_locations": "/cvmfs/grid.cern.ch, /cvmfs/dirac.egi.eu" + } + }, + "LHCb": { + "Pilot": { + "Version": "VAR_DIRAC_VERSION", + "CheckVersion": "False", + "pilotFileServer": "lbcertifdirac70.cern.ch:8443", + "GenericPilotGroup": "dteam_pilot", + "GenericPilotDN": "VAR_USERDN", + "RemoteLogging": "False", + "RemoteLoggerTimerInterval": 0, + "PilotLogLevel": "DEBUG", + "CVMFS_locations": "/cvmfs/lhcb.cern.ch/, $VO_LHCB_SW_DIR" + } + }, + "ConfigurationServers": [ + "VAR_CS" + ], + "PreferredURLPatterns": [ + "VAR_PREF_URL_PATTERN" + ] } diff --git a/tests/CI/pilot_oldSchema.json b/tests/CI/pilot_oldSchema.json index 3502e3b5..163bb387 100644 --- a/tests/CI/pilot_oldSchema.json +++ b/tests/CI/pilot_oldSchema.json @@ -75,8 +75,8 @@ ] }, "Logging": { - "LoggingType":"LOCAL_FILE", - "LocalOutputFile":"myFile" + "LoggingType": "LOCAL_FILE", + "LocalOutputFile": "myFile" }, "NagiosProbes": "WN-mjf, WN-sft-csh, WN-sft-voms, WN-sft-vo-swdir", "Version": "VAR_DIRAC_VERSION", From 63571e3b40cddd0d9bfc0152104fac0f8197acfe Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 4 Nov 2025 15:17:02 +0100 Subject: [PATCH 7/7] feat: setting /DIRAC/PreferredURLPatterns if it is in pilot.json --- .github/workflows/integration.yml | 6 ++++++ Pilot/pilotCommands.py | 2 ++ Pilot/pilotTools.py | 3 +++ Pilot/proxyTools.py | 6 +++--- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ee252cfe..a3ff15fd 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -53,6 +53,7 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) @@ -115,6 +116,7 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) @@ -164,6 +166,7 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) @@ -221,6 +224,7 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) @@ -273,6 +277,7 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) @@ -334,6 +339,7 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index c2e6cb1e..5032a02d 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -508,6 +508,8 @@ def _getBasicsCFG(self): self.cfg.append('-S "%s"' % self.pp.setup) if self.pp.configServer: self.cfg.append('-C "%s"' % self.pp.configServer) + if self.pp.preferredURLPatterns: + self.cfg.append("-o /DIRAC/PreferredURLPatterns=%s" % quote(",".join(self.pp.preferredURLPatterns))) if self.pp.releaseProject: self.cfg.append('-e "%s"' % self.pp.releaseProject) self.cfg.append("-o /LocalSite/ReleaseProject=%s" % self.pp.releaseProject) diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index b8e0f1ec..11345f46 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -908,6 +908,7 @@ def __init__(self): self.site = "" self.setup = "" self.configServer = "" + self.preferredURLPatterns = "" self.ceName = "" self.ceType = "" self.queueName = "" @@ -1306,6 +1307,8 @@ def __initJSON2(self): # the generic one (a list): self.configServer = ",".join([str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"]]) + self.preferredURLPatterns = self.pilotJSON.get("PreferredURLPatterns", self.preferredURLPatterns) + # version(a comma separated values in a string). We take the first one. (the default value defined in the code) dVersion = pilotOptions.get("Version", self.releaseVersion) if dVersion: diff --git a/Pilot/proxyTools.py b/Pilot/proxyTools.py index 4aac355d..a5fa652e 100644 --- a/Pilot/proxyTools.py +++ b/Pilot/proxyTools.py @@ -9,7 +9,7 @@ VOMS_FQANS_OID = b"1.3.6.1.4.1.8005.100.100.4" VOMS_EXTENSION_OID = b"1.3.6.1.4.1.8005.100.100.5" -RE_OPENSSL_ANS1_FORMAT = re.compile(rb"^\s*\d+:d=(\d+)\s+hl=") +RE_OPENSSL_ANS1_FORMAT = re.compile(br"^\s*\d+:d=(\d+)\s+hl=") def parseASN1(data): @@ -41,7 +41,7 @@ def getVO(proxy_data): str: A VO """ - chain = re.findall(rb"-----BEGIN CERTIFICATE-----\n.+?\n-----END CERTIFICATE-----", proxy_data, flags=re.DOTALL) + chain = re.findall(br"-----BEGIN CERTIFICATE-----\n.+?\n-----END CERTIFICATE-----", proxy_data, flags=re.DOTALL) for cert in chain: proc = Popen(["openssl", "x509", "-outform", "der"], stdin=PIPE, stdout=PIPE) out, _ = proc.communicate(cert) @@ -61,7 +61,7 @@ def getVO(proxy_data): if depth <= initial_depth: break # Look for a role, if it exists the VO is the first element - match = re.search(rb"OCTET STRING\s+:/([a-zA-Z0-9]+)/Role=", line) + match = re.search(br"OCTET STRING\s+:/([a-zA-Z0-9]+)/Role=", line) if match: return match.groups()[0].decode() raise NotImplementedError("Something went very wrong")