From 6d5db629189deed2bfbca69247e7e377798ccdae Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Mon, 22 Sep 2025 12:41:00 +0200 Subject: [PATCH 1/2] feat: removed python2 support --- .github/workflows/basic.yml | 3 - .pylintrc | 4 - Pilot/dirac-pilot.py | 14 +--- Pilot/pilotCommands.py | 29 ++----- Pilot/pilotTools.py | 114 +++++++------------------- Pilot/proxyTools.py | 2 - Pilot/tests/Test_Pilot.py | 10 ++- Pilot/tests/Test_proxyTools.py | 12 +-- Pilot/tests/Test_simplePilotLogger.py | 27 ++---- Untitled | 11 +++ environment.yml | 1 - 11 files changed, 63 insertions(+), 164 deletions(-) create mode 100644 Untitled diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index e0f9221b..629d740e 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -24,7 +24,6 @@ jobs: fail-fast: false matrix: python: - - 2.7.18 - 3.6.15 - 3.9.17 @@ -55,7 +54,6 @@ jobs: fail-fast: false matrix: python: - - 2.7.18 - 3.6.15 - 3.9.17 @@ -82,7 +80,6 @@ jobs: fail-fast: false matrix: python: - - 2.7.18 - 3.6.15 - 3.9.17 diff --git a/.pylintrc b/.pylintrc index 2bae85fd..7c158f91 100644 --- a/.pylintrc +++ b/.pylintrc @@ -18,7 +18,3 @@ dummy-variables=_ disable= invalid-name, line-too-long, # would be nice to remove this one - consider-using-f-string, # python2/3 support - unspecified-encoding, # python2/3 support - super-with-arguments, # python2/3 support - redefined-builtin, # python2/3 support \ No newline at end of file diff --git a/Pilot/dirac-pilot.py b/Pilot/dirac-pilot.py index 9c434c97..a2858c42 100644 --- a/Pilot/dirac-pilot.py +++ b/Pilot/dirac-pilot.py @@ -19,19 +19,10 @@ But, as said, all the actions are actually configurable. """ -from __future__ import absolute_import, division, print_function - import os import sys import time - -############################ -# python 2 -> 3 "hacks" - -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO +from io import StringIO try: from Pilot.pilotTools import ( @@ -41,7 +32,7 @@ getCommand, pythonPathCheck, ) -except ImportError: +except ModuleNotFoundError: from pilotTools import ( Logger, PilotParams, @@ -49,6 +40,7 @@ getCommand, pythonPathCheck, ) + ############################ if __name__ == "__main__": diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index 5032a02d..6223a0a3 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -17,8 +17,6 @@ def __init__(self, pilotParams): execution. """ -from __future__ import absolute_import, division, print_function - import filecmp import os import platform @@ -28,22 +26,9 @@ def __init__(self, pilotParams): import sys import time import traceback -import subprocess from collections import Counter - -############################ -# python 2 -> 3 "hacks" -try: - # For Python 3.0 and later - from http.client import HTTPSConnection -except ImportError: - # Fall back to Python 2 - from httplib import HTTPSConnection - -try: - from shlex import quote -except ImportError: - from pipes import quote +from http.client import HTTPSConnection +from shlex import quote try: from Pilot.pilotTools import ( @@ -53,7 +38,7 @@ def __init__(self, pilotParams): safe_listdir, sendMessage, ) -except ImportError: +except ModuleNotFoundError: from pilotTools import ( CommandBase, getSubmitterInfo, @@ -61,6 +46,7 @@ def __init__(self, pilotParams): safe_listdir, sendMessage, ) + ############################ @@ -283,7 +269,7 @@ def _getPreinstalledEnvScript(self): self.pp.installEnv["DIRAC_RC_PATH"] = preinstalledEnvScript def _localInstallDIRAC(self): - """Install python3 version of DIRAC client""" + """Install DIRAC client""" self.log.info("Installing DIRAC locally") @@ -296,10 +282,7 @@ def _localInstallDIRAC(self): # 1. Get the DIRACOS installer name # curl -O -L https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/DIRACOS-Linux-$(uname -m).sh - try: - machine = os.uname().machine # py3 - except AttributeError: - machine = os.uname()[4] # py2 + machine = os.uname().machine installerName = "DIRACOS-Linux-%s.sh" % machine diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 11345f46..10cd4bd8 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -1,9 +1,8 @@ """A set of common tools to be used in pilot commands""" -from __future__ import absolute_import, division, print_function - import fcntl import getopt +import importlib.util import json import os import re @@ -16,81 +15,26 @@ import warnings from datetime import datetime from functools import partial, wraps -from threading import RLock - -############################ -# python 2 -> 3 "hacks" -try: - from urllib.error import HTTPError, URLError - from urllib.parse import urlencode - from urllib.request import urlopen -except ImportError: - from urllib import urlencode - - from urllib2 import HTTPError, URLError, urlopen - -try: - import importlib.util - from importlib import import_module - - def load_module_from_path(module_name, path_to_module): - spec = importlib.util.spec_from_file_location(module_name, path_to_module) # pylint: disable=no-member - module = importlib.util.module_from_spec(spec) # pylint: disable=no-member - spec.loader.exec_module(module) - return module - -except ImportError: - - def import_module(module): - import imp - - impData = imp.find_module(module) - return imp.load_module(module, *impData) - - def load_module_from_path(module_name, path_to_module): - import imp - - fp, pathname, description = imp.find_module(module_name, [path_to_module]) - try: - return imp.load_module(module_name, fp, pathname, description) - finally: - if fp: - fp.close() - - -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - -try: - basestring # pylint: disable=used-before-assignment -except NameError: - basestring = str +from importlib import import_module +from io import StringIO +from threading import RLock, Timer +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import urlopen try: from Pilot.proxyTools import getVO -except ImportError: +except ModuleNotFoundError: from proxyTools import getVO -try: - FileNotFoundError # pylint: disable=used-before-assignment - # because of https://github.com/PyCQA/pylint/issues/6748 -except NameError: - FileNotFoundError = OSError - -try: - IsADirectoryError # pylint: disable=used-before-assignment -except NameError: - IsADirectoryError = IOError +# Utilities functions -# Timer 2.7 and < 3.3 versions issue where Timer is a function -if sys.version_info.major == 2 or sys.version_info.major == 3 and sys.version_info.minor < 3: - from threading import _Timer as Timer # pylint: disable=no-name-in-module -else: - from threading import Timer -# Utilities functions +def load_module_from_path(module_name, path_to_module): + spec = importlib.util.spec_from_file_location(module_name, path_to_module) # pylint: disable=no-member + module = importlib.util.module_from_spec(spec) # pylint: disable=no-member + spec.loader.exec_module(module) + return module def parseVersion(releaseVersion): @@ -399,7 +343,7 @@ def loadModule(self, modName, hideExceptions=False): def __recurseImport(self, modName, parentModule=None, hideExceptions=False): """Internal function to load modules""" - if isinstance(modName, basestring): + if isinstance(modName, str): modName = modName.split(".") try: if parentModule: @@ -713,11 +657,7 @@ def sendMessage(url, pilotUUID, wnVO, method, rawMessage): context.load_cert_chain(os.path.join(cert, "hostcert.pem"), os.path.join(cert, "hostkey.pem")) raw_data = {"method": method, "args": message, "extraCredentials": '"hosts"'} - if sys.version_info.major == 3: - data = urlencode(raw_data).encode("utf-8") # encode to bytes ! for python3 - else: - # Python2 - data = urlencode(raw_data) + data = urlencode(raw_data).encode("utf-8") # encode to bytes res = urlopen(url, data, context=context) res.close() @@ -787,6 +727,7 @@ def executeAndGetOutput(self, cmd, environDict=None): if not outChunk: continue dataWasRead = True +<<<<<<< HEAD if sys.version_info.major == 2: # Ensure outChunk is unicode in Python 2 if isinstance(outChunk, str): @@ -798,6 +739,9 @@ def executeAndGetOutput(self, cmd, environDict=None): outChunk = unicode(outChunk) # pylint: disable=undefined-variable else: outChunk = str(outChunk.replace("\ufffd", "")) # Python 3: Ensure it's a string +======= + outChunk = str(outChunk.replace("\ufffd", "")) # Ensure it's a string +>>>>>>> 7eb2ebf (feat: removed python2 support) if stream == _p.stderr: sys.stderr.write(outChunk) @@ -1458,7 +1402,7 @@ def __initJSON(self): # Commands first # FIXME: pilotSynchronizer() should publish these as comma-separated lists. We are ready for that. try: - if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType], basestring): + if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType], str): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType].split(",") @@ -1469,7 +1413,7 @@ def __initJSON(self): ] except KeyError: try: - if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"], basestring): + if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"], str): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"].split(",") @@ -1480,7 +1424,7 @@ def __initJSON(self): ] except KeyError: try: - if isinstance(self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType], basestring): + if isinstance(self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType], str): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType].split(",") @@ -1491,7 +1435,7 @@ def __initJSON(self): ] except KeyError: try: - if isinstance(self.pilotJSON["Defaults"]["Commands"]["Defaults"], basestring): + if isinstance(self.pilotJSON["Defaults"]["Commands"]["Defaults"], str): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Defaults"]["Commands"]["Defaults"].split(",") ] @@ -1507,7 +1451,7 @@ def __initJSON(self): # pilotSynchronizer() can publish this as a comma separated list. We are ready for that. try: if isinstance( - self.pilotJSON["Setups"][self.setup]["CommandExtensions"], basestring + self.pilotJSON["Setups"][self.setup]["CommandExtensions"], str ): # In the specific setup? self.commandExtensions = [ str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["CommandExtensions"].split(",") @@ -1519,7 +1463,7 @@ def __initJSON(self): except KeyError: try: if isinstance( - self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"], basestring + self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"], str ): # Or in the defaults section? self.commandExtensions = [ str(pv).strip() for pv in self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"].split(",") @@ -1536,7 +1480,7 @@ def __initJSON(self): # pilotSynchronizer() can publish this as a comma separated list. We are ready for that try: if isinstance( - self.pilotJSON["ConfigurationServers"], basestring + self.pilotJSON["ConfigurationServers"], str ): # Generic, there may also be setup-specific ones self.configServer = ",".join( [str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"].split(",")] @@ -1547,7 +1491,7 @@ def __initJSON(self): pass try: # now trying to see if there is setup-specific ones if isinstance( - self.pilotJSON["Setups"][self.setup]["ConfigurationServer"], basestring + self.pilotJSON["Setups"][self.setup]["ConfigurationServer"], str ): # In the specific setup? self.configServer = ",".join( [str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["ConfigurationServer"].split(",")] @@ -1559,7 +1503,7 @@ def __initJSON(self): except KeyError: # and if it doesn't exist try: if isinstance( - self.pilotJSON["Setups"]["Defaults"]["ConfigurationServer"], basestring + self.pilotJSON["Setups"]["Defaults"]["ConfigurationServer"], str ): # Is there one in the defaults section? self.configServer = ",".join( [ diff --git a/Pilot/proxyTools.py b/Pilot/proxyTools.py index a5fa652e..8792a34a 100644 --- a/Pilot/proxyTools.py +++ b/Pilot/proxyTools.py @@ -1,7 +1,5 @@ """few functions for dealing with proxies""" -from __future__ import absolute_import, division, print_function - import re from base64 import b16decode from subprocess import PIPE, Popen diff --git a/Pilot/tests/Test_Pilot.py b/Pilot/tests/Test_Pilot.py index 8a1b75a1..db1286eb 100644 --- a/Pilot/tests/Test_Pilot.py +++ b/Pilot/tests/Test_Pilot.py @@ -1,7 +1,5 @@ """Test class for Pilot""" -from __future__ import absolute_import, division, print_function - import json import os import shutil @@ -12,8 +10,12 @@ # imports import unittest -from Pilot.pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes -from Pilot.pilotTools import PilotParams +try: + from Pilot.pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes + from Pilot.pilotTools import PilotParams +except ModuleNotFoundError: + from pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes + from pilotTools import PilotParams class PilotTestCase(unittest.TestCase): diff --git a/Pilot/tests/Test_proxyTools.py b/Pilot/tests/Test_proxyTools.py index 7a8688cb..48280277 100644 --- a/Pilot/tests/Test_proxyTools.py +++ b/Pilot/tests/Test_proxyTools.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import, division, print_function import os import shlex @@ -6,20 +5,13 @@ import subprocess import sys import unittest +from unittest.mock import patch -############################ -# python 2 -> 3 "hacks" try: from Pilot.proxyTools import getVO, parseASN1 -except ImportError: +except ModuleNotFoundError: from proxyTools import getVO, parseASN1 -try: - from unittest.mock import patch -except ImportError: - from mock import patch - - class TestProxyTools(unittest.TestCase): def test_getVO(self): vo = "unknown" diff --git a/Pilot/tests/Test_simplePilotLogger.py b/Pilot/tests/Test_simplePilotLogger.py index df2ac0c2..f0cf0913 100644 --- a/Pilot/tests/Test_simplePilotLogger.py +++ b/Pilot/tests/Test_simplePilotLogger.py @@ -1,27 +1,18 @@ #!/usr/bin/env python -from __future__ import absolute_import, division, print_function - import json import os import random import string -import sys import tempfile +import unittest +from unittest.mock import patch try: from Pilot.pilotTools import CommandBase, Logger, PilotParams -except ImportError: +except ModuleNotFoundError: from pilotTools import CommandBase, Logger, PilotParams -import unittest - -try: - from unittest.mock import patch -except ImportError: - from mock import patch - - class TestPilotParams(unittest.TestCase): @patch("sys.argv") def test_pilotParamsInit(self, argvmock): @@ -146,16 +137,10 @@ def test_executeAndGetOutput(self, popenMock, argvmock): for size in [1000, 1024, 1025, 2005]: random_str = "".join(random.choice(string.ascii_letters + "\n") for i in range(size)) - if sys.version_info.major == 3: - random_bytes = random_str.encode("UTF-8") - self.stdout_mock.write(random_bytes) - else: - self.stdout_mock.write(random_str) + random_bytes = random_str.encode("UTF-8") + self.stdout_mock.write(random_bytes) self.stdout_mock.seek(0) - if sys.version_info.major == 3: - self.stderr_mock.write("Errare humanum est!".encode("UTF-8")) - else: - self.stderr_mock.write("Errare humanum est!") + self.stderr_mock.write("Errare humanum est!".encode("UTF-8")) self.stderr_mock.seek(0) pp = PilotParams() diff --git a/Untitled b/Untitled new file mode 100644 index 00000000..10dd1dde --- /dev/null +++ b/Untitled @@ -0,0 +1,11 @@ + UPDATE sb_SandBoxes AS s + JOIN ( + SELECT + MAX(OwnerId) AS badId, + MIN(OwnerId) AS goodId + FROM sb_Owners + GROUP BY Owner, OwnerGroup, VO + HAVING COUNT(*) > 1 + ) AS xxx + ON s.OwnerId = xxx.badId + SET s.OwnerId = xxx.goodId; diff --git a/environment.yml b/environment.yml index 41e0a564..72e2765e 100644 --- a/environment.yml +++ b/environment.yml @@ -11,7 +11,6 @@ dependencies: - requests # testing and development - pycodestyle - - caniusepython3 - coverage - mock - pylint From 36d635924172b3cc7ed3ba499d46590f024acf01 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 18 Nov 2025 16:34:29 +0100 Subject: [PATCH 2/2] fix: use absolute imports --- Pilot/__init__.py | 0 Pilot/dirac-pilot.py | 23 +- Pilot/pilotCommands.py | 23 +- Pilot/pilotTools.py | 334 +++++++++++++++++++------- Pilot/tests/Test_Pilot.py | 10 +- Pilot/tests/Test_proxyTools.py | 60 +---- Pilot/tests/Test_simplePilotLogger.py | 9 +- Untitled | 11 - 8 files changed, 281 insertions(+), 189 deletions(-) delete mode 100755 Pilot/__init__.py delete mode 100644 Untitled diff --git a/Pilot/__init__.py b/Pilot/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/Pilot/dirac-pilot.py b/Pilot/dirac-pilot.py index a2858c42..5fd39640 100644 --- a/Pilot/dirac-pilot.py +++ b/Pilot/dirac-pilot.py @@ -24,22 +24,13 @@ import time from io import StringIO -try: - from Pilot.pilotTools import ( - Logger, - PilotParams, - RemoteLogger, - getCommand, - pythonPathCheck, - ) -except ModuleNotFoundError: - from pilotTools import ( - Logger, - PilotParams, - RemoteLogger, - getCommand, - pythonPathCheck, - ) +from pilotTools import ( + Logger, + PilotParams, + RemoteLogger, + getCommand, + pythonPathCheck, +) ############################ diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index 6223a0a3..08256356 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -30,22 +30,13 @@ def __init__(self, pilotParams): from http.client import HTTPSConnection from shlex import quote -try: - from Pilot.pilotTools import ( - CommandBase, - getSubmitterInfo, - retrieveUrlTimeout, - safe_listdir, - sendMessage, - ) -except ModuleNotFoundError: - from pilotTools import ( - CommandBase, - getSubmitterInfo, - retrieveUrlTimeout, - safe_listdir, - sendMessage, - ) +from pilotTools import ( + CommandBase, + getSubmitterInfo, + retrieveUrlTimeout, + safe_listdir, + sendMessage, +) ############################ diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 10cd4bd8..79af2d86 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -22,10 +22,7 @@ from urllib.parse import urlencode from urllib.request import urlopen -try: - from Pilot.proxyTools import getVO -except ModuleNotFoundError: - from proxyTools import getVO +from proxyTools import getVO # Utilities functions @@ -42,7 +39,9 @@ def parseVersion(releaseVersion): :param str releaseVersion: The software version to use """ - VERSION_PATTERN = re.compile(r"^(?:v)?(\d+)[r\.](\d+)(?:[p\.](\d+))?(?:(?:-pre|a)?(\d+))?$") + VERSION_PATTERN = re.compile( + r"^(?:v)?(\d+)[r\.](\d+)(?:[p\.](\d+))?(?:(?:-pre|a)?(\d+))?$" + ) match = VERSION_PATTERN.match(releaseVersion) # If the regex fails just return the original version @@ -132,11 +131,17 @@ def retrieveUrlTimeout(url, fileName, log, timeout=0): signal.alarm(0) return False except URLError: - log.error('Timeout after %s seconds on transfer request for "%s"' % (str(timeout), url)) + log.error( + 'Timeout after %s seconds on transfer request for "%s"' + % (str(timeout), url) + ) return False except Exception as x: if x == "Timeout": - log.error('Timeout after %s seconds on transfer request for "%s"' % (str(timeout), url)) + log.error( + 'Timeout after %s seconds on transfer request for "%s"' + % (str(timeout), url) + ) if timeout: signal.alarm(0) raise x @@ -218,7 +223,9 @@ def getSubmitterInfo(ceName): if "SGE_TASK_ID" in os.environ: batchSystemType = "SGE" batchSystemJobID = os.environ["JOB_ID"] - batchSystemParameters["BinaryPath"] = os.environ.get("SGE_BINARY_PATH", "Unknown") + batchSystemParameters["BinaryPath"] = os.environ.get( + "SGE_BINARY_PATH", "Unknown" + ) batchSystemParameters["Queue"] = os.environ.get("QUEUE", "Unknown") flavour = "SSH%s" % batchSystemType @@ -251,7 +258,12 @@ def getSubmitterInfo(ceName): batchSystemParameters["InfoPath"] = os.environ["_CONDOR_JOB_AD"] flavour = "SSH%s" % batchSystemType - pilotReference = "sshcondor://" + ceName + "/" + os.environ.get("CONDOR_JOBID", pilotReference) + pilotReference = ( + "sshcondor://" + + ceName + + "/" + + os.environ.get("CONDOR_JOBID", pilotReference) + ) # # Local/SSH @@ -269,7 +281,12 @@ def getSubmitterInfo(ceName): if "SSHBATCH_JOBID" in os.environ and "SSH_NODE_HOST" in os.environ: flavour = "SSHBATCH" pilotReference = ( - "sshbatchhost://" + ceName + "/" + os.environ["SSH_NODE_HOST"] + "/" + os.environ["SSHBATCH_JOBID"] + "sshbatchhost://" + + ceName + + "/" + + os.environ["SSH_NODE_HOST"] + + "/" + + os.environ["SSHBATCH_JOBID"] ) # # CEs @@ -292,7 +309,11 @@ def getSubmitterInfo(ceName): return ( flavour, pilotReference, - {"Type": batchSystemType, "JobID": batchSystemJobID, "Parameters": batchSystemParameters}, + { + "Type": batchSystemType, + "JobID": batchSystemJobID, + "Parameters": batchSystemParameters, + }, ) @@ -302,7 +323,9 @@ def getFlavour(ceName): Please use getSubmitterInfo instead. """ warnings.warn( - "getFlavour() is deprecated. Please use getSubmitterInfo() instead.", category=DeprecationWarning, stacklevel=2 + "getFlavour() is deprecated. Please use getSubmitterInfo() instead.", + category=DeprecationWarning, + stacklevel=2, ) flavour, pilotReference, _ = getSubmitterInfo(ceName) return flavour, pilotReference @@ -331,7 +354,9 @@ def loadModule(self, modName, hideExceptions=False): if rootModule: impName = "%s.%s" % (rootModule, impName) self.log.debug("Trying to load %s" % impName) - module, parentPath = self.__recurseImport(impName, hideExceptions=hideExceptions) + module, parentPath = self.__recurseImport( + impName, hideExceptions=hideExceptions + ) # Error. Something cannot be imported. Return error if module is None: return None, None @@ -353,13 +378,18 @@ def __recurseImport(self, modName, parentModule=None, hideExceptions=False): except ImportError as excp: if str(excp).find("No module named %s" % modName[0]) == 0: return None, None - errMsg = "Can't load %s in %s" % (".".join(modName), parentModule.__path__[0]) + errMsg = "Can't load %s in %s" % ( + ".".join(modName), + parentModule.__path__[0], + ) if not hideExceptions: self.log.exception(errMsg) return None, None if len(modName) == 1: return impModule, parentModule.__path__[0] - return self.__recurseImport(modName[1:], impModule, hideExceptions=hideExceptions) + return self.__recurseImport( + modName[1:], impModule, hideExceptions=hideExceptions + ) def loadObject(self, package, moduleName, command): """Load an object from inside a module""" @@ -427,7 +457,9 @@ def __outputMessage(self, msg, level, header): with open(self.out, "a") as outputFile: for _line in str(msg).split("\n"): if header: - outLine = self.messageTemplate.format(level=level, message=_line) + outLine = self.messageTemplate.format( + level=level, message=_line + ) print(outLine) if self.out: outputFile.write(outLine + "\n") @@ -483,7 +515,9 @@ def __init__( self.wnVO = wnVO self.isPilotLoggerOn = isPilotLoggerOn sendToURL = partial(sendMessage, url, pilotUUID, wnVO, "sendMessage") - self.buffer = FixedSizeBuffer(sendToURL, bufsize=bufsize, autoflush=flushInterval) + self.buffer = FixedSizeBuffer( + sendToURL, bufsize=bufsize, autoflush=flushInterval + ) def debug(self, msg, header=True, _sendPilotLog=False): # TODO: Send pilot log remotely? @@ -654,7 +688,9 @@ def sendMessage(url, pilotUUID, wnVO, method, rawMessage): context.load_cert_chain(cert) # this is a proxy raw_data = {"method": method, "args": message} except IsADirectoryError: # assuming it'a dir containing cert and key - context.load_cert_chain(os.path.join(cert, "hostcert.pem"), os.path.join(cert, "hostkey.pem")) + context.load_cert_chain( + os.path.join(cert, "hostcert.pem"), os.path.join(cert, "hostkey.pem") + ) raw_data = {"method": method, "args": message, "extraCredentials": '"hosts"'} data = urlencode(raw_data).encode("utf-8") # encode to bytes @@ -710,7 +746,12 @@ def executeAndGetOutput(self, cmd, environDict=None): self.log.info("Executing command %s" % cmd) _p = subprocess.Popen( - cmd, shell=True, env=environDict, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=False + cmd, + shell=True, + env=environDict, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=False, ) # Use non-blocking I/O on the process pipes @@ -727,21 +768,6 @@ def executeAndGetOutput(self, cmd, environDict=None): if not outChunk: continue dataWasRead = True -<<<<<<< HEAD - if sys.version_info.major == 2: - # Ensure outChunk is unicode in Python 2 - if isinstance(outChunk, str): - outChunk = outChunk.decode("utf-8") - # Strip unicode replacement characters - # Ensure correct type conversion in Python 2 - outChunk = str(outChunk.replace("\ufffd", "")) - # Avoid potential str() issues in Py2 - outChunk = unicode(outChunk) # pylint: disable=undefined-variable - else: - outChunk = str(outChunk.replace("\ufffd", "")) # Python 3: Ensure it's a string -======= - outChunk = str(outChunk.replace("\ufffd", "")) # Ensure it's a string ->>>>>>> 7eb2ebf (feat: removed python2 support) if stream == _p.stderr: sys.stderr.write(outChunk) @@ -796,7 +822,12 @@ def forkAndExecute(self, cmd, logFile, environDict=None): with open(logFile, "a+", 0) as fpLogFile: try: _p = subprocess.Popen( - "%s" % cmd, shell=True, env=environDict, close_fds=False, stdout=fpLogFile, stderr=fpLogFile + "%s" % cmd, + shell=True, + env=environDict, + close_fds=False, + stdout=fpLogFile, + stderr=fpLogFile, ) # return code @@ -908,7 +939,9 @@ def __init__(self): # Set number of allocatable processors from MJF if available try: - self.pilotProcessors = int(urlopen(os.path.join(os.environ["JOBFEATURES"], "allocated_cpu")).read()) + self.pilotProcessors = int( + urlopen(os.path.join(os.environ["JOBFEATURES"], "allocated_cpu")).read() + ) except Exception: self.pilotProcessors = 1 @@ -925,7 +958,11 @@ def __init__(self): ("l:", "project=", "Project to install"), ("n:", "name=", "Set as Site Name"), ("o:", "option=", "Option=value to add"), - ("m:", "maxNumberOfProcessors=", "specify a max number of processors to use by the payload inside a pilot"), + ( + "m:", + "maxNumberOfProcessors=", + "specify a max number of processors to use by the payload inside a pilot", + ), ("", "modules=", "for installing non-released code"), ( "", @@ -948,7 +985,11 @@ def __init__(self): ("K:", "certLocation=", "Specify server certificate location"), ("M:", "MaxCycles=", "Maximum Number of JobAgent cycles to run"), ("", "PollingTime=", "JobAgent execution frequency"), - ("", "StopOnApplicationFailure=", "Stop Job Agent when encounter an application failure"), + ( + "", + "StopOnApplicationFailure=", + "Stop Job Agent when encounter an application failure", + ), ("", "StopAfterFailedMatches=", "Stop Job Agent after N failed matches"), ("N:", "Name=", "CE Name"), ("O:", "OwnerDN=", "Pilot OwnerDN (for private pilots)"), @@ -958,12 +999,20 @@ def __init__(self): ("R:", "reference=", "Use this pilot reference"), ("S:", "setup=", "DIRAC Setup to use"), ("T:", "CPUTime=", "Requested CPU Time"), - ("W:", "gateway=", "Configure as DIRAC Gateway during installation"), + ( + "W:", + "gateway=", + "Configure as DIRAC Gateway during installation", + ), ("X:", "commands=", "Pilot commands to execute"), ("Z:", "commandOptions=", "Options parsed by command modules"), ("", "pilotUUID=", "pilot UUID"), ("", "preinstalledEnv=", "preinstalled pilot environment script location"), - ("", "preinstalledEnvPrefix=", "preinstalled pilot environment area prefix"), + ( + "", + "preinstalledEnvPrefix=", + "preinstalled pilot environment area prefix", + ), ("", "architectureScript=", "architecture script to use"), ("", "CVMFS_locations=", "comma-separated list of CVMS locations"), ) @@ -1039,7 +1088,8 @@ def __checkSecurityDir(self, envName, dirName): # If so, just return if envName in os.environ and safe_listdir(os.environ[envName]): self.log.debug( - "%s is set in the host environment as %s, aligning installEnv to it" % (envName, os.environ[envName]) + "%s is set in the host environment as %s, aligning installEnv to it" + % (envName, os.environ[envName]) ) else: # None of the candidates exists, stop the program. @@ -1050,7 +1100,9 @@ def __initCommandLine1(self): """Parses and interpret options on the command line: first pass (essential things)""" self.optList, __args__ = getopt.getopt( - sys.argv[1:], "".join([opt[0] for opt in self.cmdOpts]), [opt[1] for opt in self.cmdOpts] + sys.argv[1:], + "".join([opt[0] for opt in self.cmdOpts]), + [opt[1] for opt in self.cmdOpts], ) self.log.debug("Options list: %s" % self.optList) for o, v in self.optList: @@ -1076,7 +1128,9 @@ def __initCommandLine2(self): """ self.optList, __args__ = getopt.getopt( - sys.argv[1:], "".join([opt[0] for opt in self.cmdOpts]), [opt[1] for opt in self.cmdOpts] + sys.argv[1:], + "".join([opt[0] for opt in self.cmdOpts]), + [opt[1] for opt in self.cmdOpts], ) for o, v in self.optList: if o == "-E" or o == "--commandExtensions": @@ -1085,7 +1139,9 @@ def __initCommandLine2(self): self.commands = v.split(",") elif o == "-Z" or o == "--commandOptions": for i in v.split(","): - self.commandOptions[i.split("=", 1)[0].strip()] = i.split("=", 1)[1].strip() + self.commandOptions[i.split("=", 1)[0].strip()] = i.split("=", 1)[ + 1 + ].strip() elif o == "-e" or o == "--extraPackages": self.extensions = v.split(",") elif o == "-n" or o == "--name": @@ -1204,27 +1260,40 @@ def __initJSON2(self): self.pilotLogging = pilotLogging.upper() == "TRUE" self.loggerURL = pilotOptions.get("RemoteLoggerURL") # logger buffer flush interval in seconds. - self.loggerTimerInterval = int(pilotOptions.get("RemoteLoggerTimerInterval", self.loggerTimerInterval)) + self.loggerTimerInterval = int( + pilotOptions.get("RemoteLoggerTimerInterval", self.loggerTimerInterval) + ) # logger buffer size in lines: - self.loggerBufsize = max(1, int(pilotOptions.get("RemoteLoggerBufsize", self.loggerBufsize))) + self.loggerBufsize = max( + 1, int(pilotOptions.get("RemoteLoggerBufsize", self.loggerBufsize)) + ) # logger CE white list loggerCEsWhiteList = pilotOptions.get("RemoteLoggerCEsWhiteList") # restrict remote logging to whitelisted CEs ([] or None => no restriction) self.log.debug("JSON: Remote logging CE white list: %s" % loggerCEsWhiteList) if loggerCEsWhiteList is not None: if not isinstance(loggerCEsWhiteList, list): - loggerCEsWhiteList = [elem.strip() for elem in loggerCEsWhiteList.split(",")] + loggerCEsWhiteList = [ + elem.strip() for elem in loggerCEsWhiteList.split(",") + ] if self.ceName not in loggerCEsWhiteList: self.pilotLogging = False - self.log.debug("JSON: Remote logging disabled for this CE: %s" % self.ceName) + self.log.debug( + "JSON: Remote logging disabled for this CE: %s" % self.ceName + ) pilotLogLevel = pilotOptions.get("PilotLogLevel", "INFO") if pilotLogLevel.lower() == "debug": self.debugFlag = True self.log.debug("JSON: Remote logging: %s" % self.pilotLogging) self.log.debug("JSON: Remote logging URL: %s" % self.loggerURL) - self.log.debug("JSON: Remote logging buffer flush interval in sec.(0: disabled): %s" % self.loggerTimerInterval) + self.log.debug( + "JSON: Remote logging buffer flush interval in sec.(0: disabled): %s" + % self.loggerTimerInterval + ) self.log.debug("JSON: Remote/local logging debug flag: %s" % self.debugFlag) - self.log.debug("JSON: Remote logging buffer size (lines): %s" % self.loggerBufsize) + self.log.debug( + "JSON: Remote logging buffer size (lines): %s" % self.loggerBufsize + ) # CE type if present, then Defaults, otherwise as defined in the code: if "Commands" in pilotOptions: @@ -1236,7 +1305,9 @@ def __initJSON2(self): else: # TODO: This is a workaround until the pilot JSON syncroniser is fixed self.commands = [elem.strip() for elem in commands.split(",")] - self.log.debug("Selecting commands from JSON for Grid CE type %s" % key) + self.log.debug( + "Selecting commands from JSON for Grid CE type %s" % key + ) break else: key = "CodeDefaults" @@ -1246,12 +1317,18 @@ def __initJSON2(self): # Command extensions for the commands above: commandExtOptions = pilotOptions.get("CommandExtensions") if commandExtOptions: - self.commandExtensions = [elem.strip() for elem in commandExtOptions.split(",")] + self.commandExtensions = [ + elem.strip() for elem in commandExtOptions.split(",") + ] # Configuration server (the synchroniser looks into gConfig.getServersList(), as before # the generic one (a list): - self.configServer = ",".join([str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"]]) + self.configServer = ",".join( + [str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"]] + ) - self.preferredURLPatterns = self.pilotJSON.get("PreferredURLPatterns", self.preferredURLPatterns) + self.preferredURLPatterns = self.pilotJSON.get( + "PreferredURLPatterns", self.preferredURLPatterns + ) # version(a comma separated values in a string). We take the first one. (the default value defined in the code) dVersion = pilotOptions.get("Version", self.releaseVersion) @@ -1261,13 +1338,19 @@ def __initJSON2(self): else: self.log.warn("Could not find a version in the JSON file configuration") - self.log.debug("Version: %s -> (release) %s" % (str(dVersion), self.releaseVersion)) + self.log.debug( + "Version: %s -> (release) %s" % (str(dVersion), self.releaseVersion) + ) - self.releaseProject = pilotOptions.get("Project", self.releaseProject) # default from the code. + self.releaseProject = pilotOptions.get( + "Project", self.releaseProject + ) # default from the code. self.log.debug("Release project: %s" % self.releaseProject) if "CVMFS_locations" in pilotOptions: - self.CVMFS_locations = pilotOptions["CVMFS_locations"].replace(" ", "").split(",") + self.CVMFS_locations = ( + pilotOptions["CVMFS_locations"].replace(" ", "").split(",") + ) self.log.debug("CVMFS locations: %s" % self.CVMFS_locations) def getPilotOptionsDict(self): @@ -1305,7 +1388,10 @@ def __getVO(self): with open(cert, "rb") as fp: return getVO(fp.read()) except IOError as err: - self.log.error("Could not read a proxy, setting vo to 'unknown': %s" % os.strerror(err.errno)) + self.log.error( + "Could not read a proxy, setting vo to 'unknown': %s" + % os.strerror(err.errno) + ) else: self.log.error("Could not locate a proxy via X509_USER_PROXY") @@ -1402,46 +1488,78 @@ def __initJSON(self): # Commands first # FIXME: pilotSynchronizer() should publish these as comma-separated lists. We are ready for that. try: - if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType], str): + if isinstance( + self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType], str + ): self.commands = [ str(pv).strip() - for pv in self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType].split(",") + for pv in self.pilotJSON["Setups"][self.setup]["Commands"][ + self.gridCEType + ].split(",") ] else: self.commands = [ - str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType] + str(pv).strip() + for pv in self.pilotJSON["Setups"][self.setup]["Commands"][ + self.gridCEType + ] ] except KeyError: try: - if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"], str): + if isinstance( + self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"], str + ): self.commands = [ str(pv).strip() - for pv in self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"].split(",") + for pv in self.pilotJSON["Setups"][self.setup]["Commands"][ + "Defaults" + ].split(",") ] else: self.commands = [ - str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"] + str(pv).strip() + for pv in self.pilotJSON["Setups"][self.setup]["Commands"][ + "Defaults" + ] ] except KeyError: try: - if isinstance(self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType], str): + if isinstance( + self.pilotJSON["Setups"]["Defaults"]["Commands"][ + self.gridCEType + ], + str, + ): self.commands = [ str(pv).strip() - for pv in self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType].split(",") + for pv in self.pilotJSON["Setups"]["Defaults"]["Commands"][ + self.gridCEType + ].split(",") ] else: self.commands = [ - str(pv).strip() for pv in self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType] + str(pv).strip() + for pv in self.pilotJSON["Setups"]["Defaults"]["Commands"][ + self.gridCEType + ] ] except KeyError: try: - if isinstance(self.pilotJSON["Defaults"]["Commands"]["Defaults"], str): + if isinstance( + self.pilotJSON["Defaults"]["Commands"]["Defaults"], str + ): self.commands = [ - str(pv).strip() for pv in self.pilotJSON["Defaults"]["Commands"]["Defaults"].split(",") + str(pv).strip() + for pv in self.pilotJSON["Defaults"]["Commands"][ + "Defaults" + ].split(",") ] else: self.commands = [ - str(pv).strip() for pv in self.pilotJSON["Defaults"]["Commands"]["Defaults"] + str(pv).strip() + for pv in self.pilotJSON["Defaults"]["Commands"][ + "Defaults" + ] ] except KeyError: pass @@ -1454,11 +1572,15 @@ def __initJSON(self): self.pilotJSON["Setups"][self.setup]["CommandExtensions"], str ): # In the specific setup? self.commandExtensions = [ - str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["CommandExtensions"].split(",") + str(pv).strip() + for pv in self.pilotJSON["Setups"][self.setup][ + "CommandExtensions" + ].split(",") ] else: self.commandExtensions = [ - str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["CommandExtensions"] + str(pv).strip() + for pv in self.pilotJSON["Setups"][self.setup]["CommandExtensions"] ] except KeyError: try: @@ -1466,11 +1588,17 @@ def __initJSON(self): self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"], str ): # Or in the defaults section? self.commandExtensions = [ - str(pv).strip() for pv in self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"].split(",") + str(pv).strip() + for pv in self.pilotJSON["Setups"]["Defaults"][ + "CommandExtensions" + ].split(",") ] else: self.commandExtensions = [ - str(pv).strip() for pv in self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"] + str(pv).strip() + for pv in self.pilotJSON["Setups"]["Defaults"][ + "CommandExtensions" + ] ] except KeyError: pass @@ -1483,10 +1611,15 @@ def __initJSON(self): self.pilotJSON["ConfigurationServers"], str ): # Generic, there may also be setup-specific ones self.configServer = ",".join( - [str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"].split(",")] + [ + str(pv).strip() + for pv in self.pilotJSON["ConfigurationServers"].split(",") + ] ) else: # it's a list, we suppose - self.configServer = ",".join([str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"]]) + self.configServer = ",".join( + [str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"]] + ) except KeyError: pass try: # now trying to see if there is setup-specific ones @@ -1494,11 +1627,21 @@ def __initJSON(self): self.pilotJSON["Setups"][self.setup]["ConfigurationServer"], str ): # In the specific setup? self.configServer = ",".join( - [str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["ConfigurationServer"].split(",")] + [ + str(pv).strip() + for pv in self.pilotJSON["Setups"][self.setup][ + "ConfigurationServer" + ].split(",") + ] ) else: # it's a list, we suppose self.configServer = ",".join( - [str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["ConfigurationServer"]] + [ + str(pv).strip() + for pv in self.pilotJSON["Setups"][self.setup][ + "ConfigurationServer" + ] + ] ) except KeyError: # and if it doesn't exist try: @@ -1508,12 +1651,19 @@ def __initJSON(self): self.configServer = ",".join( [ str(pv).strip() - for pv in self.pilotJSON["Setups"]["Defaults"]["ConfigurationServer"].split(",") + for pv in self.pilotJSON["Setups"]["Defaults"][ + "ConfigurationServer" + ].split(",") ] ) else: # it's a list, we suppose self.configServer = ",".join( - [str(pv).strip() for pv in self.pilotJSON["Setups"]["Defaults"]["ConfigurationServer"]] + [ + str(pv).strip() + for pv in self.pilotJSON["Setups"]["Defaults"][ + "ConfigurationServer" + ] + ] ) except KeyError: pass @@ -1523,10 +1673,18 @@ def __initJSON(self): # There may be a list of versions specified (in a string, comma separated). We just want the first one. dVersion = None try: - dVersion = [dv.strip() for dv in self.pilotJSON["Setups"][self.setup]["Version"].split(",", 1)] + dVersion = [ + dv.strip() + for dv in self.pilotJSON["Setups"][self.setup]["Version"].split(",", 1) + ] except KeyError: try: - dVersion = [dv.strip() for dv in self.pilotJSON["Setups"]["Defaults"]["Version"].split(",", 1)] + dVersion = [ + dv.strip() + for dv in self.pilotJSON["Setups"]["Defaults"]["Version"].split( + ",", 1 + ) + ] except KeyError: self.log.warn("Could not find a version in the JSON file configuration") if dVersion is not None: @@ -1537,7 +1695,9 @@ def __initJSON(self): self.releaseProject = str(self.pilotJSON["Setups"][self.setup]["Project"]) except KeyError: try: - self.releaseProject = str(self.pilotJSON["Setups"]["Defaults"]["Project"]) + self.releaseProject = str( + self.pilotJSON["Setups"]["Defaults"]["Project"] + ) except KeyError: pass self.log.debug("Release project: %s" % self.releaseProject) @@ -1557,7 +1717,9 @@ def __ceType(self): try: if not self.gridCEType: # We don't override a grid CEType given on the command line! - self.gridCEType = str(self.pilotJSON["CEs"][self.ceName]["GridCEType"]) + self.gridCEType = str( + self.pilotJSON["CEs"][self.ceName]["GridCEType"] + ) except KeyError: pass # This LocalCEType is like 'InProcess' or 'Pool' or 'Pool/Singularity' etc. @@ -1567,7 +1729,9 @@ def __ceType(self): except KeyError: pass try: - self.ceType = str(self.pilotJSON["CEs"][self.ceName][self.queueName]["LocalCEType"]) + self.ceType = str( + self.pilotJSON["CEs"][self.ceName][self.queueName]["LocalCEType"] + ) except KeyError: pass diff --git a/Pilot/tests/Test_Pilot.py b/Pilot/tests/Test_Pilot.py index db1286eb..75b600d1 100644 --- a/Pilot/tests/Test_Pilot.py +++ b/Pilot/tests/Test_Pilot.py @@ -10,12 +10,10 @@ # imports import unittest -try: - from Pilot.pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes - from Pilot.pilotTools import PilotParams -except ModuleNotFoundError: - from pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes - from pilotTools import PilotParams +sys.path.insert(0, os.getcwd() + "/Pilot") + +from pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes +from pilotTools import PilotParams class PilotTestCase(unittest.TestCase): diff --git a/Pilot/tests/Test_proxyTools.py b/Pilot/tests/Test_proxyTools.py index 48280277..86935c22 100644 --- a/Pilot/tests/Test_proxyTools.py +++ b/Pilot/tests/Test_proxyTools.py @@ -1,16 +1,13 @@ - import os -import shlex import shutil -import subprocess import sys import unittest from unittest.mock import patch -try: - from Pilot.proxyTools import getVO, parseASN1 -except ModuleNotFoundError: - from proxyTools import getVO, parseASN1 +sys.path.insert(0, os.getcwd() + "/Pilot") + +from proxyTools import getVO, parseASN1 + class TestProxyTools(unittest.TestCase): def test_getVO(self): @@ -23,7 +20,7 @@ def test_getVO(self): os.remove(cert) self.assertEqual(vo, "fakevo") - @patch("Pilot.proxyTools.Popen") + @patch("proxyTools.Popen") def test_getVOPopenFails(self, popenMock): """ Check if an exception is raised when Popen return code is not 0. @@ -51,7 +48,7 @@ def test_getVOPopenFails(self, popenMock): getVO(data) self.assertEqual(str(exc.exception), msg) - @patch("Pilot.proxyTools.Popen") + @patch("proxyTools.Popen") def test_parseASN1Fails(self, popenMock): """Should raise an exception when Popen return code is !=0""" @@ -84,46 +81,7 @@ def __createFakeProxy(self, proxyFile): """ Create a fake proxy locally. """ - basedir = os.path.dirname(__file__) - shutil.copy(basedir + "/certs/user/userkey.pem", basedir + "/certs/user/userkey400.pem") - os.chmod(basedir + "/certs/user/userkey400.pem", 0o400) - ret = self.createFakeProxy( - basedir + "/certs/user/usercert.pem", - basedir + "/certs/user/userkey400.pem", - "fakeserver.cern.ch:15000", - "fakevo", - basedir + "/certs//host/hostcert.pem", - basedir + "/certs/host/hostkey.pem", - basedir + "/certs/ca", - proxyFile, - ) - os.remove(basedir + "/certs/user/userkey400.pem") - return ret - - def createFakeProxy(self, usercert, userkey, serverURI, vo, hostcert, hostkey, CACertDir, proxyfile): - """ - voms-proxy-fake --cert usercert.pem - --key userkey.pem - -rfc - -fqan "/fakevo/Role=user/Capability=NULL" - -uri fakeserver.cern.ch:15000 - -voms fakevo - -hostcert hostcert.pem - -hostkey hostkey.pem - -certdir ca - """ - opt = ( - '--cert %s --key %s -rfc -fqan "/fakevo/Role=user/Capability=NULL" -uri %s -voms %s -hostcert %s' - " -hostkey %s -certdir %s -out %s" - % (usercert, userkey, serverURI, vo, hostcert, hostkey, CACertDir, proxyfile) - ) - proc = subprocess.Popen( - shlex.split("voms-proxy-fake " + opt), - bufsize=1, - stdout=sys.stdout, - stderr=sys.stderr, - universal_newlines=True, - ) - proc.communicate() - return proc.returncode + shutil.copy(basedir + "/certs/voms/proxy.pem", proxyFile) + return 0 + diff --git a/Pilot/tests/Test_simplePilotLogger.py b/Pilot/tests/Test_simplePilotLogger.py index f0cf0913..3a236da1 100644 --- a/Pilot/tests/Test_simplePilotLogger.py +++ b/Pilot/tests/Test_simplePilotLogger.py @@ -4,14 +4,15 @@ import os import random import string +import sys import tempfile import unittest from unittest.mock import patch -try: - from Pilot.pilotTools import CommandBase, Logger, PilotParams -except ModuleNotFoundError: - from pilotTools import CommandBase, Logger, PilotParams +sys.path.insert(0, os.getcwd() + "/Pilot") + +from pilotTools import CommandBase, Logger, PilotParams + class TestPilotParams(unittest.TestCase): @patch("sys.argv") diff --git a/Untitled b/Untitled deleted file mode 100644 index 10dd1dde..00000000 --- a/Untitled +++ /dev/null @@ -1,11 +0,0 @@ - UPDATE sb_SandBoxes AS s - JOIN ( - SELECT - MAX(OwnerId) AS badId, - MIN(OwnerId) AS goodId - FROM sb_Owners - GROUP BY Owner, OwnerGroup, VO - HAVING COUNT(*) > 1 - ) AS xxx - ON s.OwnerId = xxx.badId - SET s.OwnerId = xxx.goodId;