diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index e0f9221b..629d740e 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -24,7 +24,6 @@ jobs: fail-fast: false matrix: python: - - 2.7.18 - 3.6.15 - 3.9.17 @@ -55,7 +54,6 @@ jobs: fail-fast: false matrix: python: - - 2.7.18 - 3.6.15 - 3.9.17 @@ -82,7 +80,6 @@ jobs: fail-fast: false matrix: python: - - 2.7.18 - 3.6.15 - 3.9.17 diff --git a/.pylintrc b/.pylintrc index 2bae85fd..7c158f91 100644 --- a/.pylintrc +++ b/.pylintrc @@ -18,7 +18,3 @@ dummy-variables=_ disable= invalid-name, line-too-long, # would be nice to remove this one - consider-using-f-string, # python2/3 support - unspecified-encoding, # python2/3 support - super-with-arguments, # python2/3 support - redefined-builtin, # python2/3 support \ No newline at end of file diff --git a/Pilot/__init__.py b/Pilot/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/Pilot/dirac-pilot.py b/Pilot/dirac-pilot.py index 9c434c97..5fd39640 100644 --- a/Pilot/dirac-pilot.py +++ b/Pilot/dirac-pilot.py @@ -19,36 +19,19 @@ But, as said, all the actions are actually configurable. """ -from __future__ import absolute_import, division, print_function - import os import sys import time +from io import StringIO -############################ -# python 2 -> 3 "hacks" - -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO +from pilotTools import ( + Logger, + PilotParams, + RemoteLogger, + getCommand, + pythonPathCheck, +) -try: - from Pilot.pilotTools import ( - Logger, - PilotParams, - RemoteLogger, - getCommand, - pythonPathCheck, - ) -except ImportError: - from pilotTools import ( - Logger, - PilotParams, - RemoteLogger, - getCommand, - pythonPathCheck, - ) ############################ if __name__ == "__main__": diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index 5032a02d..08256356 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -17,8 +17,6 @@ def __init__(self, pilotParams): execution. """ -from __future__ import absolute_import, division, print_function - import filecmp import os import platform @@ -28,39 +26,18 @@ def __init__(self, pilotParams): import sys import time import traceback -import subprocess from collections import Counter +from http.client import HTTPSConnection +from shlex import quote + +from pilotTools import ( + CommandBase, + getSubmitterInfo, + retrieveUrlTimeout, + safe_listdir, + sendMessage, +) -############################ -# python 2 -> 3 "hacks" -try: - # For Python 3.0 and later - from http.client import HTTPSConnection -except ImportError: - # Fall back to Python 2 - from httplib import HTTPSConnection - -try: - from shlex import quote -except ImportError: - from pipes import quote - -try: - from Pilot.pilotTools import ( - CommandBase, - getSubmitterInfo, - retrieveUrlTimeout, - safe_listdir, - sendMessage, - ) -except ImportError: - from pilotTools import ( - CommandBase, - getSubmitterInfo, - retrieveUrlTimeout, - safe_listdir, - sendMessage, - ) ############################ @@ -283,7 +260,7 @@ def _getPreinstalledEnvScript(self): self.pp.installEnv["DIRAC_RC_PATH"] = preinstalledEnvScript def _localInstallDIRAC(self): - """Install python3 version of DIRAC client""" + """Install DIRAC client""" self.log.info("Installing DIRAC locally") @@ -296,10 +273,7 @@ def _localInstallDIRAC(self): # 1. Get the DIRACOS installer name # curl -O -L https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/DIRACOS-Linux-$(uname -m).sh - try: - machine = os.uname().machine # py3 - except AttributeError: - machine = os.uname()[4] # py2 + machine = os.uname().machine installerName = "DIRACOS-Linux-%s.sh" % machine diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 11345f46..79af2d86 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -1,9 +1,8 @@ """A set of common tools to be used in pilot commands""" -from __future__ import absolute_import, division, print_function - import fcntl import getopt +import importlib.util import json import os import re @@ -16,81 +15,23 @@ import warnings from datetime import datetime from functools import partial, wraps -from threading import RLock - -############################ -# python 2 -> 3 "hacks" -try: - from urllib.error import HTTPError, URLError - from urllib.parse import urlencode - from urllib.request import urlopen -except ImportError: - from urllib import urlencode - - from urllib2 import HTTPError, URLError, urlopen - -try: - import importlib.util - from importlib import import_module - - def load_module_from_path(module_name, path_to_module): - spec = importlib.util.spec_from_file_location(module_name, path_to_module) # pylint: disable=no-member - module = importlib.util.module_from_spec(spec) # pylint: disable=no-member - spec.loader.exec_module(module) - return module +from importlib import import_module +from io import StringIO +from threading import RLock, Timer +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import urlopen -except ImportError: +from proxyTools import getVO - def import_module(module): - import imp - - impData = imp.find_module(module) - return imp.load_module(module, *impData) +# Utilities functions - def load_module_from_path(module_name, path_to_module): - import imp - fp, pathname, description = imp.find_module(module_name, [path_to_module]) - try: - return imp.load_module(module_name, fp, pathname, description) - finally: - if fp: - fp.close() - - -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - -try: - basestring # pylint: disable=used-before-assignment -except NameError: - basestring = str - -try: - from Pilot.proxyTools import getVO -except ImportError: - from proxyTools import getVO - -try: - FileNotFoundError # pylint: disable=used-before-assignment - # because of https://github.com/PyCQA/pylint/issues/6748 -except NameError: - FileNotFoundError = OSError - -try: - IsADirectoryError # pylint: disable=used-before-assignment -except NameError: - IsADirectoryError = IOError - -# Timer 2.7 and < 3.3 versions issue where Timer is a function -if sys.version_info.major == 2 or sys.version_info.major == 3 and sys.version_info.minor < 3: - from threading import _Timer as Timer # pylint: disable=no-name-in-module -else: - from threading import Timer - -# Utilities functions +def load_module_from_path(module_name, path_to_module): + spec = importlib.util.spec_from_file_location(module_name, path_to_module) # pylint: disable=no-member + module = importlib.util.module_from_spec(spec) # pylint: disable=no-member + spec.loader.exec_module(module) + return module def parseVersion(releaseVersion): @@ -98,7 +39,9 @@ def parseVersion(releaseVersion): :param str releaseVersion: The software version to use """ - VERSION_PATTERN = re.compile(r"^(?:v)?(\d+)[r\.](\d+)(?:[p\.](\d+))?(?:(?:-pre|a)?(\d+))?$") + VERSION_PATTERN = re.compile( + r"^(?:v)?(\d+)[r\.](\d+)(?:[p\.](\d+))?(?:(?:-pre|a)?(\d+))?$" + ) match = VERSION_PATTERN.match(releaseVersion) # If the regex fails just return the original version @@ -188,11 +131,17 @@ def retrieveUrlTimeout(url, fileName, log, timeout=0): signal.alarm(0) return False except URLError: - log.error('Timeout after %s seconds on transfer request for "%s"' % (str(timeout), url)) + log.error( + 'Timeout after %s seconds on transfer request for "%s"' + % (str(timeout), url) + ) return False except Exception as x: if x == "Timeout": - log.error('Timeout after %s seconds on transfer request for "%s"' % (str(timeout), url)) + log.error( + 'Timeout after %s seconds on transfer request for "%s"' + % (str(timeout), url) + ) if timeout: signal.alarm(0) raise x @@ -274,7 +223,9 @@ def getSubmitterInfo(ceName): if "SGE_TASK_ID" in os.environ: batchSystemType = "SGE" batchSystemJobID = os.environ["JOB_ID"] - batchSystemParameters["BinaryPath"] = os.environ.get("SGE_BINARY_PATH", "Unknown") + batchSystemParameters["BinaryPath"] = os.environ.get( + "SGE_BINARY_PATH", "Unknown" + ) batchSystemParameters["Queue"] = os.environ.get("QUEUE", "Unknown") flavour = "SSH%s" % batchSystemType @@ -307,7 +258,12 @@ def getSubmitterInfo(ceName): batchSystemParameters["InfoPath"] = os.environ["_CONDOR_JOB_AD"] flavour = "SSH%s" % batchSystemType - pilotReference = "sshcondor://" + ceName + "/" + os.environ.get("CONDOR_JOBID", pilotReference) + pilotReference = ( + "sshcondor://" + + ceName + + "/" + + os.environ.get("CONDOR_JOBID", pilotReference) + ) # # Local/SSH @@ -325,7 +281,12 @@ def getSubmitterInfo(ceName): if "SSHBATCH_JOBID" in os.environ and "SSH_NODE_HOST" in os.environ: flavour = "SSHBATCH" pilotReference = ( - "sshbatchhost://" + ceName + "/" + os.environ["SSH_NODE_HOST"] + "/" + os.environ["SSHBATCH_JOBID"] + "sshbatchhost://" + + ceName + + "/" + + os.environ["SSH_NODE_HOST"] + + "/" + + os.environ["SSHBATCH_JOBID"] ) # # CEs @@ -348,7 +309,11 @@ def getSubmitterInfo(ceName): return ( flavour, pilotReference, - {"Type": batchSystemType, "JobID": batchSystemJobID, "Parameters": batchSystemParameters}, + { + "Type": batchSystemType, + "JobID": batchSystemJobID, + "Parameters": batchSystemParameters, + }, ) @@ -358,7 +323,9 @@ def getFlavour(ceName): Please use getSubmitterInfo instead. """ warnings.warn( - "getFlavour() is deprecated. Please use getSubmitterInfo() instead.", category=DeprecationWarning, stacklevel=2 + "getFlavour() is deprecated. Please use getSubmitterInfo() instead.", + category=DeprecationWarning, + stacklevel=2, ) flavour, pilotReference, _ = getSubmitterInfo(ceName) return flavour, pilotReference @@ -387,7 +354,9 @@ def loadModule(self, modName, hideExceptions=False): if rootModule: impName = "%s.%s" % (rootModule, impName) self.log.debug("Trying to load %s" % impName) - module, parentPath = self.__recurseImport(impName, hideExceptions=hideExceptions) + module, parentPath = self.__recurseImport( + impName, hideExceptions=hideExceptions + ) # Error. Something cannot be imported. Return error if module is None: return None, None @@ -399,7 +368,7 @@ def loadModule(self, modName, hideExceptions=False): def __recurseImport(self, modName, parentModule=None, hideExceptions=False): """Internal function to load modules""" - if isinstance(modName, basestring): + if isinstance(modName, str): modName = modName.split(".") try: if parentModule: @@ -409,13 +378,18 @@ def __recurseImport(self, modName, parentModule=None, hideExceptions=False): except ImportError as excp: if str(excp).find("No module named %s" % modName[0]) == 0: return None, None - errMsg = "Can't load %s in %s" % (".".join(modName), parentModule.__path__[0]) + errMsg = "Can't load %s in %s" % ( + ".".join(modName), + parentModule.__path__[0], + ) if not hideExceptions: self.log.exception(errMsg) return None, None if len(modName) == 1: return impModule, parentModule.__path__[0] - return self.__recurseImport(modName[1:], impModule, hideExceptions=hideExceptions) + return self.__recurseImport( + modName[1:], impModule, hideExceptions=hideExceptions + ) def loadObject(self, package, moduleName, command): """Load an object from inside a module""" @@ -483,7 +457,9 @@ def __outputMessage(self, msg, level, header): with open(self.out, "a") as outputFile: for _line in str(msg).split("\n"): if header: - outLine = self.messageTemplate.format(level=level, message=_line) + outLine = self.messageTemplate.format( + level=level, message=_line + ) print(outLine) if self.out: outputFile.write(outLine + "\n") @@ -539,7 +515,9 @@ def __init__( self.wnVO = wnVO self.isPilotLoggerOn = isPilotLoggerOn sendToURL = partial(sendMessage, url, pilotUUID, wnVO, "sendMessage") - self.buffer = FixedSizeBuffer(sendToURL, bufsize=bufsize, autoflush=flushInterval) + self.buffer = FixedSizeBuffer( + sendToURL, bufsize=bufsize, autoflush=flushInterval + ) def debug(self, msg, header=True, _sendPilotLog=False): # TODO: Send pilot log remotely? @@ -710,14 +688,12 @@ def sendMessage(url, pilotUUID, wnVO, method, rawMessage): context.load_cert_chain(cert) # this is a proxy raw_data = {"method": method, "args": message} except IsADirectoryError: # assuming it'a dir containing cert and key - context.load_cert_chain(os.path.join(cert, "hostcert.pem"), os.path.join(cert, "hostkey.pem")) + context.load_cert_chain( + os.path.join(cert, "hostcert.pem"), os.path.join(cert, "hostkey.pem") + ) raw_data = {"method": method, "args": message, "extraCredentials": '"hosts"'} - if sys.version_info.major == 3: - data = urlencode(raw_data).encode("utf-8") # encode to bytes ! for python3 - else: - # Python2 - data = urlencode(raw_data) + data = urlencode(raw_data).encode("utf-8") # encode to bytes res = urlopen(url, data, context=context) res.close() @@ -770,7 +746,12 @@ def executeAndGetOutput(self, cmd, environDict=None): self.log.info("Executing command %s" % cmd) _p = subprocess.Popen( - cmd, shell=True, env=environDict, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=False + cmd, + shell=True, + env=environDict, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=False, ) # Use non-blocking I/O on the process pipes @@ -787,17 +768,6 @@ def executeAndGetOutput(self, cmd, environDict=None): if not outChunk: continue dataWasRead = True - if sys.version_info.major == 2: - # Ensure outChunk is unicode in Python 2 - if isinstance(outChunk, str): - outChunk = outChunk.decode("utf-8") - # Strip unicode replacement characters - # Ensure correct type conversion in Python 2 - outChunk = str(outChunk.replace("\ufffd", "")) - # Avoid potential str() issues in Py2 - outChunk = unicode(outChunk) # pylint: disable=undefined-variable - else: - outChunk = str(outChunk.replace("\ufffd", "")) # Python 3: Ensure it's a string if stream == _p.stderr: sys.stderr.write(outChunk) @@ -852,7 +822,12 @@ def forkAndExecute(self, cmd, logFile, environDict=None): with open(logFile, "a+", 0) as fpLogFile: try: _p = subprocess.Popen( - "%s" % cmd, shell=True, env=environDict, close_fds=False, stdout=fpLogFile, stderr=fpLogFile + "%s" % cmd, + shell=True, + env=environDict, + close_fds=False, + stdout=fpLogFile, + stderr=fpLogFile, ) # return code @@ -964,7 +939,9 @@ def __init__(self): # Set number of allocatable processors from MJF if available try: - self.pilotProcessors = int(urlopen(os.path.join(os.environ["JOBFEATURES"], "allocated_cpu")).read()) + self.pilotProcessors = int( + urlopen(os.path.join(os.environ["JOBFEATURES"], "allocated_cpu")).read() + ) except Exception: self.pilotProcessors = 1 @@ -981,7 +958,11 @@ def __init__(self): ("l:", "project=", "Project to install"), ("n:", "name=", "Set as Site Name"), ("o:", "option=", "Option=value to add"), - ("m:", "maxNumberOfProcessors=", "specify a max number of processors to use by the payload inside a pilot"), + ( + "m:", + "maxNumberOfProcessors=", + "specify a max number of processors to use by the payload inside a pilot", + ), ("", "modules=", "for installing non-released code"), ( "", @@ -1004,7 +985,11 @@ def __init__(self): ("K:", "certLocation=", "Specify server certificate location"), ("M:", "MaxCycles=", "Maximum Number of JobAgent cycles to run"), ("", "PollingTime=", "JobAgent execution frequency"), - ("", "StopOnApplicationFailure=", "Stop Job Agent when encounter an application failure"), + ( + "", + "StopOnApplicationFailure=", + "Stop Job Agent when encounter an application failure", + ), ("", "StopAfterFailedMatches=", "Stop Job Agent after N failed matches"), ("N:", "Name=", "CE Name"), ("O:", "OwnerDN=", "Pilot OwnerDN (for private pilots)"), @@ -1014,12 +999,20 @@ def __init__(self): ("R:", "reference=", "Use this pilot reference"), ("S:", "setup=", "DIRAC Setup to use"), ("T:", "CPUTime=", "Requested CPU Time"), - ("W:", "gateway=", "Configure as DIRAC Gateway during installation"), + ( + "W:", + "gateway=", + "Configure as DIRAC Gateway during installation", + ), ("X:", "commands=", "Pilot commands to execute"), ("Z:", "commandOptions=", "Options parsed by command modules"), ("", "pilotUUID=", "pilot UUID"), ("", "preinstalledEnv=", "preinstalled pilot environment script location"), - ("", "preinstalledEnvPrefix=", "preinstalled pilot environment area prefix"), + ( + "", + "preinstalledEnvPrefix=", + "preinstalled pilot environment area prefix", + ), ("", "architectureScript=", "architecture script to use"), ("", "CVMFS_locations=", "comma-separated list of CVMS locations"), ) @@ -1095,7 +1088,8 @@ def __checkSecurityDir(self, envName, dirName): # If so, just return if envName in os.environ and safe_listdir(os.environ[envName]): self.log.debug( - "%s is set in the host environment as %s, aligning installEnv to it" % (envName, os.environ[envName]) + "%s is set in the host environment as %s, aligning installEnv to it" + % (envName, os.environ[envName]) ) else: # None of the candidates exists, stop the program. @@ -1106,7 +1100,9 @@ def __initCommandLine1(self): """Parses and interpret options on the command line: first pass (essential things)""" self.optList, __args__ = getopt.getopt( - sys.argv[1:], "".join([opt[0] for opt in self.cmdOpts]), [opt[1] for opt in self.cmdOpts] + sys.argv[1:], + "".join([opt[0] for opt in self.cmdOpts]), + [opt[1] for opt in self.cmdOpts], ) self.log.debug("Options list: %s" % self.optList) for o, v in self.optList: @@ -1132,7 +1128,9 @@ def __initCommandLine2(self): """ self.optList, __args__ = getopt.getopt( - sys.argv[1:], "".join([opt[0] for opt in self.cmdOpts]), [opt[1] for opt in self.cmdOpts] + sys.argv[1:], + "".join([opt[0] for opt in self.cmdOpts]), + [opt[1] for opt in self.cmdOpts], ) for o, v in self.optList: if o == "-E" or o == "--commandExtensions": @@ -1141,7 +1139,9 @@ def __initCommandLine2(self): self.commands = v.split(",") elif o == "-Z" or o == "--commandOptions": for i in v.split(","): - self.commandOptions[i.split("=", 1)[0].strip()] = i.split("=", 1)[1].strip() + self.commandOptions[i.split("=", 1)[0].strip()] = i.split("=", 1)[ + 1 + ].strip() elif o == "-e" or o == "--extraPackages": self.extensions = v.split(",") elif o == "-n" or o == "--name": @@ -1260,27 +1260,40 @@ def __initJSON2(self): self.pilotLogging = pilotLogging.upper() == "TRUE" self.loggerURL = pilotOptions.get("RemoteLoggerURL") # logger buffer flush interval in seconds. - self.loggerTimerInterval = int(pilotOptions.get("RemoteLoggerTimerInterval", self.loggerTimerInterval)) + self.loggerTimerInterval = int( + pilotOptions.get("RemoteLoggerTimerInterval", self.loggerTimerInterval) + ) # logger buffer size in lines: - self.loggerBufsize = max(1, int(pilotOptions.get("RemoteLoggerBufsize", self.loggerBufsize))) + self.loggerBufsize = max( + 1, int(pilotOptions.get("RemoteLoggerBufsize", self.loggerBufsize)) + ) # logger CE white list loggerCEsWhiteList = pilotOptions.get("RemoteLoggerCEsWhiteList") # restrict remote logging to whitelisted CEs ([] or None => no restriction) self.log.debug("JSON: Remote logging CE white list: %s" % loggerCEsWhiteList) if loggerCEsWhiteList is not None: if not isinstance(loggerCEsWhiteList, list): - loggerCEsWhiteList = [elem.strip() for elem in loggerCEsWhiteList.split(",")] + loggerCEsWhiteList = [ + elem.strip() for elem in loggerCEsWhiteList.split(",") + ] if self.ceName not in loggerCEsWhiteList: self.pilotLogging = False - self.log.debug("JSON: Remote logging disabled for this CE: %s" % self.ceName) + self.log.debug( + "JSON: Remote logging disabled for this CE: %s" % self.ceName + ) pilotLogLevel = pilotOptions.get("PilotLogLevel", "INFO") if pilotLogLevel.lower() == "debug": self.debugFlag = True self.log.debug("JSON: Remote logging: %s" % self.pilotLogging) self.log.debug("JSON: Remote logging URL: %s" % self.loggerURL) - self.log.debug("JSON: Remote logging buffer flush interval in sec.(0: disabled): %s" % self.loggerTimerInterval) + self.log.debug( + "JSON: Remote logging buffer flush interval in sec.(0: disabled): %s" + % self.loggerTimerInterval + ) self.log.debug("JSON: Remote/local logging debug flag: %s" % self.debugFlag) - self.log.debug("JSON: Remote logging buffer size (lines): %s" % self.loggerBufsize) + self.log.debug( + "JSON: Remote logging buffer size (lines): %s" % self.loggerBufsize + ) # CE type if present, then Defaults, otherwise as defined in the code: if "Commands" in pilotOptions: @@ -1292,7 +1305,9 @@ def __initJSON2(self): else: # TODO: This is a workaround until the pilot JSON syncroniser is fixed self.commands = [elem.strip() for elem in commands.split(",")] - self.log.debug("Selecting commands from JSON for Grid CE type %s" % key) + self.log.debug( + "Selecting commands from JSON for Grid CE type %s" % key + ) break else: key = "CodeDefaults" @@ -1302,12 +1317,18 @@ def __initJSON2(self): # Command extensions for the commands above: commandExtOptions = pilotOptions.get("CommandExtensions") if commandExtOptions: - self.commandExtensions = [elem.strip() for elem in commandExtOptions.split(",")] + self.commandExtensions = [ + elem.strip() for elem in commandExtOptions.split(",") + ] # Configuration server (the synchroniser looks into gConfig.getServersList(), as before # the generic one (a list): - self.configServer = ",".join([str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"]]) + self.configServer = ",".join( + [str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"]] + ) - self.preferredURLPatterns = self.pilotJSON.get("PreferredURLPatterns", self.preferredURLPatterns) + self.preferredURLPatterns = self.pilotJSON.get( + "PreferredURLPatterns", self.preferredURLPatterns + ) # version(a comma separated values in a string). We take the first one. (the default value defined in the code) dVersion = pilotOptions.get("Version", self.releaseVersion) @@ -1317,13 +1338,19 @@ def __initJSON2(self): else: self.log.warn("Could not find a version in the JSON file configuration") - self.log.debug("Version: %s -> (release) %s" % (str(dVersion), self.releaseVersion)) + self.log.debug( + "Version: %s -> (release) %s" % (str(dVersion), self.releaseVersion) + ) - self.releaseProject = pilotOptions.get("Project", self.releaseProject) # default from the code. + self.releaseProject = pilotOptions.get( + "Project", self.releaseProject + ) # default from the code. self.log.debug("Release project: %s" % self.releaseProject) if "CVMFS_locations" in pilotOptions: - self.CVMFS_locations = pilotOptions["CVMFS_locations"].replace(" ", "").split(",") + self.CVMFS_locations = ( + pilotOptions["CVMFS_locations"].replace(" ", "").split(",") + ) self.log.debug("CVMFS locations: %s" % self.CVMFS_locations) def getPilotOptionsDict(self): @@ -1361,7 +1388,10 @@ def __getVO(self): with open(cert, "rb") as fp: return getVO(fp.read()) except IOError as err: - self.log.error("Could not read a proxy, setting vo to 'unknown': %s" % os.strerror(err.errno)) + self.log.error( + "Could not read a proxy, setting vo to 'unknown': %s" + % os.strerror(err.errno) + ) else: self.log.error("Could not locate a proxy via X509_USER_PROXY") @@ -1458,46 +1488,78 @@ def __initJSON(self): # Commands first # FIXME: pilotSynchronizer() should publish these as comma-separated lists. We are ready for that. try: - if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType], basestring): + if isinstance( + self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType], str + ): self.commands = [ str(pv).strip() - for pv in self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType].split(",") + for pv in self.pilotJSON["Setups"][self.setup]["Commands"][ + self.gridCEType + ].split(",") ] else: self.commands = [ - str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType] + str(pv).strip() + for pv in self.pilotJSON["Setups"][self.setup]["Commands"][ + self.gridCEType + ] ] except KeyError: try: - if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"], basestring): + if isinstance( + self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"], str + ): self.commands = [ str(pv).strip() - for pv in self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"].split(",") + for pv in self.pilotJSON["Setups"][self.setup]["Commands"][ + "Defaults" + ].split(",") ] else: self.commands = [ - str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"] + str(pv).strip() + for pv in self.pilotJSON["Setups"][self.setup]["Commands"][ + "Defaults" + ] ] except KeyError: try: - if isinstance(self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType], basestring): + if isinstance( + self.pilotJSON["Setups"]["Defaults"]["Commands"][ + self.gridCEType + ], + str, + ): self.commands = [ str(pv).strip() - for pv in self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType].split(",") + for pv in self.pilotJSON["Setups"]["Defaults"]["Commands"][ + self.gridCEType + ].split(",") ] else: self.commands = [ - str(pv).strip() for pv in self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType] + str(pv).strip() + for pv in self.pilotJSON["Setups"]["Defaults"]["Commands"][ + self.gridCEType + ] ] except KeyError: try: - if isinstance(self.pilotJSON["Defaults"]["Commands"]["Defaults"], basestring): + if isinstance( + self.pilotJSON["Defaults"]["Commands"]["Defaults"], str + ): self.commands = [ - str(pv).strip() for pv in self.pilotJSON["Defaults"]["Commands"]["Defaults"].split(",") + str(pv).strip() + for pv in self.pilotJSON["Defaults"]["Commands"][ + "Defaults" + ].split(",") ] else: self.commands = [ - str(pv).strip() for pv in self.pilotJSON["Defaults"]["Commands"]["Defaults"] + str(pv).strip() + for pv in self.pilotJSON["Defaults"]["Commands"][ + "Defaults" + ] ] except KeyError: pass @@ -1507,26 +1569,36 @@ def __initJSON(self): # pilotSynchronizer() can publish this as a comma separated list. We are ready for that. try: if isinstance( - self.pilotJSON["Setups"][self.setup]["CommandExtensions"], basestring + self.pilotJSON["Setups"][self.setup]["CommandExtensions"], str ): # In the specific setup? self.commandExtensions = [ - str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["CommandExtensions"].split(",") + str(pv).strip() + for pv in self.pilotJSON["Setups"][self.setup][ + "CommandExtensions" + ].split(",") ] else: self.commandExtensions = [ - str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["CommandExtensions"] + str(pv).strip() + for pv in self.pilotJSON["Setups"][self.setup]["CommandExtensions"] ] except KeyError: try: if isinstance( - self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"], basestring + self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"], str ): # Or in the defaults section? self.commandExtensions = [ - str(pv).strip() for pv in self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"].split(",") + str(pv).strip() + for pv in self.pilotJSON["Setups"]["Defaults"][ + "CommandExtensions" + ].split(",") ] else: self.commandExtensions = [ - str(pv).strip() for pv in self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"] + str(pv).strip() + for pv in self.pilotJSON["Setups"]["Defaults"][ + "CommandExtensions" + ] ] except KeyError: pass @@ -1536,40 +1608,62 @@ def __initJSON(self): # pilotSynchronizer() can publish this as a comma separated list. We are ready for that try: if isinstance( - self.pilotJSON["ConfigurationServers"], basestring + self.pilotJSON["ConfigurationServers"], str ): # Generic, there may also be setup-specific ones self.configServer = ",".join( - [str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"].split(",")] + [ + str(pv).strip() + for pv in self.pilotJSON["ConfigurationServers"].split(",") + ] ) else: # it's a list, we suppose - self.configServer = ",".join([str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"]]) + self.configServer = ",".join( + [str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"]] + ) except KeyError: pass try: # now trying to see if there is setup-specific ones if isinstance( - self.pilotJSON["Setups"][self.setup]["ConfigurationServer"], basestring + self.pilotJSON["Setups"][self.setup]["ConfigurationServer"], str ): # In the specific setup? self.configServer = ",".join( - [str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["ConfigurationServer"].split(",")] + [ + str(pv).strip() + for pv in self.pilotJSON["Setups"][self.setup][ + "ConfigurationServer" + ].split(",") + ] ) else: # it's a list, we suppose self.configServer = ",".join( - [str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["ConfigurationServer"]] + [ + str(pv).strip() + for pv in self.pilotJSON["Setups"][self.setup][ + "ConfigurationServer" + ] + ] ) except KeyError: # and if it doesn't exist try: if isinstance( - self.pilotJSON["Setups"]["Defaults"]["ConfigurationServer"], basestring + self.pilotJSON["Setups"]["Defaults"]["ConfigurationServer"], str ): # Is there one in the defaults section? self.configServer = ",".join( [ str(pv).strip() - for pv in self.pilotJSON["Setups"]["Defaults"]["ConfigurationServer"].split(",") + for pv in self.pilotJSON["Setups"]["Defaults"][ + "ConfigurationServer" + ].split(",") ] ) else: # it's a list, we suppose self.configServer = ",".join( - [str(pv).strip() for pv in self.pilotJSON["Setups"]["Defaults"]["ConfigurationServer"]] + [ + str(pv).strip() + for pv in self.pilotJSON["Setups"]["Defaults"][ + "ConfigurationServer" + ] + ] ) except KeyError: pass @@ -1579,10 +1673,18 @@ def __initJSON(self): # There may be a list of versions specified (in a string, comma separated). We just want the first one. dVersion = None try: - dVersion = [dv.strip() for dv in self.pilotJSON["Setups"][self.setup]["Version"].split(",", 1)] + dVersion = [ + dv.strip() + for dv in self.pilotJSON["Setups"][self.setup]["Version"].split(",", 1) + ] except KeyError: try: - dVersion = [dv.strip() for dv in self.pilotJSON["Setups"]["Defaults"]["Version"].split(",", 1)] + dVersion = [ + dv.strip() + for dv in self.pilotJSON["Setups"]["Defaults"]["Version"].split( + ",", 1 + ) + ] except KeyError: self.log.warn("Could not find a version in the JSON file configuration") if dVersion is not None: @@ -1593,7 +1695,9 @@ def __initJSON(self): self.releaseProject = str(self.pilotJSON["Setups"][self.setup]["Project"]) except KeyError: try: - self.releaseProject = str(self.pilotJSON["Setups"]["Defaults"]["Project"]) + self.releaseProject = str( + self.pilotJSON["Setups"]["Defaults"]["Project"] + ) except KeyError: pass self.log.debug("Release project: %s" % self.releaseProject) @@ -1613,7 +1717,9 @@ def __ceType(self): try: if not self.gridCEType: # We don't override a grid CEType given on the command line! - self.gridCEType = str(self.pilotJSON["CEs"][self.ceName]["GridCEType"]) + self.gridCEType = str( + self.pilotJSON["CEs"][self.ceName]["GridCEType"] + ) except KeyError: pass # This LocalCEType is like 'InProcess' or 'Pool' or 'Pool/Singularity' etc. @@ -1623,7 +1729,9 @@ def __ceType(self): except KeyError: pass try: - self.ceType = str(self.pilotJSON["CEs"][self.ceName][self.queueName]["LocalCEType"]) + self.ceType = str( + self.pilotJSON["CEs"][self.ceName][self.queueName]["LocalCEType"] + ) except KeyError: pass diff --git a/Pilot/proxyTools.py b/Pilot/proxyTools.py index a5fa652e..8792a34a 100644 --- a/Pilot/proxyTools.py +++ b/Pilot/proxyTools.py @@ -1,7 +1,5 @@ """few functions for dealing with proxies""" -from __future__ import absolute_import, division, print_function - import re from base64 import b16decode from subprocess import PIPE, Popen diff --git a/Pilot/tests/Test_Pilot.py b/Pilot/tests/Test_Pilot.py index 8a1b75a1..75b600d1 100644 --- a/Pilot/tests/Test_Pilot.py +++ b/Pilot/tests/Test_Pilot.py @@ -1,7 +1,5 @@ """Test class for Pilot""" -from __future__ import absolute_import, division, print_function - import json import os import shutil @@ -12,8 +10,10 @@ # imports import unittest -from Pilot.pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes -from Pilot.pilotTools import PilotParams +sys.path.insert(0, os.getcwd() + "/Pilot") + +from pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes +from pilotTools import PilotParams class PilotTestCase(unittest.TestCase): diff --git a/Pilot/tests/Test_proxyTools.py b/Pilot/tests/Test_proxyTools.py index 7a8688cb..86935c22 100644 --- a/Pilot/tests/Test_proxyTools.py +++ b/Pilot/tests/Test_proxyTools.py @@ -1,23 +1,12 @@ -from __future__ import absolute_import, division, print_function - import os -import shlex import shutil -import subprocess import sys import unittest +from unittest.mock import patch -############################ -# python 2 -> 3 "hacks" -try: - from Pilot.proxyTools import getVO, parseASN1 -except ImportError: - from proxyTools import getVO, parseASN1 +sys.path.insert(0, os.getcwd() + "/Pilot") -try: - from unittest.mock import patch -except ImportError: - from mock import patch +from proxyTools import getVO, parseASN1 class TestProxyTools(unittest.TestCase): @@ -31,7 +20,7 @@ def test_getVO(self): os.remove(cert) self.assertEqual(vo, "fakevo") - @patch("Pilot.proxyTools.Popen") + @patch("proxyTools.Popen") def test_getVOPopenFails(self, popenMock): """ Check if an exception is raised when Popen return code is not 0. @@ -59,7 +48,7 @@ def test_getVOPopenFails(self, popenMock): getVO(data) self.assertEqual(str(exc.exception), msg) - @patch("Pilot.proxyTools.Popen") + @patch("proxyTools.Popen") def test_parseASN1Fails(self, popenMock): """Should raise an exception when Popen return code is !=0""" @@ -92,46 +81,7 @@ def __createFakeProxy(self, proxyFile): """ Create a fake proxy locally. """ - basedir = os.path.dirname(__file__) - shutil.copy(basedir + "/certs/user/userkey.pem", basedir + "/certs/user/userkey400.pem") - os.chmod(basedir + "/certs/user/userkey400.pem", 0o400) - ret = self.createFakeProxy( - basedir + "/certs/user/usercert.pem", - basedir + "/certs/user/userkey400.pem", - "fakeserver.cern.ch:15000", - "fakevo", - basedir + "/certs//host/hostcert.pem", - basedir + "/certs/host/hostkey.pem", - basedir + "/certs/ca", - proxyFile, - ) - os.remove(basedir + "/certs/user/userkey400.pem") - return ret - - def createFakeProxy(self, usercert, userkey, serverURI, vo, hostcert, hostkey, CACertDir, proxyfile): - """ - voms-proxy-fake --cert usercert.pem - --key userkey.pem - -rfc - -fqan "/fakevo/Role=user/Capability=NULL" - -uri fakeserver.cern.ch:15000 - -voms fakevo - -hostcert hostcert.pem - -hostkey hostkey.pem - -certdir ca - """ - opt = ( - '--cert %s --key %s -rfc -fqan "/fakevo/Role=user/Capability=NULL" -uri %s -voms %s -hostcert %s' - " -hostkey %s -certdir %s -out %s" - % (usercert, userkey, serverURI, vo, hostcert, hostkey, CACertDir, proxyfile) - ) - proc = subprocess.Popen( - shlex.split("voms-proxy-fake " + opt), - bufsize=1, - stdout=sys.stdout, - stderr=sys.stderr, - universal_newlines=True, - ) - proc.communicate() - return proc.returncode + shutil.copy(basedir + "/certs/voms/proxy.pem", proxyFile) + return 0 + diff --git a/Pilot/tests/Test_simplePilotLogger.py b/Pilot/tests/Test_simplePilotLogger.py index df2ac0c2..3a236da1 100644 --- a/Pilot/tests/Test_simplePilotLogger.py +++ b/Pilot/tests/Test_simplePilotLogger.py @@ -1,25 +1,17 @@ #!/usr/bin/env python -from __future__ import absolute_import, division, print_function - import json import os import random import string import sys import tempfile - -try: - from Pilot.pilotTools import CommandBase, Logger, PilotParams -except ImportError: - from pilotTools import CommandBase, Logger, PilotParams - import unittest +from unittest.mock import patch + +sys.path.insert(0, os.getcwd() + "/Pilot") -try: - from unittest.mock import patch -except ImportError: - from mock import patch +from pilotTools import CommandBase, Logger, PilotParams class TestPilotParams(unittest.TestCase): @@ -146,16 +138,10 @@ def test_executeAndGetOutput(self, popenMock, argvmock): for size in [1000, 1024, 1025, 2005]: random_str = "".join(random.choice(string.ascii_letters + "\n") for i in range(size)) - if sys.version_info.major == 3: - random_bytes = random_str.encode("UTF-8") - self.stdout_mock.write(random_bytes) - else: - self.stdout_mock.write(random_str) + random_bytes = random_str.encode("UTF-8") + self.stdout_mock.write(random_bytes) self.stdout_mock.seek(0) - if sys.version_info.major == 3: - self.stderr_mock.write("Errare humanum est!".encode("UTF-8")) - else: - self.stderr_mock.write("Errare humanum est!") + self.stderr_mock.write("Errare humanum est!".encode("UTF-8")) self.stderr_mock.seek(0) pp = PilotParams() diff --git a/environment.yml b/environment.yml index 41e0a564..72e2765e 100644 --- a/environment.yml +++ b/environment.yml @@ -11,7 +11,6 @@ dependencies: - requests # testing and development - pycodestyle - - caniusepython3 - coverage - mock - pylint