diff --git a/.pylintrc b/.pylintrc index 58d94461..2bae85fd 100644 --- a/.pylintrc +++ b/.pylintrc @@ -9,3 +9,16 @@ init-hook='import sys; sys.path.append("$WORKSPACE/DIRAC/WorkloadManagementSyste # Add to the black list. It should be a base name, not a # path. You may set this option multiple times. ignore=.git + +# For unused variables +[VARIABLES] +dummy-variables=_ + +[MESSAGES CONTROL] +disable= + invalid-name, + line-too-long, # would be nice to remove this one + consider-using-f-string, # python2/3 support + unspecified-encoding, # python2/3 support + super-with-arguments, # python2/3 support + redefined-builtin, # python2/3 support \ No newline at end of file diff --git a/Pilot/dirac-pilot.py b/Pilot/dirac-pilot.py index 676bc197..9c434c97 100644 --- a/Pilot/dirac-pilot.py +++ b/Pilot/dirac-pilot.py @@ -1,21 +1,22 @@ #!/usr/bin/env python -""" The dirac-pilot.py script is a steering script to execute a series of - pilot commands. The commands may be provided in the pilot input sandbox, and are coded in - the pilotCommands.py module or in any Commands.py module. - The pilot script defines two switches in order to choose a set of commands for the pilot: +"""The dirac-pilot.py script is a steering script to execute a series of +pilot commands. The commands may be provided in the pilot input sandbox, and are coded in +the pilotCommands.py module or in any PilotCommands.py module (e.g. "LHCbPilotCommands.py") +The pilot script defines two switches in order to choose a set of commands for the pilot: - -E, --commandExtensions value - where the value is a comma separated list of extension names. Modules - with names Commands.py will be searched for the commands in - the order defined in the value. By default no extensions are given - -X, --commands value - where value is a comma separated list of pilot commands. By default - the list is InstallDIRAC,ConfigureDIRAC,LaunchAgent + -E, --commandExtensions value + where the value is a comma separated list of extension names. Modules + with names PilotCommands.py will be searched for the commands in + the order defined in the value. By default no extensions are given + -X, --commands value + where value is a comma separated list of pilot commands. By default + the list is CheckWorkerNode,InstallDIRAC,ConfigureBasics,RegisterPilot,CheckCECapabilities,CheckWNCapabilities, + ConfigureSite,ConfigureArchitecture,ConfigureCPURequirements,LaunchAgent - The pilot script by default performs initial sanity checks on WN, installs and configures - DIRAC and runs the Job Agent to execute pending workloads in the DIRAC WMS. - But, as said, all the actions are actually configurable. +The pilot script by default performs initial sanity checks on WN, installs and configures +DIRAC and runs the DIRAC JobAgent (https://github.com/DIRACGrid/DIRAC/blob/integration/src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py) to execute pending workloads in the DIRAC WMS. +But, as said, all the actions are actually configurable. """ from __future__ import absolute_import, division, print_function @@ -70,8 +71,12 @@ if not sys.stdin.isatty(): receivedContent = sys.stdin.read() log = RemoteLogger( - pilotParams.loggerURL, "Pilot", bufsize=pilotParams.loggerBufsize, - pilotUUID=pilotParams.pilotUUID, debugFlag=pilotParams.debugFlag, wnVO=pilotParams.wnVO, + pilotParams.loggerURL, + "Pilot", + bufsize=pilotParams.loggerBufsize, + pilotUUID=pilotParams.pilotUUID, + debugFlag=pilotParams.debugFlag, + wnVO=pilotParams.wnVO, ) log.info("Remote logger activated") log.buffer.write(receivedContent) diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index 3647ff30..b0e65a0f 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -1,20 +1,20 @@ -""" Definitions of a standard set of pilot commands +"""Definitions of a standard set of pilot commands - Each command is represented by a class inheriting from CommandBase class. - The command class constructor takes PilotParams object which is a data - structure which keeps common parameters across all the pilot commands. +Each command is represented by a class inheriting from CommandBase class. +The command class constructor takes PilotParams object which is a data +structure which keeps common parameters across all the pilot commands. - The constructor must call the superclass constructor with the PilotParams - object and the command name as arguments, e.g.:: +The constructor must call the superclass constructor with the PilotParams +object and the command name as arguments, e.g.:: - class InstallDIRAC(CommandBase): + class InstallDIRAC(CommandBase): - def __init__(self, pilotParams): - CommandBase.__init__(self, pilotParams, 'Install') - ... + def __init__(self, pilotParams): + CommandBase.__init__(self, pilotParams, 'Install') + ... - The command class must implement execute() method for the actual command - execution. +The command class must implement execute() method for the actual command +execution. """ from __future__ import absolute_import, division, print_function @@ -65,7 +65,7 @@ def __init__(self, pilotParams): def logFinalizer(func): """ - PilotCammand decorator. It marks a log file as final so no more messages should be written to it . + PilotCommand decorator. It marks a log file as final so no more messages should be written to it. Finalising is triggered by a return statement or any sys.exit() call, so a file might be incomplete if a command throws SystemExit exception with a code =! 0. @@ -104,6 +104,7 @@ def wrapper(self): raise finally: self.log.buffer.cancelTimer() + return wrapper @@ -206,8 +207,7 @@ def execute(self): class InstallDIRAC(CommandBase): - """ Source from CVMFS, or install locally - """ + """Source from CVMFS, or install locally""" def __init__(self, pilotParams): """c'tor""" @@ -215,8 +215,7 @@ def __init__(self, pilotParams): self.pp.rootPath = self.pp.pilotRootPath def _sourceEnvironmentFile(self): - """source the $DIRAC_RC_FILE and save the created environment in self.pp.installEnv - """ + """Source the $DIRAC_RC_FILE and save the created environment in self.pp.installEnv""" retCode, output = self.executeAndGetOutput("bash -c 'source $DIRAC_RC_PATH && env'", self.pp.installEnv) if retCode: @@ -249,12 +248,12 @@ def _saveEnvInFile(self, eFile="environmentSourceDirac"): fd.write(bl) def _getPreinstalledEnvScript(self): - """ Get preinstalled environment script if any """ + """Get preinstalled environment script if any""" self.log.debug("self.pp.preinstalledEnv = %s" % self.pp.preinstalledEnv) self.log.debug("self.pp.preinstalledEnvPrefix = %s" % self.pp.preinstalledEnvPrefix) self.log.debug("self.pp.CVMFS_locations = %s" % self.pp.CVMFS_locations) - + preinstalledEnvScript = self.pp.preinstalledEnv if not preinstalledEnvScript and self.pp.preinstalledEnvPrefix: version = self.pp.releaseVersion or "pro" @@ -265,7 +264,9 @@ def _getPreinstalledEnvScript(self): for CVMFS_location in self.pp.CVMFS_locations: version = self.pp.releaseVersion or "pro" arch = platform.system() + "-" + platform.machine() - preinstalledEnvScript = os.path.join(CVMFS_location, self.pp.releaseProject.lower() + "dirac", version, arch, "diracosrc") + preinstalledEnvScript = os.path.join( + CVMFS_location, self.pp.releaseProject.lower() + "dirac", version, arch, "diracosrc" + ) if os.path.isfile(preinstalledEnvScript): break @@ -280,11 +281,11 @@ def _getPreinstalledEnvScript(self): self.pp.preinstalledEnv = preinstalledEnvScript self.pp.installEnv["DIRAC_RC_PATH"] = preinstalledEnvScript - def _localInstallDIRAC(self): + """Install python3 version of DIRAC client""" + self.log.info("Installing DIRAC locally") - """Install python3 version of DIRAC client""" # default to limit the resources used during installation to what the pilot owns installEnv = { # see https://github.com/DIRACGrid/Pilot/issues/189 @@ -353,7 +354,7 @@ def _localInstallDIRAC(self): self.pp.installEnv["DIRAC_RC_PATH"] = os.path.join(os.getcwd(), "diracos/diracosrc") self._sourceEnvironmentFile() self._saveEnvInFile() - + # 7. pip install DIRAC[pilot] pipInstalling = "pip install %s " % self.pp.pipInstallOptions @@ -401,7 +402,7 @@ def execute(self): self.log.info("NOT sourcing: starting traditional DIRAC installation") self._localInstallDIRAC() return - + # Try sourcing from CVMFS self._getPreinstalledEnvScript() if not self.pp.preinstalledEnv: @@ -413,10 +414,8 @@ def execute(self): # environment variables to add? if self.pp.userEnvVariables: # User-requested environment variables (comma-separated, name and value separated by ":::") - newEnvVars = dict( - name.split(":::", 1) for name in self.pp.userEnvVariables.replace(" ", "").split(",") - ) - self.log.info("Adding env variable(s) to the environment", newEnvVars) + newEnvVars = dict(name.split(":::", 1) for name in self.pp.userEnvVariables.replace(" ", "").split(",")) + self.log.info("Adding env variable(s) to the environment : %s" % newEnvVars) self.pp.installEnv.update(newEnvVars) except OSError as e: @@ -524,8 +523,7 @@ def _getBasicsCFG(self): self.cfg.append('-o "/Resources/Computing/CEDefaults/VirtualOrganization=%s"' % self.pp.wnVO) def _getSecurityCFG(self): - """ Sets security-related env variables, if needed - """ + """Sets security-related env variables, if needed""" # Need to know host cert and key location in case they are needed if self.pp.useServerCertificate: self.cfg.append("--UseServerCertificate") @@ -765,9 +763,13 @@ def execute(self): batchSystemParams = self.pp.batchSystemInfo.get("Parameters", {}) self.cfg.append("-o /LocalSite/BatchSystemInfo/Parameters/Queue=%s" % batchSystemParams.get("Queue", "Unknown")) - self.cfg.append("-o /LocalSite/BatchSystemInfo/Parameters/BinaryPath=%s" % batchSystemParams.get("BinaryPath", "Unknown")) + self.cfg.append( + "-o /LocalSite/BatchSystemInfo/Parameters/BinaryPath=%s" % batchSystemParams.get("BinaryPath", "Unknown") + ) self.cfg.append("-o /LocalSite/BatchSystemInfo/Parameters/Host=%s" % batchSystemParams.get("Host", "Unknown")) - self.cfg.append("-o /LocalSite/BatchSystemInfo/Parameters/InfoPath=%s" % batchSystemParams.get("InfoPath", "Unknown")) + self.cfg.append( + "-o /LocalSite/BatchSystemInfo/Parameters/InfoPath=%s" % batchSystemParams.get("InfoPath", "Unknown") + ) self.cfg.append('-n "%s"' % self.pp.site) self.cfg.append('-S "%s"' % self.pp.setup) @@ -856,7 +858,7 @@ def execute(self): localArchitecture = localArchitecture.strip().split("\n")[-1].strip() cfg.append('-S "%s"' % self.pp.setup) cfg.append("-o /LocalSite/Architecture=%s" % localArchitecture) - + # add the local platform as determined by the platform module cfg.append("-o /LocalSite/Platform=%s" % platform.machine()) @@ -1130,6 +1132,8 @@ def _runNagiosProbes(self): self.log.error("Return code = %d: %s" % (retCode, str(output).split("\n", 1)[0])) retStatus = "error" + # TODO: Do something with the retStatus (for example: log it?) + # report results to pilot logger too. Like this: # "NagiosProbes", probeCmd, retStatus, str(retCode) + ' ' + output.split('\n',1)[0] diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 35d47969..f64888e5 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -1,5 +1,4 @@ -""" A set of common tools to be used in pilot commands -""" +"""A set of common tools to be used in pilot commands""" from __future__ import absolute_import, division, print_function @@ -41,15 +40,16 @@ def load_module_from_path(module_name, path_to_module): return module except ImportError: + def import_module(module): import imp impData = imp.find_module(module) return imp.load_module(module, *impData) - def load_module_from_path(module_name, path_to_module): import imp + fp, pathname, description = imp.find_module(module_name, [path_to_module]) try: return imp.load_module(module_name, fp, pathname, description) @@ -57,6 +57,7 @@ def load_module_from_path(module_name, path_to_module): if fp: fp.close() + try: from cStringIO import StringIO except ImportError: @@ -112,6 +113,12 @@ def parseVersion(releaseVersion): def pythonPathCheck(): + """Checks where python is located + + Raises: + - An exception if getting the env path raises an error + - An exception in case there is an error while removing duplicates in the PYTHONPATH + """ try: os.umask(18) # 022 pythonpath = os.getenv("PYTHONPATH", "").split(":") @@ -123,17 +130,17 @@ def pythonPathCheck(): if os.path.normpath(p) in sys.path: # In case a given directory is twice in PYTHONPATH it has to removed only once sys.path.remove(os.path.normpath(p)) - except Exception as x: - print(x) + except Exception as pathRemovalError: + print(pathRemovalError) print("[EXCEPTION-info] Failing path:", p, os.path.normpath(p)) print("[EXCEPTION-info] sys.path:", sys.path) - raise x - except Exception as x: - print(x) + raise pathRemovalError + except Exception as envError: + print(envError) print("[EXCEPTION-info] sys.executable:", sys.executable) print("[EXCEPTION-info] sys.version:", sys.version) print("[EXCEPTION-info] os.uname():", os.uname()) - raise x + raise envError def alarmTimeoutHandler(*args): @@ -192,10 +199,14 @@ def retrieveUrlTimeout(url, fileName, log, timeout=0): def safe_listdir(directory, timeout=60): - """ This is a "safe" list directory, + """This is a "safe" list directory, for lazily-loaded File Systems like CVMFS. There's by default a 60 seconds timeout. + .. warning:: + There is no distinction between an empty directory, and a non existent one. + It will return `[]` in both cases. + :param str directory: directory to list :param int timeout: optional timeout, in seconds. Defaults to 60. """ @@ -220,7 +231,7 @@ def listdir(directory): def getSubmitterInfo(ceName): """Get information about the submitter of the pilot. - Check the environment variables to determine the type of batch system and CE used + Check the environment variables to determine the type of batch system and CE used to submit the pilot being used and return this information in a tuple. """ @@ -292,7 +303,7 @@ def getSubmitterInfo(ceName): # HTCondor if "_CONDOR_JOB_AD" in os.environ: batchSystemType = "HTCondor" - batchSystemJobID = None # Not available in the environment + batchSystemJobID = None # Not available in the environment batchSystemParameters["InfoPath"] = os.environ["_CONDOR_JOB_AD"] flavour = "SSH%s" % batchSystemType @@ -334,25 +345,34 @@ def getSubmitterInfo(ceName): flavour = "CLOUD" pilotReference = os.environ["PILOT_UUID"] - return flavour, pilotReference, {"Type": batchSystemType, "JobID": batchSystemJobID, "Parameters": batchSystemParameters} + return ( + flavour, + pilotReference, + {"Type": batchSystemType, "JobID": batchSystemJobID, "Parameters": batchSystemParameters}, + ) def getFlavour(ceName): """Old method to get the flavour of the pilot. Deprecated. - + Please use getSubmitterInfo instead. """ - warnings.warn("getFlavour() is deprecated. Please use getSubmitterInfo() instead.", category=DeprecationWarning, stacklevel=2) + warnings.warn( + "getFlavour() is deprecated. Please use getSubmitterInfo() instead.", category=DeprecationWarning, stacklevel=2 + ) flavour, pilotReference, _ = getSubmitterInfo(ceName) return flavour, pilotReference + class ObjectLoader(object): """Simplified class for loading objects from a DIRAC installation. Example: + ```py ol = ObjectLoader() object, modulePath = ol.loadObject( 'pilot', 'LaunchAgent' ) + ``` """ def __init__(self, baseModules, log): @@ -415,8 +435,8 @@ def getCommand(params, commandName): """Get an instantiated command object for execution. Commands are looked in the following modules in the order: - 1. Commands - 2. pilotCommands + 1. `PilotCommands` + 2. `PilotCommands` """ extensions = params.commandExtensions modules = [m + "Commands" for m in extensions + ["pilot"]] @@ -506,7 +526,7 @@ def __init__( pilotUUID="unknown", flushInterval=10, bufsize=1000, - wnVO = "unknown", + wnVO="unknown", ): """ c'tor @@ -520,24 +540,28 @@ def __init__( sendToURL = partial(sendMessage, url, pilotUUID, wnVO, "sendMessage") self.buffer = FixedSizeBuffer(sendToURL, bufsize=bufsize, autoflush=flushInterval) - def debug(self, msg, header=True, sendPilotLog=False): + def debug(self, msg, header=True, _sendPilotLog=False): + # TODO: Send pilot log remotely? super(RemoteLogger, self).debug(msg, header) if ( self.isPilotLoggerOn and self.debugFlag ): # the -d flag activates this debug flag in CommandBase via PilotParams self.sendMessage(self.messageTemplate.format(level="DEBUG", message=msg)) - def error(self, msg, header=True, sendPilotLog=False): + def error(self, msg, header=True, _sendPilotLog=False): + # TODO: Send pilot log remotely? super(RemoteLogger, self).error(msg, header) if self.isPilotLoggerOn: self.sendMessage(self.messageTemplate.format(level="ERROR", message=msg)) - def warn(self, msg, header=True, sendPilotLog=False): + def warn(self, msg, header=True, _sendPilotLog=False): + # TODO: Send pilot log remotely? super(RemoteLogger, self).warn(msg, header) if self.isPilotLoggerOn: self.sendMessage(self.messageTemplate.format(level="WARNING", message=msg)) - def info(self, msg, header=True, sendPilotLog=False): + def info(self, msg, header=True, _sendPilotLog=False): + # TODO: Send pilot log remotely? super(RemoteLogger, self).info(msg, header) if self.isPilotLoggerOn: self.sendMessage(self.messageTemplate.format(level="INFO", message=msg)) @@ -678,22 +702,20 @@ def sendMessage(url, pilotUUID, wnVO, method, rawMessage): context = ssl.create_default_context() context.load_verify_locations(capath=caPath) - + message = json.dumps((json.dumps(rawMessage), pilotUUID, wnVO)) try: context.load_cert_chain(cert) # this is a proxy raw_data = {"method": method, "args": message} except IsADirectoryError: # assuming it'a dir containing cert and key - context.load_cert_chain( - os.path.join(cert, "hostcert.pem"), - os.path.join(cert, "hostkey.pem") - ) + context.load_cert_chain(os.path.join(cert, "hostcert.pem"), os.path.join(cert, "hostkey.pem")) raw_data = {"method": method, "args": message, "extraCredentials": '"hosts"'} - + if sys.version_info[0] == 3: data = urlencode(raw_data).encode("utf-8") # encode to bytes ! for python3 else: + # Python2 data = urlencode(raw_data) res = urlopen(url, data, context=context) @@ -703,7 +725,7 @@ def sendMessage(url, pilotUUID, wnVO, method, rawMessage): class CommandBase(object): """CommandBase is the base class for every command in the pilot commands toolbox""" - def __init__(self, pilotParams, dummy=""): + def __init__(self, pilotParams): """ Defines the classic pilot logger and the pilot parameters. Debug level of the Logger is controlled by the -d flag in pilotParams. @@ -765,7 +787,7 @@ def executeAndGetOutput(self, cmd, environDict=None): continue dataWasRead = True # Strip unicode replacement characters - outChunk = str(outChunk.replace(u"\ufffd", "")) + outChunk = str(outChunk.replace("\ufffd", "")) if stream == _p.stderr: sys.stderr.write(outChunk) sys.stderr.flush() @@ -948,7 +970,7 @@ def __init__(self): ("n:", "name=", "Set as Site Name"), ("o:", "option=", "Option=value to add"), ("m:", "maxNumberOfProcessors=", "specify a max number of processors to use by the payload inside a pilot"), - ("", "modules=", 'for installing non-released code'), + ("", "modules=", "for installing non-released code"), ( "", "userEnvVariables=", @@ -1014,34 +1036,60 @@ def __init__(self): self.installEnv["X509_USER_PROXY"] = self.certsLocation os.environ["X509_USER_PROXY"] = self.certsLocation + def __setSecurityDir(self, envName, dirLocation): + """Set the environment variable of the `envName`, and add it also to the Pilot Parameters + + Args: + envName (str): The environment to set (ex : `X509_USER_PROXY`) + dirLocation (str): The path that corresponds to the environment variable + """ + self.log.debug("Setting %s=%s" % (envName, dirLocation)) + self.installEnv[envName] = dirLocation + os.environ[envName] = dirLocation + def __checkSecurityDir(self, envName, dirName): + """For a given environment variable (that is not necessarily set in the OS), checks if it exists *and* is not empty. + + .. example:: + ``` + self.__checkSecurityDir("X509_VOMSES", "vomses") + ``` + It will check if `X509_VOMSES` is set, if not, check if one of the CVMFS_locations with "vomses" is a valid candidate. + If let's say `/cvmfs/dirac.egi.eu/etc/grid-security/vomses` exists, *and* is not empty, sets the OS environment variable `X509_VOMSES` to `/cvmfs/dirac.egi.eu/etc/grid-security/vomses`. + + + .. warning:: + If none of the candidates work, it will stop the program. - # try and find it + Args: + envName (str): The environment name to try + dirName (str): The target folder + """ + + # Else, try to find it for candidate in self.CVMFS_locations: - candidateDir = os.path.join(candidate, - 'etc/grid-security', - dirName) - self.log.debug( - "Candidate directory for %s is %s" - % (envName, candidateDir) - ) + candidateDir = os.path.join(candidate, "etc/grid-security", dirName) + self.log.debug("Candidate directory for %s is %s" % (envName, candidateDir)) + + # Checks if the directory exists *and* isn't empty if safe_listdir(candidateDir): self.log.debug("Setting %s=%s" % (envName, candidateDir)) - self.installEnv[envName] = candidateDir - os.environ[envName] = candidateDir - return - self.log.debug("%s empty, or not found, or not a directory" % candidateDir) + # Set the environment variables to the candidate + self.__setSecurityDir(envName, candidateDir) + break + self.log.debug("%s not found or not a directory" % candidateDir) + # Check first if the environment variable is set + # If so, just return if envName in os.environ and safe_listdir(os.environ[envName]): self.log.debug( - "%s is set in the host environment as %s, aligning installEnv to it" - % (envName, os.environ[envName]) + "%s is set in the host environment as %s, aligning installEnv to it" % (envName, os.environ[envName]) ) else: + # None of the candidates exists, stop the program. self.log.error("Could not find/set %s" % envName) sys.exit(1) - def __initCommandLine1(self): """Parses and interpret options on the command line: first pass (essential things)""" @@ -1170,8 +1218,7 @@ def __loadJSON(self): """ Load JSON file and return a dict content. - :return: - :rtype: + :return: None """ self.log.debug("JSON file loaded: %s" % self.pilotCFGFile) @@ -1265,7 +1312,6 @@ def __initJSON2(self): self.CVMFS_locations = pilotOptions["CVMFS_locations"].replace(" ", "").split(",") self.log.debug("CVMFS locations: %s" % self.CVMFS_locations) - def getPilotOptionsDict(self): """ Get pilot option dictionary by searching paths in a certain order (commands, logging etc.). @@ -1301,7 +1347,7 @@ def __getVO(self): with open(cert, "rb") as fp: return getVO(fp.read()) except IOError as err: - self.log.error("Could not read a proxy, setting vo to 'unknown': ", os.strerror(err.errno)) + self.log.error("Could not read a proxy, setting vo to 'unknown': %s" % os.strerror(err.errno)) else: self.log.error("Could not locate a proxy via X509_USER_PROXY") @@ -1541,7 +1587,6 @@ def __initJSON(self): def __ceType(self): """ Set CE type and setup. - """ self.log.debug("CE name: %s" % self.ceName) if self.ceName: diff --git a/Pilot/proxyTools.py b/Pilot/proxyTools.py index a3bd262b..a5fa652e 100644 --- a/Pilot/proxyTools.py +++ b/Pilot/proxyTools.py @@ -1,5 +1,4 @@ -""" few functions for dealing with proxies -""" +"""few functions for dealing with proxies""" from __future__ import absolute_import, division, print_function @@ -29,6 +28,18 @@ def findExtension(oid, lines): def getVO(proxy_data): + """Fetches the VO in a chain certificate + + Args: + proxy_data (bytes): Bytes for the proxy chain + + Raises: + Exception: Any error related to openssl + NotImplementedError: Not documented error + + Returns: + str: A VO + """ chain = re.findall(br"-----BEGIN CERTIFICATE-----\n.+?\n-----END CERTIFICATE-----", proxy_data, flags=re.DOTALL) for cert in chain: diff --git a/Pilot/tests/Test_Pilot.py b/Pilot/tests/Test_Pilot.py index 1f5c5986..8a1b75a1 100644 --- a/Pilot/tests/Test_Pilot.py +++ b/Pilot/tests/Test_Pilot.py @@ -1,5 +1,4 @@ -""" Test class for Pilot -""" +"""Test class for Pilot""" from __future__ import absolute_import, division, print_function diff --git a/Pilot/tests/Test_simplePilotLogger.py b/Pilot/tests/Test_simplePilotLogger.py index 9c176c82..1fc448ae 100644 --- a/Pilot/tests/Test_simplePilotLogger.py +++ b/Pilot/tests/Test_simplePilotLogger.py @@ -10,9 +10,9 @@ import tempfile try: - from Pilot.pilotTools import CommandBase, PilotParams, Logger + from Pilot.pilotTools import CommandBase, Logger, PilotParams except ImportError: - from pilotTools import CommandBase, PilotParams, Logger + from pilotTools import CommandBase, Logger, PilotParams import unittest @@ -58,7 +58,7 @@ def test_getOptionForPaths(self): "/%s/Pilot" % setup, "/%s/Defaults/Pilot" % vo, "/%s/%s/Pilot" % (vo, setup), - "/%s/Pilot" % vo + "/%s/Pilot" % vo, ] with open(jsonFile, "r") as fp: jsonDict = json.load(fp) @@ -90,7 +90,7 @@ def test_pilotOptions(self, argvmock, mockPaths): "/%s/Pilot" % setup, "/%s/Defaults/Pilot" % vo, "/%s/%s/Pilot" % (vo, setup), - "/%s/Pilot" % vo + "/%s/Pilot" % vo, ] mockPaths.return_value = paths os.environ["X509_CERT_DIR"] = os.getcwd() @@ -111,7 +111,7 @@ def test_pilotOptions(self, argvmock, mockPaths): self.assertEqual(res.get("UploadPath"), "/gridpp/pilotlogs/") self.assertTrue("Commands" in res) self.assertEqual(res["Commands"].get(pp.gridCEType), lTESTcommands) - self.assertEqual(', '.join(pp.commands), lTESTcommands) + self.assertEqual(", ".join(pp.commands), lTESTcommands) self.assertEqual(pp.releaseVersion, "VAR_DIRAC_VERSION") @@ -120,9 +120,8 @@ def setUp(self): # These temporary files, opened in a binary mode, will act as standard stream pipes for `Popen` self.stdout_mock = tempfile.NamedTemporaryFile(mode="rb+", delete=False) self.stderr_mock = tempfile.NamedTemporaryFile(mode="rb+", delete=False) - os.environ["X509_CERT_DIR"] = '/some/thing/' - os.environ["X509_USER_PROXY"] = '/some/thing' - + os.environ["X509_CERT_DIR"] = "/some/thing/" + os.environ["X509_USER_PROXY"] = "/some/thing" def tearDown(self): # At the end of the test, we'll close and remove the created files @@ -144,7 +143,7 @@ def test_executeAndGetOutput(self, popenMock, argvmock): "-F", "tests/pilot.json", ] - + for size in [1000, 1024, 1025, 2005]: random_str = "".join(random.choice(string.ascii_letters + "\n") for i in range(size)) if sys.version_info.major == 3: @@ -173,11 +172,5 @@ def test_executeAndGetOutput(self, popenMock, argvmock): self.stdout_mock.truncate() self.stderr_mock.truncate() - -class TestSimplePilotLogger(unittest.TestCase): - def test_SimplePilotLogger(self): - uuid = "37356d94-15c6-11e6-a600-606c663dde16" - - if __name__ == "__main__": unittest.main()