diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 629d740e..e0f9221b 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -24,6 +24,7 @@ jobs: fail-fast: false matrix: python: + - 2.7.18 - 3.6.15 - 3.9.17 @@ -54,6 +55,7 @@ jobs: fail-fast: false matrix: python: + - 2.7.18 - 3.6.15 - 3.9.17 @@ -80,6 +82,7 @@ jobs: fail-fast: false matrix: python: + - 2.7.18 - 3.6.15 - 3.9.17 diff --git a/.pylintrc b/.pylintrc index 7c158f91..2bae85fd 100644 --- a/.pylintrc +++ b/.pylintrc @@ -18,3 +18,7 @@ dummy-variables=_ disable= invalid-name, line-too-long, # would be nice to remove this one + consider-using-f-string, # python2/3 support + unspecified-encoding, # python2/3 support + super-with-arguments, # python2/3 support + redefined-builtin, # python2/3 support \ No newline at end of file diff --git a/Pilot/dirac-pilot.py b/Pilot/dirac-pilot.py index b3f0866c..9c434c97 100644 --- a/Pilot/dirac-pilot.py +++ b/Pilot/dirac-pilot.py @@ -19,19 +19,36 @@ But, as said, all the actions are actually configurable. """ +from __future__ import absolute_import, division, print_function + import os import sys import time -from io import StringIO -from .pilotTools import ( - Logger, - PilotParams, - RemoteLogger, - getCommand, - pythonPathCheck, -) +############################ +# python 2 -> 3 "hacks" + +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO +try: + from Pilot.pilotTools import ( + Logger, + PilotParams, + RemoteLogger, + getCommand, + pythonPathCheck, + ) +except ImportError: + from pilotTools import ( + Logger, + PilotParams, + RemoteLogger, + getCommand, + pythonPathCheck, + ) ############################ if __name__ == "__main__": diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index 7411f4a5..4815f44b 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -17,6 +17,8 @@ def __init__(self, pilotParams): execution. """ +from __future__ import absolute_import, division, print_function + import filecmp import os import platform @@ -26,18 +28,39 @@ def __init__(self, pilotParams): import sys import time import traceback +import subprocess from collections import Counter -from http.client import HTTPSConnection -from shlex import quote -from .pilotTools import ( +############################ +# python 2 -> 3 "hacks" +try: + # For Python 3.0 and later + from http.client import HTTPSConnection +except ImportError: + # Fall back to Python 2 + from httplib import HTTPSConnection + +try: + from shlex import quote +except ImportError: + from pipes import quote + +try: + from Pilot.pilotTools import ( + CommandBase, + getSubmitterInfo, + retrieveUrlTimeout, + safe_listdir, + sendMessage, + ) +except ImportError: + from pilotTools import ( CommandBase, getSubmitterInfo, retrieveUrlTimeout, safe_listdir, sendMessage, ) - ############################ @@ -260,7 +283,7 @@ def _getPreinstalledEnvScript(self): self.pp.installEnv["DIRAC_RC_PATH"] = preinstalledEnvScript def _localInstallDIRAC(self): - """Install DIRAC client""" + """Install python3 version of DIRAC client""" self.log.info("Installing DIRAC locally") @@ -273,7 +296,10 @@ def _localInstallDIRAC(self): # 1. Get the DIRACOS installer name # curl -O -L https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/DIRACOS-Linux-$(uname -m).sh - machine = os.uname().machine + try: + machine = os.uname().machine # py3 + except AttributeError: + machine = os.uname()[4] # py2 installerName = "DIRACOS-Linux-%s.sh" % machine diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 7d9530ad..8afe0f62 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -1,8 +1,9 @@ """A set of common tools to be used in pilot commands""" +from __future__ import absolute_import, division, print_function + import fcntl import getopt -import importlib.util import json import os import re @@ -15,23 +16,81 @@ import warnings from datetime import datetime from functools import partial, wraps -from importlib import import_module -from io import StringIO -from threading import RLock, Timer -from urllib.error import HTTPError, URLError -from urllib.parse import urlencode -from urllib.request import urlopen +from threading import RLock -from .proxyTools import getVO +############################ +# python 2 -> 3 "hacks" +try: + from urllib.error import HTTPError, URLError + from urllib.parse import urlencode + from urllib.request import urlopen +except ImportError: + from urllib import urlencode -# Utilities functions + from urllib2 import HTTPError, URLError, urlopen + +try: + import importlib.util + from importlib import import_module + def load_module_from_path(module_name, path_to_module): + spec = importlib.util.spec_from_file_location(module_name, path_to_module) # pylint: disable=no-member + module = importlib.util.module_from_spec(spec) # pylint: disable=no-member + spec.loader.exec_module(module) + return module -def load_module_from_path(module_name, path_to_module): - spec = importlib.util.spec_from_file_location(module_name, path_to_module) # pylint: disable=no-member - module = importlib.util.module_from_spec(spec) # pylint: disable=no-member - spec.loader.exec_module(module) - return module +except ImportError: + + def import_module(module): + import imp + + impData = imp.find_module(module) + return imp.load_module(module, *impData) + + def load_module_from_path(module_name, path_to_module): + import imp + + fp, pathname, description = imp.find_module(module_name, [path_to_module]) + try: + return imp.load_module(module_name, fp, pathname, description) + finally: + if fp: + fp.close() + + +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + +try: + basestring # pylint: disable=used-before-assignment +except NameError: + basestring = str + +try: + from Pilot.proxyTools import getVO +except ImportError: + from proxyTools import getVO + +try: + FileNotFoundError # pylint: disable=used-before-assignment + # because of https://github.com/PyCQA/pylint/issues/6748 +except NameError: + FileNotFoundError = OSError + +try: + IsADirectoryError # pylint: disable=used-before-assignment +except NameError: + IsADirectoryError = IOError + +# Timer 2.7 and < 3.3 versions issue where Timer is a function +if sys.version_info.major == 2 or sys.version_info.major == 3 and sys.version_info.minor < 3: + from threading import _Timer as Timer # pylint: disable=no-name-in-module +else: + from threading import Timer + +# Utilities functions def parseVersion(releaseVersion): @@ -340,7 +399,7 @@ def loadModule(self, modName, hideExceptions=False): def __recurseImport(self, modName, parentModule=None, hideExceptions=False): """Internal function to load modules""" - if isinstance(modName, str): + if isinstance(modName, basestring): modName = modName.split(".") try: if parentModule: @@ -654,7 +713,11 @@ def sendMessage(url, pilotUUID, wnVO, method, rawMessage): context.load_cert_chain(os.path.join(cert, "hostcert.pem"), os.path.join(cert, "hostkey.pem")) raw_data = {"method": method, "args": message, "extraCredentials": '"hosts"'} - data = urlencode(raw_data).encode("utf-8") # encode to bytes + if sys.version_info.major == 3: + data = urlencode(raw_data).encode("utf-8") # encode to bytes ! for python3 + else: + # Python2 + data = urlencode(raw_data) res = urlopen(url, data, context=context) res.close() @@ -724,7 +787,17 @@ def executeAndGetOutput(self, cmd, environDict=None): if not outChunk: continue dataWasRead = True - outChunk = str(outChunk.replace("\ufffd", "")) # Ensure it's a string + if sys.version_info.major == 2: + # Ensure outChunk is unicode in Python 2 + if isinstance(outChunk, str): + outChunk = outChunk.decode("utf-8") + # Strip unicode replacement characters + # Ensure correct type conversion in Python 2 + outChunk = str(outChunk.replace(u"\ufffd", "")) + # Avoid potential str() issues in Py2 + outChunk = unicode(outChunk) # pylint: disable=undefined-variable + else: + outChunk = str(outChunk.replace("\ufffd", "")) # Python 3: Ensure it's a string if stream == _p.stderr: sys.stderr.write(outChunk) @@ -1382,7 +1455,7 @@ def __initJSON(self): # Commands first # FIXME: pilotSynchronizer() should publish these as comma-separated lists. We are ready for that. try: - if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType], str): + if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType], basestring): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType].split(",") @@ -1393,7 +1466,7 @@ def __initJSON(self): ] except KeyError: try: - if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"], str): + if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"], basestring): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"].split(",") @@ -1404,7 +1477,7 @@ def __initJSON(self): ] except KeyError: try: - if isinstance(self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType], str): + if isinstance(self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType], basestring): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType].split(",") @@ -1415,7 +1488,7 @@ def __initJSON(self): ] except KeyError: try: - if isinstance(self.pilotJSON["Defaults"]["Commands"]["Defaults"], str): + if isinstance(self.pilotJSON["Defaults"]["Commands"]["Defaults"], basestring): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Defaults"]["Commands"]["Defaults"].split(",") ] @@ -1431,7 +1504,7 @@ def __initJSON(self): # pilotSynchronizer() can publish this as a comma separated list. We are ready for that. try: if isinstance( - self.pilotJSON["Setups"][self.setup]["CommandExtensions"], str + self.pilotJSON["Setups"][self.setup]["CommandExtensions"], basestring ): # In the specific setup? self.commandExtensions = [ str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["CommandExtensions"].split(",") @@ -1443,7 +1516,7 @@ def __initJSON(self): except KeyError: try: if isinstance( - self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"], str + self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"], basestring ): # Or in the defaults section? self.commandExtensions = [ str(pv).strip() for pv in self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"].split(",") @@ -1460,7 +1533,7 @@ def __initJSON(self): # pilotSynchronizer() can publish this as a comma separated list. We are ready for that try: if isinstance( - self.pilotJSON["ConfigurationServers"], str + self.pilotJSON["ConfigurationServers"], basestring ): # Generic, there may also be setup-specific ones self.configServer = ",".join( [str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"].split(",")] @@ -1471,7 +1544,7 @@ def __initJSON(self): pass try: # now trying to see if there is setup-specific ones if isinstance( - self.pilotJSON["Setups"][self.setup]["ConfigurationServer"], str + self.pilotJSON["Setups"][self.setup]["ConfigurationServer"], basestring ): # In the specific setup? self.configServer = ",".join( [str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["ConfigurationServer"].split(",")] @@ -1483,7 +1556,7 @@ def __initJSON(self): except KeyError: # and if it doesn't exist try: if isinstance( - self.pilotJSON["Setups"]["Defaults"]["ConfigurationServer"], str + self.pilotJSON["Setups"]["Defaults"]["ConfigurationServer"], basestring ): # Is there one in the defaults section? self.configServer = ",".join( [ diff --git a/Pilot/proxyTools.py b/Pilot/proxyTools.py index 8792a34a..a5fa652e 100644 --- a/Pilot/proxyTools.py +++ b/Pilot/proxyTools.py @@ -1,5 +1,7 @@ """few functions for dealing with proxies""" +from __future__ import absolute_import, division, print_function + import re from base64 import b16decode from subprocess import PIPE, Popen diff --git a/Pilot/tests/Test_Pilot.py b/Pilot/tests/Test_Pilot.py index bf0f54c1..8a1b75a1 100644 --- a/Pilot/tests/Test_Pilot.py +++ b/Pilot/tests/Test_Pilot.py @@ -1,5 +1,7 @@ """Test class for Pilot""" +from __future__ import absolute_import, division, print_function + import json import os import shutil @@ -10,8 +12,8 @@ # imports import unittest -from ..pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes -from ..pilotTools import PilotParams +from Pilot.pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes +from Pilot.pilotTools import PilotParams class PilotTestCase(unittest.TestCase): diff --git a/Pilot/tests/Test_proxyTools.py b/Pilot/tests/Test_proxyTools.py index dc6c3a95..7a8688cb 100644 --- a/Pilot/tests/Test_proxyTools.py +++ b/Pilot/tests/Test_proxyTools.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, division, print_function import os import shlex @@ -5,9 +6,19 @@ import subprocess import sys import unittest -from unittest.mock import patch -from ..proxyTools import getVO, parseASN1 +############################ +# python 2 -> 3 "hacks" +try: + from Pilot.proxyTools import getVO, parseASN1 +except ImportError: + from proxyTools import getVO, parseASN1 + +try: + from unittest.mock import patch +except ImportError: + from mock import patch + class TestProxyTools(unittest.TestCase): def test_getVO(self): @@ -81,7 +92,46 @@ def __createFakeProxy(self, proxyFile): """ Create a fake proxy locally. """ - basedir = os.path.dirname(__file__) - shutil.copy(basedir + "/certs/voms/proxy.pem", proxyFile) - return 0 + basedir = os.path.dirname(__file__) + shutil.copy(basedir + "/certs/user/userkey.pem", basedir + "/certs/user/userkey400.pem") + os.chmod(basedir + "/certs/user/userkey400.pem", 0o400) + ret = self.createFakeProxy( + basedir + "/certs/user/usercert.pem", + basedir + "/certs/user/userkey400.pem", + "fakeserver.cern.ch:15000", + "fakevo", + basedir + "/certs//host/hostcert.pem", + basedir + "/certs/host/hostkey.pem", + basedir + "/certs/ca", + proxyFile, + ) + os.remove(basedir + "/certs/user/userkey400.pem") + return ret + + def createFakeProxy(self, usercert, userkey, serverURI, vo, hostcert, hostkey, CACertDir, proxyfile): + """ + voms-proxy-fake --cert usercert.pem + --key userkey.pem + -rfc + -fqan "/fakevo/Role=user/Capability=NULL" + -uri fakeserver.cern.ch:15000 + -voms fakevo + -hostcert hostcert.pem + -hostkey hostkey.pem + -certdir ca + """ + opt = ( + '--cert %s --key %s -rfc -fqan "/fakevo/Role=user/Capability=NULL" -uri %s -voms %s -hostcert %s' + " -hostkey %s -certdir %s -out %s" + % (usercert, userkey, serverURI, vo, hostcert, hostkey, CACertDir, proxyfile) + ) + proc = subprocess.Popen( + shlex.split("voms-proxy-fake " + opt), + bufsize=1, + stdout=sys.stdout, + stderr=sys.stderr, + universal_newlines=True, + ) + proc.communicate() + return proc.returncode diff --git a/Pilot/tests/Test_simplePilotLogger.py b/Pilot/tests/Test_simplePilotLogger.py index aec1b191..1fc448ae 100644 --- a/Pilot/tests/Test_simplePilotLogger.py +++ b/Pilot/tests/Test_simplePilotLogger.py @@ -1,14 +1,26 @@ #!/usr/bin/env python +from __future__ import absolute_import, division, print_function + import json import os import random import string +import sys import tempfile + +try: + from Pilot.pilotTools import CommandBase, Logger, PilotParams +except ImportError: + from pilotTools import CommandBase, Logger, PilotParams + import unittest -from unittest.mock import patch -from ..pilotTools import CommandBase, Logger, PilotParams +try: + from unittest.mock import patch +except ImportError: + from mock import patch + class TestPilotParams(unittest.TestCase): @patch("sys.argv") @@ -134,10 +146,16 @@ def test_executeAndGetOutput(self, popenMock, argvmock): for size in [1000, 1024, 1025, 2005]: random_str = "".join(random.choice(string.ascii_letters + "\n") for i in range(size)) - random_bytes = random_str.encode("UTF-8") - self.stdout_mock.write(random_bytes) + if sys.version_info.major == 3: + random_bytes = random_str.encode("UTF-8") + self.stdout_mock.write(random_bytes) + else: + self.stdout_mock.write(random_str) self.stdout_mock.seek(0) - self.stderr_mock.write("Errare humanum est!".encode("UTF-8")) + if sys.version_info.major == 3: + self.stderr_mock.write("Errare humanum est!".encode("UTF-8")) + else: + self.stderr_mock.write("Errare humanum est!") self.stderr_mock.seek(0) pp = PilotParams() diff --git a/environment.yml b/environment.yml index 72e2765e..41e0a564 100644 --- a/environment.yml +++ b/environment.yml @@ -11,6 +11,7 @@ dependencies: - requests # testing and development - pycodestyle + - caniusepython3 - coverage - mock - pylint