diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 64d1411f..a3ff15fd 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -14,7 +14,7 @@ jobs: pilot_schema: - pilot_oldSchema.json - pilot_newSchema.json - VO: + VO: - dteam - gridpp @@ -53,12 +53,13 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::${{ matrix.dirac_branch }} -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=${{ matrix.VO }} --pilotUUID="${pilotUUID}" --debug - release_prod_pre-cvmfs: + release_prod_pre-cvmfs_CEs: runs-on: ubuntu-latest strategy: @@ -68,7 +69,13 @@ jobs: - pilot_newSchema.json dirac_version: - prod - - pre + - old + ce: + - jenkins.cern.ch + - jenkins-singularity.cern.ch + - jenkins-mp.cern.ch + - jenkins-mp-pool.cern.ch + - jenkins-mp-pool-singularity.cern.ch steps: - uses: actions/checkout@v4 @@ -92,9 +99,9 @@ jobs: chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem if [ "${{ matrix.dirac_version }}" == "prod" ]; then - version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep -v 'a' | tail -n 1) + version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v9' | tail -n 1) else - version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'a' | tail -n 1) + version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v8' | tail -n 1) fi source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc pilot_proxy_file=$(mktemp) @@ -109,62 +116,11 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) - X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --architectureScript=dirac-platform --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --wnVO=dteam --pilotUUID="${pilotUUID}" --debug - - - release_prod-cvmfs_no_env_CEs: - runs-on: ubuntu-latest - - strategy: - matrix: - ce: - - jenkins.cern.ch - - jenkins-singularity.cern.ch - - jenkins-mp.cern.ch - - jenkins-mp-pool.cern.ch - - jenkins-mp-pool-singularity.cern.ch - - jenkins-revised.cern.ch - - steps: - - uses: actions/checkout@v4 - - uses: cvmfs-contrib/github-action-cvmfs@v3 - - - name: Test CernVM-FS - run: ls /cvmfs/dirac.egi.eu - - - name: Tests - env: - HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} - HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} - run: | - mkdir -p ${GITHUB_WORKSPACE}/Client/etc/grid-security - cd ${GITHUB_WORKSPACE}/Client - echo "$HOSTCERT_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem - echo "$HOSTKEY_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem - chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem - chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem - version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'a' | tail -n 1) - source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc - pilot_proxy_file=$(mktemp) - dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg - dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg - cd ${GITHUB_WORKSPACE}/Pilot - cp ../tests/CI/pilot_newSchema.json pilot.json - sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json - sed -i "s/VAR_JENKINS_CE/${{ matrix.ce }}/g" pilot.json - sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json - sed -i "s/VAR_DIRAC_VERSION/${version}/g" pilot.json - sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json - sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json - sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - g_job="testintegrationworkflow${GITHUB_JOB//-/}" - pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" - pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) - X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -S DIRAC-Certification -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --wnVO=dteam -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug - + X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -S DIRAC-Certification -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --architectureScript=dirac-platform --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --wnVO=dteam --pilotUUID="${pilotUUID}" --debug matching: runs-on: ubuntu-latest @@ -186,6 +142,9 @@ jobs: env: HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} + X509_CERT_DIR: /cvmfs/grid.cern.ch/etc/grid-security/certificates + X509_VOMS_DIR: /cvmfs/grid.cern.ch/etc/grid-security/vomsdir + DIRAC_VOMSES: /cvmfs/grid.cern.ch/etc/grid-security/vomses run: | mkdir -p ${GITHUB_WORKSPACE}/Client/etc/grid-security cd ${GITHUB_WORKSPACE}/Client @@ -193,7 +152,7 @@ jobs: echo "$HOSTKEY_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem - version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'a' | tail -n 1) + version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep -v 'a' | tail -n 1) source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc pilot_proxy_file=$(mktemp) dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg @@ -207,23 +166,23 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::integration -M 1 -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=dteam --pilotUUID="${pilotUUID}" --debug - release_prod_pre-cvmfs_matching: runs-on: ubuntu-latest strategy: matrix: - VO: + VO: - dteam - gridpp dirac_version: - prod - - pre + - old steps: - uses: actions/checkout@v4 @@ -236,6 +195,9 @@ jobs: env: HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} + X509_CERT_DIR: /cvmfs/grid.cern.ch/etc/grid-security/certificates + X509_VOMS_DIR: /cvmfs/grid.cern.ch/etc/grid-security/vomsdir + DIRAC_VOMSES: /cvmfs/grid.cern.ch/etc/grid-security/vomses run: | mkdir -p ${GITHUB_WORKSPACE}/Client/etc/grid-security cd ${GITHUB_WORKSPACE}/Client @@ -245,9 +207,9 @@ jobs: chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem cd ${GITHUB_WORKSPACE}/Pilot if [ "${{ matrix.dirac_version }}" == "prod" ]; then - version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep -v 'a' | tail -n 1) + version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v9' | tail -n 1) else - version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'a' | tail -n 1) + version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v8' | tail -n 1) fi source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc pilot_proxy_file=$(mktemp) @@ -262,12 +224,12 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --wnVO=${{ matrix.VO }} -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug - release_prod-cvmfs_matching_CEs: runs-on: ubuntu-latest @@ -279,7 +241,6 @@ jobs: - jenkins-mp-full.cern.ch - jenkins-mp-pool-full.cern.ch - jenkins-mp-pool-singularity-full.cern.ch - - jenkins-revised.cern.ch steps: - uses: actions/checkout@v4 @@ -292,6 +253,9 @@ jobs: env: HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} + X509_CERT_DIR: /cvmfs/grid.cern.ch/etc/grid-security/certificates + X509_VOMS_DIR: /cvmfs/grid.cern.ch/etc/grid-security/vomsdir + DIRAC_VOMSES: /cvmfs/grid.cern.ch/etc/grid-security/vomses run: | mkdir -p ${GITHUB_WORKSPACE}/Client/etc/grid-security cd ${GITHUB_WORKSPACE}/Client @@ -299,7 +263,7 @@ jobs: echo "$HOSTKEY_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem - version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'a' | tail -n 1) + version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep -v 'a' | tail -n 1) source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc pilot_proxy_file=$(mktemp) dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg @@ -313,12 +277,12 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --StopAfterFailedMatches=1 --wnVO=dteam -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug - ################################## ### # extensions tests ################################## @@ -375,63 +339,13 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --architectureScript=dirac-architecture --wnVO=LHCb -o lbRunOnly --pilotUUID="${pilotUUID}" --debug - ext-lhcb_integration_dirac_installer_no_env: - runs-on: ubuntu-latest - - strategy: - matrix: - pilot_schema: - - pilot_oldSchema.json - - pilot_newSchema.json - ce: - - jenkins-lhcb-d.cern.ch - - jenkins-lhcb-dd.cern.ch - - steps: - - uses: actions/checkout@v4 - - uses: cvmfs-contrib/github-action-cvmfs@v3 - - - name: Test CernVM-FS - run: ls /cvmfs/lhcb.cern.ch - - - name: Tests - env: - HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} - HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} - run: | - mkdir -p ${GITHUB_WORKSPACE}/Client/etc/grid-security - cd ${GITHUB_WORKSPACE}/Client - echo "$HOSTCERT_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem - echo "$HOSTKEY_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem - chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem - chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem - version=$(echo $(curl -s "https://gitlab.cern.ch/api/v4/projects/3588/releases") | jq -r '.[].name' | sort -V | grep -v 'a' | tail -n 1) - source /cvmfs/lhcb.cern.ch/lhcbdirac/${version}/Linux-x86_64/diracosrc - pilot_proxy_file=$(mktemp) - dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg - dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg - X509_CERT_DIR="/cvmfs/lhcb.cern.ch/etc/grid-security/certificates" X509_VOMS_DIR="/cvmfs/lhcb.cern.ch/etc/grid-security/vomsdir" DIRAC_VOMSES="/cvmfs/lhcb.cern.ch/etc/grid-security/vomses" dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg - cd ${GITHUB_WORKSPACE}/Pilot - curl https://lhcbdirac.s3.cern.ch/Pilot3/LHCbPilotCommands.py -o LHCbPilotCommands.py - cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json - sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json - sed -i "s/VAR_JENKINS_CE/${{ matrix.ce }}/g" pilot.json - sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json - sed -i "s/VAR_DIRAC_VERSION/${version}/g" pilot.json - sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json - sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json - sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - g_job="testintegrationworkflow${GITHUB_JOB//-/}" - pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" - pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) - X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --wnVO=LHCb -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug - - ext-lhcb_integration_no_env: + ext-lhcb_integration_cvmfs: runs-on: ubuntu-latest strategy: @@ -454,6 +368,9 @@ jobs: env: HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} + X509_CERT_DIR: /cvmfs/lhcb.cern.ch/etc/grid-security/certificates + X509_VOMS_DIR: /cvmfs/lhcb.cern.ch/etc/grid-security/vomsdir + DIRAC_VOMSES: /cvmfs/lhcb.cern.ch/etc/grid-security/vomses run: | mkdir -p ${GITHUB_WORKSPACE}/Client/etc/grid-security cd ${GITHUB_WORKSPACE}/Client diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index 4815f44b..5032a02d 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -508,6 +508,8 @@ def _getBasicsCFG(self): self.cfg.append('-S "%s"' % self.pp.setup) if self.pp.configServer: self.cfg.append('-C "%s"' % self.pp.configServer) + if self.pp.preferredURLPatterns: + self.cfg.append("-o /DIRAC/PreferredURLPatterns=%s" % quote(",".join(self.pp.preferredURLPatterns))) if self.pp.releaseProject: self.cfg.append('-e "%s"' % self.pp.releaseProject) self.cfg.append("-o /LocalSite/ReleaseProject=%s" % self.pp.releaseProject) @@ -834,7 +836,7 @@ def execute(self): archScript = self.pp.architectureScript if self.pp.architectureScript.split(" ")[0] == "dirac-apptainer-exec": archScript = " ".join(self.pp.architectureScript.split(" ")[1:]) - + architectureCmd = "%s %s -ddd" % (archScript, " ".join(cfg)) if self.pp.architectureScript.split(" ")[0] == "dirac-apptainer-exec": @@ -872,10 +874,12 @@ def execute(self): return localArchitecture + class ConfigureArchitectureWithoutCLI(CommandBase): """This command determines the platform. Separated from the ConfigureDIRAC command for easier extensibility. """ + def getPlatformString(self): # Modified to return our desired platform string, R. Graciani platformTuple = (platform.system(), platform.machine()) @@ -903,7 +907,6 @@ def execute(self): self.log.error("Configuration error [ERROR %s]" % str(e)) self.exitWithError(1) - cfg = ["-FDMH"] # force update, skip CA checks, skip CA download, skip VOMS if self.pp.useServerCertificate: cfg.append("--UseServerCertificate") diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 8afe0f62..11345f46 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -793,7 +793,7 @@ def executeAndGetOutput(self, cmd, environDict=None): outChunk = outChunk.decode("utf-8") # Strip unicode replacement characters # Ensure correct type conversion in Python 2 - outChunk = str(outChunk.replace(u"\ufffd", "")) + outChunk = str(outChunk.replace("\ufffd", "")) # Avoid potential str() issues in Py2 outChunk = unicode(outChunk) # pylint: disable=undefined-variable else: @@ -908,6 +908,7 @@ def __init__(self): self.site = "" self.setup = "" self.configServer = "" + self.preferredURLPatterns = "" self.ceName = "" self.ceType = "" self.queueName = "" @@ -1306,6 +1307,8 @@ def __initJSON2(self): # the generic one (a list): self.configServer = ",".join([str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"]]) + self.preferredURLPatterns = self.pilotJSON.get("PreferredURLPatterns", self.preferredURLPatterns) + # version(a comma separated values in a string). We take the first one. (the default value defined in the code) dVersion = pilotOptions.get("Version", self.releaseVersion) if dVersion: diff --git a/Pilot/tests/Test_simplePilotLogger.py b/Pilot/tests/Test_simplePilotLogger.py index 1fc448ae..df2ac0c2 100644 --- a/Pilot/tests/Test_simplePilotLogger.py +++ b/Pilot/tests/Test_simplePilotLogger.py @@ -172,5 +172,6 @@ def test_executeAndGetOutput(self, popenMock, argvmock): self.stdout_mock.truncate() self.stderr_mock.truncate() + if __name__ == "__main__": unittest.main() diff --git a/tests/CI/pilot_newSchema.json b/tests/CI/pilot_newSchema.json index 0c8c1d32..ee793113 100644 --- a/tests/CI/pilot_newSchema.json +++ b/tests/CI/pilot_newSchema.json @@ -1,157 +1,160 @@ { - "timestamp": "2023-02-13T14:34:26.725499", - "CEs": { - "jenkins.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST" - }, - "jenkins-full.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-FULL" - }, - "jenkins-mp.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-MP" - }, - "jenkins-mp-full.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-FULL" - }, - "jenkins-singularity.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST", - "LocalCEType": "Singularity", - "InstallDIRACInContainer": "True" - }, - "jenkins-singularity-full.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-FULL", - "LocalCEType": "Singularity" - }, - "jenkins-mp-pool.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "Queue": "VAR_JENKINS_QUEUE", - "GridCEType": "TEST-MP", - "LocalCEType": "Pool" - }, - "jenkins-mp-pool-full.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-FULL", - "LocalCEType": "InProcess", - "VAR_JENKINS_QUEUE": { - "LocalCEType": "Pool" - } - }, - "jenkins-mp-pool-sudo.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-MP", - "VAR_JENKINS_QUEUE": { - "LocalCEType": "Pool/Sudo" - } - }, - "jenkins-mp-pool-sudo-full.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-FULL", - "VAR_JENKINS_QUEUE": { - "LocalCEType": "Pool/Sudo" - } - }, - "jenkins-mp-pool-singularity.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-MP", - "VAR_JENKINS_QUEUE": { - "LocalCEType": "Pool/Singularity" - } - }, - "jenkins-mp-pool-singularity-full.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-FULL", - "VAR_JENKINS_QUEUE": { - "LocalCEType": "Pool/Singularity" - } - }, - "jenkins-lhcb.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "LHCb" - }, - "jenkins-lhcb-d.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "LHCbD" - }, - "jenkins-lhcb-dd.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "LHCbDD" - }, + "timestamp": "2023-02-13T14:34:26.725499", + "CEs": { + "jenkins.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST" + }, + "jenkins-full.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-FULL" + }, + "jenkins-mp.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-MP" + }, + "jenkins-mp-full.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-FULL" + }, + "jenkins-singularity.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST", + "LocalCEType": "Singularity", + "InstallDIRACInContainer": "True" + }, + "jenkins-singularity-full.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-FULL", + "LocalCEType": "Singularity" + }, + "jenkins-mp-pool.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "Queue": "VAR_JENKINS_QUEUE", + "GridCEType": "TEST-MP", + "LocalCEType": "Pool" + }, + "jenkins-mp-pool-full.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-FULL", + "LocalCEType": "InProcess", + "VAR_JENKINS_QUEUE": { + "LocalCEType": "Pool" + } + }, + "jenkins-mp-pool-sudo.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-MP", + "VAR_JENKINS_QUEUE": { + "LocalCEType": "Pool/Sudo" + } + }, + "jenkins-mp-pool-sudo-full.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-FULL", + "VAR_JENKINS_QUEUE": { + "LocalCEType": "Pool/Sudo" + } + }, + "jenkins-mp-pool-singularity.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-MP", + "VAR_JENKINS_QUEUE": { + "LocalCEType": "Pool/Singularity" + } + }, + "jenkins-mp-pool-singularity-full.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-FULL", + "VAR_JENKINS_QUEUE": { + "LocalCEType": "Pool/Singularity" + } + }, + "jenkins-lhcb.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "LHCb" + }, + "jenkins-lhcb-d.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "LHCbD" + }, + "jenkins-lhcb-dd.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "LHCbDD" + }, "jenkins-revised.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-REVISED-COMMANDS" + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-REVISED-COMMANDS" } - }, - "Defaults": { - "Pilot": { - "RemoteLogging": "False", - "Commands": { - "TEST": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", - "TEST-MP": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", - "TEST-FULL": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements, LaunchAgent", - "LHCb": "CheckWorkerNode, LHCbInstallDIRAC, LHCbConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", - "LHCbD": "CheckWorkerNode, InstallDIRAC, LHCbConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", - "LHCbDD": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", - "TEST-REVISED-COMMANDS": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitectureWithoutCLI, ConfigureCPURequirements" + }, + "Defaults": { + "Pilot": { + "RemoteLogging": "False", + "Commands": { + "TEST": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", + "TEST-MP": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", + "TEST-FULL": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements, LaunchAgent", + "LHCb": "CheckWorkerNode, LHCbInstallDIRAC, LHCbConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", + "LHCbD": "CheckWorkerNode, InstallDIRAC, LHCbConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", + "LHCbDD": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", + "TEST-REVISED-COMMANDS": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitectureWithoutCLI, ConfigureCPURequirements" } - } - }, - "gridpp": { - "Pilot": { - "Version": "VAR_DIRAC_VERSION", - "Extensions": "None", - "CheckVersion": "False", - "pilotFileServer": "lbcertifdirac70.cern.ch:8443", - "pilotRepoBranch": "does_not_matter", - "pilotRepo": "https://github.com/does_not_matter/Pilot.git", - "GenericPilotGroup": "gridpp_pilot", - "GenericPilotDN": "VAR_USERDN_GRIDPP", - "RemoteLogging": "False", - "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", - "UploadSE": "UKI-LT2-IC-HEP-disk", - "UploadPath": "/gridpp/pilotlogs/", - "RemoteLoggerTimerInterval": 0, - "LoggingShifterName": "GridPPLogManager", - "PilotLogLevel": "DEBUG", - "CVMFS_locations": "/cvmfs/grid.cern.ch, /cvmfs/dirac.egi.eu" - } - }, - "dteam": { - "Pilot": { - "Version": "VAR_DIRAC_VERSION", - "CheckVersion": "False", - "pilotFileServer": "lbcertifdirac70.cern.ch:8443", - "pilotRepoBranch": "should_not_matter", - "pilotRepo": "https://github.com/should_not_matter/Pilot.git", - "GenericPilotGroup": "dteam_pilot", - "GenericPilotDN": "VAR_USERDN", - "RemoteLogging": "True", - "RemoteLoggerCEsWhiteList": "jenkins.cern.ch, jenkins-mp-pool.cern.ch", - "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", - "RemoteLoggerTimerInterval": 0, - "PilotLogLevel": "DEBUG", - "CVMFS_locations": "/cvmfs/grid.cern.ch, /cvmfs/dirac.egi.eu" - } - }, - "LHCb": { - "Pilot": { - "Version": "VAR_DIRAC_VERSION", - "CheckVersion": "False", - "pilotFileServer": "lbcertifdirac70.cern.ch:8443", - "GenericPilotGroup": "dteam_pilot", - "GenericPilotDN": "VAR_USERDN", - "RemoteLogging": "False", - "RemoteLoggerTimerInterval": 0, - "PilotLogLevel": "DEBUG", - "CVMFS_locations": "/cvmfs/lhcb.cern.ch/, $VO_LHCB_SW_DIR" - } - }, - "ConfigurationServers": [ - "VAR_CS" - ] + } + }, + "gridpp": { + "Pilot": { + "Version": "VAR_DIRAC_VERSION", + "Extensions": "None", + "CheckVersion": "False", + "pilotFileServer": "lbcertifdirac70.cern.ch:8443", + "pilotRepoBranch": "does_not_matter", + "pilotRepo": "https://github.com/does_not_matter/Pilot.git", + "GenericPilotGroup": "gridpp_pilot", + "GenericPilotDN": "VAR_USERDN_GRIDPP", + "RemoteLogging": "False", + "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", + "UploadSE": "UKI-LT2-IC-HEP-disk", + "UploadPath": "/gridpp/pilotlogs/", + "RemoteLoggerTimerInterval": 0, + "LoggingShifterName": "GridPPLogManager", + "PilotLogLevel": "DEBUG", + "CVMFS_locations": "/cvmfs/grid.cern.ch, /cvmfs/dirac.egi.eu" + } + }, + "dteam": { + "Pilot": { + "Version": "VAR_DIRAC_VERSION", + "CheckVersion": "False", + "pilotFileServer": "lbcertifdirac70.cern.ch:8443", + "pilotRepoBranch": "should_not_matter", + "pilotRepo": "https://github.com/should_not_matter/Pilot.git", + "GenericPilotGroup": "dteam_pilot", + "GenericPilotDN": "VAR_USERDN", + "RemoteLogging": "True", + "RemoteLoggerCEsWhiteList": "jenkins.cern.ch, jenkins-mp-pool.cern.ch", + "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", + "RemoteLoggerTimerInterval": 0, + "PilotLogLevel": "DEBUG", + "CVMFS_locations": "/cvmfs/grid.cern.ch, /cvmfs/dirac.egi.eu" + } + }, + "LHCb": { + "Pilot": { + "Version": "VAR_DIRAC_VERSION", + "CheckVersion": "False", + "pilotFileServer": "lbcertifdirac70.cern.ch:8443", + "GenericPilotGroup": "dteam_pilot", + "GenericPilotDN": "VAR_USERDN", + "RemoteLogging": "False", + "RemoteLoggerTimerInterval": 0, + "PilotLogLevel": "DEBUG", + "CVMFS_locations": "/cvmfs/lhcb.cern.ch/, $VO_LHCB_SW_DIR" + } + }, + "ConfigurationServers": [ + "VAR_CS" + ], + "PreferredURLPatterns": [ + "VAR_PREF_URL_PATTERN" + ] } diff --git a/tests/CI/pilot_oldSchema.json b/tests/CI/pilot_oldSchema.json index 3502e3b5..163bb387 100644 --- a/tests/CI/pilot_oldSchema.json +++ b/tests/CI/pilot_oldSchema.json @@ -75,8 +75,8 @@ ] }, "Logging": { - "LoggingType":"LOCAL_FILE", - "LocalOutputFile":"myFile" + "LoggingType": "LOCAL_FILE", + "LocalOutputFile": "myFile" }, "NagiosProbes": "WN-mjf, WN-sft-csh, WN-sft-voms, WN-sft-vo-swdir", "Version": "VAR_DIRAC_VERSION",