Skip to content

Commit c54ab5e

Browse files
fstagnifstagni
committed
feat: setting /DIRAC/PreferredURLPatterns if it is in pilot.json
1 parent 724f091 commit c54ab5e

4 files changed

Lines changed: 22 additions & 14 deletions

File tree

‎.github/workflows/integration.yml‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
pilot_schema:
1515
- pilot_oldSchema.json
1616
- pilot_newSchema.json
17-
VO:
17+
VO:
1818
- dteam
1919
- gridpp
2020

@@ -53,6 +53,7 @@ jobs:
5353
sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
5454
sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
5555
sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
56+
sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json
5657
g_job="testintegrationworkflow${GITHUB_JOB//-/}"
5758
pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)"
5859
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
@@ -68,7 +69,7 @@ jobs:
6869
- pilot_newSchema.json
6970
dirac_version:
7071
- prod
71-
- pre
72+
# - pre
7273

7374
steps:
7475
- uses: actions/checkout@v4
@@ -109,12 +110,12 @@ jobs:
109110
sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
110111
sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
111112
sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
113+
sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json
112114
g_job="testintegrationworkflow${GITHUB_JOB//-/}"
113115
pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)"
114116
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
115117
X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --architectureScript=dirac-platform --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --wnVO=dteam --pilotUUID="${pilotUUID}" --debug
116118
117-
118119
release_prod-cvmfs_no_env_CEs:
119120
runs-on: ubuntu-latest
120121

@@ -146,7 +147,7 @@ jobs:
146147
echo "$HOSTKEY_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
147148
chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem
148149
chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
149-
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'a' | tail -n 1)
150+
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | tail -n 1)
150151
source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc
151152
pilot_proxy_file=$(mktemp)
152153
dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg
@@ -160,12 +161,12 @@ jobs:
160161
sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
161162
sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
162163
sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
164+
sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json
163165
g_job="testintegrationworkflow${GITHUB_JOB//-/}"
164166
pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)"
165167
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
166168
X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -S DIRAC-Certification -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --wnVO=dteam -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug
167169
168-
169170
matching:
170171
runs-on: ubuntu-latest
171172

@@ -193,7 +194,7 @@ jobs:
193194
echo "$HOSTKEY_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
194195
chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem
195196
chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
196-
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'a' | tail -n 1)
197+
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | tail -n 1)
197198
source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc
198199
pilot_proxy_file=$(mktemp)
199200
dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg
@@ -207,23 +208,23 @@ jobs:
207208
sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
208209
sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
209210
sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
211+
sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json
210212
g_job="testintegrationworkflow${GITHUB_JOB//-/}"
211213
pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)"
212214
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
213215
X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::integration -M 1 -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=dteam --pilotUUID="${pilotUUID}" --debug
214216
215-
216217
release_prod_pre-cvmfs_matching:
217218
runs-on: ubuntu-latest
218219

219220
strategy:
220221
matrix:
221-
VO:
222+
VO:
222223
- dteam
223224
- gridpp
224225
dirac_version:
225226
- prod
226-
- pre
227+
# - pre
227228

228229
steps:
229230
- uses: actions/checkout@v4
@@ -262,12 +263,12 @@ jobs:
262263
sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
263264
sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
264265
sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
266+
sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json
265267
g_job="testintegrationworkflow${GITHUB_JOB//-/}"
266268
pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)"
267269
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
268270
X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --wnVO=${{ matrix.VO }} -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug
269271
270-
271272
release_prod-cvmfs_matching_CEs:
272273
runs-on: ubuntu-latest
273274

@@ -313,12 +314,12 @@ jobs:
313314
sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
314315
sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
315316
sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
317+
sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json
316318
g_job="testintegrationworkflow${GITHUB_JOB//-/}"
317319
pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)"
318320
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
319321
X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --StopAfterFailedMatches=1 --wnVO=dteam -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug
320322
321-
322323
##################################
323324
### # extensions tests
324325
##################################
@@ -375,6 +376,7 @@ jobs:
375376
sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
376377
sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
377378
sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
379+
sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json
378380
g_job="testintegrationworkflow${GITHUB_JOB//-/}"
379381
pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)"
380382
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
@@ -426,6 +428,7 @@ jobs:
426428
sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
427429
sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
428430
sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
431+
sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json
429432
g_job="testintegrationworkflow${GITHUB_JOB//-/}"
430433
pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)"
431434
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)

‎Pilot/pilotCommands.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -508,6 +508,8 @@ def _getBasicsCFG(self):
508508
self.cfg.append('-S "%s"' % self.pp.setup)
509509
if self.pp.configServer:
510510
self.cfg.append('-C "%s"' % self.pp.configServer)
511+
if self.pp.preferredURLPatterns:
512+
self.cfg.append("-o /DIRAC/PreferredURLPatterns=%s" % quote(",".join(self.pp.preferredURLPatterns)))
511513
if self.pp.releaseProject:
512514
self.cfg.append('-e "%s"' % self.pp.releaseProject)
513515
self.cfg.append("-o /LocalSite/ReleaseProject=%s" % self.pp.releaseProject)

‎Pilot/pilotTools.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -908,6 +908,7 @@ def __init__(self):
908908
self.site = ""
909909
self.setup = ""
910910
self.configServer = ""
911+
self.preferredURLPatterns = ""
911912
self.ceName = ""
912913
self.ceType = ""
913914
self.queueName = ""
@@ -1306,6 +1307,8 @@ def __initJSON2(self):
13061307
# the generic one (a list):
13071308
self.configServer = ",".join([str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"]])
13081309

1310+
self.preferredURLPatterns = self.pilotJSON.get("PreferredURLPatterns", self.preferredURLPatterns)
1311+
13091312
# version(a comma separated values in a string). We take the first one. (the default value defined in the code)
13101313
dVersion = pilotOptions.get("Version", self.releaseVersion)
13111314
if dVersion:

‎Pilot/proxyTools.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
VOMS_FQANS_OID = b"1.3.6.1.4.1.8005.100.100.4"
1010
VOMS_EXTENSION_OID = b"1.3.6.1.4.1.8005.100.100.5"
1111

12-
RE_OPENSSL_ANS1_FORMAT = re.compile(rb"^\s*\d+:d=(\d+)\s+hl=")
12+
RE_OPENSSL_ANS1_FORMAT = re.compile(br"^\s*\d+:d=(\d+)\s+hl=")
1313

1414

1515
def parseASN1(data):
@@ -41,7 +41,7 @@ def getVO(proxy_data):
4141
str: A VO
4242
"""
4343

44-
chain = re.findall(rb"-----BEGIN CERTIFICATE-----\n.+?\n-----END CERTIFICATE-----", proxy_data, flags=re.DOTALL)
44+
chain = re.findall(br"-----BEGIN CERTIFICATE-----\n.+?\n-----END CERTIFICATE-----", proxy_data, flags=re.DOTALL)
4545
for cert in chain:
4646
proc = Popen(["openssl", "x509", "-outform", "der"], stdin=PIPE, stdout=PIPE)
4747
out, _ = proc.communicate(cert)
@@ -61,7 +61,7 @@ def getVO(proxy_data):
6161
if depth <= initial_depth:
6262
break
6363
# Look for a role, if it exists the VO is the first element
64-
match = re.search(rb"OCTET STRING\s+:/([a-zA-Z0-9]+)/Role=", line)
64+
match = re.search(br"OCTET STRING\s+:/([a-zA-Z0-9]+)/Role=", line)
6565
if match:
6666
return match.groups()[0].decode()
6767
raise NotImplementedError("Something went very wrong")

0 commit comments

Comments
 (0)