Skip to content

Commit dd4fb50

Browse files
knoepfelmambelli
knoepfel
authored and
mambelli
committed
Homogenize parameter-naming.
Includes the following changes: - For source proxy: retries -> max_attempts, retry_timeout -> retry_interval - Otherwise, nretries -> max_retries (cherry picked from commit 63f6315)
1 parent e1c28e7 commit dd4fb50

15 files changed

Lines changed: 176 additions & 145 deletions

‎src/decisionengine_modules/AWS/sources/AWSOccupancyWithSourceProxy.py‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -154,12 +154,16 @@ def acquire(self):
154154

155155
oc_list = [i.data for i in occupancy_data]
156156
# to fix the test failure
157-
return {'AWS_Occupancy': pd.DataFrame(oc_list)}
158-
159-
160-
Source.describe(AWSOccupancy,
161-
sample_config={"channel_name": "test",
162-
"source_channel": "channel_aws_config_data",
163-
"Dataproducts": ["spot_occupancy_config"],
164-
"retries": 3,
165-
"retry_timeout": 20})
157+
return {"AWS_Occupancy": pd.DataFrame(oc_list)}
158+
159+
160+
Source.describe(
161+
AWSOccupancy,
162+
sample_config={
163+
"channel_name": "test",
164+
"source_channel": "channel_aws_config_data",
165+
"Dataproducts": ["spot_occupancy_config"],
166+
"max_attempts": 3,
167+
"retry_interval": 20,
168+
},
169+
)

‎src/decisionengine_modules/AWS/sources/AWSSpotPriceWithSourceProxy.py‎

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -199,14 +199,24 @@ def acquire(self):
199199
spot_price_history)
200200

201201
sp_list = [i.data for i in sp_data]
202-
column_names = ['AccountName', 'AvailabilityZone',
203-
'InstanceType', 'ProductDescription', 'SpotPrice', 'Timestamp']
204-
return {'provisioner_resource_spot_prices': pd.DataFrame(sp_list, columns=column_names)}
205-
206-
207-
Source.describe(AWSSpotPrice,
208-
sample_config={"channel_name": "test",
209-
"source_channel": "channel_aws_config_data",
210-
"Dataproducts": ["spot_occupancy_config"],
211-
"retries": 3,
212-
"retry_timeout": 20})
202+
column_names = [
203+
"AccountName",
204+
"AvailabilityZone",
205+
"InstanceType",
206+
"ProductDescription",
207+
"SpotPrice",
208+
"Timestamp",
209+
]
210+
return {"provisioner_resource_spot_prices": pd.DataFrame(sp_list, columns=column_names)}
211+
212+
213+
Source.describe(
214+
AWSSpotPrice,
215+
sample_config={
216+
"channel_name": "test",
217+
"source_channel": "channel_aws_config_data",
218+
"Dataproducts": ["spot_occupancy_config"],
219+
"max_attempts": 3,
220+
"retry_interval": 20,
221+
},
222+
)

‎src/decisionengine_modules/AWS/tests/test_AWSOccupancyWithSourceProxy.py‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,13 @@
1111

1212
DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
1313

14-
config = {"channel_name": "test",
15-
"source_channel": "channel_aws_config_data",
16-
"Dataproducts": ["spot_occupancy_config"],
17-
"retries": 3,
18-
"retry_timeout": 20,
19-
}
14+
config = {
15+
"channel_name": "test",
16+
"source_channel": "channel_aws_config_data",
17+
"Dataproducts": ["spot_occupancy_config"],
18+
"max_attempts": 3,
19+
"retry_interval": 20,
20+
}
2021

2122
account = {'spot_occupancy_config': pd.read_csv(
2223
os.path.join(DATA_DIR, 'account_config.csv'))}

‎src/decisionengine_modules/AWS/tests/test_AWSSpotPriceWithSourceProxy.py‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,12 @@
1313

1414
DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
1515

16-
config = {"source_channel": "channel_aws_config_data",
17-
"Dataproducts": ["spot_occupancy_config"],
18-
"retries": 3,
19-
"retry_timeout": 20,
20-
}
16+
config = {
17+
"source_channel": "channel_aws_config_data",
18+
"Dataproducts": ["spot_occupancy_config"],
19+
"max_attempts": 3,
20+
"retry_interval": 20,
21+
}
2122

2223
account = {'spot_occupancy_config': pd.read_csv(os.path.join(DATA_DIR,
2324
'account_config.csv'))}

‎src/decisionengine_modules/NERSC/sources/NerscAllocationInfo.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,8 @@ def __init__(self, config):
4343
self.retry_backoff_factor = config.get("retry_backoff_factor",
4444
_RETRY_BACKOFF_FACTOR)
4545
self.newt = newt.Newt(
46-
config.get('passwd_file'),
47-
num_retries=self.max_retries,
48-
retry_backoff_factor=self.retry_backoff_factor)
46+
config.get("passwd_file"), max_retries=self.max_retries, retry_backoff_factor=self.retry_backoff_factor
47+
)
4948
self.logger = self.logger.bind(class_module=__name__.split(".")[-1], )
5049

5150
def send_query(self):

‎src/decisionengine_modules/NERSC/sources/NerscJobInfo.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,8 @@ def __init__(self, config):
3737
self.max_retries = config.get("max_retries", _MAX_RETRIES)
3838
self.retry_backoff_factor = config.get("retry_backoff_factor", _RETRY_BACKOFF_FACTOR)
3939
self.newt = newt.Newt(
40-
config.get('passwd_file'),
41-
num_retries=self.max_retries,
42-
retry_backoff_factor=self.retry_backoff_factor)
40+
config.get("passwd_file"), max_retries=self.max_retries, retry_backoff_factor=self.retry_backoff_factor
41+
)
4342

4443
def acquire(self):
4544
"""

‎src/decisionengine_modules/NERSC/util/newt.py‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,7 @@
1717

1818

1919
class Newt:
20-
21-
def __init__(self, password_file, newt_base_url=None, num_retries=0,
22-
retry_backoff_factor=0):
20+
def __init__(self, password_file, newt_base_url=None, max_retries=0, retry_backoff_factor=0):
2321
"""
2422
Constructor that takes path to password file and
2523
optional Newt base URL
@@ -36,7 +34,7 @@ def __init__(self, password_file, newt_base_url=None, num_retries=0,
3634
if not self.newt_base_url.endswith("/"):
3735
self.newt_base_url += "/"
3836
self.session = requests.Session()
39-
self.num_retries = num_retries
37+
self.max_retries = max_retries
4038
self.retry_backoff_factor = retry_backoff_factor
4139
self.expiration_time = time.time()
4240
self._add_retries_to_session()
@@ -47,7 +45,7 @@ def _add_retries_to_session(self):
4745
:return: void
4846
"""
4947
retry = Retry(
50-
status=self.num_retries,
48+
status=self.max_retries,
5149
status_forcelist=[500, 502, 503, 504, 507],
5250
backoff_factor=self.retry_backoff_factor)
5351
retry_adapter = HTTPAdapter(max_retries=retry)

‎src/decisionengine_modules/glideinwms/sources/factory_entries.py‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,14 @@
1717
'constraints': 'HTCondor classad query constraints'
1818
}
1919
"""),
20-
Parameter('nretries', default=0),
21-
Parameter('retry_interval', default=0))
22-
@Source.produces(Factory_Entries_Grid=pandas.DataFrame,
23-
Factory_Entries_AWS=pandas.DataFrame,
24-
Factory_Entries_GCE=pandas.DataFrame,
25-
Factory_Entries_LCF=pandas.DataFrame)
20+
Parameter("max_retries", default=0),
21+
Parameter("retry_interval", default=0))
22+
@Source.produces(
23+
Factory_Entries_Grid=pandas.DataFrame,
24+
Factory_Entries_AWS=pandas.DataFrame,
25+
Factory_Entries_GCE=pandas.DataFrame,
26+
Factory_Entries_LCF=pandas.DataFrame,
27+
)
2628
class FactoryEntries(Source.Source):
2729

2830
def __init__(self, config):
@@ -36,10 +38,10 @@ def __init__(self, config):
3638
'Factory_Entries_LCF': ('batch slurm',)
3739
}
3840

39-
# The combination of nretries=10 and retry_interval=2 adds up to just
41+
# The combination of max_retries=10 and retry_interval=2 adds up to just
4042
# over 15 minutes
41-
self.nretries = config.get('nretries', 0)
42-
self.retry_interval = config.get('retry_interval', 0)
43+
self.max_retries = config.get("max_retries", 0)
44+
self.retry_interval = config.get("retry_interval", 0)
4345

4446
self.subsystem_name = 'any'
4547
self.logger = self.logger.bind(class_module=__name__.split(".")[-1], )
@@ -68,10 +70,11 @@ def acquire(self):
6870
group_attr=['GLIDEIN_GridType'])
6971

7072
retry_wrapper(
71-
partial(condor_status.load,
72-
*(constraint, classad_attrs, self.condor_config)),
73-
nretries=self.nretries,
74-
retry_interval=self.retry_interval)
73+
partial(condor_status.load, *(constraint, classad_attrs, self.condor_config)),
74+
max_retries=self.max_retries,
75+
retry_interval=self.retry_interval,
76+
logger=self.logger,
77+
)
7578

7679
if correction_map is not None:
7780
for eachDict in condor_status.stored_data:

‎src/decisionengine_modules/glideinwms/sources/factory_global.py‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@
1818
'constraints': 'HTCondor classad query constraints'
1919
}
2020
"""),
21-
Parameter('nretries', default=0),
22-
Parameter('retry_interval', default=0))
21+
Parameter("max_retries", default=0),
22+
Parameter("retry_interval", default=0))
2323
@Source.produces(factoryglobal_manifests=pandas.DataFrame)
2424
class FactoryGlobalManifests(Source.Source):
2525

@@ -33,10 +33,10 @@ def __init__(self, config):
3333
self.condor_config = config.get('condor_config')
3434
self.factories = config.get('factories', [])
3535

36-
# The combination of nretries=10 and retry_interval=2 adds up to just
36+
# The combination of max_retries=10 and retry_interval=2 adds up to just
3737
# over 15 minutes
38-
self.nretries = config.get('nretries', 0)
39-
self.retry_interval = config.get('retry_interval', 0)
38+
self.max_retries = config.get("max_retries", 0)
39+
self.retry_interval = config.get("retry_interval", 0)
4040

4141
self.subsystem_name = 'any'
4242
self.logger = self.logger.bind(class_module=__name__.split(".")[-1], )
@@ -64,10 +64,11 @@ def acquire(self):
6464
group_attr=['Name'])
6565

6666
retry_wrapper(
67-
partial(condor_status.load,
68-
*(constraint, classad_attrs, self.condor_config)),
69-
nretries=self.nretries,
70-
retry_interval=self.retry_interval)
67+
partial(condor_status.load, *(constraint, classad_attrs, self.condor_config)),
68+
max_retries=self.max_retries,
69+
retry_interval=self.retry_interval,
70+
logger=self.logger,
71+
)
7172

7273
df = pandas.DataFrame(condor_status.stored_data)
7374
if not df.empty:

‎src/decisionengine_modules/htcondor/publishers/publisher.py‎

Lines changed: 23 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -12,18 +12,20 @@
1212
DEFAULT_INVALIDATE_AD_COMMAND = 'INVALIDATE_AD_GENERIC'
1313

1414

15-
@Publisher.supports_config(Parameter('condor_config', type=str),
16-
Parameter('x509_user_proxy', type=str),
17-
Parameter('nretries', type=int),
18-
Parameter('retry_interval', type=int, comment="Number of seconds to wait between retries."))
15+
@Publisher.supports_config(
16+
Parameter("condor_config", type=str),
17+
Parameter("x509_user_proxy", type=str),
18+
Parameter("max_retries", type=int),
19+
Parameter("retry_interval", type=int, comment="Number of seconds to wait between retries."),
20+
)
1921
class HTCondorManifests(Publisher.Publisher, metaclass=abc.ABCMeta):
2022

2123
def __init__(self, config):
2224
super().__init__(config)
23-
self.condor_config = config.get('condor_config')
24-
self.x509_user_proxy = config.get('x509_user_proxy')
25-
self.nretries = config.get('nretries')
26-
self.retry_interval = config.get('retry_interval')
25+
self.condor_config = config.get("condor_config")
26+
self.x509_user_proxy = config.get("x509_user_proxy")
27+
self.max_retries = config.get("max_retries")
28+
self.retry_interval = config.get("retry_interval")
2729
self.logger = self.logger.bind(class_module=__name__.split(".")[-1], )
2830
self.update_ad_command = DEFAULT_UPDATE_AD_COMMAND
2931
self.invalidate_ad_command = DEFAULT_INVALIDATE_AD_COMMAND
@@ -102,14 +104,19 @@ def _condor_advertise(self, classads, collector_host=None,
102104
raise
103105
finally:
104106
if old_condor_config_env:
105-
os.environ['CONDOR_CONFIG'] = old_condor_config_env
106-
107-
def condor_advertise(self, classads, collector_host=None,
108-
update_ad_command=DEFAULT_UPDATE_AD_COMMAND):
109-
return retry_wrapper(partial(self._condor_advertise, classads,
110-
**{"collector_host": collector_host,
111-
"update_ad_command": update_ad_command}),
112-
self.nretries, self.retry_interval)
107+
os.environ["CONDOR_CONFIG"] = old_condor_config_env
108+
109+
def condor_advertise(self, classads, collector_host=None, update_ad_command=DEFAULT_UPDATE_AD_COMMAND):
110+
return retry_wrapper(
111+
partial(
112+
self._condor_advertise,
113+
classads,
114+
**{"collector_host": collector_host, "update_ad_command": update_ad_command},
115+
),
116+
self.max_retries,
117+
self.retry_interval,
118+
logger=self.logger,
119+
)
113120

114121
def publish(self, datablock):
115122
"""

0 commit comments

Comments
 (0)