forked from aws/aws-sam-cli
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathparameter_loaders.py
More file actions
359 lines (290 loc) · 11 KB
/
Copy pathparameter_loaders.py
File metadata and controls
359 lines (290 loc) · 11 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
"""
Parameter file loading utilities for SAM CLI configuration
"""
import json
import logging
import os
from pathlib import Path
from typing import Dict, List, Optional, Tuple
from ruamel.yaml import YAML, YAMLError
from samcli.lib.config.exceptions import FileParseException
LOG = logging.getLogger(__name__)
class ParameterFileLoader:
"""
Loads parameters from external parameter files.
Supports JSON, YAML, and ENV file formats.
Follows existing SAM CLI FileManager patterns.
"""
@staticmethod
def is_file_url(parameter_value: Optional[str]) -> bool:
"""
Check if a parameter value is a file:// URL.
Parameters
----------
parameter_value : str, optional
Parameter value to check
Returns
-------
bool
True if parameter_value is a file:// URL
"""
if not isinstance(parameter_value, str):
return False
return parameter_value.startswith("file://")
@staticmethod
def parse_file_url(file_url: str) -> str:
"""
Parse file:// URL and return the file path with environment variable expansion.
Parameters
----------
file_url : str
File URL in format file://path/to/file.ext
Returns
-------
str
Expanded file path
Raises
------
ValueError
If URL format is invalid
"""
if not ParameterFileLoader.is_file_url(file_url):
raise ValueError(f"Invalid file URL format: {file_url}")
# Extract path by removing file:// prefix
# Don't use urlparse because it doesn't handle env vars and relative paths correctly
file_path = file_url[7:] # Remove 'file://' prefix
# Handle Windows paths (file:///C:/path/to/file)
if os.name == "nt" and file_path.startswith("/") and ":" in file_path[1:3]:
file_path = file_path[1:]
# Expand environment variables before returning
expanded_path = os.path.expandvars(file_path)
LOG.debug(f"Parsed file URL '{file_url}' to path '{expanded_path}'")
return expanded_path
@staticmethod
def load_from_file(file_path: str) -> Dict:
"""
Load parameters from a file based on its extension.
Supported formats:
- .json: JSON format
- .yaml/.yml: YAML format
- .env: Environment variable format
Parameters
----------
file_path : str
Path to the parameter file
Returns
-------
Dict
Dictionary of loaded parameters
Raises
------
FileNotFoundError
If file doesn't exist
FileParseException
If file format is invalid or unsupported
"""
path_obj = Path(file_path)
if not path_obj.exists():
raise FileNotFoundError(f"Parameter file not found: {file_path}")
if not path_obj.is_file():
raise FileParseException(f"Parameter path is not a file: {file_path}")
extension = path_obj.suffix.lower()
try:
if extension == ".json":
return ParameterFileLoader._load_json_file(path_obj)
elif extension in [".yaml", ".yml"]:
return ParameterFileLoader._load_yaml_file(path_obj)
elif extension == ".env":
return ParameterFileLoader._load_env_file(path_obj)
else:
raise FileParseException(
f"Unsupported parameter file format: {extension}. " f"Supported formats: .json, .yaml, .yml, .env"
)
except Exception as e:
if isinstance(e, (FileParseException, FileNotFoundError)):
raise
raise FileParseException(f"Failed to load parameter file '{file_path}': {str(e)}") from e
@staticmethod
def _load_json_file(file_path: Path) -> Dict:
"""
Load parameters from JSON file.
Parameters
----------
file_path : Path
Path to JSON file
Returns
-------
Dict
Dictionary of parameters
"""
try:
content = file_path.read_text(encoding="utf-8")
params = json.loads(content)
if not isinstance(params, dict):
raise FileParseException(f"JSON parameter file must contain an object, got {type(params)}")
LOG.debug(f"Loaded {len(params)} parameters from JSON file: {file_path}")
return params
except json.JSONDecodeError as e:
raise FileParseException(f"Invalid JSON in parameter file '{file_path}': {str(e)}") from e
@staticmethod
def _load_yaml_file(file_path: Path) -> Dict:
"""
Load parameters from YAML file.
Parameters
----------
file_path : Path
Path to YAML file
Returns
-------
Dict
Dictionary of parameters
"""
yaml = YAML(typ="safe", pure=True)
try:
content = file_path.read_text(encoding="utf-8")
params = yaml.load(content)
if params is None:
return {}
if not isinstance(params, dict):
raise FileParseException(f"YAML parameter file must contain a mapping, got {type(params)}")
LOG.debug(f"Loaded {len(params)} parameters from YAML file: {file_path}")
return params
except YAMLError as e:
raise FileParseException(f"Invalid YAML in parameter file '{file_path}': {str(e)}") from e
@staticmethod
def _load_env_file(file_path: Path) -> Dict:
"""
Load parameters from environment variable file.
Format:
KEY1=value1
KEY2=value2
# Comments are supported
MULTILINE_KEY="line1
line2
line3"
Parameters
----------
file_path : Path
Path to ENV file
Returns
-------
Dict
Dictionary of parameters
"""
params: Dict[str, str] = {}
try:
content = file_path.read_text(encoding="utf-8")
lines = content.splitlines()
current_key: Optional[str] = None
current_value: List[str] = []
in_multiline = False
for line_num, raw_line in enumerate(lines, 1):
line = raw_line.rstrip()
# Skip empty lines and comments
if not line or line.startswith("#"):
continue
# Handle multiline values
if in_multiline:
if line.endswith('"') and not line.endswith('\\"'):
# End of multiline value
current_value.append(line[:-1]) # Remove closing quote
if current_key is not None:
params[current_key] = "\n".join(current_value)
current_key = None
current_value = []
in_multiline = False
else:
current_value.append(line)
continue
# Parse key=value pairs
if "=" not in line:
LOG.warning(f"Skipping invalid line {line_num} in {file_path}: {line}")
continue
key, value = line.split("=", 1)
key = key.strip()
value = value.strip()
# Handle quoted values
if value.startswith('"'):
if value.endswith('"') and len(value) > 1 and not value.endswith('\\"'):
# Single line quoted value
params[key] = value[1:-1] # Remove quotes
else:
# Start of multiline value
current_key = key
current_value = [value[1:]] # Remove opening quote
in_multiline = True
else:
params[key] = value
if in_multiline:
raise FileParseException(f"Unterminated quoted value for key '{current_key}' in {file_path}")
LOG.debug(f"Loaded {len(params)} parameters from ENV file: {file_path}")
return params
except Exception as e:
if isinstance(e, FileParseException):
raise
raise FileParseException(f"Failed to parse ENV file '{file_path}': {str(e)}") from e
@staticmethod
def resolve_parameter_files(parameter_overrides: Optional[str]) -> Tuple[Dict, Dict]:
"""
Resolve parameter overrides that may contain file:// URLs.
Parameters
----------
parameter_overrides : str, optional
Parameter overrides string that may contain file:// URLs
Returns
-------
Tuple[Dict, Dict]
Tuple of (direct_parameters, file_parameters)
- direct_parameters: Parameters specified directly as Key=Value
- file_parameters: Parameters loaded from files
"""
if not parameter_overrides:
return {}, {}
direct_params = {}
file_params = {}
# Split parameter overrides by spaces, handling quoted values
import shlex
try:
parts = shlex.split(parameter_overrides)
except ValueError:
# Fall back to simple split if shlex fails
parts = parameter_overrides.split()
for part in parts:
if ParameterFileLoader.is_file_url(part):
# Load parameters from file
try:
file_path = ParameterFileLoader.parse_file_url(part)
loaded_params = ParameterFileLoader.load_from_file(file_path)
file_params.update(loaded_params)
LOG.info(f"Loaded {len(loaded_params)} parameters from file: {file_path}")
except Exception as e:
LOG.error(f"Failed to load parameters from {part}: {str(e)}")
raise
elif "=" in part:
# Direct parameter specification
key, value = part.split("=", 1)
direct_params[key.strip()] = value.strip()
else:
LOG.warning(f"Skipping invalid parameter format: {part}")
return direct_params, file_params
@staticmethod
def expand_environment_variables(params: Dict) -> Dict:
"""
Expand environment variables in parameter values.
Supports ${VAR_NAME} and $VAR_NAME syntax.
Parameters
----------
params : Dict
Dictionary of parameters
Returns
-------
Dict
Dictionary with expanded environment variables
"""
expanded = {}
for key, value in params.items():
if isinstance(value, str):
expanded[key] = os.path.expandvars(value)
else:
expanded[key] = value
return expanded