Skip to content

Commit 97d977d

Browse files
FabianHofmannFabianHofmann
committed
feat: split console and file logging levels
Add console_level (default WARNING) and file_level (default = level) to LoggingConfig so INFO messages stay in the rule log file while the console stays readable under parallel execution. Restore the atlite progress logs in build_renewable_profiles and determine_availability_matrix.
1 parent e355951 commit 97d977d

7 files changed

Lines changed: 99 additions & 13 deletions

File tree

‎config/config.default.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ tutorial: false
1111
# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#logging
1212
logging:
1313
level: INFO
14+
console_level: WARNING
15+
file_level:
1416
format: "%(levelname)s:%(name)s:%(message)s"
1517

1618
# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#remote

‎config/schema.default.json‎

Lines changed: 64 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2848,7 +2848,7 @@
28482848
"properties": {
28492849
"level": {
28502850
"default": "INFO",
2851-
"description": "Restrict console outputs to all infos, warning or errors only",
2851+
"description": "Root log level. Messages below this level are discarded before reaching any handler.",
28522852
"enum": [
28532853
"DEBUG",
28542854
"INFO",
@@ -2858,6 +2858,37 @@
28582858
],
28592859
"type": "string"
28602860
},
2861+
"console_level": {
2862+
"default": "WARNING",
2863+
"description": "Minimum level emitted to the console (stderr). Defaults to ``WARNING`` so parallel rule output stays readable; the full log is kept in the rule's log file.",
2864+
"enum": [
2865+
"DEBUG",
2866+
"INFO",
2867+
"WARNING",
2868+
"ERROR",
2869+
"CRITICAL"
2870+
],
2871+
"type": "string"
2872+
},
2873+
"file_level": {
2874+
"anyOf": [
2875+
{
2876+
"enum": [
2877+
"DEBUG",
2878+
"INFO",
2879+
"WARNING",
2880+
"ERROR",
2881+
"CRITICAL"
2882+
],
2883+
"type": "string"
2884+
},
2885+
{
2886+
"type": "null"
2887+
}
2888+
],
2889+
"default": null,
2890+
"description": "Minimum level written to the rule log file. Defaults to ``level`` when unset."
2891+
},
28612892
"format": {
28622893
"default": "%(levelname)s:%(name)s:%(message)s",
28632894
"description": "Custom format for log messages. See `LogRecord <https://docs.python.org/3/library/logging.html#logging.LogRecord>`_ attributes.",
@@ -8182,7 +8213,7 @@
81828213
"properties": {
81838214
"level": {
81848215
"default": "INFO",
8185-
"description": "Restrict console outputs to all infos, warning or errors only",
8216+
"description": "Root log level. Messages below this level are discarded before reaching any handler.",
81868217
"enum": [
81878218
"DEBUG",
81888219
"INFO",
@@ -8192,6 +8223,37 @@
81928223
],
81938224
"type": "string"
81948225
},
8226+
"console_level": {
8227+
"default": "WARNING",
8228+
"description": "Minimum level emitted to the console (stderr). Defaults to ``WARNING`` so parallel rule output stays readable; the full log is kept in the rule's log file.",
8229+
"enum": [
8230+
"DEBUG",
8231+
"INFO",
8232+
"WARNING",
8233+
"ERROR",
8234+
"CRITICAL"
8235+
],
8236+
"type": "string"
8237+
},
8238+
"file_level": {
8239+
"anyOf": [
8240+
{
8241+
"enum": [
8242+
"DEBUG",
8243+
"INFO",
8244+
"WARNING",
8245+
"ERROR",
8246+
"CRITICAL"
8247+
],
8248+
"type": "string"
8249+
},
8250+
{
8251+
"type": "null"
8252+
}
8253+
],
8254+
"default": null,
8255+
"description": "Minimum level written to the rule log file. Defaults to ``level`` when unset."
8256+
},
81958257
"format": {
81968258
"default": "%(levelname)s:%(name)s:%(message)s",
81978259
"description": "Custom format for log messages. See `LogRecord <https://docs.python.org/3/library/logging.html#logging.LogRecord>`_ attributes.",

‎doc/release_notes.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ Release Notes
2222

2323
* Fix xarray ``FutureWarning`` about implicit MultiIndex promotion in ``build_renewable_profiles`` (https://github.com/PyPSA/pypsa-eur/pull/2144).
2424

25+
* Split logging levels between console and log file: the console now defaults to ``WARNING`` while the rule log file still captures ``INFO``. Configure via ``logging.console_level`` and ``logging.file_level`` (https://github.com/PyPSA/pypsa-eur/pull/2144).
26+
2527
PyPSA-Eur v2026.02.0 (18th February 2026)
2628
=========================================
2729

‎scripts/_helpers.py‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,8 @@ def configure_logging(snakemake, skip_handlers=False):
262262

263263
kwargs = snakemake.config.get("logging", dict()).copy()
264264
kwargs.setdefault("level", "INFO")
265+
console_level = kwargs.pop("console_level", "WARNING")
266+
file_level = kwargs.pop("file_level", None) or kwargs["level"]
265267

266268
if skip_handlers is False:
267269
fallback_path = Path(__file__).parent.joinpath(
@@ -270,16 +272,11 @@ def configure_logging(snakemake, skip_handlers=False):
270272
logfile = snakemake.log.get(
271273
"python", snakemake.log[0] if snakemake.log else fallback_path
272274
)
273-
kwargs.update(
274-
{
275-
"handlers": [
276-
# Prefer the 'python' log, otherwise take the first log for each
277-
# Snakemake rule
278-
logging.FileHandler(logfile),
279-
logging.StreamHandler(),
280-
]
281-
}
282-
)
275+
file_handler = logging.FileHandler(logfile)
276+
file_handler.setLevel(file_level)
277+
stream_handler = logging.StreamHandler()
278+
stream_handler.setLevel(console_level)
279+
kwargs["handlers"] = [file_handler, stream_handler]
283280
logging.basicConfig(**kwargs)
284281

285282
# Setup a function to handle uncaught exceptions and include them with their stacktrace into logfiles

‎scripts/build_renewable_profiles.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,9 @@
137137
correction_factor = params.get("correction_factor", 1.0)
138138
capacity_per_sqkm = params["capacity_per_sqkm"]
139139

140+
if correction_factor != 1.0:
141+
logger.info(f"correction_factor is set as {correction_factor}")
142+
140143
dask_kwargs = setup_dask(nprocesses)
141144

142145
sns = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day)
@@ -168,6 +171,9 @@
168171

169172
func = getattr(cutout, resource.pop("method"))
170173

174+
logger.info(
175+
f"Calculate average capacity factor per grid cell for technology {technology}..."
176+
)
171177
start = time.time()
172178

173179
capacity_factor = correction_factor * func(
@@ -182,6 +188,9 @@
182188
)
183189

184190
nbins = params.get("resource_classes", 1)
191+
logger.info(
192+
f"Create masks for {nbins} resource classes for technology {technology}..."
193+
)
185194
start = time.time()
186195

187196
fn = snakemake.input.resource_regions
@@ -238,6 +247,9 @@
238247

239248
profiles = []
240249
for year, model in models.items():
250+
logger.info(
251+
f"Calculate weighted capacity factor time series for model {model} for technology {technology}..."
252+
)
241253
start = time.time()
242254

243255
resource[tech] = model
@@ -271,8 +283,10 @@
271283

272284
profiles = xr.merge(profiles)
273285

286+
logger.info(f"Calculating maximal capacity per bus for technology {technology}")
274287
p_nom_max = capacity_per_sqkm * availability * class_masks @ area
275288

289+
logger.info(f"Calculate average distances for technology {technology}.")
276290
layoutmatrix = (layout * availability * class_masks).stack(
277291
bus_bin=["bus", "bin"], spatial=["y", "x"]
278292
)

‎scripts/determine_availability_matrix.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@
158158
snakemake.input.country_shapes, buffer=buffer, invert=True
159159
)
160160

161+
logger.info(f"Calculate landuse availability for {technology}...")
161162
start = time.time()
162163

163164
kwargs = dict(nprocesses=nprocesses, disable_progressbar=noprogress)

‎scripts/lib/validation/config/_schema.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,15 @@ class LoggingConfig(ConfigModel):
4646

4747
level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = Field(
4848
"INFO",
49-
description="Restrict console outputs to all infos, warning or errors only",
49+
description="Root log level. Messages below this level are discarded before reaching any handler.",
50+
)
51+
console_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = Field(
52+
"WARNING",
53+
description="Minimum level emitted to the console (stderr). Defaults to ``WARNING`` so parallel rule output stays readable; the full log is kept in the rule's log file.",
54+
)
55+
file_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] | None = Field(
56+
None,
57+
description="Minimum level written to the rule log file. Defaults to ``level`` when unset.",
5058
)
5159
format: str = Field(
5260
"%(levelname)s:%(name)s:%(message)s",

0 commit comments

Comments
 (0)