From 6ffbb9905de39c9bd4ff44c496c5bdbff75fbb15 Mon Sep 17 00:00:00 2001 From: Rishikesh Devsot Date: Wed, 17 Jun 2026 19:43:25 -0400 Subject: [PATCH 01/35] Add log correlation context formatter --- .../clp-py-utils/clp_py_utils/clp_logging.py | 79 +++++++++++++++++-- 1 file changed, 74 insertions(+), 5 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_logging.py b/components/clp-py-utils/clp_py_utils/clp_logging.py index 353ac89d9c..a779ce052e 100644 --- a/components/clp-py-utils/clp_py_utils/clp_logging.py +++ b/components/clp-py-utils/clp_py_utils/clp_logging.py @@ -1,6 +1,9 @@ import logging import os import pathlib +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar from typing import get_args, Literal LoggingLevel = Literal[ @@ -13,11 +16,77 @@ ] -def get_logging_formatter(): - return logging.Formatter("%(asctime)s %(name)s [%(levelname)s] %(message)s") +_LOG_CONTEXT: ContextVar[dict[str, object] | None] = ContextVar("clp_log_context", default=None) +_LOG_CONTEXT_FIELD_ORDER = ( + "job_id", + "query_job_type", + "task_id", + "archive_id", + "dataset", + "partition_id", + "celery_task_id", + "clp_subprocess_pid", + "query", + "query_hash", +) -def get_logger(name: str): + +@contextmanager +def bind_log_context(**fields: object) -> Iterator[None]: + """ + Temporarily adds CLP correlation fields to log records emitted in the current context. + + Fields with ``None`` values are ignored. Nested contexts merge with outer contexts and restore + the previous context on exit. + """ + current_context = _LOG_CONTEXT.get() or {} + new_context = current_context.copy() + new_context.update({key: value for key, value in fields.items() if value is not None}) + token = _LOG_CONTEXT.set(new_context) + try: + yield + finally: + _LOG_CONTEXT.reset(token) + + +def get_log_context() -> dict[str, object]: + """Returns a copy of the active CLP log correlation context.""" + return (_LOG_CONTEXT.get() or {}).copy() + + +class ClpContextFormatter(logging.Formatter): + """Appends active CLP correlation fields to log records.""" + + def format(self, record: logging.LogRecord) -> str: + message = super().format(record) + context = get_log_context() + if not context: + return message + + fields = [f"{key}={context[key]}" for key in _LOG_CONTEXT_FIELD_ORDER if key in context] + fields.extend( + f"{key}={value}" + for key, value in sorted(context.items()) + if key not in _LOG_CONTEXT_FIELD_ORDER + ) + if len(fields) > 0: + message = f"{message} [{' '.join(fields)}]" + return message + + +def get_logging_formatter() -> logging.Formatter: + return ClpContextFormatter("%(asctime)s %(name)s [%(levelname)s] %(message)s") + + +def set_formatter_on_handlers(logger: logging.Logger) -> None: + """Sets CLP's logging formatter on all existing handlers for a logger.""" + formatter = get_logging_formatter() + for handler in logger.handlers: + handler.setFormatter(formatter) + + +def get_logger(name: str) -> logging.Logger: logger = logging.getLogger(name) # Setup console logging logging_console_handler = logging.StreamHandler() @@ -28,7 +97,7 @@ def get_logger(name: str): return logger -def set_logging_level(logger: logging.Logger, level: str | None): +def set_logging_level(logger: logging.Logger, level: str | None) -> None: if level is None: logger.setLevel(logging.INFO) return @@ -43,7 +112,7 @@ def set_logging_level(logger: logging.Logger, level: str | None): def configure_logging( logger: logging.Logger, component_name: str, -): +) -> None: """ Configures file logging and the logging level for a logger using environment variables. From fbb3df0714d11b66016e51da7f4cba330acfa667 Mon Sep 17 00:00:00 2001 From: Rishikesh Devsot Date: Thu, 18 Jun 2026 17:33:15 -0400 Subject: [PATCH 02/35] nit --- components/clp-py-utils/clp_py_utils/clp_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_logging.py b/components/clp-py-utils/clp_py_utils/clp_logging.py index a779ce052e..a2f1dfd66e 100644 --- a/components/clp-py-utils/clp_py_utils/clp_logging.py +++ b/components/clp-py-utils/clp_py_utils/clp_logging.py @@ -35,7 +35,7 @@ @contextmanager def bind_log_context(**fields: object) -> Iterator[None]: """ - Temporarily adds CLP correlation fields to log records emitted in the current context. + Adds CLP correlation fields to log records emitted in the current context. Fields with ``None`` values are ignored. Nested contexts merge with outer contexts and restore the previous context on exit. From 962ae484c4dd7e7bb33c4222e0a3387468328773 Mon Sep 17 00:00:00 2001 From: Rishikesh Devsot Date: Sun, 21 Jun 2026 14:15:24 -0400 Subject: [PATCH 03/35] add struct logging --- .../clp-py-utils/clp_py_utils/clp_logging.py | 72 +++++++++---------- components/clp-py-utils/pyproject.toml | 1 + components/clp-py-utils/uv.lock | 14 ++++ 3 files changed, 48 insertions(+), 39 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_logging.py b/components/clp-py-utils/clp_py_utils/clp_logging.py index a2f1dfd66e..52458164f2 100644 --- a/components/clp-py-utils/clp_py_utils/clp_logging.py +++ b/components/clp-py-utils/clp_py_utils/clp_logging.py @@ -3,9 +3,16 @@ import pathlib from collections.abc import Iterator from contextlib import contextmanager -from contextvars import ContextVar from typing import get_args, Literal +import structlog +from structlog.contextvars import ( + bind_contextvars, + get_contextvars, + merge_contextvars, + reset_contextvars, +) + LoggingLevel = Literal[ "INFO", "DEBUG", @@ -15,21 +22,8 @@ "CRITICAL", ] - -_LOG_CONTEXT: ContextVar[dict[str, object] | None] = ContextVar("clp_log_context", default=None) - -_LOG_CONTEXT_FIELD_ORDER = ( - "job_id", - "query_job_type", - "task_id", - "archive_id", - "dataset", - "partition_id", - "celery_task_id", - "clp_subprocess_pid", - "query", - "query_hash", -) +_TIMESTAMP_PROCESSOR = structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp") +_JSON_RENDERER = structlog.processors.JSONRenderer() @contextmanager @@ -40,43 +34,43 @@ def bind_log_context(**fields: object) -> Iterator[None]: Fields with ``None`` values are ignored. Nested contexts merge with outer contexts and restore the previous context on exit. """ - current_context = _LOG_CONTEXT.get() or {} - new_context = current_context.copy() - new_context.update({key: value for key, value in fields.items() if value is not None}) - token = _LOG_CONTEXT.set(new_context) + tokens = bind_contextvars(**{key: value for key, value in fields.items() if value is not None}) try: yield finally: - _LOG_CONTEXT.reset(token) + reset_contextvars(**tokens) def get_log_context() -> dict[str, object]: """Returns a copy of the active CLP log correlation context.""" - return (_LOG_CONTEXT.get() or {}).copy() + return get_contextvars() -class ClpContextFormatter(logging.Formatter): - """Appends active CLP correlation fields to log records.""" +class _ClpJsonFormatter(logging.Formatter): + """Formats log records as JSON and includes active CLP correlation fields.""" def format(self, record: logging.LogRecord) -> str: - message = super().format(record) - context = get_log_context() - if not context: - return message - - fields = [f"{key}={context[key]}" for key in _LOG_CONTEXT_FIELD_ORDER if key in context] - fields.extend( - f"{key}={value}" - for key, value in sorted(context.items()) - if key not in _LOG_CONTEXT_FIELD_ORDER - ) - if len(fields) > 0: - message = f"{message} [{' '.join(fields)}]" - return message + method_name = record.levelname.lower() + event_dict: dict[str, object] = { + "event": record.getMessage(), + "logger": record.name, + "level": method_name, + } + if record.exc_info: + event_dict["exception"] = self.formatException(record.exc_info) + if record.stack_info: + event_dict["stack"] = self.formatStack(record.stack_info) + + merge_contextvars(None, method_name, event_dict) + _TIMESTAMP_PROCESSOR(None, method_name, event_dict) + rendered_log = _JSON_RENDERER(None, method_name, event_dict) + if isinstance(rendered_log, bytes): + return rendered_log.decode("utf-8") + return rendered_log def get_logging_formatter() -> logging.Formatter: - return ClpContextFormatter("%(asctime)s %(name)s [%(levelname)s] %(message)s") + return _ClpJsonFormatter() def set_formatter_on_handlers(logger: logging.Logger) -> None: diff --git a/components/clp-py-utils/pyproject.toml b/components/clp-py-utils/pyproject.toml index 46bd38a9ba..6c6aaf056d 100644 --- a/components/clp-py-utils/pyproject.toml +++ b/components/clp-py-utils/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "result>=0.17.0", "sqlalchemy>=2.0.46", "StrEnum>=0.4.15", + "structlog>=25.5.0", ] [build-system] diff --git a/components/clp-py-utils/uv.lock b/components/clp-py-utils/uv.lock index 69565cd86a..ca40259c16 100644 --- a/components/clp-py-utils/uv.lock +++ b/components/clp-py-utils/uv.lock @@ -253,6 +253,7 @@ dependencies = [ { name = "result" }, { name = "sqlalchemy" }, { name = "strenum" }, + { name = "structlog" }, ] [package.dev-dependencies] @@ -276,6 +277,7 @@ requires-dist = [ { name = "result", specifier = ">=0.17.0" }, { name = "sqlalchemy", specifier = ">=2.0.46" }, { name = "strenum", specifier = ">=0.4.15" }, + { name = "structlog", specifier = ">=25.5.0" }, ] [package.metadata.requires-dev] @@ -1392,6 +1394,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/69/297302c5f5f59c862faa31e6cb9a4cd74721cd1e052b38e464c5b402df8b/StrEnum-0.4.15-py3-none-any.whl", hash = "sha256:a30cda4af7cc6b5bf52c8055bc4bf4b2b6b14a93b574626da33df53cf7740659", size = 8851, upload-time = "2023-06-29T22:02:56.947Z" }, ] +[[package]] +name = "structlog" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/89/b4a0bcfdf4f71a3dea31379f095929613d7e4528a0996bca6aa964cd0dca/structlog-26.1.0.tar.gz", hash = "sha256:f63a716cbd1b1291cf7661de7794b455acfa4c43c5bcf1630e6ad5ddc1adb3b7", size = 1459881, upload-time = "2026-06-06T07:33:39.348Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/18/489c97b834dfff9cf2fc2507cede4bcd4b11e67f84bc462acd1992496f86/structlog-26.1.0-py3-none-any.whl", hash = "sha256:e081a26d6c373e6d201eca24eede26d8ffab07f88f477822e679183428d3d91e", size = 73764, upload-time = "2026-06-06T07:33:38.046Z" }, +] + [[package]] name = "tomli" version = "2.4.0" From c13eceef5459d9c5e88dbd36c7910f0b04a46cd7 Mon Sep 17 00:00:00 2001 From: Rishikesh Devsot Date: Sun, 21 Jun 2026 20:13:08 -0400 Subject: [PATCH 04/35] linter fixes --- components/clp-mcp-server/uv.lock | 38 +++++++++++++++++++++++----- components/clp-package-utils/uv.lock | 14 ++++++++++ components/job-orchestration/uv.lock | 14 ++++++++++ integration-tests/uv.lock | 14 ++++++++++ 4 files changed, 74 insertions(+), 6 deletions(-) diff --git a/components/clp-mcp-server/uv.lock b/components/clp-mcp-server/uv.lock index 202ed21305..868c87721e 100644 --- a/components/clp-mcp-server/uv.lock +++ b/components/clp-mcp-server/uv.lock @@ -393,6 +393,7 @@ dependencies = [ { name = "result" }, { name = "sqlalchemy" }, { name = "strenum" }, + { name = "structlog" }, ] [package.metadata] @@ -406,6 +407,7 @@ requires-dist = [ { name = "result", specifier = ">=0.17.0" }, { name = "sqlalchemy", specifier = ">=2.0.46" }, { name = "strenum", specifier = ">=0.4.15" }, + { name = "structlog", specifier = ">=25.5.0" }, ] [package.metadata.requires-dev] @@ -622,7 +624,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fe/65/5b235b40581ad75ab97dcd8b4218022ae8e3ab77c13c919f1a1dfe9171fd/greenlet-3.3.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:04bee4775f40ecefcdaa9d115ab44736cd4b9c5fba733575bfe9379419582e13", size = 273723, upload-time = "2026-01-23T15:30:37.521Z" }, { url = "https://files.pythonhosted.org/packages/ce/ad/eb4729b85cba2d29499e0a04ca6fbdd8f540afd7be142fd571eea43d712f/greenlet-3.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50e1457f4fed12a50e427988a07f0f9df53cf0ee8da23fab16e6732c2ec909d4", size = 574874, upload-time = "2026-01-23T16:00:54.551Z" }, { url = "https://files.pythonhosted.org/packages/87/32/57cad7fe4c8b82fdaa098c89498ef85ad92dfbb09d5eb713adedfc2ae1f5/greenlet-3.3.1-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:070472cd156f0656f86f92e954591644e158fd65aa415ffbe2d44ca77656a8f5", size = 586309, upload-time = "2026-01-23T16:05:25.18Z" }, - { url = "https://files.pythonhosted.org/packages/66/66/f041005cb87055e62b0d68680e88ec1a57f4688523d5e2fb305841bc8307/greenlet-3.3.1-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1108b61b06b5224656121c3c8ee8876161c491cbe74e5c519e0634c837cf93d5", size = 597461, upload-time = "2026-01-23T16:15:51.943Z" }, { url = "https://files.pythonhosted.org/packages/87/eb/8a1ec2da4d55824f160594a75a9d8354a5fe0a300fb1c48e7944265217e1/greenlet-3.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a300354f27dd86bae5fbf7002e6dd2b3255cd372e9242c933faf5e859b703fe", size = 586985, upload-time = "2026-01-23T15:32:47.968Z" }, { url = "https://files.pythonhosted.org/packages/15/1c/0621dd4321dd8c351372ee8f9308136acb628600658a49be1b7504208738/greenlet-3.3.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e84b51cbebf9ae573b5fbd15df88887815e3253fc000a7d0ff95170e8f7e9729", size = 1547271, upload-time = "2026-01-23T16:04:18.977Z" }, { url = "https://files.pythonhosted.org/packages/9d/53/24047f8924c83bea7a59c8678d9571209c6bfe5f4c17c94a78c06024e9f2/greenlet-3.3.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e0093bd1a06d899892427217f0ff2a3c8f306182b8c754336d32e2d587c131b4", size = 1613427, upload-time = "2026-01-23T15:33:44.428Z" }, @@ -630,7 +631,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/e8/2e1462c8fdbe0f210feb5ac7ad2d9029af8be3bf45bd9fa39765f821642f/greenlet-3.3.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:5fd23b9bc6d37b563211c6abbb1b3cab27db385a4449af5c32e932f93017080c", size = 274974, upload-time = "2026-01-23T15:31:02.891Z" }, { url = "https://files.pythonhosted.org/packages/7e/a8/530a401419a6b302af59f67aaf0b9ba1015855ea7e56c036b5928793c5bd/greenlet-3.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09f51496a0bfbaa9d74d36a52d2580d1ef5ed4fdfcff0a73730abfbbbe1403dd", size = 577175, upload-time = "2026-01-23T16:00:56.213Z" }, { url = "https://files.pythonhosted.org/packages/8e/89/7e812bb9c05e1aaef9b597ac1d0962b9021d2c6269354966451e885c4e6b/greenlet-3.3.1-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb0feb07fe6e6a74615ee62a880007d976cf739b6669cce95daa7373d4fc69c5", size = 590401, upload-time = "2026-01-23T16:05:26.365Z" }, - { url = "https://files.pythonhosted.org/packages/70/ae/e2d5f0e59b94a2269b68a629173263fa40b63da32f5c231307c349315871/greenlet-3.3.1-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:67ea3fc73c8cd92f42467a72b75e8f05ed51a0e9b1d15398c913416f2dafd49f", size = 601161, upload-time = "2026-01-23T16:15:53.456Z" }, { url = "https://files.pythonhosted.org/packages/5c/ae/8d472e1f5ac5efe55c563f3eabb38c98a44b832602e12910750a7c025802/greenlet-3.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:39eda9ba259cc9801da05351eaa8576e9aa83eb9411e8f0c299e05d712a210f2", size = 590272, upload-time = "2026-01-23T15:32:49.411Z" }, { url = "https://files.pythonhosted.org/packages/a8/51/0fde34bebfcadc833550717eade64e35ec8738e6b097d5d248274a01258b/greenlet-3.3.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e2e7e882f83149f0a71ac822ebf156d902e7a5d22c9045e3e0d1daf59cee2cc9", size = 1550729, upload-time = "2026-01-23T16:04:20.867Z" }, { url = "https://files.pythonhosted.org/packages/16/c9/2fb47bee83b25b119d5a35d580807bb8b92480a54b68fef009a02945629f/greenlet-3.3.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:80aa4d79eb5564f2e0a6144fcc744b5a37c56c4a92d60920720e99210d88db0f", size = 1615552, upload-time = "2026-01-23T15:33:45.743Z" }, @@ -639,7 +639,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/c8/9d76a66421d1ae24340dfae7e79c313957f6e3195c144d2c73333b5bfe34/greenlet-3.3.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:7e806ca53acf6d15a888405880766ec84721aa4181261cd11a457dfe9a7a4975", size = 276443, upload-time = "2026-01-23T15:30:10.066Z" }, { url = "https://files.pythonhosted.org/packages/81/99/401ff34bb3c032d1f10477d199724f5e5f6fbfb59816ad1455c79c1eb8e7/greenlet-3.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d842c94b9155f1c9b3058036c24ffb8ff78b428414a19792b2380be9cecf4f36", size = 597359, upload-time = "2026-01-23T16:00:57.394Z" }, { url = "https://files.pythonhosted.org/packages/2b/bc/4dcc0871ed557792d304f50be0f7487a14e017952ec689effe2180a6ff35/greenlet-3.3.1-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:20fedaadd422fa02695f82093f9a98bad3dab5fcda793c658b945fcde2ab27ba", size = 607805, upload-time = "2026-01-23T16:05:28.068Z" }, - { url = "https://files.pythonhosted.org/packages/3b/cd/7a7ca57588dac3389e97f7c9521cb6641fd8b6602faf1eaa4188384757df/greenlet-3.3.1-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c620051669fd04ac6b60ebc70478210119c56e2d5d5df848baec4312e260e4ca", size = 622363, upload-time = "2026-01-23T16:15:54.754Z" }, { url = "https://files.pythonhosted.org/packages/cf/05/821587cf19e2ce1f2b24945d890b164401e5085f9d09cbd969b0c193cd20/greenlet-3.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14194f5f4305800ff329cbf02c5fcc88f01886cadd29941b807668a45f0d2336", size = 609947, upload-time = "2026-01-23T15:32:51.004Z" }, { url = "https://files.pythonhosted.org/packages/a4/52/ee8c46ed9f8babaa93a19e577f26e3d28a519feac6350ed6f25f1afee7e9/greenlet-3.3.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7b2fe4150a0cf59f847a67db8c155ac36aed89080a6a639e9f16df5d6c6096f1", size = 1567487, upload-time = "2026-01-23T16:04:22.125Z" }, { url = "https://files.pythonhosted.org/packages/8f/7c/456a74f07029597626f3a6db71b273a3632aecb9afafeeca452cfa633197/greenlet-3.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:49f4ad195d45f4a66a0eb9c1ba4832bb380570d361912fa3554746830d332149", size = 1636087, upload-time = "2026-01-23T15:33:47.486Z" }, @@ -648,7 +647,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/ab/d26750f2b7242c2b90ea2ad71de70cfcd73a948a49513188a0fc0d6fc15a/greenlet-3.3.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:7ab327905cabb0622adca5971e488064e35115430cec2c35a50fd36e72a315b3", size = 275205, upload-time = "2026-01-23T15:30:24.556Z" }, { url = "https://files.pythonhosted.org/packages/10/d3/be7d19e8fad7c5a78eeefb2d896a08cd4643e1e90c605c4be3b46264998f/greenlet-3.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65be2f026ca6a176f88fb935ee23c18333ccea97048076aef4db1ef5bc0713ac", size = 599284, upload-time = "2026-01-23T16:00:58.584Z" }, { url = "https://files.pythonhosted.org/packages/ae/21/fe703aaa056fdb0f17e5afd4b5c80195bbdab701208918938bd15b00d39b/greenlet-3.3.1-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7a3ae05b3d225b4155bda56b072ceb09d05e974bc74be6c3fc15463cf69f33fd", size = 610274, upload-time = "2026-01-23T16:05:29.312Z" }, - { url = "https://files.pythonhosted.org/packages/06/00/95df0b6a935103c0452dad2203f5be8377e551b8466a29650c4c5a5af6cc/greenlet-3.3.1-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:12184c61e5d64268a160226fb4818af4df02cfead8379d7f8b99a56c3a54ff3e", size = 624375, upload-time = "2026-01-23T16:15:55.915Z" }, { url = "https://files.pythonhosted.org/packages/cb/86/5c6ab23bb3c28c21ed6bebad006515cfe08b04613eb105ca0041fecca852/greenlet-3.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6423481193bbbe871313de5fd06a082f2649e7ce6e08015d2a76c1e9186ca5b3", size = 612904, upload-time = "2026-01-23T15:32:52.317Z" }, { url = "https://files.pythonhosted.org/packages/c2/f3/7949994264e22639e40718c2daf6f6df5169bf48fb038c008a489ec53a50/greenlet-3.3.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:33a956fe78bbbda82bfc95e128d61129b32d66bcf0a20a1f0c08aa4839ffa951", size = 1567316, upload-time = "2026-01-23T16:04:23.316Z" }, { url = "https://files.pythonhosted.org/packages/8d/6e/d73c94d13b6465e9f7cd6231c68abde838bb22408596c05d9059830b7872/greenlet-3.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4b065d3284be43728dd280f6f9a13990b56470b81be20375a207cdc814a983f2", size = 1636549, upload-time = "2026-01-23T15:33:48.643Z" }, @@ -657,7 +655,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/fb/011c7c717213182caf78084a9bea51c8590b0afda98001f69d9f853a495b/greenlet-3.3.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:bd59acd8529b372775cd0fcbc5f420ae20681c5b045ce25bd453ed8455ab99b5", size = 275737, upload-time = "2026-01-23T15:32:16.889Z" }, { url = "https://files.pythonhosted.org/packages/41/2e/a3a417d620363fdbb08a48b1dd582956a46a61bf8fd27ee8164f9dfe87c2/greenlet-3.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b31c05dd84ef6871dd47120386aed35323c944d86c3d91a17c4b8d23df62f15b", size = 646422, upload-time = "2026-01-23T16:01:00.354Z" }, { url = "https://files.pythonhosted.org/packages/b4/09/c6c4a0db47defafd2d6bab8ddfe47ad19963b4e30f5bed84d75328059f8c/greenlet-3.3.1-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:02925a0bfffc41e542c70aa14c7eda3593e4d7e274bfcccca1827e6c0875902e", size = 658219, upload-time = "2026-01-23T16:05:30.956Z" }, - { url = "https://files.pythonhosted.org/packages/e2/89/b95f2ddcc5f3c2bc09c8ee8d77be312df7f9e7175703ab780f2014a0e781/greenlet-3.3.1-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3e0f3878ca3a3ff63ab4ea478585942b53df66ddde327b59ecb191b19dbbd62d", size = 671455, upload-time = "2026-01-23T16:15:57.232Z" }, { url = "https://files.pythonhosted.org/packages/80/38/9d42d60dffb04b45f03dbab9430898352dba277758640751dc5cc316c521/greenlet-3.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34a729e2e4e4ffe9ae2408d5ecaf12f944853f40ad724929b7585bca808a9d6f", size = 660237, upload-time = "2026-01-23T15:32:53.967Z" }, { url = "https://files.pythonhosted.org/packages/96/61/373c30b7197f9e756e4c81ae90a8d55dc3598c17673f91f4d31c3c689c3f/greenlet-3.3.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:aec9ab04e82918e623415947921dea15851b152b822661cce3f8e4393c3df683", size = 1615261, upload-time = "2026-01-23T16:04:25.066Z" }, { url = "https://files.pythonhosted.org/packages/fd/d3/ca534310343f5945316f9451e953dcd89b36fe7a19de652a1dc5a0eeef3f/greenlet-3.3.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:71c767cf281a80d02b6c1bdc41c9468e1f5a494fb11bc8688c360524e273d7b1", size = 1683719, upload-time = "2026-01-23T15:33:50.61Z" }, @@ -666,7 +663,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/28/24/cbbec49bacdcc9ec652a81d3efef7b59f326697e7edf6ed775a5e08e54c2/greenlet-3.3.1-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:3e63252943c921b90abb035ebe9de832c436401d9c45f262d80e2d06cc659242", size = 282706, upload-time = "2026-01-23T15:33:05.525Z" }, { url = "https://files.pythonhosted.org/packages/86/2e/4f2b9323c144c4fe8842a4e0d92121465485c3c2c5b9e9b30a52e80f523f/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:76e39058e68eb125de10c92524573924e827927df5d3891fbc97bd55764a8774", size = 651209, upload-time = "2026-01-23T16:01:01.517Z" }, { url = "https://files.pythonhosted.org/packages/d9/87/50ca60e515f5bb55a2fbc5f0c9b5b156de7d2fc51a0a69abc9d23914a237/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9f9d5e7a9310b7a2f416dd13d2e3fd8b42d803968ea580b7c0f322ccb389b97", size = 654300, upload-time = "2026-01-23T16:05:32.199Z" }, - { url = "https://files.pythonhosted.org/packages/7c/25/c51a63f3f463171e09cb586eb64db0861eb06667ab01a7968371a24c4f3b/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b9721549a95db96689458a1e0ae32412ca18776ed004463df3a9299c1b257ab", size = 662574, upload-time = "2026-01-23T16:15:58.364Z" }, { url = "https://files.pythonhosted.org/packages/1d/94/74310866dfa2b73dd08659a3d18762f83985ad3281901ba0ee9a815194fb/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92497c78adf3ac703b57f1e3813c2d874f27f71a178f9ea5887855da413cd6d2", size = 653842, upload-time = "2026-01-23T15:32:55.671Z" }, { url = "https://files.pythonhosted.org/packages/97/43/8bf0ffa3d498eeee4c58c212a3905dd6146c01c8dc0b0a046481ca29b18c/greenlet-3.3.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ed6b402bc74d6557a705e197d47f9063733091ed6357b3de33619d8a8d93ac53", size = 1614917, upload-time = "2026-01-23T16:04:26.276Z" }, { url = "https://files.pythonhosted.org/packages/89/90/a3be7a5f378fc6e84abe4dcfb2ba32b07786861172e502388b4c90000d1b/greenlet-3.3.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:59913f1e5ada20fde795ba906916aea25d442abcc0593fba7e26c92b7ad76249", size = 1676092, upload-time = "2026-01-23T15:33:52.176Z" }, @@ -1308,6 +1304,7 @@ wheels = [ name = "mysql-connector-python" version = "9.6.0" source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6e/c89babc7de3df01467d159854414659c885152579903a8220c8db02a3835/mysql_connector_python-9.6.0.tar.gz", hash = "sha256:c453bb55347174d87504b534246fb10c589daf5d057515bf615627198a3c7ef1", size = 12254999, upload-time = "2026-02-10T12:04:52.63Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/16/27/106f5b7a69381c58cb0ba6bf44e6488969ce6cd9f69f62df340f379141ee/mysql_connector_python-9.6.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:478e035ebcf734b3a1497bfd3eb72ce3632da6384545b08cf6329471b3849b6e", size = 17582676, upload-time = "2026-01-21T09:04:35.755Z" }, { url = "https://files.pythonhosted.org/packages/17/97/d3cbab27663d7b5063d891c6bd322db6cf1cdb42d4e0d2d2ec9a1952c67f/mysql_connector_python-9.6.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:228000bb951810dad724821d04000174ffcc7fa94b4dcef884b17a3cdae07283", size = 18449353, upload-time = "2026-01-21T09:04:38.032Z" }, @@ -1318,6 +1315,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/93/aa/3dd4db039fc6a9bcbdbade83be9914ead6786c0be4918170dfaf89327b76/mysql_connector_python-9.6.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:b5212372aff6833473d2560ac87d3df9fb2498d0faacb7ebf231d947175fa36a", size = 18449358, upload-time = "2026-01-21T09:04:50.278Z" }, { url = "https://files.pythonhosted.org/packages/53/38/ecd6d35382b6265ff5f030464d53b45e51ff2c2523ab88771c277fd84c05/mysql_connector_python-9.6.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:61deca6e243fafbb3cf08ae27bd0c83d0f8188de8456e46aeba0d3db15bb7230", size = 34169309, upload-time = "2026-01-21T09:04:52.402Z" }, { url = "https://files.pythonhosted.org/packages/18/1d/fe1133eb76089342854d8fbe88e28598f7e06bc684a763d21fc7b23f1d5e/mysql_connector_python-9.6.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:adabbc5e1475cdf5fb6f1902a25edc3bd1e0726fa45f01ab1b8f479ff43b3337", size = 34541101, upload-time = "2026-01-21T09:04:55.897Z" }, + { url = "https://files.pythonhosted.org/packages/3f/99/da0f55beb970ca049fd7d37a6391d686222af89a8b13e636d8e9bbd06536/mysql_connector_python-9.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:8732ca0b7417b45238bcbfc7e64d9c4d62c759672207c6284f0921c366efddc7", size = 16514767, upload-time = "2026-02-10T12:03:50.584Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d9/2a4b4d90b52f4241f0f71618cd4bd8779dd6d18db8058b0a4dd83ec0541c/mysql_connector_python-9.6.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9664e217c72dd6fb700f4c8512af90261f72d2f5d7c00c4e13e4c1e09bfa3d5e", size = 17585672, upload-time = "2026-02-10T12:03:52.955Z" }, + { url = "https://files.pythonhosted.org/packages/33/91/2495835733a054e716a17dc28404748b33f2dc1da1ae4396fb45574adf40/mysql_connector_python-9.6.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:1ed4b5c4761e5333035293e746683890e4ef2e818e515d14023fd80293bc31fa", size = 18452624, upload-time = "2026-02-10T12:03:56.153Z" }, + { url = "https://files.pythonhosted.org/packages/7a/69/e83abbbbf7f8eed855b5a5ff7285bc0afb1199418ac036c7691edf41e154/mysql_connector_python-9.6.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5095758dcb89a6bce2379f349da336c268c407129002b595c5dba82ce387e2a5", size = 34169154, upload-time = "2026-02-10T12:03:58.831Z" }, + { url = "https://files.pythonhosted.org/packages/82/44/67bb61c71f398fbc739d07e8dcadad94e2f655874cb32ae851454066bea0/mysql_connector_python-9.6.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:4ae4e7780fad950a4f267dea5851048d160f5b71314a342cdbf30b154f1c74f7", size = 34542947, upload-time = "2026-02-10T12:04:02.408Z" }, + { url = "https://files.pythonhosted.org/packages/ba/39/994c4f7e9c59d3ca534a831d18442ac4c529865db20aeaa4fd94e2af5efd/mysql_connector_python-9.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:c180e0b4100d7402e03993bfac5c97d18e01d7ca9d198d742fffc245077f8ffe", size = 16515709, upload-time = "2026-02-10T12:04:04.924Z" }, + { url = "https://files.pythonhosted.org/packages/2f/58/9521aa678708ec6cebfd40524c14c3d151e4f29e3774e6086aa0a30d203b/mysql_connector_python-9.6.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e86e45a7b540ca09af8a18ecfa761e0cdeccfdb62818331614ec030ae44bfd26", size = 17585837, upload-time = "2026-02-10T12:04:07.004Z" }, + { url = "https://files.pythonhosted.org/packages/39/8d/b108f9bcce9780f6a1f91decb2af54defdaf845e237ddc42f2b4578f1cd7/mysql_connector_python-9.6.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:8d3e9252384e1b7f95b07020664f2673d9c29c5e95eeda2e048b3331e190b9d4", size = 18452844, upload-time = "2026-02-10T12:04:09.418Z" }, + { url = "https://files.pythonhosted.org/packages/d6/28/735cd93d16e76dc2feb4abb3f1229a1d9475af34d80c26712fec6abe1d70/mysql_connector_python-9.6.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:0fa18ead33cb699ea92005695077cef09aa494eebf51164ee30c891c3eaea90c", size = 34169374, upload-time = "2026-02-10T12:04:12.13Z" }, + { url = "https://files.pythonhosted.org/packages/42/07/069983799cf4050c68f61a494f94b06f095fee6026ab0dd863a14de30867/mysql_connector_python-9.6.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:a26490cb029bf7b18a1d2093101105b3526a1036b51ad01553d30138f5beb8d2", size = 34543019, upload-time = "2026-02-10T12:04:15.065Z" }, + { url = "https://files.pythonhosted.org/packages/32/00/fbeb7d666ab8153f719e620bac5abfbc74640e8ec511612493110a75fe66/mysql_connector_python-9.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:3460ed976e1b88b7284335d9397a3c519dff56d71580ca1f76ff1c0c7714c813", size = 16515701, upload-time = "2026-02-10T12:04:19.26Z" }, + { url = "https://files.pythonhosted.org/packages/70/51/13cc90b2a703784cd9a0aa0a6fce07946cf6a2abe7c8fd0b585562e250fc/mysql_connector_python-9.6.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:e2cc13cd3dcdb845d636e52c4e7a9509b63da09bec6ce1b3696be53a79847e2d", size = 17585800, upload-time = "2026-02-10T12:04:21.6Z" }, + { url = "https://files.pythonhosted.org/packages/c8/6b/ce7ab998fbdd17f35a1b54624365d039045cbb2d42bbc7b03f50d7597c7b/mysql_connector_python-9.6.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:a08c2149d4b52a010c4353f18c84716d18114a4ecd00b466ea34138de2c640f2", size = 18452823, upload-time = "2026-02-10T12:04:23.995Z" }, + { url = "https://files.pythonhosted.org/packages/f9/bf/8157ed61d17878c33511dcb97c68ecaaaf6220bea5a2944ea4eba73cc63a/mysql_connector_python-9.6.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:b00228b985edd208b20f45c5e684c54e08e31e01bc1d8c3c18a36641c3be5bf7", size = 34171594, upload-time = "2026-02-10T12:04:27.401Z" }, + { url = "https://files.pythonhosted.org/packages/f7/06/5efdd28819afdb9f1487a62842fda4277febe128a3cd6e9090dbe0a6524e/mysql_connector_python-9.6.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4617ef5216da7ca32dd46afda61a1552807762434127413bba46fbe4379f59d4", size = 34542851, upload-time = "2026-02-10T12:04:31.021Z" }, + { url = "https://files.pythonhosted.org/packages/40/6a/26e08a4a79f159cd8e5b64eb10bd056e7735b65d4464d98641f59eb9ca3a/mysql_connector_python-9.6.0-cp314-cp314-win_amd64.whl", hash = "sha256:bc782f64ca00b6b933d4c6a35568f1349d115cc4434c849b5b9edc015bee3e62", size = 17002947, upload-time = "2026-02-10T12:04:34.386Z" }, + { url = "https://files.pythonhosted.org/packages/15/dd/b3250826c29cee7816de4409a2fe5e469a68b9a89f6bfaa5eed74f05532c/mysql_connector_python-9.6.0-py2.py3-none-any.whl", hash = "sha256:44b0fb57207ebc6ae05b5b21b7968a9ed33b29187fe87b38951bad2a334d75d5", size = 480527, upload-time = "2026-02-10T12:04:36.176Z" }, ] [[package]] @@ -2471,6 +2485,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/69/297302c5f5f59c862faa31e6cb9a4cd74721cd1e052b38e464c5b402df8b/StrEnum-0.4.15-py3-none-any.whl", hash = "sha256:a30cda4af7cc6b5bf52c8055bc4bf4b2b6b14a93b574626da33df53cf7740659", size = 8851, upload-time = "2023-06-29T22:02:56.947Z" }, ] +[[package]] +name = "structlog" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/89/b4a0bcfdf4f71a3dea31379f095929613d7e4528a0996bca6aa964cd0dca/structlog-26.1.0.tar.gz", hash = "sha256:f63a716cbd1b1291cf7661de7794b455acfa4c43c5bcf1630e6ad5ddc1adb3b7", size = 1459881, upload-time = "2026-06-06T07:33:39.348Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/18/489c97b834dfff9cf2fc2507cede4bcd4b11e67f84bc462acd1992496f86/structlog-26.1.0-py3-none-any.whl", hash = "sha256:e081a26d6c373e6d201eca24eede26d8ffab07f88f477822e679183428d3d91e", size = 73764, upload-time = "2026-06-06T07:33:38.046Z" }, +] + [[package]] name = "tomli" version = "2.4.0" diff --git a/components/clp-package-utils/uv.lock b/components/clp-package-utils/uv.lock index 1f8244f4c7..84c02cbffc 100644 --- a/components/clp-package-utils/uv.lock +++ b/components/clp-package-utils/uv.lock @@ -263,6 +263,7 @@ dependencies = [ { name = "result" }, { name = "sqlalchemy" }, { name = "strenum" }, + { name = "structlog" }, ] [package.metadata] @@ -276,6 +277,7 @@ requires-dist = [ { name = "result", specifier = ">=0.17.0" }, { name = "sqlalchemy", specifier = ">=2.0.46" }, { name = "strenum", specifier = ">=0.4.15" }, + { name = "structlog", specifier = ">=25.5.0" }, ] [package.metadata.requires-dev] @@ -1392,6 +1394,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/69/297302c5f5f59c862faa31e6cb9a4cd74721cd1e052b38e464c5b402df8b/StrEnum-0.4.15-py3-none-any.whl", hash = "sha256:a30cda4af7cc6b5bf52c8055bc4bf4b2b6b14a93b574626da33df53cf7740659", size = 8851, upload-time = "2023-06-29T22:02:56.947Z" }, ] +[[package]] +name = "structlog" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/89/b4a0bcfdf4f71a3dea31379f095929613d7e4528a0996bca6aa964cd0dca/structlog-26.1.0.tar.gz", hash = "sha256:f63a716cbd1b1291cf7661de7794b455acfa4c43c5bcf1630e6ad5ddc1adb3b7", size = 1459881, upload-time = "2026-06-06T07:33:39.348Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/18/489c97b834dfff9cf2fc2507cede4bcd4b11e67f84bc462acd1992496f86/structlog-26.1.0-py3-none-any.whl", hash = "sha256:e081a26d6c373e6d201eca24eede26d8ffab07f88f477822e679183428d3d91e", size = 73764, upload-time = "2026-06-06T07:33:38.046Z" }, +] + [[package]] name = "tomli" version = "2.4.0" diff --git a/components/job-orchestration/uv.lock b/components/job-orchestration/uv.lock index 7853c9ff3a..06759dd3e7 100644 --- a/components/job-orchestration/uv.lock +++ b/components/job-orchestration/uv.lock @@ -253,6 +253,7 @@ dependencies = [ { name = "result" }, { name = "sqlalchemy" }, { name = "strenum" }, + { name = "structlog" }, ] [package.metadata] @@ -266,6 +267,7 @@ requires-dist = [ { name = "result", specifier = ">=0.17.0" }, { name = "sqlalchemy", specifier = ">=2.0.46" }, { name = "strenum", specifier = ">=0.4.15" }, + { name = "structlog", specifier = ">=25.5.0" }, ] [package.metadata.requires-dev] @@ -1392,6 +1394,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/69/297302c5f5f59c862faa31e6cb9a4cd74721cd1e052b38e464c5b402df8b/StrEnum-0.4.15-py3-none-any.whl", hash = "sha256:a30cda4af7cc6b5bf52c8055bc4bf4b2b6b14a93b574626da33df53cf7740659", size = 8851, upload-time = "2023-06-29T22:02:56.947Z" }, ] +[[package]] +name = "structlog" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/89/b4a0bcfdf4f71a3dea31379f095929613d7e4528a0996bca6aa964cd0dca/structlog-26.1.0.tar.gz", hash = "sha256:f63a716cbd1b1291cf7661de7794b455acfa4c43c5bcf1630e6ad5ddc1adb3b7", size = 1459881, upload-time = "2026-06-06T07:33:39.348Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/18/489c97b834dfff9cf2fc2507cede4bcd4b11e67f84bc462acd1992496f86/structlog-26.1.0-py3-none-any.whl", hash = "sha256:e081a26d6c373e6d201eca24eede26d8ffab07f88f477822e679183428d3d91e", size = 73764, upload-time = "2026-06-06T07:33:38.046Z" }, +] + [[package]] name = "tomli" version = "2.4.0" diff --git a/integration-tests/uv.lock b/integration-tests/uv.lock index 34513f78e1..3acccd3f81 100644 --- a/integration-tests/uv.lock +++ b/integration-tests/uv.lock @@ -550,6 +550,7 @@ dependencies = [ { name = "result" }, { name = "sqlalchemy" }, { name = "strenum" }, + { name = "structlog" }, ] [package.metadata] @@ -563,6 +564,7 @@ requires-dist = [ { name = "result", specifier = ">=0.17.0" }, { name = "sqlalchemy", specifier = ">=2.0.46" }, { name = "strenum", specifier = ">=0.4.15" }, + { name = "structlog", specifier = ">=25.5.0" }, ] [package.metadata.requires-dev] @@ -2794,6 +2796,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/69/297302c5f5f59c862faa31e6cb9a4cd74721cd1e052b38e464c5b402df8b/StrEnum-0.4.15-py3-none-any.whl", hash = "sha256:a30cda4af7cc6b5bf52c8055bc4bf4b2b6b14a93b574626da33df53cf7740659", size = 8851, upload-time = "2023-06-29T22:02:56.947Z" }, ] +[[package]] +name = "structlog" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/89/b4a0bcfdf4f71a3dea31379f095929613d7e4528a0996bca6aa964cd0dca/structlog-26.1.0.tar.gz", hash = "sha256:f63a716cbd1b1291cf7661de7794b455acfa4c43c5bcf1630e6ad5ddc1adb3b7", size = 1459881, upload-time = "2026-06-06T07:33:39.348Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/18/489c97b834dfff9cf2fc2507cede4bcd4b11e67f84bc462acd1992496f86/structlog-26.1.0-py3-none-any.whl", hash = "sha256:e081a26d6c373e6d201eca24eede26d8ffab07f88f477822e679183428d3d91e", size = 73764, upload-time = "2026-06-06T07:33:38.046Z" }, +] + [[package]] name = "tomli" version = "2.4.0" From b325d9b925d2400c1d3a99abde69c15b73474151 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 23 Jun 2026 14:09:07 -0400 Subject: [PATCH 05/35] feat(logging): Implement structured JSON logging for CLP components --- .../clp-py-utils/clp_py_utils/clp_logging.py | 95 ++++++++----------- .../executor/compress/celery.py | 14 ++- .../executor/query/celery.py | 14 ++- 3 files changed, 65 insertions(+), 58 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_logging.py b/components/clp-py-utils/clp_py_utils/clp_logging.py index 52458164f2..7607c652c1 100644 --- a/components/clp-py-utils/clp_py_utils/clp_logging.py +++ b/components/clp-py-utils/clp_py_utils/clp_logging.py @@ -1,17 +1,12 @@ +"""Shared logging helpers for CLP Python components.""" + import logging import os import pathlib -from collections.abc import Iterator -from contextlib import contextmanager from typing import get_args, Literal import structlog -from structlog.contextvars import ( - bind_contextvars, - get_contextvars, - merge_contextvars, - reset_contextvars, -) +from structlog.typing import Processor LoggingLevel = Literal[ "INFO", @@ -22,65 +17,53 @@ "CRITICAL", ] -_TIMESTAMP_PROCESSOR = structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp") -_JSON_RENDERER = structlog.processors.JSONRenderer() - - -@contextmanager -def bind_log_context(**fields: object) -> Iterator[None]: - """ - Adds CLP correlation fields to log records emitted in the current context. - - Fields with ``None`` values are ignored. Nested contexts merge with outer contexts and restore - the previous context on exit. - """ - tokens = bind_contextvars(**{key: value for key, value in fields.items() if value is not None}) - try: - yield - finally: - reset_contextvars(**tokens) - - -def get_log_context() -> dict[str, object]: - """Returns a copy of the active CLP log correlation context.""" - return get_contextvars() - - -class _ClpJsonFormatter(logging.Formatter): - """Formats log records as JSON and includes active CLP correlation fields.""" - - def format(self, record: logging.LogRecord) -> str: - method_name = record.levelname.lower() - event_dict: dict[str, object] = { - "event": record.getMessage(), - "logger": record.name, - "level": method_name, - } - if record.exc_info: - event_dict["exception"] = self.formatException(record.exc_info) - if record.stack_info: - event_dict["stack"] = self.formatStack(record.stack_info) +_STRUCTLOG_PROCESSORS: tuple[Processor, ...] = ( + structlog.stdlib.add_logger_name, + structlog.stdlib.add_log_level, + structlog.stdlib.PositionalArgumentsFormatter(), + structlog.processors.StackInfoRenderer(), + structlog.processors.format_exc_info, + structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"), +) +_FOREIGN_PRE_CHAIN: tuple[Processor, ...] = ( + structlog.stdlib.add_logger_name, + structlog.stdlib.add_log_level, + structlog.stdlib.ExtraAdder(), + structlog.stdlib.PositionalArgumentsFormatter(), + structlog.processors.StackInfoRenderer(), + structlog.processors.format_exc_info, + structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"), +) - merge_contextvars(None, method_name, event_dict) - _TIMESTAMP_PROCESSOR(None, method_name, event_dict) - rendered_log = _JSON_RENDERER(None, method_name, event_dict) - if isinstance(rendered_log, bytes): - return rendered_log.decode("utf-8") - return rendered_log +structlog.configure( + processors=[ + *_STRUCTLOG_PROCESSORS, + structlog.stdlib.ProcessorFormatter.wrap_for_formatter, + ], + logger_factory=structlog.stdlib.LoggerFactory(), + cache_logger_on_first_use=True, +) def get_logging_formatter() -> logging.Formatter: - return _ClpJsonFormatter() + return structlog.stdlib.ProcessorFormatter( + foreign_pre_chain=_FOREIGN_PRE_CHAIN, + processors=[ + structlog.stdlib.ProcessorFormatter.remove_processors_meta, + structlog.processors.JSONRenderer(), + ], + ) -def set_formatter_on_handlers(logger: logging.Logger) -> None: - """Sets CLP's logging formatter on all existing handlers for a logger.""" +def set_json_formatter_on_handlers(logger: logging.Logger) -> None: + """Set CLP's JSON log formatter on all handlers currently attached to *logger*.""" formatter = get_logging_formatter() for handler in logger.handlers: handler.setFormatter(formatter) def get_logger(name: str) -> logging.Logger: + """Return a logger configured with CLP's console formatter.""" logger = logging.getLogger(name) # Setup console logging logging_console_handler = logging.StreamHandler() @@ -96,7 +79,7 @@ def set_logging_level(logger: logging.Logger, level: str | None) -> None: logger.setLevel(logging.INFO) return if level not in get_args(LoggingLevel): - logger.warning(f"Invalid logging level: {level}, using INFO as default") + logger.warning("Invalid logging level: %s, using INFO as default", level) logger.setLevel(logging.INFO) return diff --git a/components/job-orchestration/job_orchestration/executor/compress/celery.py b/components/job-orchestration/job_orchestration/executor/compress/celery.py index db5dc6a5c3..71eb296319 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/celery.py +++ b/components/job-orchestration/job_orchestration/executor/compress/celery.py @@ -1,9 +1,21 @@ -from celery import Celery +import logging + +from celery import Celery, signals +from clp_py_utils.clp_logging import set_json_formatter_on_handlers from job_orchestration.executor.compress import celeryconfig app = Celery("compress") app.config_from_object(celeryconfig) + +@signals.after_setup_logger.connect +@signals.after_setup_task_logger.connect +def setup_json_logging(logger: logging.Logger | None = None, **_: object) -> None: + """Use CLP's JSON formatter for loggers configured by Celery.""" + if logger is not None: + set_json_formatter_on_handlers(logger) + + if "__main__" == __name__: app.start() diff --git a/components/job-orchestration/job_orchestration/executor/query/celery.py b/components/job-orchestration/job_orchestration/executor/query/celery.py index d0a0f239d2..4e8d79aa88 100644 --- a/components/job-orchestration/job_orchestration/executor/query/celery.py +++ b/components/job-orchestration/job_orchestration/executor/query/celery.py @@ -1,9 +1,21 @@ -from celery import Celery +import logging + +from celery import Celery, signals +from clp_py_utils.clp_logging import set_json_formatter_on_handlers from job_orchestration.executor.query import celeryconfig app = Celery("query") app.config_from_object(celeryconfig) + +@signals.after_setup_logger.connect +@signals.after_setup_task_logger.connect +def setup_json_logging(logger: logging.Logger | None = None, **_: object) -> None: + """Use CLP's JSON formatter for loggers configured by Celery.""" + if logger is not None: + set_json_formatter_on_handlers(logger) + + if "__main__" == __name__: app.start() From b2c8a338358bbd682269d0ebd97bec00ee3ff3c9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 23 Jun 2026 14:13:44 -0400 Subject: [PATCH 06/35] docs(logging): Add logging documentation outlining component-specific logging setups and controls --- docs/src/dev-docs/index.md | 1 + docs/src/dev-docs/logging.md | 109 +++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 docs/src/dev-docs/logging.md diff --git a/docs/src/dev-docs/index.md b/docs/src/dev-docs/index.md index c9146763b1..f22ae29f37 100644 --- a/docs/src/dev-docs/index.md +++ b/docs/src/dev-docs/index.md @@ -96,6 +96,7 @@ tooling-gh-workflows design-project-structure design-deployment-orchestration +logging design-kv-ir-streams/index design-metadata-db diff --git a/docs/src/dev-docs/logging.md b/docs/src/dev-docs/logging.md new file mode 100644 index 0000000000..a58fd6df73 --- /dev/null +++ b/docs/src/dev-docs/logging.md @@ -0,0 +1,109 @@ +# Logging + +The CLP package contains multiple runtime stacks, and each stack owns its logging setup. This page +summarizes the current behavior and the controls developers/operators should use. + +## Component summary + +| Component family | Examples | Logger | Format | Level control | +|-------------------------------|---------------------------------------------------------------------|---------------------------------------|----------------------------------|----------------------------| +| Python orchestration services | Schedulers, workers, reducer wrapper, garbage collector, MCP server | `clp_py_utils.clp_logging` | JSON | `CLP_LOGGING_LEVEL` | +| Rust HTTP services | API server, log ingestor | `clp_rust_utils::logging` / `tracing` | JSON | `RUST_LOG` | +| WebUI server | Fastify server | Fastify/Pino | JSON in prod; pretty text in dev | `LOG_LEVEL` | +| WebUI client | Browser app | Browser console | Browser console output | Browser/devtools dependent | +| Native core binaries | `clp`, `clp-s`, `glt`, native `reducer_server` | `spdlog` | Text | Binary-specific | +| Package/setup tools | Package controller, DB initialization scripts | Python stdlib logging | Text | Script-specific | + +There is no single project-wide JSON schema. Python, Rust, and Pino logs are all line-delimited JSON +in packaged non-interactive service runtimes, but each stack uses its own field names. + +## Python orchestration services + +These services use `clp_py_utils.clp_logging` and emit one JSON object per log record: + +* `compression_scheduler` +* `query_scheduler` +* `compression_worker` +* `query_worker` +* `reducer` +* `garbage_collector` +* `mcp_server` + +Python service records include: + +| Field | Description | +|--------------|----------------------------------------------------------| +| `timestamp` | ISO-8601 UTC timestamp. | +| `event` | Formatted log message. | +| `logger` | Logger name. | +| `level` | Lowercase log level. | +| `exception` | Present when `exc_info` is logged. | +| `stack` | Present when `stack_info` is logged. | +| Extra fields | Fields passed through stdlib logging's `extra` argument. | + +Example: + + +```json +{"timestamp":"2026-06-22T17:03:21.123456Z","event":"Compression job 1 submitted.","logger":"compression_scheduler","level":"info"} +``` + + +Controls: + +* `CLP_LOGGING_LEVEL` supports `DEBUG`, `INFO`, `WARN`, `WARNING`, `ERROR`, and `CRITICAL`. + Missing or invalid values default to `INFO`. +* `CLP_LOGS_DIR`, when set, adds a file handler at `/.log` in + addition to stdout. + +## Rust HTTP services + +`api_server` and `log_ingestor` use `clp_rust_utils::logging::set_up_logging`, which configures +`tracing_subscriber` JSON output. + +Rust service records include `timestamp`, `level`, `fields`, `filename`, and `line_number`. +Structured `tracing` fields are nested under `fields`. + +Controls: + +* `RUST_LOG` uses `tracing_subscriber::EnvFilter` syntax. +* `CLP_LOGS_DIR`, when set, adds an hourly non-blocking rolling file appender. + +In the package manifests, `log_ingestor` exposes a deployment setting through +`CLP_LOG_INGESTOR_LOGGING_LEVEL` in Docker Compose and `clpConfig.log_ingestor.logging_level` in +Helm. `api_server` currently runs with `RUST_LOG=INFO`. + +## WebUI + +The WebUI server uses Fastify's Pino logger: + +* Non-interactive runtimes emit Pino JSON to stdout. +* Interactive terminals use `pino-pretty` for developer-readable output. +* `LOG_LEVEL` controls the server log level and defaults to `info`. + +The WebUI client logs to the browser console. Treat client `console.*` output as browser diagnostics, +not service logs. + +## Native core binaries and package tools + +Native core binaries use `spdlog` text output. Package controller commands and one-shot setup scripts +also use human-readable stdlib logging. These tools are not covered by the Python/Rust/WebUI service +JSON contracts. + +## Deployment notes + +* Docker Compose service stdout is available through `docker compose logs`. + * Compose services that set `CLP_LOGS_DIR` also write under the mounted CLP log directory, which + defaults to `./var/log` through `CLP_LOGS_DIR_HOST`. +* Helm deployments should primarily use pod stdout through `kubectl logs` or the cluster log + collector. File logging is template-specific. + +## Development guidance + +* New Python orchestration services should use `clp_py_utils.clp_logging.get_logger` and + `clp_py_utils.clp_logging.configure_logging`. +* New Rust services should use `clp_rust_utils::logging::set_up_logging`. +* WebUI server code should log through Fastify's logger (`request.log` or `app.log`). +* WebUI client `console.*` calls should stay limited to browser diagnostics. +* Native core binaries should keep their existing `spdlog` setup unless a separate change migrates + them to structured logging. From dd31655d89ec652e612a58c0c92b2f7af3359a58 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 23 Jun 2026 14:15:57 -0400 Subject: [PATCH 07/35] chore(dependencies): Update structlog dependency to version 26.1.0 --- components/clp-mcp-server/uv.lock | 2 +- components/clp-package-utils/uv.lock | 2 +- components/clp-py-utils/pyproject.toml | 2 +- components/clp-py-utils/uv.lock | 2 +- components/job-orchestration/uv.lock | 2 +- integration-tests/uv.lock | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/clp-mcp-server/uv.lock b/components/clp-mcp-server/uv.lock index 868c87721e..c91130bb97 100644 --- a/components/clp-mcp-server/uv.lock +++ b/components/clp-mcp-server/uv.lock @@ -407,7 +407,7 @@ requires-dist = [ { name = "result", specifier = ">=0.17.0" }, { name = "sqlalchemy", specifier = ">=2.0.46" }, { name = "strenum", specifier = ">=0.4.15" }, - { name = "structlog", specifier = ">=25.5.0" }, + { name = "structlog", specifier = ">=26.1.0" }, ] [package.metadata.requires-dev] diff --git a/components/clp-package-utils/uv.lock b/components/clp-package-utils/uv.lock index 84c02cbffc..b970f39c45 100644 --- a/components/clp-package-utils/uv.lock +++ b/components/clp-package-utils/uv.lock @@ -277,7 +277,7 @@ requires-dist = [ { name = "result", specifier = ">=0.17.0" }, { name = "sqlalchemy", specifier = ">=2.0.46" }, { name = "strenum", specifier = ">=0.4.15" }, - { name = "structlog", specifier = ">=25.5.0" }, + { name = "structlog", specifier = ">=26.1.0" }, ] [package.metadata.requires-dev] diff --git a/components/clp-py-utils/pyproject.toml b/components/clp-py-utils/pyproject.toml index 6c6aaf056d..25433a46ab 100644 --- a/components/clp-py-utils/pyproject.toml +++ b/components/clp-py-utils/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ "result>=0.17.0", "sqlalchemy>=2.0.46", "StrEnum>=0.4.15", - "structlog>=25.5.0", + "structlog>=26.1.0", ] [build-system] diff --git a/components/clp-py-utils/uv.lock b/components/clp-py-utils/uv.lock index ca40259c16..6f2e0a1584 100644 --- a/components/clp-py-utils/uv.lock +++ b/components/clp-py-utils/uv.lock @@ -277,7 +277,7 @@ requires-dist = [ { name = "result", specifier = ">=0.17.0" }, { name = "sqlalchemy", specifier = ">=2.0.46" }, { name = "strenum", specifier = ">=0.4.15" }, - { name = "structlog", specifier = ">=25.5.0" }, + { name = "structlog", specifier = ">=26.1.0" }, ] [package.metadata.requires-dev] diff --git a/components/job-orchestration/uv.lock b/components/job-orchestration/uv.lock index 06759dd3e7..23f980abdd 100644 --- a/components/job-orchestration/uv.lock +++ b/components/job-orchestration/uv.lock @@ -267,7 +267,7 @@ requires-dist = [ { name = "result", specifier = ">=0.17.0" }, { name = "sqlalchemy", specifier = ">=2.0.46" }, { name = "strenum", specifier = ">=0.4.15" }, - { name = "structlog", specifier = ">=25.5.0" }, + { name = "structlog", specifier = ">=26.1.0" }, ] [package.metadata.requires-dev] diff --git a/integration-tests/uv.lock b/integration-tests/uv.lock index 3acccd3f81..63f09ad182 100644 --- a/integration-tests/uv.lock +++ b/integration-tests/uv.lock @@ -564,7 +564,7 @@ requires-dist = [ { name = "result", specifier = ">=0.17.0" }, { name = "sqlalchemy", specifier = ">=2.0.46" }, { name = "strenum", specifier = ">=0.4.15" }, - { name = "structlog", specifier = ">=25.5.0" }, + { name = "structlog", specifier = ">=26.1.0" }, ] [package.metadata.requires-dev] From 63ea1a178fc80b5dd4e5e13545efdd71758fdd1b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 23 Jun 2026 14:23:39 -0400 Subject: [PATCH 08/35] feat(logging): Enhance logging processors with context variable merging and level filtering --- components/clp-py-utils/clp_py_utils/clp_logging.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/clp-py-utils/clp_py_utils/clp_logging.py b/components/clp-py-utils/clp_py_utils/clp_logging.py index 7607c652c1..06bd8a56f0 100644 --- a/components/clp-py-utils/clp_py_utils/clp_logging.py +++ b/components/clp-py-utils/clp_py_utils/clp_logging.py @@ -18,6 +18,8 @@ ] _STRUCTLOG_PROCESSORS: tuple[Processor, ...] = ( + structlog.contextvars.merge_contextvars, + structlog.stdlib.filter_by_level, structlog.stdlib.add_logger_name, structlog.stdlib.add_log_level, structlog.stdlib.PositionalArgumentsFormatter(), @@ -26,6 +28,7 @@ structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"), ) _FOREIGN_PRE_CHAIN: tuple[Processor, ...] = ( + structlog.contextvars.merge_contextvars, structlog.stdlib.add_logger_name, structlog.stdlib.add_log_level, structlog.stdlib.ExtraAdder(), From 79fd7d37299217dcef45c193d434ecf83aabed01 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 23 Jun 2026 14:26:06 -0400 Subject: [PATCH 09/35] docs(logging): Update logging documentation to include links for component-specific loggers --- docs/src/dev-docs/logging.md | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/docs/src/dev-docs/logging.md b/docs/src/dev-docs/logging.md index a58fd6df73..87330b893d 100644 --- a/docs/src/dev-docs/logging.md +++ b/docs/src/dev-docs/logging.md @@ -5,21 +5,22 @@ summarizes the current behavior and the controls developers/operators should use ## Component summary -| Component family | Examples | Logger | Format | Level control | -|-------------------------------|---------------------------------------------------------------------|---------------------------------------|----------------------------------|----------------------------| -| Python orchestration services | Schedulers, workers, reducer wrapper, garbage collector, MCP server | `clp_py_utils.clp_logging` | JSON | `CLP_LOGGING_LEVEL` | -| Rust HTTP services | API server, log ingestor | `clp_rust_utils::logging` / `tracing` | JSON | `RUST_LOG` | -| WebUI server | Fastify server | Fastify/Pino | JSON in prod; pretty text in dev | `LOG_LEVEL` | -| WebUI client | Browser app | Browser console | Browser console output | Browser/devtools dependent | -| Native core binaries | `clp`, `clp-s`, `glt`, native `reducer_server` | `spdlog` | Text | Binary-specific | -| Package/setup tools | Package controller, DB initialization scripts | Python stdlib logging | Text | Script-specific | +| Component family | Examples | Logger | Format | Level control | +|-------------------------------|---------------------------------------------------------------------|-----------------------------------------------------------|----------------------------------|----------------------------| +| Python orchestration services | Schedulers, workers, reducer wrapper, garbage collector, MCP server | [`clp_py_utils.clp_logging`][clp-py-logging] | JSON | `CLP_LOGGING_LEVEL` | +| Rust HTTP services | API server, log ingestor | [`clp_rust_utils::logging`][clp-rust-logging] / `tracing` | JSON | `RUST_LOG` | +| WebUI server | Fastify server | Fastify/Pino | JSON in prod; pretty text in dev | `LOG_LEVEL` | +| WebUI client | Browser app | Browser console | Browser console output | Browser/devtools dependent | +| Native core binaries | `clp`, `clp-s`, `glt`, native `reducer_server` | `spdlog` | Text | Binary-specific | +| Package/setup tools | Package controller, DB initialization scripts | Python stdlib logging | Text | Script-specific | There is no single project-wide JSON schema. Python, Rust, and Pino logs are all line-delimited JSON in packaged non-interactive service runtimes, but each stack uses its own field names. ## Python orchestration services -These services use `clp_py_utils.clp_logging` and emit one JSON object per log record: +These services use [`clp_py_utils.clp_logging`][clp-py-logging] and emit one JSON object per log +record: * `compression_scheduler` * `query_scheduler` @@ -58,7 +59,8 @@ Controls: ## Rust HTTP services -`api_server` and `log_ingestor` use `clp_rust_utils::logging::set_up_logging`, which configures +`api_server` and `log_ingestor` use +[`clp_rust_utils::logging::set_up_logging`][clp-rust-logging], which configures `tracing_subscriber` JSON output. Rust service records include `timestamp`, `level`, `fields`, `filename`, and `line_number`. @@ -100,10 +102,14 @@ JSON contracts. ## Development guidance -* New Python orchestration services should use `clp_py_utils.clp_logging.get_logger` and - `clp_py_utils.clp_logging.configure_logging`. -* New Rust services should use `clp_rust_utils::logging::set_up_logging`. +* New Python orchestration services should use + [`clp_py_utils.clp_logging.get_logger`][clp-py-logging] and + [`clp_py_utils.clp_logging.configure_logging`][clp-py-logging]. +* New Rust services should use [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging]. * WebUI server code should log through Fastify's logger (`request.log` or `app.log`). * WebUI client `console.*` calls should stay limited to browser diagnostics. * Native core binaries should keep their existing `spdlog` setup unless a separate change migrates them to structured logging. + +[clp-py-logging]: https://github.com/y-scope/clp/blob/main/components/clp-py-utils/clp_py_utils/clp_logging.py +[clp-rust-logging]: https://github.com/y-scope/clp/blob/main/components/clp-rust-utils/src/logging.rs From 0c1b40c13b346e24e66266913bea1d56a48211df Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 23 Jun 2026 14:37:45 -0400 Subject: [PATCH 10/35] docs(logging): Add timestamp convention guidelines for structured service logs --- docs/src/dev-docs/logging.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/src/dev-docs/logging.md b/docs/src/dev-docs/logging.md index 87330b893d..b54c3485f8 100644 --- a/docs/src/dev-docs/logging.md +++ b/docs/src/dev-docs/logging.md @@ -17,6 +17,13 @@ summarizes the current behavior and the controls developers/operators should use There is no single project-wide JSON schema. Python, Rust, and Pino logs are all line-delimited JSON in packaged non-interactive service runtimes, but each stack uses its own field names. +## Timestamp convention + +Structured service logs should use UTC or another unambiguous absolute timestamp format. Python +service logs use UTC ISO-8601 timestamps, Rust service logs use `tracing_subscriber` timestamps, and +WebUI server logs use Pino epoch-millisecond timestamps. Prefer converting to local time in the log +viewer or aggregator rather than emitting ambiguous local-time strings from services. + ## Python orchestration services These services use [`clp_py_utils.clp_logging`][clp-py-logging] and emit one JSON object per log From 966fd23f610aee9c26c873c8b94fc18429ee05a7 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 23 Jun 2026 14:41:05 -0400 Subject: [PATCH 11/35] docs(logging): reorder --- docs/src/dev-docs/logging.md | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/docs/src/dev-docs/logging.md b/docs/src/dev-docs/logging.md index b54c3485f8..8bc4f7290e 100644 --- a/docs/src/dev-docs/logging.md +++ b/docs/src/dev-docs/logging.md @@ -24,7 +24,11 @@ service logs use UTC ISO-8601 timestamps, Rust service logs use `tracing_subscri WebUI server logs use Pino epoch-millisecond timestamps. Prefer converting to local time in the log viewer or aggregator rather than emitting ambiguous local-time strings from services. -## Python orchestration services +## Component details + +The following sections describe the behavior and controls for each logging stack. + +### Python orchestration services These services use [`clp_py_utils.clp_logging`][clp-py-logging] and emit one JSON object per log record: @@ -64,7 +68,7 @@ Controls: * `CLP_LOGS_DIR`, when set, adds a file handler at `/.log` in addition to stdout. -## Rust HTTP services +### Rust HTTP services `api_server` and `log_ingestor` use [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging], which configures @@ -82,7 +86,7 @@ In the package manifests, `log_ingestor` exposes a deployment setting through `CLP_LOG_INGESTOR_LOGGING_LEVEL` in Docker Compose and `clpConfig.log_ingestor.logging_level` in Helm. `api_server` currently runs with `RUST_LOG=INFO`. -## WebUI +### WebUI The WebUI server uses Fastify's Pino logger: @@ -93,20 +97,12 @@ The WebUI server uses Fastify's Pino logger: The WebUI client logs to the browser console. Treat client `console.*` output as browser diagnostics, not service logs. -## Native core binaries and package tools +### Native core binaries and package tools Native core binaries use `spdlog` text output. Package controller commands and one-shot setup scripts also use human-readable stdlib logging. These tools are not covered by the Python/Rust/WebUI service JSON contracts. -## Deployment notes - -* Docker Compose service stdout is available through `docker compose logs`. - * Compose services that set `CLP_LOGS_DIR` also write under the mounted CLP log directory, which - defaults to `./var/log` through `CLP_LOGS_DIR_HOST`. -* Helm deployments should primarily use pod stdout through `kubectl logs` or the cluster log - collector. File logging is template-specific. - ## Development guidance * New Python orchestration services should use @@ -118,5 +114,13 @@ JSON contracts. * Native core binaries should keep their existing `spdlog` setup unless a separate change migrates them to structured logging. -[clp-py-logging]: https://github.com/y-scope/clp/blob/main/components/clp-py-utils/clp_py_utils/clp_logging.py -[clp-rust-logging]: https://github.com/y-scope/clp/blob/main/components/clp-rust-utils/src/logging.rs +## Deployment notes + +* Docker Compose service stdout is available through `docker compose logs`. + * Compose services that set `CLP_LOGS_DIR` also write under the mounted CLP log directory, which + defaults to `./var/log` through `CLP_LOGS_DIR_HOST`. +* Helm deployments should primarily use pod stdout through `kubectl logs` or the cluster log + collector. File logging is template-specific. + +[clp-py-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-py-utils/clp_py_utils/clp_logging.py +[clp-rust-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-rust-utils/src/logging.rs From 718f6b6d47b36a282bbbce4aea33f63ce3fba1fe Mon Sep 17 00:00:00 2001 From: Rishikesh Devsot Date: Wed, 24 Jun 2026 00:50:59 -0400 Subject: [PATCH 12/35] added comments and re-ordered processor chains --- .../clp-py-utils/clp_py_utils/clp_logging.py | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_logging.py b/components/clp-py-utils/clp_py_utils/clp_logging.py index 06bd8a56f0..88d5b0a145 100644 --- a/components/clp-py-utils/clp_py_utils/clp_logging.py +++ b/components/clp-py-utils/clp_py_utils/clp_logging.py @@ -17,25 +17,45 @@ "CRITICAL", ] +# Processor chain for events emitted through structlog loggers before they reach +# ``ProcessorFormatter``. _STRUCTLOG_PROCESSORS: tuple[Processor, ...] = ( - structlog.contextvars.merge_contextvars, structlog.stdlib.filter_by_level, + structlog.contextvars.merge_contextvars, structlog.stdlib.add_logger_name, structlog.stdlib.add_log_level, structlog.stdlib.PositionalArgumentsFormatter(), + structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"), structlog.processors.StackInfoRenderer(), structlog.processors.format_exc_info, - structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"), + structlog.processors.UnicodeDecoder(), + structlog.processors.CallsiteParameterAdder( + { + structlog.processors.CallsiteParameter.FILENAME, + structlog.processors.CallsiteParameter.FUNC_NAME, + structlog.processors.CallsiteParameter.LINENO, + } + ), ) + +# Processor chain for stdlib ``LogRecord`` objects that did not originate from structlog. _FOREIGN_PRE_CHAIN: tuple[Processor, ...] = ( structlog.contextvars.merge_contextvars, structlog.stdlib.add_logger_name, structlog.stdlib.add_log_level, structlog.stdlib.ExtraAdder(), structlog.stdlib.PositionalArgumentsFormatter(), + structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"), structlog.processors.StackInfoRenderer(), structlog.processors.format_exc_info, - structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"), + structlog.processors.UnicodeDecoder(), + structlog.processors.CallsiteParameterAdder( + { + structlog.processors.CallsiteParameter.FILENAME, + structlog.processors.CallsiteParameter.FUNC_NAME, + structlog.processors.CallsiteParameter.LINENO, + } + ), ) structlog.configure( @@ -49,7 +69,10 @@ def get_logging_formatter() -> logging.Formatter: + """Return a JSON formatter for both structlog-originated and stdlib log records.""" return structlog.stdlib.ProcessorFormatter( + # foreign_pre_chain is run for stdlib LogRecord objects that do not go through + # structlog's processor chain. foreign_pre_chain=_FOREIGN_PRE_CHAIN, processors=[ structlog.stdlib.ProcessorFormatter.remove_processors_meta, From caa20a11c7f25cc396f30707da5ff8de519036dc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 24 Jun 2026 15:23:56 -0400 Subject: [PATCH 13/35] docs(logging): Expand docstrings with parameter and return value descriptions --- .../clp-py-utils/clp_py_utils/clp_logging.py | 20 ++++++++++++++++--- .../executor/compress/celery.py | 7 ++++++- .../executor/query/celery.py | 7 ++++++- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_logging.py b/components/clp-py-utils/clp_py_utils/clp_logging.py index 88d5b0a145..828172e4f8 100644 --- a/components/clp-py-utils/clp_py_utils/clp_logging.py +++ b/components/clp-py-utils/clp_py_utils/clp_logging.py @@ -69,7 +69,7 @@ def get_logging_formatter() -> logging.Formatter: - """Return a JSON formatter for both structlog-originated and stdlib log records.""" + """:return: A JSON log formatter configured with CLP's structlog processors.""" return structlog.stdlib.ProcessorFormatter( # foreign_pre_chain is run for stdlib LogRecord objects that do not go through # structlog's processor chain. @@ -82,14 +82,21 @@ def get_logging_formatter() -> logging.Formatter: def set_json_formatter_on_handlers(logger: logging.Logger) -> None: - """Set CLP's JSON log formatter on all handlers currently attached to *logger*.""" + """ + Set CLP's JSON log formatter on all handlers currently attached to the logger. + + :param logger: Logger whose handlers should use CLP's JSON formatter. + """ formatter = get_logging_formatter() for handler in logger.handlers: handler.setFormatter(formatter) def get_logger(name: str) -> logging.Logger: - """Return a logger configured with CLP's console formatter.""" + """ + :param name: Name of the logger to create or retrieve. + :return: A logger configured with CLP's JSON console formatter. + """ logger = logging.getLogger(name) # Setup console logging logging_console_handler = logging.StreamHandler() @@ -101,6 +108,13 @@ def get_logger(name: str) -> logging.Logger: def set_logging_level(logger: logging.Logger, level: str | None) -> None: + """ + Set a logger to the requested logging level, defaulting to INFO when the level is unset or + invalid. + + :param logger: Logger whose level should be updated. + :param level: Requested logging level, or `None` to use INFO. + """ if level is None: logger.setLevel(logging.INFO) return diff --git a/components/job-orchestration/job_orchestration/executor/compress/celery.py b/components/job-orchestration/job_orchestration/executor/compress/celery.py index 71eb296319..55bc132b96 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/celery.py +++ b/components/job-orchestration/job_orchestration/executor/compress/celery.py @@ -12,7 +12,12 @@ @signals.after_setup_logger.connect @signals.after_setup_task_logger.connect def setup_json_logging(logger: logging.Logger | None = None, **_: object) -> None: - """Use CLP's JSON formatter for loggers configured by Celery.""" + """ + Use CLP's JSON formatter for loggers configured by Celery. + + :param logger: Logger configured by Celery, if one was provided by the signal. + :param _: Additional Celery signal keyword arguments. Unused. + """ if logger is not None: set_json_formatter_on_handlers(logger) diff --git a/components/job-orchestration/job_orchestration/executor/query/celery.py b/components/job-orchestration/job_orchestration/executor/query/celery.py index 4e8d79aa88..c981b8ac8d 100644 --- a/components/job-orchestration/job_orchestration/executor/query/celery.py +++ b/components/job-orchestration/job_orchestration/executor/query/celery.py @@ -12,7 +12,12 @@ @signals.after_setup_logger.connect @signals.after_setup_task_logger.connect def setup_json_logging(logger: logging.Logger | None = None, **_: object) -> None: - """Use CLP's JSON formatter for loggers configured by Celery.""" + """ + Use CLP's JSON formatter for loggers configured by Celery. + + :param logger: Logger configured by Celery, if one was provided by the signal. + :param _: Additional Celery signal keyword arguments. Unused. + """ if logger is not None: set_json_formatter_on_handlers(logger) From 1d2d21a723cebafd4a5ea43327afd09affd4f1a0 Mon Sep 17 00:00:00 2001 From: Rishikesh Devsot Date: Fri, 26 Jun 2026 11:35:12 -0400 Subject: [PATCH 14/35] clean up doc, future support for logging python services --- .../clp-py-utils/clp_py_utils/clp_logging.py | 37 +++-- docs/src/dev-docs/logging.md | 146 +++++++++++------- 2 files changed, 116 insertions(+), 67 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_logging.py b/components/clp-py-utils/clp_py_utils/clp_logging.py index 828172e4f8..a0ada2cf07 100644 --- a/components/clp-py-utils/clp_py_utils/clp_logging.py +++ b/components/clp-py-utils/clp_py_utils/clp_logging.py @@ -6,7 +6,7 @@ from typing import get_args, Literal import structlog -from structlog.typing import Processor +from structlog.typing import FilteringBoundLogger, Processor LoggingLevel = Literal[ "INFO", @@ -58,14 +58,33 @@ ), ) -structlog.configure( - processors=[ - *_STRUCTLOG_PROCESSORS, - structlog.stdlib.ProcessorFormatter.wrap_for_formatter, - ], - logger_factory=structlog.stdlib.LoggerFactory(), - cache_logger_on_first_use=True, -) +def configure_structlog() -> None: + """Configure structlog with CLP's default processor chain if it is not already configured.""" + if not structlog.is_configured(): + structlog.configure( + processors=[ + *_STRUCTLOG_PROCESSORS, + structlog.stdlib.ProcessorFormatter.wrap_for_formatter, + ], + logger_factory=structlog.stdlib.LoggerFactory(), + cache_logger_on_first_use=True, + ) + + +def get_structlog_logger(name: str) -> FilteringBoundLogger: + """ + Configure CLP's structured logging defaults and return a structlog logger for a component. + + CLP uses structlog's stdlib integration, so this first configures the stdlib logger's + handlers with structlog's `ProcessFormatter`. Use the returned structlog logger for log calls. + + :param name: Name of the logger to create or retrieve. + :return: A structlog logger configured to emit through CLP's JSON formatter. + """ + configure_structlog() + stdlib_logger = get_logger(name) + configure_logging(stdlib_logger, name) + return structlog.get_logger(name) def get_logging_formatter() -> logging.Formatter: diff --git a/docs/src/dev-docs/logging.md b/docs/src/dev-docs/logging.md index 3d44d3b30f..4a5086316d 100644 --- a/docs/src/dev-docs/logging.md +++ b/docs/src/dev-docs/logging.md @@ -1,14 +1,47 @@ -# Logging +# CLP Package Logging The CLP package contains multiple runtime stacks, and each stack owns its logging setup. This page summarizes the current behavior and the controls developers/operators should use. -## Component summary -This section explains the logging setup of the different components in CLP package. -| Component family | Examples | Logger | Format | Level control | +## Developer guidance +This section gives guidance on how to set up and use loggers when writing a new component. + +* New Python orchestration services should use: + ```python + from clp_py_utils.clp_logging import get_structlog_logger + + log = get_structlog_logger("service_name") + log.info("hello, %s!", "world") + ``` +* New Rust HTTP services should initialize `tracing` at process startup with + [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging] and keep the returned guard alive + for the lifetime of the process: + ```rust + let _guard = clp_rust_utils::logging::set_up_logging("service_name.log"); + tracing::info!("Server started at {addr}"); + ``` +* WebUI server code should log through Fastify's Pino logger. Use `request.log` for request-scoped + logs and `app.log` for startup, shutdown, and application-level logs: + ```typescript + request.log.info({searchJobId}, "Search submitted"); + request.log.error(err, "Failed to submit search"); + ``` +* WebUI client `console.*` calls should stay limited to browser diagnostics. Do not rely on browser + console output for service logs, audit events, or telemetry that operators need to collect. +* Native core binaries should continue using `spdlog` and their existing entry-point logger setup. + Prefer `spdlog` from core C++ code rather than introducing another logging stack unless a separate + migration moves that binary family to structured logging. + +:::{note} +Prefer UTC service timestamps. Convert to local time in log viewers or aggregation systems. +::: + +## Component-specific logging +This section explains the logging setup for CLP package components. +| Component family | Components | Logger | Format | Level control | |-------------------------------|---------------------------------------------------------------------|-----------------------------------------------------------|----------------------------------|----------------------------| -| Python orchestration services | Schedulers, workers, reducer wrapper, garbage collector, MCP server | [`clp_py_utils.clp_logging`][clp-py-logging] | JSON | `CLP_LOGGING_LEVEL` | -| Rust HTTP services | API server, log ingestor | [`clp_rust_utils::logging`][clp-rust-logging] / `tracing` | JSON | `RUST_LOG` | +| Python orchestration services | `compression_scheduler`, `query_scheduler`, `compression_worker`, `query_worker`, `reducer`, `garbage_collector`, `mcp_server` | [`structlog` with stdlib logging compatibility][clp-py-logging] | JSON | `CLP_LOGGING_LEVEL` | +| Rust HTTP services | `api_server`, `log_ingestor` | [`clp_rust_utils::logging`][clp-rust-logging] / `tracing` | JSON | `RUST_LOG` | | WebUI server | Fastify server | Fastify/Pino | JSON in prod; pretty text in dev | `LOG_LEVEL` | | WebUI client | Browser app | Browser console | Browser console output | Browser/devtools dependent | | Native core binaries | `clp`, `clp-s`, `glt`, native `reducer_server` | `spdlog` | Text | Binary-specific | @@ -17,62 +50,38 @@ This section explains the logging setup of the different components in CLP packa There is no single project-wide JSON schema. Python, Rust, and Pino logs are all line-delimited JSON in packaged non-interactive service runtimes, but each stack uses its own field names. -## Development guidance - -* New Python orchestration services should use - [`structlog.get_logger`][clp-py-logging] and - [`clp_py_utils.clp_logging.configure_logging`][clp-py-logging]. -* New Rust services should use [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging]. -* WebUI server code should log through Fastify's logger (`request.log` or `app.log`). -* WebUI client `console.*` calls should stay limited to browser diagnostics. -* Native core binaries should keep their existing `spdlog` setup unless a separate change migrates - them to structured logging. - -## Timestamp convention - -Structured service logs should use UTC or another unambiguous absolute timestamp format. Python -service logs use UTC ISO-8601 timestamps, Rust service logs use `tracing_subscriber` timestamps, and -WebUI server logs use Pino epoch-millisecond timestamps. Prefer converting to local time in the log -viewer or aggregator rather than emitting ambiguous local-time strings from services. - ## Component details -The following sections describe the behavior and controls for each logging stack. +The following sections describe the behavior and controls for each component family. ### Python orchestration services These services use [`clp_py_utils.clp_logging`][clp-py-logging] and emit one JSON object per log -record: - -* `compression_scheduler` -* `query_scheduler` -* `compression_worker` -* `query_worker` -* `reducer` -* `garbage_collector` -* `mcp_server` - -Python service records include: - -| Field | Description | -|--------------|----------------------------------------------------------| -| `timestamp` | ISO-8601 UTC timestamp. | -| `event` | Formatted log message. | -| `logger` | Logger name. | -| `level` | Lowercase log level. | -| `exception` | Present when `exc_info` is logged. | -| `stack` | Present when `stack_info` is logged. | -| Extra fields | Fields passed through stdlib logging's `extra` argument. | +record. Records include: + +| Field | Description | +|-------------------|----------------------------------------------------------------------| +| `timestamp` | ISO-8601 UTC timestamp. | +| `event` | Formatted log message. | +| `logger` | Logger name. | +| `level` | Log level, emitted as values such as `info`, `warning`, and `error`. | +| `filename` | Source filename for the log call. | +| `func_name` | Source function name for the log call. | +| `lineno` | Source line number for the log call. | +| `exception` | Present when `exc_info` is logged. | +| `stack` | Present when `stack_info` is logged. | +| Context variables | Fields bound through structlog context variables. | +| Extra fields | Fields passed through stdlib logging's `extra` argument. | Example: ```json -{"timestamp":"2026-06-22T17:03:21.123456Z","event":"Compression job 1 submitted.","logger":"compression_scheduler","level":"info"} +{"timestamp":"2026-06-22T17:03:21.123456Z","event":"Compression job 1 submitted.","logger":"compression_scheduler","level":"info","filename":"compression_scheduler.py","func_name":"main","lineno":616} ``` -Controls: +Configuration: * `CLP_LOGGING_LEVEL` supports `DEBUG`, `INFO`, `WARN`, `WARNING`, `ERROR`, and `CRITICAL`. Missing or invalid values default to `INFO`. @@ -81,22 +90,35 @@ Controls: ### Rust HTTP services -`api_server` and `log_ingestor` use +These services use [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging], which configures -`tracing_subscriber` JSON output. +`tracing_subscriber` JSON output that includes: + +| Field | Description | +|---------------|------------------------------------------------------------| +| `timestamp` | Timestamp emitted by `tracing_subscriber`. | +| `level` | Log level, emitted as values such as `INFO`, `WARN`, and `ERROR`. | +| `fields` | Structured `tracing` fields, including the log message. | +| `filename` | Source filename for the log call. | +| `line_number` | Source line number for the log call. | -Rust service records include `timestamp`, `level`, `fields`, `filename`, and `line_number`. -Structured `tracing` fields are nested under `fields`. +Example: -Controls: + +```json +{"timestamp":"2026-06-26T13:06:48.621307Z","level":"INFO","fields":{"message":"Server started at 0.0.0.0:3001"},"filename":"components/api-server/src/bin/api_server.rs","line_number":81} +``` + + +Configuration: * `RUST_LOG` uses `tracing_subscriber::EnvFilter` syntax. +* In the package manifests, `log_ingestor` exposes a deployment setting through + `CLP_LOG_INGESTOR_LOGGING_LEVEL` in Docker Compose and + `clpConfig.log_ingestor.logging_level` in Helm. `api_server` currently runs with + `RUST_LOG=INFO`. * `CLP_LOGS_DIR`, when set, adds an hourly non-blocking rolling file appender. -In the package manifests, `log_ingestor` exposes a deployment setting through -`CLP_LOG_INGESTOR_LOGGING_LEVEL` in Docker Compose and `clpConfig.log_ingestor.logging_level` in -Helm. `api_server` currently runs with `RUST_LOG=INFO`. - ### WebUI The WebUI server uses Fastify's Pino logger: @@ -108,6 +130,14 @@ The WebUI server uses Fastify's Pino logger: The WebUI client logs to the browser console. Treat client `console.*` output as browser diagnostics, not service logs. +Example: + + +```json +{"level":30,"time":1782480774533,"pid":1,"hostname":"webui","reqId":"req-1h","res":{"statusCode":200},"responseTime":1.0457000732421875,"msg":"request completed"} +``` + + ### Native core binaries and package tools Native core binaries use `spdlog` text output. Package controller commands and one-shot setup scripts @@ -120,7 +150,7 @@ JSON contracts. * Docker Compose service stdout is available through `docker compose logs`. * Compose services that set `CLP_LOGS_DIR` also write under the mounted CLP log directory, which defaults to `./var/log` through `CLP_LOGS_DIR_HOST`. -* Helm deployments should primarily use pod stdout through `kubectl logs` or the cluster log +* Helm deployments should use pod stdout through `kubectl logs` or the cluster log collector. File logging is template-specific. [clp-py-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-py-utils/clp_py_utils/clp_logging.py From 021334dc6b2b96bab312671b3c80c615895977d9 Mon Sep 17 00:00:00 2001 From: Rishikesh Devsot Date: Fri, 26 Jun 2026 12:32:16 -0400 Subject: [PATCH 15/35] lint fix --- components/clp-py-utils/clp_py_utils/clp_logging.py | 1 + 1 file changed, 1 insertion(+) diff --git a/components/clp-py-utils/clp_py_utils/clp_logging.py b/components/clp-py-utils/clp_py_utils/clp_logging.py index a0ada2cf07..aa243d2ece 100644 --- a/components/clp-py-utils/clp_py_utils/clp_logging.py +++ b/components/clp-py-utils/clp_py_utils/clp_logging.py @@ -58,6 +58,7 @@ ), ) + def configure_structlog() -> None: """Configure structlog with CLP's default processor chain if it is not already configured.""" if not structlog.is_configured(): From 911bda35d861d93d4feb656f3e5aa057aa166bb8 Mon Sep 17 00:00:00 2001 From: Rishikesh Devsot Date: Fri, 26 Jun 2026 12:34:36 -0400 Subject: [PATCH 16/35] linting fixes and comment fixes --- components/clp-py-utils/clp_py_utils/clp_logging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_logging.py b/components/clp-py-utils/clp_py_utils/clp_logging.py index aa243d2ece..05555cc058 100644 --- a/components/clp-py-utils/clp_py_utils/clp_logging.py +++ b/components/clp-py-utils/clp_py_utils/clp_logging.py @@ -18,7 +18,7 @@ ] # Processor chain for events emitted through structlog loggers before they reach -# ``ProcessorFormatter``. +# `ProcessorFormatter`. _STRUCTLOG_PROCESSORS: tuple[Processor, ...] = ( structlog.stdlib.filter_by_level, structlog.contextvars.merge_contextvars, @@ -38,7 +38,7 @@ ), ) -# Processor chain for stdlib ``LogRecord`` objects that did not originate from structlog. +# Processor chain for stdlib `LogRecord` objects that did not originate from structlog. _FOREIGN_PRE_CHAIN: tuple[Processor, ...] = ( structlog.contextvars.merge_contextvars, structlog.stdlib.add_logger_name, From 5a1989019cedf51c7a476cf0ddd10a749e729b4c Mon Sep 17 00:00:00 2001 From: Rishikesh Devsot Date: Fri, 26 Jun 2026 15:50:28 -0400 Subject: [PATCH 17/35] fix doc to be consistent --- docs/src/dev-docs/logging.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/dev-docs/logging.md b/docs/src/dev-docs/logging.md index 4a5086316d..15d1205c6b 100644 --- a/docs/src/dev-docs/logging.md +++ b/docs/src/dev-docs/logging.md @@ -1,6 +1,6 @@ # CLP Package Logging -The CLP package contains multiple runtime stacks, and each stack owns its logging setup. This page +The CLP package contains multiple components, and each component owns its logging setup. This page summarizes the current behavior and the controls developers/operators should use. ## Developer guidance @@ -48,7 +48,7 @@ This section explains the logging setup for CLP package components. | Package/setup tools | Package controller, DB initialization scripts | Python stdlib logging | Text | Script-specific | There is no single project-wide JSON schema. Python, Rust, and Pino logs are all line-delimited JSON -in packaged non-interactive service runtimes, but each stack uses its own field names. +in packaged non-interactive service runtimes, but each component family uses its own field names. ## Component details From 9134e5d09f3ba0fd5019657bec9061c5da8e2236 Mon Sep 17 00:00:00 2001 From: Rishikesh Devsot Date: Fri, 26 Jun 2026 15:59:09 -0400 Subject: [PATCH 18/35] lint fixes --- .../job_orchestration/executor/compress/celery.py | 1 + 1 file changed, 1 insertion(+) diff --git a/components/job-orchestration/job_orchestration/executor/compress/celery.py b/components/job-orchestration/job_orchestration/executor/compress/celery.py index 231370a923..aad69ce77e 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/celery.py +++ b/components/job-orchestration/job_orchestration/executor/compress/celery.py @@ -22,6 +22,7 @@ def setup_json_logging(logger: logging.Logger | None = None, **_: object) -> Non if logger is not None: set_json_formatter_on_handlers(logger) + @signals.worker_process_init.connect def setup_telemetry(**kwargs) -> None: init_telemetry() From 29568099540566bbab7d7671b1eb62e8e40f842d Mon Sep 17 00:00:00 2001 From: rishikeshdevsot <33337841+rishikeshdevsot@users.noreply.github.com> Date: Mon, 29 Jun 2026 10:04:30 -0400 Subject: [PATCH 19/35] Apply suggestions from code review Co-authored-by: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> --- docs/src/dev-docs/logging.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/src/dev-docs/logging.md b/docs/src/dev-docs/logging.md index 15d1205c6b..ee7c81687d 100644 --- a/docs/src/dev-docs/logging.md +++ b/docs/src/dev-docs/logging.md @@ -20,7 +20,8 @@ This section gives guidance on how to set up and use loggers when writing a new let _guard = clp_rust_utils::logging::set_up_logging("service_name.log"); tracing::info!("Server started at {addr}"); ``` -* WebUI server code should log through Fastify's Pino logger. Use `request.log` for request-scoped + * Prefer structured logging over formatting values directly into the message field, for example: + logs and `app.log` for startup, shutdown, and application-level logs: ```typescript request.log.info({searchJobId}, "Search submitted"); @@ -57,7 +58,7 @@ The following sections describe the behavior and controls for each component fam ### Python orchestration services These services use [`clp_py_utils.clp_logging`][clp-py-logging] and emit one JSON object per log -record. Records include: +record. Each record includes the following fields: | Field | Description | |-------------------|----------------------------------------------------------------------| @@ -90,9 +91,9 @@ Configuration: ### Rust HTTP services -These services use -[`clp_rust_utils::logging::set_up_logging`][clp-rust-logging], which configures -`tracing_subscriber` JSON output that includes: +These services use [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging], which configures +`tracing_subscriber` to emit one JSON object per log record. Each record includes the following +fields: | Field | Description | |---------------|------------------------------------------------------------| @@ -106,13 +107,15 @@ Example: ```json -{"timestamp":"2026-06-26T13:06:48.621307Z","level":"INFO","fields":{"message":"Server started at 0.0.0.0:3001"},"filename":"components/api-server/src/bin/api_server.rs","line_number":81} +{"timestamp":"2026-06-26T13:06:48.621307","level":"INFO","fields":{"message":"Spawned SQS listener task.","job_id":"3","task_id":"0"},"filename":"components/log-ingestor/src/ingestion_job/sqs_listener.rs","line_number":320} ``` Configuration: -* `RUST_LOG` uses `tracing_subscriber::EnvFilter` syntax. +* Configure log filtering with [tracing_subscriber::EnvFilter](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html). + Filter directives are read from the `RUST_LOG` environment variable to determine which spans and + events are enabled. * In the package manifests, `log_ingestor` exposes a deployment setting through `CLP_LOG_INGESTOR_LOGGING_LEVEL` in Docker Compose and `clpConfig.log_ingestor.logging_level` in Helm. `api_server` currently runs with From 6924a1dfc37d6059fab8dae2b6b23f2e82f47ba4 Mon Sep 17 00:00:00 2001 From: rishikeshdevsot <33337841+rishikeshdevsot@users.noreply.github.com> Date: Mon, 29 Jun 2026 10:28:12 -0400 Subject: [PATCH 20/35] Apply suggestions from code review Co-authored-by: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> --- docs/src/dev-docs/logging.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/src/dev-docs/logging.md b/docs/src/dev-docs/logging.md index ee7c81687d..89ffad7705 100644 --- a/docs/src/dev-docs/logging.md +++ b/docs/src/dev-docs/logging.md @@ -116,10 +116,13 @@ Configuration: * Configure log filtering with [tracing_subscriber::EnvFilter](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html). Filter directives are read from the `RUST_LOG` environment variable to determine which spans and events are enabled. -* In the package manifests, `log_ingestor` exposes a deployment setting through - `CLP_LOG_INGESTOR_LOGGING_LEVEL` in Docker Compose and - `clpConfig.log_ingestor.logging_level` in Helm. `api_server` currently runs with - `RUST_LOG=INFO`. +* In the package manifests, `log_ingestor` exposes its logging level as a deployment setting: + * `CLP_LOG_INGESTOR_LOGGING_LEVEL` in Docker Compose. + * `clpConfig.log_ingestor.logging_level` in Helm. + + The value is passed to `RUST_LOG` to configure the log filtering. +* `api_server` currently runs with `RUST_LOG=INFO`, but does not expose the logging level as a + configurable deployment setting. * `CLP_LOGS_DIR`, when set, adds an hourly non-blocking rolling file appender. ### WebUI From 8d05764a3cef5c3e7553d408f531038d87ed26ad Mon Sep 17 00:00:00 2001 From: Rishikesh Devsot Date: Mon, 29 Jun 2026 10:33:49 -0400 Subject: [PATCH 21/35] added fixes from review --- docs/src/dev-docs/logging.md | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/docs/src/dev-docs/logging.md b/docs/src/dev-docs/logging.md index ee7c81687d..2959b33dd2 100644 --- a/docs/src/dev-docs/logging.md +++ b/docs/src/dev-docs/logging.md @@ -4,34 +4,46 @@ The CLP package contains multiple components, and each component owns its loggin summarizes the current behavior and the controls developers/operators should use. ## Developer guidance + This section gives guidance on how to set up and use loggers when writing a new component. -* New Python orchestration services should use: +:::{note} +When modifying an existing service, follow its current logging setup. +::: + +### Python +New Python orchestration services should use: ```python from clp_py_utils.clp_logging import get_structlog_logger log = get_structlog_logger("service_name") log.info("hello, %s!", "world") ``` -* New Rust HTTP services should initialize `tracing` at process startup with + +### Rust +New Rust HTTP services should initialize `tracing` at process startup with [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging] and keep the returned guard alive for the lifetime of the process: ```rust let _guard = clp_rust_utils::logging::set_up_logging("service_name.log"); tracing::info!("Server started at {addr}"); ``` - * Prefer structured logging over formatting values directly into the message field, for example: - - logs and `app.log` for startup, shutdown, and application-level logs: +Prefer structured logging over formatting values directly into the message field. + +### WebUI +WebUI server code should use Fastify's Pino logger. Use `request.log` for request-scoped +logs and `app.log` for startup, shutdown, and application-level logs: ```typescript request.log.info({searchJobId}, "Search submitted"); request.log.error(err, "Failed to submit search"); ``` -* WebUI client `console.*` calls should stay limited to browser diagnostics. Do not rely on browser - console output for service logs, audit events, or telemetry that operators need to collect. -* Native core binaries should continue using `spdlog` and their existing entry-point logger setup. - Prefer `spdlog` from core C++ code rather than introducing another logging stack unless a separate - migration moves that binary family to structured logging. + +WebUI client `console.*` calls should stay limited to browser diagnostics. Do not rely on browser +console output for service logs, audit events, or telemetry that operators need to collect. + +### Core +Native core binaries should continue using `spdlog` and their existing entry-point logger setup. +Prefer `spdlog` from core C++ code rather than introducing another logging stack. :::{note} Prefer UTC service timestamps. Convert to local time in log viewers or aggregation systems. @@ -113,7 +125,7 @@ Example: Configuration: -* Configure log filtering with [tracing_subscriber::EnvFilter](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html). +* Configure log filtering with [tracing_subscriber::EnvFilter][EnvFilter]. Filter directives are read from the `RUST_LOG` environment variable to determine which spans and events are enabled. * In the package manifests, `log_ingestor` exposes a deployment setting through @@ -156,5 +168,6 @@ JSON contracts. * Helm deployments should use pod stdout through `kubectl logs` or the cluster log collector. File logging is template-specific. +[EnvFilter]: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html [clp-py-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-py-utils/clp_py_utils/clp_logging.py [clp-rust-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-rust-utils/src/logging.rs From d3eb1d9e86d25179598fdbbec03c4779351569ce Mon Sep 17 00:00:00 2001 From: Rishikesh Devsot Date: Mon, 29 Jun 2026 11:31:34 -0400 Subject: [PATCH 22/35] ran doc linter --- docs/src/dev-docs/logging.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/src/dev-docs/logging.md b/docs/src/dev-docs/logging.md index d1351ea67d..b550660998 100644 --- a/docs/src/dev-docs/logging.md +++ b/docs/src/dev-docs/logging.md @@ -12,7 +12,9 @@ When modifying an existing service, follow its current logging setup. ::: ### Python + New Python orchestration services should use: + ```python from clp_py_utils.clp_logging import get_structlog_logger @@ -21,18 +23,23 @@ New Python orchestration services should use: ``` ### Rust + New Rust HTTP services should initialize `tracing` at process startup with [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging] and keep the returned guard alive for the lifetime of the process: + ```rust let _guard = clp_rust_utils::logging::set_up_logging("service_name.log"); tracing::info!("Server started at {addr}"); ``` + Prefer structured logging over formatting values directly into the message field. ### WebUI + WebUI server code should use Fastify's Pino logger. Use `request.log` for request-scoped logs and `app.log` for startup, shutdown, and application-level logs: + ```typescript request.log.info({searchJobId}, "Search submitted"); request.log.error(err, "Failed to submit search"); @@ -42,6 +49,7 @@ WebUI client `console.*` calls should stay limited to browser diagnostics. Do no console output for service logs, audit events, or telemetry that operators need to collect. ### Core + Native core binaries should continue using `spdlog` and their existing entry-point logger setup. Prefer `spdlog` from core C++ code rather than introducing another logging stack. @@ -50,7 +58,9 @@ Prefer UTC service timestamps. Convert to local time in log viewers or aggregati ::: ## Component-specific logging + This section explains the logging setup for CLP package components. + | Component family | Components | Logger | Format | Level control | |-------------------------------|---------------------------------------------------------------------|-----------------------------------------------------------|----------------------------------|----------------------------| | Python orchestration services | `compression_scheduler`, `query_scheduler`, `compression_worker`, `query_worker`, `reducer`, `garbage_collector`, `mcp_server` | [`structlog` with stdlib logging compatibility][clp-py-logging] | JSON | `CLP_LOGGING_LEVEL` | @@ -162,7 +172,6 @@ Native core binaries use `spdlog` text output. Package controller commands and o also use human-readable stdlib logging. These tools are not covered by the Python/Rust/WebUI service JSON contracts. - ## Deployment notes * Docker Compose service stdout is available through `docker compose logs`. From 665a313e65792c97cd1da0ad2de660ae55d46a7f Mon Sep 17 00:00:00 2001 From: Rishikesh Devsot Date: Mon, 29 Jun 2026 11:56:37 -0400 Subject: [PATCH 23/35] nit --- docs/src/dev-docs/logging.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/dev-docs/logging.md b/docs/src/dev-docs/logging.md index b550660998..ccbcdcf8c0 100644 --- a/docs/src/dev-docs/logging.md +++ b/docs/src/dev-docs/logging.md @@ -8,7 +8,7 @@ summarizes the current behavior and the controls developers/operators should use This section gives guidance on how to set up and use loggers when writing a new component. :::{note} -When modifying an existing service, follow its current logging setup. +When modifying an existing component, follow its current logging setup. ::: ### Python From 74b367b19a144dfac2b18cf917015ca2703ea4c5 Mon Sep 17 00:00:00 2001 From: Rishikesh Devsot Date: Mon, 29 Jun 2026 12:04:48 -0400 Subject: [PATCH 24/35] cleanup logging.md --- docs/src/dev-docs/logging.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/src/dev-docs/logging.md b/docs/src/dev-docs/logging.md index ccbcdcf8c0..881c2d2357 100644 --- a/docs/src/dev-docs/logging.md +++ b/docs/src/dev-docs/logging.md @@ -51,7 +51,6 @@ console output for service logs, audit events, or telemetry that operators need ### Core Native core binaries should continue using `spdlog` and their existing entry-point logger setup. -Prefer `spdlog` from core C++ code rather than introducing another logging stack. :::{note} Prefer UTC service timestamps. Convert to local time in log viewers or aggregation systems. @@ -155,9 +154,6 @@ The WebUI server uses Fastify's Pino logger: * Interactive terminals use `pino-pretty` for developer-readable output. * `LOG_LEVEL` controls the server log level and defaults to `info`. -The WebUI client logs to the browser console. Treat client `console.*` output as browser diagnostics, -not service logs. - Example: @@ -166,7 +162,10 @@ Example: ``` -### Native core binaries and package tools +The WebUI client logs to the browser console. Treat client `console.*` output as browser diagnostics, +not service logs. + +### Core and package tools Native core binaries use `spdlog` text output. Package controller commands and one-shot setup scripts also use human-readable stdlib logging. These tools are not covered by the Python/Rust/WebUI service From d630c5e48d2cecabc5dc7d9eb04917ae3ba8e61e Mon Sep 17 00:00:00 2001 From: Rishikesh Devsot Date: Mon, 29 Jun 2026 13:44:35 -0400 Subject: [PATCH 25/35] restructured the logging docs --- docs/src/dev-docs/logging-developer-guide.md | 65 +++++++ docs/src/dev-docs/logging-operator-guide.md | 135 ++++++++++++++ docs/src/dev-docs/logging.md | 185 +------------------ 3 files changed, 208 insertions(+), 177 deletions(-) create mode 100644 docs/src/dev-docs/logging-developer-guide.md create mode 100644 docs/src/dev-docs/logging-operator-guide.md diff --git a/docs/src/dev-docs/logging-developer-guide.md b/docs/src/dev-docs/logging-developer-guide.md new file mode 100644 index 0000000000..a24d303ae9 --- /dev/null +++ b/docs/src/dev-docs/logging-developer-guide.md @@ -0,0 +1,65 @@ +# Developer Guide: Writing Logs + +This guide provides the language-specific setup instructions and standards required +to initialize loggers and emit diagnostic logs when developing or modifying CLP components. + +:::{note} +Always prefer UTC service timestamps. Timezone conversions should be handled +downstream in log viewers or aggregation systems. +::: + +## Python + +New Python orchestration services should use `structlog` for structured JSON logging +and bind context variables where appropriate. + + ```python + from clp_py_utils.clp_logging import get_structlog_logger + + log = get_structlog_logger("service_name") + log.info("hello, %s!", "world") + ``` + +:::{note} +Existing Python services use stdlib loggers whose handlers are configured with structlog's +`ProcessorFormatter`. Follow the established convention when modifying those services. +::: + +## Rust + +New Rust HTTP services should initialize `tracing` at process startup using + [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging] and keep the returned guard alive + for the lifetime of the process: + + ```rust + let _guard = clp_rust_utils::logging::set_up_logging("service_name.log"); + + // Choose structured logging over formatting values directly into the message field. + tracing::info!(server_address = %addr, "Server started."); + ``` + +## WebUI + +WebUI server code should use Fastify's Pino logger. Use `request.log` for request-scoped +logs and `app.log` for startup, shutdown, and application-level logs: + + ```typescript + // Request-scoped + request.log.info({searchJobId}, "Search submitted"); + request.log.error(err, "Failed to submit search"); + + // Application-scoped + app.log.info("WebUI server listening on port 3000"); + ``` + +WebUI client code may use `console.*` for browser diagnostics. Logs that operators need to +collect, search, or alert on should be emitted by the WebUI server. + +## Core & Setup Tool + +* Native core binaries (`clp`, `clp-s`, `glt`, native `reducer_server`) should continue using spdlog +and their existing entry-point logger setup. +* Package/setup tools (DB initialization scripts, package controllers) should use standard Python +logger. + +[clp-rust-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-rust-utils/src/logging.rs diff --git a/docs/src/dev-docs/logging-operator-guide.md b/docs/src/dev-docs/logging-operator-guide.md new file mode 100644 index 0000000000..301ca18744 --- /dev/null +++ b/docs/src/dev-docs/logging-operator-guide.md @@ -0,0 +1,135 @@ +# Operator Guide: Consuming CLP Logs + +This guide details how to configure internal log levels, capture service logs, +and understand the component-specific log structures emitted by a running CLP deployment. + +| Component family | Components | Logger | Format | Level control | +|-------------------------------|---------------------------------------------------------------------|-----------------------------------------------------------|----------------------------------|----------------------------| +| Python orchestration services | `compression_scheduler`, `query_scheduler`, `compression_worker`, `query_worker`, `reducer`, `garbage_collector`, `mcp_server` | [`structlog` with stdlib logging compatibility][clp-py-logging] | JSON | `CLP_LOGGING_LEVEL` | +| Rust HTTP services | `api_server`, `log_ingestor` | [`clp_rust_utils::logging`][clp-rust-logging] / `tracing` | JSON | `RUST_LOG` | +| WebUI server | Fastify server | Fastify/Pino | JSON in prod; pretty text in dev | `LOG_LEVEL` | +| WebUI client | Browser app | Browser console | Browser console output | Browser/devtools dependent | +| Native core binaries | `clp`, `clp-s`, `glt`, native `reducer_server` | `spdlog` | Text | Binary-specific | +| Package/setup tools | Package controller, DB initialization scripts | Python stdlib logging | Text | Script-specific | + +:::{note} +There is no single project-wide JSON schema. Python, Rust, and Pino logs are all line-delimited JSON +in packaged non-interactive service runtimes, but each component family uses its own field names. +::: + +## Configuration + +### Log Level Configuration + +* **Python orchestration services**: `CLP_LOGGING_LEVEL` supports `DEBUG`, `INFO`, `WARN`, `WARNING`, `ERROR`, and `CRITICAL`. + Missing or invalid values default to `INFO`. + * `CLP_LOGS_DIR`, when set, adds a file handler at `/.log` in + addition to stdout. +* **Rust HTTP services**: Configure log filtering with [tracing_subscriber::EnvFilter][EnvFilter]. + Filter directives are read from the `RUST_LOG` environment variable to determine which spans and + events are enabled. + * *Note on `log_ingestor`*: Level is exposed via CLP_LOG_INGESTOR_LOGGING_LEVEL (Docker Compose) or clpConfig.log_ingestor.logging_level (Helm), which is then passed to RUST_LOG. + * *Note on `api_server`*: Currently runs hardcoded at INFO and does not expose a deployment setting. +* **WebUI**: `LOG_LEVEL` controls the Pino server log level (defaults to `info`). *Warning: Be mindful of environment variable collisions with LOG_LEVEL in shared container spaces.* + +### Deployment Notes and Log output + +* **`docker-compose`**: Service stdout is available via `docker compose logs`. If `CLP_LOGS_DIR` is set, +logs are additionally written to `/.log` (which mounts to `./var/log` +via `CLP_LOGS_DIR_HOST`). +*Note: For Rust components, this adds an hourly non-blocking rolling file appender.* +* **`helm`**: Rely on pod stdout via `kubectl logs` or a cluster log collector (e.g., Fluent Bit). +File logging is template-specific. +* **WebUI**: + * **Server**: Server: In non-interactive deployments, the Fastify server emits Pino JSON directly to `stdout`. If run in an interactive terminal (e.g., local development), it uses pino-pretty for human-readable output. + * **Client**: Browser logs (`console.*`) remain local to the user's browser devtools and are *not* captured by backend service telemetry. Treat these purely as local diagnostics. +* **Core**: Native binaries (clp, clp-s, etc.) emit standard human-readable text. Currently, Python orchestration services invoke native core binaries (clp, clp-s, etc.) as subprocesses and the core binaries' human-readable `spdlog` text is piped into the Python logger's output stream as a single multi-line JSON record. +* **Setup tools**: Package controller scripts emit standard human-readable text. They do not adhere to the JSON schemas used by the orchestration services and rely strictly on standard output streams. + +## Component-Specific Logging details + +The following sections describe the behavior for each component family. + +### Python orchestration services + +These services use [`clp_py_utils.clp_logging`][clp-py-logging] and emit one JSON object per log +record. Each record includes the following fields: + +| Field | Description | +|-------------------|----------------------------------------------------------------------| +| `timestamp` | ISO-8601 UTC timestamp. | +| `event` | Formatted log message. | +| `logger` | Logger name. | +| `level` | Log level, emitted as values such as `info`, `warning`, and `error`. | +| `filename` | Source filename for the log call. | +| `func_name` | Source function name for the log call. | +| `lineno` | Source line number for the log call. | +| `exception` | Present when `exc_info` is logged. | +| `stack` | Present when `stack_info` is logged. | +| Context variables | Fields bound through structlog context variables. | +| Extra fields | Fields passed through stdlib logging's `extra` argument. | + +Example: + + +```json +{"timestamp":"2026-06-22T17:03:21.123456Z","event":"Compression job 1 submitted.","logger":"compression_scheduler","level":"info","filename":"compression_scheduler.py","func_name":"main","lineno":616} +``` + + +### Rust HTTP services + +These services use [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging], which configures +`tracing_subscriber` to emit one JSON object per log record. Each record includes the following +fields: + +| Field | Description | +|---------------|------------------------------------------------------------| +| `timestamp` | Timestamp emitted by `tracing_subscriber`. | +| `level` | Log level, emitted as values such as `INFO`, `WARN`, and `ERROR`. | +| `fields` | Structured `tracing` fields, including the log message. | +| `filename` | Source filename for the log call. | +| `line_number` | Source line number for the log call. | + +Example: + + +```json +{"timestamp":"2026-06-26T13:06:48.621307","level":"INFO","fields":{"message":"Spawned SQS listener task.","job_id":"3","task_id":"0"},"filename":"components/log-ingestor/src/ingestion_job/sqs_listener.rs","line_number":320} +``` + + +### WebUI + +The WebUI server uses Fastify's Pino logger where each record includes the +following fields: + +| Field | Description | +|----------------|---------------------------------------------------------| +| `level` | Numeric Pino log level. | +| `time` | Unix timestamp in milliseconds. | +| `pid` | Process ID. | +| `hostname` | Hostname of the process emitting the log. | +| `reqId` | Fastify request ID, when the record is request-scoped. | +| `req`, `res` | Request and response metadata, when available. | +| `responseTime` | Request duration in milliseconds, when available. | +| `msg` | Log message. | +| Extra fields | Structured fields passed to the Pino log call. | + +Example: + + +```json +{"level":30,"time":1782480774533,"pid":1,"hostname":"webui","reqId":"req-1h","res":{"statusCode":200},"responseTime":1.0457000732421875,"msg":"request completed"} +``` + + +### Core and package tools + +Native core binaries use `spdlog` text output. Package controller commands and one-shot setup scripts +also use human-readable stdlib logging. These tools are not covered by the Python/Rust/WebUI service +JSON contracts. + +[EnvFilter]: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html +[clp-py-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-py-utils/clp_py_utils/clp_logging.py +[clp-rust-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-rust-utils/src/logging.rs diff --git a/docs/src/dev-docs/logging.md b/docs/src/dev-docs/logging.md index 881c2d2357..02d5fdc744 100644 --- a/docs/src/dev-docs/logging.md +++ b/docs/src/dev-docs/logging.md @@ -1,184 +1,15 @@ # CLP Package Logging -The CLP package contains multiple components, and each component owns its logging setup. This page -summarizes the current behavior and the controls developers/operators should use. +The CLP package utilizes a polyglot architecture, meaning each component family manages its own logging stack. There is **no single project-wide JSON schema**. -## Developer guidance +This document is divided into two sections: -This section gives guidance on how to set up and use loggers when writing a new component. +* [Developer guide](logging-developer-guide.md): How to set up and write logs when modifying CLP components. +* [Operator guide](logging-operator-guide.md): How to configure log levels, capture service logs, and understand the component-specific log structures in deployment -:::{note} -When modifying an existing component, follow its current logging setup. -::: - -### Python - -New Python orchestration services should use: - - ```python - from clp_py_utils.clp_logging import get_structlog_logger - - log = get_structlog_logger("service_name") - log.info("hello, %s!", "world") - ``` - -### Rust - -New Rust HTTP services should initialize `tracing` at process startup with - [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging] and keep the returned guard alive - for the lifetime of the process: - - ```rust - let _guard = clp_rust_utils::logging::set_up_logging("service_name.log"); - tracing::info!("Server started at {addr}"); - ``` - -Prefer structured logging over formatting values directly into the message field. - -### WebUI - -WebUI server code should use Fastify's Pino logger. Use `request.log` for request-scoped -logs and `app.log` for startup, shutdown, and application-level logs: - - ```typescript - request.log.info({searchJobId}, "Search submitted"); - request.log.error(err, "Failed to submit search"); - ``` +:::{toctree} +:hidden: -WebUI client `console.*` calls should stay limited to browser diagnostics. Do not rely on browser -console output for service logs, audit events, or telemetry that operators need to collect. - -### Core - -Native core binaries should continue using `spdlog` and their existing entry-point logger setup. - -:::{note} -Prefer UTC service timestamps. Convert to local time in log viewers or aggregation systems. +logging-developer-guide +logging-operator-guide ::: - -## Component-specific logging - -This section explains the logging setup for CLP package components. - -| Component family | Components | Logger | Format | Level control | -|-------------------------------|---------------------------------------------------------------------|-----------------------------------------------------------|----------------------------------|----------------------------| -| Python orchestration services | `compression_scheduler`, `query_scheduler`, `compression_worker`, `query_worker`, `reducer`, `garbage_collector`, `mcp_server` | [`structlog` with stdlib logging compatibility][clp-py-logging] | JSON | `CLP_LOGGING_LEVEL` | -| Rust HTTP services | `api_server`, `log_ingestor` | [`clp_rust_utils::logging`][clp-rust-logging] / `tracing` | JSON | `RUST_LOG` | -| WebUI server | Fastify server | Fastify/Pino | JSON in prod; pretty text in dev | `LOG_LEVEL` | -| WebUI client | Browser app | Browser console | Browser console output | Browser/devtools dependent | -| Native core binaries | `clp`, `clp-s`, `glt`, native `reducer_server` | `spdlog` | Text | Binary-specific | -| Package/setup tools | Package controller, DB initialization scripts | Python stdlib logging | Text | Script-specific | - -There is no single project-wide JSON schema. Python, Rust, and Pino logs are all line-delimited JSON -in packaged non-interactive service runtimes, but each component family uses its own field names. - -## Component details - -The following sections describe the behavior and controls for each component family. - -### Python orchestration services - -These services use [`clp_py_utils.clp_logging`][clp-py-logging] and emit one JSON object per log -record. Each record includes the following fields: - -| Field | Description | -|-------------------|----------------------------------------------------------------------| -| `timestamp` | ISO-8601 UTC timestamp. | -| `event` | Formatted log message. | -| `logger` | Logger name. | -| `level` | Log level, emitted as values such as `info`, `warning`, and `error`. | -| `filename` | Source filename for the log call. | -| `func_name` | Source function name for the log call. | -| `lineno` | Source line number for the log call. | -| `exception` | Present when `exc_info` is logged. | -| `stack` | Present when `stack_info` is logged. | -| Context variables | Fields bound through structlog context variables. | -| Extra fields | Fields passed through stdlib logging's `extra` argument. | - -Example: - - -```json -{"timestamp":"2026-06-22T17:03:21.123456Z","event":"Compression job 1 submitted.","logger":"compression_scheduler","level":"info","filename":"compression_scheduler.py","func_name":"main","lineno":616} -``` - - -Configuration: - -* `CLP_LOGGING_LEVEL` supports `DEBUG`, `INFO`, `WARN`, `WARNING`, `ERROR`, and `CRITICAL`. - Missing or invalid values default to `INFO`. -* `CLP_LOGS_DIR`, when set, adds a file handler at `/.log` in - addition to stdout. - -### Rust HTTP services - -These services use [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging], which configures -`tracing_subscriber` to emit one JSON object per log record. Each record includes the following -fields: - -| Field | Description | -|---------------|------------------------------------------------------------| -| `timestamp` | Timestamp emitted by `tracing_subscriber`. | -| `level` | Log level, emitted as values such as `INFO`, `WARN`, and `ERROR`. | -| `fields` | Structured `tracing` fields, including the log message. | -| `filename` | Source filename for the log call. | -| `line_number` | Source line number for the log call. | - -Example: - - -```json -{"timestamp":"2026-06-26T13:06:48.621307","level":"INFO","fields":{"message":"Spawned SQS listener task.","job_id":"3","task_id":"0"},"filename":"components/log-ingestor/src/ingestion_job/sqs_listener.rs","line_number":320} -``` - - -Configuration: - -* Configure log filtering with [tracing_subscriber::EnvFilter][EnvFilter]. - Filter directives are read from the `RUST_LOG` environment variable to determine which spans and - events are enabled. -* In the package manifests, `log_ingestor` exposes its logging level as a deployment setting: - * `CLP_LOG_INGESTOR_LOGGING_LEVEL` in Docker Compose. - * `clpConfig.log_ingestor.logging_level` in Helm. - - The value is passed to `RUST_LOG` to configure the log filtering. -* `api_server` currently runs with `RUST_LOG=INFO`, but does not expose the logging level as a - configurable deployment setting. -* `CLP_LOGS_DIR`, when set, adds an hourly non-blocking rolling file appender. - -### WebUI - -The WebUI server uses Fastify's Pino logger: - -* Non-interactive runtimes emit Pino JSON to stdout. -* Interactive terminals use `pino-pretty` for developer-readable output. -* `LOG_LEVEL` controls the server log level and defaults to `info`. - -Example: - - -```json -{"level":30,"time":1782480774533,"pid":1,"hostname":"webui","reqId":"req-1h","res":{"statusCode":200},"responseTime":1.0457000732421875,"msg":"request completed"} -``` - - -The WebUI client logs to the browser console. Treat client `console.*` output as browser diagnostics, -not service logs. - -### Core and package tools - -Native core binaries use `spdlog` text output. Package controller commands and one-shot setup scripts -also use human-readable stdlib logging. These tools are not covered by the Python/Rust/WebUI service -JSON contracts. - -## Deployment notes - -* Docker Compose service stdout is available through `docker compose logs`. - * Compose services that set `CLP_LOGS_DIR` also write under the mounted CLP log directory, which - defaults to `./var/log` through `CLP_LOGS_DIR_HOST`. -* Helm deployments should use pod stdout through `kubectl logs` or the cluster log - collector. File logging is template-specific. - -[EnvFilter]: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html -[clp-py-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-py-utils/clp_py_utils/clp_logging.py -[clp-rust-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-rust-utils/src/logging.rs From 91b152ed446456a2593341dc7f1ce86b6c872b5a Mon Sep 17 00:00:00 2001 From: Rishikesh Devsot Date: Mon, 29 Jun 2026 15:34:45 -0400 Subject: [PATCH 26/35] final developer guide --- docs/src/dev-docs/logging-developer-guide.md | 27 ++++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/src/dev-docs/logging-developer-guide.md b/docs/src/dev-docs/logging-developer-guide.md index a24d303ae9..aef462d2d7 100644 --- a/docs/src/dev-docs/logging-developer-guide.md +++ b/docs/src/dev-docs/logging-developer-guide.md @@ -1,10 +1,10 @@ -# Developer Guide: Writing Logs +# Developer Guide: Writing logs -This guide provides the language-specific setup instructions and standards required -to initialize loggers and emit diagnostic logs when developing or modifying CLP components. +This guide provides language-specific setup instructions and standards for initializing loggers and +emitting diagnostic logs when developing or modifying CLP components. :::{note} -Always prefer UTC service timestamps. Timezone conversions should be handled +Always prefer UTC service timestamps. Time zone conversions should be handled downstream in log viewers or aggregation systems. ::: @@ -52,14 +52,19 @@ logs and `app.log` for startup, shutdown, and application-level logs: app.log.info("WebUI server listening on port 3000"); ``` -WebUI client code may use `console.*` for browser diagnostics. Logs that operators need to -collect, search, or alert on should be emitted by the WebUI server. +WebUI client code should use `console.*` for browser diagnostics: -## Core & Setup Tool + ```typescript + console.error("Failed to submit query:", err); + ``` + +## Core and setup tools -* Native core binaries (`clp`, `clp-s`, `glt`, native `reducer_server`) should continue using spdlog -and their existing entry-point logger setup. -* Package/setup tools (DB initialization scripts, package controllers) should use standard Python -logger. +* Native core binaries (`clp`, `clp-s`, `glt`, native `reducer_server`) should continue using + [`spdlog`][spdlog] and the `spdlog` initialization already defined in each binary's startup code. +* Package/setup tools (DB initialization scripts, package controllers) should use the + [standard Python logger][stdlogger]. [clp-rust-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-rust-utils/src/logging.rs +[spdlog]: https://github.com/gabime/spdlog +[stdlogger]: https://docs.python.org/3/library/logging.html From 1f7e0eb5f09445a0152b6515c4c7bf7a63c5620e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 29 Jun 2026 17:36:35 -0400 Subject: [PATCH 27/35] docs(logging): Update logging guides for consistency and clarity --- docs/src/dev-docs/logging-developer-guide.md | 46 +++---- docs/src/dev-docs/logging-operator-guide.md | 133 ++++++++++++------- docs/src/dev-docs/logging.md | 11 +- 3 files changed, 113 insertions(+), 77 deletions(-) diff --git a/docs/src/dev-docs/logging-developer-guide.md b/docs/src/dev-docs/logging-developer-guide.md index a24d303ae9..b42a9602d7 100644 --- a/docs/src/dev-docs/logging-developer-guide.md +++ b/docs/src/dev-docs/logging-developer-guide.md @@ -1,4 +1,4 @@ -# Developer Guide: Writing Logs +# Developer guide: writing logs This guide provides the language-specific setup instructions and standards required to initialize loggers and emit diagnostic logs when developing or modifying CLP components. @@ -13,12 +13,12 @@ downstream in log viewers or aggregation systems. New Python orchestration services should use `structlog` for structured JSON logging and bind context variables where appropriate. - ```python - from clp_py_utils.clp_logging import get_structlog_logger +```python +from clp_py_utils.clp_logging import get_structlog_logger - log = get_structlog_logger("service_name") - log.info("hello, %s!", "world") - ``` +log = get_structlog_logger("service_name") +log.info("hello, %s!", "world") +``` :::{note} Existing Python services use stdlib loggers whose handlers are configured with structlog's @@ -28,38 +28,38 @@ Existing Python services use stdlib loggers whose handlers are configured with s ## Rust New Rust HTTP services should initialize `tracing` at process startup using - [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging] and keep the returned guard alive - for the lifetime of the process: +[`clp_rust_utils::logging::set_up_logging`][clp-rust-logging] and keep the returned guard alive +for the lifetime of the process: - ```rust - let _guard = clp_rust_utils::logging::set_up_logging("service_name.log"); - - // Choose structured logging over formatting values directly into the message field. - tracing::info!(server_address = %addr, "Server started."); - ``` +```rust +let _guard = clp_rust_utils::logging::set_up_logging("service_name.log"); + +// Choose structured logging over formatting values directly into the message field. +tracing::info!(server_address = %addr, "Server started."); +``` ## WebUI WebUI server code should use Fastify's Pino logger. Use `request.log` for request-scoped logs and `app.log` for startup, shutdown, and application-level logs: - ```typescript - // Request-scoped - request.log.info({searchJobId}, "Search submitted"); - request.log.error(err, "Failed to submit search"); +```typescript +// Request-scoped +request.log.info({searchJobId}, "Search submitted"); +request.log.error(err, "Failed to submit search"); - // Application-scoped - app.log.info("WebUI server listening on port 3000"); - ``` +// Application-scoped +app.log.info("WebUI server listening on port 3000"); +``` WebUI client code may use `console.*` for browser diagnostics. Logs that operators need to collect, search, or alert on should be emitted by the WebUI server. -## Core & Setup Tool +## Core & setup tool * Native core binaries (`clp`, `clp-s`, `glt`, native `reducer_server`) should continue using spdlog and their existing entry-point logger setup. * Package/setup tools (DB initialization scripts, package controllers) should use standard Python -logger. +logging. [clp-rust-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-rust-utils/src/logging.rs diff --git a/docs/src/dev-docs/logging-operator-guide.md b/docs/src/dev-docs/logging-operator-guide.md index 301ca18744..c70e9d0be7 100644 --- a/docs/src/dev-docs/logging-operator-guide.md +++ b/docs/src/dev-docs/logging-operator-guide.md @@ -1,16 +1,16 @@ -# Operator Guide: Consuming CLP Logs +# Operator guide: consuming CLP logs -This guide details how to configure internal log levels, capture service logs, -and understand the component-specific log structures emitted by a running CLP deployment. +This guide details how to configure internal log levels, capture service logs, and understand the +component-specific log structures emitted by a running CLP deployment. -| Component family | Components | Logger | Format | Level control | -|-------------------------------|---------------------------------------------------------------------|-----------------------------------------------------------|----------------------------------|----------------------------| -| Python orchestration services | `compression_scheduler`, `query_scheduler`, `compression_worker`, `query_worker`, `reducer`, `garbage_collector`, `mcp_server` | [`structlog` with stdlib logging compatibility][clp-py-logging] | JSON | `CLP_LOGGING_LEVEL` | -| Rust HTTP services | `api_server`, `log_ingestor` | [`clp_rust_utils::logging`][clp-rust-logging] / `tracing` | JSON | `RUST_LOG` | -| WebUI server | Fastify server | Fastify/Pino | JSON in prod; pretty text in dev | `LOG_LEVEL` | -| WebUI client | Browser app | Browser console | Browser console output | Browser/devtools dependent | -| Native core binaries | `clp`, `clp-s`, `glt`, native `reducer_server` | `spdlog` | Text | Binary-specific | -| Package/setup tools | Package controller, DB initialization scripts | Python stdlib logging | Text | Script-specific | +| Component family | Components | Logger | Format | Level control | +|-------------------------------|--------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------|----------------------------------|----------------------------| +| Python orchestration services | `compression_scheduler`, `query_scheduler`, `compression_worker`, `query_worker`, `reducer`, `garbage_collector`, `mcp_server` | [`structlog` with stdlib logging compatibility][clp-py-logging] | JSON | `CLP_LOGGING_LEVEL` | +| Rust HTTP services | `api_server`, `log_ingestor` | [`clp_rust_utils::logging`][clp-rust-logging] / `tracing` | JSON | `RUST_LOG` | +| WebUI server | Fastify server | Fastify/Pino | JSON in prod; pretty text in dev | `LOG_LEVEL` | +| WebUI client | Browser app | Browser console | Browser console output | Browser/devtools dependent | +| Native core binaries | `clp`, `clp-s`, `glt`, native `reducer_server` | `spdlog` | Text | Binary-specific | +| Package/setup tools | Package controller, DB initialization scripts | Python stdlib logging | Text | Script-specific | :::{note} There is no single project-wide JSON schema. Python, Rust, and Pino logs are all line-delimited JSON @@ -19,34 +19,46 @@ in packaged non-interactive service runtimes, but each component family uses its ## Configuration -### Log Level Configuration +### Log level configuration -* **Python orchestration services**: `CLP_LOGGING_LEVEL` supports `DEBUG`, `INFO`, `WARN`, `WARNING`, `ERROR`, and `CRITICAL`. - Missing or invalid values default to `INFO`. +* **Python orchestration services**: `CLP_LOGGING_LEVEL` supports `DEBUG`, `INFO`, `WARN`, + `WARNING`, `ERROR`, and `CRITICAL`. Missing or invalid values default to `INFO`. * `CLP_LOGS_DIR`, when set, adds a file handler at `/.log` in - addition to stdout. + addition to stdout. * **Rust HTTP services**: Configure log filtering with [tracing_subscriber::EnvFilter][EnvFilter]. Filter directives are read from the `RUST_LOG` environment variable to determine which spans and events are enabled. - * *Note on `log_ingestor`*: Level is exposed via CLP_LOG_INGESTOR_LOGGING_LEVEL (Docker Compose) or clpConfig.log_ingestor.logging_level (Helm), which is then passed to RUST_LOG. - * *Note on `api_server`*: Currently runs hardcoded at INFO and does not expose a deployment setting. -* **WebUI**: `LOG_LEVEL` controls the Pino server log level (defaults to `info`). *Warning: Be mindful of environment variable collisions with LOG_LEVEL in shared container spaces.* - -### Deployment Notes and Log output - -* **`docker-compose`**: Service stdout is available via `docker compose logs`. If `CLP_LOGS_DIR` is set, -logs are additionally written to `/.log` (which mounts to `./var/log` -via `CLP_LOGS_DIR_HOST`). -*Note: For Rust components, this adds an hourly non-blocking rolling file appender.* + * *Note on `log_ingestor`*: Level is exposed via `CLP_LOG_INGESTOR_LOGGING_LEVEL` (Docker Compose) + or `clpConfig.log_ingestor.logging_level` (Helm), which is then passed to `RUST_LOG`. + * *Note on `api_server`*: Currently runs hardcoded at INFO and does not expose a deployment + setting. +* **WebUI**: `LOG_LEVEL` controls the Pino server log level (defaults to `info`). + * *Warning*: Be mindful of environment variable collisions with LOG_LEVEL in shared container + spaces. + +### Deployment notes and log output + +* **`docker-compose`**: Service stdout is available via `docker compose logs`. If `CLP_LOGS_DIR` is + set, logs are additionally written to `/.log` (which mounts to + `./var/log` via `CLP_LOGS_DIR_HOST`). For Rust components, this adds an hourly non-blocking + rolling file appender. * **`helm`**: Rely on pod stdout via `kubectl logs` or a cluster log collector (e.g., Fluent Bit). -File logging is template-specific. + File logging is template-specific. * **WebUI**: - * **Server**: Server: In non-interactive deployments, the Fastify server emits Pino JSON directly to `stdout`. If run in an interactive terminal (e.g., local development), it uses pino-pretty for human-readable output. - * **Client**: Browser logs (`console.*`) remain local to the user's browser devtools and are *not* captured by backend service telemetry. Treat these purely as local diagnostics. -* **Core**: Native binaries (clp, clp-s, etc.) emit standard human-readable text. Currently, Python orchestration services invoke native core binaries (clp, clp-s, etc.) as subprocesses and the core binaries' human-readable `spdlog` text is piped into the Python logger's output stream as a single multi-line JSON record. -* **Setup tools**: Package controller scripts emit standard human-readable text. They do not adhere to the JSON schemas used by the orchestration services and rely strictly on standard output streams. - -## Component-Specific Logging details + * **Server**: In non-interactive deployments, the Fastify server emits Pino JSON directly to + `stdout`. If run in an interactive terminal (e.g., local development), it uses `pino-pretty` for + human-readable output. + * **Client**: Browser logs (`console.*`) remain local to the user's browser devtools and are *not* + captured by backend service telemetry. Treat these purely as local diagnostics. +* **Core**: Native binaries (clp, clp-s, etc.) emit standard human-readable text. Currently, Python + orchestration services invoke native core binaries (clp, clp-s, etc.) as subprocesses and the core + binaries' human-readable `spdlog` text is piped into the Python logger's output stream as a single + multi-line JSON record. +* **Setup tools**: Package controller scripts emit standard human-readable text. They do not adhere + to the JSON schemas used by the orchestration services and rely strictly on standard output + streams. + +## Component-specific logging details The following sections describe the behavior for each component family. @@ -71,11 +83,17 @@ record. Each record includes the following fields: Example: - ```json -{"timestamp":"2026-06-22T17:03:21.123456Z","event":"Compression job 1 submitted.","logger":"compression_scheduler","level":"info","filename":"compression_scheduler.py","func_name":"main","lineno":616} +{ + "timestamp":"2026-06-22T17:03:21.123456Z", + "event":"Compression job 1 submitted.", + "logger":"compression_scheduler", + "level":"info", + "filename":"compression_scheduler.py", + "func_name":"main", + "lineno":616 +} ``` - ### Rust HTTP services @@ -83,21 +101,29 @@ These services use [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging] `tracing_subscriber` to emit one JSON object per log record. Each record includes the following fields: -| Field | Description | -|---------------|------------------------------------------------------------| -| `timestamp` | Timestamp emitted by `tracing_subscriber`. | +| Field | Description | +|---------------|-------------------------------------------------------------------| +| `timestamp` | Timestamp emitted by `tracing_subscriber`. | | `level` | Log level, emitted as values such as `INFO`, `WARN`, and `ERROR`. | -| `fields` | Structured `tracing` fields, including the log message. | -| `filename` | Source filename for the log call. | -| `line_number` | Source line number for the log call. | +| `fields` | Structured `tracing` fields, including the log message. | +| `filename` | Source filename for the log call. | +| `line_number` | Source line number for the log call. | Example: - ```json -{"timestamp":"2026-06-26T13:06:48.621307","level":"INFO","fields":{"message":"Spawned SQS listener task.","job_id":"3","task_id":"0"},"filename":"components/log-ingestor/src/ingestion_job/sqs_listener.rs","line_number":320} +{ + "timestamp":"2026-06-26T13:06:48.621307", + "level":"INFO", + "fields":{ + "message":"Spawned SQS listener task.", + "job_id":"3", + "task_id":"0" + }, + "filename":"components/log-ingestor/src/ingestion_job/sqs_listener.rs", + "line_number":320 +} ``` - ### WebUI @@ -118,17 +144,24 @@ following fields: Example: - ```json -{"level":30,"time":1782480774533,"pid":1,"hostname":"webui","reqId":"req-1h","res":{"statusCode":200},"responseTime":1.0457000732421875,"msg":"request completed"} +{ + "level":30, + "time":1782480774533, + "pid":1, + "hostname":"webui", + "reqId":"req-1h", + "res":{"statusCode":200}, + "responseTime":1.0457000732421875, + "msg":"request completed" +} ``` - ### Core and package tools -Native core binaries use `spdlog` text output. Package controller commands and one-shot setup scripts -also use human-readable stdlib logging. These tools are not covered by the Python/Rust/WebUI service -JSON contracts. +Native core binaries use `spdlog` text output. Package controller commands and one-shot setup +scripts also use human-readable stdlib logging. These tools are not covered by the Python/Rust/WebUI +service JSON contracts. [EnvFilter]: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html [clp-py-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-py-utils/clp_py_utils/clp_logging.py diff --git a/docs/src/dev-docs/logging.md b/docs/src/dev-docs/logging.md index 02d5fdc744..377351b075 100644 --- a/docs/src/dev-docs/logging.md +++ b/docs/src/dev-docs/logging.md @@ -1,11 +1,14 @@ -# CLP Package Logging +# CLP Package logging -The CLP package utilizes a polyglot architecture, meaning each component family manages its own logging stack. There is **no single project-wide JSON schema**. +The CLP Package uses a polyglot architecture, meaning each component family manages its own logging +stack. There is **no single project-wide JSON schema**. This document is divided into two sections: -* [Developer guide](logging-developer-guide.md): How to set up and write logs when modifying CLP components. -* [Operator guide](logging-operator-guide.md): How to configure log levels, capture service logs, and understand the component-specific log structures in deployment +* [Developer guide](logging-developer-guide.md): How to set up and write logs when modifying CLP + components. +* [Operator guide](logging-operator-guide.md): How to configure log levels, capture service logs, + and understand the component-specific log structures in deployment. :::{toctree} :hidden: From 93450523c440eb20465fb4d783858310f33883cc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 29 Jun 2026 17:38:23 -0400 Subject: [PATCH 28/35] docs(logging): Correct typo in logging configuration documentation --- components/clp-py-utils/clp_py_utils/clp_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_logging.py b/components/clp-py-utils/clp_py_utils/clp_logging.py index 05555cc058..d81086a3d2 100644 --- a/components/clp-py-utils/clp_py_utils/clp_logging.py +++ b/components/clp-py-utils/clp_py_utils/clp_logging.py @@ -77,7 +77,7 @@ def get_structlog_logger(name: str) -> FilteringBoundLogger: Configure CLP's structured logging defaults and return a structlog logger for a component. CLP uses structlog's stdlib integration, so this first configures the stdlib logger's - handlers with structlog's `ProcessFormatter`. Use the returned structlog logger for log calls. + handlers with structlog's `ProcessorFormatter`. Use the returned structlog logger for log calls. :param name: Name of the logger to create or retrieve. :return: A structlog logger configured to emit through CLP's JSON formatter. From f20f3a494ff6abee3697b815733c2eb0263fb917 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 29 Jun 2026 17:41:46 -0400 Subject: [PATCH 29/35] Revert "docs(logging): Update logging guides for consistency and clarity" This reverts commit 1f7e0eb5f09445a0152b6515c4c7bf7a63c5620e. --- docs/src/dev-docs/logging-developer-guide.md | 46 +++---- docs/src/dev-docs/logging-operator-guide.md | 133 +++++++------------ docs/src/dev-docs/logging.md | 11 +- 3 files changed, 77 insertions(+), 113 deletions(-) diff --git a/docs/src/dev-docs/logging-developer-guide.md b/docs/src/dev-docs/logging-developer-guide.md index b42a9602d7..a24d303ae9 100644 --- a/docs/src/dev-docs/logging-developer-guide.md +++ b/docs/src/dev-docs/logging-developer-guide.md @@ -1,4 +1,4 @@ -# Developer guide: writing logs +# Developer Guide: Writing Logs This guide provides the language-specific setup instructions and standards required to initialize loggers and emit diagnostic logs when developing or modifying CLP components. @@ -13,12 +13,12 @@ downstream in log viewers or aggregation systems. New Python orchestration services should use `structlog` for structured JSON logging and bind context variables where appropriate. -```python -from clp_py_utils.clp_logging import get_structlog_logger + ```python + from clp_py_utils.clp_logging import get_structlog_logger -log = get_structlog_logger("service_name") -log.info("hello, %s!", "world") -``` + log = get_structlog_logger("service_name") + log.info("hello, %s!", "world") + ``` :::{note} Existing Python services use stdlib loggers whose handlers are configured with structlog's @@ -28,38 +28,38 @@ Existing Python services use stdlib loggers whose handlers are configured with s ## Rust New Rust HTTP services should initialize `tracing` at process startup using -[`clp_rust_utils::logging::set_up_logging`][clp-rust-logging] and keep the returned guard alive -for the lifetime of the process: + [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging] and keep the returned guard alive + for the lifetime of the process: -```rust -let _guard = clp_rust_utils::logging::set_up_logging("service_name.log"); - -// Choose structured logging over formatting values directly into the message field. -tracing::info!(server_address = %addr, "Server started."); -``` + ```rust + let _guard = clp_rust_utils::logging::set_up_logging("service_name.log"); + + // Choose structured logging over formatting values directly into the message field. + tracing::info!(server_address = %addr, "Server started."); + ``` ## WebUI WebUI server code should use Fastify's Pino logger. Use `request.log` for request-scoped logs and `app.log` for startup, shutdown, and application-level logs: -```typescript -// Request-scoped -request.log.info({searchJobId}, "Search submitted"); -request.log.error(err, "Failed to submit search"); + ```typescript + // Request-scoped + request.log.info({searchJobId}, "Search submitted"); + request.log.error(err, "Failed to submit search"); -// Application-scoped -app.log.info("WebUI server listening on port 3000"); -``` + // Application-scoped + app.log.info("WebUI server listening on port 3000"); + ``` WebUI client code may use `console.*` for browser diagnostics. Logs that operators need to collect, search, or alert on should be emitted by the WebUI server. -## Core & setup tool +## Core & Setup Tool * Native core binaries (`clp`, `clp-s`, `glt`, native `reducer_server`) should continue using spdlog and their existing entry-point logger setup. * Package/setup tools (DB initialization scripts, package controllers) should use standard Python -logging. +logger. [clp-rust-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-rust-utils/src/logging.rs diff --git a/docs/src/dev-docs/logging-operator-guide.md b/docs/src/dev-docs/logging-operator-guide.md index c70e9d0be7..301ca18744 100644 --- a/docs/src/dev-docs/logging-operator-guide.md +++ b/docs/src/dev-docs/logging-operator-guide.md @@ -1,16 +1,16 @@ -# Operator guide: consuming CLP logs +# Operator Guide: Consuming CLP Logs -This guide details how to configure internal log levels, capture service logs, and understand the -component-specific log structures emitted by a running CLP deployment. +This guide details how to configure internal log levels, capture service logs, +and understand the component-specific log structures emitted by a running CLP deployment. -| Component family | Components | Logger | Format | Level control | -|-------------------------------|--------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------|----------------------------------|----------------------------| -| Python orchestration services | `compression_scheduler`, `query_scheduler`, `compression_worker`, `query_worker`, `reducer`, `garbage_collector`, `mcp_server` | [`structlog` with stdlib logging compatibility][clp-py-logging] | JSON | `CLP_LOGGING_LEVEL` | -| Rust HTTP services | `api_server`, `log_ingestor` | [`clp_rust_utils::logging`][clp-rust-logging] / `tracing` | JSON | `RUST_LOG` | -| WebUI server | Fastify server | Fastify/Pino | JSON in prod; pretty text in dev | `LOG_LEVEL` | -| WebUI client | Browser app | Browser console | Browser console output | Browser/devtools dependent | -| Native core binaries | `clp`, `clp-s`, `glt`, native `reducer_server` | `spdlog` | Text | Binary-specific | -| Package/setup tools | Package controller, DB initialization scripts | Python stdlib logging | Text | Script-specific | +| Component family | Components | Logger | Format | Level control | +|-------------------------------|---------------------------------------------------------------------|-----------------------------------------------------------|----------------------------------|----------------------------| +| Python orchestration services | `compression_scheduler`, `query_scheduler`, `compression_worker`, `query_worker`, `reducer`, `garbage_collector`, `mcp_server` | [`structlog` with stdlib logging compatibility][clp-py-logging] | JSON | `CLP_LOGGING_LEVEL` | +| Rust HTTP services | `api_server`, `log_ingestor` | [`clp_rust_utils::logging`][clp-rust-logging] / `tracing` | JSON | `RUST_LOG` | +| WebUI server | Fastify server | Fastify/Pino | JSON in prod; pretty text in dev | `LOG_LEVEL` | +| WebUI client | Browser app | Browser console | Browser console output | Browser/devtools dependent | +| Native core binaries | `clp`, `clp-s`, `glt`, native `reducer_server` | `spdlog` | Text | Binary-specific | +| Package/setup tools | Package controller, DB initialization scripts | Python stdlib logging | Text | Script-specific | :::{note} There is no single project-wide JSON schema. Python, Rust, and Pino logs are all line-delimited JSON @@ -19,46 +19,34 @@ in packaged non-interactive service runtimes, but each component family uses its ## Configuration -### Log level configuration +### Log Level Configuration -* **Python orchestration services**: `CLP_LOGGING_LEVEL` supports `DEBUG`, `INFO`, `WARN`, - `WARNING`, `ERROR`, and `CRITICAL`. Missing or invalid values default to `INFO`. +* **Python orchestration services**: `CLP_LOGGING_LEVEL` supports `DEBUG`, `INFO`, `WARN`, `WARNING`, `ERROR`, and `CRITICAL`. + Missing or invalid values default to `INFO`. * `CLP_LOGS_DIR`, when set, adds a file handler at `/.log` in - addition to stdout. + addition to stdout. * **Rust HTTP services**: Configure log filtering with [tracing_subscriber::EnvFilter][EnvFilter]. Filter directives are read from the `RUST_LOG` environment variable to determine which spans and events are enabled. - * *Note on `log_ingestor`*: Level is exposed via `CLP_LOG_INGESTOR_LOGGING_LEVEL` (Docker Compose) - or `clpConfig.log_ingestor.logging_level` (Helm), which is then passed to `RUST_LOG`. - * *Note on `api_server`*: Currently runs hardcoded at INFO and does not expose a deployment - setting. -* **WebUI**: `LOG_LEVEL` controls the Pino server log level (defaults to `info`). - * *Warning*: Be mindful of environment variable collisions with LOG_LEVEL in shared container - spaces. - -### Deployment notes and log output - -* **`docker-compose`**: Service stdout is available via `docker compose logs`. If `CLP_LOGS_DIR` is - set, logs are additionally written to `/.log` (which mounts to - `./var/log` via `CLP_LOGS_DIR_HOST`). For Rust components, this adds an hourly non-blocking - rolling file appender. + * *Note on `log_ingestor`*: Level is exposed via CLP_LOG_INGESTOR_LOGGING_LEVEL (Docker Compose) or clpConfig.log_ingestor.logging_level (Helm), which is then passed to RUST_LOG. + * *Note on `api_server`*: Currently runs hardcoded at INFO and does not expose a deployment setting. +* **WebUI**: `LOG_LEVEL` controls the Pino server log level (defaults to `info`). *Warning: Be mindful of environment variable collisions with LOG_LEVEL in shared container spaces.* + +### Deployment Notes and Log output + +* **`docker-compose`**: Service stdout is available via `docker compose logs`. If `CLP_LOGS_DIR` is set, +logs are additionally written to `/.log` (which mounts to `./var/log` +via `CLP_LOGS_DIR_HOST`). +*Note: For Rust components, this adds an hourly non-blocking rolling file appender.* * **`helm`**: Rely on pod stdout via `kubectl logs` or a cluster log collector (e.g., Fluent Bit). - File logging is template-specific. +File logging is template-specific. * **WebUI**: - * **Server**: In non-interactive deployments, the Fastify server emits Pino JSON directly to - `stdout`. If run in an interactive terminal (e.g., local development), it uses `pino-pretty` for - human-readable output. - * **Client**: Browser logs (`console.*`) remain local to the user's browser devtools and are *not* - captured by backend service telemetry. Treat these purely as local diagnostics. -* **Core**: Native binaries (clp, clp-s, etc.) emit standard human-readable text. Currently, Python - orchestration services invoke native core binaries (clp, clp-s, etc.) as subprocesses and the core - binaries' human-readable `spdlog` text is piped into the Python logger's output stream as a single - multi-line JSON record. -* **Setup tools**: Package controller scripts emit standard human-readable text. They do not adhere - to the JSON schemas used by the orchestration services and rely strictly on standard output - streams. - -## Component-specific logging details + * **Server**: Server: In non-interactive deployments, the Fastify server emits Pino JSON directly to `stdout`. If run in an interactive terminal (e.g., local development), it uses pino-pretty for human-readable output. + * **Client**: Browser logs (`console.*`) remain local to the user's browser devtools and are *not* captured by backend service telemetry. Treat these purely as local diagnostics. +* **Core**: Native binaries (clp, clp-s, etc.) emit standard human-readable text. Currently, Python orchestration services invoke native core binaries (clp, clp-s, etc.) as subprocesses and the core binaries' human-readable `spdlog` text is piped into the Python logger's output stream as a single multi-line JSON record. +* **Setup tools**: Package controller scripts emit standard human-readable text. They do not adhere to the JSON schemas used by the orchestration services and rely strictly on standard output streams. + +## Component-Specific Logging details The following sections describe the behavior for each component family. @@ -83,17 +71,11 @@ record. Each record includes the following fields: Example: + ```json -{ - "timestamp":"2026-06-22T17:03:21.123456Z", - "event":"Compression job 1 submitted.", - "logger":"compression_scheduler", - "level":"info", - "filename":"compression_scheduler.py", - "func_name":"main", - "lineno":616 -} +{"timestamp":"2026-06-22T17:03:21.123456Z","event":"Compression job 1 submitted.","logger":"compression_scheduler","level":"info","filename":"compression_scheduler.py","func_name":"main","lineno":616} ``` + ### Rust HTTP services @@ -101,29 +83,21 @@ These services use [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging] `tracing_subscriber` to emit one JSON object per log record. Each record includes the following fields: -| Field | Description | -|---------------|-------------------------------------------------------------------| -| `timestamp` | Timestamp emitted by `tracing_subscriber`. | +| Field | Description | +|---------------|------------------------------------------------------------| +| `timestamp` | Timestamp emitted by `tracing_subscriber`. | | `level` | Log level, emitted as values such as `INFO`, `WARN`, and `ERROR`. | -| `fields` | Structured `tracing` fields, including the log message. | -| `filename` | Source filename for the log call. | -| `line_number` | Source line number for the log call. | +| `fields` | Structured `tracing` fields, including the log message. | +| `filename` | Source filename for the log call. | +| `line_number` | Source line number for the log call. | Example: + ```json -{ - "timestamp":"2026-06-26T13:06:48.621307", - "level":"INFO", - "fields":{ - "message":"Spawned SQS listener task.", - "job_id":"3", - "task_id":"0" - }, - "filename":"components/log-ingestor/src/ingestion_job/sqs_listener.rs", - "line_number":320 -} +{"timestamp":"2026-06-26T13:06:48.621307","level":"INFO","fields":{"message":"Spawned SQS listener task.","job_id":"3","task_id":"0"},"filename":"components/log-ingestor/src/ingestion_job/sqs_listener.rs","line_number":320} ``` + ### WebUI @@ -144,24 +118,17 @@ following fields: Example: + ```json -{ - "level":30, - "time":1782480774533, - "pid":1, - "hostname":"webui", - "reqId":"req-1h", - "res":{"statusCode":200}, - "responseTime":1.0457000732421875, - "msg":"request completed" -} +{"level":30,"time":1782480774533,"pid":1,"hostname":"webui","reqId":"req-1h","res":{"statusCode":200},"responseTime":1.0457000732421875,"msg":"request completed"} ``` + ### Core and package tools -Native core binaries use `spdlog` text output. Package controller commands and one-shot setup -scripts also use human-readable stdlib logging. These tools are not covered by the Python/Rust/WebUI -service JSON contracts. +Native core binaries use `spdlog` text output. Package controller commands and one-shot setup scripts +also use human-readable stdlib logging. These tools are not covered by the Python/Rust/WebUI service +JSON contracts. [EnvFilter]: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html [clp-py-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-py-utils/clp_py_utils/clp_logging.py diff --git a/docs/src/dev-docs/logging.md b/docs/src/dev-docs/logging.md index 377351b075..02d5fdc744 100644 --- a/docs/src/dev-docs/logging.md +++ b/docs/src/dev-docs/logging.md @@ -1,14 +1,11 @@ -# CLP Package logging +# CLP Package Logging -The CLP Package uses a polyglot architecture, meaning each component family manages its own logging -stack. There is **no single project-wide JSON schema**. +The CLP package utilizes a polyglot architecture, meaning each component family manages its own logging stack. There is **no single project-wide JSON schema**. This document is divided into two sections: -* [Developer guide](logging-developer-guide.md): How to set up and write logs when modifying CLP - components. -* [Operator guide](logging-operator-guide.md): How to configure log levels, capture service logs, - and understand the component-specific log structures in deployment. +* [Developer guide](logging-developer-guide.md): How to set up and write logs when modifying CLP components. +* [Operator guide](logging-operator-guide.md): How to configure log levels, capture service logs, and understand the component-specific log structures in deployment :::{toctree} :hidden: From ae977c0b53171de5debf6bbae5e4df380d790726 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 29 Jun 2026 17:43:31 -0400 Subject: [PATCH 30/35] refactor(logging): Update logger implementation to use BoundLogger --- components/clp-py-utils/clp_py_utils/clp_logging.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_logging.py b/components/clp-py-utils/clp_py_utils/clp_logging.py index d81086a3d2..cd66152029 100644 --- a/components/clp-py-utils/clp_py_utils/clp_logging.py +++ b/components/clp-py-utils/clp_py_utils/clp_logging.py @@ -6,7 +6,8 @@ from typing import get_args, Literal import structlog -from structlog.typing import FilteringBoundLogger, Processor +from structlog.stdlib import BoundLogger +from structlog.typing import Processor LoggingLevel = Literal[ "INFO", @@ -68,11 +69,12 @@ def configure_structlog() -> None: structlog.stdlib.ProcessorFormatter.wrap_for_formatter, ], logger_factory=structlog.stdlib.LoggerFactory(), + wrapper_class=BoundLogger, cache_logger_on_first_use=True, ) -def get_structlog_logger(name: str) -> FilteringBoundLogger: +def get_structlog_logger(name: str) -> BoundLogger: """ Configure CLP's structured logging defaults and return a structlog logger for a component. @@ -85,7 +87,7 @@ def get_structlog_logger(name: str) -> FilteringBoundLogger: configure_structlog() stdlib_logger = get_logger(name) configure_logging(stdlib_logger, name) - return structlog.get_logger(name) + return structlog.stdlib.get_logger(name) def get_logging_formatter() -> logging.Formatter: From 2f6336835f77cf7b58b56b98d3c6b0b7ab8b8cb1 Mon Sep 17 00:00:00 2001 From: Rishikesh Devsot Date: Mon, 29 Jun 2026 17:44:48 -0400 Subject: [PATCH 31/35] polishing doc --- docs/src/dev-docs/logging-developer-guide.md | 4 +- docs/src/dev-docs/logging-operator-guide.md | 99 +++++++++++++------- docs/src/dev-docs/logging.md | 4 +- 3 files changed, 68 insertions(+), 39 deletions(-) diff --git a/docs/src/dev-docs/logging-developer-guide.md b/docs/src/dev-docs/logging-developer-guide.md index aef462d2d7..fd9e72eebc 100644 --- a/docs/src/dev-docs/logging-developer-guide.md +++ b/docs/src/dev-docs/logging-developer-guide.md @@ -1,4 +1,4 @@ -# Developer Guide: Writing logs +# Developer guide: Writing logs This guide provides language-specific setup instructions and standards for initializing loggers and emitting diagnostic logs when developing or modifying CLP components. @@ -33,7 +33,7 @@ New Rust HTTP services should initialize `tracing` at process startup using ```rust let _guard = clp_rust_utils::logging::set_up_logging("service_name.log"); - + // Choose structured logging over formatting values directly into the message field. tracing::info!(server_address = %addr, "Server started."); ``` diff --git a/docs/src/dev-docs/logging-operator-guide.md b/docs/src/dev-docs/logging-operator-guide.md index 301ca18744..907bc09935 100644 --- a/docs/src/dev-docs/logging-operator-guide.md +++ b/docs/src/dev-docs/logging-operator-guide.md @@ -1,54 +1,79 @@ -# Operator Guide: Consuming CLP Logs +# Operator guide: Consuming CLP logs This guide details how to configure internal log levels, capture service logs, and understand the component-specific log structures emitted by a running CLP deployment. +The table below provides a high-level overview of the logging behaviors +across all component families: | Component family | Components | Logger | Format | Level control | |-------------------------------|---------------------------------------------------------------------|-----------------------------------------------------------|----------------------------------|----------------------------| | Python orchestration services | `compression_scheduler`, `query_scheduler`, `compression_worker`, `query_worker`, `reducer`, `garbage_collector`, `mcp_server` | [`structlog` with stdlib logging compatibility][clp-py-logging] | JSON | `CLP_LOGGING_LEVEL` | | Rust HTTP services | `api_server`, `log_ingestor` | [`clp_rust_utils::logging`][clp-rust-logging] / `tracing` | JSON | `RUST_LOG` | -| WebUI server | Fastify server | Fastify/Pino | JSON in prod; pretty text in dev | `LOG_LEVEL` | +| WebUI server | Fastify server | [Fastify/Pino][pino] | JSON or pretty text | `LOG_LEVEL` | | WebUI client | Browser app | Browser console | Browser console output | Browser/devtools dependent | | Native core binaries | `clp`, `clp-s`, `glt`, native `reducer_server` | `spdlog` | Text | Binary-specific | | Package/setup tools | Package controller, DB initialization scripts | Python stdlib logging | Text | Script-specific | -:::{note} -There is no single project-wide JSON schema. Python, Rust, and Pino logs are all line-delimited JSON -in packaged non-interactive service runtimes, but each component family uses its own field names. -::: - -## Configuration +## Log level configuration -### Log Level Configuration +This section covers how to modify logging verbosity for different components. +Use the following environment variables and settings to filter +logs for each component family: * **Python orchestration services**: `CLP_LOGGING_LEVEL` supports `DEBUG`, `INFO`, `WARN`, `WARNING`, `ERROR`, and `CRITICAL`. Missing or invalid values default to `INFO`. - * `CLP_LOGS_DIR`, when set, adds a file handler at `/.log` in - addition to stdout. * **Rust HTTP services**: Configure log filtering with [tracing_subscriber::EnvFilter][EnvFilter]. Filter directives are read from the `RUST_LOG` environment variable to determine which spans and events are enabled. - * *Note on `log_ingestor`*: Level is exposed via CLP_LOG_INGESTOR_LOGGING_LEVEL (Docker Compose) or clpConfig.log_ingestor.logging_level (Helm), which is then passed to RUST_LOG. - * *Note on `api_server`*: Currently runs hardcoded at INFO and does not expose a deployment setting. -* **WebUI**: `LOG_LEVEL` controls the Pino server log level (defaults to `info`). *Warning: Be mindful of environment variable collisions with LOG_LEVEL in shared container spaces.* + * *`log-ingestor`*: Log level is exposed via `CLP_LOG_INGESTOR_LOGGING_LEVEL` + (Docker Compose) or `clpConfig.log_ingestor.logging_level` (Helm), which is + then passed to `RUST_LOG`. + * *`api-server`*: Log level is hardcoded to `INFO` and does not expose a deployment + variable for configuration. +* **WebUI**: `LOG_LEVEL` controls the Pino server log level (defaults to `info`). +:::{warning} +`LOG_LEVEL` is a generic environment variable. Be mindful of environment +variable collisions with other tools or container settings. +::: + +## Log routing and collection + +Access to CLP package logs is entirely dependent on where the component is executed. + +### Containerized services + +For continuously running deployed services such as Python orchestration services, Rust HTTP services, and WebUI server, log routing depends on the orchestrator: -### Deployment Notes and Log output +* **Docker Compose**: Logs are available using `docker compose logs`. If `CLP_LOGS_DIR` is set, +logs are additionally written to `/.log`. This directory mounts to the host via `CLP_LOGS_DIR_HOST` (defaults to `./var/log`). +:::{note} +For Rust services, setting `CLP_LOGS_DIR` enables file logging with hourly log rotation and a +non-blocking file writer. +::: +* **Kubernetes/Helm**: Logs are accessible using `kubectl logs` or a cluster log collector (e.g., Fluent Bit). +File logging is only enabled for templates that set `CLP_LOGS_DIR` and mount a log volume. + +### Standalone tools -* **`docker-compose`**: Service stdout is available via `docker compose logs`. If `CLP_LOGS_DIR` is set, -logs are additionally written to `/.log` (which mounts to `./var/log` -via `CLP_LOGS_DIR_HOST`). -*Note: For Rust components, this adds an hourly non-blocking rolling file appender.* -* **`helm`**: Rely on pod stdout via `kubectl logs` or a cluster log collector (e.g., Fluent Bit). -File logging is template-specific. -* **WebUI**: - * **Server**: Server: In non-interactive deployments, the Fastify server emits Pino JSON directly to `stdout`. If run in an interactive terminal (e.g., local development), it uses pino-pretty for human-readable output. - * **Client**: Browser logs (`console.*`) remain local to the user's browser devtools and are *not* captured by backend service telemetry. Treat these purely as local diagnostics. -* **Core**: Native binaries (clp, clp-s, etc.) emit standard human-readable text. Currently, Python orchestration services invoke native core binaries (clp, clp-s, etc.) as subprocesses and the core binaries' human-readable `spdlog` text is piped into the Python logger's output stream as a single multi-line JSON record. -* **Setup tools**: Package controller scripts emit standard human-readable text. They do not adhere to the JSON schemas used by the orchestration services and rely strictly on standard output streams. +* **Core binaries**: Running standalone binaries (`clp`, `clp-s`, etc.) emits +unstructured `spdlog` text to `stdout`. +:::{note} +*When deployed*, Python orchestration services invoke these binaries as subprocesses. +Consequently, the core binaries' unstructured log text is captured and written to the +Python logger's output stream as a single multi-line JSON record. These appear alongside +normal container logs in Docker or Kubernetes. +::: +* **WebUI client**: WebUI client (`console.*`) logs remain local to the user's browser devtools +and are *not* captured by any backend telemetry service. They should be treated +purely as local diagnostics. +* **Package scripts**: Package controller commands and one-shot +setup scripts emit unstructured Python stdlib logs to `stdout`. -## Component-Specific Logging details +## Component-specific log schemas -The following sections describe the behavior for each component family. +There is no single project-wide JSON schema. Python, Rust, and Pino logs are all line-delimited JSON +in packaged non-interactive service runtimes, but each component family uses its own field names. +The following sections describe the log schema for each component family. ### Python orchestration services @@ -99,10 +124,13 @@ Example: ``` -### WebUI +### WebUI server + +The WebUI server uses Fastify's Pino logger. When the server runs without an interactive terminal, +such as in Docker Compose or Kubernetes, it emits one JSON object per log record. When `stdout` is an +interactive terminal, it uses `pino-pretty` for human-readable output. -The WebUI server uses Fastify's Pino logger where each record includes the -following fields: +Each JSON record includes the following fields: | Field | Description | |----------------|---------------------------------------------------------| @@ -126,10 +154,11 @@ Example: ### Core and package tools -Native core binaries use `spdlog` text output. Package controller commands and one-shot setup scripts -also use human-readable stdlib logging. These tools are not covered by the Python/Rust/WebUI service -JSON contracts. +Native core binaries emit unstructured `spdlog` text logs. Package controller commands and one-shot +setup scripts also emit unstructured Python stdlib logs. These tools do not follow the JSON log +formats described for Python orchestration services, Rust HTTP services, or the WebUI server. -[EnvFilter]: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html [clp-py-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-py-utils/clp_py_utils/clp_logging.py [clp-rust-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-rust-utils/src/logging.rs +[pino]: https://fastify.dev/docs/v2.15.x/Documentation/Logging/ +[EnvFilter]: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html diff --git a/docs/src/dev-docs/logging.md b/docs/src/dev-docs/logging.md index 02d5fdc744..df2b782e20 100644 --- a/docs/src/dev-docs/logging.md +++ b/docs/src/dev-docs/logging.md @@ -1,11 +1,11 @@ -# CLP Package Logging +# CLP package logging The CLP package utilizes a polyglot architecture, meaning each component family manages its own logging stack. There is **no single project-wide JSON schema**. This document is divided into two sections: * [Developer guide](logging-developer-guide.md): How to set up and write logs when modifying CLP components. -* [Operator guide](logging-operator-guide.md): How to configure log levels, capture service logs, and understand the component-specific log structures in deployment +* [Operator guide](logging-operator-guide.md): How to configure log levels, capture service logs, and understand the component-specific log structures in deployment. :::{toctree} :hidden: From 437c6de2c247fe1300e46ac4e59112aeba928fef Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 29 Jun 2026 17:45:50 -0400 Subject: [PATCH 32/35] docs(celery): Add module docstring for task execution configuration --- .../job_orchestration/executor/compress/celery.py | 2 ++ .../job_orchestration/executor/query/celery.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/components/job-orchestration/job_orchestration/executor/compress/celery.py b/components/job-orchestration/job_orchestration/executor/compress/celery.py index aad69ce77e..1bbb84bcd1 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/celery.py +++ b/components/job-orchestration/job_orchestration/executor/compress/celery.py @@ -1,3 +1,5 @@ +"""Celery app configuration for compression task execution.""" + import logging from celery import Celery, signals diff --git a/components/job-orchestration/job_orchestration/executor/query/celery.py b/components/job-orchestration/job_orchestration/executor/query/celery.py index c981b8ac8d..e624c97fba 100644 --- a/components/job-orchestration/job_orchestration/executor/query/celery.py +++ b/components/job-orchestration/job_orchestration/executor/query/celery.py @@ -1,3 +1,5 @@ +"""Celery app configuration for query task execution.""" + import logging from celery import Celery, signals From 1706cf9ae072c5a6ed727dbcd2e55bd50dc4d1bf Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 29 Jun 2026 17:59:15 -0400 Subject: [PATCH 33/35] docs(logging): Improve clarity and consistency in logging guides --- docs/src/dev-docs/logging-developer-guide.md | 62 ++++---- docs/src/dev-docs/logging-operator-guide.md | 150 +++++++++++-------- docs/src/dev-docs/logging.md | 11 +- 3 files changed, 124 insertions(+), 99 deletions(-) diff --git a/docs/src/dev-docs/logging-developer-guide.md b/docs/src/dev-docs/logging-developer-guide.md index fd9e72eebc..865240a8f0 100644 --- a/docs/src/dev-docs/logging-developer-guide.md +++ b/docs/src/dev-docs/logging-developer-guide.md @@ -1,24 +1,24 @@ -# Developer guide: Writing logs +# Developer guide: writing logs This guide provides language-specific setup instructions and standards for initializing loggers and emitting diagnostic logs when developing or modifying CLP components. :::{note} -Always prefer UTC service timestamps. Time zone conversions should be handled -downstream in log viewers or aggregation systems. +Always prefer UTC service timestamps. Time zone conversions should be handled downstream in log +viewers or aggregation systems. ::: ## Python -New Python orchestration services should use `structlog` for structured JSON logging -and bind context variables where appropriate. +New Python orchestration services should use `structlog` for structured JSON logging and bind +context variables where appropriate. - ```python - from clp_py_utils.clp_logging import get_structlog_logger +```python +from clp_py_utils.clp_logging import get_structlog_logger - log = get_structlog_logger("service_name") - log.info("hello, %s!", "world") - ``` +log = get_structlog_logger("service_name") +log.info("hello, %s!", "world") +``` :::{note} Existing Python services use stdlib loggers whose handlers are configured with structlog's @@ -28,43 +28,43 @@ Existing Python services use stdlib loggers whose handlers are configured with s ## Rust New Rust HTTP services should initialize `tracing` at process startup using - [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging] and keep the returned guard alive - for the lifetime of the process: +[`clp_rust_utils::logging::set_up_logging`][clp-rust-logging] and keep the returned guard alive +for the lifetime of the process: - ```rust - let _guard = clp_rust_utils::logging::set_up_logging("service_name.log"); +```rust +let _guard = clp_rust_utils::logging::set_up_logging("service_name.log"); - // Choose structured logging over formatting values directly into the message field. - tracing::info!(server_address = %addr, "Server started."); - ``` +// Choose structured logging over formatting values directly into the message field. +tracing::info!(server_address = %addr, "Server started."); +``` ## WebUI -WebUI server code should use Fastify's Pino logger. Use `request.log` for request-scoped -logs and `app.log` for startup, shutdown, and application-level logs: +WebUI server code should use Fastify's Pino logger. Use `request.log` for request-scoped logs and +`app.log` for startup, shutdown, and application-level logs: - ```typescript - // Request-scoped - request.log.info({searchJobId}, "Search submitted"); - request.log.error(err, "Failed to submit search"); +```typescript +// Request-scoped +request.log.info({searchJobId}, "Search submitted"); +request.log.error(err, "Failed to submit search"); - // Application-scoped - app.log.info("WebUI server listening on port 3000"); - ``` +// Application-scoped +app.log.info("WebUI server listening on port 3000"); +``` WebUI client code should use `console.*` for browser diagnostics: - ```typescript - console.error("Failed to submit query:", err); - ``` +```typescript +console.error("Failed to submit query:", err); +``` ## Core and setup tools * Native core binaries (`clp`, `clp-s`, `glt`, native `reducer_server`) should continue using [`spdlog`][spdlog] and the `spdlog` initialization already defined in each binary's startup code. * Package/setup tools (DB initialization scripts, package controllers) should use the - [standard Python logger][stdlogger]. + [standard Python logger][std-logger]. [clp-rust-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-rust-utils/src/logging.rs [spdlog]: https://github.com/gabime/spdlog -[stdlogger]: https://docs.python.org/3/library/logging.html +[std-logger]: https://docs.python.org/3/library/logging.html diff --git a/docs/src/dev-docs/logging-operator-guide.md b/docs/src/dev-docs/logging-operator-guide.md index 907bc09935..4a61a8bb84 100644 --- a/docs/src/dev-docs/logging-operator-guide.md +++ b/docs/src/dev-docs/logging-operator-guide.md @@ -1,44 +1,42 @@ -# Operator guide: Consuming CLP logs - -This guide details how to configure internal log levels, capture service logs, -and understand the component-specific log structures emitted by a running CLP deployment. -The table below provides a high-level overview of the logging behaviors -across all component families: - -| Component family | Components | Logger | Format | Level control | -|-------------------------------|---------------------------------------------------------------------|-----------------------------------------------------------|----------------------------------|----------------------------| -| Python orchestration services | `compression_scheduler`, `query_scheduler`, `compression_worker`, `query_worker`, `reducer`, `garbage_collector`, `mcp_server` | [`structlog` with stdlib logging compatibility][clp-py-logging] | JSON | `CLP_LOGGING_LEVEL` | -| Rust HTTP services | `api_server`, `log_ingestor` | [`clp_rust_utils::logging`][clp-rust-logging] / `tracing` | JSON | `RUST_LOG` | -| WebUI server | Fastify server | [Fastify/Pino][pino] | JSON or pretty text | `LOG_LEVEL` | -| WebUI client | Browser app | Browser console | Browser console output | Browser/devtools dependent | -| Native core binaries | `clp`, `clp-s`, `glt`, native `reducer_server` | `spdlog` | Text | Binary-specific | -| Package/setup tools | Package controller, DB initialization scripts | Python stdlib logging | Text | Script-specific | +# Operator guide: consuming CLP logs + +This guide details how to configure internal log levels, capture service logs, and understand the +component-specific log structures emitted by a running CLP deployment. The table below provides a +high-level overview of the logging behaviors across all component families: + +| Component family | Components | Logger | Format | Level control | +|-------------------------------|--------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------|------------------------|----------------------------| +| Python orchestration services | `compression_scheduler`, `query_scheduler`, `compression_worker`, `query_worker`, `reducer`, `garbage_collector`, `mcp_server` | [`structlog` with stdlib logging compatibility][clp-py-logging] | JSON | `CLP_LOGGING_LEVEL` | +| Rust HTTP services | `api_server`, `log_ingestor` | [`clp_rust_utils::logging`][clp-rust-logging] / `tracing` | JSON | `RUST_LOG` | +| WebUI server | Fastify server | [Fastify/Pino][pino] | JSON or pretty text | `LOG_LEVEL` | +| WebUI client | Browser app | Browser console | Browser console output | Browser/devtools dependent | +| Native core binaries | `clp`, `clp-s`, `glt`, native `reducer_server` | `spdlog` | Text | Binary-specific | +| Package/setup tools | Package controller, DB initialization scripts | Python stdlib logging | Text | Script-specific | ## Log level configuration -This section covers how to modify logging verbosity for different components. -Use the following environment variables and settings to filter -logs for each component family: +This section covers how to modify logging verbosity for different components. Use the following +environment variables and settings to filter logs for each component family: -* **Python orchestration services**: `CLP_LOGGING_LEVEL` supports `DEBUG`, `INFO`, `WARN`, `WARNING`, `ERROR`, and `CRITICAL`. - Missing or invalid values default to `INFO`. +* **Python orchestration services**: `CLP_LOGGING_LEVEL` supports `DEBUG`, `INFO`, `WARN`, + `WARNING`, `ERROR`, and `CRITICAL`. Missing or invalid values default to `INFO`. * **Rust HTTP services**: Configure log filtering with [tracing_subscriber::EnvFilter][EnvFilter]. Filter directives are read from the `RUST_LOG` environment variable to determine which spans and events are enabled. - * *`log-ingestor`*: Log level is exposed via `CLP_LOG_INGESTOR_LOGGING_LEVEL` - (Docker Compose) or `clpConfig.log_ingestor.logging_level` (Helm), which is - then passed to `RUST_LOG`. - * *`api-server`*: Log level is hardcoded to `INFO` and does not expose a deployment - variable for configuration. + * *`log-ingestor`*: Log level is exposed via `CLP_LOG_INGESTOR_LOGGING_LEVEL` (Docker Compose) + or `clpConfig.log_ingestor.logging_level` (Helm), which is then passed to `RUST_LOG`. + * *`api-server`*: Log level is hardcoded to `INFO` and does not expose a deployment variable + for configuration. * **WebUI**: `LOG_LEVEL` controls the Pino server log level (defaults to `info`). -:::{warning} -`LOG_LEVEL` is a generic environment variable. Be mindful of environment -variable collisions with other tools or container settings. -::: + + :::{warning} + `LOG_LEVEL` is a generic environment variable. Be mindful of environment + variable collisions with other tools or container settings. + ::: ## Log routing and collection -Access to CLP package logs is entirely dependent on where the component is executed. +Access to the CLP Package logs is entirely dependent on where the component is executed. ### Containerized services @@ -46,28 +44,28 @@ For continuously running deployed services such as Python orchestration services * **Docker Compose**: Logs are available using `docker compose logs`. If `CLP_LOGS_DIR` is set, logs are additionally written to `/.log`. This directory mounts to the host via `CLP_LOGS_DIR_HOST` (defaults to `./var/log`). -:::{note} -For Rust services, setting `CLP_LOGS_DIR` enables file logging with hourly log rotation and a -non-blocking file writer. -::: -* **Kubernetes/Helm**: Logs are accessible using `kubectl logs` or a cluster log collector (e.g., Fluent Bit). -File logging is only enabled for templates that set `CLP_LOGS_DIR` and mount a log volume. + :::{note} + For Rust services, setting `CLP_LOGS_DIR` enables file logging with hourly log rotation and a + non-blocking file writer. + ::: +* **Kubernetes/Helm**: Logs are accessible using `kubectl logs` or a cluster log collector (e.g., Fluent Bit). File + logging is only enabled for templates that set `CLP_LOGS_DIR` and mount a log volume. ### Standalone tools * **Core binaries**: Running standalone binaries (`clp`, `clp-s`, etc.) emits -unstructured `spdlog` text to `stdout`. -:::{note} -*When deployed*, Python orchestration services invoke these binaries as subprocesses. -Consequently, the core binaries' unstructured log text is captured and written to the -Python logger's output stream as a single multi-line JSON record. These appear alongside -normal container logs in Docker or Kubernetes. -::: + unstructured `spdlog` text to `stdout`. + :::{note} + *When deployed*, Python orchestration services invoke these binaries as subprocesses. + Consequently, the core binaries' unstructured log text is captured and written to the Python + logger's output stream as a single multi-line JSON record. These appear alongside normal container + logs in Docker or Kubernetes. + ::: * **WebUI client**: WebUI client (`console.*`) logs remain local to the user's browser devtools -and are *not* captured by any backend telemetry service. They should be treated -purely as local diagnostics. -* **Package scripts**: Package controller commands and one-shot -setup scripts emit unstructured Python stdlib logs to `stdout`. + and are *not* captured by any backend telemetry service. They should be treated purely as local + diagnostics. +* **Package scripts**: Package controller commands and one-shot setup scripts emit unstructured + Python stdlib logs to `stdout`. ## Component-specific log schemas @@ -96,11 +94,17 @@ record. Each record includes the following fields: Example: - ```json -{"timestamp":"2026-06-22T17:03:21.123456Z","event":"Compression job 1 submitted.","logger":"compression_scheduler","level":"info","filename":"compression_scheduler.py","func_name":"main","lineno":616} +{ + "timestamp": "2026-06-22T17:03:21.123456Z", + "event": "Compression job 1 submitted.", + "logger": "compression_scheduler", + "level": "info", + "filename": "compression_scheduler.py", + "func_name": "main", + "lineno": 616 +} ``` - ### Rust HTTP services @@ -108,27 +112,35 @@ These services use [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging] `tracing_subscriber` to emit one JSON object per log record. Each record includes the following fields: -| Field | Description | -|---------------|------------------------------------------------------------| -| `timestamp` | Timestamp emitted by `tracing_subscriber`. | +| Field | Description | +|---------------|-------------------------------------------------------------------| +| `timestamp` | Timestamp emitted by `tracing_subscriber`. | | `level` | Log level, emitted as values such as `INFO`, `WARN`, and `ERROR`. | -| `fields` | Structured `tracing` fields, including the log message. | -| `filename` | Source filename for the log call. | -| `line_number` | Source line number for the log call. | +| `fields` | Structured `tracing` fields, including the log message. | +| `filename` | Source filename for the log call. | +| `line_number` | Source line number for the log call. | Example: - ```json -{"timestamp":"2026-06-26T13:06:48.621307","level":"INFO","fields":{"message":"Spawned SQS listener task.","job_id":"3","task_id":"0"},"filename":"components/log-ingestor/src/ingestion_job/sqs_listener.rs","line_number":320} +{ + "timestamp": "2026-06-26T13:06:48.621307", + "level": "INFO", + "fields": { + "message": "Spawned SQS listener task.", + "job_id": "3", + "task_id": "0" + }, + "filename": "components/log-ingestor/src/ingestion_job/sqs_listener.rs", + "line_number": 320 +} ``` - ### WebUI server The WebUI server uses Fastify's Pino logger. When the server runs without an interactive terminal, -such as in Docker Compose or Kubernetes, it emits one JSON object per log record. When `stdout` is an -interactive terminal, it uses `pino-pretty` for human-readable output. +such as in Docker Compose or Kubernetes, it emits one JSON object per log record. When `stdout` is +an interactive terminal, it uses `pino-pretty` for human-readable output. Each JSON record includes the following fields: @@ -146,11 +158,20 @@ Each JSON record includes the following fields: Example: - ```json -{"level":30,"time":1782480774533,"pid":1,"hostname":"webui","reqId":"req-1h","res":{"statusCode":200},"responseTime":1.0457000732421875,"msg":"request completed"} +{ + "level": 30, + "time": 1782480774533, + "pid": 1, + "hostname": "webui", + "reqId": "req-1h", + "res": { + "statusCode": 200 + }, + "responseTime": 1.0457000732421875, + "msg": "request completed" +} ``` - ### Core and package tools @@ -158,6 +179,7 @@ Native core binaries emit unstructured `spdlog` text logs. Package controller co setup scripts also emit unstructured Python stdlib logs. These tools do not follow the JSON log formats described for Python orchestration services, Rust HTTP services, or the WebUI server. + [clp-py-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-py-utils/clp_py_utils/clp_logging.py [clp-rust-logging]: https://github.com/y-scope/clp/blob/DOCS_VAR_CLP_GIT_REF/components/clp-rust-utils/src/logging.rs [pino]: https://fastify.dev/docs/v2.15.x/Documentation/Logging/ diff --git a/docs/src/dev-docs/logging.md b/docs/src/dev-docs/logging.md index df2b782e20..bdfe64894c 100644 --- a/docs/src/dev-docs/logging.md +++ b/docs/src/dev-docs/logging.md @@ -1,11 +1,14 @@ -# CLP package logging +# CLP Package logging -The CLP package utilizes a polyglot architecture, meaning each component family manages its own logging stack. There is **no single project-wide JSON schema**. +The CLP Package utilizes a polyglot architecture, meaning each component family manages its own +logging stack. There is **no single project-wide JSON schema**. This document is divided into two sections: -* [Developer guide](logging-developer-guide.md): How to set up and write logs when modifying CLP components. -* [Operator guide](logging-operator-guide.md): How to configure log levels, capture service logs, and understand the component-specific log structures in deployment. +* [Developer guide](logging-developer-guide.md): How to set up and write logs when modifying CLP + components. +* [Operator guide](logging-operator-guide.md): How to configure log levels, capture service logs, + and understand the component-specific log structures when deploying CLP package services. :::{toctree} :hidden: From 958a3a37199095947e5546186d4fbb66d13f78b3 Mon Sep 17 00:00:00 2001 From: Rishikesh Devsot Date: Tue, 30 Jun 2026 00:45:31 -0400 Subject: [PATCH 34/35] nits --- docs/src/dev-docs/logging-developer-guide.md | 4 ++-- docs/src/dev-docs/logging.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/dev-docs/logging-developer-guide.md b/docs/src/dev-docs/logging-developer-guide.md index 865240a8f0..ae8d6948f4 100644 --- a/docs/src/dev-docs/logging-developer-guide.md +++ b/docs/src/dev-docs/logging-developer-guide.md @@ -10,7 +10,7 @@ viewers or aggregation systems. ## Python -New Python orchestration services should use `structlog` for structured JSON logging and bind +Python orchestration services should use `structlog` for structured JSON logging and bind context variables where appropriate. ```python @@ -27,7 +27,7 @@ Existing Python services use stdlib loggers whose handlers are configured with s ## Rust -New Rust HTTP services should initialize `tracing` at process startup using +Rust HTTP services should initialize `tracing` at process startup using [`clp_rust_utils::logging::set_up_logging`][clp-rust-logging] and keep the returned guard alive for the lifetime of the process: diff --git a/docs/src/dev-docs/logging.md b/docs/src/dev-docs/logging.md index bdfe64894c..f3ac0f2c31 100644 --- a/docs/src/dev-docs/logging.md +++ b/docs/src/dev-docs/logging.md @@ -1,6 +1,6 @@ # CLP Package logging -The CLP Package utilizes a polyglot architecture, meaning each component family manages its own +The CLP Package utilizes a polyglot architecture, meaning each component manages its own logging stack. There is **no single project-wide JSON schema**. This document is divided into two sections: From 1166348a1c2f6e16884e5bdf031e2b25eb79965c Mon Sep 17 00:00:00 2001 From: rishikeshdevsot <33337841+rishikeshdevsot@users.noreply.github.com> Date: Tue, 30 Jun 2026 10:48:51 -0400 Subject: [PATCH 35/35] Update logging example in developer guide --- docs/src/dev-docs/logging-developer-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/dev-docs/logging-developer-guide.md b/docs/src/dev-docs/logging-developer-guide.md index ae8d6948f4..f12af53db2 100644 --- a/docs/src/dev-docs/logging-developer-guide.md +++ b/docs/src/dev-docs/logging-developer-guide.md @@ -35,7 +35,7 @@ for the lifetime of the process: let _guard = clp_rust_utils::logging::set_up_logging("service_name.log"); // Choose structured logging over formatting values directly into the message field. -tracing::info!(server_address = %addr, "Server started."); +tracing::info!(job_id = compression_job_id, "Compression job completed."); ``` ## WebUI