From 2e64c208fbe2df3c7033358bae216f2c740ae295 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Mon, 13 Jul 2026 12:41:41 +0000 Subject: [PATCH 01/50] feat: add persisted logs --- .../renku_data_services/migrations/env.py | 2 + .../eadfb5e7e7cb_feat_add_persisted_logs.py | 91 +++++++++++++++++++ .../persisted_logs/__init__.py | 4 + .../renku_data_services/persisted_logs/orm.py | 72 +++++++++++++++ 4 files changed, 169 insertions(+) create mode 100644 components/renku_data_services/migrations/versions/eadfb5e7e7cb_feat_add_persisted_logs.py create mode 100644 components/renku_data_services/persisted_logs/__init__.py create mode 100644 components/renku_data_services/persisted_logs/orm.py diff --git a/components/renku_data_services/migrations/env.py b/components/renku_data_services/migrations/env.py index c4807e42b..0f92a3223 100644 --- a/components/renku_data_services/migrations/env.py +++ b/components/renku_data_services/migrations/env.py @@ -11,6 +11,7 @@ from renku_data_services.migrations.utils import run_migrations from renku_data_services.namespace.orm import BaseORM as namespaces from renku_data_services.notifications.orm import BaseORM as notifications +from renku_data_services.persisted_logs.orm import BaseORM as persisted_logs from renku_data_services.platform.orm import BaseORM as platform from renku_data_services.project.orm import BaseORM as project from renku_data_services.resource_usage.orm import BaseORM as resource_usage @@ -31,6 +32,7 @@ metrics.metadata, namespaces.metadata, notifications.metadata, + persisted_logs.metadata, platform.metadata, project.metadata, search.metadata, diff --git a/components/renku_data_services/migrations/versions/eadfb5e7e7cb_feat_add_persisted_logs.py b/components/renku_data_services/migrations/versions/eadfb5e7e7cb_feat_add_persisted_logs.py new file mode 100644 index 000000000..6103710e3 --- /dev/null +++ b/components/renku_data_services/migrations/versions/eadfb5e7e7cb_feat_add_persisted_logs.py @@ -0,0 +1,91 @@ +"""feat: add persisted logs + +Revision ID: eadfb5e7e7cb +Revises: 0a6cee40fe0d +Create Date: 2026-07-13 12:35:55.042555 + +""" + +import sqlalchemy as sa +from alembic import op + +from renku_data_services.utils.sqlalchemy import ULIDType + +# revision identifiers, used by Alembic. +revision = "eadfb5e7e7cb" +down_revision = "0a6cee40fe0d" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "session_runs", + sa.Column("id", ULIDType(), nullable=False), + sa.Column("user_id", sa.String(length=36), nullable=False), + sa.Column("launch_id", sa.String(), nullable=False), + sa.Column("launcher_id", ULIDType(), nullable=False), + sa.Column("submission_id", sa.String(), nullable=True), + sa.Column("first_log", sa.DateTime(timezone=True), nullable=False), + sa.Column("last_log", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint( + ["launcher_id"], + ["sessions.launchers.id"], + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.users.keycloak_id"], + ), + sa.PrimaryKeyConstraint("id"), + schema="persisted_logs", + ) + op.create_index( + op.f("ix_persisted_logs_session_runs_launcher_id"), + "session_runs", + ["launcher_id"], + unique=False, + schema="persisted_logs", + ) + op.create_index( + op.f("ix_persisted_logs_session_runs_user_id"), + "session_runs", + ["user_id"], + unique=False, + schema="persisted_logs", + ) + op.create_table( + "amalthea_session_logs", + sa.Column("id", ULIDType(), server_default=sa.text("generate_ulid()"), nullable=False), + sa.Column("run_id", ULIDType(), nullable=False), + sa.Column("container", sa.String(), nullable=False), + sa.Column("timestamp", sa.DateTime(timezone=True), nullable=False), + sa.Column("log_line", sa.String(), nullable=False), + sa.ForeignKeyConstraint(["run_id"], ["persisted_logs.session_runs.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + schema="persisted_logs", + ) + op.create_index( + op.f("ix_persisted_logs_amalthea_session_logs_run_id"), + "amalthea_session_logs", + ["run_id"], + unique=False, + schema="persisted_logs", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + op.f("ix_persisted_logs_amalthea_session_logs_run_id"), + table_name="amalthea_session_logs", + schema="persisted_logs", + ) + op.drop_table("amalthea_session_logs", schema="persisted_logs") + op.drop_index(op.f("ix_persisted_logs_session_runs_user_id"), table_name="session_runs", schema="persisted_logs") + op.drop_index( + op.f("ix_persisted_logs_session_runs_launcher_id"), table_name="session_runs", schema="persisted_logs" + ) + op.drop_table("session_runs", schema="persisted_logs") + # ### end Alembic commands ### diff --git a/components/renku_data_services/persisted_logs/__init__.py b/components/renku_data_services/persisted_logs/__init__.py new file mode 100644 index 000000000..83f61e78c --- /dev/null +++ b/components/renku_data_services/persisted_logs/__init__.py @@ -0,0 +1,4 @@ +"""Persisted logs module. + +Provides persisted logs for user workloads: interactive sessions, offline jobs, image builds, etc. +""" diff --git a/components/renku_data_services/persisted_logs/orm.py b/components/renku_data_services/persisted_logs/orm.py new file mode 100644 index 000000000..a57f9d683 --- /dev/null +++ b/components/renku_data_services/persisted_logs/orm.py @@ -0,0 +1,72 @@ +"""SQLAlchemy schemas for the data connectors database.""" + +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import DateTime, ForeignKey, MetaData +from sqlalchemy.orm import DeclarativeBase, Mapped, MappedAsDataclass, mapped_column, relationship +from ulid import ULID + +from renku_data_services.base_orm.registry import COMMON_ORM_REGISTRY +from renku_data_services.session.orm import SessionLauncherORM +from renku_data_services.users.orm import UserORM +from renku_data_services.utils.sqlalchemy import ULIDType + + +class BaseORM(MappedAsDataclass, DeclarativeBase): + """Base class for all ORM classes.""" + + metadata = MetaData(schema="persisted_logs") + registry = COMMON_ORM_REGISTRY + + +class SessionRunsORM(BaseORM): + """A session run, which is the continuous execution of a session.""" + + __tablename__ = "session_runs" + + id: Mapped[ULID] = mapped_column("id", ULIDType, primary_key=True) + """ID of a session run.""" + + user_id: Mapped[str] = mapped_column(ForeignKey(UserORM.keycloak_id), index=True, nullable=False) + """User ID of the owner of the session.""" + + launch_id: Mapped[str] = mapped_column(nullable=False) + """The launch ID for this session run.""" + + launcher_id: Mapped[ULID] = mapped_column(ULIDType, ForeignKey(SessionLauncherORM.id), index=True, nullable=False) + """The session launcher ID of the session.""" + + submission_id: Mapped[str | None] = mapped_column(nullable=True) + """The submission ID, if the session run corresponds to an offline job.""" + + first_log: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + """The timestamp of the first log line.""" + + last_log: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + """The timestamp of the last log line.""" + + +class AmaltheaSessionLogsORM(BaseORM): + """A log line from an Amalthea session.""" + + __tablename__ = "amalthea_session_logs" + + id: Mapped[ULID] = mapped_column("id", ULIDType, primary_key=True, default_factory=lambda: str(ULID()), init=False) + """ID of the log line.""" + + run_id: Mapped[ULID] = mapped_column(ForeignKey(SessionRunsORM.id, ondelete="CASCADE"), index=True, nullable=False) + """ID of the session run.""" + + session_run: Mapped[SessionRunsORM] = relationship(lazy="select", init=False, repr=False, viewonly=True) + """The session run this log line belongs to.""" + + container: Mapped[str] = mapped_column(nullable=False) + """The container this log line belongs to.""" + + timestamp: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + """The timestamp of the log line.""" + + log_line: Mapped[str] = mapped_column(nullable=False) + """The contents of the log line.""" From 6b6525a4ecfa074e30993f5c9dbf7b0504e79fd3 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 14 Jul 2026 12:28:20 +0000 Subject: [PATCH 02/50] wip: forward logs from loki --- .../persisted_logs/collector.py | 72 +++++++++++++++++++ .../persisted_logs/collector_t.py | 25 +++++++ .../persisted_logs/config.py | 26 +++++++ .../persisted_logs/constants.py | 12 ++++ .../renku_data_services/persisted_logs/db.py | 29 ++++++++ .../persisted_logs/loki_api.py | 70 ++++++++++++++++++ .../renku_data_services/persisted_logs/orm.py | 4 +- pyproject.toml | 1 + 8 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 components/renku_data_services/persisted_logs/collector.py create mode 100644 components/renku_data_services/persisted_logs/collector_t.py create mode 100644 components/renku_data_services/persisted_logs/config.py create mode 100644 components/renku_data_services/persisted_logs/constants.py create mode 100644 components/renku_data_services/persisted_logs/db.py create mode 100644 components/renku_data_services/persisted_logs/loki_api.py diff --git a/components/renku_data_services/persisted_logs/collector.py b/components/renku_data_services/persisted_logs/collector.py new file mode 100644 index 000000000..5262c46c2 --- /dev/null +++ b/components/renku_data_services/persisted_logs/collector.py @@ -0,0 +1,72 @@ +"""Collector for gathering persisted logs.""" + +import httpx +from pydantic import ValidationError + +from renku_data_services.app_config import logging +from renku_data_services.persisted_logs import loki_api +from renku_data_services.persisted_logs.config import PersistedLogsConfig +from renku_data_services.persisted_logs.constants import ( + PERSISTED_LOGS_NAMESPACE_LABEL_KEY, + PERSISTED_LOGS_SESSIONS_LABEL_KEY, + PERSISTED_LOGS_SESSIONS_LABEL_VALUE, +) + +logger = logging.getLogger(__name__) + + +class LokiLogReader: + """Read logs from loki.""" + + def __init__(self, config: PersistedLogsConfig, client: httpx.AsyncClient) -> None: + self.config = config + self.client = client + self.client.base_url = httpx.URL(config.loki_read_base_url) + + async def get_amalthea_session_logs(self) -> None: + """Returns log streams from Amalthea sessions.""" + params: dict[str, str] = dict() + params["query"] = ( + "{" + f'{PERSISTED_LOGS_SESSIONS_LABEL_KEY}="{PERSISTED_LOGS_SESSIONS_LABEL_VALUE}",' + f'{PERSISTED_LOGS_NAMESPACE_LABEL_KEY}="{self.config.namespace}"' + "}" + ) + params["direction"] = "forward" + logger.info(params) + res = await self.client.get("loki/api/v1/query_range", params=params) + res.raise_for_status() + logger.info(res) + result = loki_api.LokiQueryRangeResponse.model_validate_json(res.content) + # logger.info(result) + + log_line_ids: set[str] = set() + + for entry in result.data.result: + # logger.info(entry) + stream: loki_api.AmaltheaSessionStream | None = None + try: + stream = loki_api.AmaltheaSessionStream.model_validate(entry.stream) + except ValidationError as err: + logger.warning(f"Skipping entry {entry.stream} because of validation error: {err}") + continue + + logger.info(stream) + for nano_ts, log_line in entry.values: + logger.info(nano_ts.get_value()) + logger.info(log_line) + + log_line_id = f"{nano_ts.root}::{stream.container}::{stream.pod}" + + if log_line_id in log_line_ids: + logger.info(f"Already saw: {log_line_id}") + + log_line_ids.add(log_line_id) + + pass + + +class PersistedLogsCollector: + """Collector for gathering persisted logs.""" + + pass diff --git a/components/renku_data_services/persisted_logs/collector_t.py b/components/renku_data_services/persisted_logs/collector_t.py new file mode 100644 index 000000000..a13967379 --- /dev/null +++ b/components/renku_data_services/persisted_logs/collector_t.py @@ -0,0 +1,25 @@ +"""Temp: for testing with local port-forwarding...""" + +import asyncio + +import httpx + +from renku_data_services.app_config import logging +from renku_data_services.persisted_logs.collector import LokiLogReader +from renku_data_services.persisted_logs.config import PersistedLogsConfig + + +def _main() -> None: + logging.configure_logging() + config = PersistedLogsConfig( + enabled=True, + # loki_read_base_url="http://host.docker.internal:3100/", + loki_read_base_url="http://10.6.0.96:3100/", + namespace="renku", + ) + reader = LokiLogReader(config, client=httpx.AsyncClient()) + asyncio.run(reader.get_amalthea_session_logs()) + + +if __name__ == "__main__": + _main() diff --git a/components/renku_data_services/persisted_logs/config.py b/components/renku_data_services/persisted_logs/config.py new file mode 100644 index 000000000..5f1d4bffe --- /dev/null +++ b/components/renku_data_services/persisted_logs/config.py @@ -0,0 +1,26 @@ +"""Configuration for persisted logs.""" + +from dataclasses import dataclass + + +@dataclass(eq=True, frozen=True, kw_only=True) +class PersistedLogsConfig: + """Configuration for persisted logs.""" + + enabled: bool + loki_read_base_url: str + namespace: str + + @classmethod + def from_env(cls, namespace: str) -> "PersistedLogsConfig": + """Create a config from environment variables.""" + # enabled = os.environ.get("PERSISTED_LOG_ENABLED", "false").lower() == "true" + # return cls( + # enabled=enabled, + # ) + + return cls( + enabled=True, + loki_read_base_url="http://loki-read.monitoring.cluster.local.svc:3100/", + namespace=namespace, + ) diff --git a/components/renku_data_services/persisted_logs/constants.py b/components/renku_data_services/persisted_logs/constants.py new file mode 100644 index 000000000..898b05142 --- /dev/null +++ b/components/renku_data_services/persisted_logs/constants.py @@ -0,0 +1,12 @@ +"""Constants for persisted logs.""" + +from typing import Final + +PERSISTED_LOGS_SESSIONS_LABEL_KEY: Final[str] = "app" +"""The loki label key to select session logs streams.""" + +PERSISTED_LOGS_SESSIONS_LABEL_VALUE: Final[str] = "AmaltheaSession" +"""The loki label value to select session logs streams.""" + +PERSISTED_LOGS_NAMESPACE_LABEL_KEY: Final[str] = "namespace" +"""The loki label key to select logs streams from a specific kubernetes namespace.""" diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py new file mode 100644 index 000000000..cbc4e9c3c --- /dev/null +++ b/components/renku_data_services/persisted_logs/db.py @@ -0,0 +1,29 @@ +"""Adapters for persisted logs database classes.""" + +from datetime import datetime + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from renku_data_services.persisted_logs import orm as schemas + + +class AmaltheaSessionPersistedLogsRepository: + """Repository for persisted logs of Amalthea sessions.""" + + # loki: 1783946464779229935 <- nano + # python: 1783948286.9942 <- seconds + + async def get_latest_log_timestamp(self, session: AsyncSession) -> datetime | None: + """Returns the latest log timestamp.""" + stmt = ( + select(schemas.AmaltheaSessionLogsORM.timestamp) + .select_from(schemas.AmaltheaSessionLogsORM) + .order_by(schemas.AmaltheaSessionLogsORM.timestamp.desc) + .limit(1) + ) + res = await session.scalars(stmt) + timestamp = res.one_or_none() + return timestamp + + pass diff --git a/components/renku_data_services/persisted_logs/loki_api.py b/components/renku_data_services/persisted_logs/loki_api.py new file mode 100644 index 000000000..a45e907b8 --- /dev/null +++ b/components/renku_data_services/persisted_logs/loki_api.py @@ -0,0 +1,70 @@ +"""Pydantic models for the Loki API.""" + +from __future__ import annotations + +from enum import StrEnum + +from pydantic import BaseModel, ConfigDict, Field, RootModel + + +class Base(BaseModel): + """Base CRD specification.""" + + model_config = ConfigDict( + # Do not exclude unknown properties. + extra="allow" + ) + + +class LokiQueryRangeResponse(Base): + """Response from the query range endpoint (streams only).""" + + status: LokiQueryRangeResponseStatus + data: LokiQueryRangeResponseData + + +class LokiQueryRangeResponseStatus(StrEnum): + """Response status.""" + + success = "success" + + +class LokiQueryRangeResponseData(Base): + """Response data from the query range endpoint (streams only).""" + + result_type: LokiQueryRangeResponseResultType = Field(..., alias="resultType") + result: list[LokiQueryRangeResponseStream] + + +class LokiQueryRangeResponseResultType(StrEnum): + """Result type.""" + + streams = "streams" + + +class LokiQueryRangeResponseStream(Base): + """Loki log stream.""" + + stream: dict[str, str] + values: list[tuple[NanoTimestamp, str]] + + +class NanoTimestamp(RootModel[str]): + """Unix timestamp in nanoseconds.""" + + root: str = Field(..., pattern="\\d+") + + def get_value(self) -> int: + """Return the timestamp as a big integer.""" + return int(self.root) + + +class AmaltheaSessionStream(Base): + """Loki stream labels for logs extracted from an Amalthea session.""" + + container: str + pod: str + renku_io_launcher_id: str + renku_io_project_id: str | None = None + renku_io_safe_username: str + renku_io_session_type: str | None = None diff --git a/components/renku_data_services/persisted_logs/orm.py b/components/renku_data_services/persisted_logs/orm.py index a57f9d683..33feb64c8 100644 --- a/components/renku_data_services/persisted_logs/orm.py +++ b/components/renku_data_services/persisted_logs/orm.py @@ -1,4 +1,4 @@ -"""SQLAlchemy schemas for the data connectors database.""" +"""SQLAlchemy schemas for the peristed logs database.""" from __future__ import annotations @@ -65,7 +65,7 @@ class AmaltheaSessionLogsORM(BaseORM): container: Mapped[str] = mapped_column(nullable=False) """The container this log line belongs to.""" - timestamp: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + timestamp: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True, nullable=False) """The timestamp of the log line.""" log_line: Mapped[str] = mapped_column(nullable=False) diff --git a/pyproject.toml b/pyproject.toml index 807fbf23f..0d4f83f89 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ packages = [ { include = "renku_data_services/metrics", from = "components" }, { include = "renku_data_services/notifications", from = "components" }, { include = "renku_data_services/resource_usage", from = "components" }, + { include = "renku_data_services/persisted_logs", from = "components" }, ] [tool.poetry.dependencies] From 2b812b304a3e950cc71931057e3c6c29efefc2cb Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 14 Jul 2026 13:01:54 +0000 Subject: [PATCH 03/50] fix: import persisted_logs module --- projects/renku_data_service/pyproject.toml | 1 + projects/renku_data_tasks/pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/projects/renku_data_service/pyproject.toml b/projects/renku_data_service/pyproject.toml index 034163357..db85b6d59 100644 --- a/projects/renku_data_service/pyproject.toml +++ b/projects/renku_data_service/pyproject.toml @@ -44,6 +44,7 @@ packages = [ { include = "renku_data_services/metrics", from = "../../components" }, { include = "renku_data_services/capacity_reservation", from = "../../components" }, { include = "renku_data_services/resource_usage", from = "../../components" }, + { include = "renku_data_services/persisted_logs", from = "../../components" }, ] [tool.poetry.dependencies] diff --git a/projects/renku_data_tasks/pyproject.toml b/projects/renku_data_tasks/pyproject.toml index d816a4284..9a5dc783f 100644 --- a/projects/renku_data_tasks/pyproject.toml +++ b/projects/renku_data_tasks/pyproject.toml @@ -44,6 +44,7 @@ packages = [ { include = "renku_data_services/metrics", from = "../../components" }, { include = "renku_data_services/capacity_reservation", from = "../../components" }, { include = "renku_data_services/resource_usage", from = "../../components" }, + { include = "renku_data_services/persisted_logs", from = "../../components" }, ] [tool.poetry.dependencies] From 8046d0e93bbe74b436bcbb0fdc53ffdd9757a8b0 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 15 Jul 2026 09:31:41 +0000 Subject: [PATCH 04/50] wip: can produce unsaved log lines --- .../persisted_logs/collector.py | 38 +++++++++++++------ .../persisted_logs/collector_t.py | 9 ++++- .../persisted_logs/loki_api.py | 1 + .../persisted_logs/models.py | 25 ++++++++++++ 4 files changed, 61 insertions(+), 12 deletions(-) create mode 100644 components/renku_data_services/persisted_logs/models.py diff --git a/components/renku_data_services/persisted_logs/collector.py b/components/renku_data_services/persisted_logs/collector.py index 5262c46c2..c9499af2c 100644 --- a/components/renku_data_services/persisted_logs/collector.py +++ b/components/renku_data_services/persisted_logs/collector.py @@ -1,10 +1,13 @@ """Collector for gathering persisted logs.""" +from collections.abc import AsyncGenerator + import httpx from pydantic import ValidationError +from ulid import ULID from renku_data_services.app_config import logging -from renku_data_services.persisted_logs import loki_api +from renku_data_services.persisted_logs import loki_api, models from renku_data_services.persisted_logs.config import PersistedLogsConfig from renku_data_services.persisted_logs.constants import ( PERSISTED_LOGS_NAMESPACE_LABEL_KEY, @@ -23,8 +26,8 @@ def __init__(self, config: PersistedLogsConfig, client: httpx.AsyncClient) -> No self.client = client self.client.base_url = httpx.URL(config.loki_read_base_url) - async def get_amalthea_session_logs(self) -> None: - """Returns log streams from Amalthea sessions.""" + async def get_amalthea_session_logs(self) -> AsyncGenerator[models.UnsavedLogLine, None]: + """Fetches Amalthea session logs from Loki.""" params: dict[str, str] = dict() params["query"] = ( "{" @@ -38,12 +41,9 @@ async def get_amalthea_session_logs(self) -> None: res.raise_for_status() logger.info(res) result = loki_api.LokiQueryRangeResponse.model_validate_json(res.content) - # logger.info(result) - log_line_ids: set[str] = set() for entry in result.data.result: - # logger.info(entry) stream: loki_api.AmaltheaSessionStream | None = None try: stream = loki_api.AmaltheaSessionStream.model_validate(entry.stream) @@ -51,19 +51,35 @@ async def get_amalthea_session_logs(self) -> None: logger.warning(f"Skipping entry {entry.stream} because of validation error: {err}") continue + try: + launcher_id = ULID.from_str(stream.renku_io_launcher_id) + except ValueError as err: + logger.warning( + f"Skipping entry {entry.stream} because renku_io_launcher_id='{stream.renku_io_launcher_id}' " + f"is not a valid ULID: {err}" + ) + continue + logger.info(stream) for nano_ts, log_line in entry.values: - logger.info(nano_ts.get_value()) - logger.info(log_line) - log_line_id = f"{nano_ts.root}::{stream.container}::{stream.pod}" if log_line_id in log_line_ids: logger.info(f"Already saw: {log_line_id}") + continue log_line_ids.add(log_line_id) - - pass + yield models.UnsavedLogLine( + id=log_line_id, + run_id=stream.renku_io_launcher_id, # TODO: fix + user_id=stream.renku_io_safe_username, + launch_id=stream.renku_io_launcher_id, # TODO: fix + launcher_id=launcher_id, + submission_id=stream.renku_io_submission_id, + container=stream.container, + timestamp=nano_ts.get_value(), + log_line=log_line, + ) class PersistedLogsCollector: diff --git a/components/renku_data_services/persisted_logs/collector_t.py b/components/renku_data_services/persisted_logs/collector_t.py index a13967379..d3fd8d091 100644 --- a/components/renku_data_services/persisted_logs/collector_t.py +++ b/components/renku_data_services/persisted_logs/collector_t.py @@ -8,6 +8,8 @@ from renku_data_services.persisted_logs.collector import LokiLogReader from renku_data_services.persisted_logs.config import PersistedLogsConfig +logger = logging.getLogger(__name__) + def _main() -> None: logging.configure_logging() @@ -18,7 +20,12 @@ def _main() -> None: namespace="renku", ) reader = LokiLogReader(config, client=httpx.AsyncClient()) - asyncio.run(reader.get_amalthea_session_logs()) + + async def get_some_logs() -> None: + async for log_line in reader.get_amalthea_session_logs(): + logger.info(log_line) + + asyncio.run(get_some_logs()) if __name__ == "__main__": diff --git a/components/renku_data_services/persisted_logs/loki_api.py b/components/renku_data_services/persisted_logs/loki_api.py index a45e907b8..417a2f6ac 100644 --- a/components/renku_data_services/persisted_logs/loki_api.py +++ b/components/renku_data_services/persisted_logs/loki_api.py @@ -68,3 +68,4 @@ class AmaltheaSessionStream(Base): renku_io_project_id: str | None = None renku_io_safe_username: str renku_io_session_type: str | None = None + renku_io_submission_id: str | None = None diff --git a/components/renku_data_services/persisted_logs/models.py b/components/renku_data_services/persisted_logs/models.py new file mode 100644 index 000000000..b35a6df1a --- /dev/null +++ b/components/renku_data_services/persisted_logs/models.py @@ -0,0 +1,25 @@ +"""Models for persisted logs.""" + +from dataclasses import dataclass + +from ulid import ULID + + +@dataclass +class UnsavedLogLine: + """Represents an unsaved log line.""" + + id: str + """The ID of the log line. + + This is used to de-duplicate log lines. + """ + + run_id: str + user_id: str + launch_id: str + launcher_id: ULID + submission_id: str | None + container: str + timestamp: int + log_line: str From bd511ec306995e1107db9f768486e4efe00b963d Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 15 Jul 2026 12:47:33 +0000 Subject: [PATCH 05/50] wip: collect session logs --- .../renku_data_services/data_tasks/config.py | 4 + .../data_tasks/dependencies.py | 8 ++ .../data_tasks/task_defs.py | 10 ++ .../01180f797019_wip_redefine_tables.py | 101 ++++++++++++++++++ .../migrations/versions/2537a8e1df45_wip.py | 74 +++++++++++++ .../persisted_logs/collector.py | 73 ++++++++++++- .../renku_data_services/persisted_logs/db.py | 43 ++++++-- .../persisted_logs/models.py | 4 +- .../renku_data_services/persisted_logs/orm.py | 16 +-- 9 files changed, 309 insertions(+), 24 deletions(-) create mode 100644 components/renku_data_services/migrations/versions/01180f797019_wip_redefine_tables.py create mode 100644 components/renku_data_services/migrations/versions/2537a8e1df45_wip.py diff --git a/bases/renku_data_services/data_tasks/config.py b/bases/renku_data_services/data_tasks/config.py index 7d0934065..8d4b890af 100644 --- a/bases/renku_data_services/data_tasks/config.py +++ b/bases/renku_data_services/data_tasks/config.py @@ -8,6 +8,7 @@ from renku_data_services.app_config.config import KeycloakConfig from renku_data_services.authz.config import AuthzConfig from renku_data_services.db_config.config import DBConfig +from renku_data_services.persisted_logs.config import PersistedLogsConfig from renku_data_services.solr.solr_client import SolrClientConfig @@ -42,6 +43,7 @@ class Config: posthog: PosthogConfig authz: AuthzConfig keycloak: KeycloakConfig | None + persisted_logs: PersistedLogsConfig k8s_config_root: str dummy_stores: bool max_retry_wait_seconds: int @@ -77,6 +79,7 @@ def from_env(cls) -> Config: session_quota_alert_remaining_threshold = int(os.environ.get("SESSION_QUOTA_ALERT_REMAINING_THRESHOLD_P", 20)) session_quota_alert_critical = int(os.environ.get("SESSION_QUOTA_ALERT_CRITICAL_M", 10)) + k8s_namespace = os.environ.get("KUBERNETES_NAMESPACE", "default") k8s_config_root = os.environ.get("K8S_CONFIG_ROOT", "/secrets/kube_configs") enable_resource_request_tracking = os.environ.get("ENABLE_RESOURCE_REQUEST_TRACKING", "false").lower() == "true" @@ -91,6 +94,7 @@ def from_env(cls) -> Config: posthog=posthog_config, authz=authz, keycloak=keycloak, + persisted_logs=PersistedLogsConfig.from_env(namespace=k8s_namespace), k8s_config_root=k8s_config_root, tcp_host=tcp_host, tcp_port=tcp_port, diff --git a/bases/renku_data_services/data_tasks/dependencies.py b/bases/renku_data_services/data_tasks/dependencies.py index c7a9ac036..26d9f42f6 100644 --- a/bases/renku_data_services/data_tasks/dependencies.py +++ b/bases/renku_data_services/data_tasks/dependencies.py @@ -17,6 +17,7 @@ from renku_data_services.namespace.db import GroupRepository from renku_data_services.notebooks.constants import AMALTHEA_SESSION_GVK from renku_data_services.notifications.db import NotificationsRepository +from renku_data_services.persisted_logs.collector import PersistedLogsCollector from renku_data_services.project.db import ProjectRepository from renku_data_services.resource_usage.core import ( DefaultResourcesRequestRecorder, @@ -56,6 +57,7 @@ class DependencyManager: notifications_repo: NotificationsRepository resource_usage_service: ResourceUsageService resource_requests_repo: ResourceRequestsRepo + persisted_logs_collector: PersistedLogsCollector @classmethod def from_env(cls, cfg: Config | None = None) -> "DependencyManager": @@ -151,6 +153,11 @@ def from_env(cls, cfg: Config | None = None) -> "DependencyManager": realm=cfg.keycloak.realm, ) + persisted_logs_collector = PersistedLogsCollector.from_config( + config=cfg.persisted_logs, + session_maker=cfg.db.async_session_maker, + ) + return cls( config=cfg, search_updates_repo=search_updates_repo, @@ -167,4 +174,5 @@ def from_env(cls, cfg: Config | None = None) -> "DependencyManager": notifications_repo=notifications_repo, resource_usage_service=resource_usage_service, resource_requests_repo=resource_requests_repo, + persisted_logs_collector=persisted_logs_collector, ) diff --git a/bases/renku_data_services/data_tasks/task_defs.py b/bases/renku_data_services/data_tasks/task_defs.py index 903c2a977..4b33dcf99 100644 --- a/bases/renku_data_services/data_tasks/task_defs.py +++ b/bases/renku_data_services/data_tasks/task_defs.py @@ -577,6 +577,15 @@ async def monitor_session_quota_and_send_alerts(dm: DependencyManager) -> None: await asyncio.sleep(dm.config.session_quota_alert_check_interval_s) +async def collect_persisted_logs(dm: DependencyManager) -> None: + """Collect persisted logs from Loki.""" + # NOTE: just run once (testing) + try: + await dm.persisted_logs_collector.collect_persisted_logs() + except Exception as e: + logger.warning(f"Failed to collect persisted logs: {e}", exc_info=True) + + def all_tasks(dm: DependencyManager) -> TaskDefininions: """A dict of task factories to be managed in main.""" # Impl. note: We pass the entire config to the coroutines, because @@ -603,5 +612,6 @@ def all_tasks(dm: DependencyManager) -> TaskDefininions: "cleanup_orphaned_capacity_reservations": lambda: cleanup_orphaned_capacity_reservations(dm), "record_resource_requests": lambda: record_resource_requests(dm), "monitor_session_quota_and_send_alerts": lambda: monitor_session_quota_and_send_alerts(dm), + "collect_persisted_logs": lambda: collect_persisted_logs(dm), } ) diff --git a/components/renku_data_services/migrations/versions/01180f797019_wip_redefine_tables.py b/components/renku_data_services/migrations/versions/01180f797019_wip_redefine_tables.py new file mode 100644 index 000000000..f884dac00 --- /dev/null +++ b/components/renku_data_services/migrations/versions/01180f797019_wip_redefine_tables.py @@ -0,0 +1,101 @@ +"""wip: redefine tables + +Revision ID: 01180f797019 +Revises: 2537a8e1df45 +Create Date: 2026-07-15 12:50:59.650960 + +""" + +import sqlalchemy as sa +from alembic import op + +from renku_data_services.utils.sqlalchemy import ULIDType + +# revision identifiers, used by Alembic. +revision = "01180f797019" +down_revision = "2537a8e1df45" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "session_runs", + sa.Column("id", ULIDType(), nullable=False), + sa.Column("user_id", sa.String(length=36), nullable=False), + sa.Column("launch_id", sa.String(), nullable=False), + sa.Column("launcher_id", ULIDType(), nullable=False), + sa.Column("submission_id", sa.String(), nullable=True), + sa.ForeignKeyConstraint( + ["launcher_id"], + ["sessions.launchers.id"], + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.users.keycloak_id"], + ), + sa.PrimaryKeyConstraint("id"), + schema="persisted_logs", + ) + op.create_index( + op.f("ix_persisted_logs_session_runs_launcher_id"), + "session_runs", + ["launcher_id"], + unique=False, + schema="persisted_logs", + ) + op.create_index( + op.f("ix_persisted_logs_session_runs_user_id"), + "session_runs", + ["user_id"], + unique=False, + schema="persisted_logs", + ) + op.create_table( + "amalthea_session_logs", + sa.Column("id", sa.String(), nullable=False), + sa.Column("run_id", ULIDType(), nullable=False), + sa.Column("container", sa.String(), nullable=False), + sa.Column("timestamp", sa.BigInteger(), nullable=False), + sa.Column("log_line", sa.String(), nullable=False), + sa.ForeignKeyConstraint(["run_id"], ["persisted_logs.session_runs.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + schema="persisted_logs", + ) + op.create_index( + op.f("ix_persisted_logs_amalthea_session_logs_run_id"), + "amalthea_session_logs", + ["run_id"], + unique=False, + schema="persisted_logs", + ) + op.create_index( + op.f("ix_persisted_logs_amalthea_session_logs_timestamp"), + "amalthea_session_logs", + ["timestamp"], + unique=False, + schema="persisted_logs", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + op.f("ix_persisted_logs_amalthea_session_logs_timestamp"), + table_name="amalthea_session_logs", + schema="persisted_logs", + ) + op.drop_index( + op.f("ix_persisted_logs_amalthea_session_logs_run_id"), + table_name="amalthea_session_logs", + schema="persisted_logs", + ) + op.drop_table("amalthea_session_logs", schema="persisted_logs") + op.drop_index(op.f("ix_persisted_logs_session_runs_user_id"), table_name="session_runs", schema="persisted_logs") + op.drop_index( + op.f("ix_persisted_logs_session_runs_launcher_id"), table_name="session_runs", schema="persisted_logs" + ) + op.drop_table("session_runs", schema="persisted_logs") + # ### end Alembic commands ### diff --git a/components/renku_data_services/migrations/versions/2537a8e1df45_wip.py b/components/renku_data_services/migrations/versions/2537a8e1df45_wip.py new file mode 100644 index 000000000..5ee54f5c9 --- /dev/null +++ b/components/renku_data_services/migrations/versions/2537a8e1df45_wip.py @@ -0,0 +1,74 @@ +"""wip + +Revision ID: 2537a8e1df45 +Revises: eadfb5e7e7cb +Create Date: 2026-07-15 11:42:06.232593 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "2537a8e1df45" +down_revision = "eadfb5e7e7cb" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.drop_index( + "ix_persisted_logs_amalthea_session_logs_run_id", table_name="amalthea_session_logs", schema="persisted_logs" + ) + op.drop_table("amalthea_session_logs", schema="persisted_logs") + op.drop_index("ix_persisted_logs_session_runs_launcher_id", table_name="session_runs", schema="persisted_logs") + op.drop_index("ix_persisted_logs_session_runs_user_id", table_name="session_runs", schema="persisted_logs") + op.drop_table("session_runs", schema="persisted_logs") + + +def downgrade() -> None: + op.create_table( + "session_runs", + sa.Column("id", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column("user_id", sa.VARCHAR(length=36), autoincrement=False, nullable=False), + sa.Column("launch_id", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column("launcher_id", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column("submission_id", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column("first_log", postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False), + sa.Column("last_log", postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint(["launcher_id"], ["sessions.launchers.id"], name="session_runs_launcher_id_fkey"), + sa.ForeignKeyConstraint(["user_id"], ["users.users.keycloak_id"], name="session_runs_user_id_fkey"), + sa.PrimaryKeyConstraint("id", name="session_runs_pkey"), + schema="persisted_logs", + ) + op.create_index( + "ix_persisted_logs_session_runs_user_id", "session_runs", ["user_id"], unique=False, schema="persisted_logs" + ) + op.create_index( + "ix_persisted_logs_session_runs_launcher_id", + "session_runs", + ["launcher_id"], + unique=False, + schema="persisted_logs", + ) + op.create_table( + "amalthea_session_logs", + sa.Column("id", sa.VARCHAR(), server_default=sa.text("generate_ulid()"), autoincrement=False, nullable=False), + sa.Column("run_id", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column("container", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column("timestamp", postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False), + sa.Column("log_line", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint( + ["run_id"], ["persisted_logs.session_runs.id"], name="amalthea_session_logs_run_id_fkey", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id", name="amalthea_session_logs_pkey"), + schema="persisted_logs", + ) + op.create_index( + "ix_persisted_logs_amalthea_session_logs_run_id", + "amalthea_session_logs", + ["run_id"], + unique=False, + schema="persisted_logs", + ) diff --git a/components/renku_data_services/persisted_logs/collector.py b/components/renku_data_services/persisted_logs/collector.py index c9499af2c..8ed15a1b7 100644 --- a/components/renku_data_services/persisted_logs/collector.py +++ b/components/renku_data_services/persisted_logs/collector.py @@ -1,9 +1,11 @@ """Collector for gathering persisted logs.""" -from collections.abc import AsyncGenerator +from abc import abstractmethod +from collections.abc import AsyncIterator, Callable import httpx from pydantic import ValidationError +from sqlalchemy.ext.asyncio import AsyncSession from ulid import ULID from renku_data_services.app_config import logging @@ -14,6 +16,7 @@ PERSISTED_LOGS_SESSIONS_LABEL_KEY, PERSISTED_LOGS_SESSIONS_LABEL_VALUE, ) +from renku_data_services.persisted_logs.db import AmaltheaSessionPersistedLogsRepository logger = logging.getLogger(__name__) @@ -26,7 +29,7 @@ def __init__(self, config: PersistedLogsConfig, client: httpx.AsyncClient) -> No self.client = client self.client.base_url = httpx.URL(config.loki_read_base_url) - async def get_amalthea_session_logs(self) -> AsyncGenerator[models.UnsavedLogLine, None]: + async def get_amalthea_session_logs(self) -> AsyncIterator[models.UnsavedLogLine]: """Fetches Amalthea session logs from Loki.""" params: dict[str, str] = dict() params["query"] = ( @@ -71,7 +74,7 @@ async def get_amalthea_session_logs(self) -> AsyncGenerator[models.UnsavedLogLin log_line_ids.add(log_line_id) yield models.UnsavedLogLine( id=log_line_id, - run_id=stream.renku_io_launcher_id, # TODO: fix + run_id=launcher_id, # TODO: fix user_id=stream.renku_io_safe_username, launch_id=stream.renku_io_launcher_id, # TODO: fix launcher_id=launcher_id, @@ -83,6 +86,68 @@ async def get_amalthea_session_logs(self) -> AsyncGenerator[models.UnsavedLogLin class PersistedLogsCollector: + """Abstract class for gathering persisted logs.""" + + @abstractmethod + async def collect_persisted_logs(self) -> None: + """Collect persisted logs from Amalthea sessions and image builds.""" + ... + + @staticmethod + def from_config( + config: PersistedLogsConfig, + session_maker: Callable[..., AsyncSession], + http_client: httpx.AsyncClient | None = None, + ) -> "PersistedLogsCollector": + """Construct a PersistedLogsCollector from a configuration object.""" + if config.enabled: + if http_client is None: + http_client = httpx.AsyncClient() + reader = LokiLogReader(config=config, client=http_client) + return DefaultPersistedLogsCollector( + session_maker=session_maker, + reader=reader, + session_logs_repo=AmaltheaSessionPersistedLogsRepository(), + ) + return NoopPersistedLogsCollector() + + +class NoopPersistedLogsCollector(PersistedLogsCollector): + """No-op collector.""" + + async def collect_persisted_logs(self) -> None: + """Collect persisted logs from Amalthea sessions and image builds.""" + return None + + +class DefaultPersistedLogsCollector(PersistedLogsCollector): """Collector for gathering persisted logs.""" - pass + def __init__( + self, + session_maker: Callable[..., AsyncSession], + reader: LokiLogReader, + session_logs_repo: AmaltheaSessionPersistedLogsRepository, + ) -> None: + self.session_maker = session_maker + self.reader = reader + self.session_logs_repo = session_logs_repo + + async def collect_persisted_logs(self) -> None: + """Collect persisted logs from Amalthea sessions and image builds.""" + await self.collect_sessions_persisted_logs() + return None + + async def collect_sessions_persisted_logs(self) -> None: + """Collect persisted logs from Amalthea sessions.""" + + logs_stream = self.reader.get_amalthea_session_logs() + + async with self.session_maker() as session: + async with session.begin(): + await self.session_logs_repo.insert_session_logs(session=session, logs_stream=logs_stream) + async with session.begin(): + ts = await self.session_logs_repo.get_latest_log_timestamp(session=session) + logger.info(f"Latest session log timestamp: {ts}") + + return None diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index cbc4e9c3c..ce93594de 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -1,20 +1,18 @@ """Adapters for persisted logs database classes.""" -from datetime import datetime +from collections.abc import AsyncIterator from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from renku_data_services.persisted_logs import models from renku_data_services.persisted_logs import orm as schemas class AmaltheaSessionPersistedLogsRepository: """Repository for persisted logs of Amalthea sessions.""" - # loki: 1783946464779229935 <- nano - # python: 1783948286.9942 <- seconds - - async def get_latest_log_timestamp(self, session: AsyncSession) -> datetime | None: + async def get_latest_log_timestamp(self, session: AsyncSession) -> int | None: """Returns the latest log timestamp.""" stmt = ( select(schemas.AmaltheaSessionLogsORM.timestamp) @@ -26,4 +24,37 @@ async def get_latest_log_timestamp(self, session: AsyncSession) -> datetime | No timestamp = res.one_or_none() return timestamp - pass + async def insert_session_logs( + self, session: AsyncSession, logs_stream: AsyncIterator[models.UnsavedLogLine] + ) -> None: + """Insert sessions logs into the persisted logs database.""" + async for log in logs_stream: + existing_log_res = await session.scalars(select(schemas.AmaltheaSessionLogsORM.id)) + existing_log_orm = existing_log_res.one_or_none() + if existing_log_orm: + continue + + session_run_res = await session.scalars( + select(schemas.SessionRunsORM).where(schemas.SessionRunsORM.id == log.run_id) + ) + session_run_orm = session_run_res.one_or_none() + if session_run_orm is None: + session_run_orm = schemas.SessionRunsORM( + id=log.run_id, + user_id=log.user_id, + launch_id=log.launch_id, + launcher_id=log.launcher_id, + submission_id=log.submission_id, + ) + session.add(session_run_orm) + + log_orm = schemas.AmaltheaSessionLogsORM( + id=log.id, + run_id=log.run_id, + container=log.container, + timestamp=log.timestamp, + log_line=log.log_line, + ) + session.add(log_orm) + + await session.flush() diff --git a/components/renku_data_services/persisted_logs/models.py b/components/renku_data_services/persisted_logs/models.py index b35a6df1a..00b4cebf7 100644 --- a/components/renku_data_services/persisted_logs/models.py +++ b/components/renku_data_services/persisted_logs/models.py @@ -5,7 +5,7 @@ from ulid import ULID -@dataclass +@dataclass(eq=True, frozen=True, kw_only=True) class UnsavedLogLine: """Represents an unsaved log line.""" @@ -15,7 +15,7 @@ class UnsavedLogLine: This is used to de-duplicate log lines. """ - run_id: str + run_id: ULID user_id: str launch_id: str launcher_id: ULID diff --git a/components/renku_data_services/persisted_logs/orm.py b/components/renku_data_services/persisted_logs/orm.py index 33feb64c8..914068f49 100644 --- a/components/renku_data_services/persisted_logs/orm.py +++ b/components/renku_data_services/persisted_logs/orm.py @@ -2,9 +2,7 @@ from __future__ import annotations -from datetime import datetime - -from sqlalchemy import DateTime, ForeignKey, MetaData +from sqlalchemy import BigInteger, ForeignKey, MetaData from sqlalchemy.orm import DeclarativeBase, Mapped, MappedAsDataclass, mapped_column, relationship from ulid import ULID @@ -41,19 +39,13 @@ class SessionRunsORM(BaseORM): submission_id: Mapped[str | None] = mapped_column(nullable=True) """The submission ID, if the session run corresponds to an offline job.""" - first_log: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) - """The timestamp of the first log line.""" - - last_log: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) - """The timestamp of the last log line.""" - class AmaltheaSessionLogsORM(BaseORM): """A log line from an Amalthea session.""" __tablename__ = "amalthea_session_logs" - id: Mapped[ULID] = mapped_column("id", ULIDType, primary_key=True, default_factory=lambda: str(ULID()), init=False) + id: Mapped[str] = mapped_column("id", primary_key=True, nullable=False) """ID of the log line.""" run_id: Mapped[ULID] = mapped_column(ForeignKey(SessionRunsORM.id, ondelete="CASCADE"), index=True, nullable=False) @@ -65,8 +57,8 @@ class AmaltheaSessionLogsORM(BaseORM): container: Mapped[str] = mapped_column(nullable=False) """The container this log line belongs to.""" - timestamp: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True, nullable=False) - """The timestamp of the log line.""" + timestamp: Mapped[int] = mapped_column(BigInteger, index=True, nullable=False) + """The timestamp of the log line (nanosecond timestamp).""" log_line: Mapped[str] = mapped_column(nullable=False) """The contents of the log line.""" From 702c72f85a7711b10e9673e69f8bc9080e5f6bb6 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 15 Jul 2026 13:05:09 +0000 Subject: [PATCH 06/50] fix config --- bases/renku_data_services/data_tasks/config.py | 2 +- components/renku_data_services/persisted_logs/config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bases/renku_data_services/data_tasks/config.py b/bases/renku_data_services/data_tasks/config.py index 8d4b890af..7c0f4d03a 100644 --- a/bases/renku_data_services/data_tasks/config.py +++ b/bases/renku_data_services/data_tasks/config.py @@ -79,7 +79,7 @@ def from_env(cls) -> Config: session_quota_alert_remaining_threshold = int(os.environ.get("SESSION_QUOTA_ALERT_REMAINING_THRESHOLD_P", 20)) session_quota_alert_critical = int(os.environ.get("SESSION_QUOTA_ALERT_CRITICAL_M", 10)) - k8s_namespace = os.environ.get("KUBERNETES_NAMESPACE", "default") + k8s_namespace = os.environ.get("KUBERNETES_NAMESPACE", os.environ.get("K8S_NAMESPACE", "default")) k8s_config_root = os.environ.get("K8S_CONFIG_ROOT", "/secrets/kube_configs") enable_resource_request_tracking = os.environ.get("ENABLE_RESOURCE_REQUEST_TRACKING", "false").lower() == "true" diff --git a/components/renku_data_services/persisted_logs/config.py b/components/renku_data_services/persisted_logs/config.py index 5f1d4bffe..2acb7547d 100644 --- a/components/renku_data_services/persisted_logs/config.py +++ b/components/renku_data_services/persisted_logs/config.py @@ -21,6 +21,6 @@ def from_env(cls, namespace: str) -> "PersistedLogsConfig": return cls( enabled=True, - loki_read_base_url="http://loki-read.monitoring.cluster.local.svc:3100/", + loki_read_base_url="http://loki-read.monitoring.svc.cluster.local:3100/", namespace=namespace, ) From 42c83e63346238e4d517cfe0a7faa4cdd784f738 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 15 Jul 2026 13:17:27 +0000 Subject: [PATCH 07/50] fix get_latest_log_timestamp() --- components/renku_data_services/persisted_logs/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index ce93594de..224a2a8c0 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -17,7 +17,7 @@ async def get_latest_log_timestamp(self, session: AsyncSession) -> int | None: stmt = ( select(schemas.AmaltheaSessionLogsORM.timestamp) .select_from(schemas.AmaltheaSessionLogsORM) - .order_by(schemas.AmaltheaSessionLogsORM.timestamp.desc) + .order_by(schemas.AmaltheaSessionLogsORM.timestamp.desc()) .limit(1) ) res = await session.scalars(stmt) From 9437729119cd33888e5394e89715f2246886946d Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 15 Jul 2026 13:37:26 +0000 Subject: [PATCH 08/50] fixes --- components/renku_data_services/persisted_logs/db.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index 224a2a8c0..bde52bfbc 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -29,7 +29,9 @@ async def insert_session_logs( ) -> None: """Insert sessions logs into the persisted logs database.""" async for log in logs_stream: - existing_log_res = await session.scalars(select(schemas.AmaltheaSessionLogsORM.id)) + existing_log_res = await session.scalars( + select(schemas.AmaltheaSessionLogsORM.id).where(schemas.AmaltheaSessionLogsORM.id == log.id) + ) existing_log_orm = existing_log_res.one_or_none() if existing_log_orm: continue @@ -47,6 +49,7 @@ async def insert_session_logs( submission_id=log.submission_id, ) session.add(session_run_orm) + await session.flush() log_orm = schemas.AmaltheaSessionLogsORM( id=log.id, @@ -56,5 +59,4 @@ async def insert_session_logs( log_line=log.log_line, ) session.add(log_orm) - - await session.flush() + await session.flush() From 7024719f317fa3491bf79f961068cb27baa4898f Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 15 Jul 2026 13:50:49 +0000 Subject: [PATCH 09/50] logs --- components/renku_data_services/persisted_logs/db.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index bde52bfbc..b7bd9a77c 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -5,9 +5,12 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from renku_data_services.app_config import logging from renku_data_services.persisted_logs import models from renku_data_services.persisted_logs import orm as schemas +logger = logging.getLogger(__name__) + class AmaltheaSessionPersistedLogsRepository: """Repository for persisted logs of Amalthea sessions.""" @@ -34,8 +37,11 @@ async def insert_session_logs( ) existing_log_orm = existing_log_res.one_or_none() if existing_log_orm: + logger.info(f"Skipping log line {log.id}") continue + logger.info(f"Processing log line {log}") + session_run_res = await session.scalars( select(schemas.SessionRunsORM).where(schemas.SessionRunsORM.id == log.run_id) ) From 32626637f1c39104bc745d94e97bfbdaa7a1e26e Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 16 Jul 2026 07:05:59 +0000 Subject: [PATCH 10/50] enable loop --- bases/renku_data_services/data_tasks/task_defs.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bases/renku_data_services/data_tasks/task_defs.py b/bases/renku_data_services/data_tasks/task_defs.py index 4b33dcf99..9fb6f785b 100644 --- a/bases/renku_data_services/data_tasks/task_defs.py +++ b/bases/renku_data_services/data_tasks/task_defs.py @@ -579,11 +579,13 @@ async def monitor_session_quota_and_send_alerts(dm: DependencyManager) -> None: async def collect_persisted_logs(dm: DependencyManager) -> None: """Collect persisted logs from Loki.""" - # NOTE: just run once (testing) - try: - await dm.persisted_logs_collector.collect_persisted_logs() - except Exception as e: - logger.warning(f"Failed to collect persisted logs: {e}", exc_info=True) + while True: + try: + await dm.persisted_logs_collector.collect_persisted_logs() + except Exception as e: + logger.warning(f"Failed to collect persisted logs: {e}", exc_info=True) + else: + await asyncio.sleep(1) def all_tasks(dm: DependencyManager) -> TaskDefininions: From 47a57d718751c24b7c676df26632b64e4f463be0 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 16 Jul 2026 08:13:45 +0000 Subject: [PATCH 11/50] query logs only from the last one --- .../persisted_logs/collector.py | 37 ++++++++++++++++--- .../persisted_logs/constants.py | 3 ++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/components/renku_data_services/persisted_logs/collector.py b/components/renku_data_services/persisted_logs/collector.py index 8ed15a1b7..4ccc717d0 100644 --- a/components/renku_data_services/persisted_logs/collector.py +++ b/components/renku_data_services/persisted_logs/collector.py @@ -2,6 +2,7 @@ from abc import abstractmethod from collections.abc import AsyncIterator, Callable +from datetime import UTC, datetime, timedelta import httpx from pydantic import ValidationError @@ -12,6 +13,7 @@ from renku_data_services.persisted_logs import loki_api, models from renku_data_services.persisted_logs.config import PersistedLogsConfig from renku_data_services.persisted_logs.constants import ( + ONE_SECOND_IN_NANOS, PERSISTED_LOGS_NAMESPACE_LABEL_KEY, PERSISTED_LOGS_SESSIONS_LABEL_KEY, PERSISTED_LOGS_SESSIONS_LABEL_VALUE, @@ -29,9 +31,19 @@ def __init__(self, config: PersistedLogsConfig, client: httpx.AsyncClient) -> No self.client = client self.client.base_url = httpx.URL(config.loki_read_base_url) - async def get_amalthea_session_logs(self) -> AsyncIterator[models.UnsavedLogLine]: - """Fetches Amalthea session logs from Loki.""" - params: dict[str, str] = dict() + async def get_amalthea_session_logs( + self, limit: int = 1000, start: int | None = None, end: int | None = None + ) -> AsyncIterator[models.UnsavedLogLine]: + """Fetches Amalthea session logs from Loki. + + Parameters: + - limit: max number of entries to return + - start: start timestamp as a Unix nano timestamp + - end: end timestamp as a Unix nano timestamp + + See also https://grafana.com/docs/loki/latest/reference/loki-http-api/#query-logs-within-a-range-of-time + """ + params: dict[str, str | int] = dict() params["query"] = ( "{" f'{PERSISTED_LOGS_SESSIONS_LABEL_KEY}="{PERSISTED_LOGS_SESSIONS_LABEL_VALUE}",' @@ -39,6 +51,11 @@ async def get_amalthea_session_logs(self) -> AsyncIterator[models.UnsavedLogLine "}" ) params["direction"] = "forward" + params["limit"] = limit + if start: + params["start"] = str(start) + if end: + params["end"] = str(end) logger.info(params) res = await self.client.get("loki/api/v1/query_range", params=params) res.raise_for_status() @@ -141,9 +158,13 @@ async def collect_persisted_logs(self) -> None: async def collect_sessions_persisted_logs(self) -> None: """Collect persisted logs from Amalthea sessions.""" - logs_stream = self.reader.get_amalthea_session_logs() - async with self.session_maker() as session: + async with session.begin(): + ts = await self.session_logs_repo.get_latest_log_timestamp(session=session) + start = _one_hour_ago_in_nanos() + if ts is not None and ts > start: + start = ts - ONE_SECOND_IN_NANOS + logs_stream = self.reader.get_amalthea_session_logs(start=start) async with session.begin(): await self.session_logs_repo.insert_session_logs(session=session, logs_stream=logs_stream) async with session.begin(): @@ -151,3 +172,9 @@ async def collect_sessions_persisted_logs(self) -> None: logger.info(f"Latest session log timestamp: {ts}") return None + + +def _one_hour_ago_in_nanos() -> int: + """Returns the Unix nano timestamp corresponding to one hour ago.""" + dt = datetime.now(tz=UTC) - timedelta(hours=1) + return int(dt.timestamp() * 1e6) * 1000 diff --git a/components/renku_data_services/persisted_logs/constants.py b/components/renku_data_services/persisted_logs/constants.py index 898b05142..85b1e76cc 100644 --- a/components/renku_data_services/persisted_logs/constants.py +++ b/components/renku_data_services/persisted_logs/constants.py @@ -10,3 +10,6 @@ PERSISTED_LOGS_NAMESPACE_LABEL_KEY: Final[str] = "namespace" """The loki label key to select logs streams from a specific kubernetes namespace.""" + +ONE_SECOND_IN_NANOS: Final[int] = int(1e9) +"""One second as nanoseconds (for Loki).""" From 92d9cbebf33b6c996eb1fee3e6773c84b04aac2f Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 16 Jul 2026 11:53:09 +0000 Subject: [PATCH 12/50] fix: use start = ts --- components/renku_data_services/persisted_logs/collector.py | 4 ++-- components/renku_data_services/persisted_logs/constants.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/renku_data_services/persisted_logs/collector.py b/components/renku_data_services/persisted_logs/collector.py index 4ccc717d0..9ba5fa3cd 100644 --- a/components/renku_data_services/persisted_logs/collector.py +++ b/components/renku_data_services/persisted_logs/collector.py @@ -13,7 +13,6 @@ from renku_data_services.persisted_logs import loki_api, models from renku_data_services.persisted_logs.config import PersistedLogsConfig from renku_data_services.persisted_logs.constants import ( - ONE_SECOND_IN_NANOS, PERSISTED_LOGS_NAMESPACE_LABEL_KEY, PERSISTED_LOGS_SESSIONS_LABEL_KEY, PERSISTED_LOGS_SESSIONS_LABEL_VALUE, @@ -163,7 +162,8 @@ async def collect_sessions_persisted_logs(self) -> None: ts = await self.session_logs_repo.get_latest_log_timestamp(session=session) start = _one_hour_ago_in_nanos() if ts is not None and ts > start: - start = ts - ONE_SECOND_IN_NANOS + # start = ts - ONE_SECOND_IN_NANOS + start = ts logs_stream = self.reader.get_amalthea_session_logs(start=start) async with session.begin(): await self.session_logs_repo.insert_session_logs(session=session, logs_stream=logs_stream) diff --git a/components/renku_data_services/persisted_logs/constants.py b/components/renku_data_services/persisted_logs/constants.py index 85b1e76cc..cd907b075 100644 --- a/components/renku_data_services/persisted_logs/constants.py +++ b/components/renku_data_services/persisted_logs/constants.py @@ -11,5 +11,5 @@ PERSISTED_LOGS_NAMESPACE_LABEL_KEY: Final[str] = "namespace" """The loki label key to select logs streams from a specific kubernetes namespace.""" -ONE_SECOND_IN_NANOS: Final[int] = int(1e9) -"""One second as nanoseconds (for Loki).""" +# ONE_SECOND_IN_NANOS: Final[int] = int(1e9) +# """One second as nanoseconds (for Loki).""" From cdcaf0efca1cc8a3bce5e6bab26216939bc7301d Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 16 Jul 2026 12:46:22 +0000 Subject: [PATCH 13/50] handle log bursts (?) --- .../persisted_logs/collector.py | 18 +++++++++++++----- .../persisted_logs/constants.py | 4 ++-- .../renku_data_services/persisted_logs/db.py | 9 ++++++++- .../persisted_logs/models.py | 8 ++++++++ 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/components/renku_data_services/persisted_logs/collector.py b/components/renku_data_services/persisted_logs/collector.py index 9ba5fa3cd..82b2f8504 100644 --- a/components/renku_data_services/persisted_logs/collector.py +++ b/components/renku_data_services/persisted_logs/collector.py @@ -13,6 +13,7 @@ from renku_data_services.persisted_logs import loki_api, models from renku_data_services.persisted_logs.config import PersistedLogsConfig from renku_data_services.persisted_logs.constants import ( + ONE_SECOND_IN_NANOS, PERSISTED_LOGS_NAMESPACE_LABEL_KEY, PERSISTED_LOGS_SESSIONS_LABEL_KEY, PERSISTED_LOGS_SESSIONS_LABEL_VALUE, @@ -162,11 +163,18 @@ async def collect_sessions_persisted_logs(self) -> None: ts = await self.session_logs_repo.get_latest_log_timestamp(session=session) start = _one_hour_ago_in_nanos() if ts is not None and ts > start: - # start = ts - ONE_SECOND_IN_NANOS - start = ts - logs_stream = self.reader.get_amalthea_session_logs(start=start) - async with session.begin(): - await self.session_logs_repo.insert_session_logs(session=session, logs_stream=logs_stream) + start = ts - ONE_SECOND_IN_NANOS + + # Loop to collect all logs, including late entries + has_more = True + current_start = start + while has_more: + logs_stream = self.reader.get_amalthea_session_logs(start=current_start) + async with session.begin(): + result = await self.session_logs_repo.insert_session_logs(session=session, logs_stream=logs_stream) + current_start = result.last_timestamp + 1 + has_more = result.log_count > 1 + async with session.begin(): ts = await self.session_logs_repo.get_latest_log_timestamp(session=session) logger.info(f"Latest session log timestamp: {ts}") diff --git a/components/renku_data_services/persisted_logs/constants.py b/components/renku_data_services/persisted_logs/constants.py index cd907b075..a7dd8af1c 100644 --- a/components/renku_data_services/persisted_logs/constants.py +++ b/components/renku_data_services/persisted_logs/constants.py @@ -11,5 +11,5 @@ PERSISTED_LOGS_NAMESPACE_LABEL_KEY: Final[str] = "namespace" """The loki label key to select logs streams from a specific kubernetes namespace.""" -# ONE_SECOND_IN_NANOS: Final[int] = int(1e9) -# """One second as nanoseconds (for Loki).""" +ONE_SECOND_IN_NANOS: Final[int] = 1_000_000_000 +"""One second as nanoseconds (for Loki).""" diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index b7bd9a77c..0456fc3ea 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -29,9 +29,15 @@ async def get_latest_log_timestamp(self, session: AsyncSession) -> int | None: async def insert_session_logs( self, session: AsyncSession, logs_stream: AsyncIterator[models.UnsavedLogLine] - ) -> None: + ) -> models.InsertLogsResult: """Insert sessions logs into the persisted logs database.""" + log_count = 0 + last_timestamp = 0 async for log in logs_stream: + log_count += 1 + if log.timestamp > last_timestamp: + last_timestamp = log.timestamp + existing_log_res = await session.scalars( select(schemas.AmaltheaSessionLogsORM.id).where(schemas.AmaltheaSessionLogsORM.id == log.id) ) @@ -66,3 +72,4 @@ async def insert_session_logs( ) session.add(log_orm) await session.flush() + return models.InsertLogsResult(log_count=log_count, last_timestamp=last_timestamp) diff --git a/components/renku_data_services/persisted_logs/models.py b/components/renku_data_services/persisted_logs/models.py index 00b4cebf7..b8d4f6e55 100644 --- a/components/renku_data_services/persisted_logs/models.py +++ b/components/renku_data_services/persisted_logs/models.py @@ -23,3 +23,11 @@ class UnsavedLogLine: container: str timestamp: int log_line: str + + +@dataclass(eq=True, frozen=True, kw_only=True) +class InsertLogsResult: + """Result of inserting a log stream in the database.""" + + log_count: int + last_timestamp: int From 6c997ca26b37c05fdc141d6397ea4afa919473b3 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Mon, 20 Jul 2026 12:37:49 +0000 Subject: [PATCH 14/50] wip: get persisted logs from the API --- Makefile | 1 + bases/renku_data_services/data_api/app.py | 9 ++ .../data_api/dependencies.py | 5 + .../persisted_logs/api.spec.yaml | 104 ++++++++++++++++++ .../persisted_logs/apispec.py | 49 +++++++++ .../persisted_logs/apispec_base.py | 12 ++ .../persisted_logs/blueprints.py | 34 ++++++ .../renku_data_services/persisted_logs/db.py | 79 ++++++++++++- 8 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 components/renku_data_services/persisted_logs/api.spec.yaml create mode 100644 components/renku_data_services/persisted_logs/apispec.py create mode 100644 components/renku_data_services/persisted_logs/apispec_base.py create mode 100644 components/renku_data_services/persisted_logs/blueprints.py diff --git a/Makefile b/Makefile index d50c13841..3e607162f 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,7 @@ API_SPECS := \ components/renku_data_services/notifications/apispec.py \ components/renku_data_services/capacity_reservation/apispec.py \ components/renku_data_services/resource_usage/apispec.py \ + components/renku_data_services/persisted_logs/apispec.py \ components/renku_data_services/authn/api/apispec.py schemas: ${API_SPECS} ## Generate pydantic classes from apispec yaml files diff --git a/bases/renku_data_services/data_api/app.py b/bases/renku_data_services/data_api/app.py index e260a71da..bfe221f4f 100644 --- a/bases/renku_data_services/data_api/app.py +++ b/bases/renku_data_services/data_api/app.py @@ -30,6 +30,7 @@ from renku_data_services.namespace.blueprints import GroupsBP from renku_data_services.notebooks.blueprints import NotebooksNewBP from renku_data_services.notifications.blueprints import NotificationsBP +from renku_data_services.persisted_logs.blueprints import PersistedLogsBP from renku_data_services.platform.blueprints import PlatformConfigBP, PlatformUrlRedirectBP from renku_data_services.project.blueprints import ProjectsBP, ProjectSessionSecretBP from renku_data_services.repositories.blueprints import RepositoriesBP @@ -307,6 +308,13 @@ def register_all_handlers(app: Sanic, dm: DependencyManager) -> Sanic: authenticator=dm.authenticator, rp_repo=dm.rp_repo, ) + persisted_logs = PersistedLogsBP( + name="persisted_logs", + url_prefix=url_prefix, + session_logs_repo=dm.session_logs_repo, + authenticator=dm.authenticator, + session_maker=dm.config.db.async_session_maker, + ) internal_authentication = InternalAuthenticationBP( name="internal_authentication", url_prefix=url_prefix, @@ -345,6 +353,7 @@ def register_all_handlers(app: Sanic, dm: DependencyManager) -> Sanic: notifications.blueprint(), capacity_reservation.blueprint(), resource_usage.blueprint(), + persisted_logs.blueprint(), internal_authentication.blueprint(), ] ) diff --git a/bases/renku_data_services/data_api/dependencies.py b/bases/renku_data_services/data_api/dependencies.py index 16447f714..b0353c052 100644 --- a/bases/renku_data_services/data_api/dependencies.py +++ b/bases/renku_data_services/data_api/dependencies.py @@ -60,6 +60,7 @@ from renku_data_services.notebooks.data_sources import DataSourceRepository from renku_data_services.notebooks.image_check import ImageCheckRepository from renku_data_services.notifications.db import NotificationsRepository +from renku_data_services.persisted_logs.db import AmaltheaSessionPersistedLogsReadRepository from renku_data_services.platform.db import PlatformRepository, UrlRedirectRepository from renku_data_services.project.db import ( ProjectMemberRepository, @@ -170,6 +171,7 @@ class DependencyManager: occurrence_repo: OccurrenceRepository resource_requests_repo: ResourceRequestsRepo resource_usage_service: ResourceUsageService + session_logs_repo: AmaltheaSessionPersistedLogsReadRepository zenodo_client: ZenodoAPIClient job_client: DepositUploadJobClient secret_client: K8sSecretClient @@ -205,6 +207,7 @@ def load_apispec() -> dict[str, Any]: renku_data_services.notifications.__file__, renku_data_services.capacity_reservation.__file__, renku_data_services.resource_usage.__file__, + renku_data_services.persisted_logs.__file__, renku_data_services.authn.api.__file__, ] @@ -468,6 +471,7 @@ def from_env(cls) -> DependencyManager: occurrence_repo = OccurrenceRepository( session_maker=config.db.async_session_maker, ) + session_logs_repo = AmaltheaSessionPersistedLogsReadRepository(authz=authz) return cls( config, k8s_client=client, @@ -514,6 +518,7 @@ def from_env(cls) -> DependencyManager: occurrence_repo=occurrence_repo, resource_requests_repo=resource_requests_repo, resource_usage_service=resource_usage_service, + session_logs_repo=session_logs_repo, zenodo_client=ZenodoAPIClient(), job_client=job_client, secret_client=secret_client, diff --git a/components/renku_data_services/persisted_logs/api.spec.yaml b/components/renku_data_services/persisted_logs/api.spec.yaml new file mode 100644 index 000000000..59cb4efde --- /dev/null +++ b/components/renku_data_services/persisted_logs/api.spec.yaml @@ -0,0 +1,104 @@ +openapi: 3.0.2 +info: + title: Renku Data Services API + description: | + This service is the main backend for Renku. It provides information about users, projects, + cloud storage, access to compute resources and many other things. + version: v1 +servers: + - url: /api/data +paths: + /persisted_logs/sessions/{launcher_id}: + get: + parameters: + - in: path + name: launcher_id + required: true + schema: + $ref: "#/components/schemas/Ulid" + responses: + "200": + description: | + The session logs from the corresponding run and container, paginated. + content: + application/json: + schema: + $ref: "#/components/schemas/PersistedSessionLogs" + "404": + description: The session launcher does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + default: + $ref: "#/components/responses/Error" + /persisted_logs/sessions/{launcher_id}/runs: + get: + parameters: + - in: path + name: launcher_id + required: true + schema: + $ref: "#/components/schemas/Ulid" + responses: + "200": + description: | + The session runs for which logs exist. + content: + application/json: + schema: + $ref: "#/components/schemas/SessionRuns" + "404": + description: The session launcher does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + default: + $ref: "#/components/responses/Error" +components: + schemas: + PersistedSessionLogs: + description: Logs for a session + type: object + additionalProperties: true + SessionRuns: + description: A session run + type: object + additionalProperties: true + Ulid: + description: ULID identifier + type: string + minLength: 26 + maxLength: 26 + pattern: "^[0-7][0-9A-HJKMNP-TV-Z]{25}$" # This is case-insensitive + ErrorResponse: + type: object + properties: + error: + type: object + properties: + code: + type: integer + minimum: 0 + exclusiveMinimum: true + example: 1404 + detail: + type: string + example: "A more detailed optional message showing what the problem was" + message: + type: string + example: "Something went wrong - please try again later" + trace_id: + type: string + example: "ac93950e9e114a55c67fb8e5ef519bbe" + description: Sentry trace ID for linking to corresponding log entries + required: ["code", "message"] + required: ["error"] + responses: + Error: + description: The schema for all 4xx and 5xx responses + content: + "application/json": + schema: + $ref: "#/components/schemas/ErrorResponse" diff --git a/components/renku_data_services/persisted_logs/apispec.py b/components/renku_data_services/persisted_logs/apispec.py new file mode 100644 index 000000000..8e56e6bfa --- /dev/null +++ b/components/renku_data_services/persisted_logs/apispec.py @@ -0,0 +1,49 @@ +# generated by datamodel-codegen: +# filename: api.spec.yaml +# timestamp: 2026-07-20T11:41:32+00:00 + +from __future__ import annotations + +from pydantic import ConfigDict, Field, RootModel +from renku_data_services.persisted_logs.apispec_base import BaseAPISpec + + +class PersistedSessionLogs(BaseAPISpec): + model_config = ConfigDict( + extra="allow", + ) + + +class SessionRuns(BaseAPISpec): + model_config = ConfigDict( + extra="allow", + ) + + +class Ulid(RootModel[str]): + root: str = Field( + ..., + description="ULID identifier", + max_length=26, + min_length=26, + pattern="^[0-7][0-9A-HJKMNP-TV-Z]{25}$", + ) + + +class Error(BaseAPISpec): + code: int = Field(..., examples=[1404], gt=0) + detail: str | None = Field( + None, examples=["A more detailed optional message showing what the problem was"] + ) + message: str = Field( + ..., examples=["Something went wrong - please try again later"] + ) + trace_id: str | None = Field( + None, + description="Sentry trace ID for linking to corresponding log entries", + examples=["ac93950e9e114a55c67fb8e5ef519bbe"], + ) + + +class ErrorResponse(BaseAPISpec): + error: Error diff --git a/components/renku_data_services/persisted_logs/apispec_base.py b/components/renku_data_services/persisted_logs/apispec_base.py new file mode 100644 index 000000000..d857d8fc9 --- /dev/null +++ b/components/renku_data_services/persisted_logs/apispec_base.py @@ -0,0 +1,12 @@ +"""Base models for API specifications.""" + +from pydantic import BaseModel, ConfigDict + + +class BaseAPISpec(BaseModel): + """Base API specification.""" + + model_config = ConfigDict( + # Enables orm mode for pydantic.""" + from_attributes=True, + ) diff --git a/components/renku_data_services/persisted_logs/blueprints.py b/components/renku_data_services/persisted_logs/blueprints.py new file mode 100644 index 000000000..4856cec30 --- /dev/null +++ b/components/renku_data_services/persisted_logs/blueprints.py @@ -0,0 +1,34 @@ +"""Persisted logs blueprints.""" + +from collections.abc import Callable +from dataclasses import dataclass + +from sanic import Request +from sanic.response import JSONResponse +from sqlalchemy.ext.asyncio import AsyncSession + +from renku_data_services import base_models +from renku_data_services.base_api.auth import authenticate, only_authenticated +from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint +from renku_data_services.base_models.validation import validated_json +from renku_data_services.persisted_logs import apispec +from renku_data_services.persisted_logs.db import AmaltheaSessionPersistedLogsReadRepository + + +@dataclass(kw_only=True) +class PersistedLogsBP(CustomBlueprint): + """Handlers for querying persisted logs.""" + + session_logs_repo: AmaltheaSessionPersistedLogsReadRepository + authenticator: base_models.Authenticator + session_maker: Callable[..., AsyncSession] + + def get_session_logs(self) -> BlueprintFactoryResponse: + """Get persisted sessions logs.""" + + @authenticate(self.authenticator) + @only_authenticated + async def _get_session_logs(_: Request, user: base_models.APIUser) -> JSONResponse: + return validated_json(apispec.PersistedSessionLogs, {}) + + return "/persisted_logs/sessions/", ["GET"], _get_session_logs diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index 0456fc3ea..28612cc62 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -1,17 +1,94 @@ """Adapters for persisted logs database classes.""" -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Sequence from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from ulid import ULID +from renku_data_services import base_models, errors from renku_data_services.app_config import logging +from renku_data_services.authz.authz import Authz, ResourceType +from renku_data_services.authz.models import Scope from renku_data_services.persisted_logs import models from renku_data_services.persisted_logs import orm as schemas +from renku_data_services.session import orm as session_schemas logger = logging.getLogger(__name__) +class AmaltheaSessionPersistedLogsReadRepository: + """Repository for persisted logs of Amalthea sessions.""" + + def __init__(self, authz: Authz) -> None: + self.authz: Authz = authz + + async def get_session_logs(self, session: AsyncSession, user: base_models.APIUser, launcher_id: ULID) -> None: + """Returns persisted session logs for the given launcher.""" + if not user.is_authenticated or not user.id: + raise errors.UnauthorizedError(message="You have to be authenticated to perform this operation.") + await self._check_session_launcher(session=session, user=user, launcher_id=launcher_id) + latest_run = await self._get_session_run(session=session, user_id=user.id, launcher_id=launcher_id) + + # TODO + if latest_run is None: + return + + logger.info(f"latest_run = {str(latest_run)}") + + containers = await self._get_containers(session=session, run_id=latest_run) + + logger.info(f"containers = {str(containers)}") + + # TODO + pass + + async def _check_session_launcher( + self, session: AsyncSession, user: base_models.APIUser, launcher_id: ULID + ) -> None: + """Check that the session launcher exists and the user has access to it.""" + stmt = select(session_schemas.SessionLauncherORM).where(session_schemas.SessionLauncherORM.id == launcher_id) + res = await session.scalars(stmt) + launcher_orm = res.one_or_none() + authorized = ( + await self.authz.has_permission(user, ResourceType.project, launcher_orm.project_id, Scope.READ) + if launcher_orm is not None + else False + ) + if not authorized or launcher_orm is None: + raise errors.MissingResourceError( + message=f"Session launcher with id '{launcher_id}' does not exist or you do not have access to it." + ) + + async def _get_session_run(self, session: AsyncSession, user_id: str, launcher_id: ULID) -> ULID | None: + """Get a specific session run from the persisted logs database.""" + stmt = ( + select(schemas.SessionRunsORM) + .where(schemas.SessionRunsORM.user_id == user_id) + .where(schemas.SessionRunsORM.launcher_id == launcher_id) + .order_by(schemas.SessionRunsORM.id.desc()) + .limit(1) + ) + res = await session.scalars(stmt) + session_run = res.one_or_none() + if session_run is None: + return None + # TODO: return a model instance with .dump() + return session_run.id + + async def _get_containers(self, session: AsyncSession, run_id: ULID) -> Sequence[str]: + """Get the list of pod containers from the persisted logs database.""" + stmt = ( + select(schemas.AmaltheaSessionLogsORM.container.distinct()) + .select_from(schemas.AmaltheaSessionLogsORM) + .where(schemas.AmaltheaSessionLogsORM.run_id == run_id) + .order_by(schemas.AmaltheaSessionLogsORM.container) + ) + res = await session.scalars(stmt) + containers = res.all() + return containers + + class AmaltheaSessionPersistedLogsRepository: """Repository for persisted logs of Amalthea sessions.""" From e5289bb0506d10b5f89faf9f20fb797dd34ace9c Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Mon, 20 Jul 2026 12:55:07 +0000 Subject: [PATCH 15/50] fix get_session_logs() --- components/renku_data_services/persisted_logs/blueprints.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/components/renku_data_services/persisted_logs/blueprints.py b/components/renku_data_services/persisted_logs/blueprints.py index 4856cec30..34865ea7f 100644 --- a/components/renku_data_services/persisted_logs/blueprints.py +++ b/components/renku_data_services/persisted_logs/blueprints.py @@ -6,6 +6,7 @@ from sanic import Request from sanic.response import JSONResponse from sqlalchemy.ext.asyncio import AsyncSession +from ulid import ULID from renku_data_services import base_models from renku_data_services.base_api.auth import authenticate, only_authenticated @@ -28,7 +29,9 @@ def get_session_logs(self) -> BlueprintFactoryResponse: @authenticate(self.authenticator) @only_authenticated - async def _get_session_logs(_: Request, user: base_models.APIUser) -> JSONResponse: + async def _get_session_logs(_: Request, user: base_models.APIUser, launcher_id: ULID) -> JSONResponse: + async with self.session_maker() as session, session.begin(): + await self.session_logs_repo.get_session_logs(session=session, user=user, launcher_id=launcher_id) return validated_json(apispec.PersistedSessionLogs, {}) return "/persisted_logs/sessions/", ["GET"], _get_session_logs From d57e3934529b0088fd4695033d758fe2ced91434 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Mon, 20 Jul 2026 13:45:29 +0000 Subject: [PATCH 16/50] wip get session logs --- .../persisted_logs/constants.py | 3 + .../renku_data_services/persisted_logs/db.py | 74 +++++++++++++------ .../persisted_logs/models.py | 24 ++++++ .../renku_data_services/persisted_logs/orm.py | 11 +++ 4 files changed, 91 insertions(+), 21 deletions(-) diff --git a/components/renku_data_services/persisted_logs/constants.py b/components/renku_data_services/persisted_logs/constants.py index a7dd8af1c..b4f1ee0d5 100644 --- a/components/renku_data_services/persisted_logs/constants.py +++ b/components/renku_data_services/persisted_logs/constants.py @@ -13,3 +13,6 @@ ONE_SECOND_IN_NANOS: Final[int] = 1_000_000_000 """One second as nanoseconds (for Loki).""" + +SESSION_MAIN_CONTAINER: Final[str] = "amalthea-session" +"""The name of the main pod container for Amalthea sessions.""" diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index 28612cc62..e727428a3 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -1,6 +1,6 @@ """Adapters for persisted logs database classes.""" -from collections.abc import AsyncIterator, Sequence +from collections.abc import AsyncIterator from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -12,6 +12,7 @@ from renku_data_services.authz.models import Scope from renku_data_services.persisted_logs import models from renku_data_services.persisted_logs import orm as schemas +from renku_data_services.persisted_logs.constants import SESSION_MAIN_CONTAINER from renku_data_services.session import orm as session_schemas logger = logging.getLogger(__name__) @@ -23,22 +24,30 @@ class AmaltheaSessionPersistedLogsReadRepository: def __init__(self, authz: Authz) -> None: self.authz: Authz = authz - async def get_session_logs(self, session: AsyncSession, user: base_models.APIUser, launcher_id: ULID) -> None: + async def get_session_logs( + self, session: AsyncSession, user: base_models.APIUser, launcher_id: ULID, run_id: ULID | None = None + ) -> None: """Returns persisted session logs for the given launcher.""" if not user.is_authenticated or not user.id: raise errors.UnauthorizedError(message="You have to be authenticated to perform this operation.") await self._check_session_launcher(session=session, user=user, launcher_id=launcher_id) - latest_run = await self._get_session_run(session=session, user_id=user.id, launcher_id=launcher_id) + session_run = await self._get_session_run( + session=session, user_id=user.id, launcher_id=launcher_id, run_id=run_id + ) # TODO - if latest_run is None: + if session_run is None: return - logger.info(f"latest_run = {str(latest_run)}") + logger.info(f"session_run = {str(session_run)}") - containers = await self._get_containers(session=session, run_id=latest_run) + # containers = await self._get_containers(session=session, run_id=session_run.id) + # logger.info(f"containers = {str(containers)}") - logger.info(f"containers = {str(containers)}") + logs_per_container = await self._get_logs_per_container(session=session, run_id=session_run.id) + containers = logs_per_container.keys() + logger.info(f"containers = {list(*containers)}") + logger.info(f"logs_per_container = {logs_per_container}") # TODO pass @@ -60,8 +69,13 @@ async def _check_session_launcher( message=f"Session launcher with id '{launcher_id}' does not exist or you do not have access to it." ) - async def _get_session_run(self, session: AsyncSession, user_id: str, launcher_id: ULID) -> ULID | None: - """Get a specific session run from the persisted logs database.""" + async def _get_session_run( + self, session: AsyncSession, user_id: str, launcher_id: ULID, run_id: ULID | None = None + ) -> models.SessionRun | None: + """Get a specific session run from the persisted logs database. + + If no `run_id` is specified, then return the latest session run. + """ stmt = ( select(schemas.SessionRunsORM) .where(schemas.SessionRunsORM.user_id == user_id) @@ -69,24 +83,42 @@ async def _get_session_run(self, session: AsyncSession, user_id: str, launcher_i .order_by(schemas.SessionRunsORM.id.desc()) .limit(1) ) + if run_id: + stmt = stmt.where(schemas.SessionRunsORM.id == run_id) res = await session.scalars(stmt) - session_run = res.one_or_none() - if session_run is None: + session_run_orm = res.one_or_none() + if session_run_orm is None: return None - # TODO: return a model instance with .dump() - return session_run.id + return session_run_orm.dump() - async def _get_containers(self, session: AsyncSession, run_id: ULID) -> Sequence[str]: - """Get the list of pod containers from the persisted logs database.""" + async def _get_logs_per_container(self, session: AsyncSession, run_id: ULID) -> models.SessionRunLogs: + """Get the logs of a specific session run, organized by container.""" + # TODO: handle pagination? stmt = ( - select(schemas.AmaltheaSessionLogsORM.container.distinct()) - .select_from(schemas.AmaltheaSessionLogsORM) + select(schemas.AmaltheaSessionLogsORM) .where(schemas.AmaltheaSessionLogsORM.run_id == run_id) - .order_by(schemas.AmaltheaSessionLogsORM.container) + .order_by(schemas.AmaltheaSessionLogsORM.id.asc()) ) - res = await session.scalars(stmt) - containers = res.all() - return containers + res = await session.stream_scalars(stmt) + logs_per_container: dict[str, list[models.LogLine]] = dict() + async for log_entry in res: + container = log_entry.container + logs = logs_per_container.get(container) + if logs is None: + logs = list[models.LogLine]() + logs_per_container[container] = logs + logs.append(models.LogLine(timestamp=log_entry.timestamp, log_line=log_entry.log_line)) + # Sort container by name, forcing "amalthea-session" to be the first item (main container) + containers_set = set(logs_per_container.keys()) + containers: list[str] = list() + if SESSION_MAIN_CONTAINER in containers_set: + containers.append(SESSION_MAIN_CONTAINER) + containers_set.remove(SESSION_MAIN_CONTAINER) + containers.extend(sorted(containers_set)) + result: dict[str, list[models.LogLine]] = dict() + for container in containers: + result[container] = logs_per_container[container] + return result class AmaltheaSessionPersistedLogsRepository: diff --git a/components/renku_data_services/persisted_logs/models.py b/components/renku_data_services/persisted_logs/models.py index b8d4f6e55..a120f45d4 100644 --- a/components/renku_data_services/persisted_logs/models.py +++ b/components/renku_data_services/persisted_logs/models.py @@ -1,5 +1,6 @@ """Models for persisted logs.""" +from collections.abc import Mapping, Sequence from dataclasses import dataclass from ulid import ULID @@ -25,6 +26,29 @@ class UnsavedLogLine: log_line: str +@dataclass(eq=True, frozen=True, kw_only=True) +class SessionRun: + """The continuous execution span of a session.""" + + id: ULID + user_id: str + launch_id: str + launcher_id: ULID + submission_id: str | None + + +@dataclass(eq=True, frozen=True, kw_only=True) +class LogLine: + """A single log line.""" + + timestamp: int + log_line: str + + +type SessionRunLogs = Mapping[str, Sequence[LogLine]] +"""Logs of a session run, organized by pod container.""" + + @dataclass(eq=True, frozen=True, kw_only=True) class InsertLogsResult: """Result of inserting a log stream in the database.""" diff --git a/components/renku_data_services/persisted_logs/orm.py b/components/renku_data_services/persisted_logs/orm.py index 914068f49..2431aa467 100644 --- a/components/renku_data_services/persisted_logs/orm.py +++ b/components/renku_data_services/persisted_logs/orm.py @@ -7,6 +7,7 @@ from ulid import ULID from renku_data_services.base_orm.registry import COMMON_ORM_REGISTRY +from renku_data_services.persisted_logs import models from renku_data_services.session.orm import SessionLauncherORM from renku_data_services.users.orm import UserORM from renku_data_services.utils.sqlalchemy import ULIDType @@ -39,6 +40,16 @@ class SessionRunsORM(BaseORM): submission_id: Mapped[str | None] = mapped_column(nullable=True) """The submission ID, if the session run corresponds to an offline job.""" + def dump(self) -> models.SessionRun: + """Create a session run model from the SessionRunsORM.""" + return models.SessionRun( + id=self.id, + user_id=self.user_id, + launch_id=self.launch_id, + launcher_id=self.launcher_id, + submission_id=self.submission_id, + ) + class AmaltheaSessionLogsORM(BaseORM): """A log line from an Amalthea session.""" From 5e9488757aec28e1a6f5a104c661123310b2f64b Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Mon, 20 Jul 2026 13:56:34 +0000 Subject: [PATCH 17/50] oops --- components/renku_data_services/persisted_logs/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index e727428a3..03ebff56a 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -46,7 +46,7 @@ async def get_session_logs( logs_per_container = await self._get_logs_per_container(session=session, run_id=session_run.id) containers = logs_per_container.keys() - logger.info(f"containers = {list(*containers)}") + logger.info(f"containers = {list(containers)}") logger.info(f"logs_per_container = {logs_per_container}") # TODO From 6dfbfb86a50a03a22ff52fd7c550816c34b9338a Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 21 Jul 2026 07:23:31 +0000 Subject: [PATCH 18/50] wip: return logs --- .../persisted_logs/blueprints.py | 14 +++++++++---- .../renku_data_services/persisted_logs/db.py | 21 ++++++------------- .../persisted_logs/models.py | 8 +++++++ 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/components/renku_data_services/persisted_logs/blueprints.py b/components/renku_data_services/persisted_logs/blueprints.py index 34865ea7f..5fb63e405 100644 --- a/components/renku_data_services/persisted_logs/blueprints.py +++ b/components/renku_data_services/persisted_logs/blueprints.py @@ -1,14 +1,14 @@ """Persisted logs blueprints.""" from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import asdict, dataclass from sanic import Request from sanic.response import JSONResponse from sqlalchemy.ext.asyncio import AsyncSession from ulid import ULID -from renku_data_services import base_models +from renku_data_services import base_models, errors from renku_data_services.base_api.auth import authenticate, only_authenticated from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint from renku_data_services.base_models.validation import validated_json @@ -31,7 +31,13 @@ def get_session_logs(self) -> BlueprintFactoryResponse: @only_authenticated async def _get_session_logs(_: Request, user: base_models.APIUser, launcher_id: ULID) -> JSONResponse: async with self.session_maker() as session, session.begin(): - await self.session_logs_repo.get_session_logs(session=session, user=user, launcher_id=launcher_id) - return validated_json(apispec.PersistedSessionLogs, {}) + result = await self.session_logs_repo.get_session_logs( + session=session, user=user, launcher_id=launcher_id + ) + if result is None: + raise errors.MissingResourceError( + message=f"Session launcher with id '{launcher_id}' does not have logs yet." + ) + return validated_json(apispec.PersistedSessionLogs, asdict(result)) return "/persisted_logs/sessions/", ["GET"], _get_session_logs diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index 03ebff56a..2f156fec6 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -26,7 +26,7 @@ def __init__(self, authz: Authz) -> None: async def get_session_logs( self, session: AsyncSession, user: base_models.APIUser, launcher_id: ULID, run_id: ULID | None = None - ) -> None: + ) -> models.GetSessionLogsResult | None: """Returns persisted session logs for the given launcher.""" if not user.is_authenticated or not user.id: raise errors.UnauthorizedError(message="You have to be authenticated to perform this operation.") @@ -34,23 +34,14 @@ async def get_session_logs( session_run = await self._get_session_run( session=session, user_id=user.id, launcher_id=launcher_id, run_id=run_id ) - - # TODO if session_run is None: - return - - logger.info(f"session_run = {str(session_run)}") - - # containers = await self._get_containers(session=session, run_id=session_run.id) - # logger.info(f"containers = {str(containers)}") + return None logs_per_container = await self._get_logs_per_container(session=session, run_id=session_run.id) - containers = logs_per_container.keys() - logger.info(f"containers = {list(containers)}") - logger.info(f"logs_per_container = {logs_per_container}") - - # TODO - pass + return models.GetSessionLogsResult( + run=session_run, + logs=logs_per_container, + ) async def _check_session_launcher( self, session: AsyncSession, user: base_models.APIUser, launcher_id: ULID diff --git a/components/renku_data_services/persisted_logs/models.py b/components/renku_data_services/persisted_logs/models.py index a120f45d4..5ff0565c9 100644 --- a/components/renku_data_services/persisted_logs/models.py +++ b/components/renku_data_services/persisted_logs/models.py @@ -49,6 +49,14 @@ class LogLine: """Logs of a session run, organized by pod container.""" +@dataclass(eq=True, frozen=True, kw_only=True) +class GetSessionLogsResult: + """Result of getting session logs from the database.""" + + run: SessionRun + logs: SessionRunLogs + + @dataclass(eq=True, frozen=True, kw_only=True) class InsertLogsResult: """Result of inserting a log stream in the database.""" From ab7beb0d95c58a32c44394481128a999c243b422 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 21 Jul 2026 07:40:17 +0000 Subject: [PATCH 19/50] fix ULID serialization --- .../persisted_logs/apispec_base.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/components/renku_data_services/persisted_logs/apispec_base.py b/components/renku_data_services/persisted_logs/apispec_base.py index d857d8fc9..6a842832f 100644 --- a/components/renku_data_services/persisted_logs/apispec_base.py +++ b/components/renku_data_services/persisted_logs/apispec_base.py @@ -1,6 +1,9 @@ """Base models for API specifications.""" -from pydantic import BaseModel, ConfigDict +from typing import Any + +from pydantic import BaseModel, ConfigDict, field_validator +from ulid import ULID class BaseAPISpec(BaseModel): @@ -10,3 +13,11 @@ class BaseAPISpec(BaseModel): # Enables orm mode for pydantic.""" from_attributes=True, ) + + @field_validator("*", mode="before", check_fields=False) + @classmethod + def serialize_ulid(cls, value: Any) -> Any: + """Handle ULIDs.""" + if isinstance(value, ULID): + return str(value) + return value From a5bad1f303e746935e0cfc4935af16506571c755 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 21 Jul 2026 08:02:55 +0000 Subject: [PATCH 20/50] temp json --- .../renku_data_services/persisted_logs/blueprints.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/components/renku_data_services/persisted_logs/blueprints.py b/components/renku_data_services/persisted_logs/blueprints.py index 5fb63e405..aeb1177aa 100644 --- a/components/renku_data_services/persisted_logs/blueprints.py +++ b/components/renku_data_services/persisted_logs/blueprints.py @@ -3,7 +3,7 @@ from collections.abc import Callable from dataclasses import asdict, dataclass -from sanic import Request +from sanic import Request, json from sanic.response import JSONResponse from sqlalchemy.ext.asyncio import AsyncSession from ulid import ULID @@ -11,8 +11,6 @@ from renku_data_services import base_models, errors from renku_data_services.base_api.auth import authenticate, only_authenticated from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint -from renku_data_services.base_models.validation import validated_json -from renku_data_services.persisted_logs import apispec from renku_data_services.persisted_logs.db import AmaltheaSessionPersistedLogsReadRepository @@ -38,6 +36,7 @@ async def _get_session_logs(_: Request, user: base_models.APIUser, launcher_id: raise errors.MissingResourceError( message=f"Session launcher with id '{launcher_id}' does not have logs yet." ) - return validated_json(apispec.PersistedSessionLogs, asdict(result)) + return json(asdict(result), status=200, content_type="application/json") + # return validated_json(apispec.PersistedSessionLogs, asdict(result)) return "/persisted_logs/sessions/", ["GET"], _get_session_logs From dd2fdd82baac9bc70ff4dd3c7bd1931df21693a5 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 21 Jul 2026 08:47:11 +0000 Subject: [PATCH 21/50] wip: API --- .../persisted_logs/api.spec.yaml | 61 ++++++++++++++++- .../persisted_logs/apispec.py | 68 ++++++++++++------- .../persisted_logs/blueprints.py | 26 +++++-- .../renku_data_services/persisted_logs/db.py | 4 +- .../persisted_logs/models.py | 2 +- 5 files changed, 128 insertions(+), 33 deletions(-) diff --git a/components/renku_data_services/persisted_logs/api.spec.yaml b/components/renku_data_services/persisted_logs/api.spec.yaml index 59cb4efde..c310c48c8 100644 --- a/components/renku_data_services/persisted_logs/api.spec.yaml +++ b/components/renku_data_services/persisted_logs/api.spec.yaml @@ -32,6 +32,8 @@ paths: $ref: "#/components/schemas/ErrorResponse" default: $ref: "#/components/responses/Error" + tags: + - persisted_logs /persisted_logs/sessions/{launcher_id}/runs: get: parameters: @@ -56,22 +58,77 @@ paths: $ref: "#/components/schemas/ErrorResponse" default: $ref: "#/components/responses/Error" + tags: + - persisted_logs components: schemas: PersistedSessionLogs: description: Logs for a session type: object - additionalProperties: true + properties: + run: + $ref: "#/components/schemas/SessionRun" + logs: + $ref: "#/components/schemas/SessionRunLogs" + required: + - run + - logs SessionRuns: + description: A list of session runs + type: array + items: + $ref: "#/components/schemas/SessionRun" + SessionRun: description: A session run type: object - additionalProperties: true + properties: + id: + $ref: "#/components/schemas/Ulid" + launch_id: + type: string + launcher_id: + $ref: "#/components/schemas/Ulid" + submission_id: + type: string + required: + - id + - launch_id + - launcher_id + SessionRunLogs: + description: Logs of a session run, organized by pod container. + type: array + items: + $ref: "#/components/schemas/SessionRunLogsItem" + SessionRunLogsItem: + type: object + properties: + container: + type: string + logs: + $ref: "#/components/schemas/PersistedLogLines" + required: + - container + - logs + PersistedLogLines: + type: object + properties: + timestamp: + $ref: "#/components/schemas/NanoTimestamp" + log_line: + type: string + required: + - timestamp + - log_line Ulid: description: ULID identifier type: string minLength: 26 maxLength: 26 pattern: "^[0-7][0-9A-HJKMNP-TV-Z]{25}$" # This is case-insensitive + NanoTimestamp: + description: UNIX timestamp in nanoseconds + type: string + pattern: "[0-9]+" ErrorResponse: type: object properties: diff --git a/components/renku_data_services/persisted_logs/apispec.py b/components/renku_data_services/persisted_logs/apispec.py index 8e56e6bfa..7932028fe 100644 --- a/components/renku_data_services/persisted_logs/apispec.py +++ b/components/renku_data_services/persisted_logs/apispec.py @@ -1,35 +1,13 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2026-07-20T11:41:32+00:00 +# timestamp: 2026-07-21T08:24:35+00:00 from __future__ import annotations -from pydantic import ConfigDict, Field, RootModel +from pydantic import Field, RootModel from renku_data_services.persisted_logs.apispec_base import BaseAPISpec -class PersistedSessionLogs(BaseAPISpec): - model_config = ConfigDict( - extra="allow", - ) - - -class SessionRuns(BaseAPISpec): - model_config = ConfigDict( - extra="allow", - ) - - -class Ulid(RootModel[str]): - root: str = Field( - ..., - description="ULID identifier", - max_length=26, - min_length=26, - pattern="^[0-7][0-9A-HJKMNP-TV-Z]{25}$", - ) - - class Error(BaseAPISpec): code: int = Field(..., examples=[1404], gt=0) detail: str | None = Field( @@ -47,3 +25,45 @@ class Error(BaseAPISpec): class ErrorResponse(BaseAPISpec): error: Error + + +class SessionRun(BaseAPISpec): + id: str = Field( + ..., + description="ULID identifier", + max_length=26, + min_length=26, + pattern="^[0-7][0-9A-HJKMNP-TV-Z]{25}$", + ) + launch_id: str + launcher_id: str = Field( + ..., + description="ULID identifier", + max_length=26, + min_length=26, + pattern="^[0-7][0-9A-HJKMNP-TV-Z]{25}$", + ) + submission_id: str | None = None + + +class PersistedLogLines(BaseAPISpec): + timestamp: str = Field( + ..., description="UNIX timestamp in nanoseconds", pattern="[0-9]+" + ) + log_line: str + + +class SessionRuns(RootModel[list[SessionRun]]): + root: list[SessionRun] = Field(..., description="A list of session runs") + + +class SessionRunLogsItem(BaseAPISpec): + container: str + logs: PersistedLogLines + + +class PersistedSessionLogs(BaseAPISpec): + run: SessionRun + logs: list[SessionRunLogsItem] = Field( + ..., description="Logs of a session run, organized by pod container." + ) diff --git a/components/renku_data_services/persisted_logs/blueprints.py b/components/renku_data_services/persisted_logs/blueprints.py index aeb1177aa..419256d8a 100644 --- a/components/renku_data_services/persisted_logs/blueprints.py +++ b/components/renku_data_services/persisted_logs/blueprints.py @@ -1,9 +1,10 @@ """Persisted logs blueprints.""" from collections.abc import Callable -from dataclasses import asdict, dataclass +from dataclasses import dataclass +from typing import Any -from sanic import Request, json +from sanic import Request from sanic.response import JSONResponse from sqlalchemy.ext.asyncio import AsyncSession from ulid import ULID @@ -11,6 +12,8 @@ from renku_data_services import base_models, errors from renku_data_services.base_api.auth import authenticate, only_authenticated from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint +from renku_data_services.base_models.validation import validated_json +from renku_data_services.persisted_logs import apispec, models from renku_data_services.persisted_logs.db import AmaltheaSessionPersistedLogsReadRepository @@ -36,7 +39,22 @@ async def _get_session_logs(_: Request, user: base_models.APIUser, launcher_id: raise errors.MissingResourceError( message=f"Session launcher with id '{launcher_id}' does not have logs yet." ) - return json(asdict(result), status=200, content_type="application/json") - # return validated_json(apispec.PersistedSessionLogs, asdict(result)) + return validated_json(apispec.PersistedSessionLogs, self._dump_persisted_session_logs(result)) return "/persisted_logs/sessions/", ["GET"], _get_session_logs + + @staticmethod + def _dump_persisted_session_logs(session_log: models.PersistedSessionLogs) -> dict[str, Any]: + """Dump persisted session logs for API responses.""" + return dict(run=PersistedLogsBP._dump_session_run(session_log.run), logs=list(session_log.logs.items())) + + @staticmethod + def _dump_session_run(session_run: models.SessionRun) -> dict[str, Any]: + """Dump a session run for API responses.""" + # NOTE: omit the "user_id" field + return dict( + id=session_run.id, + launch_id=session_run.launch_id, + launcher_id=session_run.launcher_id, + submission_id=session_run.submission_id, + ) diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index 2f156fec6..08cb5ea76 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -26,7 +26,7 @@ def __init__(self, authz: Authz) -> None: async def get_session_logs( self, session: AsyncSession, user: base_models.APIUser, launcher_id: ULID, run_id: ULID | None = None - ) -> models.GetSessionLogsResult | None: + ) -> models.PersistedSessionLogs | None: """Returns persisted session logs for the given launcher.""" if not user.is_authenticated or not user.id: raise errors.UnauthorizedError(message="You have to be authenticated to perform this operation.") @@ -38,7 +38,7 @@ async def get_session_logs( return None logs_per_container = await self._get_logs_per_container(session=session, run_id=session_run.id) - return models.GetSessionLogsResult( + return models.PersistedSessionLogs( run=session_run, logs=logs_per_container, ) diff --git a/components/renku_data_services/persisted_logs/models.py b/components/renku_data_services/persisted_logs/models.py index 5ff0565c9..e0bc9020a 100644 --- a/components/renku_data_services/persisted_logs/models.py +++ b/components/renku_data_services/persisted_logs/models.py @@ -50,7 +50,7 @@ class LogLine: @dataclass(eq=True, frozen=True, kw_only=True) -class GetSessionLogsResult: +class PersistedSessionLogs: """Result of getting session logs from the database.""" run: SessionRun From 8650392e6eb63ef549a60b97bf4630e507b819a2 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 21 Jul 2026 09:45:00 +0000 Subject: [PATCH 22/50] another fix --- .../renku_data_services/persisted_logs/blueprints.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/components/renku_data_services/persisted_logs/blueprints.py b/components/renku_data_services/persisted_logs/blueprints.py index 419256d8a..959f6b4e2 100644 --- a/components/renku_data_services/persisted_logs/blueprints.py +++ b/components/renku_data_services/persisted_logs/blueprints.py @@ -46,7 +46,10 @@ async def _get_session_logs(_: Request, user: base_models.APIUser, launcher_id: @staticmethod def _dump_persisted_session_logs(session_log: models.PersistedSessionLogs) -> dict[str, Any]: """Dump persisted session logs for API responses.""" - return dict(run=PersistedLogsBP._dump_session_run(session_log.run), logs=list(session_log.logs.items())) + return dict( + run=PersistedLogsBP._dump_session_run(session_log.run), + logs=PersistedLogsBP._dump_session_run_logs(session_log.logs), + ) @staticmethod def _dump_session_run(session_run: models.SessionRun) -> dict[str, Any]: @@ -58,3 +61,8 @@ def _dump_session_run(session_run: models.SessionRun) -> dict[str, Any]: launcher_id=session_run.launcher_id, submission_id=session_run.submission_id, ) + + @staticmethod + def _dump_session_run_logs(logs: models.SessionRunLogs) -> list[dict[str, Any]]: + """Dump the logs of a session run, organized by pod container, for API responses.""" + return list(dict(container=container, logs=log_lines) for container, log_lines in logs.items()) From eb0ce2c6f1f11fd7293ac620fee8d2a7dd55ce1e Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 21 Jul 2026 10:04:05 +0000 Subject: [PATCH 23/50] fix api spec --- .../persisted_logs/api.spec.yaml | 13 ++++++++++--- .../renku_data_services/persisted_logs/apispec.py | 12 ++++++------ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/components/renku_data_services/persisted_logs/api.spec.yaml b/components/renku_data_services/persisted_logs/api.spec.yaml index c310c48c8..87b8c0e15 100644 --- a/components/renku_data_services/persisted_logs/api.spec.yaml +++ b/components/renku_data_services/persisted_logs/api.spec.yaml @@ -95,11 +95,12 @@ components: - launch_id - launcher_id SessionRunLogs: - description: Logs of a session run, organized by pod container. + description: Logs of a session run, organized by pod container type: array items: - $ref: "#/components/schemas/SessionRunLogsItem" - SessionRunLogsItem: + $ref: "#/components/schemas/SessionRunLog" + SessionRunLog: + description: Logs of a single container type: object properties: container: @@ -110,6 +111,12 @@ components: - container - logs PersistedLogLines: + description: Stream of log lines + type: array + items: + $ref: "#/components/schemas/PersistedLogLine" + PersistedLogLine: + description: A timestamped log line type: object properties: timestamp: diff --git a/components/renku_data_services/persisted_logs/apispec.py b/components/renku_data_services/persisted_logs/apispec.py index 7932028fe..f1c45d028 100644 --- a/components/renku_data_services/persisted_logs/apispec.py +++ b/components/renku_data_services/persisted_logs/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2026-07-21T08:24:35+00:00 +# timestamp: 2026-07-21T10:01:25+00:00 from __future__ import annotations @@ -46,7 +46,7 @@ class SessionRun(BaseAPISpec): submission_id: str | None = None -class PersistedLogLines(BaseAPISpec): +class PersistedLogLine(BaseAPISpec): timestamp: str = Field( ..., description="UNIX timestamp in nanoseconds", pattern="[0-9]+" ) @@ -57,13 +57,13 @@ class SessionRuns(RootModel[list[SessionRun]]): root: list[SessionRun] = Field(..., description="A list of session runs") -class SessionRunLogsItem(BaseAPISpec): +class SessionRunLog(BaseAPISpec): container: str - logs: PersistedLogLines + logs: list[PersistedLogLine] = Field(..., description="Stream of log lines") class PersistedSessionLogs(BaseAPISpec): run: SessionRun - logs: list[SessionRunLogsItem] = Field( - ..., description="Logs of a session run, organized by pod container." + logs: list[SessionRunLog] = Field( + ..., description="Logs of a session run, organized by pod container" ) From 9aa8267d6aaa26cb9e1ce1f2f0e75d906530405e Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 21 Jul 2026 11:03:03 +0000 Subject: [PATCH 24/50] fix nanotimestamp serializing --- .../renku_data_services/persisted_logs/blueprints.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/components/renku_data_services/persisted_logs/blueprints.py b/components/renku_data_services/persisted_logs/blueprints.py index 959f6b4e2..fb42a4024 100644 --- a/components/renku_data_services/persisted_logs/blueprints.py +++ b/components/renku_data_services/persisted_logs/blueprints.py @@ -65,4 +65,12 @@ def _dump_session_run(session_run: models.SessionRun) -> dict[str, Any]: @staticmethod def _dump_session_run_logs(logs: models.SessionRunLogs) -> list[dict[str, Any]]: """Dump the logs of a session run, organized by pod container, for API responses.""" - return list(dict(container=container, logs=log_lines) for container, log_lines in logs.items()) + return [ + dict(container=container, logs=[PersistedLogsBP._dump_log_line(log_line) for log_line in log_lines]) + for container, log_lines in logs.items() + ] + + @staticmethod + def _dump_log_line(log_line: models.LogLine) -> dict[str, str]: + """Dump a log line for API responses.""" + return dict(timestamp=str(log_line.timestamp), log_line=log_line.log_line) From 48eb6c3746908941007983167e22d51f978efa5b Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 21 Jul 2026 11:42:25 +0000 Subject: [PATCH 25/50] temp fix for jobs --- .../renku_data_services/persisted_logs/db.py | 51 ++++++++++++++----- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index 08cb5ea76..fdbdf2343 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -148,24 +148,47 @@ async def insert_session_logs( logger.info(f"Processing log line {log}") - session_run_res = await session.scalars( - select(schemas.SessionRunsORM).where(schemas.SessionRunsORM.id == log.run_id) - ) - session_run_orm = session_run_res.one_or_none() - if session_run_orm is None: - session_run_orm = schemas.SessionRunsORM( - id=log.run_id, - user_id=log.user_id, - launch_id=log.launch_id, - launcher_id=log.launcher_id, - submission_id=log.submission_id, + # NOTE: temporary code for finding a session run for jobs (one per submission) + run_id = log.run_id + if log.submission_id: + session_run_res = await session.scalars( + select(schemas.SessionRunsORM) + .where(schemas.SessionRunsORM.user_id == log.user_id) + .where(schemas.SessionRunsORM.submission_id == log.submission_id) + ) + session_run_orm = session_run_res.one_or_none() + if session_run_orm is None: + run_id = ULID() + session_run_orm = schemas.SessionRunsORM( + id=run_id, + user_id=log.user_id, + launch_id=log.launch_id, + launcher_id=log.launcher_id, + submission_id=log.submission_id, + ) + session.add(session_run_orm) + await session.flush() + else: + run_id = session_run_orm.id + else: + session_run_res = await session.scalars( + select(schemas.SessionRunsORM).where(schemas.SessionRunsORM.id == log.run_id) ) - session.add(session_run_orm) - await session.flush() + session_run_orm = session_run_res.one_or_none() + if session_run_orm is None: + session_run_orm = schemas.SessionRunsORM( + id=log.run_id, + user_id=log.user_id, + launch_id=log.launch_id, + launcher_id=log.launcher_id, + submission_id=log.submission_id, + ) + session.add(session_run_orm) + await session.flush() log_orm = schemas.AmaltheaSessionLogsORM( id=log.id, - run_id=log.run_id, + run_id=run_id, container=log.container, timestamp=log.timestamp, log_line=log.log_line, From 2d0fbce7b3000f6749c2fc1605bd418b6c27fcd3 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 21 Jul 2026 12:34:30 +0000 Subject: [PATCH 26/50] add query params --- .../persisted_logs/api.spec.yaml | 14 ++++++++++++++ .../persisted_logs/apispec.py | 17 ++++++++++++++++- .../persisted_logs/blueprints.py | 13 +++++++++++-- .../renku_data_services/persisted_logs/db.py | 18 +++++++++++++++--- 4 files changed, 56 insertions(+), 6 deletions(-) diff --git a/components/renku_data_services/persisted_logs/api.spec.yaml b/components/renku_data_services/persisted_logs/api.spec.yaml index 87b8c0e15..97f53c989 100644 --- a/components/renku_data_services/persisted_logs/api.spec.yaml +++ b/components/renku_data_services/persisted_logs/api.spec.yaml @@ -16,6 +16,12 @@ paths: required: true schema: $ref: "#/components/schemas/Ulid" + - in: query + name: params + style: form + explode: true + schema: + $ref: "#/components/schemas/PersistedLogsGetQuery" responses: "200": description: | @@ -136,6 +142,14 @@ components: description: UNIX timestamp in nanoseconds type: string pattern: "[0-9]+" + PersistedLogsGetQuery: + description: Query params for querying persisted logs + type: object + properties: + run_id: + $ref: "#/components/schemas/Ulid" + submission_id: + type: string ErrorResponse: type: object properties: diff --git a/components/renku_data_services/persisted_logs/apispec.py b/components/renku_data_services/persisted_logs/apispec.py index f1c45d028..239c6cecf 100644 --- a/components/renku_data_services/persisted_logs/apispec.py +++ b/components/renku_data_services/persisted_logs/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2026-07-21T10:01:25+00:00 +# timestamp: 2026-07-21T12:27:04+00:00 from __future__ import annotations @@ -8,6 +8,17 @@ from renku_data_services.persisted_logs.apispec_base import BaseAPISpec +class PersistedLogsGetQuery(BaseAPISpec): + run_id: str | None = Field( + None, + description="ULID identifier", + max_length=26, + min_length=26, + pattern="^[0-7][0-9A-HJKMNP-TV-Z]{25}$", + ) + submission_id: str | None = None + + class Error(BaseAPISpec): code: int = Field(..., examples=[1404], gt=0) detail: str | None = Field( @@ -27,6 +38,10 @@ class ErrorResponse(BaseAPISpec): error: Error +class PersistedLogsSessionsLauncherIdGetParametersQuery(BaseAPISpec): + params: PersistedLogsGetQuery | None = None + + class SessionRun(BaseAPISpec): id: str = Field( ..., diff --git a/components/renku_data_services/persisted_logs/blueprints.py b/components/renku_data_services/persisted_logs/blueprints.py index fb42a4024..67e0d9597 100644 --- a/components/renku_data_services/persisted_logs/blueprints.py +++ b/components/renku_data_services/persisted_logs/blueprints.py @@ -12,6 +12,7 @@ from renku_data_services import base_models, errors from renku_data_services.base_api.auth import authenticate, only_authenticated from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint +from renku_data_services.base_api.misc import validate_query from renku_data_services.base_models.validation import validated_json from renku_data_services.persisted_logs import apispec, models from renku_data_services.persisted_logs.db import AmaltheaSessionPersistedLogsReadRepository @@ -30,10 +31,18 @@ def get_session_logs(self) -> BlueprintFactoryResponse: @authenticate(self.authenticator) @only_authenticated - async def _get_session_logs(_: Request, user: base_models.APIUser, launcher_id: ULID) -> JSONResponse: + @validate_query(query=apispec.PersistedLogsGetQuery) + async def _get_session_logs( + _: Request, user: base_models.APIUser, launcher_id: ULID, query: apispec.PersistedLogsGetQuery + ) -> JSONResponse: + run_id = ULID.from_str(query.run_id) async with self.session_maker() as session, session.begin(): result = await self.session_logs_repo.get_session_logs( - session=session, user=user, launcher_id=launcher_id + session=session, + user=user, + launcher_id=launcher_id, + run_id=run_id, + submission_id=query.submission_id, ) if result is None: raise errors.MissingResourceError( diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index fdbdf2343..52bca5fb1 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -25,14 +25,19 @@ def __init__(self, authz: Authz) -> None: self.authz: Authz = authz async def get_session_logs( - self, session: AsyncSession, user: base_models.APIUser, launcher_id: ULID, run_id: ULID | None = None + self, + session: AsyncSession, + user: base_models.APIUser, + launcher_id: ULID, + run_id: ULID | None = None, + submission_id: str | None = None, ) -> models.PersistedSessionLogs | None: """Returns persisted session logs for the given launcher.""" if not user.is_authenticated or not user.id: raise errors.UnauthorizedError(message="You have to be authenticated to perform this operation.") await self._check_session_launcher(session=session, user=user, launcher_id=launcher_id) session_run = await self._get_session_run( - session=session, user_id=user.id, launcher_id=launcher_id, run_id=run_id + session=session, user_id=user.id, launcher_id=launcher_id, run_id=run_id, submission_id=submission_id ) if session_run is None: return None @@ -61,7 +66,12 @@ async def _check_session_launcher( ) async def _get_session_run( - self, session: AsyncSession, user_id: str, launcher_id: ULID, run_id: ULID | None = None + self, + session: AsyncSession, + user_id: str, + launcher_id: ULID, + run_id: ULID | None = None, + submission_id: str | None = None, ) -> models.SessionRun | None: """Get a specific session run from the persisted logs database. @@ -76,6 +86,8 @@ async def _get_session_run( ) if run_id: stmt = stmt.where(schemas.SessionRunsORM.id == run_id) + if submission_id: + stmt = stmt.where(schemas.SessionRunsORM.submission_id == submission_id) res = await session.scalars(stmt) session_run_orm = res.one_or_none() if session_run_orm is None: From af2b3cbc7d550ae7f081d7a18da50783a3d52a0a Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 21 Jul 2026 12:49:21 +0000 Subject: [PATCH 27/50] add runs API endpoint --- .../persisted_logs/blueprints.py | 17 ++++++++++++++++ .../renku_data_services/persisted_logs/db.py | 20 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/components/renku_data_services/persisted_logs/blueprints.py b/components/renku_data_services/persisted_logs/blueprints.py index 67e0d9597..b6ec1032d 100644 --- a/components/renku_data_services/persisted_logs/blueprints.py +++ b/components/renku_data_services/persisted_logs/blueprints.py @@ -52,6 +52,23 @@ async def _get_session_logs( return "/persisted_logs/sessions/", ["GET"], _get_session_logs + def get_session_runs(self) -> BlueprintFactoryResponse: + """Get the session runs for a given session launcher.""" + + @authenticate(self.authenticator) + @only_authenticated + async def _get_session_runs(_: Request, user: base_models.APIUser, launcher_id: ULID) -> JSONResponse: + async with self.session_maker() as session, session.begin(): + session_runs = self.session_logs_repo.get_session_runs( + session=session, user=user, launcher_id=launcher_id + ) + result: list[models.SessionRun] = [] + async for item in session_runs: + result.append(item) + return validated_json(apispec.SessionRuns, result) + + return "/persisted_logs/sessions//runs:", ["GET"], _get_session_runs + @staticmethod def _dump_persisted_session_logs(session_log: models.PersistedSessionLogs) -> dict[str, Any]: """Dump persisted session logs for API responses.""" diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index 52bca5fb1..04df83f01 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -48,6 +48,26 @@ async def get_session_logs( logs=logs_per_container, ) + async def get_session_runs( + self, + session: AsyncSession, + user: base_models.APIUser, + launcher_id: ULID, + ) -> AsyncIterator[models.SessionRun]: + """Returns the session runs for the given launcher.""" + if not user.is_authenticated or not user.id: + raise errors.UnauthorizedError(message="You have to be authenticated to perform this operation.") + await self._check_session_launcher(session=session, user=user, launcher_id=launcher_id) + stmt = ( + select(schemas.SessionRunsORM) + .where(schemas.SessionRunsORM.user_id == user.id) + .where(schemas.SessionRunsORM.launcher_id == launcher_id) + .order_by(schemas.SessionRunsORM.id.desc()) + ) + res = await session.stream_scalars(stmt) + async for session_run_orm in res: + yield session_run_orm.dump() + async def _check_session_launcher( self, session: AsyncSession, user: base_models.APIUser, launcher_id: ULID ) -> None: From 83f13bed90f4aa3aece012d228e0ef04f0d5ade8 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 21 Jul 2026 12:52:15 +0000 Subject: [PATCH 28/50] oops --- components/renku_data_services/persisted_logs/blueprints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/renku_data_services/persisted_logs/blueprints.py b/components/renku_data_services/persisted_logs/blueprints.py index b6ec1032d..7d435a633 100644 --- a/components/renku_data_services/persisted_logs/blueprints.py +++ b/components/renku_data_services/persisted_logs/blueprints.py @@ -35,7 +35,7 @@ def get_session_logs(self) -> BlueprintFactoryResponse: async def _get_session_logs( _: Request, user: base_models.APIUser, launcher_id: ULID, query: apispec.PersistedLogsGetQuery ) -> JSONResponse: - run_id = ULID.from_str(query.run_id) + run_id = ULID.from_str(query.run_id) if query.run_id else None async with self.session_maker() as session, session.begin(): result = await self.session_logs_repo.get_session_logs( session=session, From 325224be14308becb83e771158d2adf53f53a492 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 21 Jul 2026 13:06:33 +0000 Subject: [PATCH 29/50] oops typo --- components/renku_data_services/persisted_logs/blueprints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/renku_data_services/persisted_logs/blueprints.py b/components/renku_data_services/persisted_logs/blueprints.py index 7d435a633..be781c60f 100644 --- a/components/renku_data_services/persisted_logs/blueprints.py +++ b/components/renku_data_services/persisted_logs/blueprints.py @@ -67,7 +67,7 @@ async def _get_session_runs(_: Request, user: base_models.APIUser, launcher_id: result.append(item) return validated_json(apispec.SessionRuns, result) - return "/persisted_logs/sessions//runs:", ["GET"], _get_session_runs + return "/persisted_logs/sessions//runs", ["GET"], _get_session_runs @staticmethod def _dump_persisted_session_logs(session_log: models.PersistedSessionLogs) -> dict[str, Any]: From b8938099a3bfc8a61d7c9ad4714fef480a680a55 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 22 Jul 2026 08:12:49 +0000 Subject: [PATCH 30/50] remove debug logs --- components/renku_data_services/persisted_logs/collector.py | 4 ++-- components/renku_data_services/persisted_logs/db.py | 6 ------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/components/renku_data_services/persisted_logs/collector.py b/components/renku_data_services/persisted_logs/collector.py index 82b2f8504..b8ca719e6 100644 --- a/components/renku_data_services/persisted_logs/collector.py +++ b/components/renku_data_services/persisted_logs/collector.py @@ -80,12 +80,12 @@ async def get_amalthea_session_logs( ) continue - logger.info(stream) + # logger.info(stream) for nano_ts, log_line in entry.values: log_line_id = f"{nano_ts.root}::{stream.container}::{stream.pod}" if log_line_id in log_line_ids: - logger.info(f"Already saw: {log_line_id}") + # logger.info(f"Already saw: {log_line_id}") continue log_line_ids.add(log_line_id) diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index 04df83f01..c46c973c8 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -7,7 +7,6 @@ from ulid import ULID from renku_data_services import base_models, errors -from renku_data_services.app_config import logging from renku_data_services.authz.authz import Authz, ResourceType from renku_data_services.authz.models import Scope from renku_data_services.persisted_logs import models @@ -15,8 +14,6 @@ from renku_data_services.persisted_logs.constants import SESSION_MAIN_CONTAINER from renku_data_services.session import orm as session_schemas -logger = logging.getLogger(__name__) - class AmaltheaSessionPersistedLogsReadRepository: """Repository for persisted logs of Amalthea sessions.""" @@ -175,11 +172,8 @@ async def insert_session_logs( ) existing_log_orm = existing_log_res.one_or_none() if existing_log_orm: - logger.info(f"Skipping log line {log.id}") continue - logger.info(f"Processing log line {log}") - # NOTE: temporary code for finding a session run for jobs (one per submission) run_id = log.run_id if log.submission_id: From e43b37de62f81170adfee0a00dcb01932ba27666 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 22 Jul 2026 08:54:27 +0000 Subject: [PATCH 31/50] use real run_id --- .../906ec89ea06f_wip_update_persisted_logs.py | 31 +++++++++++ .../persisted_logs/api.spec.yaml | 3 +- .../persisted_logs/apispec.py | 4 +- .../persisted_logs/blueprints.py | 2 +- .../persisted_logs/collector.py | 16 ++++-- .../renku_data_services/persisted_logs/db.py | 51 +++++-------------- .../persisted_logs/loki_api.py | 2 + .../persisted_logs/models.py | 6 +-- .../renku_data_services/persisted_logs/orm.py | 6 +-- 9 files changed, 69 insertions(+), 52 deletions(-) create mode 100644 components/renku_data_services/migrations/versions/906ec89ea06f_wip_update_persisted_logs.py diff --git a/components/renku_data_services/migrations/versions/906ec89ea06f_wip_update_persisted_logs.py b/components/renku_data_services/migrations/versions/906ec89ea06f_wip_update_persisted_logs.py new file mode 100644 index 000000000..2908836e3 --- /dev/null +++ b/components/renku_data_services/migrations/versions/906ec89ea06f_wip_update_persisted_logs.py @@ -0,0 +1,31 @@ +"""wip: update persisted logs + +Revision ID: 906ec89ea06f +Revises: 01180f797019 +Create Date: 2026-07-22 08:50:21.380775 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "906ec89ea06f" +down_revision = "01180f797019" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("DELETE FROM persisted_logs.session_runs") + op.add_column("session_runs", sa.Column("session_uid", sa.String(), nullable=True), schema="persisted_logs") + op.drop_column("session_runs", "launch_id", schema="persisted_logs") + + +def downgrade() -> None: + op.add_column( + "session_runs", + sa.Column("launch_id", sa.VARCHAR(), autoincrement=False, nullable=False), + schema="persisted_logs", + ) + op.drop_column("session_runs", "session_uid", schema="persisted_logs") diff --git a/components/renku_data_services/persisted_logs/api.spec.yaml b/components/renku_data_services/persisted_logs/api.spec.yaml index 97f53c989..5b09e6eb0 100644 --- a/components/renku_data_services/persisted_logs/api.spec.yaml +++ b/components/renku_data_services/persisted_logs/api.spec.yaml @@ -90,7 +90,7 @@ components: properties: id: $ref: "#/components/schemas/Ulid" - launch_id: + session_uid: type: string launcher_id: $ref: "#/components/schemas/Ulid" @@ -98,7 +98,6 @@ components: type: string required: - id - - launch_id - launcher_id SessionRunLogs: description: Logs of a session run, organized by pod container diff --git a/components/renku_data_services/persisted_logs/apispec.py b/components/renku_data_services/persisted_logs/apispec.py index 239c6cecf..72e92875b 100644 --- a/components/renku_data_services/persisted_logs/apispec.py +++ b/components/renku_data_services/persisted_logs/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2026-07-21T12:27:04+00:00 +# timestamp: 2026-07-22T08:48:21+00:00 from __future__ import annotations @@ -50,7 +50,7 @@ class SessionRun(BaseAPISpec): min_length=26, pattern="^[0-7][0-9A-HJKMNP-TV-Z]{25}$", ) - launch_id: str + session_uid: str | None = None launcher_id: str = Field( ..., description="ULID identifier", diff --git a/components/renku_data_services/persisted_logs/blueprints.py b/components/renku_data_services/persisted_logs/blueprints.py index be781c60f..4952aca28 100644 --- a/components/renku_data_services/persisted_logs/blueprints.py +++ b/components/renku_data_services/persisted_logs/blueprints.py @@ -83,7 +83,7 @@ def _dump_session_run(session_run: models.SessionRun) -> dict[str, Any]: # NOTE: omit the "user_id" field return dict( id=session_run.id, - launch_id=session_run.launch_id, + session_uid=session_run.session_uid, launcher_id=session_run.launcher_id, submission_id=session_run.submission_id, ) diff --git a/components/renku_data_services/persisted_logs/collector.py b/components/renku_data_services/persisted_logs/collector.py index b8ca719e6..c0945dc00 100644 --- a/components/renku_data_services/persisted_logs/collector.py +++ b/components/renku_data_services/persisted_logs/collector.py @@ -72,7 +72,7 @@ async def get_amalthea_session_logs( continue try: - launcher_id = ULID.from_str(stream.renku_io_launcher_id) + launcher_id = ULID.from_str(stream.renku_io_launcher_id.upper()) except ValueError as err: logger.warning( f"Skipping entry {entry.stream} because renku_io_launcher_id='{stream.renku_io_launcher_id}' " @@ -80,20 +80,28 @@ async def get_amalthea_session_logs( ) continue + try: + run_id = ULID.from_str(stream.renku_io_run_id.upper()) + except ValueError as err: + logger.warning( + f"Skipping entry {entry.stream} because renku_io_run_id='{stream.renku_io_run_id}' " + f"is not a valid ULID: {err}" + ) + continue + # logger.info(stream) for nano_ts, log_line in entry.values: log_line_id = f"{nano_ts.root}::{stream.container}::{stream.pod}" if log_line_id in log_line_ids: - # logger.info(f"Already saw: {log_line_id}") continue log_line_ids.add(log_line_id) yield models.UnsavedLogLine( id=log_line_id, - run_id=launcher_id, # TODO: fix user_id=stream.renku_io_safe_username, - launch_id=stream.renku_io_launcher_id, # TODO: fix + run_id=run_id, + session_uid=stream.renku_io_session_uid, launcher_id=launcher_id, submission_id=stream.renku_io_submission_id, container=stream.container, diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index c46c973c8..63be7edf2 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -174,47 +174,24 @@ async def insert_session_logs( if existing_log_orm: continue - # NOTE: temporary code for finding a session run for jobs (one per submission) - run_id = log.run_id - if log.submission_id: - session_run_res = await session.scalars( - select(schemas.SessionRunsORM) - .where(schemas.SessionRunsORM.user_id == log.user_id) - .where(schemas.SessionRunsORM.submission_id == log.submission_id) - ) - session_run_orm = session_run_res.one_or_none() - if session_run_orm is None: - run_id = ULID() - session_run_orm = schemas.SessionRunsORM( - id=run_id, - user_id=log.user_id, - launch_id=log.launch_id, - launcher_id=log.launcher_id, - submission_id=log.submission_id, - ) - session.add(session_run_orm) - await session.flush() - else: - run_id = session_run_orm.id - else: - session_run_res = await session.scalars( - select(schemas.SessionRunsORM).where(schemas.SessionRunsORM.id == log.run_id) + session_run_res = await session.scalars( + select(schemas.SessionRunsORM).where(schemas.SessionRunsORM.id == log.run_id) + ) + session_run_orm = session_run_res.one_or_none() + if session_run_orm is None: + session_run_orm = schemas.SessionRunsORM( + id=log.run_id, + user_id=log.user_id, + session_uid=log.session_uid, + launcher_id=log.launcher_id, + submission_id=log.submission_id, ) - session_run_orm = session_run_res.one_or_none() - if session_run_orm is None: - session_run_orm = schemas.SessionRunsORM( - id=log.run_id, - user_id=log.user_id, - launch_id=log.launch_id, - launcher_id=log.launcher_id, - submission_id=log.submission_id, - ) - session.add(session_run_orm) - await session.flush() + session.add(session_run_orm) + await session.flush() log_orm = schemas.AmaltheaSessionLogsORM( id=log.id, - run_id=run_id, + run_id=log.run_id, container=log.container, timestamp=log.timestamp, log_line=log.log_line, diff --git a/components/renku_data_services/persisted_logs/loki_api.py b/components/renku_data_services/persisted_logs/loki_api.py index 417a2f6ac..7e47a07c9 100644 --- a/components/renku_data_services/persisted_logs/loki_api.py +++ b/components/renku_data_services/persisted_logs/loki_api.py @@ -66,6 +66,8 @@ class AmaltheaSessionStream(Base): pod: str renku_io_launcher_id: str renku_io_project_id: str | None = None + renku_io_run_id: str renku_io_safe_username: str renku_io_session_type: str | None = None + renku_io_session_uid: str | None = None renku_io_submission_id: str | None = None diff --git a/components/renku_data_services/persisted_logs/models.py b/components/renku_data_services/persisted_logs/models.py index e0bc9020a..91d4b0787 100644 --- a/components/renku_data_services/persisted_logs/models.py +++ b/components/renku_data_services/persisted_logs/models.py @@ -16,9 +16,9 @@ class UnsavedLogLine: This is used to de-duplicate log lines. """ - run_id: ULID user_id: str - launch_id: str + run_id: ULID + session_uid: str | None launcher_id: ULID submission_id: str | None container: str @@ -32,7 +32,7 @@ class SessionRun: id: ULID user_id: str - launch_id: str + session_uid: str | None launcher_id: ULID submission_id: str | None diff --git a/components/renku_data_services/persisted_logs/orm.py b/components/renku_data_services/persisted_logs/orm.py index 2431aa467..ee8bcf001 100644 --- a/components/renku_data_services/persisted_logs/orm.py +++ b/components/renku_data_services/persisted_logs/orm.py @@ -31,8 +31,8 @@ class SessionRunsORM(BaseORM): user_id: Mapped[str] = mapped_column(ForeignKey(UserORM.keycloak_id), index=True, nullable=False) """User ID of the owner of the session.""" - launch_id: Mapped[str] = mapped_column(nullable=False) - """The launch ID for this session run.""" + session_uid: Mapped[str | None] = mapped_column(nullable=True) + """The session UID for this session run.""" launcher_id: Mapped[ULID] = mapped_column(ULIDType, ForeignKey(SessionLauncherORM.id), index=True, nullable=False) """The session launcher ID of the session.""" @@ -45,7 +45,7 @@ def dump(self) -> models.SessionRun: return models.SessionRun( id=self.id, user_id=self.user_id, - launch_id=self.launch_id, + session_uid=self.session_uid, launcher_id=self.launcher_id, submission_id=self.submission_id, ) From 2866c94dbb859041172fb47598ad34f7abba3452 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Fri, 24 Jul 2026 07:01:19 +0000 Subject: [PATCH 32/50] remove debug logs --- components/renku_data_services/persisted_logs/collector.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/components/renku_data_services/persisted_logs/collector.py b/components/renku_data_services/persisted_logs/collector.py index c0945dc00..2d1cd25a2 100644 --- a/components/renku_data_services/persisted_logs/collector.py +++ b/components/renku_data_services/persisted_logs/collector.py @@ -56,10 +56,8 @@ async def get_amalthea_session_logs( params["start"] = str(start) if end: params["end"] = str(end) - logger.info(params) res = await self.client.get("loki/api/v1/query_range", params=params) res.raise_for_status() - logger.info(res) result = loki_api.LokiQueryRangeResponse.model_validate_json(res.content) log_line_ids: set[str] = set() @@ -183,10 +181,6 @@ async def collect_sessions_persisted_logs(self) -> None: current_start = result.last_timestamp + 1 has_more = result.log_count > 1 - async with session.begin(): - ts = await self.session_logs_repo.get_latest_log_timestamp(session=session) - logger.info(f"Latest session log timestamp: {ts}") - return None From 94c3d1a56de60308a51cd557e266cc943a8ff107 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Fri, 24 Jul 2026 08:15:32 +0000 Subject: [PATCH 33/50] feat: purge old logs --- .../data_tasks/task_defs.py | 12 +++++++ .../persisted_logs/collector.py | 25 ++++++++++++++ .../persisted_logs/collector_t.py | 2 ++ .../persisted_logs/config.py | 3 ++ .../persisted_logs/core.py | 19 +++++++++++ .../renku_data_services/persisted_logs/db.py | 34 +++++++++++++++++-- 6 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 components/renku_data_services/persisted_logs/core.py diff --git a/bases/renku_data_services/data_tasks/task_defs.py b/bases/renku_data_services/data_tasks/task_defs.py index 9fb6f785b..e762fe851 100644 --- a/bases/renku_data_services/data_tasks/task_defs.py +++ b/bases/renku_data_services/data_tasks/task_defs.py @@ -588,6 +588,17 @@ async def collect_persisted_logs(dm: DependencyManager) -> None: await asyncio.sleep(1) +async def purge_expired_persisted_logs(dm: DependencyManager) -> None: + """Purge expired persisted logs from the database.""" + while True: + try: + await dm.persisted_logs_collector.purge_expired_logs() + except Exception as e: + logger.warning(f"Failed to purge expired persisted logs: {e}", exc_info=True) + else: + await asyncio.sleep(dm.config.long_task_period_s) + + def all_tasks(dm: DependencyManager) -> TaskDefininions: """A dict of task factories to be managed in main.""" # Impl. note: We pass the entire config to the coroutines, because @@ -615,5 +626,6 @@ def all_tasks(dm: DependencyManager) -> TaskDefininions: "record_resource_requests": lambda: record_resource_requests(dm), "monitor_session_quota_and_send_alerts": lambda: monitor_session_quota_and_send_alerts(dm), "collect_persisted_logs": lambda: collect_persisted_logs(dm), + "purge_expired_persisted_logs": lambda: purge_expired_persisted_logs(dm), } ) diff --git a/components/renku_data_services/persisted_logs/collector.py b/components/renku_data_services/persisted_logs/collector.py index 2d1cd25a2..1b2bc0e67 100644 --- a/components/renku_data_services/persisted_logs/collector.py +++ b/components/renku_data_services/persisted_logs/collector.py @@ -116,6 +116,11 @@ async def collect_persisted_logs(self) -> None: """Collect persisted logs from Amalthea sessions and image builds.""" ... + @abstractmethod + async def purge_expired_logs(self) -> None: + """Purge expired persisted logs from the database.""" + ... + @staticmethod def from_config( config: PersistedLogsConfig, @@ -129,6 +134,7 @@ def from_config( reader = LokiLogReader(config=config, client=http_client) return DefaultPersistedLogsCollector( session_maker=session_maker, + config=config, reader=reader, session_logs_repo=AmaltheaSessionPersistedLogsRepository(), ) @@ -142,6 +148,10 @@ async def collect_persisted_logs(self) -> None: """Collect persisted logs from Amalthea sessions and image builds.""" return None + async def purge_expired_logs(self) -> None: + """Purge expired persisted logs from the database.""" + return None + class DefaultPersistedLogsCollector(PersistedLogsCollector): """Collector for gathering persisted logs.""" @@ -149,10 +159,12 @@ class DefaultPersistedLogsCollector(PersistedLogsCollector): def __init__( self, session_maker: Callable[..., AsyncSession], + config: PersistedLogsConfig, reader: LokiLogReader, session_logs_repo: AmaltheaSessionPersistedLogsRepository, ) -> None: self.session_maker = session_maker + self.config = config self.reader = reader self.session_logs_repo = session_logs_repo @@ -183,6 +195,19 @@ async def collect_sessions_persisted_logs(self) -> None: return None + async def purge_expired_logs(self) -> None: + """Purge expired persisted logs from the database.""" + await self.purge_expired_session_logs() + return None + + async def purge_expired_session_logs(self) -> None: + """Purge expired session logs from the database.""" + now = datetime.now(tz=UTC) + async with self.session_maker() as session, session.begin(): + await self.session_logs_repo.delete_expired_session_logs(session=session, before=now) + + return None + def _one_hour_ago_in_nanos() -> int: """Returns the Unix nano timestamp corresponding to one hour ago.""" diff --git a/components/renku_data_services/persisted_logs/collector_t.py b/components/renku_data_services/persisted_logs/collector_t.py index d3fd8d091..72c9eb241 100644 --- a/components/renku_data_services/persisted_logs/collector_t.py +++ b/components/renku_data_services/persisted_logs/collector_t.py @@ -1,6 +1,7 @@ """Temp: for testing with local port-forwarding...""" import asyncio +from datetime import timedelta import httpx @@ -18,6 +19,7 @@ def _main() -> None: # loki_read_base_url="http://host.docker.internal:3100/", loki_read_base_url="http://10.6.0.96:3100/", namespace="renku", + logs_ttl=timedelta(days=1), ) reader = LokiLogReader(config, client=httpx.AsyncClient()) diff --git a/components/renku_data_services/persisted_logs/config.py b/components/renku_data_services/persisted_logs/config.py index 2acb7547d..18a964ecb 100644 --- a/components/renku_data_services/persisted_logs/config.py +++ b/components/renku_data_services/persisted_logs/config.py @@ -1,6 +1,7 @@ """Configuration for persisted logs.""" from dataclasses import dataclass +from datetime import timedelta @dataclass(eq=True, frozen=True, kw_only=True) @@ -10,6 +11,7 @@ class PersistedLogsConfig: enabled: bool loki_read_base_url: str namespace: str + logs_ttl: timedelta @classmethod def from_env(cls, namespace: str) -> "PersistedLogsConfig": @@ -23,4 +25,5 @@ def from_env(cls, namespace: str) -> "PersistedLogsConfig": enabled=True, loki_read_base_url="http://loki-read.monitoring.svc.cluster.local:3100/", namespace=namespace, + logs_ttl=timedelta(days=1), ) diff --git a/components/renku_data_services/persisted_logs/core.py b/components/renku_data_services/persisted_logs/core.py new file mode 100644 index 000000000..4c98ba6a2 --- /dev/null +++ b/components/renku_data_services/persisted_logs/core.py @@ -0,0 +1,19 @@ +"""Business logic for persisted logs.""" + +from datetime import UTC, datetime +from typing import Self + +from renku_data_services.persisted_logs.constants import ONE_SECOND_IN_NANOS + + +class NanoTimestamp(int): + """Unix timestamp in nanoseconds.""" + + def to_datetime(self) -> datetime: + """Return the corresponding datetime, trucated to xxx precision.""" + return datetime.fromtimestamp(self / float(ONE_SECOND_IN_NANOS), tz=UTC) + + @classmethod + def from_datetime(cls, dt: datetime) -> Self: + """Create a nano timestamp from a datetime object.""" + return cls(int(dt.timestamp() * 1e6) * 1000) diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index 63be7edf2..837f5f208 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -1,19 +1,23 @@ """Adapters for persisted logs database classes.""" from collections.abc import AsyncIterator +from datetime import datetime -from sqlalchemy import select +from sqlalchemy import delete, select from sqlalchemy.ext.asyncio import AsyncSession from ulid import ULID from renku_data_services import base_models, errors +from renku_data_services.app_config import logging from renku_data_services.authz.authz import Authz, ResourceType from renku_data_services.authz.models import Scope -from renku_data_services.persisted_logs import models +from renku_data_services.persisted_logs import core, models from renku_data_services.persisted_logs import orm as schemas from renku_data_services.persisted_logs.constants import SESSION_MAIN_CONTAINER from renku_data_services.session import orm as session_schemas +logger = logging.getLogger(__name__) + class AmaltheaSessionPersistedLogsReadRepository: """Repository for persisted logs of Amalthea sessions.""" @@ -199,3 +203,29 @@ async def insert_session_logs( session.add(log_orm) await session.flush() return models.InsertLogsResult(log_count=log_count, last_timestamp=last_timestamp) + + async def delete_expired_session_logs(self, session: AsyncSession, before: datetime) -> int: + """Remove expired session logs from the database.""" + nano_ts = core.NanoTimestamp.from_datetime(before) + delete_logs_stmt = delete(schemas.AmaltheaSessionLogsORM).where( + schemas.AmaltheaSessionLogsORM.timestamp < nano_ts + ) + res = await session.execute(delete_logs_stmt) + deleted_logs_count = res.rowcount + logger.info(f"deleted_logs_count = {type(deleted_logs_count)} {deleted_logs_count}") + + # Remove orphaned session runs + stmt = ( + select(schemas.SessionRunsORM) + .join( + schemas.AmaltheaSessionLogsORM, + schemas.SessionRunsORM.id == schemas.AmaltheaSessionLogsORM.run_id, + isouter=True, # isouter makes it a left-join, not an outer join + ) + .where(schemas.AmaltheaSessionLogsORM.id.is_(None)) + ) + session_runs_res = await session.stream_scalars(stmt) + async for session_run in session_runs_res: + logger.info(f"Orphaned session run: {session_run.dump()}") + + return 0 From d82a353a577cfcf7078a23fda52a44c3c095e4e6 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Fri, 24 Jul 2026 10:21:10 +0000 Subject: [PATCH 34/50] fix cutoff --- components/renku_data_services/persisted_logs/collector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/renku_data_services/persisted_logs/collector.py b/components/renku_data_services/persisted_logs/collector.py index 1b2bc0e67..92a1a94da 100644 --- a/components/renku_data_services/persisted_logs/collector.py +++ b/components/renku_data_services/persisted_logs/collector.py @@ -203,9 +203,9 @@ async def purge_expired_logs(self) -> None: async def purge_expired_session_logs(self) -> None: """Purge expired session logs from the database.""" now = datetime.now(tz=UTC) + cutoff = now - self.config.logs_ttl async with self.session_maker() as session, session.begin(): - await self.session_logs_repo.delete_expired_session_logs(session=session, before=now) - + await self.session_logs_repo.delete_expired_session_logs(session=session, before=cutoff) return None From 62cff129fa155ffc0cfc2972c7b5da77948f1b98 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Fri, 24 Jul 2026 10:58:08 +0000 Subject: [PATCH 35/50] remove old runs; backtrack more --- .../renku_data_services/persisted_logs/collector.py | 4 ++-- .../renku_data_services/persisted_logs/constants.py | 3 +++ components/renku_data_services/persisted_logs/db.py | 11 +++++------ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/components/renku_data_services/persisted_logs/collector.py b/components/renku_data_services/persisted_logs/collector.py index 92a1a94da..45826bef6 100644 --- a/components/renku_data_services/persisted_logs/collector.py +++ b/components/renku_data_services/persisted_logs/collector.py @@ -13,7 +13,7 @@ from renku_data_services.persisted_logs import loki_api, models from renku_data_services.persisted_logs.config import PersistedLogsConfig from renku_data_services.persisted_logs.constants import ( - ONE_SECOND_IN_NANOS, + ONE_MINUTE_IN_NANOS, PERSISTED_LOGS_NAMESPACE_LABEL_KEY, PERSISTED_LOGS_SESSIONS_LABEL_KEY, PERSISTED_LOGS_SESSIONS_LABEL_VALUE, @@ -181,7 +181,7 @@ async def collect_sessions_persisted_logs(self) -> None: ts = await self.session_logs_repo.get_latest_log_timestamp(session=session) start = _one_hour_ago_in_nanos() if ts is not None and ts > start: - start = ts - ONE_SECOND_IN_NANOS + start = ts - ONE_MINUTE_IN_NANOS # Loop to collect all logs, including late entries has_more = True diff --git a/components/renku_data_services/persisted_logs/constants.py b/components/renku_data_services/persisted_logs/constants.py index b4f1ee0d5..470051f58 100644 --- a/components/renku_data_services/persisted_logs/constants.py +++ b/components/renku_data_services/persisted_logs/constants.py @@ -14,5 +14,8 @@ ONE_SECOND_IN_NANOS: Final[int] = 1_000_000_000 """One second as nanoseconds (for Loki).""" +ONE_MINUTE_IN_NANOS: Final[int] = 60 * ONE_SECOND_IN_NANOS +"""One minute as nanoseconds (for Loki).""" + SESSION_MAIN_CONTAINER: Final[str] = "amalthea-session" """The name of the main pod container for Amalthea sessions.""" diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index 837f5f208..b964153a7 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -212,11 +212,10 @@ async def delete_expired_session_logs(self, session: AsyncSession, before: datet ) res = await session.execute(delete_logs_stmt) deleted_logs_count = res.rowcount - logger.info(f"deleted_logs_count = {type(deleted_logs_count)} {deleted_logs_count}") # Remove orphaned session runs stmt = ( - select(schemas.SessionRunsORM) + select(schemas.SessionRunsORM.id) .join( schemas.AmaltheaSessionLogsORM, schemas.SessionRunsORM.id == schemas.AmaltheaSessionLogsORM.run_id, @@ -224,8 +223,8 @@ async def delete_expired_session_logs(self, session: AsyncSession, before: datet ) .where(schemas.AmaltheaSessionLogsORM.id.is_(None)) ) - session_runs_res = await session.stream_scalars(stmt) - async for session_run in session_runs_res: - logger.info(f"Orphaned session run: {session_run.dump()}") + session_runs_res = await session.scalars(stmt) + session_run_ids = session_runs_res.all() + await session.execute(delete(schemas.SessionRunsORM).where(schemas.SessionRunsORM.id.in_(session_run_ids))) - return 0 + return deleted_logs_count From 56301a29f25d3c5a9e68b676b1a50e64e051fffd Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 28 Jul 2026 08:10:41 +0000 Subject: [PATCH 36/50] wip: image build logs --- ...223ca0978_feat_add_persisted_build_logs.py | 56 +++++++++++ .../persisted_logs/collector.py | 96 ++++++++++++++++--- .../persisted_logs/constants.py | 6 ++ .../persisted_logs/loki_api.py | 8 ++ .../persisted_logs/models.py | 16 ++++ .../renku_data_services/persisted_logs/orm.py | 23 ++++- 6 files changed, 191 insertions(+), 14 deletions(-) create mode 100644 components/renku_data_services/migrations/versions/f65223ca0978_feat_add_persisted_build_logs.py diff --git a/components/renku_data_services/migrations/versions/f65223ca0978_feat_add_persisted_build_logs.py b/components/renku_data_services/migrations/versions/f65223ca0978_feat_add_persisted_build_logs.py new file mode 100644 index 000000000..497aec5a8 --- /dev/null +++ b/components/renku_data_services/migrations/versions/f65223ca0978_feat_add_persisted_build_logs.py @@ -0,0 +1,56 @@ +"""feat: add persisted build logs + +Revision ID: f65223ca0978 +Revises: 906ec89ea06f +Create Date: 2026-07-28 07:59:44.706230 + +""" + +import sqlalchemy as sa +from alembic import op + +from renku_data_services.utils.sqlalchemy import ULIDType + +# revision identifiers, used by Alembic. +revision = "f65223ca0978" +down_revision = "906ec89ea06f" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "image_build_logs", + sa.Column("id", sa.String(), nullable=False), + sa.Column("build_id", ULIDType(), nullable=False), + sa.Column("container", sa.String(), nullable=False), + sa.Column("timestamp", sa.BigInteger(), nullable=False), + sa.Column("log_line", sa.String(), nullable=False), + sa.ForeignKeyConstraint(["build_id"], ["sessions.builds.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + schema="persisted_logs", + ) + op.create_index( + op.f("ix_persisted_logs_image_build_logs_build_id"), + "image_build_logs", + ["build_id"], + unique=False, + schema="persisted_logs", + ) + op.create_index( + op.f("ix_persisted_logs_image_build_logs_timestamp"), + "image_build_logs", + ["timestamp"], + unique=False, + schema="persisted_logs", + ) + + +def downgrade() -> None: + op.drop_index( + op.f("ix_persisted_logs_image_build_logs_timestamp"), table_name="image_build_logs", schema="persisted_logs" + ) + op.drop_index( + op.f("ix_persisted_logs_image_build_logs_build_id"), table_name="image_build_logs", schema="persisted_logs" + ) + op.drop_table("image_build_logs", schema="persisted_logs") diff --git a/components/renku_data_services/persisted_logs/collector.py b/components/renku_data_services/persisted_logs/collector.py index 45826bef6..325df9ae5 100644 --- a/components/renku_data_services/persisted_logs/collector.py +++ b/components/renku_data_services/persisted_logs/collector.py @@ -14,6 +14,8 @@ from renku_data_services.persisted_logs.config import PersistedLogsConfig from renku_data_services.persisted_logs.constants import ( ONE_MINUTE_IN_NANOS, + PERSISTED_LOGS_BUILD_LABEL_KEY, + PERSISTED_LOGS_BUILD_LABEL_VALUE, PERSISTED_LOGS_NAMESPACE_LABEL_KEY, PERSISTED_LOGS_SESSIONS_LABEL_KEY, PERSISTED_LOGS_SESSIONS_LABEL_VALUE, @@ -43,25 +45,16 @@ async def get_amalthea_session_logs( See also https://grafana.com/docs/loki/latest/reference/loki-http-api/#query-logs-within-a-range-of-time """ - params: dict[str, str | int] = dict() - params["query"] = ( + query = ( "{" f'{PERSISTED_LOGS_SESSIONS_LABEL_KEY}="{PERSISTED_LOGS_SESSIONS_LABEL_VALUE}",' f'{PERSISTED_LOGS_NAMESPACE_LABEL_KEY}="{self.config.namespace}"' "}" ) - params["direction"] = "forward" - params["limit"] = limit - if start: - params["start"] = str(start) - if end: - params["end"] = str(end) - res = await self.client.get("loki/api/v1/query_range", params=params) - res.raise_for_status() - result = loki_api.LokiQueryRangeResponse.model_validate_json(res.content) + response = await self._get_logs(query=query, limit=limit, start=start, end=end) log_line_ids: set[str] = set() - for entry in result.data.result: + for entry in response.data.result: stream: loki_api.AmaltheaSessionStream | None = None try: stream = loki_api.AmaltheaSessionStream.model_validate(entry.stream) @@ -87,7 +80,6 @@ async def get_amalthea_session_logs( ) continue - # logger.info(stream) for nano_ts, log_line in entry.values: log_line_id = f"{nano_ts.root}::{stream.container}::{stream.pod}" @@ -107,6 +99,84 @@ async def get_amalthea_session_logs( log_line=log_line, ) + async def get_image_build_logs( + self, limit: int = 1000, start: int | None = None, end: int | None = None + ) -> AsyncIterator[models.UnsavedBuildLogLine]: + """Fetchesimage build logs from Loki. + + Parameters: + - limit: max number of entries to return + - start: start timestamp as a Unix nano timestamp + - end: end timestamp as a Unix nano timestamp + + See also https://grafana.com/docs/loki/latest/reference/loki-http-api/#query-logs-within-a-range-of-time + """ + query = ( + "{" + f'{PERSISTED_LOGS_BUILD_LABEL_KEY}="{PERSISTED_LOGS_BUILD_LABEL_VALUE}",' + f'{PERSISTED_LOGS_NAMESPACE_LABEL_KEY}="{self.config.namespace}"' + "}" + ) + response = await self._get_logs(query=query, limit=limit, start=start, end=end) + log_line_ids: set[str] = set() + + for entry in response.data.result: + stream: loki_api.ShipwrightBuildRunStream | None = None + try: + stream = loki_api.ShipwrightBuildRunStream.model_validate(entry.stream) + except ValidationError as err: + logger.warning(f"Skipping entry {entry.stream} because of validation error: {err}") + continue + + try: + build_id = ULID.from_str(stream.renku_io_buildrun_name.upper().removeprefix("RENKU-")) + except ValueError as err: + logger.warning( + f"Skipping entry {entry.stream} because renku_io_buildrun_name='{stream.renku_io_buildrun_name}' " + f"does not contain a valid ULID: {err}" + ) + continue + + for nano_ts, log_line in entry.values: + log_line_id = f"{nano_ts.root}::{stream.container}::{stream.pod}" + + if log_line_id in log_line_ids: + continue + + log_line_ids.add(log_line_id) + yield models.UnsavedBuildLogLine( + id=log_line_id, + build_id=build_id, + container=stream.container, + timestamp=nano_ts.get_value(), + log_line=log_line, + ) + + async def _get_logs( + self, query: str, limit: int = 1000, start: int | None = None, end: int | None = None + ) -> loki_api.LokiQueryRangeResponse: + """Fetches logs from Loki, using the passed in query. + + Parameters: + - query: the Loki query + - limit: max number of entries to return + - start: start timestamp as a Unix nano timestamp + - end: end timestamp as a Unix nano timestamp + + See also https://grafana.com/docs/loki/latest/reference/loki-http-api/#query-logs-within-a-range-of-time + """ + params: dict[str, str | int] = dict() + params["query"] = query + params["direction"] = "forward" + params["limit"] = limit + if start: + params["start"] = str(start) + if end: + params["end"] = str(end) + res = await self.client.get("loki/api/v1/query_range", params=params) + res.raise_for_status() + return loki_api.LokiQueryRangeResponse.model_validate_json(res.content) + class PersistedLogsCollector: """Abstract class for gathering persisted logs.""" diff --git a/components/renku_data_services/persisted_logs/constants.py b/components/renku_data_services/persisted_logs/constants.py index 470051f58..170785e63 100644 --- a/components/renku_data_services/persisted_logs/constants.py +++ b/components/renku_data_services/persisted_logs/constants.py @@ -8,6 +8,12 @@ PERSISTED_LOGS_SESSIONS_LABEL_VALUE: Final[str] = "AmaltheaSession" """The loki label value to select session logs streams.""" +PERSISTED_LOGS_BUILD_LABEL_KEY: Final[str] = "app" +"""The loki label key to select build logs streams.""" + +PERSISTED_LOGS_BUILD_LABEL_VALUE: Final[str] = "ShipwrightBuildRun" +"""The loki label value to select build logs streams.""" + PERSISTED_LOGS_NAMESPACE_LABEL_KEY: Final[str] = "namespace" """The loki label key to select logs streams from a specific kubernetes namespace.""" diff --git a/components/renku_data_services/persisted_logs/loki_api.py b/components/renku_data_services/persisted_logs/loki_api.py index 7e47a07c9..3ce17eda9 100644 --- a/components/renku_data_services/persisted_logs/loki_api.py +++ b/components/renku_data_services/persisted_logs/loki_api.py @@ -71,3 +71,11 @@ class AmaltheaSessionStream(Base): renku_io_session_type: str | None = None renku_io_session_uid: str | None = None renku_io_submission_id: str | None = None + + +class ShipwrightBuildRunStream(Base): + """Loki stream labels for logs extracted from a Shipwright build run.""" + + container: str + pod: str + renku_io_buildrun_name: str diff --git a/components/renku_data_services/persisted_logs/models.py b/components/renku_data_services/persisted_logs/models.py index 91d4b0787..02ab912dd 100644 --- a/components/renku_data_services/persisted_logs/models.py +++ b/components/renku_data_services/persisted_logs/models.py @@ -63,3 +63,19 @@ class InsertLogsResult: log_count: int last_timestamp: int + + +@dataclass(eq=True, frozen=True, kw_only=True) +class UnsavedBuildLogLine: + """Represents an unsaved image build log line.""" + + id: str + """The ID of the log line. + + This is used to de-duplicate log lines. + """ + + build_id: ULID + container: str + timestamp: int + log_line: str diff --git a/components/renku_data_services/persisted_logs/orm.py b/components/renku_data_services/persisted_logs/orm.py index ee8bcf001..a1ef095b7 100644 --- a/components/renku_data_services/persisted_logs/orm.py +++ b/components/renku_data_services/persisted_logs/orm.py @@ -8,7 +8,7 @@ from renku_data_services.base_orm.registry import COMMON_ORM_REGISTRY from renku_data_services.persisted_logs import models -from renku_data_services.session.orm import SessionLauncherORM +from renku_data_services.session.orm import BuildORM, SessionLauncherORM from renku_data_services.users.orm import UserORM from renku_data_services.utils.sqlalchemy import ULIDType @@ -73,3 +73,24 @@ class AmaltheaSessionLogsORM(BaseORM): log_line: Mapped[str] = mapped_column(nullable=False) """The contents of the log line.""" + + +class ImageBuildLogsORM(BaseORM): + """A log line from an image build.""" + + __tablename__ = "image_build_logs" + + id: Mapped[str] = mapped_column("id", primary_key=True, nullable=False) + """ID of the log line.""" + + build_id: Mapped[ULID] = mapped_column(ForeignKey(BuildORM.id, ondelete="CASCADE"), index=True, nullable=False) + """ID of the image build.""" + + container: Mapped[str] = mapped_column(nullable=False) + """The container this log line belongs to.""" + + timestamp: Mapped[int] = mapped_column(BigInteger, index=True, nullable=False) + """The timestamp of the log line (nanosecond timestamp).""" + + log_line: Mapped[str] = mapped_column(nullable=False) + """The contents of the log line.""" From 01bfd98f59769c42047be6d6ad8609f489394bda Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 28 Jul 2026 09:49:02 +0000 Subject: [PATCH 37/50] wip: image build logs --- .../persisted_logs/collector.py | 43 +++++++++++++-- .../renku_data_services/persisted_logs/db.py | 53 +++++++++++++++++++ 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/components/renku_data_services/persisted_logs/collector.py b/components/renku_data_services/persisted_logs/collector.py index 325df9ae5..51a494a23 100644 --- a/components/renku_data_services/persisted_logs/collector.py +++ b/components/renku_data_services/persisted_logs/collector.py @@ -1,5 +1,6 @@ """Collector for gathering persisted logs.""" +import asyncio from abc import abstractmethod from collections.abc import AsyncIterator, Callable from datetime import UTC, datetime, timedelta @@ -20,7 +21,10 @@ PERSISTED_LOGS_SESSIONS_LABEL_KEY, PERSISTED_LOGS_SESSIONS_LABEL_VALUE, ) -from renku_data_services.persisted_logs.db import AmaltheaSessionPersistedLogsRepository +from renku_data_services.persisted_logs.db import ( + AmaltheaSessionPersistedLogsRepository, + ImageBuildPersistedLogsWriteRepository, +) logger = logging.getLogger(__name__) @@ -207,6 +211,7 @@ def from_config( config=config, reader=reader, session_logs_repo=AmaltheaSessionPersistedLogsRepository(), + build_logs_repo=ImageBuildPersistedLogsWriteRepository(), ) return NoopPersistedLogsCollector() @@ -232,15 +237,17 @@ def __init__( config: PersistedLogsConfig, reader: LokiLogReader, session_logs_repo: AmaltheaSessionPersistedLogsRepository, + build_logs_repo: ImageBuildPersistedLogsWriteRepository, ) -> None: self.session_maker = session_maker self.config = config self.reader = reader self.session_logs_repo = session_logs_repo + self.build_logs_repo = build_logs_repo async def collect_persisted_logs(self) -> None: """Collect persisted logs from Amalthea sessions and image builds.""" - await self.collect_sessions_persisted_logs() + await asyncio.gather(self.collect_sessions_persisted_logs(), self.collect_build_persisted_logs()) return None async def collect_sessions_persisted_logs(self) -> None: @@ -265,9 +272,31 @@ async def collect_sessions_persisted_logs(self) -> None: return None + async def collect_build_persisted_logs(self) -> None: + """Collect persisted logs from image builds.""" + + async with self.session_maker() as session: + async with session.begin(): + ts = await self.build_logs_repo.get_latest_log_timestamp(session=session) + start = _one_hour_ago_in_nanos() + if ts is not None and ts > start: + start = ts - ONE_MINUTE_IN_NANOS + + # Loop to collect all logs, including late entries + has_more = True + current_start = start + while has_more: + logs_stream = self.reader.get_image_build_logs(start=current_start) + async with session.begin(): + result = await self.build_logs_repo.insert_build_logs(session=session, logs_stream=logs_stream) + current_start = result.last_timestamp + 1 + has_more = result.log_count > 1 + + return None + async def purge_expired_logs(self) -> None: """Purge expired persisted logs from the database.""" - await self.purge_expired_session_logs() + await asyncio.gather(self.purge_expired_session_logs(), self.purge_expired_build_logs()) return None async def purge_expired_session_logs(self) -> None: @@ -278,6 +307,14 @@ async def purge_expired_session_logs(self) -> None: await self.session_logs_repo.delete_expired_session_logs(session=session, before=cutoff) return None + async def purge_expired_build_logs(self) -> None: + """Purge expired image build logs from the database.""" + now = datetime.now(tz=UTC) + cutoff = now - self.config.logs_ttl + async with self.session_maker() as session, session.begin(): + await self.build_logs_repo.delete_expired_build_logs(session=session, before=cutoff) + return None + def _one_hour_ago_in_nanos() -> int: """Returns the Unix nano timestamp corresponding to one hour ago.""" diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index b964153a7..56763b873 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -228,3 +228,56 @@ async def delete_expired_session_logs(self, session: AsyncSession, before: datet await session.execute(delete(schemas.SessionRunsORM).where(schemas.SessionRunsORM.id.in_(session_run_ids))) return deleted_logs_count + + +class ImageBuildPersistedLogsWriteRepository: + """Repository for writing persisted logs of image builds.""" + + async def get_latest_log_timestamp(self, session: AsyncSession) -> int | None: + """Returns the latest log timestamp.""" + stmt = ( + select(schemas.ImageBuildLogsORM.timestamp) + .select_from(schemas.ImageBuildLogsORM) + .order_by(schemas.ImageBuildLogsORM.timestamp.desc()) + .limit(1) + ) + res = await session.scalars(stmt) + timestamp = res.one_or_none() + return timestamp + + async def insert_build_logs( + self, session: AsyncSession, logs_stream: AsyncIterator[models.UnsavedBuildLogLine] + ) -> models.InsertLogsResult: + """Insert sessions logs into the persisted logs database.""" + log_count = 0 + last_timestamp = 0 + async for log in logs_stream: + log_count += 1 + if log.timestamp > last_timestamp: + last_timestamp = log.timestamp + + existing_log_res = await session.scalars( + select(schemas.ImageBuildLogsORM.id).where(schemas.ImageBuildLogsORM.id == log.id) + ) + existing_log_orm = existing_log_res.one_or_none() + if existing_log_orm: + continue + + log_orm = schemas.ImageBuildLogsORM( + id=log.id, + build_id=log.build_id, + container=log.container, + timestamp=log.timestamp, + log_line=log.log_line, + ) + session.add(log_orm) + await session.flush() + return models.InsertLogsResult(log_count=log_count, last_timestamp=last_timestamp) + + async def delete_expired_build_logs(self, session: AsyncSession, before: datetime) -> int: + """Remove expired build logs from the database.""" + nano_ts = core.NanoTimestamp.from_datetime(before) + delete_logs_stmt = delete(schemas.ImageBuildLogsORM).where(schemas.ImageBuildLogsORM.timestamp < nano_ts) + res = await session.execute(delete_logs_stmt) + deleted_logs_count = res.rowcount + return deleted_logs_count From 3cfff9365ea923ba1a05ede7af7795c191e2dc55 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 28 Jul 2026 11:46:53 +0000 Subject: [PATCH 38/50] draft: persisted build logs --- bases/renku_data_services/data_api/app.py | 1 + .../data_api/dependencies.py | 8 +- .../persisted_logs/api.spec.yaml | 34 ++++++++ .../persisted_logs/apispec.py | 8 +- .../persisted_logs/blueprints.py | 25 +++++- .../persisted_logs/constants.py | 3 + .../renku_data_services/persisted_logs/db.py | 85 ++++++++++++++++++- 7 files changed, 160 insertions(+), 4 deletions(-) diff --git a/bases/renku_data_services/data_api/app.py b/bases/renku_data_services/data_api/app.py index bfe221f4f..e2fc871cb 100644 --- a/bases/renku_data_services/data_api/app.py +++ b/bases/renku_data_services/data_api/app.py @@ -312,6 +312,7 @@ def register_all_handlers(app: Sanic, dm: DependencyManager) -> Sanic: name="persisted_logs", url_prefix=url_prefix, session_logs_repo=dm.session_logs_repo, + build_logs_repo=dm.build_logs_repo, authenticator=dm.authenticator, session_maker=dm.config.db.async_session_maker, ) diff --git a/bases/renku_data_services/data_api/dependencies.py b/bases/renku_data_services/data_api/dependencies.py index b0353c052..ba23f6315 100644 --- a/bases/renku_data_services/data_api/dependencies.py +++ b/bases/renku_data_services/data_api/dependencies.py @@ -60,7 +60,10 @@ from renku_data_services.notebooks.data_sources import DataSourceRepository from renku_data_services.notebooks.image_check import ImageCheckRepository from renku_data_services.notifications.db import NotificationsRepository -from renku_data_services.persisted_logs.db import AmaltheaSessionPersistedLogsReadRepository +from renku_data_services.persisted_logs.db import ( + AmaltheaSessionPersistedLogsReadRepository, + ImageBuildPersistedLogsReadRepository, +) from renku_data_services.platform.db import PlatformRepository, UrlRedirectRepository from renku_data_services.project.db import ( ProjectMemberRepository, @@ -172,6 +175,7 @@ class DependencyManager: resource_requests_repo: ResourceRequestsRepo resource_usage_service: ResourceUsageService session_logs_repo: AmaltheaSessionPersistedLogsReadRepository + build_logs_repo: ImageBuildPersistedLogsReadRepository zenodo_client: ZenodoAPIClient job_client: DepositUploadJobClient secret_client: K8sSecretClient @@ -472,6 +476,7 @@ def from_env(cls) -> DependencyManager: session_maker=config.db.async_session_maker, ) session_logs_repo = AmaltheaSessionPersistedLogsReadRepository(authz=authz) + build_logs_repo = ImageBuildPersistedLogsReadRepository(authz=authz) return cls( config, k8s_client=client, @@ -519,6 +524,7 @@ def from_env(cls) -> DependencyManager: resource_requests_repo=resource_requests_repo, resource_usage_service=resource_usage_service, session_logs_repo=session_logs_repo, + build_logs_repo=build_logs_repo, zenodo_client=ZenodoAPIClient(), job_client=job_client, secret_client=secret_client, diff --git a/components/renku_data_services/persisted_logs/api.spec.yaml b/components/renku_data_services/persisted_logs/api.spec.yaml index 5b09e6eb0..af9dee9f6 100644 --- a/components/renku_data_services/persisted_logs/api.spec.yaml +++ b/components/renku_data_services/persisted_logs/api.spec.yaml @@ -66,6 +66,32 @@ paths: $ref: "#/components/responses/Error" tags: - persisted_logs + /persisted_logs/builds/{build_id}: + get: + parameters: + - in: path + name: build_id + required: true + schema: + $ref: "#/components/schemas/Ulid" + responses: + "200": + description: | + The image build logs from the corresponding image build. + content: + application/json: + schema: + $ref: "#/components/schemas/PersistedBuildLogs" + "404": + description: The image build does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + default: + $ref: "#/components/responses/Error" + tags: + - persisted_logs components: schemas: PersistedSessionLogs: @@ -99,6 +125,14 @@ components: required: - id - launcher_id + PersistedBuildLogs: + description: Logs for an image build + type: object + properties: + logs: + $ref: "#/components/schemas/SessionRunLogs" + required: + - logs SessionRunLogs: description: Logs of a session run, organized by pod container type: array diff --git a/components/renku_data_services/persisted_logs/apispec.py b/components/renku_data_services/persisted_logs/apispec.py index 72e92875b..9d84944cb 100644 --- a/components/renku_data_services/persisted_logs/apispec.py +++ b/components/renku_data_services/persisted_logs/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2026-07-22T08:48:21+00:00 +# timestamp: 2026-07-28T11:20:45+00:00 from __future__ import annotations @@ -82,3 +82,9 @@ class PersistedSessionLogs(BaseAPISpec): logs: list[SessionRunLog] = Field( ..., description="Logs of a session run, organized by pod container" ) + + +class PersistedBuildLogs(BaseAPISpec): + logs: list[SessionRunLog] = Field( + ..., description="Logs of a session run, organized by pod container" + ) diff --git a/components/renku_data_services/persisted_logs/blueprints.py b/components/renku_data_services/persisted_logs/blueprints.py index 4952aca28..3fc20b7eb 100644 --- a/components/renku_data_services/persisted_logs/blueprints.py +++ b/components/renku_data_services/persisted_logs/blueprints.py @@ -15,7 +15,10 @@ from renku_data_services.base_api.misc import validate_query from renku_data_services.base_models.validation import validated_json from renku_data_services.persisted_logs import apispec, models -from renku_data_services.persisted_logs.db import AmaltheaSessionPersistedLogsReadRepository +from renku_data_services.persisted_logs.db import ( + AmaltheaSessionPersistedLogsReadRepository, + ImageBuildPersistedLogsReadRepository, +) @dataclass(kw_only=True) @@ -23,6 +26,7 @@ class PersistedLogsBP(CustomBlueprint): """Handlers for querying persisted logs.""" session_logs_repo: AmaltheaSessionPersistedLogsReadRepository + build_logs_repo: ImageBuildPersistedLogsReadRepository authenticator: base_models.Authenticator session_maker: Callable[..., AsyncSession] @@ -69,6 +73,18 @@ async def _get_session_runs(_: Request, user: base_models.APIUser, launcher_id: return "/persisted_logs/sessions//runs", ["GET"], _get_session_runs + def get_build_logs(self) -> BlueprintFactoryResponse: + """Get persisted image build logs.""" + + @authenticate(self.authenticator) + @only_authenticated + async def _get_build_logs(_: Request, user: base_models.APIUser, build_id: ULID) -> JSONResponse: + async with self.session_maker() as session, session.begin(): + result = await self.build_logs_repo.get_build_logs(session=session, user=user, build_id=build_id) + return validated_json(apispec.PersistedBuildLogs, self._dump_persisted_build_logs(result)) + + return "/persisted_logs/builds/", ["GET"], _get_build_logs + @staticmethod def _dump_persisted_session_logs(session_log: models.PersistedSessionLogs) -> dict[str, Any]: """Dump persisted session logs for API responses.""" @@ -100,3 +116,10 @@ def _dump_session_run_logs(logs: models.SessionRunLogs) -> list[dict[str, Any]]: def _dump_log_line(log_line: models.LogLine) -> dict[str, str]: """Dump a log line for API responses.""" return dict(timestamp=str(log_line.timestamp), log_line=log_line.log_line) + + @staticmethod + def _dump_persisted_build_logs(build_logs: models.SessionRunLogs) -> dict[str, Any]: + """Dump persisted session logs for API responses.""" + return dict( + logs=PersistedLogsBP._dump_session_run_logs(build_logs), + ) diff --git a/components/renku_data_services/persisted_logs/constants.py b/components/renku_data_services/persisted_logs/constants.py index 170785e63..e55b44201 100644 --- a/components/renku_data_services/persisted_logs/constants.py +++ b/components/renku_data_services/persisted_logs/constants.py @@ -25,3 +25,6 @@ SESSION_MAIN_CONTAINER: Final[str] = "amalthea-session" """The name of the main pod container for Amalthea sessions.""" + +BUILD_MAIN_CONTAINER: Final[str] = "step-build-and-push" +"""The name of the main pod container for image builds.""" diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index 56763b873..e0da08c7f 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -13,7 +13,8 @@ from renku_data_services.authz.models import Scope from renku_data_services.persisted_logs import core, models from renku_data_services.persisted_logs import orm as schemas -from renku_data_services.persisted_logs.constants import SESSION_MAIN_CONTAINER +from renku_data_services.persisted_logs.constants import BUILD_MAIN_CONTAINER, SESSION_MAIN_CONTAINER +from renku_data_services.session import models as session_models from renku_data_services.session import orm as session_schemas logger = logging.getLogger(__name__) @@ -230,6 +231,88 @@ async def delete_expired_session_logs(self, session: AsyncSession, before: datet return deleted_logs_count +class ImageBuildPersistedLogsReadRepository: + """Repository for persisted logs of image builds.""" + + def __init__(self, authz: Authz) -> None: + self.authz: Authz = authz + + async def get_build_logs( + self, session: AsyncSession, user: base_models.APIUser, build_id: ULID + ) -> models.SessionRunLogs: + """Returns persisted session logs for the given image build.""" + if not user.is_authenticated or not user.id: + raise errors.UnauthorizedError(message="You have to be authenticated to perform this operation.") + await self._check_build(session=session, user=user, build_id=build_id) + logs_per_container = await self._get_logs_per_container(session=session, build_id=build_id) + return logs_per_container + + async def _check_build(self, session: AsyncSession, user: base_models.APIUser, build_id: ULID) -> None: + """Check that the image build exists and the user has access to it.""" + stmt = select(session_schemas.BuildORM).where(session_schemas.BuildORM.id == build_id) + res = await session.scalars(stmt) + build_orm = res.one_or_none() + authorized = ( + await self._check_environment( + session=session, user=user, environment=build_orm.environment, scope=Scope.READ + ) + if build_orm is not None + else False + ) + if not authorized or build_orm is None: + raise errors.MissingResourceError( + message=f"Build with id '{build_id}' does not exist or you do not have access to it." + ) + + async def _check_environment( + self, + session: AsyncSession, + user: base_models.APIUser, + environment: session_schemas.EnvironmentORM, + scope: Scope, + ) -> bool: + """Checks whether the provided user has a specific permission on a session environment.""" + if environment.environment_kind == session_models.EnvironmentKind.GLOBAL: + return scope == Scope.READ or user.is_admin + + launcher = await session.scalar( + select(schemas.SessionLauncherORM).where(schemas.SessionLauncherORM.environment_id == environment.id) + ) + authorized = False + if launcher: + authorized = await self.authz.has_permission(user, ResourceType.project, launcher.project_id, scope) + return authorized + + async def _get_logs_per_container(self, session: AsyncSession, build_id: ULID) -> models.SessionRunLogs: + """Get the logs of a specific image build, organized by container.""" + # TODO: handle pagination? + stmt = ( + select(schemas.ImageBuildLogsORM) + .where(schemas.ImageBuildLogsORM.build_id == build_id) + .order_by(schemas.ImageBuildLogsORM.id.asc()) + ) + res = await session.stream_scalars(stmt) + logs_per_container: dict[str, list[models.LogLine]] = dict() + async for log_entry in res: + container = log_entry.container + logs = logs_per_container.get(container) + if logs is None: + logs = list[models.LogLine]() + logs_per_container[container] = logs + logs.append(models.LogLine(timestamp=log_entry.timestamp, log_line=log_entry.log_line)) + # Sort container by name, forcing "step-build-and-push" to be the first item (main container) + containers_set = set(logs_per_container.keys()) + containers: list[str] = list() + if BUILD_MAIN_CONTAINER in containers_set: + containers.append(BUILD_MAIN_CONTAINER) + containers_set.remove(BUILD_MAIN_CONTAINER) + containers.extend(sorted(containers_set)) + result: dict[str, list[models.LogLine]] = dict() + for container in containers: + result[container] = logs_per_container[container] + return result + + class ImageBuildPersistedLogsWriteRepository: """Repository for writing persisted logs of image builds.""" From 639056923becbe7f2c354ae6320f51fd6d277018 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 30 Jul 2026 11:36:58 +0000 Subject: [PATCH 39/50] improve SessionRun model --- .vscode/settings.json | 2 ++ .../persisted_logs/blueprints.py | 13 +------------ .../renku_data_services/persisted_logs/models.py | 1 - .../renku_data_services/persisted_logs/orm.py | 1 - 4 files changed, 3 insertions(+), 14 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 32c4a1c12..b5d233914 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -21,4 +21,6 @@ "bases", "components" ], + "python-envs.defaultEnvManager": "ms-python.python:poetry", + "python-envs.defaultPackageManager": "ms-python.python:poetry", } diff --git a/components/renku_data_services/persisted_logs/blueprints.py b/components/renku_data_services/persisted_logs/blueprints.py index 3fc20b7eb..8dc777db0 100644 --- a/components/renku_data_services/persisted_logs/blueprints.py +++ b/components/renku_data_services/persisted_logs/blueprints.py @@ -89,21 +89,10 @@ async def _get_build_logs(_: Request, user: base_models.APIUser, build_id: ULID) def _dump_persisted_session_logs(session_log: models.PersistedSessionLogs) -> dict[str, Any]: """Dump persisted session logs for API responses.""" return dict( - run=PersistedLogsBP._dump_session_run(session_log.run), + run=session_log.run, logs=PersistedLogsBP._dump_session_run_logs(session_log.logs), ) - @staticmethod - def _dump_session_run(session_run: models.SessionRun) -> dict[str, Any]: - """Dump a session run for API responses.""" - # NOTE: omit the "user_id" field - return dict( - id=session_run.id, - session_uid=session_run.session_uid, - launcher_id=session_run.launcher_id, - submission_id=session_run.submission_id, - ) - @staticmethod def _dump_session_run_logs(logs: models.SessionRunLogs) -> list[dict[str, Any]]: """Dump the logs of a session run, organized by pod container, for API responses.""" diff --git a/components/renku_data_services/persisted_logs/models.py b/components/renku_data_services/persisted_logs/models.py index 02ab912dd..4fba260bd 100644 --- a/components/renku_data_services/persisted_logs/models.py +++ b/components/renku_data_services/persisted_logs/models.py @@ -31,7 +31,6 @@ class SessionRun: """The continuous execution span of a session.""" id: ULID - user_id: str session_uid: str | None launcher_id: ULID submission_id: str | None diff --git a/components/renku_data_services/persisted_logs/orm.py b/components/renku_data_services/persisted_logs/orm.py index a1ef095b7..0005a7a89 100644 --- a/components/renku_data_services/persisted_logs/orm.py +++ b/components/renku_data_services/persisted_logs/orm.py @@ -44,7 +44,6 @@ def dump(self) -> models.SessionRun: """Create a session run model from the SessionRunsORM.""" return models.SessionRun( id=self.id, - user_id=self.user_id, session_uid=self.session_uid, launcher_id=self.launcher_id, submission_id=self.submission_id, From 9dedb3b43fcf0a36c3159cce6b834c9f9daee78c Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 30 Jul 2026 11:56:16 +0000 Subject: [PATCH 40/50] improve more models --- .../persisted_logs/api.spec.yaml | 48 ++++++++++++++----- .../persisted_logs/apispec.py | 16 +++---- .../persisted_logs/blueprints.py | 39 ++------------- .../renku_data_services/persisted_logs/db.py | 28 +++++------ .../persisted_logs/models.py | 12 +++-- 5 files changed, 67 insertions(+), 76 deletions(-) diff --git a/components/renku_data_services/persisted_logs/api.spec.yaml b/components/renku_data_services/persisted_logs/api.spec.yaml index af9dee9f6..c9a83afbc 100644 --- a/components/renku_data_services/persisted_logs/api.spec.yaml +++ b/components/renku_data_services/persisted_logs/api.spec.yaml @@ -10,6 +10,16 @@ servers: paths: /persisted_logs/sessions/{launcher_id}: get: + summary: Get persisted logs for a session + description: | + Returns logs for a given session launcher belonging to the current user. + + * If the `run_id` is not specified, logs from the most recent run will be returned. + * For offline jobs (`launcher_type: non-interactive`), the `submission_id` can + also be used to get the corresponding logs. + + NOTE: logs are persisted for a limited amount of time. Once expired, logs are + purged from the database. parameters: - in: path name: launcher_id @@ -21,17 +31,18 @@ paths: style: form explode: true schema: - $ref: "#/components/schemas/PersistedLogsGetQuery" + $ref: "#/components/schemas/PersistedSessionLogsGetQuery" responses: "200": description: | - The session logs from the corresponding run and container, paginated. + The persisted logs from the corresponding session run. content: application/json: schema: $ref: "#/components/schemas/PersistedSessionLogs" "404": - description: The session launcher does not exist + description: | + The session launcher does not exist or there are no logs persisted for it at the moment. content: application/json: schema: @@ -42,6 +53,11 @@ paths: - persisted_logs /persisted_logs/sessions/{launcher_id}/runs: get: + summary: Get the list of session runs for a given session launcher + description: | + Returns the list of session runs for the given session launcher. + + NOTE: session runs are removed when the corresponding logs have expired. parameters: - in: path name: launcher_id @@ -68,6 +84,12 @@ paths: - persisted_logs /persisted_logs/builds/{build_id}: get: + summary: Get persisted logs for a build + description: | + Returns logs for a given image build. + + NOTE: logs are persisted for a limited amount of time. Once expired, logs are + purged from the database. parameters: - in: path name: build_id @@ -95,13 +117,13 @@ paths: components: schemas: PersistedSessionLogs: - description: Logs for a session + description: Persisted logs for a session type: object properties: run: $ref: "#/components/schemas/SessionRun" logs: - $ref: "#/components/schemas/SessionRunLogs" + $ref: "#/components/schemas/LogsPerContainer" required: - run - logs @@ -126,19 +148,19 @@ components: - id - launcher_id PersistedBuildLogs: - description: Logs for an image build + description: Persisted logs for an image build type: object properties: logs: - $ref: "#/components/schemas/SessionRunLogs" + $ref: "#/components/schemas/LogsPerContainer" required: - logs - SessionRunLogs: - description: Logs of a session run, organized by pod container + LogsPerContainer: + description: Logs organized by pod container type: array items: - $ref: "#/components/schemas/SessionRunLog" - SessionRunLog: + $ref: "#/components/schemas/ContainerLogs" + ContainerLogs: description: Logs of a single container type: object properties: @@ -175,8 +197,8 @@ components: description: UNIX timestamp in nanoseconds type: string pattern: "[0-9]+" - PersistedLogsGetQuery: - description: Query params for querying persisted logs + PersistedSessionLogsGetQuery: + description: Query params for querying persisted logs of a session type: object properties: run_id: diff --git a/components/renku_data_services/persisted_logs/apispec.py b/components/renku_data_services/persisted_logs/apispec.py index 9d84944cb..84aff8f2b 100644 --- a/components/renku_data_services/persisted_logs/apispec.py +++ b/components/renku_data_services/persisted_logs/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2026-07-28T11:20:45+00:00 +# timestamp: 2026-07-30T11:38:47+00:00 from __future__ import annotations @@ -8,7 +8,7 @@ from renku_data_services.persisted_logs.apispec_base import BaseAPISpec -class PersistedLogsGetQuery(BaseAPISpec): +class PersistedSessionLogsGetQuery(BaseAPISpec): run_id: str | None = Field( None, description="ULID identifier", @@ -39,7 +39,7 @@ class ErrorResponse(BaseAPISpec): class PersistedLogsSessionsLauncherIdGetParametersQuery(BaseAPISpec): - params: PersistedLogsGetQuery | None = None + params: PersistedSessionLogsGetQuery | None = None class SessionRun(BaseAPISpec): @@ -72,19 +72,19 @@ class SessionRuns(RootModel[list[SessionRun]]): root: list[SessionRun] = Field(..., description="A list of session runs") -class SessionRunLog(BaseAPISpec): +class ContainerLogs(BaseAPISpec): container: str logs: list[PersistedLogLine] = Field(..., description="Stream of log lines") class PersistedSessionLogs(BaseAPISpec): run: SessionRun - logs: list[SessionRunLog] = Field( - ..., description="Logs of a session run, organized by pod container" + logs: list[ContainerLogs] = Field( + ..., description="Logs organized by pod container" ) class PersistedBuildLogs(BaseAPISpec): - logs: list[SessionRunLog] = Field( - ..., description="Logs of a session run, organized by pod container" + logs: list[ContainerLogs] = Field( + ..., description="Logs organized by pod container" ) diff --git a/components/renku_data_services/persisted_logs/blueprints.py b/components/renku_data_services/persisted_logs/blueprints.py index 8dc777db0..41636e10c 100644 --- a/components/renku_data_services/persisted_logs/blueprints.py +++ b/components/renku_data_services/persisted_logs/blueprints.py @@ -2,7 +2,6 @@ from collections.abc import Callable from dataclasses import dataclass -from typing import Any from sanic import Request from sanic.response import JSONResponse @@ -35,9 +34,9 @@ def get_session_logs(self) -> BlueprintFactoryResponse: @authenticate(self.authenticator) @only_authenticated - @validate_query(query=apispec.PersistedLogsGetQuery) + @validate_query(query=apispec.PersistedSessionLogsGetQuery) async def _get_session_logs( - _: Request, user: base_models.APIUser, launcher_id: ULID, query: apispec.PersistedLogsGetQuery + _: Request, user: base_models.APIUser, launcher_id: ULID, query: apispec.PersistedSessionLogsGetQuery ) -> JSONResponse: run_id = ULID.from_str(query.run_id) if query.run_id else None async with self.session_maker() as session, session.begin(): @@ -50,9 +49,9 @@ async def _get_session_logs( ) if result is None: raise errors.MissingResourceError( - message=f"Session launcher with id '{launcher_id}' does not have logs yet." + message=f"Session launcher with id '{launcher_id}' does not have persisted." ) - return validated_json(apispec.PersistedSessionLogs, self._dump_persisted_session_logs(result)) + return validated_json(apispec.PersistedSessionLogs, result) return "/persisted_logs/sessions/", ["GET"], _get_session_logs @@ -81,34 +80,6 @@ def get_build_logs(self) -> BlueprintFactoryResponse: async def _get_build_logs(_: Request, user: base_models.APIUser, build_id: ULID) -> JSONResponse: async with self.session_maker() as session, session.begin(): result = await self.build_logs_repo.get_build_logs(session=session, user=user, build_id=build_id) - return validated_json(apispec.PersistedBuildLogs, self._dump_persisted_build_logs(result)) + return validated_json(apispec.PersistedBuildLogs, dict(logs=result)) return "/persisted_logs/builds/", ["GET"], _get_build_logs - - @staticmethod - def _dump_persisted_session_logs(session_log: models.PersistedSessionLogs) -> dict[str, Any]: - """Dump persisted session logs for API responses.""" - return dict( - run=session_log.run, - logs=PersistedLogsBP._dump_session_run_logs(session_log.logs), - ) - - @staticmethod - def _dump_session_run_logs(logs: models.SessionRunLogs) -> list[dict[str, Any]]: - """Dump the logs of a session run, organized by pod container, for API responses.""" - return [ - dict(container=container, logs=[PersistedLogsBP._dump_log_line(log_line) for log_line in log_lines]) - for container, log_lines in logs.items() - ] - - @staticmethod - def _dump_log_line(log_line: models.LogLine) -> dict[str, str]: - """Dump a log line for API responses.""" - return dict(timestamp=str(log_line.timestamp), log_line=log_line.log_line) - - @staticmethod - def _dump_persisted_build_logs(build_logs: models.SessionRunLogs) -> dict[str, Any]: - """Dump persisted session logs for API responses.""" - return dict( - logs=PersistedLogsBP._dump_session_run_logs(build_logs), - ) diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index e0da08c7f..9b7d5cfa8 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -1,6 +1,6 @@ """Adapters for persisted logs database classes.""" -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Sequence from datetime import datetime from sqlalchemy import delete, select @@ -8,7 +8,6 @@ from ulid import ULID from renku_data_services import base_models, errors -from renku_data_services.app_config import logging from renku_data_services.authz.authz import Authz, ResourceType from renku_data_services.authz.models import Scope from renku_data_services.persisted_logs import core, models @@ -17,8 +16,6 @@ from renku_data_services.session import models as session_models from renku_data_services.session import orm as session_schemas -logger = logging.getLogger(__name__) - class AmaltheaSessionPersistedLogsReadRepository: """Repository for persisted logs of Amalthea sessions.""" @@ -43,7 +40,6 @@ async def get_session_logs( ) if session_run is None: return None - logs_per_container = await self._get_logs_per_container(session=session, run_id=session_run.id) return models.PersistedSessionLogs( run=session_run, @@ -116,7 +112,7 @@ async def _get_session_run( return None return session_run_orm.dump() - async def _get_logs_per_container(self, session: AsyncSession, run_id: ULID) -> models.SessionRunLogs: + async def _get_logs_per_container(self, session: AsyncSession, run_id: ULID) -> Sequence[models.ContainerLogs]: """Get the logs of a specific session run, organized by container.""" # TODO: handle pagination? stmt = ( @@ -133,17 +129,16 @@ async def _get_logs_per_container(self, session: AsyncSession, run_id: ULID) -> logs = list[models.LogLine]() logs_per_container[container] = logs logs.append(models.LogLine(timestamp=log_entry.timestamp, log_line=log_entry.log_line)) - # Sort container by name, forcing "amalthea-session" to be the first item (main container) + # Sort containers by name, forcing "amalthea-session" to be the first item (main container) containers_set = set(logs_per_container.keys()) containers: list[str] = list() if SESSION_MAIN_CONTAINER in containers_set: containers.append(SESSION_MAIN_CONTAINER) containers_set.remove(SESSION_MAIN_CONTAINER) containers.extend(sorted(containers_set)) - result: dict[str, list[models.LogLine]] = dict() - for container in containers: - result[container] = logs_per_container[container] - return result + return [ + models.ContainerLogs(container=container, logs=logs_per_container[container]) for container in containers + ] class AmaltheaSessionPersistedLogsRepository: @@ -239,7 +234,7 @@ def __init__(self, authz: Authz) -> None: async def get_build_logs( self, session: AsyncSession, user: base_models.APIUser, build_id: ULID - ) -> models.SessionRunLogs: + ) -> Sequence[models.ContainerLogs]: """Returns persisted session logs for the given image build.""" if not user.is_authenticated or not user.id: raise errors.UnauthorizedError(message="You have to be authenticated to perform this operation.") @@ -283,7 +278,7 @@ async def _check_environment( authorized = await self.authz.has_permission(user, ResourceType.project, launcher.project_id, scope) return authorized - async def _get_logs_per_container(self, session: AsyncSession, build_id: ULID) -> models.SessionRunLogs: + async def _get_logs_per_container(self, session: AsyncSession, build_id: ULID) -> Sequence[models.ContainerLogs]: """Get the logs of a specific image build, organized by container.""" # TODO: handle pagination? stmt = ( @@ -307,10 +302,9 @@ async def _get_logs_per_container(self, session: AsyncSession, build_id: ULID) - containers.append(BUILD_MAIN_CONTAINER) containers_set.remove(BUILD_MAIN_CONTAINER) containers.extend(sorted(containers_set)) - result: dict[str, list[models.LogLine]] = dict() - for container in containers: - result[container] = logs_per_container[container] - return result + return [ + models.ContainerLogs(container=container, logs=logs_per_container[container]) for container in containers + ] class ImageBuildPersistedLogsWriteRepository: diff --git a/components/renku_data_services/persisted_logs/models.py b/components/renku_data_services/persisted_logs/models.py index 4fba260bd..d9fe7d70d 100644 --- a/components/renku_data_services/persisted_logs/models.py +++ b/components/renku_data_services/persisted_logs/models.py @@ -1,6 +1,6 @@ """Models for persisted logs.""" -from collections.abc import Mapping, Sequence +from collections.abc import Sequence from dataclasses import dataclass from ulid import ULID @@ -44,8 +44,12 @@ class LogLine: log_line: str -type SessionRunLogs = Mapping[str, Sequence[LogLine]] -"""Logs of a session run, organized by pod container.""" +@dataclass(eq=True, frozen=True, kw_only=True) +class ContainerLogs: + """Logs of a single container.""" + + container: str + logs: Sequence[LogLine] @dataclass(eq=True, frozen=True, kw_only=True) @@ -53,7 +57,7 @@ class PersistedSessionLogs: """Result of getting session logs from the database.""" run: SessionRun - logs: SessionRunLogs + logs: Sequence[ContainerLogs] @dataclass(eq=True, frozen=True, kw_only=True) From 8a4cbb01bd2d7db31f7732add0a21151741c6b97 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 30 Jul 2026 12:01:03 +0000 Subject: [PATCH 41/50] rename orm --- .../persisted_logs/collector.py | 4 +- .../renku_data_services/persisted_logs/db.py | 74 +++++++++---------- .../persisted_logs/models.py | 2 +- .../renku_data_services/persisted_logs/orm.py | 12 +-- 4 files changed, 46 insertions(+), 46 deletions(-) diff --git a/components/renku_data_services/persisted_logs/collector.py b/components/renku_data_services/persisted_logs/collector.py index 51a494a23..ac2a99e1d 100644 --- a/components/renku_data_services/persisted_logs/collector.py +++ b/components/renku_data_services/persisted_logs/collector.py @@ -39,7 +39,7 @@ def __init__(self, config: PersistedLogsConfig, client: httpx.AsyncClient) -> No async def get_amalthea_session_logs( self, limit: int = 1000, start: int | None = None, end: int | None = None - ) -> AsyncIterator[models.UnsavedLogLine]: + ) -> AsyncIterator[models.UnsavedSessionLogLine]: """Fetches Amalthea session logs from Loki. Parameters: @@ -91,7 +91,7 @@ async def get_amalthea_session_logs( continue log_line_ids.add(log_line_id) - yield models.UnsavedLogLine( + yield models.UnsavedSessionLogLine( id=log_line_id, user_id=stream.renku_io_safe_username, run_id=run_id, diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index 9b7d5cfa8..a6681f2ac 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -57,10 +57,10 @@ async def get_session_runs( raise errors.UnauthorizedError(message="You have to be authenticated to perform this operation.") await self._check_session_launcher(session=session, user=user, launcher_id=launcher_id) stmt = ( - select(schemas.SessionRunsORM) - .where(schemas.SessionRunsORM.user_id == user.id) - .where(schemas.SessionRunsORM.launcher_id == launcher_id) - .order_by(schemas.SessionRunsORM.id.desc()) + select(schemas.SessionRunORM) + .where(schemas.SessionRunORM.user_id == user.id) + .where(schemas.SessionRunORM.launcher_id == launcher_id) + .order_by(schemas.SessionRunORM.id.desc()) ) res = await session.stream_scalars(stmt) async for session_run_orm in res: @@ -96,16 +96,16 @@ async def _get_session_run( If no `run_id` is specified, then return the latest session run. """ stmt = ( - select(schemas.SessionRunsORM) - .where(schemas.SessionRunsORM.user_id == user_id) - .where(schemas.SessionRunsORM.launcher_id == launcher_id) - .order_by(schemas.SessionRunsORM.id.desc()) + select(schemas.SessionRunORM) + .where(schemas.SessionRunORM.user_id == user_id) + .where(schemas.SessionRunORM.launcher_id == launcher_id) + .order_by(schemas.SessionRunORM.id.desc()) .limit(1) ) if run_id: - stmt = stmt.where(schemas.SessionRunsORM.id == run_id) + stmt = stmt.where(schemas.SessionRunORM.id == run_id) if submission_id: - stmt = stmt.where(schemas.SessionRunsORM.submission_id == submission_id) + stmt = stmt.where(schemas.SessionRunORM.submission_id == submission_id) res = await session.scalars(stmt) session_run_orm = res.one_or_none() if session_run_orm is None: @@ -116,9 +116,9 @@ async def _get_logs_per_container(self, session: AsyncSession, run_id: ULID) -> """Get the logs of a specific session run, organized by container.""" # TODO: handle pagination? stmt = ( - select(schemas.AmaltheaSessionLogsORM) - .where(schemas.AmaltheaSessionLogsORM.run_id == run_id) - .order_by(schemas.AmaltheaSessionLogsORM.id.asc()) + select(schemas.AmaltheaSessionLogORM) + .where(schemas.AmaltheaSessionLogORM.run_id == run_id) + .order_by(schemas.AmaltheaSessionLogORM.id.asc()) ) res = await session.stream_scalars(stmt) logs_per_container: dict[str, list[models.LogLine]] = dict() @@ -147,9 +147,9 @@ class AmaltheaSessionPersistedLogsRepository: async def get_latest_log_timestamp(self, session: AsyncSession) -> int | None: """Returns the latest log timestamp.""" stmt = ( - select(schemas.AmaltheaSessionLogsORM.timestamp) - .select_from(schemas.AmaltheaSessionLogsORM) - .order_by(schemas.AmaltheaSessionLogsORM.timestamp.desc()) + select(schemas.AmaltheaSessionLogORM.timestamp) + .select_from(schemas.AmaltheaSessionLogORM) + .order_by(schemas.AmaltheaSessionLogORM.timestamp.desc()) .limit(1) ) res = await session.scalars(stmt) @@ -157,7 +157,7 @@ async def get_latest_log_timestamp(self, session: AsyncSession) -> int | None: return timestamp async def insert_session_logs( - self, session: AsyncSession, logs_stream: AsyncIterator[models.UnsavedLogLine] + self, session: AsyncSession, logs_stream: AsyncIterator[models.UnsavedSessionLogLine] ) -> models.InsertLogsResult: """Insert sessions logs into the persisted logs database.""" log_count = 0 @@ -168,18 +168,18 @@ async def insert_session_logs( last_timestamp = log.timestamp existing_log_res = await session.scalars( - select(schemas.AmaltheaSessionLogsORM.id).where(schemas.AmaltheaSessionLogsORM.id == log.id) + select(schemas.AmaltheaSessionLogORM.id).where(schemas.AmaltheaSessionLogORM.id == log.id) ) existing_log_orm = existing_log_res.one_or_none() if existing_log_orm: continue session_run_res = await session.scalars( - select(schemas.SessionRunsORM).where(schemas.SessionRunsORM.id == log.run_id) + select(schemas.SessionRunORM).where(schemas.SessionRunORM.id == log.run_id) ) session_run_orm = session_run_res.one_or_none() if session_run_orm is None: - session_run_orm = schemas.SessionRunsORM( + session_run_orm = schemas.SessionRunORM( id=log.run_id, user_id=log.user_id, session_uid=log.session_uid, @@ -189,7 +189,7 @@ async def insert_session_logs( session.add(session_run_orm) await session.flush() - log_orm = schemas.AmaltheaSessionLogsORM( + log_orm = schemas.AmaltheaSessionLogORM( id=log.id, run_id=log.run_id, container=log.container, @@ -203,25 +203,25 @@ async def insert_session_logs( async def delete_expired_session_logs(self, session: AsyncSession, before: datetime) -> int: """Remove expired session logs from the database.""" nano_ts = core.NanoTimestamp.from_datetime(before) - delete_logs_stmt = delete(schemas.AmaltheaSessionLogsORM).where( - schemas.AmaltheaSessionLogsORM.timestamp < nano_ts + delete_logs_stmt = delete(schemas.AmaltheaSessionLogORM).where( + schemas.AmaltheaSessionLogORM.timestamp < nano_ts ) res = await session.execute(delete_logs_stmt) deleted_logs_count = res.rowcount # Remove orphaned session runs stmt = ( - select(schemas.SessionRunsORM.id) + select(schemas.SessionRunORM.id) .join( - schemas.AmaltheaSessionLogsORM, - schemas.SessionRunsORM.id == schemas.AmaltheaSessionLogsORM.run_id, + schemas.AmaltheaSessionLogORM, + schemas.SessionRunORM.id == schemas.AmaltheaSessionLogORM.run_id, isouter=True, # isouter makes it a left-join, not an outer join ) - .where(schemas.AmaltheaSessionLogsORM.id.is_(None)) + .where(schemas.AmaltheaSessionLogORM.id.is_(None)) ) session_runs_res = await session.scalars(stmt) session_run_ids = session_runs_res.all() - await session.execute(delete(schemas.SessionRunsORM).where(schemas.SessionRunsORM.id.in_(session_run_ids))) + await session.execute(delete(schemas.SessionRunORM).where(schemas.SessionRunORM.id.in_(session_run_ids))) return deleted_logs_count @@ -282,9 +282,9 @@ async def _get_logs_per_container(self, session: AsyncSession, build_id: ULID) - """Get the logs of a specific image build, organized by container.""" # TODO: handle pagination? stmt = ( - select(schemas.ImageBuildLogsORM) - .where(schemas.ImageBuildLogsORM.build_id == build_id) - .order_by(schemas.ImageBuildLogsORM.id.asc()) + select(schemas.ImageBuildLogORM) + .where(schemas.ImageBuildLogORM.build_id == build_id) + .order_by(schemas.ImageBuildLogORM.id.asc()) ) res = await session.stream_scalars(stmt) logs_per_container: dict[str, list[models.LogLine]] = dict() @@ -313,9 +313,9 @@ class ImageBuildPersistedLogsWriteRepository: async def get_latest_log_timestamp(self, session: AsyncSession) -> int | None: """Returns the latest log timestamp.""" stmt = ( - select(schemas.ImageBuildLogsORM.timestamp) - .select_from(schemas.ImageBuildLogsORM) - .order_by(schemas.ImageBuildLogsORM.timestamp.desc()) + select(schemas.ImageBuildLogORM.timestamp) + .select_from(schemas.ImageBuildLogORM) + .order_by(schemas.ImageBuildLogORM.timestamp.desc()) .limit(1) ) res = await session.scalars(stmt) @@ -334,13 +334,13 @@ async def insert_build_logs( last_timestamp = log.timestamp existing_log_res = await session.scalars( - select(schemas.ImageBuildLogsORM.id).where(schemas.ImageBuildLogsORM.id == log.id) + select(schemas.ImageBuildLogORM.id).where(schemas.ImageBuildLogORM.id == log.id) ) existing_log_orm = existing_log_res.one_or_none() if existing_log_orm: continue - log_orm = schemas.ImageBuildLogsORM( + log_orm = schemas.ImageBuildLogORM( id=log.id, build_id=log.build_id, container=log.container, @@ -354,7 +354,7 @@ async def insert_build_logs( async def delete_expired_build_logs(self, session: AsyncSession, before: datetime) -> int: """Remove expired build logs from the database.""" nano_ts = core.NanoTimestamp.from_datetime(before) - delete_logs_stmt = delete(schemas.ImageBuildLogsORM).where(schemas.ImageBuildLogsORM.timestamp < nano_ts) + delete_logs_stmt = delete(schemas.ImageBuildLogORM).where(schemas.ImageBuildLogORM.timestamp < nano_ts) res = await session.execute(delete_logs_stmt) deleted_logs_count = res.rowcount return deleted_logs_count diff --git a/components/renku_data_services/persisted_logs/models.py b/components/renku_data_services/persisted_logs/models.py index d9fe7d70d..3a43a6656 100644 --- a/components/renku_data_services/persisted_logs/models.py +++ b/components/renku_data_services/persisted_logs/models.py @@ -7,7 +7,7 @@ @dataclass(eq=True, frozen=True, kw_only=True) -class UnsavedLogLine: +class UnsavedSessionLogLine: """Represents an unsaved log line.""" id: str diff --git a/components/renku_data_services/persisted_logs/orm.py b/components/renku_data_services/persisted_logs/orm.py index 0005a7a89..9c6476d5f 100644 --- a/components/renku_data_services/persisted_logs/orm.py +++ b/components/renku_data_services/persisted_logs/orm.py @@ -20,7 +20,7 @@ class BaseORM(MappedAsDataclass, DeclarativeBase): registry = COMMON_ORM_REGISTRY -class SessionRunsORM(BaseORM): +class SessionRunORM(BaseORM): """A session run, which is the continuous execution of a session.""" __tablename__ = "session_runs" @@ -41,7 +41,7 @@ class SessionRunsORM(BaseORM): """The submission ID, if the session run corresponds to an offline job.""" def dump(self) -> models.SessionRun: - """Create a session run model from the SessionRunsORM.""" + """Create a session run model from the SessionRunORM.""" return models.SessionRun( id=self.id, session_uid=self.session_uid, @@ -50,7 +50,7 @@ def dump(self) -> models.SessionRun: ) -class AmaltheaSessionLogsORM(BaseORM): +class AmaltheaSessionLogORM(BaseORM): """A log line from an Amalthea session.""" __tablename__ = "amalthea_session_logs" @@ -58,10 +58,10 @@ class AmaltheaSessionLogsORM(BaseORM): id: Mapped[str] = mapped_column("id", primary_key=True, nullable=False) """ID of the log line.""" - run_id: Mapped[ULID] = mapped_column(ForeignKey(SessionRunsORM.id, ondelete="CASCADE"), index=True, nullable=False) + run_id: Mapped[ULID] = mapped_column(ForeignKey(SessionRunORM.id, ondelete="CASCADE"), index=True, nullable=False) """ID of the session run.""" - session_run: Mapped[SessionRunsORM] = relationship(lazy="select", init=False, repr=False, viewonly=True) + session_run: Mapped[SessionRunORM] = relationship(lazy="select", init=False, repr=False, viewonly=True) """The session run this log line belongs to.""" container: Mapped[str] = mapped_column(nullable=False) @@ -74,7 +74,7 @@ class AmaltheaSessionLogsORM(BaseORM): """The contents of the log line.""" -class ImageBuildLogsORM(BaseORM): +class ImageBuildLogORM(BaseORM): """A log line from an image build.""" __tablename__ = "image_build_logs" From 45ab4755b390f1c646244f510f59eae6d504be24 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 30 Jul 2026 12:58:03 +0000 Subject: [PATCH 42/50] factor db.py code --- .../persisted_logs/core.py | 19 ----- .../renku_data_services/persisted_logs/db.py | 71 ++++++++----------- .../persisted_logs/models.py | 15 ++++ 3 files changed, 46 insertions(+), 59 deletions(-) delete mode 100644 components/renku_data_services/persisted_logs/core.py diff --git a/components/renku_data_services/persisted_logs/core.py b/components/renku_data_services/persisted_logs/core.py deleted file mode 100644 index 4c98ba6a2..000000000 --- a/components/renku_data_services/persisted_logs/core.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Business logic for persisted logs.""" - -from datetime import UTC, datetime -from typing import Self - -from renku_data_services.persisted_logs.constants import ONE_SECOND_IN_NANOS - - -class NanoTimestamp(int): - """Unix timestamp in nanoseconds.""" - - def to_datetime(self) -> datetime: - """Return the corresponding datetime, trucated to xxx precision.""" - return datetime.fromtimestamp(self / float(ONE_SECOND_IN_NANOS), tz=UTC) - - @classmethod - def from_datetime(cls, dt: datetime) -> Self: - """Create a nano timestamp from a datetime object.""" - return cls(int(dt.timestamp() * 1e6) * 1000) diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index a6681f2ac..e472ed786 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -4,13 +4,13 @@ from datetime import datetime from sqlalchemy import delete, select -from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.ext.asyncio import AsyncScalarResult, AsyncSession from ulid import ULID from renku_data_services import base_models, errors from renku_data_services.authz.authz import Authz, ResourceType from renku_data_services.authz.models import Scope -from renku_data_services.persisted_logs import core, models +from renku_data_services.persisted_logs import models from renku_data_services.persisted_logs import orm as schemas from renku_data_services.persisted_logs.constants import BUILD_MAIN_CONTAINER, SESSION_MAIN_CONTAINER from renku_data_services.session import models as session_models @@ -121,24 +121,8 @@ async def _get_logs_per_container(self, session: AsyncSession, run_id: ULID) -> .order_by(schemas.AmaltheaSessionLogORM.id.asc()) ) res = await session.stream_scalars(stmt) - logs_per_container: dict[str, list[models.LogLine]] = dict() - async for log_entry in res: - container = log_entry.container - logs = logs_per_container.get(container) - if logs is None: - logs = list[models.LogLine]() - logs_per_container[container] = logs - logs.append(models.LogLine(timestamp=log_entry.timestamp, log_line=log_entry.log_line)) - # Sort containers by name, forcing "amalthea-session" to be the first item (main container) - containers_set = set(logs_per_container.keys()) - containers: list[str] = list() - if SESSION_MAIN_CONTAINER in containers_set: - containers.append(SESSION_MAIN_CONTAINER) - containers_set.remove(SESSION_MAIN_CONTAINER) - containers.extend(sorted(containers_set)) - return [ - models.ContainerLogs(container=container, logs=logs_per_container[container]) for container in containers - ] + # Sort logs by container name, forcing "amalthea-session" to be the first item (main container) + return await _sort_logs_per_container(res, main_container=SESSION_MAIN_CONTAINER) class AmaltheaSessionPersistedLogsRepository: @@ -202,7 +186,7 @@ async def insert_session_logs( async def delete_expired_session_logs(self, session: AsyncSession, before: datetime) -> int: """Remove expired session logs from the database.""" - nano_ts = core.NanoTimestamp.from_datetime(before) + nano_ts = models.NanoTimestamp.from_datetime(before) delete_logs_stmt = delete(schemas.AmaltheaSessionLogORM).where( schemas.AmaltheaSessionLogORM.timestamp < nano_ts ) @@ -287,24 +271,8 @@ async def _get_logs_per_container(self, session: AsyncSession, build_id: ULID) - .order_by(schemas.ImageBuildLogORM.id.asc()) ) res = await session.stream_scalars(stmt) - logs_per_container: dict[str, list[models.LogLine]] = dict() - async for log_entry in res: - container = log_entry.container - logs = logs_per_container.get(container) - if logs is None: - logs = list[models.LogLine]() - logs_per_container[container] = logs - logs.append(models.LogLine(timestamp=log_entry.timestamp, log_line=log_entry.log_line)) - # Sort container by name, forcing "step-build-and-push" to be the first item (main container) - containers_set = set(logs_per_container.keys()) - containers: list[str] = list() - if BUILD_MAIN_CONTAINER in containers_set: - containers.append(BUILD_MAIN_CONTAINER) - containers_set.remove(BUILD_MAIN_CONTAINER) - containers.extend(sorted(containers_set)) - return [ - models.ContainerLogs(container=container, logs=logs_per_container[container]) for container in containers - ] + # Sort logs by container name, forcing "step-build-and-push" to be the first item (main container) + return await _sort_logs_per_container(res, main_container=BUILD_MAIN_CONTAINER) class ImageBuildPersistedLogsWriteRepository: @@ -353,8 +321,31 @@ async def insert_build_logs( async def delete_expired_build_logs(self, session: AsyncSession, before: datetime) -> int: """Remove expired build logs from the database.""" - nano_ts = core.NanoTimestamp.from_datetime(before) + nano_ts = models.NanoTimestamp.from_datetime(before) delete_logs_stmt = delete(schemas.ImageBuildLogORM).where(schemas.ImageBuildLogORM.timestamp < nano_ts) res = await session.execute(delete_logs_stmt) deleted_logs_count = res.rowcount return deleted_logs_count + + +async def _sort_logs_per_container( + result: AsyncScalarResult[schemas.AmaltheaSessionLogORM] | AsyncScalarResult[schemas.ImageBuildLogORM], + main_container: str | None = None, +) -> Sequence[models.ContainerLogs]: + """Organize logs per container.""" + logs_per_container: dict[str, list[models.LogLine]] = dict() + async for log_entry in result: + container = log_entry.container + logs = logs_per_container.get(container) + if logs is None: + logs = list[models.LogLine]() + logs_per_container[container] = logs + logs.append(models.LogLine(timestamp=log_entry.timestamp, log_line=log_entry.log_line)) + # Sort containers by name, forcing `main_container` to be the first item + containers_set = set(logs_per_container.keys()) + containers: list[str] = list() + if main_container and main_container in containers_set: + containers.append(main_container) + containers_set.remove(main_container) + containers.extend(sorted(containers_set)) + return [models.ContainerLogs(container=container, logs=logs_per_container[container]) for container in containers] diff --git a/components/renku_data_services/persisted_logs/models.py b/components/renku_data_services/persisted_logs/models.py index 3a43a6656..a7982e642 100644 --- a/components/renku_data_services/persisted_logs/models.py +++ b/components/renku_data_services/persisted_logs/models.py @@ -2,10 +2,25 @@ from collections.abc import Sequence from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Self from ulid import ULID +class NanoTimestamp(int): + """Unix timestamp in nanoseconds.""" + + def to_datetime(self) -> datetime: + """Return the corresponding datetime, trucated to microsecond precision.""" + return datetime.fromtimestamp((self // 1_000) / 1e6, tz=UTC) + + @classmethod + def from_datetime(cls, dt: datetime) -> Self: + """Create a nano timestamp from a datetime object.""" + return cls(int(dt.timestamp() * 1e6) * 1000) + + @dataclass(eq=True, frozen=True, kw_only=True) class UnsavedSessionLogLine: """Represents an unsaved log line.""" From b41fdc2cbaad3e6d567edf8d9881cbe57eeb3586 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 30 Jul 2026 13:18:06 +0000 Subject: [PATCH 43/50] fix timestamp --- .../renku_data_services/persisted_logs/apispec_base.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/components/renku_data_services/persisted_logs/apispec_base.py b/components/renku_data_services/persisted_logs/apispec_base.py index 6a842832f..edb200ef9 100644 --- a/components/renku_data_services/persisted_logs/apispec_base.py +++ b/components/renku_data_services/persisted_logs/apispec_base.py @@ -21,3 +21,11 @@ def serialize_ulid(cls, value: Any) -> Any: if isinstance(value, ULID): return str(value) return value + + @field_validator("timestamp", mode="before", check_fields=False) + @classmethod + def serialize_nano_timestamp(cls, value: Any) -> Any: + """Handle serializing nanosecond timestamps to string.""" + if isinstance(value, int): + return str(value) + return value From d55a41db16831415b5fd5b6629657fa12ba782c1 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 30 Jul 2026 13:39:27 +0000 Subject: [PATCH 44/50] implement strict check for private builds --- .../data_api/dependencies.py | 6 ++- .../renku_data_services/persisted_logs/db.py | 41 ++++++++++++++++++- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/bases/renku_data_services/data_api/dependencies.py b/bases/renku_data_services/data_api/dependencies.py index 81191af2a..4c753bdcd 100644 --- a/bases/renku_data_services/data_api/dependencies.py +++ b/bases/renku_data_services/data_api/dependencies.py @@ -470,7 +470,11 @@ def from_env(cls) -> DependencyManager: session_maker=config.db.async_session_maker, ) session_logs_repo = AmaltheaSessionPersistedLogsReadRepository(authz=authz) - build_logs_repo = ImageBuildPersistedLogsReadRepository(authz=authz) + build_logs_repo = ImageBuildPersistedLogsReadRepository( + authz=authz, + builds_config=config.builds, + git_repositories_repo=git_repositories_repo, + ) return cls( config, k8s_client=client, diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index e472ed786..c34632aa3 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -1,7 +1,10 @@ """Adapters for persisted logs database classes.""" +from __future__ import annotations + from collections.abc import AsyncIterator, Sequence from datetime import datetime +from typing import TYPE_CHECKING from sqlalchemy import delete, select from sqlalchemy.ext.asyncio import AsyncScalarResult, AsyncSession @@ -13,12 +16,17 @@ from renku_data_services.persisted_logs import models from renku_data_services.persisted_logs import orm as schemas from renku_data_services.persisted_logs.constants import BUILD_MAIN_CONTAINER, SESSION_MAIN_CONTAINER +from renku_data_services.repositories import models as repo_models from renku_data_services.session import models as session_models from renku_data_services.session import orm as session_schemas +if TYPE_CHECKING: + from renku_data_services.repositories.db import GitRepositoriesRepository + from renku_data_services.session.config import BuildsConfig + class AmaltheaSessionPersistedLogsReadRepository: - """Repository for persisted logs of Amalthea sessions.""" + """Repository for reading persisted logs of Amalthea sessions.""" def __init__(self, authz: Authz) -> None: self.authz: Authz = authz @@ -213,8 +221,15 @@ async def delete_expired_session_logs(self, session: AsyncSession, before: datet class ImageBuildPersistedLogsReadRepository: """Repository for persisted logs of image builds.""" - def __init__(self, authz: Authz) -> None: + def __init__( + self, + authz: Authz, + builds_config: BuildsConfig, + git_repositories_repo: GitRepositoriesRepository, + ) -> None: self.authz: Authz = authz + self.builds_config = builds_config + self.git_repositories_repo = git_repositories_repo async def get_build_logs( self, session: AsyncSession, user: base_models.APIUser, build_id: ULID @@ -238,6 +253,28 @@ async def _check_build(self, session: AsyncSession, user: base_models.APIUser, b if build_orm is not None else False ) + + # If the output image is private, check that the user can read the source repository + if build_orm is None or build_orm.result_image is None: + authorized = False + else: + if self.builds_config.private_builds_enabled and build_orm.result_image.startswith( + self.builds_config.build_output_private_image_prefix + ): + if build_orm.result_repository_url is None: + authorized = False + else: + repo_data = await self.git_repositories_repo.get_repository( + repository_url=build_orm.result_repository_url, + user=user, + etag=None, + ) + if ( + not isinstance(repo_data.metadata, repo_models.Metadata) + or not repo_data.metadata.pull_permission + ): + authorized = False + if not authorized or build_orm is None: raise errors.MissingResourceError( message=f"Build with id '{build_id}' does not exist or you do not have access to it." From ba06d3cd23e03ea4c4ad2f62d055b71b636300e9 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Fri, 31 Jul 2026 06:48:07 +0000 Subject: [PATCH 45/50] small fixes --- bases/renku_data_services/data_api/app.py | 3 +- bases/renku_data_services/data_api/config.py | 8 ++++- .../persisted_logs/blueprints.py | 3 +- .../persisted_logs/collector.py | 6 ++-- .../persisted_logs/collector_t.py | 34 ------------------- .../persisted_logs/config.py | 2 +- .../renku_data_services/persisted_logs/db.py | 14 +++++--- .../persisted_logs/models.py | 32 ++++++++--------- 8 files changed, 41 insertions(+), 61 deletions(-) delete mode 100644 components/renku_data_services/persisted_logs/collector_t.py diff --git a/bases/renku_data_services/data_api/app.py b/bases/renku_data_services/data_api/app.py index a9d540641..781f40786 100644 --- a/bases/renku_data_services/data_api/app.py +++ b/bases/renku_data_services/data_api/app.py @@ -347,12 +347,13 @@ def register_all_handlers(app: Sanic, dm: DependencyManager) -> Sanic: notifications.blueprint(), capacity_reservation.blueprint(), resource_usage.blueprint(), - persisted_logs.blueprint(), internal_authentication.blueprint(), ] ) if builds is not None: app.blueprint(builds.blueprint()) + if dm.config.persisted_logs.enabled: + app.blueprint(persisted_logs.blueprint()) # We need to patch sanic_ext as since version 24.12 they only send a string representation of errors import sanic_ext.extras.validation.setup diff --git a/bases/renku_data_services/data_api/config.py b/bases/renku_data_services/data_api/config.py index 045176de5..c1ec557b2 100644 --- a/bases/renku_data_services/data_api/config.py +++ b/bases/renku_data_services/data_api/config.py @@ -17,6 +17,7 @@ from renku_data_services.data_connectors.config import DepositConfig from renku_data_services.db_config.config import DBConfig from renku_data_services.notebooks.config import NotebooksConfig +from renku_data_services.persisted_logs.config import PersistedLogsConfig from renku_data_services.secrets.config import PublicSecretsConfig from renku_data_services.session.config import BuildsConfig from renku_data_services.solr.solr_client import SolrClientConfig @@ -48,6 +49,7 @@ class Config: version: str alertmanager_webhook_role: str deposit_config: DepositConfig + persisted_logs: PersistedLogsConfig @classmethod def from_env(cls, db: DBConfig | None = None) -> Self: @@ -73,11 +75,14 @@ def from_env(cls, db: DBConfig | None = None) -> Self: gitlab_url = None nb_config = NotebooksConfig.from_env(db, authz_config, enable_internal_gitlab=enable_internal_gitlab) + + k8s_namespace = os.environ.get("K8S_NAMESPACE", "default") + return cls( enable_internal_gitlab=enable_internal_gitlab, version=os.environ.get("VERSION", "0.0.1"), dummy_stores=dummy_stores, - k8s_namespace=os.environ.get("K8S_NAMESPACE", "default"), + k8s_namespace=k8s_namespace, k8s_config_root=os.environ.get("K8S_CONFIGS_ROOT", "/secrets/kube_configs"), db=db, builds=BuildsConfig.from_env(), @@ -95,4 +100,5 @@ def from_env(cls, db: DBConfig | None = None) -> Self: log_cfg=LoggingConfig.from_env(), alertmanager_webhook_role=os.environ.get("ALERTMANAGER_WEBHOOK_ROLE", "alertmanager-webhook"), deposit_config=DepositConfig.from_env(nb_config.sessions.renku_url), + persisted_logs=PersistedLogsConfig.from_env(namespace=k8s_namespace), ) diff --git a/components/renku_data_services/persisted_logs/blueprints.py b/components/renku_data_services/persisted_logs/blueprints.py index 41636e10c..bcf6f1990 100644 --- a/components/renku_data_services/persisted_logs/blueprints.py +++ b/components/renku_data_services/persisted_logs/blueprints.py @@ -48,8 +48,9 @@ async def _get_session_logs( submission_id=query.submission_id, ) if result is None: + # TODO: adjust error message when params are passed in the query raise errors.MissingResourceError( - message=f"Session launcher with id '{launcher_id}' does not have persisted." + message=f"Session launcher with id '{launcher_id}' does not have persisted logs." ) return validated_json(apispec.PersistedSessionLogs, result) diff --git a/components/renku_data_services/persisted_logs/collector.py b/components/renku_data_services/persisted_logs/collector.py index ac2a99e1d..c382c3f31 100644 --- a/components/renku_data_services/persisted_logs/collector.py +++ b/components/renku_data_services/persisted_logs/collector.py @@ -22,7 +22,7 @@ PERSISTED_LOGS_SESSIONS_LABEL_VALUE, ) from renku_data_services.persisted_logs.db import ( - AmaltheaSessionPersistedLogsRepository, + AmaltheaSessionPersistedLogsWriteRepository, ImageBuildPersistedLogsWriteRepository, ) @@ -210,7 +210,7 @@ def from_config( session_maker=session_maker, config=config, reader=reader, - session_logs_repo=AmaltheaSessionPersistedLogsRepository(), + session_logs_repo=AmaltheaSessionPersistedLogsWriteRepository(), build_logs_repo=ImageBuildPersistedLogsWriteRepository(), ) return NoopPersistedLogsCollector() @@ -236,7 +236,7 @@ def __init__( session_maker: Callable[..., AsyncSession], config: PersistedLogsConfig, reader: LokiLogReader, - session_logs_repo: AmaltheaSessionPersistedLogsRepository, + session_logs_repo: AmaltheaSessionPersistedLogsWriteRepository, build_logs_repo: ImageBuildPersistedLogsWriteRepository, ) -> None: self.session_maker = session_maker diff --git a/components/renku_data_services/persisted_logs/collector_t.py b/components/renku_data_services/persisted_logs/collector_t.py deleted file mode 100644 index 72c9eb241..000000000 --- a/components/renku_data_services/persisted_logs/collector_t.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Temp: for testing with local port-forwarding...""" - -import asyncio -from datetime import timedelta - -import httpx - -from renku_data_services.app_config import logging -from renku_data_services.persisted_logs.collector import LokiLogReader -from renku_data_services.persisted_logs.config import PersistedLogsConfig - -logger = logging.getLogger(__name__) - - -def _main() -> None: - logging.configure_logging() - config = PersistedLogsConfig( - enabled=True, - # loki_read_base_url="http://host.docker.internal:3100/", - loki_read_base_url="http://10.6.0.96:3100/", - namespace="renku", - logs_ttl=timedelta(days=1), - ) - reader = LokiLogReader(config, client=httpx.AsyncClient()) - - async def get_some_logs() -> None: - async for log_line in reader.get_amalthea_session_logs(): - logger.info(log_line) - - asyncio.run(get_some_logs()) - - -if __name__ == "__main__": - _main() diff --git a/components/renku_data_services/persisted_logs/config.py b/components/renku_data_services/persisted_logs/config.py index 18a964ecb..bde4897c4 100644 --- a/components/renku_data_services/persisted_logs/config.py +++ b/components/renku_data_services/persisted_logs/config.py @@ -20,7 +20,7 @@ def from_env(cls, namespace: str) -> "PersistedLogsConfig": # return cls( # enabled=enabled, # ) - + # TODO: load config from env vars return cls( enabled=True, loki_read_base_url="http://loki-read.monitoring.svc.cluster.local:3100/", diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index c34632aa3..c9e715d39 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -133,8 +133,11 @@ async def _get_logs_per_container(self, session: AsyncSession, run_id: ULID) -> return await _sort_logs_per_container(res, main_container=SESSION_MAIN_CONTAINER) -class AmaltheaSessionPersistedLogsRepository: - """Repository for persisted logs of Amalthea sessions.""" +class AmaltheaSessionPersistedLogsWriteRepository: + """Repository for writing persisted logs of Amalthea sessions. + + The write side is performed as a background task and does not access authz. + """ async def get_latest_log_timestamp(self, session: AsyncSession) -> int | None: """Returns the latest log timestamp.""" @@ -219,7 +222,7 @@ async def delete_expired_session_logs(self, session: AsyncSession, before: datet class ImageBuildPersistedLogsReadRepository: - """Repository for persisted logs of image builds.""" + """Repository for reading persisted logs of image builds.""" def __init__( self, @@ -313,7 +316,10 @@ async def _get_logs_per_container(self, session: AsyncSession, build_id: ULID) - class ImageBuildPersistedLogsWriteRepository: - """Repository for writing persisted logs of image builds.""" + """Repository for writing persisted logs of image builds. + + The write side is performed as a background task and does not access authz. + """ async def get_latest_log_timestamp(self, session: AsyncSession) -> int | None: """Returns the latest log timestamp.""" diff --git a/components/renku_data_services/persisted_logs/models.py b/components/renku_data_services/persisted_logs/models.py index a7982e642..94ff3ace1 100644 --- a/components/renku_data_services/persisted_logs/models.py +++ b/components/renku_data_services/persisted_logs/models.py @@ -51,6 +51,22 @@ class SessionRun: submission_id: str | None +@dataclass(eq=True, frozen=True, kw_only=True) +class UnsavedBuildLogLine: + """Represents an unsaved image build log line.""" + + id: str + """The ID of the log line. + + This is used to de-duplicate log lines. + """ + + build_id: ULID + container: str + timestamp: int + log_line: str + + @dataclass(eq=True, frozen=True, kw_only=True) class LogLine: """A single log line.""" @@ -81,19 +97,3 @@ class InsertLogsResult: log_count: int last_timestamp: int - - -@dataclass(eq=True, frozen=True, kw_only=True) -class UnsavedBuildLogLine: - """Represents an unsaved image build log line.""" - - id: str - """The ID of the log line. - - This is used to de-duplicate log lines. - """ - - build_id: ULID - container: str - timestamp: int - log_line: str From 22bb3f68d28efd796320cbea3ee065c944194649 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Fri, 31 Jul 2026 08:01:57 +0000 Subject: [PATCH 46/50] add test for collector --- .../persisted_logs/collector.py | 102 ++++--- .../persisted_logs/__init__.py | 1 + .../persisted_logs/test_collector.py | 257 ++++++++++++++++++ test/utils.py | 12 + 4 files changed, 327 insertions(+), 45 deletions(-) create mode 100644 test/components/renku_data_services/persisted_logs/__init__.py create mode 100644 test/components/renku_data_services/persisted_logs/test_collector.py diff --git a/components/renku_data_services/persisted_logs/collector.py b/components/renku_data_services/persisted_logs/collector.py index c382c3f31..83dfcf353 100644 --- a/components/renku_data_services/persisted_logs/collector.py +++ b/components/renku_data_services/persisted_logs/collector.py @@ -56,8 +56,61 @@ async def get_amalthea_session_logs( "}" ) response = await self._get_logs(query=query, limit=limit, start=start, end=end) - log_line_ids: set[str] = set() + async for item in self._process_session_logs(response): + yield item + + async def get_image_build_logs( + self, limit: int = 1000, start: int | None = None, end: int | None = None + ) -> AsyncIterator[models.UnsavedBuildLogLine]: + """Fetchesimage build logs from Loki. + + Parameters: + - limit: max number of entries to return + - start: start timestamp as a Unix nano timestamp + - end: end timestamp as a Unix nano timestamp + + See also https://grafana.com/docs/loki/latest/reference/loki-http-api/#query-logs-within-a-range-of-time + """ + query = ( + "{" + f'{PERSISTED_LOGS_BUILD_LABEL_KEY}="{PERSISTED_LOGS_BUILD_LABEL_VALUE}",' + f'{PERSISTED_LOGS_NAMESPACE_LABEL_KEY}="{self.config.namespace}"' + "}" + ) + response = await self._get_logs(query=query, limit=limit, start=start, end=end) + async for item in self._process_image_build_logs(response): + yield item + async def _get_logs( + self, query: str, limit: int = 1000, start: int | None = None, end: int | None = None + ) -> loki_api.LokiQueryRangeResponse: + """Fetches logs from Loki, using the passed in query. + + Parameters: + - query: the Loki query + - limit: max number of entries to return + - start: start timestamp as a Unix nano timestamp + - end: end timestamp as a Unix nano timestamp + + See also https://grafana.com/docs/loki/latest/reference/loki-http-api/#query-logs-within-a-range-of-time + """ + params: dict[str, str | int] = dict() + params["query"] = query + params["direction"] = "forward" + params["limit"] = limit + if start: + params["start"] = str(start) + if end: + params["end"] = str(end) + res = await self.client.get("loki/api/v1/query_range", params=params) + res.raise_for_status() + return loki_api.LokiQueryRangeResponse.model_validate_json(res.content) + + @staticmethod + async def _process_session_logs( + response: loki_api.LokiQueryRangeResponse, + ) -> AsyncIterator[models.UnsavedSessionLogLine]: + log_line_ids: set[str] = set() for entry in response.data.result: stream: loki_api.AmaltheaSessionStream | None = None try: @@ -103,27 +156,11 @@ async def get_amalthea_session_logs( log_line=log_line, ) - async def get_image_build_logs( - self, limit: int = 1000, start: int | None = None, end: int | None = None + @staticmethod + async def _process_image_build_logs( + response: loki_api.LokiQueryRangeResponse, ) -> AsyncIterator[models.UnsavedBuildLogLine]: - """Fetchesimage build logs from Loki. - - Parameters: - - limit: max number of entries to return - - start: start timestamp as a Unix nano timestamp - - end: end timestamp as a Unix nano timestamp - - See also https://grafana.com/docs/loki/latest/reference/loki-http-api/#query-logs-within-a-range-of-time - """ - query = ( - "{" - f'{PERSISTED_LOGS_BUILD_LABEL_KEY}="{PERSISTED_LOGS_BUILD_LABEL_VALUE}",' - f'{PERSISTED_LOGS_NAMESPACE_LABEL_KEY}="{self.config.namespace}"' - "}" - ) - response = await self._get_logs(query=query, limit=limit, start=start, end=end) log_line_ids: set[str] = set() - for entry in response.data.result: stream: loki_api.ShipwrightBuildRunStream | None = None try: @@ -156,31 +193,6 @@ async def get_image_build_logs( log_line=log_line, ) - async def _get_logs( - self, query: str, limit: int = 1000, start: int | None = None, end: int | None = None - ) -> loki_api.LokiQueryRangeResponse: - """Fetches logs from Loki, using the passed in query. - - Parameters: - - query: the Loki query - - limit: max number of entries to return - - start: start timestamp as a Unix nano timestamp - - end: end timestamp as a Unix nano timestamp - - See also https://grafana.com/docs/loki/latest/reference/loki-http-api/#query-logs-within-a-range-of-time - """ - params: dict[str, str | int] = dict() - params["query"] = query - params["direction"] = "forward" - params["limit"] = limit - if start: - params["start"] = str(start) - if end: - params["end"] = str(end) - res = await self.client.get("loki/api/v1/query_range", params=params) - res.raise_for_status() - return loki_api.LokiQueryRangeResponse.model_validate_json(res.content) - class PersistedLogsCollector: """Abstract class for gathering persisted logs.""" diff --git a/test/components/renku_data_services/persisted_logs/__init__.py b/test/components/renku_data_services/persisted_logs/__init__.py new file mode 100644 index 000000000..0835fac71 --- /dev/null +++ b/test/components/renku_data_services/persisted_logs/__init__.py @@ -0,0 +1 @@ +"""Tests for persisted logs.""" diff --git a/test/components/renku_data_services/persisted_logs/test_collector.py b/test/components/renku_data_services/persisted_logs/test_collector.py new file mode 100644 index 000000000..0938a9256 --- /dev/null +++ b/test/components/renku_data_services/persisted_logs/test_collector.py @@ -0,0 +1,257 @@ +"""Tests for the logs collector.""" + +import pytest +from ulid import ULID + +from renku_data_services.persisted_logs import loki_api, models +from renku_data_services.persisted_logs.collector import LokiLogReader + + +@pytest.fixture +def session_logs_response() -> loki_api.LokiQueryRangeResponse: + json_content = """ +{ + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "app": "AmaltheaSession", + "container": "git-clone", + "container_runtime": "containerd", + "detected_level": "unknown", + "instance": "renku-ci-ds-1383/j-flora-thie-a8944af936b5-7mjnh:git-clone", + "job": "renku-ci-ds-1383/git-clone", + "namespace": "renku-ci-ds-1383", + "pod": "j-flora-thie-a8944af936b5-7mjnh", + "renku_io_launcher_id": "01KXNAFYMJ42QCEGG6739T28RS", + "renku_io_pod_uid": "c7341da8-5333-47fc-87f4-b364b0ea9c1c", + "renku_io_project_id": "01KXJZF4YH8G2CP9TDNJJPAWNF", + "renku_io_run_id": "01KYVGCNJ3CJ1EQKF343JEEV6T", + "renku_io_safe_username": "d62fb7cb-7893-4149-8917-19e8d882cdd0", + "renku_io_session_type": "non_interactive", + "renku_io_session_uid": "6c5596f3-b27d-4b71-8d37-e672eb66b866", + "renku_io_submission_id": "run-8ej4lg", + "service_name": "AmaltheaSession" + }, + "values": [ + [ + "1785482086378524994", + "2026/07/31 07:14:46 Setting up git proxy to http://localhost:65480\\n" + ], + [ + "1785482086364418022", + "2026/07/31 07:14:46 Dealing with submodules\\n" + ], + [ + "1785482086339950490", + "2026/07/31 07:14:46 Checking out branch main\\n" + ], + [ + "1785482086339935443", + "2026/07/31 07:14:46 Default branch is main\\n" + ], + [ + "1785482085028615483", + "2026/07/31 07:14:45 Cloning repository /home/renku/work/renku-envs from https://gitlab.com/leafty/renku-envs.git\\n" + ], + [ + "1785482085026311874", + "2026/07/31 07:14:45 Setting name Flora Thiebaut in git config\\n" + ], + [ + "1785482085025300000", + "2026/07/31 07:14:45 Setting email flora.thiebaut@sdsc.ethz.ch in git config\\n" + ], + [ + "1785482085020557046", + "2026/07/31 07:14:45 Initializing repo\\n" + ], + [ + "1785482085020548955", + "2026/07/31 07:14:45 Setting up repository.\\n" + ], + [ + "1785482085019553389", + "2026/07/31 07:14:45 Processing https://gitlab.com/leafty/renku-envs.git\\n" + ], + [ + "1785482085018064534", + "2026/07/31 07:14:45 Creating clone path\\n" + ], + [ + "1785482085018035543", + "2026/07/31 07:14:45 Checking if clone path: /home/renku/work/renku-envs exists\\n" + ] + ] + }, + { + "stream": { + "app": "AmaltheaSession", + "container": "amalthea-session", + "container_runtime": "containerd", + "detected_level": "unknown", + "instance": "renku-ci-ds-1383/j-flora-thie-a8944af936b5-7mjnh:amalthea-session", + "job": "renku-ci-ds-1383/amalthea-session", + "namespace": "renku-ci-ds-1383", + "pod": "j-flora-thie-a8944af936b5-7mjnh", + "renku_io_launcher_id": "01KXNAFYMJ42QCEGG6739T28RS", + "renku_io_pod_uid": "c7341da8-5333-47fc-87f4-b364b0ea9c1c", + "renku_io_project_id": "01KXJZF4YH8G2CP9TDNJJPAWNF", + "renku_io_run_id": "01KYVGCNJ3CJ1EQKF343JEEV6T", + "renku_io_safe_username": "d62fb7cb-7893-4149-8917-19e8d882cdd0", + "renku_io_session_type": "non_interactive", + "renku_io_session_uid": "6c5596f3-b27d-4b71-8d37-e672eb66b866", + "renku_io_submission_id": "run-8ej4lg", + "service_name": "AmaltheaSession" + }, + "values": [ + [ + "1785482091170416212", + "10/10\\n" + ], + [ + "1785482091170414253", + "9/10\\n" + ], + [ + "1785482091170412277", + "8/10\\n" + ], + [ + "1785482091170410038", + "7/10\\n" + ], + [ + "1785482091170408130", + "6/10\\n" + ], + [ + "1785482091170406092", + "5/10\\n" + ], + [ + "1785482091170404020", + "4/10\\n" + ], + [ + "1785482091170401840", + "3/10\\n" + ], + [ + "1785482091170397828", + "2/10\\n" + ], + [ + "1785482091170339759", + "1/10\\n" + ] + ] + } + ] + } +} +""" + return loki_api.LokiQueryRangeResponse.model_validate_json(json_content) + + +@pytest.fixture +def build_logs_response() -> loki_api.LokiQueryRangeResponse: + json_content = """ +{ + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "app": "ShipwrightBuildRun", + "container": "step-build-and-push", + "container_runtime": "containerd", + "detected_level": "unknown", + "instance": "renku-ci-ds-1383/renku-01kyvgffxtxv4qk0dyjkx0zsa5-ttr4v-pod:step-build-and-push", + "job": "renku-ci-ds-1383/step-build-and-push", + "namespace": "renku-ci-ds-1383", + "pod": "renku-01kyvgffxtxv4qk0dyjkx0zsa5-ttr4v-pod", + "renku_io_buildrun_name": "renku-01kyvgffxtxv4qk0dyjkx0zsa5", + "renku_io_pod_uid": "a533d0d6-c485-4671-b3d6-643ad34bd0b6", + "service_name": "ShipwrightBuildRun" + }, + "values": [ + [ + "1785482346922298906", + " harbor.dev.renku.ch/renku-build/renku-build:renku-01kyvgffxtxv4qk0dyjkx0zsa5\\n" + ], + [ + "1785482346922274287", + "*** Images (sha256:77281bcd4ffcc16bd9942dd280d1a9ef78c64fd97274cd1ec4b0d4a0c4084fef):\\n" + ], + [ + "1785482342006624181", + "Saving harbor.dev.renku.ch/renku-build/renku-build:renku-01kyvgffxtxv4qk0dyjkx0zsa5...\\n" + ] + ] + } + ] + } +} +""" + return loki_api.LokiQueryRangeResponse.model_validate_json(json_content) + + +@pytest.mark.asyncio +async def test_process_session_logs(session_logs_response: loki_api.LokiQueryRangeResponse) -> None: + log_stream = LokiLogReader._process_session_logs(session_logs_response) + unsaved_log_lines: list[models.UnsavedSessionLogLine] = [] + async for item in log_stream: + unsaved_log_lines.append(item) + + assert unsaved_log_lines is not None + assert len(unsaved_log_lines) == 22 + + expected_log_line_1 = models.UnsavedSessionLogLine( + id="1785482085020548955::git-clone::j-flora-thie-a8944af936b5-7mjnh", + user_id="d62fb7cb-7893-4149-8917-19e8d882cdd0", + run_id=ULID.from_str("01KYVGCNJ3CJ1EQKF343JEEV6T"), + session_uid="6c5596f3-b27d-4b71-8d37-e672eb66b866", + launcher_id=ULID.from_str("01KXNAFYMJ42QCEGG6739T28RS"), + submission_id="run-8ej4lg", + container="git-clone", + timestamp=1785482085020548955, + log_line="2026/07/31 07:14:45 Setting up repository.\n", + ) + assert expected_log_line_1 in unsaved_log_lines + + expected_log_line_2 = models.UnsavedSessionLogLine( + id="1785482091170410038::amalthea-session::j-flora-thie-a8944af936b5-7mjnh", + user_id="d62fb7cb-7893-4149-8917-19e8d882cdd0", + run_id=ULID.from_str("01KYVGCNJ3CJ1EQKF343JEEV6T"), + session_uid="6c5596f3-b27d-4b71-8d37-e672eb66b866", + launcher_id=ULID.from_str("01KXNAFYMJ42QCEGG6739T28RS"), + submission_id="run-8ej4lg", + container="amalthea-session", + timestamp=1785482091170410038, + log_line="7/10\n", + ) + assert expected_log_line_2 in unsaved_log_lines + + +@pytest.mark.asyncio +async def test_process_build_logs(build_logs_response: loki_api.LokiQueryRangeResponse) -> None: + log_stream = LokiLogReader._process_image_build_logs(build_logs_response) + unsaved_log_lines: list[models.UnsavedBuildLogLine] = [] + async for item in log_stream: + unsaved_log_lines.append(item) + + assert unsaved_log_lines is not None + assert len(unsaved_log_lines) == 3 + + expected_log_line = models.UnsavedBuildLogLine( + id="1785482342006624181::step-build-and-push::renku-01kyvgffxtxv4qk0dyjkx0zsa5-ttr4v-pod", + build_id=ULID.from_str("01KYVGFFXTXV4QK0DYJKX0ZSA5"), + container="step-build-and-push", + timestamp=1785482342006624181, + log_line="Saving harbor.dev.renku.ch/renku-build/renku-build:renku-01kyvgffxtxv4qk0dyjkx0zsa5...\n", + ) + assert expected_log_line in unsaved_log_lines diff --git a/test/utils.py b/test/utils.py index 0ad39e445..dd1c926b3 100644 --- a/test/utils.py +++ b/test/utils.py @@ -56,6 +56,10 @@ from renku_data_services.notebooks.data_sources import DataSourceRepository from renku_data_services.notebooks.image_check import ImageCheckRepository from renku_data_services.notifications.db import NotificationsRepository +from renku_data_services.persisted_logs.db import ( + AmaltheaSessionPersistedLogsReadRepository, + ImageBuildPersistedLogsReadRepository, +) from renku_data_services.platform.db import PlatformRepository, UrlRedirectRepository from renku_data_services.project.db import ( ProjectMemberRepository, @@ -364,6 +368,12 @@ def from_env( occurrence_repo = OccurrenceRepository(session_maker=config.db.async_session_maker) resource_requests_repo = ResourceRequestsRepo(session_maker=config.db.async_session_maker) resource_usage_service = ResourceUsageService(resource_requests_repo) + session_logs_repo = AmaltheaSessionPersistedLogsReadRepository(authz=authz) + build_logs_repo = ImageBuildPersistedLogsReadRepository( + authz=authz, + builds_config=config.builds, + git_repositories_repo=git_repositories_repo, + ) return cls( config=config, @@ -410,6 +420,8 @@ def from_env( occurrence_repo=occurrence_repo, resource_requests_repo=resource_requests_repo, resource_usage_service=resource_usage_service, + session_logs_repo=session_logs_repo, + build_logs_repo=build_logs_repo, zenodo_client=ZenodoAPIClient(), envidat_client=EnvidatClient(), job_client=job_client, From 71c40556e43d63bd4f974f1494e56108bb841493 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Fri, 31 Jul 2026 08:52:25 +0000 Subject: [PATCH 47/50] wip: db tests --- .../persisted_logs/conftest.py | 198 ++++++++++++++++++ .../persisted_logs/test_collector.py | 193 ----------------- .../persisted_logs/test_db.py | 54 +++++ 3 files changed, 252 insertions(+), 193 deletions(-) create mode 100644 test/components/renku_data_services/persisted_logs/conftest.py create mode 100644 test/components/renku_data_services/persisted_logs/test_db.py diff --git a/test/components/renku_data_services/persisted_logs/conftest.py b/test/components/renku_data_services/persisted_logs/conftest.py new file mode 100644 index 000000000..7e0a25669 --- /dev/null +++ b/test/components/renku_data_services/persisted_logs/conftest.py @@ -0,0 +1,198 @@ +"""Tests for the logs collector.""" + +import pytest + +from renku_data_services.persisted_logs import loki_api + + +@pytest.fixture +def session_logs_response() -> loki_api.LokiQueryRangeResponse: + json_content = """ +{ + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "app": "AmaltheaSession", + "container": "git-clone", + "container_runtime": "containerd", + "detected_level": "unknown", + "instance": "renku-ci-ds-1383/j-flora-thie-a8944af936b5-7mjnh:git-clone", + "job": "renku-ci-ds-1383/git-clone", + "namespace": "renku-ci-ds-1383", + "pod": "j-flora-thie-a8944af936b5-7mjnh", + "renku_io_launcher_id": "01KXNAFYMJ42QCEGG6739T28RS", + "renku_io_pod_uid": "c7341da8-5333-47fc-87f4-b364b0ea9c1c", + "renku_io_project_id": "01KXJZF4YH8G2CP9TDNJJPAWNF", + "renku_io_run_id": "01KYVGCNJ3CJ1EQKF343JEEV6T", + "renku_io_safe_username": "d62fb7cb-7893-4149-8917-19e8d882cdd0", + "renku_io_session_type": "non_interactive", + "renku_io_session_uid": "6c5596f3-b27d-4b71-8d37-e672eb66b866", + "renku_io_submission_id": "run-8ej4lg", + "service_name": "AmaltheaSession" + }, + "values": [ + [ + "1785482086378524994", + "2026/07/31 07:14:46 Setting up git proxy to http://localhost:65480\\n" + ], + [ + "1785482086364418022", + "2026/07/31 07:14:46 Dealing with submodules\\n" + ], + [ + "1785482086339950490", + "2026/07/31 07:14:46 Checking out branch main\\n" + ], + [ + "1785482086339935443", + "2026/07/31 07:14:46 Default branch is main\\n" + ], + [ + "1785482085028615483", + "2026/07/31 07:14:45 Cloning repository /home/renku/work/renku-envs from https://gitlab.com/leafty/renku-envs.git\\n" + ], + [ + "1785482085026311874", + "2026/07/31 07:14:45 Setting name Flora Thiebaut in git config\\n" + ], + [ + "1785482085025300000", + "2026/07/31 07:14:45 Setting email flora.thiebaut@sdsc.ethz.ch in git config\\n" + ], + [ + "1785482085020557046", + "2026/07/31 07:14:45 Initializing repo\\n" + ], + [ + "1785482085020548955", + "2026/07/31 07:14:45 Setting up repository.\\n" + ], + [ + "1785482085019553389", + "2026/07/31 07:14:45 Processing https://gitlab.com/leafty/renku-envs.git\\n" + ], + [ + "1785482085018064534", + "2026/07/31 07:14:45 Creating clone path\\n" + ], + [ + "1785482085018035543", + "2026/07/31 07:14:45 Checking if clone path: /home/renku/work/renku-envs exists\\n" + ] + ] + }, + { + "stream": { + "app": "AmaltheaSession", + "container": "amalthea-session", + "container_runtime": "containerd", + "detected_level": "unknown", + "instance": "renku-ci-ds-1383/j-flora-thie-a8944af936b5-7mjnh:amalthea-session", + "job": "renku-ci-ds-1383/amalthea-session", + "namespace": "renku-ci-ds-1383", + "pod": "j-flora-thie-a8944af936b5-7mjnh", + "renku_io_launcher_id": "01KXNAFYMJ42QCEGG6739T28RS", + "renku_io_pod_uid": "c7341da8-5333-47fc-87f4-b364b0ea9c1c", + "renku_io_project_id": "01KXJZF4YH8G2CP9TDNJJPAWNF", + "renku_io_run_id": "01KYVGCNJ3CJ1EQKF343JEEV6T", + "renku_io_safe_username": "d62fb7cb-7893-4149-8917-19e8d882cdd0", + "renku_io_session_type": "non_interactive", + "renku_io_session_uid": "6c5596f3-b27d-4b71-8d37-e672eb66b866", + "renku_io_submission_id": "run-8ej4lg", + "service_name": "AmaltheaSession" + }, + "values": [ + [ + "1785482091170416212", + "10/10\\n" + ], + [ + "1785482091170414253", + "9/10\\n" + ], + [ + "1785482091170412277", + "8/10\\n" + ], + [ + "1785482091170410038", + "7/10\\n" + ], + [ + "1785482091170408130", + "6/10\\n" + ], + [ + "1785482091170406092", + "5/10\\n" + ], + [ + "1785482091170404020", + "4/10\\n" + ], + [ + "1785482091170401840", + "3/10\\n" + ], + [ + "1785482091170397828", + "2/10\\n" + ], + [ + "1785482091170339759", + "1/10\\n" + ] + ] + } + ] + } +} +""" + return loki_api.LokiQueryRangeResponse.model_validate_json(json_content) + + +@pytest.fixture +def build_logs_response() -> loki_api.LokiQueryRangeResponse: + json_content = """ +{ + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "app": "ShipwrightBuildRun", + "container": "step-build-and-push", + "container_runtime": "containerd", + "detected_level": "unknown", + "instance": "renku-ci-ds-1383/renku-01kyvgffxtxv4qk0dyjkx0zsa5-ttr4v-pod:step-build-and-push", + "job": "renku-ci-ds-1383/step-build-and-push", + "namespace": "renku-ci-ds-1383", + "pod": "renku-01kyvgffxtxv4qk0dyjkx0zsa5-ttr4v-pod", + "renku_io_buildrun_name": "renku-01kyvgffxtxv4qk0dyjkx0zsa5", + "renku_io_pod_uid": "a533d0d6-c485-4671-b3d6-643ad34bd0b6", + "service_name": "ShipwrightBuildRun" + }, + "values": [ + [ + "1785482346922298906", + " harbor.dev.renku.ch/renku-build/renku-build:renku-01kyvgffxtxv4qk0dyjkx0zsa5\\n" + ], + [ + "1785482346922274287", + "*** Images (sha256:77281bcd4ffcc16bd9942dd280d1a9ef78c64fd97274cd1ec4b0d4a0c4084fef):\\n" + ], + [ + "1785482342006624181", + "Saving harbor.dev.renku.ch/renku-build/renku-build:renku-01kyvgffxtxv4qk0dyjkx0zsa5...\\n" + ] + ] + } + ] + } +} +""" + return loki_api.LokiQueryRangeResponse.model_validate_json(json_content) diff --git a/test/components/renku_data_services/persisted_logs/test_collector.py b/test/components/renku_data_services/persisted_logs/test_collector.py index 0938a9256..2554f9b9d 100644 --- a/test/components/renku_data_services/persisted_logs/test_collector.py +++ b/test/components/renku_data_services/persisted_logs/test_collector.py @@ -7,199 +7,6 @@ from renku_data_services.persisted_logs.collector import LokiLogReader -@pytest.fixture -def session_logs_response() -> loki_api.LokiQueryRangeResponse: - json_content = """ -{ - "status": "success", - "data": { - "resultType": "streams", - "result": [ - { - "stream": { - "app": "AmaltheaSession", - "container": "git-clone", - "container_runtime": "containerd", - "detected_level": "unknown", - "instance": "renku-ci-ds-1383/j-flora-thie-a8944af936b5-7mjnh:git-clone", - "job": "renku-ci-ds-1383/git-clone", - "namespace": "renku-ci-ds-1383", - "pod": "j-flora-thie-a8944af936b5-7mjnh", - "renku_io_launcher_id": "01KXNAFYMJ42QCEGG6739T28RS", - "renku_io_pod_uid": "c7341da8-5333-47fc-87f4-b364b0ea9c1c", - "renku_io_project_id": "01KXJZF4YH8G2CP9TDNJJPAWNF", - "renku_io_run_id": "01KYVGCNJ3CJ1EQKF343JEEV6T", - "renku_io_safe_username": "d62fb7cb-7893-4149-8917-19e8d882cdd0", - "renku_io_session_type": "non_interactive", - "renku_io_session_uid": "6c5596f3-b27d-4b71-8d37-e672eb66b866", - "renku_io_submission_id": "run-8ej4lg", - "service_name": "AmaltheaSession" - }, - "values": [ - [ - "1785482086378524994", - "2026/07/31 07:14:46 Setting up git proxy to http://localhost:65480\\n" - ], - [ - "1785482086364418022", - "2026/07/31 07:14:46 Dealing with submodules\\n" - ], - [ - "1785482086339950490", - "2026/07/31 07:14:46 Checking out branch main\\n" - ], - [ - "1785482086339935443", - "2026/07/31 07:14:46 Default branch is main\\n" - ], - [ - "1785482085028615483", - "2026/07/31 07:14:45 Cloning repository /home/renku/work/renku-envs from https://gitlab.com/leafty/renku-envs.git\\n" - ], - [ - "1785482085026311874", - "2026/07/31 07:14:45 Setting name Flora Thiebaut in git config\\n" - ], - [ - "1785482085025300000", - "2026/07/31 07:14:45 Setting email flora.thiebaut@sdsc.ethz.ch in git config\\n" - ], - [ - "1785482085020557046", - "2026/07/31 07:14:45 Initializing repo\\n" - ], - [ - "1785482085020548955", - "2026/07/31 07:14:45 Setting up repository.\\n" - ], - [ - "1785482085019553389", - "2026/07/31 07:14:45 Processing https://gitlab.com/leafty/renku-envs.git\\n" - ], - [ - "1785482085018064534", - "2026/07/31 07:14:45 Creating clone path\\n" - ], - [ - "1785482085018035543", - "2026/07/31 07:14:45 Checking if clone path: /home/renku/work/renku-envs exists\\n" - ] - ] - }, - { - "stream": { - "app": "AmaltheaSession", - "container": "amalthea-session", - "container_runtime": "containerd", - "detected_level": "unknown", - "instance": "renku-ci-ds-1383/j-flora-thie-a8944af936b5-7mjnh:amalthea-session", - "job": "renku-ci-ds-1383/amalthea-session", - "namespace": "renku-ci-ds-1383", - "pod": "j-flora-thie-a8944af936b5-7mjnh", - "renku_io_launcher_id": "01KXNAFYMJ42QCEGG6739T28RS", - "renku_io_pod_uid": "c7341da8-5333-47fc-87f4-b364b0ea9c1c", - "renku_io_project_id": "01KXJZF4YH8G2CP9TDNJJPAWNF", - "renku_io_run_id": "01KYVGCNJ3CJ1EQKF343JEEV6T", - "renku_io_safe_username": "d62fb7cb-7893-4149-8917-19e8d882cdd0", - "renku_io_session_type": "non_interactive", - "renku_io_session_uid": "6c5596f3-b27d-4b71-8d37-e672eb66b866", - "renku_io_submission_id": "run-8ej4lg", - "service_name": "AmaltheaSession" - }, - "values": [ - [ - "1785482091170416212", - "10/10\\n" - ], - [ - "1785482091170414253", - "9/10\\n" - ], - [ - "1785482091170412277", - "8/10\\n" - ], - [ - "1785482091170410038", - "7/10\\n" - ], - [ - "1785482091170408130", - "6/10\\n" - ], - [ - "1785482091170406092", - "5/10\\n" - ], - [ - "1785482091170404020", - "4/10\\n" - ], - [ - "1785482091170401840", - "3/10\\n" - ], - [ - "1785482091170397828", - "2/10\\n" - ], - [ - "1785482091170339759", - "1/10\\n" - ] - ] - } - ] - } -} -""" - return loki_api.LokiQueryRangeResponse.model_validate_json(json_content) - - -@pytest.fixture -def build_logs_response() -> loki_api.LokiQueryRangeResponse: - json_content = """ -{ - "status": "success", - "data": { - "resultType": "streams", - "result": [ - { - "stream": { - "app": "ShipwrightBuildRun", - "container": "step-build-and-push", - "container_runtime": "containerd", - "detected_level": "unknown", - "instance": "renku-ci-ds-1383/renku-01kyvgffxtxv4qk0dyjkx0zsa5-ttr4v-pod:step-build-and-push", - "job": "renku-ci-ds-1383/step-build-and-push", - "namespace": "renku-ci-ds-1383", - "pod": "renku-01kyvgffxtxv4qk0dyjkx0zsa5-ttr4v-pod", - "renku_io_buildrun_name": "renku-01kyvgffxtxv4qk0dyjkx0zsa5", - "renku_io_pod_uid": "a533d0d6-c485-4671-b3d6-643ad34bd0b6", - "service_name": "ShipwrightBuildRun" - }, - "values": [ - [ - "1785482346922298906", - " harbor.dev.renku.ch/renku-build/renku-build:renku-01kyvgffxtxv4qk0dyjkx0zsa5\\n" - ], - [ - "1785482346922274287", - "*** Images (sha256:77281bcd4ffcc16bd9942dd280d1a9ef78c64fd97274cd1ec4b0d4a0c4084fef):\\n" - ], - [ - "1785482342006624181", - "Saving harbor.dev.renku.ch/renku-build/renku-build:renku-01kyvgffxtxv4qk0dyjkx0zsa5...\\n" - ] - ] - } - ] - } -} -""" - return loki_api.LokiQueryRangeResponse.model_validate_json(json_content) - - @pytest.mark.asyncio async def test_process_session_logs(session_logs_response: loki_api.LokiQueryRangeResponse) -> None: log_stream = LokiLogReader._process_session_logs(session_logs_response) diff --git a/test/components/renku_data_services/persisted_logs/test_db.py b/test/components/renku_data_services/persisted_logs/test_db.py new file mode 100644 index 000000000..39bc86a9c --- /dev/null +++ b/test/components/renku_data_services/persisted_logs/test_db.py @@ -0,0 +1,54 @@ +"""Tests for the persisted logs database.""" + +import pytest +import pytest_asyncio + +from renku_data_services.data_api.dependencies import DependencyManager +from renku_data_services.migrations.core import run_migrations_for_app +from renku_data_services.persisted_logs import loki_api +from renku_data_services.persisted_logs.collector import LokiLogReader +from renku_data_services.persisted_logs.db import ( + AmaltheaSessionPersistedLogsWriteRepository, + ImageBuildPersistedLogsWriteRepository, +) + + +@pytest_asyncio.fixture +async def dependency_manager(app_manager_instance: DependencyManager) -> DependencyManager: + run_migrations_for_app("common") + return app_manager_instance + + +@pytest.fixture +def session_logs_repo() -> AmaltheaSessionPersistedLogsWriteRepository: + return AmaltheaSessionPersistedLogsWriteRepository() + + +@pytest.fixture +def build_logs_repo() -> ImageBuildPersistedLogsWriteRepository: + return ImageBuildPersistedLogsWriteRepository() + + +@pytest.mark.asyncio +async def test_session_latest_log_timestamp_is_none_at_startup( + session_logs_repo: AmaltheaSessionPersistedLogsWriteRepository, dependency_manager: DependencyManager +): + async_session_maker = dependency_manager.config.db.async_session_maker + async with async_session_maker() as session, session.begin(): + ts = await session_logs_repo.get_latest_log_timestamp(session=session) + assert ts is None + + +@pytest.mark.asyncio +async def test_insert_session_logs( + session_logs_response: loki_api.LokiQueryRangeResponse, + session_logs_repo: AmaltheaSessionPersistedLogsWriteRepository, + dependency_manager: DependencyManager, +): + # TODO: Setup the database with a session launcher + # dependency_manager.session_repo.insert_launcher() + logs_stream = LokiLogReader._process_session_logs(session_logs_response) + async_session_maker = dependency_manager.config.db.async_session_maker + async with async_session_maker() as session, session.begin(): + result = await session_logs_repo.insert_session_logs(session=session, logs_stream=logs_stream) + assert result is None From 009cf513ffcd0961c316654fd4384aec008f5238 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Fri, 31 Jul 2026 09:32:42 +0000 Subject: [PATCH 48/50] wip: tests --- .../persisted_logs/test_db.py | 98 ++++++++++++++++++- 1 file changed, 93 insertions(+), 5 deletions(-) diff --git a/test/components/renku_data_services/persisted_logs/test_db.py b/test/components/renku_data_services/persisted_logs/test_db.py index 39bc86a9c..bae074ad7 100644 --- a/test/components/renku_data_services/persisted_logs/test_db.py +++ b/test/components/renku_data_services/persisted_logs/test_db.py @@ -1,16 +1,30 @@ """Tests for the persisted logs database.""" +from collections.abc import AsyncIterator +from dataclasses import replace + import pytest import pytest_asyncio +from renku_data_services import base_models from renku_data_services.data_api.dependencies import DependencyManager from renku_data_services.migrations.core import run_migrations_for_app -from renku_data_services.persisted_logs import loki_api +from renku_data_services.persisted_logs import loki_api, models from renku_data_services.persisted_logs.collector import LokiLogReader from renku_data_services.persisted_logs.db import ( AmaltheaSessionPersistedLogsWriteRepository, ImageBuildPersistedLogsWriteRepository, ) +from renku_data_services.project.models import Project, UnsavedProject, Visibility +from renku_data_services.session.models import ( + EnvironmentImageSource, + EnvironmentKind, + LauncherType, + SessionLauncher, + UnsavedEnvironment, + UnsavedSessionLauncher, +) +from renku_data_services.users.models import UserInfo @pytest_asyncio.fixture @@ -39,16 +53,90 @@ async def test_session_latest_log_timestamp_is_none_at_startup( assert ts is None +@pytest_asyncio.fixture +async def regular_user(dependency_manager: DependencyManager) -> base_models.AuthenticatedAPIUser: + api_user = base_models.AuthenticatedAPIUser( + id="jane_doe", email="jane.doe@example.org", access_token="my_access_token" + ) + user_info = await dependency_manager.kc_user_repo.get_or_create_user(requested_by=api_user, id=api_user.id) + assert user_info is not None + return api_user + + +@pytest_asyncio.fixture +async def regular_user_info( + dependency_manager: DependencyManager, regular_user: base_models.AuthenticatedAPIUser +) -> UserInfo: + user_info = await dependency_manager.kc_user_repo.get_user(id=regular_user.id) + assert user_info is not None + return user_info + + +@pytest_asyncio.fixture +async def my_project( + dependency_manager: DependencyManager, + regular_user: base_models.AuthenticatedAPIUser, + regular_user_info: UserInfo, +) -> Project: + project = await dependency_manager.project_repo.insert_project( + user=regular_user, + project=UnsavedProject( + name="My Project", + slug="my-project", + visibility=Visibility.PRIVATE, + created_by=regular_user.id, + namespace=regular_user_info.namespace.path.serialize(), + ), + ) + assert project is not None + return project + + +@pytest_asyncio.fixture +async def my_session_launcher( + dependency_manager: DependencyManager, + regular_user: base_models.AuthenticatedAPIUser, + my_project: Project, +) -> SessionLauncher: + environment = UnsavedEnvironment( + name="My Environment", + container_image="renku:test", + default_url="/", + environment_kind=EnvironmentKind.CUSTOM, + environment_image_source=EnvironmentImageSource.image, + ) + launcher = await dependency_manager.session_repo.insert_launcher( + user=regular_user, + launcher=UnsavedSessionLauncher( + project_id=my_project.id, + name="My Session", + description=None, + resource_class_id=None, + disk_storage=None, + env_variables=None, + environment=environment, + launcher_type=LauncherType.interactive, + ), + ) + assert launcher is not None + return launcher + + @pytest.mark.asyncio async def test_insert_session_logs( session_logs_response: loki_api.LokiQueryRangeResponse, session_logs_repo: AmaltheaSessionPersistedLogsWriteRepository, dependency_manager: DependencyManager, + regular_user: base_models.AuthenticatedAPIUser, + my_session_launcher: SessionLauncher, ): - # TODO: Setup the database with a session launcher - # dependency_manager.session_repo.insert_launcher() - logs_stream = LokiLogReader._process_session_logs(session_logs_response) + # Replace the log line metadata for the test + async def _make_logs_stream() -> AsyncIterator[models.UnsavedSessionLogLine]: + source = LokiLogReader._process_session_logs(session_logs_response) + async for item in source: + yield replace(item, user_id=regular_user.id, launcher_id=my_session_launcher.id) + async_session_maker = dependency_manager.config.db.async_session_maker async with async_session_maker() as session, session.begin(): - result = await session_logs_repo.insert_session_logs(session=session, logs_stream=logs_stream) + result = await session_logs_repo.insert_session_logs(session=session, logs_stream=_make_logs_stream()) assert result is None From f6fd0026030e634928a97719912ae37eed20da6c Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Mon, 3 Aug 2026 08:01:45 +0000 Subject: [PATCH 49/50] wip: db tests --- .../renku_data_services/persisted_logs/db.py | 85 +++++++++++-------- .../persisted_logs/models.py | 7 +- .../persisted_logs/test_db.py | 73 +++++++++++++--- 3 files changed, 117 insertions(+), 48 deletions(-) diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index c9e715d39..0eac82494 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -7,10 +7,12 @@ from typing import TYPE_CHECKING from sqlalchemy import delete, select +from sqlalchemy.exc import DatabaseError from sqlalchemy.ext.asyncio import AsyncScalarResult, AsyncSession from ulid import ULID from renku_data_services import base_models, errors +from renku_data_services.app_config import logging from renku_data_services.authz.authz import Authz, ResourceType from renku_data_services.authz.models import Scope from renku_data_services.persisted_logs import models @@ -23,6 +25,7 @@ if TYPE_CHECKING: from renku_data_services.repositories.db import GitRepositoriesRepository from renku_data_services.session.config import BuildsConfig +logger = logging.getLogger(__name__) class AmaltheaSessionPersistedLogsReadRepository: @@ -153,7 +156,7 @@ async def get_latest_log_timestamp(self, session: AsyncSession) -> int | None: async def insert_session_logs( self, session: AsyncSession, logs_stream: AsyncIterator[models.UnsavedSessionLogLine] - ) -> models.InsertLogsResult: + ) -> models.LogStreamMetadata: """Insert sessions logs into the persisted logs database.""" log_count = 0 last_timestamp = 0 @@ -161,39 +164,12 @@ async def insert_session_logs( log_count += 1 if log.timestamp > last_timestamp: last_timestamp = log.timestamp + try: + await self._insert_log_line(session=session, log=log) + except DatabaseError as err: + logger.warning(f"Could not process log line {log.id}: {err}") - existing_log_res = await session.scalars( - select(schemas.AmaltheaSessionLogORM.id).where(schemas.AmaltheaSessionLogORM.id == log.id) - ) - existing_log_orm = existing_log_res.one_or_none() - if existing_log_orm: - continue - - session_run_res = await session.scalars( - select(schemas.SessionRunORM).where(schemas.SessionRunORM.id == log.run_id) - ) - session_run_orm = session_run_res.one_or_none() - if session_run_orm is None: - session_run_orm = schemas.SessionRunORM( - id=log.run_id, - user_id=log.user_id, - session_uid=log.session_uid, - launcher_id=log.launcher_id, - submission_id=log.submission_id, - ) - session.add(session_run_orm) - await session.flush() - - log_orm = schemas.AmaltheaSessionLogORM( - id=log.id, - run_id=log.run_id, - container=log.container, - timestamp=log.timestamp, - log_line=log.log_line, - ) - session.add(log_orm) - await session.flush() - return models.InsertLogsResult(log_count=log_count, last_timestamp=last_timestamp) + return models.LogStreamMetadata(log_count=log_count, last_timestamp=last_timestamp) async def delete_expired_session_logs(self, session: AsyncSession, before: datetime) -> int: """Remove expired session logs from the database.""" @@ -220,6 +196,45 @@ async def delete_expired_session_logs(self, session: AsyncSession, before: datet return deleted_logs_count + async def _insert_log_line(self, session: AsyncSession, log: models.UnsavedSessionLogLine) -> bool: + """Insert a single session log line into the persisted logs database. + + Returns true if the log line was inserted into the database and false otherwise (the log line already exists). + """ + existing_log_res = await session.scalars( + select(schemas.AmaltheaSessionLogORM.id).where(schemas.AmaltheaSessionLogORM.id == log.id) + ) + existing_log_orm = existing_log_res.one_or_none() + if existing_log_orm: + return False + + session_run_res = await session.scalars( + select(schemas.SessionRunORM).where(schemas.SessionRunORM.id == log.run_id) + ) + session_run_orm = session_run_res.one_or_none() + if session_run_orm is None: + async with session.begin_nested(): + session_run_orm = schemas.SessionRunORM( + id=log.run_id, + user_id=log.user_id, + session_uid=log.session_uid, + launcher_id=log.launcher_id, + submission_id=log.submission_id, + ) + session.add(session_run_orm) + await session.flush() + + log_orm = schemas.AmaltheaSessionLogORM( + id=log.id, + run_id=log.run_id, + container=log.container, + timestamp=log.timestamp, + log_line=log.log_line, + ) + session.add(log_orm) + await session.flush() + return True + class ImageBuildPersistedLogsReadRepository: """Repository for reading persisted logs of image builds.""" @@ -335,7 +350,7 @@ async def get_latest_log_timestamp(self, session: AsyncSession) -> int | None: async def insert_build_logs( self, session: AsyncSession, logs_stream: AsyncIterator[models.UnsavedBuildLogLine] - ) -> models.InsertLogsResult: + ) -> models.LogStreamMetadata: """Insert sessions logs into the persisted logs database.""" log_count = 0 last_timestamp = 0 @@ -360,7 +375,7 @@ async def insert_build_logs( ) session.add(log_orm) await session.flush() - return models.InsertLogsResult(log_count=log_count, last_timestamp=last_timestamp) + return models.LogStreamMetadata(log_count=log_count, last_timestamp=last_timestamp) async def delete_expired_build_logs(self, session: AsyncSession, before: datetime) -> int: """Remove expired build logs from the database.""" diff --git a/components/renku_data_services/persisted_logs/models.py b/components/renku_data_services/persisted_logs/models.py index 94ff3ace1..ea48a6093 100644 --- a/components/renku_data_services/persisted_logs/models.py +++ b/components/renku_data_services/persisted_logs/models.py @@ -92,8 +92,11 @@ class PersistedSessionLogs: @dataclass(eq=True, frozen=True, kw_only=True) -class InsertLogsResult: - """Result of inserting a log stream in the database.""" +class LogStreamMetadata: + """Log stream metadata. + + Used to know if there are more logs to fetch and if so, where to continue from. + """ log_count: int last_timestamp: int diff --git a/test/components/renku_data_services/persisted_logs/test_db.py b/test/components/renku_data_services/persisted_logs/test_db.py index bae074ad7..06b75d547 100644 --- a/test/components/renku_data_services/persisted_logs/test_db.py +++ b/test/components/renku_data_services/persisted_logs/test_db.py @@ -5,11 +5,14 @@ import pytest import pytest_asyncio +from sqlalchemy import select +from ulid import ULID from renku_data_services import base_models from renku_data_services.data_api.dependencies import DependencyManager from renku_data_services.migrations.core import run_migrations_for_app from renku_data_services.persisted_logs import loki_api, models +from renku_data_services.persisted_logs import orm as schemas from renku_data_services.persisted_logs.collector import LokiLogReader from renku_data_services.persisted_logs.db import ( AmaltheaSessionPersistedLogsWriteRepository, @@ -43,16 +46,6 @@ def build_logs_repo() -> ImageBuildPersistedLogsWriteRepository: return ImageBuildPersistedLogsWriteRepository() -@pytest.mark.asyncio -async def test_session_latest_log_timestamp_is_none_at_startup( - session_logs_repo: AmaltheaSessionPersistedLogsWriteRepository, dependency_manager: DependencyManager -): - async_session_maker = dependency_manager.config.db.async_session_maker - async with async_session_maker() as session, session.begin(): - ts = await session_logs_repo.get_latest_log_timestamp(session=session) - assert ts is None - - @pytest_asyncio.fixture async def regular_user(dependency_manager: DependencyManager) -> base_models.AuthenticatedAPIUser: api_user = base_models.AuthenticatedAPIUser( @@ -122,6 +115,16 @@ async def my_session_launcher( return launcher +@pytest.mark.asyncio +async def test_session_latest_log_timestamp_is_none_at_startup( + session_logs_repo: AmaltheaSessionPersistedLogsWriteRepository, dependency_manager: DependencyManager +): + async_session_maker = dependency_manager.config.db.async_session_maker + async with async_session_maker() as session, session.begin(): + ts = await session_logs_repo.get_latest_log_timestamp(session=session) + assert ts is None + + @pytest.mark.asyncio async def test_insert_session_logs( session_logs_response: loki_api.LokiQueryRangeResponse, @@ -139,4 +142,52 @@ async def _make_logs_stream() -> AsyncIterator[models.UnsavedSessionLogLine]: async_session_maker = dependency_manager.config.db.async_session_maker async with async_session_maker() as session, session.begin(): result = await session_logs_repo.insert_session_logs(session=session, logs_stream=_make_logs_stream()) - assert result is None + + expected = models.LogStreamMetadata(log_count=22, last_timestamp=1785482091170416212) + assert result == expected + + # Check the result of get_latest_log_timestamp() + async with async_session_maker() as session, session.begin(): + ts = await session_logs_repo.get_latest_log_timestamp(session=session) + assert ts == expected.last_timestamp + + +@pytest.mark.asyncio +async def test_insert_session_logs_with_db_failures( + session_logs_response: loki_api.LokiQueryRangeResponse, + session_logs_repo: AmaltheaSessionPersistedLogsWriteRepository, + dependency_manager: DependencyManager, + regular_user: base_models.AuthenticatedAPIUser, + my_session_launcher: SessionLauncher, +): + # Replace the log line metadata for the test + async def _make_logs_stream() -> AsyncIterator[models.UnsavedSessionLogLine]: + source = LokiLogReader._process_session_logs(session_logs_response) + alt_run_id = ULID() + idx = 0 + async for item in source: + # # Use correct foreign keys only on half of the log lines + if idx % 2 == 0: + yield replace(item, user_id=regular_user.id, run_id=alt_run_id, launcher_id=my_session_launcher.id) + else: + yield item + idx += 1 + + async_session_maker = dependency_manager.config.db.async_session_maker + async with async_session_maker() as session, session.begin(): + result = await session_logs_repo.insert_session_logs(session=session, logs_stream=_make_logs_stream()) + + expected = models.LogStreamMetadata(log_count=22, last_timestamp=1785482091170416212) + assert result == expected + + async with async_session_maker() as session, session.begin(): + stmt = select(schemas.AmaltheaSessionLogORM) + res = await session.scalars(stmt) + session_logs_orm = res.all() + + assert len(session_logs_orm) == 11 + + # Check the result of get_latest_log_timestamp() + async with async_session_maker() as session, session.begin(): + ts = await session_logs_repo.get_latest_log_timestamp(session=session) + assert ts == expected.last_timestamp From 1585f417a4cec52394a5f7275a7ba01d9b1d0bc2 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Mon, 3 Aug 2026 08:36:07 +0000 Subject: [PATCH 50/50] done: db tests --- .../renku_data_services/persisted_logs/db.py | 42 ++++--- .../persisted_logs/test_db.py | 108 ++++++++++++++++-- 2 files changed, 125 insertions(+), 25 deletions(-) diff --git a/components/renku_data_services/persisted_logs/db.py b/components/renku_data_services/persisted_logs/db.py index 0eac82494..263536401 100644 --- a/components/renku_data_services/persisted_logs/db.py +++ b/components/renku_data_services/persisted_logs/db.py @@ -358,14 +358,34 @@ async def insert_build_logs( log_count += 1 if log.timestamp > last_timestamp: last_timestamp = log.timestamp + try: + await self._insert_log_line(session=session, log=log) + except DatabaseError as err: + logger.warning(f"Could not process log line {log.id}: {err}") - existing_log_res = await session.scalars( - select(schemas.ImageBuildLogORM.id).where(schemas.ImageBuildLogORM.id == log.id) - ) - existing_log_orm = existing_log_res.one_or_none() - if existing_log_orm: - continue + return models.LogStreamMetadata(log_count=log_count, last_timestamp=last_timestamp) + async def delete_expired_build_logs(self, session: AsyncSession, before: datetime) -> int: + """Remove expired build logs from the database.""" + nano_ts = models.NanoTimestamp.from_datetime(before) + delete_logs_stmt = delete(schemas.ImageBuildLogORM).where(schemas.ImageBuildLogORM.timestamp < nano_ts) + res = await session.execute(delete_logs_stmt) + deleted_logs_count = res.rowcount + return deleted_logs_count + + async def _insert_log_line(self, session: AsyncSession, log: models.UnsavedBuildLogLine) -> bool: + """Insert a single session log line into the persisted logs database. + + Returns true if the log line was inserted into the database and false otherwise (the log line already exists). + """ + existing_log_res = await session.scalars( + select(schemas.ImageBuildLogORM.id).where(schemas.ImageBuildLogORM.id == log.id) + ) + existing_log_orm = existing_log_res.one_or_none() + if existing_log_orm: + return False + + async with session.begin_nested(): log_orm = schemas.ImageBuildLogORM( id=log.id, build_id=log.build_id, @@ -375,15 +395,7 @@ async def insert_build_logs( ) session.add(log_orm) await session.flush() - return models.LogStreamMetadata(log_count=log_count, last_timestamp=last_timestamp) - - async def delete_expired_build_logs(self, session: AsyncSession, before: datetime) -> int: - """Remove expired build logs from the database.""" - nano_ts = models.NanoTimestamp.from_datetime(before) - delete_logs_stmt = delete(schemas.ImageBuildLogORM).where(schemas.ImageBuildLogORM.timestamp < nano_ts) - res = await session.execute(delete_logs_stmt) - deleted_logs_count = res.rowcount - return deleted_logs_count + return True async def _sort_logs_per_container( diff --git a/test/components/renku_data_services/persisted_logs/test_db.py b/test/components/renku_data_services/persisted_logs/test_db.py index 06b75d547..6ee2e1d1b 100644 --- a/test/components/renku_data_services/persisted_logs/test_db.py +++ b/test/components/renku_data_services/persisted_logs/test_db.py @@ -20,11 +20,11 @@ ) from renku_data_services.project.models import Project, UnsavedProject, Visibility from renku_data_services.session.models import ( - EnvironmentImageSource, - EnvironmentKind, + Build, LauncherType, + Platform, SessionLauncher, - UnsavedEnvironment, + UnsavedBuildParameters, UnsavedSessionLauncher, ) from renku_data_services.users.models import UserInfo @@ -91,12 +91,11 @@ async def my_session_launcher( regular_user: base_models.AuthenticatedAPIUser, my_project: Project, ) -> SessionLauncher: - environment = UnsavedEnvironment( - name="My Environment", - container_image="renku:test", - default_url="/", - environment_kind=EnvironmentKind.CUSTOM, - environment_image_source=EnvironmentImageSource.image, + build_parameters = UnsavedBuildParameters( + repository="https://example.org/repo.git", + platforms=[Platform.linux_amd64], + builder_variant="python", + frontend_variant="vscodium", ) launcher = await dependency_manager.session_repo.insert_launcher( user=regular_user, @@ -107,7 +106,7 @@ async def my_session_launcher( resource_class_id=None, disk_storage=None, env_variables=None, - environment=environment, + environment=build_parameters, launcher_type=LauncherType.interactive, ), ) @@ -115,6 +114,20 @@ async def my_session_launcher( return launcher +@pytest_asyncio.fixture +async def my_build( + dependency_manager: DependencyManager, + regular_user: base_models.AuthenticatedAPIUser, + my_session_launcher: SessionLauncher, +) -> Build: + builds = await dependency_manager.session_repo.get_environment_builds( + user=regular_user, environment_id=my_session_launcher.environment.id + ) + assert builds is not None + assert len(builds) == 1 + return builds[0] + + @pytest.mark.asyncio async def test_session_latest_log_timestamp_is_none_at_startup( session_logs_repo: AmaltheaSessionPersistedLogsWriteRepository, dependency_manager: DependencyManager @@ -191,3 +204,78 @@ async def _make_logs_stream() -> AsyncIterator[models.UnsavedSessionLogLine]: async with async_session_maker() as session, session.begin(): ts = await session_logs_repo.get_latest_log_timestamp(session=session) assert ts == expected.last_timestamp + + +@pytest.mark.asyncio +async def test_build_latest_log_timestamp_is_none_at_startup( + build_logs_repo: ImageBuildPersistedLogsWriteRepository, dependency_manager: DependencyManager +): + async_session_maker = dependency_manager.config.db.async_session_maker + async with async_session_maker() as session, session.begin(): + ts = await build_logs_repo.get_latest_log_timestamp(session=session) + assert ts is None + + +@pytest.mark.asyncio +async def test_insert_build_logs( + build_logs_response: loki_api.LokiQueryRangeResponse, + build_logs_repo: ImageBuildPersistedLogsWriteRepository, + dependency_manager: DependencyManager, + my_build: Build, +): + # Replace the log line metadata for the test + async def _make_logs_stream() -> AsyncIterator[models.UnsavedBuildLogLine]: + source = LokiLogReader._process_image_build_logs(build_logs_response) + async for item in source: + yield replace(item, build_id=my_build.id) + + async_session_maker = dependency_manager.config.db.async_session_maker + async with async_session_maker() as session, session.begin(): + result = await build_logs_repo.insert_build_logs(session=session, logs_stream=_make_logs_stream()) + + expected = models.LogStreamMetadata(log_count=3, last_timestamp=1785482346922298906) + assert result == expected + + # Check the result of get_latest_log_timestamp() + async with async_session_maker() as session, session.begin(): + ts = await build_logs_repo.get_latest_log_timestamp(session=session) + assert ts == expected.last_timestamp + + +@pytest.mark.asyncio +async def test_insert_build_logs_with_db_failures( + build_logs_response: loki_api.LokiQueryRangeResponse, + build_logs_repo: ImageBuildPersistedLogsWriteRepository, + dependency_manager: DependencyManager, + my_build: Build, +): + # Replace the log line metadata for the test + async def _make_logs_stream() -> AsyncIterator[models.UnsavedBuildLogLine]: + source = LokiLogReader._process_image_build_logs(build_logs_response) + idx = 0 + async for item in source: + # Use correct foreign keys only on half of the log lines + if idx % 2 == 0: + yield replace(item, build_id=my_build.id) + else: + yield item + idx += 1 + + async_session_maker = dependency_manager.config.db.async_session_maker + async with async_session_maker() as session, session.begin(): + result = await build_logs_repo.insert_build_logs(session=session, logs_stream=_make_logs_stream()) + + expected = models.LogStreamMetadata(log_count=3, last_timestamp=1785482346922298906) + assert result == expected + + async with async_session_maker() as session, session.begin(): + stmt = select(schemas.ImageBuildLogORM) + res = await session.scalars(stmt) + build_logs_orm = res.all() + + assert len(build_logs_orm) == 2 + + # Check the result of get_latest_log_timestamp() + async with async_session_maker() as session, session.begin(): + ts = await build_logs_repo.get_latest_log_timestamp(session=session) + assert ts == expected.last_timestamp