Return logs in base64 for backward compatibility (#2910)

* Return logs in base64 for backward compatibility

* Fix newlines in logs UI

* Fix lint

* Fix lint

Author: r4victor

frontend/src/libs/index.ts

@@ -91,6 +91,15 @@ export const riseRouterException = (status = 404, json = 'Not Found'): never =>
     throw new Response(json, { status });
 };
 
+export const base64ToArrayBuffer = (base64: string) => {
+    const binaryString = atob(base64);
+    const bytes = new Uint8Array(binaryString.length);
+    for (let i = 0; i < binaryString.length; i++) {
+        bytes[i] = binaryString.charCodeAt(i);
+    }
+    return bytes;
+};
+
 export const isValidUrl = (urlString: string) => {
     try {
         return Boolean(new URL(urlString));

frontend/src/services/project.ts

@@ -1,8 +1,11 @@
 import { API } from 'api';
 import { createApi, fetchBaseQuery } from '@reduxjs/toolkit/query/react';
 
+import { base64ToArrayBuffer } from 'libs';
 import fetchBaseQueryHeaders from 'libs/fetchBaseQueryHeaders';
 
+const decoder = new TextDecoder('utf-8');
+
 // Helper function to transform backend response to frontend format
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 const transformProjectResponse = (project: any): IProject => ({
@@ -130,7 +133,7 @@ export const projectApi = createApi({
             transformResponse: (response: { logs: ILogItem[]; next_token: string }) => {
                 const logs = response.logs.map((logItem) => ({
                     ...logItem,
-                    message: logItem.message,
+                    message: decoder.decode(base64ToArrayBuffer(logItem.message)),
                 }));
 
                 return {

src/dstack/_internal/server/services/logs/__init__.py

@@ -8,7 +8,11 @@
 from dstack._internal.server.schemas.logs import PollLogsRequest
 from dstack._internal.server.schemas.runner import LogEvent as RunnerLogEvent
 from dstack._internal.server.services.logs.aws import BOTO_AVAILABLE, CloudWatchLogStorage
-from dstack._internal.server.services.logs.base import LogStorage, LogStorageError
+from dstack._internal.server.services.logs.base import (
+    LogStorage,
+    LogStorageError,
+    b64encode_raw_message,
+)
 from dstack._internal.server.services.logs.filelog import FileLogStorage
 from dstack._internal.server.services.logs.gcp import GCP_LOGGING_AVAILABLE, GCPLogStorage
 from dstack._internal.utils.common import run_async
@@ -75,4 +79,13 @@ def write_logs(
 
 
 async def poll_logs_async(project: ProjectModel, request: PollLogsRequest) -> JobSubmissionLogs:
-    return await run_async(get_log_storage().poll_logs, project=project, request=request)
+    job_submission_logs = await run_async(
+        get_log_storage().poll_logs, project=project, request=request
+    )
+    # Logs are stored in plaintext but transmitted in base64 for API/CLI backward compatibility.
+    # Old logs stored in base64 are encoded twice for transmission and shown as base64 in CLI/UI.
+    # We live with that.
+    # TODO: Drop base64 encoding in 0.20.
+    for log_event in job_submission_logs.logs:
+        log_event.message = b64encode_raw_message(log_event.message.encode())
+    return job_submission_logs

src/dstack/api/_public/runs.py

@@ -1,3 +1,4 @@
+import base64
 import queue
 import tempfile
 import threading
@@ -228,7 +229,7 @@ def logs(
                     ),
                 )
                 for log in resp.logs:
-                    yield log.message.encode()
+                    yield base64.b64decode(log.message)
                 next_token = resp.next_token
                 if next_token is None:
                     break

src/tests/_internal/server/routers/test_logs.py

@@ -62,17 +62,17 @@ async def test_returns_logs(
                 {
                     "timestamp": "2023-10-06T10:01:53.234234+00:00",
                     "log_source": "stdout",
-                    "message": "Hello",
+                    "message": "SGVsbG8=",
                 },
                 {
                     "timestamp": "2023-10-06T10:01:53.234235+00:00",
                     "log_source": "stdout",
-                    "message": "World",
+                    "message": "V29ybGQ=",
                 },
                 {
                     "timestamp": "2023-10-06T10:01:53.234236+00:00",
                     "log_source": "stdout",
-                    "message": "!",
+                    "message": "IQ==",
                 },
             ],
             "next_token": None,
@@ -93,7 +93,7 @@ async def test_returns_logs(
                 {
                     "timestamp": "2023-10-06T10:01:53.234236+00:00",
                     "log_source": "stdout",
-                    "message": "!",
+                    "message": "IQ==",
                 },
             ],
             "next_token": None,