[UI] Query logs using descending (#2915)

- [x] Allowed `descending` in `PollLogsRequest`
- [x] Supported `descending` in `FileLogStorage` (implemented an algorithm that reads lines from the end of t he file)
- [x] Updated `CloudWatchLogStorage` to support `MAX_RETRIES` (to skip empty pages)
- [x] Re-implemented to make `_read_lines_reversed` fast regardless of `start_offset`

---------

Co-authored-by: Oleg Vavilov <vavilovolegik@gmail.com>

Author: peterschmidt85

frontend/package-lock.json

@@ -105,8 +105,6 @@
     "@cloudscape-design/global-styles": "^1.0.33",
     "@hookform/resolvers": "^2.9.10",
     "@reduxjs/toolkit": "^1.9.1",
-    "@xterm/addon-fit": "^0.10.0",
-    "@xterm/xterm": "^5.5.0",
     "ace-builds": "^1.36.3",
     "classnames": "^2.5.1",
     "css-minimizer-webpack-plugin": "^4.2.2",

frontend/package.json

@@ -1,4 +1,4 @@
-import React from 'react';
+import React, { forwardRef } from 'react';
 import classNames from 'classnames';
 import Box from '@cloudscape-design/components/box';
 
@@ -8,12 +8,12 @@ export interface Props extends React.PropsWithChildren {
     className?: string;
 }
 
-export const Code: React.FC<Props> = ({ children, className }) => {
+export const Code = forwardRef<HTMLDivElement, Props>(({ children, className }, ref) => {
     return (
-        <div className={classNames(styles.code, className)}>
+        <div ref={ref} className={classNames(styles.code, className)}>
             <Box variant="code" color="text-status-inactive">
                 {children}
             </Box>
         </div>
     );
-};
+});

frontend/src/components/Code/index.tsx

@@ -56,7 +56,6 @@ export { default as PropertyFilter } from '@cloudscape-design/components/propert
 export type { PropertyFilterProps } from '@cloudscape-design/components/property-filter';
 export type { LineChartProps } from '@cloudscape-design/components/line-chart/interfaces';
 export type { ModalProps } from '@cloudscape-design/components/modal';
-export type { TilesProps } from '@cloudscape-design/components/tiles';
 
 // custom components
 export { NavigateLink } from './NavigateLink';

frontend/src/components/index.ts

@@ -12,7 +12,6 @@ import 'ace-builds/css/ace.css';
 import 'ace-builds/css/theme/cloud_editor.css';
 import 'ace-builds/css/theme/cloud_editor_dark.css';
 import 'assets/css/index.css';
-import '@xterm/xterm/css/xterm.css';
 
 import 'locale';
 

frontend/src/index.tsx

@@ -7,3 +7,18 @@ export const getJobSubmissionId = (run?: IRun): string | undefined => {
 
     return lastJob.job_submissions[lastJob.job_submissions.length - 1]?.id;
 };
+
+export const decodeLogs = (logs: ILogItem[]): ILogItem[] => {
+    return logs.map((log: ILogItem) => {
+        let { message } = log;
+
+        try {
+            message = atob(message);
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        } catch (e) {
+            return log;
+        }
+
+        return { ...log, message };
+    });
+};

frontend/src/pages/Runs/Details/Logs/helpers.ts

@@ -1,43 +1,66 @@
-import React, { useEffect, useRef, useState } from 'react';
+import React, { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import classNames from 'classnames';
-import { Mode } from '@cloudscape-design/global-styles';
-import { FitAddon } from '@xterm/addon-fit';
-import { Terminal } from '@xterm/xterm';
 
-import { Container, Header, ListEmptyMessage, Loader, TextContent } from 'components';
+import { Box, Button, Code, Container, Header, ListEmptyMessage, Loader, TextContent } from 'components';
 
-import { useAppSelector } from 'hooks';
+import { useLocalStorageState } from 'hooks/useLocalStorageState';
 import { useLazyGetProjectLogsQuery } from 'services/project';
 
-import { selectSystemMode } from 'App/slice';
+import { decodeLogs } from './helpers';
 
 import { IProps } from './types';
 
 import styles from './styles.module.scss';
 
-const LIMIT_LOG_ROWS = 1000;
+const LIMIT_LOG_ROWS = 100;
+const LOADING_SCROLL_GAP = 300;
 
 export const Logs: React.FC<IProps> = ({ className, projectName, runName, jobSubmissionId }) => {
     const { t } = useTranslation();
-    const appliedTheme = useAppSelector(selectSystemMode);
+    const codeRef = useRef<HTMLDivElement>(null);
+    const nextTokenRef = useRef<string | undefined>(undefined);
+    const scrollPositionByBottom = useRef<number>(0);
 
-    const terminalInstance = useRef<Terminal>(new Terminal({ scrollback: 10000000 }));
-    const fitAddonInstance = useRef<FitAddon>(new FitAddon());
     const [logsData, setLogsData] = useState<ILogItem[]>([]);
     const [isLoading, setIsLoading] = useState(false);
-
     const [getProjectLogs] = useLazyGetProjectLogsQuery();
+    const [isEnabledDecoding, setIsEnabledDecoding] = useLocalStorageState('enable-encode-logs', false);
+    // const [isShowTimestamp, setIsShowTimestamp] = useLocalStorageState('enable-showing-timestamp-logs', false);
+
+    const logsForView = useMemo(() => {
+        if (isEnabledDecoding) {
+            return decodeLogs(logsData);
+        }
+
+        return logsData;
+    }, [logsData, isEnabledDecoding]);
+
+    const saveScrollPositionByBottom = () => {
+        if (!codeRef.current) return;
 
-    const writeDataToTerminal = (logs: ILogItem[]) => {
-        logs.forEach((logItem) => {
-            terminalInstance.current.write(logItem.message.replace(/(?<!\r)\n/g, '\r\n'));
-        });
+        const { clientHeight, scrollHeight, scrollTop } = codeRef.current;
+        scrollPositionByBottom.current = scrollHeight - clientHeight - scrollTop;
+    };
+
+    const restoreScrollPositionByBottom = () => {
+        if (!codeRef.current) return;
 
-        fitAddonInstance.current.fit();
+        const { clientHeight, scrollHeight } = codeRef.current;
+        codeRef.current.scrollTo(0, scrollHeight - clientHeight - scrollPositionByBottom.current);
     };
 
-    const getNextLogItems = (nextToken?: string) => {
+    const checkNeedMoreLoadingData = () => {
+        if (!codeRef.current) return;
+
+        const { clientHeight, scrollHeight } = codeRef.current;
+
+        if (scrollHeight - clientHeight <= LOADING_SCROLL_GAP) {
+            getLogItems();
+        }
+    };
+
+    const getLogItems = (nextToken?: string) => {
         setIsLoading(true);
 
         if (!jobSubmissionId) {
@@ -47,86 +70,131 @@ export const Logs: React.FC<IProps> = ({ className, projectName, runName, jobSub
         getProjectLogs({
             project_name: projectName,
             run_name: runName,
-            descending: false,
-            job_submission_id: jobSubmissionId ?? '',
+            descending: true,
+            job_submission_id: jobSubmissionId,
             next_token: nextToken,
             limit: LIMIT_LOG_ROWS,
         })
             .unwrap()
             .then((response) => {
-                setLogsData((old) => [...old, ...response.logs]);
-
-                writeDataToTerminal(response.logs);
-
-                if (response.next_token) {
-                    getNextLogItems(response.next_token);
-                } else {
-                    setIsLoading(false);
-                }
+                saveScrollPositionByBottom();
+                const reversed = response.logs.toReversed();
+                setLogsData((old) => [...reversed, ...old]);
+                nextTokenRef.current = response.next_token;
+                setIsLoading(false);
             })
             .catch(() => setIsLoading(false));
     };
 
+    const getNextLogItems = () => {
+        if (nextTokenRef.current) {
+            getLogItems(nextTokenRef.current);
+        }
+    };
+
+    const toggleDecodeLogs = () => {
+        saveScrollPositionByBottom();
+        setIsEnabledDecoding(!isEnabledDecoding);
+    };
+
     useEffect(() => {
-        if (appliedTheme === Mode.Light) {
-            terminalInstance.current.options.theme = {
-                foreground: '#000716',
-                background: '#ffffff',
-                selectionBackground: '#B4D5FE',
-            };
+        getLogItems();
+    }, []);
+
+    useLayoutEffect(() => {
+        if (logsForView.length && logsForView.length <= LIMIT_LOG_ROWS) {
+            scrollToBottom();
         } else {
-            terminalInstance.current.options.theme = {
-                foreground: '#b6bec9',
-                background: '#161d26',
-            };
+            restoreScrollPositionByBottom();
         }
-    }, [appliedTheme]);
 
-    useEffect(() => {
-        terminalInstance.current.loadAddon(fitAddonInstance.current);
+        if (logsForView.length) checkNeedMoreLoadingData();
+    }, [logsForView]);
 
-        getNextLogItems();
+    const onScroll = useCallback<EventListener>(
+        (event) => {
+            const element = event.target as HTMLDivElement;
 
-        const onResize = () => {
-            fitAddonInstance.current.fit();
-        };
+            if (element.scrollTop <= LOADING_SCROLL_GAP && !isLoading) {
+                getNextLogItems();
+            }
+        },
+        [isLoading, logsForView],
+    );
+
+    useEffect(() => {
+        if (!codeRef.current) return;
 
-        window.addEventListener('resize', onResize);
+        codeRef.current.addEventListener('scroll', onScroll);
 
         return () => {
-            window.removeEventListener('resize', onResize);
+            if (codeRef.current) codeRef.current.removeEventListener('scroll', onScroll);
         };
-    }, []);
+    }, [codeRef.current, onScroll]);
 
-    useEffect(() => {
-        const element = document.getElementById('terminal');
+    const scrollToBottom = () => {
+        if (!codeRef.current) return;
 
-        if (terminalInstance.current && element) {
-            terminalInstance.current.open(element);
-        }
-    }, []);
+        const { clientHeight, scrollHeight } = codeRef.current;
+        codeRef.current.scrollTo(0, scrollHeight - clientHeight);
+    };
 
     return (
         <div className={classNames(styles.logs, className)}>
             <Container
                 header={
-                    <Header variant="h2">
-                        <div className={styles.headerContainer}>
-                            {t('projects.run.log')}
-                            <Loader show={isLoading} padding={'n'} className={classNames(styles.loader)} loadingText={''} />
+                    <div className={styles.headerContainer}>
+                        <div className={styles.headerTitle}>
+                            <Header variant="h2">{t('projects.run.log')}</Header>
                         </div>
-                    </Header>
+
+                        <Loader
+                            show={isLoading && Boolean(logsForView.length)}
+                            padding={'n'}
+                            className={styles.loader}
+                            loadingText={''}
+                        />
+
+                        <div className={styles.switchers}>
+                            <Box>
+                                <Button
+                                    ariaLabel="Legacy mode"
+                                    formAction="none"
+                                    iconName="gen-ai"
+                                    variant={isEnabledDecoding ? 'primary' : 'icon'}
+                                    onClick={toggleDecodeLogs}
+                                />
+                            </Box>
+
+                            {/*<Box>*/}
+                            {/*    <Toggle onChange={({ detail }) => setIsShowTimestamp(detail.checked)} checked={isShowTimestamp}>*/}
+                            {/*        Show timestamp*/}
+                            {/*    </Toggle>*/}
+                            {/*</Box>*/}
+                        </div>
+                    </div>
                 }
             >
                 <TextContent>
-                    {!isLoading && !logsData.length && (
+                    {!isLoading && !logsForView.length && (
                         <ListEmptyMessage
                             title={t('projects.run.log_empty_message_title')}
                             message={t('projects.run.log_empty_message_text')}
                         />
                     )}
 
-                    <div className={styles.terminal} id="terminal" />
+                    {!logsForView.length && <Loader show={isLoading} className={styles.mainLoader} />}
+
+                    {Boolean(logsForView.length) && (
+                        <Code className={styles.terminal} ref={codeRef}>
+                            {logsForView.map((log, i) => (
+                                <p key={i}>
+                                    {/*{isShowTimestamp && <span className={styles.timestamp}>{log.timestamp}</span>}*/}
+                                    {log.message}
+                                </p>
+                            ))}
+                        </Code>
+                    )}
                 </TextContent>
             </Container>
         </div>