Add client side logging (#72)

nithinkdb · web-flow · commit 067ce23c7f2f · 2022-10-24T12:18:21.000-07:00
* Initial commit

Signed-off-by: nithinkdb &lt;nithin.krishnamurthi@databricks.com&gt;

* Fixed comments ran linter

Signed-off-by: nithinkdb &lt;nithin.krishnamurthi@databricks.com&gt;

* Changed logging of uuids

Signed-off-by: nithinkdb &lt;nithin.krishnamurthi@databricks.com&gt;

* Fixed up comments, added functionality

Signed-off-by: nithinkdb &lt;nithin.krishnamurthi@databricks.com&gt;

* Fixed bugs

Signed-off-by: nithinkdb &lt;nithin.krishnamurthi@databricks.com&gt;

* Fixed import

Signed-off-by: nithinkdb &lt;nithin.krishnamurthi@databricks.com&gt;

* Ran linter

Signed-off-by: nithinkdb &lt;nithin.krishnamurthi@databricks.com&gt;

* Fixed up comments

Signed-off-by: nithinkdb &lt;nithin.krishnamurthi@databricks.com&gt;

* Added in substitution I'd forgotten

Signed-off-by: nithinkdb &lt;nithin.krishnamurthi@databricks.com&gt;

* Fixed code style

Signed-off-by: nithinkdb &lt;nithin.krishnamurthi@databricks.com&gt;

* Changed comment

Signed-off-by: nithinkdb &lt;nithin.krishnamurthi@databricks.com&gt;

* Ran Linter

Signed-off-by: nithinkdb &lt;nithin.krishnamurthi@databricks.com&gt;

Signed-off-by: nithinkdb &lt;nithin.krishnamurthi@databricks.com&gt;
diff --git a/.eslintrc b/.eslintrc
@@ -12,7 +12,8 @@
         "consistent-return": "off",
         "no-param-reassign": "off",
         "no-bitwise": "off",
-        "@typescript-eslint/no-throw-literal": "off"
+        "@typescript-eslint/no-throw-literal": "off",
+        "no-restricted-syntax": "off"
       }
     }
   ]
diff --git a/examples/logging.js b/examples/logging.js
@@ -0,0 +1,38 @@
+const { LOADIPHLPAPI } = require('dns');
+const { DBSQLClient, DBSQLLogger, LogLevel } = require('../');
+
+// This logger will emit logs to console and log.txt
+//
+const logger = new DBSQLLogger('log.txt', LogLevel.info);
+
+const client = new DBSQLClient({ logger: logger });
+
+client
+  .connect({
+    host: 'eng-academy.staging.cloud.databricks.com',
+    path: '/sql/1.0/endpoints/410886e8ec4b0796',
+    token: 'dapi5aef9f11ea0f3003440555526226ef7f',
+  })
+  .then(async (client) => {
+    const session = await client.openSession();
+
+    let queryOperation = await session.executeStatement('SELECT "Hello, World!"', { runAsync: true });
+    let result = await queryOperation.fetchAll();
+    await queryOperation.close();
+
+    console.table(result);
+
+    // Set logger to different level.
+    //
+    logger.setLevel(LogLevel.debug);
+
+    queryOperation = await session.executeStatement('SELECT "Hello, World!"', { runAsync: true });
+    result = await queryOperation.fetchAll();
+    await queryOperation.close();
+
+    await session.close();
+    await client.close();
+  })
+  .catch((error) => {
+    console.log(error);
+  });
diff --git a/lib/DBSQLClient.ts b/lib/DBSQLClient.ts
@@ -3,7 +3,7 @@ import thrift from 'thrift';
 import { EventEmitter } from 'events';
 import TCLIService from '../thrift/TCLIService';
 import { TProtocolVersion } from '../thrift/TCLIService_types';
-import IDBSQLClient, { ConnectionOptions, OpenSessionRequest } from './contracts/IDBSQLClient';
+import IDBSQLClient, { ConnectionOptions, OpenSessionRequest, ClientOptions } from './contracts/IDBSQLClient';
 import HiveDriver from './hive/HiveDriver';
 import { Int64 } from './hive/Types';
 import DBSQLSession from './DBSQLSession';
@@ -18,6 +18,8 @@ import StatusFactory from './factory/StatusFactory';
 import HiveDriverError from './errors/HiveDriverError';
 import { buildUserAgentString, definedOrError } from './utils';
 import PlainHttpAuthentication from './connection/auth/PlainHttpAuthentication';
+import IDBSQLLogger, { LogLevel } from './contracts/IDBSQLLogger';
+import DBSQLLogger from './DBSQLLogger';
 
 function getInitialNamespaceOptions(catalogName?: string, schemaName?: string) {
   if (!catalogName && !schemaName) {
@@ -43,15 +45,19 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient {
 
   private authProvider: IAuthentication;
 
+  private logger: IDBSQLLogger;
+
   private thrift = thrift;
 
-  constructor() {
+  constructor(options: ClientOptions) {
     super();
     this.connectionProvider = new HttpConnection();
     this.authProvider = new NoSaslAuthentication();
     this.statusFactory = new StatusFactory();
+    this.logger = options?.logger || new DBSQLLogger();
     this.client = null;
     this.connection = null;
+    this.logger.log(LogLevel.info, 'Created DBSQLClient');
   }
 
   private getConnectionOptions(options: ConnectionOptions): IConnectionOptions {
@@ -88,18 +94,22 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient {
     this.client = this.thrift.createClient(TCLIService, this.connection.getConnection());
 
     this.connection.getConnection().on('error', (error: Error) => {
+      this.logger.log(LogLevel.error, JSON.stringify(error));
       this.emit('error', error);
     });
 
     this.connection.getConnection().on('reconnecting', (params: { delay: number; attempt: number }) => {
+      this.logger.log(LogLevel.debug, `Reconnecting, params: ${JSON.stringify(params)}`);
       this.emit('reconnecting', params);
     });
 
     this.connection.getConnection().on('close', () => {
+      this.logger.log(LogLevel.debug, 'Closing connection.');
       this.emit('close');
     });
 
     this.connection.getConnection().on('timeout', () => {
+      this.logger.log(LogLevel.debug, 'Connection timed out.');
       this.emit('timeout');
     });
 
@@ -129,7 +139,7 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient {
       })
       .then((response) => {
         this.statusFactory.create(response.status);
-        return new DBSQLSession(driver, definedOrError(response.sessionHandle));
+        return new DBSQLSession(driver, definedOrError(response.sessionHandle), this.logger);
       });
   }
 
diff --git a/lib/DBSQLLogger.ts b/lib/DBSQLLogger.ts
@@ -0,0 +1,31 @@
+import winston, { Logger } from 'winston';
+import IDBSQLLogger, { LogLevel } from './contracts/IDBSQLLogger';
+
+export default class DBSQLLogger implements IDBSQLLogger {
+  logger: Logger;
+
+  transports: any;
+
+  constructor(filepath?: string, level = LogLevel.info) {
+    this.transports = {
+      console: new winston.transports.Console({ handleExceptions: true, level }),
+    };
+    this.logger = winston.createLogger({
+      transports: [this.transports.console],
+    });
+    if (filepath) {
+      this.transports.file = new winston.transports.File({ filename: filepath, handleExceptions: true, level });
+      this.logger.add(this.transports.file);
+    }
+  }
+
+  async log(level: LogLevel, message: string) {
+    this.logger.log({ level, message });
+  }
+
+  setLevel(level: LogLevel) {
+    for (const key of Object.keys(this.transports)) {
+      this.transports[key].level = level;
+    }
+  }
+}
diff --git a/lib/DBSQLOperation/index.ts b/lib/DBSQLOperation/index.ts
@@ -1,3 +1,4 @@
+import { stringify, NIL, parse } from 'uuid';
 import IOperation, { FetchOptions, GetSchemaOptions, FinishedOptions, defaultMaxRows } from '../contracts/IOperation';
 import HiveDriver from '../hive/HiveDriver';
 import {
@@ -13,12 +14,15 @@ import OperationStatusHelper from './OperationStatusHelper';
 import SchemaHelper from './SchemaHelper';
 import FetchResultsHelper from './FetchResultsHelper';
 import CompleteOperationHelper from './CompleteOperationHelper';
+import IDBSQLLogger, { LogLevel } from '../contracts/IDBSQLLogger';
 
 export default class DBSQLOperation implements IOperation {
   private driver: HiveDriver;
 
   private operationHandle: TOperationHandle;
 
+  private logger: IDBSQLLogger;
+
   private _status: OperationStatusHelper;
 
   private _schema: SchemaHelper;
@@ -27,9 +31,15 @@ export default class DBSQLOperation implements IOperation {
 
   private _completeOperation: CompleteOperationHelper;
 
-  constructor(driver: HiveDriver, operationHandle: TOperationHandle, directResults?: TSparkDirectResults) {
+  constructor(
+    driver: HiveDriver,
+    operationHandle: TOperationHandle,
+    logger: IDBSQLLogger,
+    directResults?: TSparkDirectResults,
+  ) {
     this.driver = driver;
     this.operationHandle = operationHandle;
+    this.logger = logger;
     this._status = new OperationStatusHelper(this.driver, this.operationHandle, directResults?.operationStatus);
     this._schema = new SchemaHelper(this.driver, this.operationHandle, directResults?.resultSetMetadata);
     this._data = new FetchResultsHelper(this.driver, this.operationHandle, [directResults?.resultSet]);
@@ -38,6 +48,11 @@ export default class DBSQLOperation implements IOperation {
       this.operationHandle,
       directResults?.closeOperation,
     );
+    this.logger.log(LogLevel.debug, `Operation created with id: ${this.getId()}`);
+  }
+
+  getId() {
+    return stringify(this.operationHandle?.operationId?.guid || parse(NIL));
   }
 
   /**
@@ -56,6 +71,7 @@ export default class DBSQLOperation implements IOperation {
       const chunk = await this.fetchChunk(options);
       data.push(chunk);
     } while (await this.hasMoreRows()); // eslint-disable-line no-await-in-loop
+    this.logger?.log(LogLevel.debug, `Fetched all data from operation with id: ${this.getId()}`);
 
     return data.flat();
   }
@@ -79,6 +95,10 @@ export default class DBSQLOperation implements IOperation {
     return Promise.all([this._schema.fetch(), this._data.fetch(options?.maxRows || defaultMaxRows)]).then(
       ([schema, data]) => {
         const result = getResult(schema, data ? [data] : []);
+        this.logger?.log(
+          LogLevel.debug,
+          `Fetched chunk of size: ${options?.maxRows || defaultMaxRows} from operation with id: ${this.getId()}`,
+        );
         return Promise.resolve(result);
       },
     );
@@ -90,6 +110,7 @@ export default class DBSQLOperation implements IOperation {
    * @throws {StatusError}
    */
   async status(progress: boolean = false): Promise<TGetOperationStatusResp> {
+    this.logger?.log(LogLevel.debug, `Fetching status for operation with id: ${this.getId()}`);
     return this._status.status(progress);
   }
 
@@ -98,6 +119,7 @@ export default class DBSQLOperation implements IOperation {
    * @throws {StatusError}
    */
   cancel(): Promise<Status> {
+    this.logger?.log(LogLevel.debug, `Operation with id: ${this.getId()} canceled.`);
     return this._completeOperation.cancel();
   }
 
@@ -106,6 +128,7 @@ export default class DBSQLOperation implements IOperation {
    * @throws {StatusError}
    */
   close(): Promise<Status> {
+    this.logger?.log(LogLevel.debug, `Closing operation with id: ${this.getId()}`);
     return this._completeOperation.close();
   }
 
@@ -126,6 +149,7 @@ export default class DBSQLOperation implements IOperation {
     }
 
     await this._status.waitUntilReady(options);
+    this.logger?.log(LogLevel.debug, `Fetching schema for operation with id: ${this.getId()}`);
 
     return this._schema.fetch();
   }
diff --git a/lib/DBSQLSession.ts b/lib/DBSQLSession.ts
@@ -1,3 +1,4 @@
+import { stringify, NIL, parse } from 'uuid';
 import { TSessionHandle, TStatus, TOperationHandle, TSparkDirectResults } from '../thrift/TCLIService_types';
 import HiveDriver from './hive/HiveDriver';
 import { Int64 } from './hive/Types';
@@ -19,6 +20,7 @@ import Status from './dto/Status';
 import StatusFactory from './factory/StatusFactory';
 import InfoValue from './dto/InfoValue';
 import { definedOrError } from './utils';
+import IDBSQLLogger, { LogLevel } from './contracts/IDBSQLLogger';
 
 interface OperationResponseShape {
   status: TStatus;
@@ -45,10 +47,18 @@ export default class DBSQLSession implements IDBSQLSession {
 
   private statusFactory: StatusFactory;
 
-  constructor(driver: HiveDriver, sessionHandle: TSessionHandle) {
+  private logger: IDBSQLLogger;
+
+  constructor(driver: HiveDriver, sessionHandle: TSessionHandle, logger: IDBSQLLogger) {
     this.driver = driver;
     this.sessionHandle = sessionHandle;
     this.statusFactory = new StatusFactory();
+    this.logger = logger;
+    this.logger.log(LogLevel.debug, `Session created with id: ${this.getId()}`);
+  }
+
+  getId() {
+    return stringify(this.sessionHandle?.sessionId?.guid || parse(NIL));
   }
 
   /**
@@ -322,13 +332,16 @@ export default class DBSQLSession implements IDBSQLSession {
       .closeSession({
         sessionHandle: this.sessionHandle,
       })
-      .then((response) => this.statusFactory.create(response.status));
+      .then((response) => {
+        this.logger.log(LogLevel.debug, `Session closed with id: ${this.getId()}`);
+        return this.statusFactory.create(response.status);
+      });
   }
 
   private createOperation(response: OperationResponseShape): IOperation {
     this.assertStatus(response.status);
     const handle = definedOrError(response.operationHandle);
-    return new DBSQLOperation(this.driver, handle, response.directResults);
+    return new DBSQLOperation(this.driver, handle, this.logger, response.directResults);
   }
 
   private assertStatus(responseStatus: TStatus): void {
diff --git a/lib/contracts/IDBSQLClient.ts b/lib/contracts/IDBSQLClient.ts
@@ -1,5 +1,10 @@
+import IDBSQLLogger from './IDBSQLLogger';
 import IDBSQLSession from './IDBSQLSession';
 
+export interface ClientOptions {
+  logger?: IDBSQLLogger;
+}
+
 export interface ConnectionOptions {
   host: string;
   port?: number;
diff --git a/lib/contracts/IDBSQLLogger.ts b/lib/contracts/IDBSQLLogger.ts
@@ -0,0 +1,10 @@
+export default interface IDBSQLLogger {
+  log(level: LogLevel, message: string): void;
+}
+
+export enum LogLevel {
+  error = 'error',
+  warn = 'warn',
+  info = 'info',
+  debug = 'debug',
+}
diff --git a/lib/index.ts b/lib/index.ts
@@ -2,10 +2,12 @@ import TCLIService from '../thrift/TCLIService';
 import TCLIService_types from '../thrift/TCLIService_types';
 import DBSQLClient from './DBSQLClient';
 import DBSQLSession from './DBSQLSession';
+import DBSQLLogger from './DBSQLLogger';
 import NoSaslAuthentication from './connection/auth/NoSaslAuthentication';
 import PlainHttpAuthentication from './connection/auth/PlainHttpAuthentication';
 import HttpConnection from './connection/connections/HttpConnection';
 import { formatProgress } from './utils';
+import { LogLevel } from './contracts/IDBSQLLogger';
 
 export const auth = {
   NoSaslAuthentication,
@@ -25,4 +27,4 @@ export const utils = {
   formatProgress,
 };
 
-export { DBSQLClient, DBSQLSession };
+export { DBSQLClient, DBSQLSession, DBSQLLogger, LogLevel };
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
diff --git a/tests/unit/DBSQLOperation.test.js b/tests/unit/DBSQLOperation.test.js
diff --git a/tests/unit/DBSQLSession.test.js b/tests/unit/DBSQLSession.test.js

Original file line number	Diff line number	Diff line change
`@@ -12,7 +12,8 @@`
`12`	`12`	`"consistent-return": "off",`
`13`	`13`	`"no-param-reassign": "off",`
`14`	`14`	`"no-bitwise": "off",`
`15`		`- "@typescript-eslint/no-throw-literal": "off"`
	`15`	`+ "@typescript-eslint/no-throw-literal": "off",`
	`16`	`+ "no-restricted-syntax": "off"`
`16`	`17`	`}`
`17`	`18`	`}`
`18`	`19`	`]`