Skip to content

Commit bb945d4

Browse files
NightlngaleNightlngale
authored
HCK-16468: Improve logging and run queries sequential (#195)
* HCK-16468: Make all call sequential * HCK-16468: Skip memory optimized query if unsupported * HCK-16468: Fix remarks * HCK-16468: Logged normalized query * HCK-16468: Improve query formatting in logs
1 parent 647d49e commit bb945d4

2 files changed

Lines changed: 118 additions & 40 deletions

File tree

‎reverse_engineering/databaseService/databaseService.js‎

Lines changed: 89 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,15 @@ const { parseProcedure } = require('../helpers/parsers/parseProcedure');
99

1010
const PERMISSION_DENIED_CODE = 297;
1111

12+
const normalizeQueryForLogging = queryParams =>
13+
queryParams.map(param => {
14+
if (typeof param === 'string') {
15+
return param.replace(/\t/g, ' ');
16+
}
17+
18+
return param;
19+
});
20+
1221
const addPermissionDeniedMetaData = ({ error, meta }) => {
1322
error.message =
1423
'The user does not have permission to perform ' +
@@ -33,7 +42,12 @@ const getClient = async ({ client, dbName, meta, logger }) => {
3342
},
3443
async query(...queryParams) {
3544
try {
36-
return await currentDbConnectionClient.query(...queryParams);
45+
logger.log('info', { query: normalizeQueryForLogging(queryParams) }, 'Performing query');
46+
const start = Date.now();
47+
const result = await currentDbConnectionClient.query(...queryParams);
48+
const duration = Date.now() - start;
49+
logger.log('info', { 'action': meta.action, duration: `${duration}ms` }, 'Query executed');
50+
return result;
3751
} catch (error) {
3852
if (meta) {
3953
if (error.number === PERMISSION_DENIED_CODE) {
@@ -265,6 +279,7 @@ const getDatabaseIndexes = async ({ client, dbName, tablesInfo, logger }) => {
265279
});
266280

267281
logger.log('info', { message: `Get '${dbName}' database indexes.` }, 'Reverse Engineering');
282+
logger.progress({ message: 'Discovering table index metadata', containerName: dbName, entityName: '' });
268283

269284
const tableAlias = 't';
270285
const leftJoinClauseParts = Object.entries(tablesInfo).map(([schemaName, tableNames]) => {
@@ -315,6 +330,8 @@ const getIndexesBucketCount = async ({ client, dbName, indexesId, logger }) => {
315330
logger,
316331
});
317332

333+
logger.log('info', { message: `Get '${dbName}' database indexes bucket count.` }, 'Reverse Engineering');
334+
318335
return mapResponse(
319336
await currentDbConnectionClient.query(`
320337
SELECT hs.total_bucket_count, hs.index_id
@@ -343,6 +360,7 @@ const getSpatialIndexes = async ({ client, dbName, allUniqueSchemasAndTables, lo
343360
});
344361

345362
logger.log('info', { message: `Get '${dbName}' database spatial indexes.` }, 'Reverse Engineering');
363+
logger.progress({ message: 'Discovering spatial index metadata', containerName: dbName, entityName: '' });
346364

347365
const tableAlias = 't';
348366
const whereClauseParts = getWhereClauseForUniqueSchemasAndTables({ tableAlias, allUniqueSchemasAndTables });
@@ -400,6 +418,7 @@ const getFullTextIndexes = async ({ client, dbName, allUniqueSchemasAndTables, l
400418
});
401419

402420
logger.log('info', { message: `Get '${dbName}' database full text indexes.` }, 'Reverse Engineering');
421+
logger.progress({ message: 'Discovering full-text index metadata', containerName: dbName, entityName: '' });
403422

404423
const tableAlias = 'F';
405424
const whereClauseParts = getWhereClauseForUniqueSchemasAndTables({ tableAlias, allUniqueSchemasAndTables });
@@ -458,6 +477,7 @@ const getViewsIndexes = async ({ client, dbName, logger }) => {
458477
});
459478

460479
logger.log('info', { message: `Get '${dbName}' database views indexes.` }, 'Reverse Engineering');
480+
logger.progress({ message: 'Discovering view index metadata', containerName: dbName, entityName: '' });
461481

462482
const tableAlias = 'ind';
463483

@@ -536,21 +556,56 @@ const getDatabaseMemoryOptimizedTables = async ({ client, dbName, logger }) => {
536556
logger,
537557
});
538558

539-
logger.log('info', { message: `Get '${dbName}' database memory optimized indexes.` }, 'Reverse Engineering');
559+
logger.log(
560+
'info',
561+
{ message: `Detecting if '${dbName}' database supports memory optimized tables.` },
562+
'Reverse Engineering',
563+
);
564+
logger.progress({ message: 'Discovering memory-optimized table metadata', containerName: dbName, entityName: '' });
565+
566+
const { isMemoryOptimizedTableSupported, isHistoryTableSupported } = await getSysTablesCatalogCapabilities({
567+
client,
568+
dbName,
569+
});
570+
571+
if (!isMemoryOptimizedTableSupported) {
572+
logger.log(
573+
'info',
574+
{ message: `Memory optimized tables are not supported in '${dbName}' database.` },
575+
'Reverse Engineering',
576+
);
577+
return [];
578+
}
579+
580+
if (isHistoryTableSupported) {
581+
return mapResponse(
582+
await currentDbConnectionClient.query`
583+
SELECT
584+
T.name,
585+
T.durability,
586+
T.durability_desc,
587+
OBJECT_NAME(T.history_table_id) AS history_table,
588+
SCHEMA_NAME(O.schema_id) AS history_schema,
589+
T.temporal_type_desc,
590+
T.is_memory_optimized
591+
FROM sys.tables T
592+
LEFT JOIN sys.objects O
593+
ON T.history_table_id = O.object_id
594+
WHERE T.is_memory_optimized = 1
595+
`,
596+
);
597+
}
598+
599+
logger.log('info', { message: `History tables are not supported in '${dbName}' database.` }, 'Reverse Engineering');
540600

541601
return mapResponse(
542602
await currentDbConnectionClient.query`
543603
SELECT
544604
T.name,
545605
T.durability,
546606
T.durability_desc,
547-
OBJECT_NAME(T.history_table_id) AS history_table,
548-
SCHEMA_NAME(O.schema_id) AS history_schema,
549-
T.temporal_type_desc,
550607
T.is_memory_optimized
551608
FROM sys.tables T
552-
LEFT JOIN sys.objects O
553-
ON T.history_table_id = O.object_id
554609
WHERE T.is_memory_optimized = 1
555610
`,
556611
);
@@ -569,6 +624,7 @@ const getDatabaseCheckConstraints = async ({ client, dbName, allUniqueSchemasAnd
569624
});
570625

571626
logger.log('info', { message: `Get '${dbName}' database check constraints.` }, 'Reverse Engineering');
627+
logger.progress({ message: 'Discovering check constraint metadata', containerName: dbName, entityName: '' });
572628

573629
const tableAlias = 't';
574630
const whereClauseParts = getWhereClauseForUniqueSchemasAndTables({ tableAlias, allUniqueSchemasAndTables });
@@ -790,11 +846,10 @@ const getDatabaseXmlSchemaCollection = async ({ client, dbName, allUniqueSchemas
790846
});
791847

792848
logger.log('info', { message: `Get '${dbName}' database xml schema collection.` }, 'Reverse Engineering');
849+
logger.progress({ message: 'Discovering XML schema collection metadata', containerName: dbName, entityName: '' });
793850

794-
const schemaAlias = 'xsc';
795851
const tableAlias = 'xcu';
796852
const whereClauseParts = getWhereClauseForUniqueSchemasAndTables({
797-
schemaAlias,
798853
tableAlias,
799854
allUniqueSchemasAndTables,
800855
});
@@ -860,6 +915,7 @@ const getDatabaseUserDefinedTypes = async ({ client, dbName, logger }) => {
860915
});
861916

862917
logger.log('info', { message: `Get '${dbName}' database UDTs.` }, 'Reverse Engineering');
918+
logger.progress({ message: 'Discovering user-defined type metadata', containerName: dbName, entityName: '' });
863919

864920
return mapResponse(currentDbConnectionClient.query`
865921
SELECT * FROM sys.types
@@ -889,6 +945,24 @@ const mapResponse = async (response = Promise.resolve({})) => {
889945
return resp.recordset ? resp.recordset : resp;
890946
};
891947

948+
const getSysTablesCatalogCapabilities = async ({ client, dbName }) => {
949+
const connectionClient = await getNewConnectionClientByDb(client, dbName);
950+
const [{ Has_HistoryTableId, Has_IsMemoryOptimized } = {}] = await mapResponse(
951+
connectionClient.query(`
952+
SELECT
953+
COUNT(CASE WHEN name = 'history_table_id' THEN 1 END) AS Has_HistoryTableId,
954+
COUNT(CASE WHEN name = 'is_memory_optimized' THEN 1 END) AS Has_IsMemoryOptimized
955+
FROM sys.all_columns
956+
WHERE object_id = OBJECT_ID('sys.tables')
957+
`),
958+
);
959+
960+
return {
961+
isHistoryTableSupported: Boolean(Has_HistoryTableId),
962+
isMemoryOptimizedTableSupported: Boolean(Has_IsMemoryOptimized),
963+
};
964+
};
965+
892966
const getDescriptionComments = async ({ client, dbName, schema, entity, logger }) => {
893967
const currentDbConnectionClient = await getClient({
894968
client,
@@ -922,12 +996,8 @@ const buildDescriptionCommentsRetrieveQuery = ({ schema, entity }) => {
922996
return `SELECT objtype, objname, value FROM fn_listextendedproperty ('MS_Description', ${schemaTemplate}, ${entityTemplate});`;
923997
};
924998

925-
const getWhereClauseForUniqueSchemasAndTables = ({
926-
schemaAlias,
927-
tableAlias,
928-
allUniqueSchemasAndTables: { schemas, tables },
929-
}) =>
930-
`OBJECT_SCHEMA_NAME(${schemaAlias || tableAlias}.object_id) IN (${[...schemas].join(', ')})
999+
const getWhereClauseForUniqueSchemasAndTables = ({ tableAlias, allUniqueSchemasAndTables: { schemas, tables } }) =>
1000+
`OBJECT_SCHEMA_NAME(${tableAlias}.object_id) IN (${[...schemas].join(', ')})
9311001
AND OBJECT_NAME(${tableAlias}.object_id) IN (${[...tables].join(', ')})`;
9321002

9331003
const getDatabaseProcedures = async ({ client, dbName, logger }) => {
@@ -943,6 +1013,7 @@ const getDatabaseProcedures = async ({ client, dbName, logger }) => {
9431013
});
9441014

9451015
logger.log('info', { message: `Get '${dbName}' database procedures.` }, 'Reverse Engineering');
1016+
logger.progress({ message: 'Discovering stored procedure metadata', containerName: dbName, entityName: '' });
9461017

9471018
const response = await currentDbConnectionClient.query(`
9481019
SELECT
@@ -951,11 +1022,11 @@ const getDatabaseProcedures = async ({ client, dbName, logger }) => {
9511022
sm.definition AS procedure_body,
9521023
ep.value AS description
9531024
FROM sys.procedures p
954-
JOIN sys.schemas s
1025+
JOIN sys.schemas s
9551026
ON p.schema_id = s.schema_id
956-
LEFT JOIN sys.sql_modules sm
1027+
LEFT JOIN sys.sql_modules sm
9571028
ON p.object_id = sm.object_id
958-
LEFT JOIN sys.extended_properties ep
1029+
LEFT JOIN sys.extended_properties ep
9591030
ON ep.major_id = p.object_id
9601031
AND ep.minor_id = 0
9611032
AND ep.name = 'MS_Description'

‎reverse_engineering/reverseEngineeringService/reverseEngineeringService.js‎

Lines changed: 29 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -288,34 +288,41 @@ const addTotalBucketCountToDatabaseIndexes = ({ databaseIndexes, indexesBucketCo
288288
const fetchDatabaseMetadata = async ({ client, dbName, tablesInfo, logger }) => {
289289
const allUniqueSchemasAndTables = getAllUniqueSchemasAndTables({ tablesInfo });
290290

291-
const [
292-
rawDatabaseIndexes,
293-
databaseMemoryOptimizedTables,
294-
databaseCheckConstraints,
295-
xmlSchemaCollections,
296-
databaseUDT,
297-
viewsIndexes,
298-
fullTextIndexes,
299-
spatialIndexes,
300-
procedures,
301-
] = await Promise.all([
302-
getDatabaseIndexes({ client, dbName, tablesInfo, logger }),
303-
getDatabaseMemoryOptimizedTables({ client, dbName, logger }),
304-
getDatabaseCheckConstraints({ client, dbName, allUniqueSchemasAndTables, logger }),
305-
getDatabaseXmlSchemaCollection({ client, dbName, allUniqueSchemasAndTables, logger }),
306-
getDatabaseUserDefinedTypes({ client, dbName, logger }),
307-
getViewsIndexes({ client, dbName, logger }),
308-
getFullTextIndexes({ client, dbName, allUniqueSchemasAndTables, logger }),
309-
getSpatialIndexes({ client, dbName, allUniqueSchemasAndTables, logger }),
310-
getDatabaseProcedures({ client, dbName, logger }),
311-
]);
312-
291+
const rawDatabaseIndexes = await getDatabaseIndexes({ client, dbName, tablesInfo, logger });
292+
const databaseCheckConstraints = await getDatabaseCheckConstraints({
293+
client,
294+
dbName,
295+
allUniqueSchemasAndTables,
296+
logger,
297+
});
298+
const viewsIndexes = await getViewsIndexes({ client, dbName, logger });
299+
const fullTextIndexes = await getFullTextIndexes({
300+
client,
301+
dbName,
302+
allUniqueSchemasAndTables,
303+
logger,
304+
});
305+
const spatialIndexes = await getSpatialIndexes({
306+
client,
307+
dbName,
308+
allUniqueSchemasAndTables,
309+
logger,
310+
});
311+
const procedures = await getDatabaseProcedures({ client, dbName, logger });
313312
const indexesBucketCount = await getIndexesBucketCount({
314313
client,
315314
dbName,
316315
indexesId: rawDatabaseIndexes.map(i => i.index_id),
317316
logger,
318317
});
318+
const databaseUDT = await getDatabaseUserDefinedTypes({ client, dbName, logger });
319+
const databaseMemoryOptimizedTables = await getDatabaseMemoryOptimizedTables({ client, dbName, logger });
320+
const xmlSchemaCollections = await getDatabaseXmlSchemaCollection({
321+
client,
322+
dbName,
323+
allUniqueSchemasAndTables,
324+
logger,
325+
});
319326

320327
const uniqueDatabaseIndexesColumns = getUniqueIndexesColumns({ indexesColumns: rawDatabaseIndexes });
321328
const databaseIndexes = addTotalBucketCountToDatabaseIndexes({

0 commit comments

Comments
 (0)