Merge PR #307: New approach to function call matching

Author: nene

src/formatter/AliasAs.ts

@@ -99,7 +99,8 @@ export default class AliasAs {
       isToken.CAST(this.getPreviousReservedToken()) &&
       isToken.AS(this.lookAhead()) &&
       (this.lookAhead(2).type === TokenType.IDENTIFIER ||
-        this.lookAhead(2).type === TokenType.RESERVED_KEYWORD) &&
+        this.lookAhead(2).type === TokenType.RESERVED_KEYWORD ||
+        this.lookAhead(2).type === TokenType.RESERVED_FUNCTION_NAME) &&
       this.lookAhead(3).value === ')'
     );
   }

src/formatter/ExpressionFormatter.ts

@@ -200,6 +200,7 @@ export default class ExpressionFormatter {
       case TokenType.RESERVED_LOGICAL_OPERATOR:
         return this.formatLogicalOperator(token);
       case TokenType.RESERVED_KEYWORD:
+      case TokenType.RESERVED_FUNCTION_NAME:
         return this.formatKeyword(token);
       case TokenType.RESERVED_CASE_START:
         return this.formatCaseStart(token);

src/languages/bigquery.formatter.ts

@@ -69,7 +69,7 @@ const reservedFunctions = {
   ],
   // https://cloud.google.com/bigquery/docs/reference/standard-sql/array_functions
   array: [
-    'ARRAY',
+    // 'ARRAY',
     'ARRAY_CONCAT',
     'ARRAY_LENGTH',
     'ARRAY_TO_STRING',
@@ -545,6 +545,9 @@ const reservedFunctions = {
   ],
   other: ['BQ.JOBS.CANCEL', 'BQ.REFRESH_MATERIALIZED_VIEW'],
   pivot: ['PIVOT', 'UNPIVOT'],
+  // Data types with parameters like VARCHAR(100)
+  // https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#parameterized_data_types
+  dataTypes: ['BYTES', 'NUMERIC', 'DECIMAL', 'BIGNUMERIC', 'BIGDECIMAL', 'STRING'],
 };
 
 /**
@@ -837,10 +840,8 @@ export default class BigQueryFormatter extends Formatter {
       reservedBinaryCommands,
       reservedJoins,
       reservedDependentClauses,
-      reservedKeywords: dedupe([
-        ...Object.values(reservedFunctions).flat(),
-        ...Object.values(reservedKeywords).flat(),
-      ]),
+      reservedKeywords: dedupe(Object.values(reservedKeywords).flat()),
+      reservedFunctionNames: dedupe(Object.values(reservedFunctions).flat()),
       openParens: ['(', '['],
       closeParens: [')', ']'],
       stringTypes: [
@@ -867,14 +868,14 @@ function postProcess(tokens: Token[]): Token[] {
   return detectArraySubscripts(combineParameterizedTypes(tokens));
 }
 
-// Converts OFFSET token inside array from RESERVED_COMMAND to RESERVED_KEYWORD
+// Converts OFFSET token inside array from RESERVED_COMMAND to RESERVED_FUNCTION_NAME
 // See: https://cloud.google.com/bigquery/docs/reference/standard-sql/functions-and-operators#array_subscript_operator
 function detectArraySubscripts(tokens: Token[]) {
   let prevToken = EOF_TOKEN;
   return tokens.map(token => {
     if (token.value === 'OFFSET' && prevToken.value === '[') {
       prevToken = token;
-      return { ...token, type: TokenType.RESERVED_KEYWORD };
+      return { ...token, type: TokenType.RESERVED_FUNCTION_NAME };
     } else {
       prevToken = token;
       return token;

src/languages/db2.formatter.ts

@@ -310,6 +310,8 @@ const reservedFunctions = {
   row: ['UNPACK'],
   // https://www.ibm.com/docs/en/db2-for-zos/12?topic=expressions-olap-specification
   olap: ['FIRST_VALUE', 'LAG', 'LAST_VALUE', 'LEAD', 'NTH_VALUE', 'NTILE', 'RATIO_TO_REPORT'],
+  // Type casting
+  cast: ['CAST'],
 };
 
 /**
@@ -342,7 +344,7 @@ const reservedKeywords = {
     'BY',
     'CAPTURE',
     'CASCADED',
-    'CAST',
+    // 'CAST',
     'CCSID',
     'CHARACTER',
     'CHECK',
@@ -862,10 +864,8 @@ export default class Db2Formatter extends Formatter {
       reservedBinaryCommands,
       reservedJoins,
       reservedDependentClauses,
-      reservedKeywords: dedupe([
-        ...Object.values(reservedFunctions).flat(),
-        ...Object.values(reservedKeywords).flat(),
-      ]),
+      reservedKeywords: dedupe(Object.values(reservedKeywords).flat()),
+      reservedFunctionNames: dedupe(Object.values(reservedFunctions).flat()),
       stringTypes: [{ quote: "''", prefixes: ['X', 'G', 'N', 'GX', 'UX', 'U&'] }],
       identTypes: [`""`],
       positionalParams: true,

src/languages/hive.formatter.ts

@@ -190,6 +190,24 @@ const reservedFunctions = {
     'VARIANCE',
   ],
   table: ['EXPLODE', 'INLINE', 'JSON_TUPLE', 'PARSE_URL_TUPLE', 'POSEXPLODE', 'STACK'],
+  // https://cwiki.apache.org/confluence/display/Hive/LanguageManual+WindowingAndAnalytics
+  window: [
+    'LEAD',
+    'LAG',
+    'FIRST_VALUE',
+    'LAST_VALUE',
+    'RANK',
+    'ROW_NUMBER',
+    'DENSE_RANK',
+    'CUME_DIST',
+    'PERCENT_RANK',
+    'NTILE',
+  ],
+  // Parameterized data types
+  // https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=82706456
+  // Though in reality Hive only supports parameters for DECIMAL(),
+  // it doesn't hurt to allow others in here as well.
+  dataTypes: ['DECIMAL', 'NUMERIC', 'VARCHAR', 'CHAR'],
 };
 
 /**
@@ -619,10 +637,8 @@ export default class HiveFormatter extends Formatter {
       reservedBinaryCommands,
       reservedJoins,
       reservedDependentClauses,
-      reservedKeywords: dedupe([
-        ...Object.values(reservedFunctions).flat(),
-        ...Object.values(reservedKeywords).flat(),
-      ]),
+      reservedKeywords: dedupe(Object.values(reservedKeywords).flat()),
+      reservedFunctionNames: dedupe(Object.values(reservedFunctions).flat()),
       openParens: ['(', '['],
       closeParens: [')', ']'],
       stringTypes: ['""', "''"],

src/languages/mariadb.formatter.ts

@@ -180,7 +180,6 @@ const reservedFunctions = [
   'NAME_CONST',
   'NVL',
   'NVL2',
-  'NULLIF',
   'OCT',
   'OCTET_LENGTH',
   'ORD',
@@ -244,6 +243,41 @@ const reservedFunctions = [
   'WSREP_LAST_SEEN_GTID',
   'WSREP_SYNC_WAIT_UPTO_GTID',
   'YEARWEEK',
+  // CASE expression shorthands
+  'COALESCE',
+  'NULLIF',
+  // Data types with parameters
+  // https://mariadb.com/kb/en/data-types/
+  'TINYINT',
+  'SMALLINT',
+  'MEDIUMINT',
+  'INT',
+  'INTEGER',
+  'BIGINT',
+  'DECIMAL',
+  'DEC',
+  'NUMERIC',
+  'FIXED',
+  // 'NUMBER', // ?? In oracle mode only
+  'FLOAT',
+  'DOUBLE',
+  'DOUBLE PRECISION',
+  'REAL',
+  'BIT',
+  'BINARY',
+  'BLOB',
+  'CHAR',
+  'NATIONAL CHAR',
+  'CHAR BYTE',
+  'ENUM',
+  'VARBINARY',
+  'VARCHAR',
+  'NATIONAL VARCHAR',
+  // 'SET' // handled as special-case in postProcess
+  'TIME',
+  'DATETIME',
+  'TIMESTAMP',
+  'YEAR',
 ];
 
 /**
@@ -1160,7 +1194,8 @@ export default class MariaDbFormatter extends Formatter {
       reservedJoins,
       reservedDependentClauses,
       reservedLogicalOperators: ['AND', 'OR', 'XOR'],
-      reservedKeywords: dedupe([...reservedKeywords, ...reservedFunctions]),
+      reservedKeywords: dedupe(reservedKeywords),
+      reservedFunctionNames: dedupe(reservedFunctions),
       stringTypes: ['""', { quote: "''", prefixes: ['X'] }],
       identTypes: ['``'],
       identChars: { first: '$', rest: '$' },
@@ -1183,7 +1218,7 @@ function postProcess(tokens: Token[]) {
     const nextToken = tokens[i + 1] || EOF_TOKEN;
     if (isToken.SET(token) && nextToken.value === '(') {
       // This is SET datatype, not SET statement
-      return { ...token, type: TokenType.RESERVED_KEYWORD };
+      return { ...token, type: TokenType.RESERVED_FUNCTION_NAME };
     }
     return token;
   });

src/languages/mysql.formatter.ts

@@ -115,7 +115,7 @@ const reservedFunctions = [
   'ICU_VERSION',
   'IF',
   'IFNULL',
-  'IN',
+  // 'IN',
   'INET_ATON',
   'INET_NTOA',
   'INET6_ATON',
@@ -432,6 +432,37 @@ const reservedFunctions = [
   // 'XOR',
   'YEAR',
   'YEARWEEK',
+  // Data types with parameters
+  // https://dev.mysql.com/doc/refman/8.0/en/data-types.html
+  'BIT',
+  'TINYINT',
+  'SMALLINT',
+  'MEDIUMINT',
+  'INT',
+  'INTEGER',
+  'BIGINT',
+  'DECIMAL',
+  'DEC',
+  'NUMERIC',
+  'FIXED',
+  'FLOAT',
+  'DOUBLE',
+  'DOUBLE PRECISION',
+  'REAL',
+  'DATETIME',
+  'TIMESTAMP',
+  'TIME',
+  'YEAR',
+  'CHAR',
+  'NATIONAL CHAR',
+  'VARCHAR',
+  'NATIONAL VARCHAR',
+  'BINARY',
+  'VARBINARY',
+  'BLOB',
+  'TEXT',
+  'ENUM',
+  // 'SET' // handled as special-case in postProcess
 ];
 
 /**
@@ -656,6 +687,7 @@ const reservedKeywords = [
   'INSERT_METHOD',
   'INSTALL',
   'INSTANCE',
+  'IN', // <-- moved over from functions
   'INT',
   'INT1',
   'INT2',
@@ -1325,7 +1357,8 @@ export default class MySqlFormatter extends Formatter {
       reservedJoins,
       reservedDependentClauses,
       reservedLogicalOperators: ['AND', 'OR', 'XOR'],
-      reservedKeywords: dedupe([...reservedKeywords, ...reservedFunctions]),
+      reservedKeywords: dedupe(reservedKeywords),
+      reservedFunctionNames: dedupe(reservedFunctions),
       stringTypes: ['""', { quote: "''", prefixes: ['X'] }],
       identTypes: ['``'],
       identChars: { first: '$', rest: '$' },
@@ -1348,7 +1381,7 @@ function postProcess(tokens: Token[]) {
     const nextToken = tokens[i + 1] || EOF_TOKEN;
     if (isToken.SET(token) && nextToken.value === '(') {
       // This is SET datatype, not SET statement
-      return { ...token, type: TokenType.RESERVED_KEYWORD };
+      return { ...token, type: TokenType.RESERVED_FUNCTION_NAME };
     }
     return token;
   });

src/languages/n1ql.formatter.ts

@@ -234,6 +234,10 @@ const reservedFunctions = [
   'VAR_SAMP',
   'WEEKDAY_MILLIS',
   'WEEKDAY_STR',
+  // type casting
+  // not implemented in N1QL, but added here now for the sake of tests
+  // https://docs.couchbase.com/server/current/analytics/3_query.html#Vs_SQL-92
+  'CAST',
 ];
 
 /**
@@ -518,7 +522,8 @@ export default class N1qlFormatter extends Formatter {
       reservedJoins,
       reservedDependentClauses,
       reservedLogicalOperators: ['AND', 'OR', 'XOR'],
-      reservedKeywords: dedupe([...reservedKeywords, ...reservedFunctions]),
+      reservedKeywords: dedupe(reservedKeywords),
+      reservedFunctionNames: dedupe(reservedFunctions),
       // NOTE: single quotes are actually not supported in N1QL,
       // but we support them anyway as all other SQL dialects do,
       // which simplifies writing tests that are shared between all dialects.

src/languages/plsql.formatter.ts

@@ -234,6 +234,36 @@ const reservedFunctions = {
   window: ['FIRST_VALUE', 'LAG', 'LAST_VALUE', 'LEAD', 'NTILE', 'RATIO_TO_REPORT', 'ROW_NUMBER'],
   objectReference: ['DEREF', 'MAKE_REF', 'REF', 'REFTOHEX', 'VALUE'],
   model: ['CV', 'ITERATION_NUMBER', 'PRESENTNNV', 'PRESENTV', 'PREVIOUS'],
+  // Parameterized data types
+  // https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html
+  dataTypes: [
+    // Oracle builtin data types
+    'VARCHAR2',
+    'NVARCHAR2',
+    'NUMBER',
+    'FLOAT',
+    'TIMESTAMP',
+    'INTERVAL YEAR',
+    'INTERVAL DAY',
+    'RAW',
+    'UROWID',
+    'NCHAR',
+    // ANSI Data Types
+    'CHARACTER',
+    'CHAR',
+    'CHARACTER VARYING',
+    'CHAR VARYING',
+    'NATIONAL CHARACTER',
+    'NATIONAL CHAR',
+    'NATIONAL CHARACTER VARYING',
+    'NATIONAL CHAR VARYING',
+    'NCHAR VARYING',
+    'NUMERIC',
+    'DECIMAL',
+    'FLOAT',
+    // SQL/DS and DB2 Data Types
+    'VARCHAR',
+  ],
 };
 
 /**
@@ -691,7 +721,8 @@ export default class PlSqlFormatter extends Formatter {
       reservedJoins,
       reservedDependentClauses,
       reservedLogicalOperators: ['AND', 'OR', 'XOR'],
-      reservedKeywords: dedupe([...reservedKeywords, ...Object.values(reservedFunctions).flat()]),
+      reservedKeywords: dedupe(reservedKeywords),
+      reservedFunctionNames: dedupe(Object.values(reservedFunctions).flat()),
       // TODO: support custom-delimited strings: Q'{..}' q'<..>' etc
       stringTypes: [{ quote: "''", prefixes: ['N'] }],
       identTypes: [`""`],