CASSANDRA-19985: Enhance CQLSH to support machine-readable output formatting (csv, json)

Author: arvindKandpal-ksolves

conf/cqlshrc.sample

@@ -37,6 +37,9 @@
 ; version = None
 
 [ui]
+;; The format of the output. Valid values are tabular, csv, and json.
+; mode = tabular
+
 ;; Whether or not to display query results with colors
 ; color = on
 

doc/modules/cassandra/pages/managing/tools/cqlsh.adoc

@@ -98,6 +98,8 @@ Options:
   Collect coverage data
 `--encoding=ENCODING`::
   Specify a non-default encoding for output. (Default: utf-8)
+`--mode=MODE`::
+  Specify the output display format. Valid values are `tabular` (default), `csv`, and `json`.
 `--cqlshrc=CQLSHRC`::
   Specify an alternative cqlshrc file location.
 `--credentials=CREDENTIALS`::

pylib/cqlshlib/cqlshmain.py

@@ -46,7 +46,8 @@
 from cqlshlib import cql3handling, pylexotron, sslhandling, cqlshhandling, authproviderhandling
 from cqlshlib.copyutil import ExportTask, ImportTask
 from cqlshlib.displaying import (ANSI_RESET, BLUE, COLUMN_NAME_COLORS, CYAN,
-                                 RED, WHITE, FormattedValue, colorme)
+                                 RED, WHITE, FormattedValue, colorme,
+                                 TabularTablePrinter, CsvTablePrinter, JsonTablePrinter)
 from cqlshlib.formatting import (DEFAULT_DATE_FORMAT, DEFAULT_NANOTIME_FORMAT,
                                  DEFAULT_TIMESTAMP_FORMAT, CqlType, DateTimeFormat,
                                  format_by_type)
@@ -284,13 +285,18 @@ def __init__(self, hostname, port, config_file, color=False,
                  connect_timeout=DEFAULT_CONNECT_TIMEOUT_SECONDS,
                  is_subshell=False,
                  auth_provider=None,
-                 disable_history=False):
+                 disable_history=False,
+                 mode='tabular'):
         cmd.Cmd.__init__(self, completekey=completekey)
         self.hostname = hostname
         self.port = port
         self.auth_provider = auth_provider
         self.username = username
         self.config_file = config_file
+        self.mode = mode
+
+        if self.mode in ('csv', 'json'):
+            self.color = False
 
         if isinstance(auth_provider, PlainTextAuthProvider):
             self.username = auth_provider.username
@@ -956,32 +962,37 @@ def perform_simple_statement(self, statement):
     def print_result(self, result, table_meta):
         self.decoding_errors = []
 
-        self.writeresult("")
+        if self.mode == 'csv':
+            printer = CsvTablePrinter(self)
+        elif self.mode == 'json':
+            printer = JsonTablePrinter(self)
+        else:
+            self.writeresult("")
+            printer = TabularTablePrinter(self, self.tty)
 
-        def print_all(result, table_meta, tty):
-            # Return the number of rows in total
+        def print_all(result, table_meta, tty, printer):
             num_rows = 0
             is_first = True
             while True:
-                # Always print for the first page even it is empty
                 if result.current_rows or is_first:
                     with_header = is_first or tty
-                    self.print_static_result(result, table_meta, with_header, tty, num_rows)
+                    self.print_static_result(result, table_meta, with_header, tty, num_rows, printer)
                     num_rows += len(result.current_rows)
                 if result.has_more_pages:
                     if self.shunted_query_out is None and tty:
-                        # Only pause when not capturing.
                         input("---MORE---")
                     result.fetch_next_page()
                 else:
-                    if not tty:
+                    if not tty and self.mode not in ('csv', 'json'):
                         self.writeresult("")
                     break
                 is_first = False
             return num_rows
 
-        num_rows = print_all(result, table_meta, self.tty)
-        self.writeresult("(%d rows)" % num_rows)
+        num_rows = print_all(result, table_meta, self.tty, printer)
+        printer.finish()
+        if self.mode not in ('csv', 'json'):
+            self.writeresult("(%d rows)" % num_rows)
 
         if self.decoding_errors:
             for err in self.decoding_errors[:2]:
@@ -990,15 +1001,18 @@ def print_all(result, table_meta, tty):
                 self.writeresult('%d more decoding errors suppressed.'
                                  % (len(self.decoding_errors) - 2), color=RED)
 
-    def print_static_result(self, result, table_meta, with_header, tty, row_count_offset=0):
+    def print_static_result(self, result, table_meta, with_header, tty, row_count_offset=0, printer=None):
         if not result.column_names and not table_meta:
             return
 
         column_names = result.column_names or list(table_meta.columns.keys())
         formatted_names = [self.myformat_colname(name, table_meta) for name in column_names]
+
         if not result.current_rows:
-            # print header only
-            self.print_formatted_result(formatted_names, None, with_header=True, tty=tty)
+            if printer is None or isinstance(printer, TabularTablePrinter):
+                self.print_formatted_result(formatted_names, None, with_header=True, tty=tty)
+            elif with_header:
+                printer.print_header(formatted_names)
             return
 
         cql_types = []
@@ -1009,10 +1023,15 @@ def print_static_result(self, result, table_meta, with_header, tty, row_count_of
 
         formatted_values = [list(map(self.myformat_value, [row[c] for c in column_names], cql_types)) for row in result.current_rows]
 
-        if self.expand_enabled:
-            self.print_formatted_result_vertically(formatted_names, formatted_values, row_count_offset)
+        if printer is None:
+            if self.expand_enabled:
+                self.print_formatted_result_vertically(formatted_names, formatted_values, row_count_offset)
+            else:
+                self.print_formatted_result(formatted_names, formatted_values, with_header, tty)
         else:
-            self.print_formatted_result(formatted_names, formatted_values, with_header, tty)
+            if with_header:
+                printer.print_header(formatted_names)
+            printer.print_rows(formatted_names, formatted_values)
 
     def print_formatted_result(self, formatted_names, formatted_values, with_header, tty):
         # determine column widths
@@ -2026,6 +2045,7 @@ def read_options(cmdlineargs, parser, config_file, cql_dir, environment=os.envir
     argvalues.completekey = option_with_default(configs.get, 'ui', 'completekey',
                                                 DEFAULT_COMPLETEKEY)
     argvalues.color = option_with_default(configs.getboolean, 'ui', 'color')
+    argvalues.mode = option_with_default(configs.get, 'ui', 'mode', 'tabular')
     argvalues.time_format = raw_option_with_default(configs, 'ui', 'time_format',
                                                     DEFAULT_TIMESTAMP_FORMAT)
     argvalues.nanotime_format = raw_option_with_default(configs, 'ui', 'nanotime_format',
@@ -2230,6 +2250,8 @@ def main(cmdline, pkgpath):
                         help='Force tty mode (command prompt).')
     parser.add_argument('--disable-history', default=False, action='store_true',
                         help='Disable saving of history (existing history will still be loaded)')
+    parser.add_argument('--mode', choices=['tabular', 'csv', 'json'],
+                        help='Specify the output format (tabular, csv, json). Default is tabular.')
 
     # This is a hidden option to suppress the warning when the -p/--password command line option is used.
     # Power users may use this option if they know no other people has access to the system where cqlsh is run or don't care about security.
@@ -2357,6 +2379,7 @@ def main(cmdline, pkgpath):
                       display_double_precision=options.double_precision,
                       display_timezone=timezone,
                       max_trace_wait=options.max_trace_wait,
+                      mode=options.mode,
                       ssl=options.ssl,
                       single_statement=options.execute,
                       request_timeout=options.request_timeout,

pylib/cqlshlib/displaying.py

@@ -126,3 +126,86 @@ def color_ljust(self, width, fill=' '):
                                  )
 
 NO_COLOR_MAP = dict()
+
+class TablePrinter:
+    def print_header(self, formatted_names):
+        raise NotImplementedError
+
+    def print_rows(self, formatted_names, formatted_values):
+        raise NotImplementedError
+
+    def finish(self):
+        pass
+
+class TabularTablePrinter(TablePrinter):
+    def __init__(self, shell, tty, row_count_offset=0):
+        self._shell = shell
+        self._tty = tty
+        self._row_count_offset = row_count_offset
+        self._with_header = True
+
+    def print_header(self, formatted_names):
+        pass
+
+    def print_rows(self, formatted_names, formatted_values, with_header=None):
+        if with_header is None:
+            with_header = self._with_header
+        if self._shell.expand_enabled:
+            self._shell.print_formatted_result_vertically(
+                formatted_names, formatted_values, self._row_count_offset)
+        else:
+            self._shell.print_formatted_result(
+                formatted_names, formatted_values, with_header, self._tty)
+        if formatted_values:
+            self._row_count_offset += len(formatted_values)
+        self._with_header = self._tty
+
+    def finish(self):
+        pass
+
+class CsvTablePrinter(TablePrinter):
+    def __init__(self, shell):
+        import csv
+        self._writer = csv.writer(shell.query_out)
+        self._header_written = False
+
+    def print_header(self, formatted_names):
+        self._colnames = [n.strval for n in formatted_names]
+
+    def print_rows(self, formatted_names, formatted_values):
+        if not self._header_written:
+            self._writer.writerow(self._colnames)
+            self._header_written = True
+        if formatted_values is None:
+            return
+        for row in formatted_values:
+            self._writer.writerow([col.strval for col in row])
+
+    def finish(self):
+        pass
+
+class JsonTablePrinter(TablePrinter):
+    def __init__(self, shell):
+        self._shell = shell
+        self._colnames = None
+        self._first_row = True
+
+    def print_header(self, formatted_names):
+        self._colnames = [n.strval for n in formatted_names]
+        self._shell.writeresult('[')
+
+    def print_rows(self, formatted_names, formatted_values):
+        import json
+        if formatted_values is None:
+            return
+        for row in formatted_values:
+            row_dict = {self._colnames[i]: col.strval for i, col in enumerate(row)}
+            serialized = json.dumps(row_dict)
+            if self._first_row:
+                self._shell.writeresult('  ' + serialized, newline=False)
+                self._first_row = False
+            else:
+                self._shell.writeresult(',\n  ' + serialized, newline=False)
+
+    def finish(self):
+        self._shell.writeresult('\n]')

pylib/cqlshlib/test/test_cqlsh_output.py

@@ -1017,3 +1017,84 @@ def test_quoted_output_text_in_udts(self):
                                         tty=False, input=query)
         self.assertEqual(0, result)
         self.assertEqual(output.splitlines()[3].strip(), "{data: 'I''m newb'}")
+
+    def test_csv_output(self):
+        ks = get_keyspace()
+        query = "SELECT a, b FROM twenty_rows_table WHERE a IN ('1', '2');"
+
+        output, result = cqlsh_testcall(args=('--mode', 'csv'), prompt=None, env=self.default_env,
+                                        tty=False, input=query + '\n')
+        self.assertEqual(0, result)
+
+        lines = output.strip().splitlines()
+        self.assertEqual(lines[0].strip(), 'a,b')
+        self.assertIn('1,1', [l.strip() for l in lines])
+        self.assertIn('2,2', [l.strip() for l in lines])
+
+        query2 = "SELECT num, setcol FROM has_all_types WHERE num = 0;"
+        output2, result2 = cqlsh_testcall(args=('--mode', 'csv'), prompt=None, env=self.default_env,
+                                          tty=False, input=query2 + '\n')
+        self.assertEqual(0, result2)
+        import csv, io
+        reader = csv.reader(io.StringIO(output2.strip()))
+        rows = list(reader)
+        self.assertEqual(rows[0], ['num', 'setcol'])
+        for row in rows[1:]:
+            self.assertEqual(len(row), 2,
+                             msg='CSV row has wrong field count (commas inside setcol not quoted?): %r' % row)
+
+        query3 = "SELECT num, varintcol FROM has_all_types WHERE num = 0;"
+        output3, result3 = cqlsh_testcall(args=('--mode', 'csv'), prompt=None, env=self.default_env,
+                                          tty=False, input=query3 + '\n')
+        self.assertEqual(0, result3)
+        reader3 = csv.reader(io.StringIO(output3.strip()))
+        rows3 = list(reader3)
+        varint_val = rows3[1][1]
+        self.assertNotIn(',', varint_val,
+                         msg='Large varint should not contain thousands separator in CSV: %r' % varint_val)
+
+    def test_json_output(self):
+        ks = get_keyspace()
+        query = "SELECT a, b FROM twenty_rows_table WHERE a IN ('1', '2');"
+
+        output, result = cqlsh_testcall(args=('--mode', 'json'), prompt=None, env=self.default_env,
+                                        tty=False, input=query + '\n')
+        self.assertEqual(0, result)
+
+        import json
+        try:
+            parsed_json = json.loads(output)
+            self.assertEqual(len(parsed_json), 2)
+
+            results = { (item['a'], item['b']) for item in parsed_json }
+            self.assertIn(('1', '1'), results)
+            self.assertIn(('2', '2'), results)
+        except ValueError as e:
+            self.fail("Output is not valid JSON: %s\nOutput was:\n%s" % (e, output))
+
+        query2 = "SELECT num, setcol, listcol, mapcol FROM has_all_types WHERE num = 0;"
+        output2, result2 = cqlsh_testcall(args=('--mode', 'json'), prompt=None, env=self.default_env,
+                                          tty=False, input=query2 + '\n')
+        self.assertEqual(0, result2)
+        try:
+            rows2 = json.loads(output2)
+            self.assertEqual(len(rows2), 1)
+            row = rows2[0]
+            self.assertIsInstance(row['setcol'], str,
+                                  msg='setcol should be a JSON string, got: %r' % type(row['setcol']))
+            self.assertIsInstance(row['listcol'], str,
+                                  msg='listcol should be a JSON string, got: %r' % type(row['listcol']))
+            self.assertIsInstance(row['mapcol'], str,
+                                  msg='mapcol should be a JSON string, got: %r' % type(row['mapcol']))
+        except ValueError as e:
+            self.fail("Output is not valid JSON: %s\nOutput was:\n%s" % (e, output2))
+
+        query3 = "SELECT num, varintcol FROM has_all_types WHERE num = 0;"
+        output3, result3 = cqlsh_testcall(args=('--mode', 'json'), prompt=None, env=self.default_env,
+                                          tty=False, input=query3 + '\n')
+        self.assertEqual(0, result3)
+        try:
+            rows3 = json.loads(output3)
+            self.assertEqual(rows3[0]['varintcol'], '10000000000000000000000000')
+        except ValueError as e:
+            self.fail("Output is not valid JSON: %s\nOutput was:\n%s" % (e, output3))