diff --git a/sdks/java/extensions/sql/jdbc/build.gradle b/sdks/java/extensions/sql/jdbc/build.gradle index 91c48b635d50..da5c9e58992f 100644 --- a/sdks/java/extensions/sql/jdbc/build.gradle +++ b/sdks/java/extensions/sql/jdbc/build.gradle @@ -37,7 +37,7 @@ dependencies { implementation project(":sdks:java:extensions:sql") implementation "jline:jline:2.14.6" permitUnusedDeclared "jline:jline:2.14.6" // BEAM-11761 - implementation "sqlline:sqlline:1.4.0" + implementation "sqlline:sqlline:1.12.0" implementation library.java.vendored_calcite_1_40_0 permitUnusedDeclared library.java.vendored_calcite_1_40_0 testImplementation project(path: ":sdks:java:core", configuration: "shadow") diff --git a/sdks/java/extensions/sql/jdbc/src/main/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLine.java b/sdks/java/extensions/sql/jdbc/src/main/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLine.java index 8c87343cd7c1..955c25c6e2f2 100644 --- a/sdks/java/extensions/sql/jdbc/src/main/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLine.java +++ b/sdks/java/extensions/sql/jdbc/src/main/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLine.java @@ -35,6 +35,8 @@ public class BeamSqlLine { private static final String NICKNAME = "BeamSQL"; + private static final String DEFAULT_USER = "beam"; + private static final String DEFAULT_PASSWORD = "beam"; public static void main(String[] args) throws IOException { runSqlLine(args, null, System.out, System.err); @@ -53,6 +55,17 @@ private static String[] checkConnectionArgs(String[] args) { argsList.add(CONNECT_STRING_PREFIX); } + // Add default credentials to prevent interactive prompts + if (!argsList.contains("-n")) { + argsList.add("-n"); + argsList.add(DEFAULT_USER); + } + + if (!argsList.contains("-p")) { + argsList.add("-p"); + argsList.add(DEFAULT_PASSWORD); + } + return argsList.toArray(new String[argsList.size()]); } diff --git a/sdks/java/extensions/sql/jdbc/src/main/resources/sqlline/SqlLine.properties b/sdks/java/extensions/sql/jdbc/src/main/resources/sqlline/SqlLine.properties index c1013f61eb0b..44e01f498e1b 100644 --- a/sdks/java/extensions/sql/jdbc/src/main/resources/sqlline/SqlLine.properties +++ b/sdks/java/extensions/sql/jdbc/src/main/resources/sqlline/SqlLine.properties @@ -19,106 +19,11 @@ # TODO: Only store the modifications here, and wrap the ResourceBundle ########################################################################## -# Printed when BeamSqlLine starts up -app-introduction: Welcome to Beam SQL @beam.version@ (based on sqlline version {1}) -# Not generally applicable to Beam, but still available for power users or vendors: -# -# -u the JDBC URL to connect to\n \ -# -n the username to connect as\n \ -# -p the password to connect as\n \ -# -d the driver class to use\n \ -# -nn nickname for the connection\n \ -# --autoCommit=[true/false] enable/disable automatic transaction commit\n \ -# --isolation=LEVEL set the transaction isolation level\n \ -cmd-usage: Usage: java -jar beam-sdks-java-extensions-sql-jdbc-@beam.version@.jar \n \ -\ Usage: java org.apache.beam.sdk.java.extensions.sql.jdbc.BeamSqlLine \n \ -\ -f script file to execute (same as --run)\n \ -\ --color=[true/false] control whether color is used for display\n \ -\ --showHeader=[true/false] show column names in query results\n \ -\ --headerInterval=ROWS the interval between which headers are displayed\n \ -\ --fastConnect=[true/false] skip building table/column list for tab-completion\n \ -\ --verbose=[true/false] show verbose error messages and debug info\n \ -\ --showTime=[true/false] display execution time when verbose\n \ -\ --showWarnings=[true/false] display connection warnings\n \ -\ --showNestedErrs=[true/false] display nested errors\n \ -\ --numberFormat=[pattern] format numbers using DecimalFormat pattern\n \ -\ --force=[true/false] continue running script even after errors\n \ -\ --maxWidth=MAXWIDTH the maximum width of the terminal\n \ -\ --maxColumnWidth=MAXCOLWIDTH the maximum width to use when displaying columns\n \ -\ --silent=[true/false] be more silent\n \ -\ --autosave=[true/false] automatically save preferences\n \ -\ --outputformat=[table/vertical/csv/tsv] format mode for result display\n \ -\ --run=/path/to/file run one script and then exit -\ --help display this message - -# Similar to command line, hiding help for irrelevant things, -# some of which already do not work (like !set propertiesFile) -# -# \nautoCommit true/false Enable/disable automatic\ -# \n transaction commit\ -# \nisolation LEVEL Set transaction isolation level\ -# \npropertiesFile path File from which SqlLine reads\ -# properties on startup; default is\ -# $HOME/.sqlline/sqlline.properties\ -# (UNIX, Linux, Mac OS),\ -# $HOME/sqlline/sqlline.properties\ -# (Windows)\ -help-set: Set a sqlline variable\ -\n\ -\nVariable Value Description\ -\n=============== ========== ================================\ -\nautoSave true/false Automatically save preferences\ -\ncolor true/false Control whether color is used\ -\n for display\ -\nfastConnect true/false Skip building table/column list\ -\n for tab-completion\ -\nforce true/false Continue running script even\ -\n after errors\ -\nheaderInterval integer The interval between which\ -\n headers are displayed\ -\nhistoryFile path File in which to save command\ -\n history. Default is\ -\n $HOME/.sqlline/history (UNIX,\ -\n Linux, Mac OS),\ -\n $HOME/sqlline/history (Windows)\ -\nincremental true/false Do not receive all rows from\ -\n server before printing the first\ -\n row. Uses fewer resources,\ -\n especially for long-running\ -\n queries, but column widths may\ -\n be incorrect.\ -\nmaxColumnWidth integer The maximum width to use when\ -\n displaying columns\ -\nmaxHeight integer The maximum height of the\ -\n terminal\ -\nmaxWidth integer The maximum width of the\ -\n terminal\ -\nnumberFormat pattern Format numbers using\ -\n DecimalFormat pattern\ -\noutputFormat table/vertical/csv/tsv Format mode for\ -\n result display\ -\nrowLimit integer Maximum number of rows returned\ -\n from a query; zero means no\ -\n limit\ -\nshowElapsedTime true/false Display execution time when\ -\n verbose\ -\nshowHeader true/false Show column names in query\ -\n results\ -\nshowNestedErrs true/false Display nested errors\ -\nshowWarnings true/false Display connection warnings\ -\nsilent true/false Be more silent\ -\ntimeout integer Query timeout in seconds; less\ -\n than zero means no timeout\ -\ntrimScripts true/false Remove trailing spaces from\ -\n lines read from script files\ -\nverbose true/false Show verbose error messages and\ -\n debug info - -#### UNMODIFIED PROPERTIES BELOW HERE #### jline-version: The version of the required {0} library is too old. Version \ "{1}" was found, but "{2}" is required. +rerun-max-offset: Current maximum offset is {0} enter-for-more: [ Hit "enter" for more ("q" to exit) ] no-manual: Could not find manual resource. executing-command: Executing command: {0} @@ -130,8 +35,32 @@ reconnecting: Reconnecting to "{0}"... connecting: Connecting to "{0}"... no-driver: No known driver to handle "{0}" setting-prop: Setting property: {0} +unknown-prop: Unknown property: {0} +wrong-prop-type: Property: {0} should be of type {1} saving-options: Saving preferences to: {0} +saving-to-dev-null-not-supported: Saving to /dev/null not supported loaded-options: Loaded preferences from: {0} +no-specified-driver: Could not find driver {0} +no-specified-driver-use-existing: Could not find driver {0}; using registered driver {1} instead +no-specified-prop: Specified property [{0}] does not exist. \ + Use !set command to get list of all available properties. +no-url: Property "url" is required +no-prop-not-supported: "--no flag for property {0} not supported" +reset-all-props: All properties were reset to their defaults. +reset-prop: [{0}] was reset to [{1}] +not-a-number: Value for property {0} should be a number. Specified value is: {1}. +command-name: Command name +connections: Connections +keyword: Keyword +function: Function +schema: Schema +table: Table +column: Column +new-size-after-resize: New size: height = {0}, width = {1} +empty-value-not-supported: Empty value for type {0} not supported +no-file: File {0} does not exist or is a directory +not-supported-script-engine: Not found script engine "{0}", available values: {1} +not-supported-script-engine-no-available: Not found script engine "{0}", no available script engines jdbc-level: JDBC level compliant: Compliant @@ -140,7 +69,9 @@ driver-class: Driver Class help-quit: Exits the program help-dropall: Drop all tables in the current database -help-connect: Open a new connection to the database. +help-showconfconnections: Show file content with configured connections +help-connect: Open a new connection to the database +help-commandhandler: Add a command handler help-manual: Display the SQLLine manual help-typeinfo: Display the type map for the current connection help-describe: Describe a table @@ -148,6 +79,8 @@ help-reconnect: Reconnect to the database help-metadata: Obtain metadata information help-dbinfo: Give metadata information about the database help-rehash: Fetch table and column names for command completion +help-resize: Reset max height/width based on terminal height/width +help-rereadconfconnections: Reset connections from config help-verbose: Set verbose mode on help-run: Run a script from the specified file help-list: List the current connections @@ -159,19 +92,157 @@ help-close: Close the current connection to the database help-closeall: Close all current open connections help-isolation: Set the transaction isolation for this connection help-nativesql: Show the native SQL for the specified statement +help-prompthandler: Set custom prompt handler class name help-call: Execute a callable statement help-autocommit: Set autocommit mode on or off +help-readonly: Set readonly mode on or off help-commit: Commit the current transaction (if autocommit is off) help-rollback: Roll back the current transaction (if autocommit is off) help-batch: Start or execute a batch of statements help-help: Print a summary of command usage -help-save: Save the current variabes and aliases +help-appconfig: Set custom application configuration class name +help-set: List / set a sqlline variable +help-rerun: Execute previous command from the history file +help-reset: Reset a sqlline variable +help-confirm: Require user to answer 'Are you sure?' before executing 'dangerous' commands (as defined by confirmPattern) +help-confirmPattern: Defines 'dangerous' commands that require user to answer 'Are you sure?' before proceeding (by default, DELETE and DROP) +help-connectInteractionMode: Defines interaction mode for !connect command +help-schemas: List all the schemas in the database +variables:\ +\n\ +\nVariables:\ +\n\ +\nVariable Value Description\ +\n=============== ========== ==================================================\ +\nautoCommit true/false Enable/disable automatic transaction commit\ +\nautoPairing true/false Enable/disable widget that auto-closes, deletes \ +\n and skips over matching delimiters\ +\nautoResize true/false Enable/disable automatic resizing of\ +\n max height/width based on terminal size\ +\nautoSave true/false Automatically save preferences\ +\ncolor true/false Control whether color is used for display\ +\ncolorScheme chester/dark/dracula/geshi/light/obsidian/solarized/vs2010\ +\n Syntax highlight schema\ +\nconfirm true/false Whether to prompt for confirmation before running\ +\n commands specified in confirmPattern (default:\ +\n false)\ +\nconfirmPattern pattern A regexp that defines the 'dangerous' commands for\ +\n which to prompt 'Are you sure?' before execution;\ +\n (default: DELETE and DROP)\ +\ncsvDelimiter String Delimiter in csv outputFormat\ +\nconnectInteractionMode askCredentials/notAskCredentials/useNPTogetherOrEmpty\ +\n Defines interaction mode for !connect command\ +\nconnectionConfig filePath Path to file with saved connection settings.\ +\ncsvQuoteCharacter char Quote character in csv outputFormat\ +\ndateFormat pattern Format dates using SimpleDateFormat pattern\ +\nescapeOutput true/false Escape control symbols in output\ +\nfastConnect true/false Skip building table/column list for tab-completion\ +\nforce true/false Continue running script even after errors\ +\nheaderInterval integer The interval between which headers are displayed\ +\nhistoryFile path File in which to save command history. Default is\ +\n $HOME/.sqlline/history (UNIX, Linux, Mac OS),\ +\n $HOME/sqlline/history (Windows)\ +\nhistoryFlags String Default flags for !history command\ +\nincremental true/false Do not receive all rows from server before\ +\n printing the first row; uses fewer resources,\ +\n especially for long-running queries, but column\ +\n widths may be incorrect\ +\nincrementalBufferRows integer Threshold at which to switch to incremental\ +\n mode; query starts in non-incremental mode, but\ +\n after the this many rows, switches to incremental\ +\nisolation LEVEL Set transaction isolation level\ +\nliveTemplates path File with live templates\ +\nkeepSemicolon true/false Keep semicolon in queries\ +\nmaxColumnWidth integer The maximum width to use when displaying columns\ +\nmaxHeight integer The maximum height of the terminal\ +\nmaxWidth integer The maximum width of the terminal\ +\nmaxHistoryFileRows integer The maximum number of history rows \ +\n to store in history file\ +\nmaxHistoryRows integer The maximum number of history rows \ +\n to store in memory\ +\nmode emacs/vi The editing mode\ +\nnullValue String Use String in place of NULL values\ +\nnumberFormat pattern Format numbers using DecimalFormat pattern\ +\noutputFormat table/vertical/csv/tsv/xmlattrs/xmlelements/json/ansiconsole\ +\n Format mode for result display\ +\nprompt pattern Format prompt\ +\npromptScript String Script code to execute to generate a prompt\ +\npropertiesFile path File from which SQLLine reads properties on\ +\n startup; default is\ +\n $HOME/.sqlline/sqlline.properties (UNIX, Linux,\ +\n macOS), $HOME/sqlline/sqlline.properties (Windows)\ +\nreadOnly true/false Enable/disable readonly connection\ +\nrightPrompt pattern Format right prompt\ +\nrowLimit integer Maximum number of rows returned from a query; zero\ +\n means no limit\ +\nscriptEngine String Script engine name\ +\nshowCompletionDesc true/false Display help for completions\ +\nshowElapsedTime true/false Display execution time when verbose\ +\nshowHeader true/false Show column names in query results\ +\nshowLineNumbers true/false Show line numbers while multiline queries\ +\nshowNestedErrs true/false Display nested errors\ +\nshowTypes true/false Display column types\ +\nshowWarnings true/false Display connection warnings\ +\nsilent true/false Be more silent\ +\nstrictJdbc true/false Use strict JDBC\ +\ntableStyle [default/solid/double_solid/round_corners/bold_header]\ +\n Table output style\ +\ntimeFormat pattern Format times using SimpleDateFormat pattern\ +\ntimeout integer Query timeout in seconds; less than zero means no\ +\n timeout\ +\ntimestampFormat pattern Format timestamps using SimpleDateFormat pattern\ +\ntrimScripts true/false Remove trailing spaces from lines read from script\ +\n files\ +\nuseLineContinuation true/false Use line continuation\ +\nverbose true/false Show verbose error messages and debug info\ +\nversion version Show the current sqlline version.\ +\n The property is read only.\ +\n\ +\nKey-strokes:\ +\nalt-b Backward word\ +\nalt-c Capitalize word\ +\nalt-d Kill word\ +\nalt-f Forward word\ +\nalt-h Next color scheme\ +\nalt-l Lowercase word\ +\nalt-n History search forward\ +\nalt-p History search backward\ +\nalt-t Transpose words\ +\nalt-u Uppercase word\ +\n\ +\nctrl-a To the beginning of line\ +\nctrl-b Backward char\ +\nctrl-d Delete char\ +\nctrl-e To the end of line\ +\nctrl-f Forward char\ +\nctrl-h Backward delete char\ +\nctrl-i Complete\ +\nctrl-j Enter\ +\nctrl-k Kill the line\ +\nctrl-m Enter\ +\nctrl-n Down line or history\ +\nctrl-p Up line or history\ +\nctrl-r History incremental search backward\ +\nctrl-s History incremental search forward\ +\nctrl-t Transpose chars\ +\nctrl-u Kill the whole line\ +\nctrl-w Backward kill the line\ +\n\ +\nalt-ctrl-n Show line numbers +\nalt-ctrl-p Enable/Disable pairing +help-save: Save the current variables and aliases help-native: Show the database''s native SQL for a command help-alias: Create a new command alias help-unalias: Unset a command alias help-scan: Scan for installed JDBC drivers help-sql: Execute a SQL command -help-history: Display the command history +help-history: Display the command history\n\ +-d Print timestamps for each event (default for !history) \n\ +-f Print full time date stamps in the US format \n\ +-E Print full time date stamps in the European format \n\ +-i Print full time date stamps in ISO8601 format \n\ +-n Suppresses command numbers \n\ +-r Reverses the order of the commands help-record: Record all output to the specified file help-indexes: List all the indexes for the specified table help-primarykeys: List all the primary keys for the specified table @@ -181,7 +252,7 @@ help-procedures: List all the procedures help-tables: List all the tables in the database help-columns: List all the columns for the specified table help-properties: Connect to the database specified in the properties file(s) -help-outputformat: Set the output format for displaying results (table,vertical,csv,tsv,xmlattrs,xmlelements) +help-outputformat: Set the output format for displaying results (table, vertical, csv, tsv, xmlattrs, xmlelements, json) help-nickname: Create a friendly name for the connection (updates command prompt) jline-missing: SQLLine static class check reports the {0} class was not found. Please ensure JLine is on classpath. @@ -198,14 +269,19 @@ possible-methods: Possible methods: closing: Closing: {0} already-closed: Connection is already closed. error-setting: Error setting configuration: {0}: {1} +property-readonly: {0} property is read only no-method: No method matching "{0}" was found in {1}. +method-requires-arguments: Method "{0}" requires arguments "{1}" connected: Connected to: {0} (version {1}) driver: Driver: {0} (version {1}) autocommit-status: Autocommit status: {0} isolation-status: Transaction isolation: {0} -unknown-format: Unknown output format "{0}". Possible values: {1} +isolation-level-not-supported: +unknown-value: Unknown {0} "{1}". Possible values: {2} + +readonly-status: Readonly status: {0} closed: closed open: open @@ -218,7 +294,7 @@ done: Done state: state code: code -invalid-connections: Invalid connection: {0} +invalid-connection: Invalid connection: {0} script-closed: Script closed. Enter "run {0}" to replay it. script-already-running: Script ({0}) is already running. Enter "script" with no arguments to stop it. @@ -245,12 +321,86 @@ abort-on-error: Aborting command set because "force" is false and \ multiple-matches: Ambiguous command: {0} -really-drop-all: Really drop every table in the database? (y/n)\ +really-drop-all: Really drop every table in the database? (y/n) abort-drop-all: Aborting drop all tables. +really-perform-action: Really perform the action on this table? (y/n) +abort-action: Aborting the action. + +default-confirm-pattern: ^(?i:(DROP|DELETE)) + drivers-found-count: 0#No driver classes found|1#{0} driver class found|1<{0} driver classes found rows-selected: 0#No rows selected|1#{0} row selected|1<{0} rows selected rows-affected: 0#No rows affected|1#{0} row affected|1<{0} rows affected|0>Unknown rows affected active-connections: 0#No active connections|1#{0} active connection:|1<{0} active connections: +script-executed: Script executed time-ms: ({0,number,#.###} seconds) + +cmd-usage: Usage: java -jar beam-sdks-java-extensions-sql-jdbc-@beam.version@.jar \n \ +\ -u the JDBC URL to connect to\n \ +\ -n the username to connect as\n \ +\ -p the password to connect as\n \ +\ -d the driver class to use\n \ +\ -e the command to execute\n \ +\ -nn nickname for the connection\n \ +\ -ch [,]* a custom command handler to use\n \ +\ -f script file to execute (same as --run)\n \ +\ -log file to write output\n \ +\ -ac application configuration class name\n \ +\ -ph prompt handler class name\n \ +\ --color=[true/false] control whether color is used for display\n \ +\ --colorScheme=[chester/dark/dracula/geshi/light/obsidian/solarized/vs2010]\ +\ syntax highlight schema\n \ +\ --confirm=[true/false] confirm before executing commands specified in confirmPattern\n \ +\ --confirmPattern=[pattern] pattern of commands to prompt confirmation\n \ +\ --connectInteractionMode=[askCredentials/notAskCredentials/useNPTogetherOrEmpty]\n \ +\ interaction mode for !connect command\n \ +\ --csvDelimiter=[delimiter] delimiter in csv outputFormat\n \ +\ --csvQuoteCharacter=[char] quote character in csv outputFormat\n \ +\ --escapeOutput=[true/false] escape control symbols in output\n \ +\ --showHeader=[true/false] show column names in query results\n \ +\ --headerInterval=ROWS the interval between which headers are displayed\n \ +\ --fastConnect=[true/false] skip building table/column list for tab-completion\n \ +\ --autoCommit=[true/false] enable/disable automatic transaction commit\n \ +\ --readOnly=[true/false] enable/disable readonly connection\n \ +\ --verbose=[true/false] show verbose error messages and debug info\n \ +\ --scriptEngine=[string] script engine name\n \ +\ --showCompletionDesc=[true/false] display help for completions\n \ +\ --showLineNumbers=[true/false] show line numbers while multiline queries\n \ +\ --showTime=[true/false] display execution time when verbose\n \ +\ --showWarnings=[true/false] display connection warnings\n \ +\ --showNestedErrs=[true/false] display nested errors\n \ +\ --showTypes=[true/false] display column types\n \ +\ --strictJdbc=[true/false] use strict JDBC\n \ +\ --nullValue=[string] use string in place of NULL values\n \ +\ --numberFormat=[pattern] format numbers using DecimalFormat pattern\n \ +\ --dateFormat=[pattern] format dates using SimpleDateFormat pattern\n \ +\ --timeFormat=[pattern] format times using SimpleDateFormat pattern\n \ +\ --timestampFormat=[pattern] format timestamps using SimpleDateFormat pattern\n \ +\ --force=[true/false] continue running script even after errors\n \ +\ --maxWidth=MAXWIDTH the maximum width of the terminal\n \ +\ --maxColumnWidth=MAXCOLWIDTH the maximum width to use when displaying columns\n \ +\ --autoResize=[true/false] enable/disable automatic resizing of\n \ +\ max height/width based on terminal size\n \ +\ --maxHistoryFileRows=ROWS the maximum number of history rows to store in history file\n \ +\ --maxHistoryRows=ROWS the maximum number of history rows to store in memory\n \ +\ --historyFlags=FLAGS default flags for !history command\n \ +\ --mode=[emacs/vi] the editing mode\n \ +\ --silent=[true/false] be more silent\n \ +\ --autosave=[true/false] automatically save preferences\n \ +\ --outputformat=[table/vertical/csv/tsv/xmlattrs/xmlelements/json/ansiconsole]\n \ +\ format mode for result display\n \ +\ --isolation=LEVEL set the transaction isolation level\n \ +\ --run=/path/to/file run one script and then exit\n \ +\ --historyfile=/path/to/file use or create history file in specified path\n \ +\ --useLineContinuation=[true/false] Use line continuation\n \ +\ --incremental=[true/false] display result rows immediately as they are\n \ +\ fetched, yielding lower latency and memory\n \ +\ usage at the price of extra display column padding\n \ +\ --incrementalBufferRows integer threshold at which to switch to incremental mode\n \ +\ --tableStyle=[default/solid/double_solid/round_corners/bold_header]\n \ +\ table output style\ +\ --keepSemicolon=[true/false] keep semicolon in queries\n \ +\ --liveTemplates=/path/to/file file with live templates\n \ +\ --help display this message diff --git a/sdks/java/extensions/sql/jdbc/src/test/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLineTest.java b/sdks/java/extensions/sql/jdbc/src/test/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLineTest.java index 367336e1e7a0..00fdb8bb7414 100644 --- a/sdks/java/extensions/sql/jdbc/src/test/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLineTest.java +++ b/sdks/java/extensions/sql/jdbc/src/test/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLineTest.java @@ -94,8 +94,7 @@ public void testSqlLine_select() throws Exception { List> lines = toLines(byteArrayOutputStream); assertThat( - Arrays.asList(Arrays.asList("3", "hello", "2018-05-28")), - everyItem(is(oneOf(lines.toArray())))); + lines, everyItem(is(oneOf(Arrays.asList(Arrays.asList("3", "hello", "2018-05-28")))))); } @Test @@ -112,10 +111,13 @@ public void testSqlLine_selectFromTable() throws Exception { List> lines = toLines(byteArrayOutputStream); assertThat( - Arrays.asList( - Arrays.asList("col_a", "col_b", "col_c", "col_x", "col_y", "col_z"), - Arrays.asList("a", "b", "c", "1", "2", "3")), - everyItem(is(oneOf(lines.toArray())))); + lines, + everyItem( + is( + oneOf( + Arrays.asList( + Arrays.asList("col_a", "col_b", "col_c", "col_x", "col_y", "col_z"), + Arrays.asList("a", "b", "c", "1", "2", "3")))))); } @Test @@ -130,7 +132,7 @@ public void testSqlLine_insertSelect() throws Exception { BeamSqlLine.runSqlLine(args, null, byteArrayOutputStream, null); List> lines = toLines(byteArrayOutputStream); - assertThat(Arrays.asList(Arrays.asList("3", "hello")), everyItem(is(oneOf(lines.toArray())))); + assertThat(lines, everyItem(is(oneOf(Arrays.asList(Arrays.asList("3", "hello")))))); } @Test @@ -147,9 +149,10 @@ public void testSqlLine_GroupBy() throws Exception { BeamSqlLine.runSqlLine(args, null, byteArrayOutputStream, null); List> lines = toLines(byteArrayOutputStream); + // Verify that the GROUP BY query returns the expected results assertThat( - Arrays.asList(Arrays.asList("3", "2"), Arrays.asList("4", "1")), - everyItem(is(oneOf(lines.toArray())))); + lines, + everyItem(is(oneOf(Arrays.asList(Arrays.asList("3", "2"), Arrays.asList("4", "1")))))); } @Test @@ -167,10 +170,13 @@ public void testSqlLine_fixedWindow() throws Exception { List> lines = toLines(byteArrayOutputStream); assertThat( - Arrays.asList( - Arrays.asList("2018-07-01 21:26:06.000000", "1"), - Arrays.asList("2018-07-01 21:26:07.000000", "1")), - everyItem(is(oneOf(lines.toArray())))); + lines, + everyItem( + is( + oneOf( + Arrays.asList( + Arrays.asList("2018-07-01 21:26:06.000000", "1"), + Arrays.asList("2018-07-01 21:26:07.000000", "1")))))); } @Test @@ -190,12 +196,15 @@ public void testSqlLine_slidingWindow() throws Exception { List> lines = toLines(byteArrayOutputStream); assertThat( - Arrays.asList( - Arrays.asList("2018-07-01 21:26:07.000000", "1"), - Arrays.asList("2018-07-01 21:26:08.000000", "2"), - Arrays.asList("2018-07-01 21:26:09.000000", "2"), - Arrays.asList("2018-07-01 21:26:10.000000", "2"), - Arrays.asList("2018-07-01 21:26:11.000000", "1")), - everyItem(is(oneOf(lines.toArray())))); + lines, + everyItem( + is( + oneOf( + Arrays.asList( + Arrays.asList("2018-07-01 21:26:07.000000", "1"), + Arrays.asList("2018-07-01 21:26:08.000000", "2"), + Arrays.asList("2018-07-01 21:26:09.000000", "2"), + Arrays.asList("2018-07-01 21:26:10.000000", "2"), + Arrays.asList("2018-07-01 21:26:11.000000", "1")))))); } } diff --git a/sdks/java/extensions/sql/jdbc/src/test/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLineTestingUtils.java b/sdks/java/extensions/sql/jdbc/src/test/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLineTestingUtils.java index 96452deef528..051cff743684 100644 --- a/sdks/java/extensions/sql/jdbc/src/test/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLineTestingUtils.java +++ b/sdks/java/extensions/sql/jdbc/src/test/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLineTestingUtils.java @@ -49,13 +49,38 @@ public static List> toLines(ByteArrayOutputStream outputStream) { } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } - return outputLines.stream().map(BeamSqlLineTestingUtils::splitFields).collect(toList()); + return outputLines.stream() + .map(BeamSqlLineTestingUtils::parseSqllineOutput) + .filter(line -> !line.isEmpty()) + .collect(toList()); } - private static List splitFields(String outputLine) { - return Arrays.stream(outputLine.split("\\|")) - .map(field -> field.trim()) - .filter(field -> field.length() != 0) - .collect(toList()); + private static List parseSqllineOutput(String outputLine) { + // Handle sqlline 1.12 table format with borders like +--+, | |, etc. + String trimmed = outputLine.trim(); + + // Skip table borders and empty lines + if (trimmed.isEmpty() || trimmed.matches("^[+\\-|\\s]+$")) { + return Arrays.asList(); + } + + // Parse data rows that contain actual values + if (trimmed.startsWith("|") && trimmed.endsWith("|")) { + // Remove the outer | characters and split by |, preserving trailing empty fields + String content = trimmed.substring(1, trimmed.length() - 1); + return Arrays.stream(content.split("\\|", -1)) // -1 preserves trailing empty strings + .map(field -> field.trim()) + .collect(toList()); // Don't filter empty fields - they represent NULL values + } + + // For non-table format, try the old parsing method + if (trimmed.contains("|")) { + return Arrays.stream(trimmed.split("\\|", -1)) // -1 preserves trailing empty strings + .map(field -> field.trim()) + .collect(toList()); // Don't filter empty fields - they represent NULL values + } + + // Single value or non-table format (trimmed is not empty at this point) + return Arrays.asList(trimmed); } }