Skip to content

Commit 03bff7c

Browse files
patrickerijokarumawak
patricker
authored and
ijokarumawak
committed
NIFI-2829 Date and Time Format Support for PutSQL
This closes #1524. Signed-off-by: Koji Kawamura <ijokarumawak@apache.org>
1 parent 3da8b94 commit 03bff7c

2 files changed

Lines changed: 208 additions & 7 deletions

File tree

  • nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src

‎nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSQL.java‎

Lines changed: 52 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,9 @@
7272
import java.text.ParseException;
7373
import java.text.SimpleDateFormat;
7474
import java.time.Instant;
75+
import java.time.LocalDate;
76+
import java.time.LocalTime;
77+
import java.time.ZoneId;
7578
import java.time.format.DateTimeFormatter;
7679
import java.time.temporal.TemporalAccessor;
7780
import java.util.ArrayList;
@@ -110,11 +113,13 @@
110113
+ "sql.args.1.value, sql.args.2.value, sql.args.3.value, and so on. The type of the sql.args.1.value Parameter is specified by the sql.args.1.type attribute."),
111114
@ReadsAttribute(attribute = "sql.args.N.format", description = "This attribute is always optional, but default options may not always work for your data. "
112115
+ "Incoming FlowFiles are expected to be parametrized SQL statements. In some cases "
113-
+ "a format option needs to be specified, currently this is only applicable for binary data types and timestamps. For binary data types "
114-
+ "available options are 'ascii', 'base64' and 'hex'. In 'ascii' format each string character in your attribute value represents a single byte, this is the default format "
115-
+ "and the format provided by Avro Processors. In 'base64' format your string is a Base64 encoded string. In 'hex' format the string is hex encoded with all "
116-
+ "letters in upper case and no '0x' at the beginning. For timestamps, the format can be specified according to java.time.format.DateTimeFormatter."
117-
+ "Customer and named patterns are accepted i.e. ('yyyy-MM-dd','ISO_OFFSET_DATE_TIME')")
116+
+ "a format option needs to be specified, currently this is only applicable for binary data types, dates, times and timestamps. Binary Data Types (defaults to 'ascii') - "
117+
+ "ascii: each string character in your attribute value represents a single byte. This is the format provided by Avro Processors. "
118+
+ "base64: the string is a Base64 encoded string that can be decoded to bytes. "
119+
+ "hex: the string is hex encoded with all letters in upper case and no '0x' at the beginning. "
120+
+ "Dates/Times/Timestamps - "
121+
+ "Date, Time and Timestamp formats all support both custom formats or named format ('yyyy-MM-dd','ISO_OFFSET_DATE_TIME') "
122+
+ "as specified according to java.time.format.DateTimeFormatter.")
118123
})
119124
@WritesAttributes({
120125
@WritesAttribute(attribute = "sql.generated.key", description = "If the database generated a key for an INSERT statement and the Obtain Generated Keys property is set to true, "
@@ -828,10 +833,50 @@ private void setParameter(final PreparedStatement stmt, final String attrName, f
828833
stmt.setBigDecimal(parameterIndex, new BigDecimal(parameterValue));
829834
break;
830835
case Types.DATE:
831-
stmt.setDate(parameterIndex, new Date(Long.parseLong(parameterValue)));
836+
Date date;
837+
838+
if (valueFormat.equals("")) {
839+
if(LONG_PATTERN.matcher(parameterValue).matches()){
840+
date = new Date(Long.parseLong(parameterValue));
841+
}else {
842+
String dateFormatString = "yyyy-MM-dd";
843+
if (!valueFormat.isEmpty()) {
844+
dateFormatString = valueFormat;
845+
}
846+
SimpleDateFormat dateFormat = new SimpleDateFormat(dateFormatString);
847+
java.util.Date parsedDate = dateFormat.parse(parameterValue);
848+
date = new Date(parsedDate.getTime());
849+
}
850+
} else {
851+
final DateTimeFormatter dtFormatter = getDateTimeFormatter(valueFormat);
852+
LocalDate parsedDate = LocalDate.parse(parameterValue, dtFormatter);
853+
date = new Date(Date.from(parsedDate.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant()).getTime());
854+
}
855+
856+
stmt.setDate(parameterIndex, date);
832857
break;
833858
case Types.TIME:
834-
stmt.setTime(parameterIndex, new Time(Long.parseLong(parameterValue)));
859+
Time time;
860+
861+
if (valueFormat.equals("")) {
862+
if (LONG_PATTERN.matcher(parameterValue).matches()) {
863+
time = new Time(Long.parseLong(parameterValue));
864+
} else {
865+
String timeFormatString = "HH:mm:ss.SSS";
866+
if (!valueFormat.isEmpty()) {
867+
timeFormatString = valueFormat;
868+
}
869+
SimpleDateFormat dateFormat = new SimpleDateFormat(timeFormatString);
870+
java.util.Date parsedDate = dateFormat.parse(parameterValue);
871+
time = new Time(parsedDate.getTime());
872+
}
873+
} else {
874+
final DateTimeFormatter dtFormatter = getDateTimeFormatter(valueFormat);
875+
LocalTime parsedTime = LocalTime.parse(parameterValue, dtFormatter);
876+
time = Time.valueOf(parsedTime);
877+
}
878+
879+
stmt.setTime(parameterIndex, time);
835880
break;
836881
case Types.TIMESTAMP:
837882
long lTimestamp=0L;

‎nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSQL.java‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -451,6 +451,77 @@ public void testUsingTimestampValuesWithFormatAttribute() throws InitializationE
451451
}
452452
}
453453

454+
@Test
455+
public void testUsingDateTimeValuesWithFormatAttribute() throws InitializationException, ProcessException, SQLException, IOException, ParseException {
456+
final TestRunner runner = TestRunners.newTestRunner(PutSQL.class);
457+
try (final Connection conn = service.getConnection()) {
458+
try (final Statement stmt = conn.createStatement()) {
459+
stmt.executeUpdate("CREATE TABLE TIMESTAMPTEST3 (id integer primary key, ts1 TIME, ts2 DATE)");
460+
}
461+
}
462+
463+
runner.addControllerService("dbcp", service);
464+
runner.enableControllerService(service);
465+
runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp");
466+
467+
final String dateStr = "2002-02-02";
468+
final String timeStr = "12:02:02";
469+
470+
Map<String, String> attributes = new HashMap<>();
471+
attributes.put("sql.args.1.type", String.valueOf(Types.TIME));
472+
attributes.put("sql.args.1.value", timeStr);
473+
attributes.put("sql.args.1.format", "ISO_LOCAL_TIME");
474+
attributes.put("sql.args.2.type", String.valueOf(Types.DATE));
475+
attributes.put("sql.args.2.value", dateStr);
476+
attributes.put("sql.args.2.format", "ISO_LOCAL_DATE");
477+
478+
runner.enqueue("INSERT INTO TIMESTAMPTEST3 (ID, ts1, ts2) VALUES (1, ?, ?)".getBytes(), attributes);
479+
480+
attributes = new HashMap<>();
481+
attributes.put("sql.args.1.type", String.valueOf(Types.TIME));
482+
attributes.put("sql.args.1.value", "68522000");
483+
attributes.put("sql.args.2.type", String.valueOf(Types.DATE));
484+
attributes.put("sql.args.2.value", "1012633200000");
485+
486+
runner.enqueue("INSERT INTO TIMESTAMPTEST3 (ID, ts1, ts2) VALUES (2, ?, ?)".getBytes(), attributes);
487+
488+
attributes = new HashMap<>();
489+
attributes.put("sql.args.1.type", String.valueOf(Types.TIME));
490+
attributes.put("sql.args.1.value", "120202000");
491+
attributes.put("sql.args.1.format", "HHmmssSSS");
492+
attributes.put("sql.args.2.type", String.valueOf(Types.DATE));
493+
attributes.put("sql.args.2.value", "20020202");
494+
attributes.put("sql.args.2.format", "yyyyMMdd");
495+
496+
runner.enqueue("INSERT INTO TIMESTAMPTEST3 (ID, ts1, ts2) VALUES (3, ?, ?)".getBytes(), attributes);
497+
498+
runner.run();
499+
500+
runner.assertAllFlowFilesTransferred(PutSQL.REL_SUCCESS, 3);
501+
502+
try (final Connection conn = service.getConnection()) {
503+
try (final Statement stmt = conn.createStatement()) {
504+
final ResultSet rs = stmt.executeQuery("SELECT * FROM TIMESTAMPTEST3 ORDER BY ID");
505+
assertTrue(rs.next());
506+
assertEquals(1, rs.getInt(1));
507+
assertEquals(68522000L, rs.getTime(2).getTime());
508+
assertEquals(1012633200000L, rs.getDate(3).getTime());
509+
510+
assertTrue(rs.next());
511+
assertEquals(2, rs.getInt(1));
512+
assertEquals(68522000L, rs.getTime(2).getTime());
513+
assertEquals(1012633200000L, rs.getDate(3).getTime());
514+
515+
assertTrue(rs.next());
516+
assertEquals(3, rs.getInt(1));
517+
assertEquals(68522000L, rs.getTime(2).getTime());
518+
assertEquals(1012633200000L, rs.getDate(3).getTime());
519+
520+
assertFalse(rs.next());
521+
}
522+
}
523+
}
524+
454525
@Test
455526
public void testBitType() throws SQLException, InitializationException {
456527
final TestRunner runner = TestRunners.newTestRunner(PutSQL.class);
@@ -568,6 +639,91 @@ public void testBitType() throws SQLException, InitializationException {
568639

569640
}
570641

642+
@Test
643+
public void testUsingTimeValuesEpochAndString() throws InitializationException, ProcessException, SQLException, IOException, ParseException {
644+
final TestRunner runner = TestRunners.newTestRunner(PutSQL.class);
645+
try (final Connection conn = service.getConnection()) {
646+
try (final Statement stmt = conn.createStatement()) {
647+
stmt.executeUpdate("CREATE TABLE TIMETESTS (id integer primary key, ts1 time, ts2 time)");
648+
}
649+
}
650+
651+
runner.addControllerService("dbcp", service);
652+
runner.enableControllerService(service);
653+
runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp");
654+
655+
final String arg2TS = "00:01:01";
656+
final String art3TS = "12:02:02";
657+
final String timeFormatString = "HH:mm:ss";
658+
SimpleDateFormat dateFormat = new SimpleDateFormat(timeFormatString);
659+
java.util.Date parsedDate = dateFormat.parse(arg2TS);
660+
661+
final Map<String, String> attributes = new HashMap<>();
662+
attributes.put("sql.args.1.type", String.valueOf(Types.TIME));
663+
attributes.put("sql.args.1.value", Long.toString(parsedDate.getTime()));
664+
665+
attributes.put("sql.args.2.type", String.valueOf(Types.TIME));
666+
attributes.put("sql.args.2.value", art3TS);
667+
attributes.put("sql.args.2.format", timeFormatString);
668+
669+
runner.enqueue("INSERT INTO TIMETESTS (ID, ts1, ts2) VALUES (1, ?, ?)".getBytes(), attributes);
670+
runner.run();
671+
672+
runner.assertAllFlowFilesTransferred(PutSQL.REL_SUCCESS, 1);
673+
674+
try (final Connection conn = service.getConnection()) {
675+
try (final Statement stmt = conn.createStatement()) {
676+
final ResultSet rs = stmt.executeQuery("SELECT * FROM TIMETESTS");
677+
assertTrue(rs.next());
678+
assertEquals(1, rs.getInt(1));
679+
assertEquals(arg2TS, rs.getString(2));
680+
assertEquals(art3TS, rs.getString(3));
681+
assertFalse(rs.next());
682+
}
683+
}
684+
}
685+
686+
@Test
687+
public void testUsingDateValuesEpochAndString() throws InitializationException, ProcessException, SQLException, IOException, ParseException {
688+
final TestRunner runner = TestRunners.newTestRunner(PutSQL.class);
689+
try (final Connection conn = service.getConnection()) {
690+
try (final Statement stmt = conn.createStatement()) {
691+
stmt.executeUpdate("CREATE TABLE DATETESTS (id integer primary key, ts1 date, ts2 date)");
692+
}
693+
}
694+
695+
runner.addControllerService("dbcp", service);
696+
runner.enableControllerService(service);
697+
runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp");
698+
699+
final String arg2TS = "2001-01-01";
700+
final String art3TS = "2002-02-02";
701+
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
702+
java.util.Date parsedDate = dateFormat.parse(arg2TS);
703+
704+
final Map<String, String> attributes = new HashMap<>();
705+
attributes.put("sql.args.1.type", String.valueOf(Types.DATE));
706+
attributes.put("sql.args.1.value", Long.toString(parsedDate.getTime()));
707+
attributes.put("sql.args.2.type", String.valueOf(Types.DATE));
708+
attributes.put("sql.args.2.value", art3TS);
709+
710+
runner.enqueue("INSERT INTO DATETESTS (ID, ts1, ts2) VALUES (1, ?, ?)".getBytes(), attributes);
711+
runner.run();
712+
713+
runner.assertAllFlowFilesTransferred(PutSQL.REL_SUCCESS, 1);
714+
715+
try (final Connection conn = service.getConnection()) {
716+
try (final Statement stmt = conn.createStatement()) {
717+
final ResultSet rs = stmt.executeQuery("SELECT * FROM DATETESTS");
718+
assertTrue(rs.next());
719+
assertEquals(1, rs.getInt(1));
720+
assertEquals(arg2TS, rs.getString(2));
721+
assertEquals(art3TS, rs.getString(3));
722+
assertFalse(rs.next());
723+
}
724+
}
725+
}
726+
571727
@Test
572728
public void testBinaryColumnTypes() throws InitializationException, ProcessException, SQLException, IOException, ParseException {
573729
final TestRunner runner = TestRunners.newTestRunner(PutSQL.class);

0 commit comments

Comments
 (0)