From 5424f76279051232fcf44f5069c8f454150c2e04 Mon Sep 17 00:00:00 2001 From: nikitapaliwal Date: Mon, 16 Jan 2023 16:35:12 +0530 Subject: [PATCH 1/4] Added the case for reading blank cells as empty strings in place of 'NULL' strings. --- .../plugin/batch/source/ExcelInputFormat.java | 5 +++++ .../plugin/batch/source/ExcelInputReader.java | 8 +++++--- .../batch/source/ExcelInputReaderTest.java | 2 +- .../test/resources/civil_test_data_two.xlsx | Bin 4007 -> 7755 bytes 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputFormat.java b/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputFormat.java index c8accfb18..487e4949e 100644 --- a/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputFormat.java +++ b/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputFormat.java @@ -199,6 +199,7 @@ public boolean nextKeyValue() throws IOException, InterruptedException { while (cellIterator.hasNext()) { Cell cell = cellIterator.next(); String colName = CellReference.convertNumToColString(cell.getColumnIndex()); + String colValue = ""; switch (cell.getCellType()) { case Cell.CELL_TYPE_STRING: sb.append(colName) @@ -218,6 +219,10 @@ public boolean nextKeyValue() throws IOException, InterruptedException { .append(COLUMN_SEPERATOR).append(cell.getNumericCellValue()).append(CELL_SEPERATOR); } break; + + case Cell.CELL_TYPE_BLANK: + sb.append(colName).append(COLUMN_SEPERATOR).append(colValue).append(CELL_SEPERATOR); + break; } } value = new Text(sb.toString()); diff --git a/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputReader.java b/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputReader.java index 65665ee7c..56804c44d 100644 --- a/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputReader.java +++ b/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputReader.java @@ -201,10 +201,12 @@ public void transform(KeyValue input, Emitter 1) { + if (columnValue.length >= 1) { String name = columnValue[0]; - String value = columnValue[1]; - + String value = ""; + if (columnValue.length > 1) { + value = columnValue[1]; + } if (columnMapping.containsKey(name)) { excelColumnValueMap.put(columnMapping.get(name), value); } else { diff --git a/core-plugins/src/test/java/io/cdap/plugin/batch/source/ExcelInputReaderTest.java b/core-plugins/src/test/java/io/cdap/plugin/batch/source/ExcelInputReaderTest.java index dc7dcb9e4..1a81a4bcb 100644 --- a/core-plugins/src/test/java/io/cdap/plugin/batch/source/ExcelInputReaderTest.java +++ b/core-plugins/src/test/java/io/cdap/plugin/batch/source/ExcelInputReaderTest.java @@ -180,7 +180,7 @@ public void testExcelInputReader() throws Exception { nameIdMap.put("john", "3.0"); nameIdMap.put("romy", "1.0"); nameIdMap.put("Paulo", "11.0"); - nameIdMap.put("Ruskin", "10.0"); + nameIdMap.put("", "10.0"); nameIdMap.put("Alan", "8.0"); nameIdMap.put("Bill", "13.0"); nameIdMap.put("Ada", "14.0"); diff --git a/core-plugins/src/test/resources/civil_test_data_two.xlsx b/core-plugins/src/test/resources/civil_test_data_two.xlsx index d0b859766507b7977853d4275d6ee31ac6ca49b9..1dc4d76004db18dbe2261924fb587218c6323dfe 100644 GIT binary patch literal 7755 zcmd^E2UL?=wuS)G0z`U1nm|C18bG8a%cf76uY;(qJeJr=86hX_RO!`?Q5AXSv;( zlkRs?KAQ~fzb4@vgwN{MCBMJX+BtFkRkFy~4cHBlg4#g+vp<%UQ3Lte@75$lK! zRrAC)_iq(o@uJ?^yz^^8mU(4oL+-VhT{DuEniU0p8myrBz@D2dRX~cq>Su(0wW9L* z3Q2lU>)UL|{7Mr+I>EyBvDQfB8*#Edg{*T2k{prrwg*Ie4zq;?#Kn5Dx`+7fVd2nN zz+H()d~a7TjHxRg$&gAOGwnS{xEBSR{s+12UQOV2iT5366)LnkQ%1T#*n1tBYIf4iu5097t4^Q`hQzgv9`<9QB zy}hrG(D$z&s+^i|&*-2#)5r5knxytV>T)k*a`<>c|0JEtxFwX@jg(#`;<>=bH4T<- zq+R00y>=UBBfwzMsNQE2_h_FY;h8Rz*b)*a~FU`A@W_nl_ z$_cu|?$_QZO4(jp@7npcytEJ~(h$s8QJdC2R;Zn_y*hzf01AzWcC1XjTa9bW7;ej3 zocH@Ci3;?B&a98mm|KKR24}hLFFMsM?z9C~n$}I!zcmrab{N0yDPj>^Z6YH!)ZpwC zz`-K$ag{1t?zkl5^Fnc))oYL9ss`CFmDx+%V;=?&b{7ZF$zvwovTb|Aj#9EyE?LDj zM7oBs$)??us$V|V;60hqw2`2gvwkTfA1YW|wBHt*`$DT+=fg6$cj$reVpuT)EW!L<{;Bddc* zc~+Y2_STO#&AtwbEIVr+glN`oeCuuhkS7u#4?!QVNG&>R229^ThDqhh#TR@NIa(GS z>~P%RNWQeac6jJpS0imYLp9X{J3-CB5U#8Z$HCJ&dD$Dpva-7?t2cSxiAhk=m3#18 z%Clt+R#DYoW@W7}^8aRXHaqI>*v5SwSp53C_;2nD<*V@#?hE~k^9ohHY8?@6*c(Gj z9%m=;e{KqVTEw$9f_QwJmNI+tBh{n8KKFxHJbQTyvEF^|Wq#ZBQ-`%v5vz5bY9rF0 z_IJEgq)mps+U8y+@KYK;bTho&@bVzokT|E|CH8;+x!O=U8Mby`nL|&74mx5XOw};` z&@Q=4FN5zUQ^;_Fv{dlikpfC##3Mn2GR*AQIyoY{qlzNz5QdOBR`BolYha){d9E2I zbk8icLim>ulc*8`oZj7M z7X^WerQsk*w^;hriSJwe7E-7dPq)}(;lbc$O#(8g7H_xMQ{jbs07qKMXN_PYG_R*B z97>j{H~RxZf=hvfx$s5_5F;^Q^@=qs35Ea;XDb?`1Y#ry+)}U>B*75Do7hVj2_X@U4j@Jfz$*o72@=d1_#Au5Il|nCMn4cE6<}4t zT8;!m49DjvVJCz{Hb#LM&jR=rtyM`dBybLn5-!5r$i_z?MjF5^MQd#m3@KcJqlA|b za;GsD#7GBtrD$zLf+2(3aFkpm%)Qh22E@n!SXH#PB*Bow;T$ExgpjDlb`T>Nz^`QO zK!O1#WpR{2mHncSXwVq=ytI;?WAxtx&ckX6;Ly)|>|ZlLhZZEq5`*8aFRB}zI0pXbgCX~cX6%|^rkif`^*6JL z&`6lsW)XexYoZxftcL!TJ^!7fX-4agg2|+nY#ezpz&lwvB~V(wsDCFIY^*l(zJKrb zmw1*C$BvEB@g(Ty9+nW8+RSv)zwvR(pay2KfID z6dnbik#4beWW)d?vTRDAK)#20h%G=)cbkZe=`yJn_LDnn` zIx|b-e^)pN@%~P<*qUmj`wLcU^;{}fR<5o7iH!KCv(1a$y}>P3;#AOb<~x#?`Ex!H z372&)qbRA-Q(koh^fYGi<>g#9IV4(jFSW&Pc3$$z47LN8>NTe(&yb%TRzJfDd+e$m z$N_mQve-V(iPpoj&}HY7>YE z@Bx?S@pxTJvLn5+Ma=8i9U7btit<;r%CRHl!={~ooO06q(1EX$y_>zz_s{RyW{pHIod6RIZWX<13}j`>TsQ0=T2O*zPA_B}jc?*fQ7s=n^)W~d17)%a6f zyI`~TEcVhhiA2e~?CiT$yj`Q0AIc-Hf8$kZ@ToV(%QCN&1t;DoO3=&qlvYM1A-n+< z%}y>{vsl<)Z~N#qYNNrG=^_;l7U>Dk}T+y)y^s=x8O zEr|BxeqLC(trBFNv00Nsg81!qM15wYh-pfzYhqv!LxuP=t7*V~}7+zzm@b>wHjhHctyjQu3lY&pSMamp#f@5X9_h2$sxbdmG84Bon-Nc`mvc#~UiBs#%c=O7CAy zl6&4yAUQyDlJG$HxcLInz39aF^s{sAOYVjk3*#NVa{+719Y6tn17q54(buyzQ+*=u zbJBK=?c=ZdYwo7x*Yry72=>GP%2I3Fri+oxg$@gahyCvaO~!SMV@4PVBgVZm$6YhW z4c6YyMl`0n%jIN9r}Y>cMI0Ab-h;hZ-#Ko~637U{SsK}rwfF^WVW*n%&n%7hho$*A z*?8OAUHA3I70YlnGW)(J?G|C+nOUA$mZy`ZWa=b2VuPu(WW@fz<$Fyy#ay>_EcDO; zozo5c;y89}U)Rvej!s1JU>19d5dcqh^ukcr{k@b`;tSCNQq0tfLdNXXmyFY+RajJ3 z7Q1U@_*7be{R4smzg0qW&k2l(>cl<*zV8P%isVlezccO+d|<0iRQ{a8 zp^CVa-os%s6dP|Ozw^3{UK`XrK0g{x@R>%@{fIwkdL=`Eqf-62`4zil3uFjEl~cK} zeM^Jb@y7hv$C}%k_B6IA=T3?ClP!azy8Saa_1sSyESGf+I~=v_P?@Wo%EaXCcloGVK&W1SUgi>ov9+=`&)lbwEpDGs{rPH| zW*RQWltJ{j;{*|7n4(WrxFusng|+w|?>LK39bK*0T%r-iyRP0QT|*i?WWwOcGc&5U-ix_5-!D6V6Z#BkCagBZtrdITCk)ANj2t7}W1=;& zg&Ashd9yS;kuZ<@u#0?HSP1O&FEA~7fzQ0a?Q)oYJ05g8MDvT5W!vhVRoCyvvr2}w74M%u?T@;O;?`PtWe4+1HtzP^(6 zV!4qVtXNc1MBv-dmeDhD^RnJ-$m?8UO_WoT@Kr;_WW`DQx3s}$_f~woI(8#SX~8qh zS5xsa2iQ0aWH3_vJzezImY?0bzGlyI?FM}B$%TElfCjpoVt3!&Te~|jRhzLs?{%C) z$Mea}Ecx*n3(_0y;2ZR_o4cjdJofTV_hR5Sy$=Bq~33eOi2&=^;0E;y%NaB{m!3Ey3fOmP}aasXumIyS`zX zo;f)ZS@dDt6p5k(j4|lEWBb^v_7P3`ej*XU*Dm!;ESd~RrC}V=h8PHF;S@ZIi5e$J z$pJ*G4jQJ{>$W`PzNEPHQP|v+5C65cIdIfJ!oN@EX-K{Si)y+vo2M%2MmYb1z9clU z?~1My(HF%GX%5aMM(}!XN+2A`B#?0JmH%2DL=%CCs~T%G(PXmFiaIK9|Fq@Z^?tjS zjnzk^3brc{bj%j>;z6l=zIa%0+`gH)CKr?ndV~Ze*~U)X&1)A@r}sWNPu+P2>`T21Z3oxjD4P z4|cO8W~D%i43;q`ZHFg)PUS~Tfm`qkuqOWeSZcyR8rP3Eh&Jw~eB%M7RaDhbvC`d3 zwYf3lW?4GY3LU#GLQc753g?XN)puC%?5id@D?1~`L6L5fYozLyP@_`K*QJ)W1lF~( zi>)(E+yE%tg{|@PLT4@&Q3wA)A3*ZW}U=%{Ytzw`w1Oo35|m zoE~=N{&Dn0j=kfDy8b)TRGaHmN zwvpO4;x{=}*aw+>R!6X?x6@!Qr+incs>fS|%Y!rfg1vi_4a~YBsfGQ3;AiC4Yb%>< z88z;@{^xIJ_qVgrS;cT^cnFi3K#sZOr=#01<}|)a4wEsd!w!v@s2?t@5V7L4X{FybnE4wbd&@Sm2#8lQP%rkhVDX$mGT=AXM?1a95kqQzM zkH)xGQ#cQ=-SrKQ;F9(+V##D9qBLClEws*hVd#qD5xRioOX58?e1@T>ptu%*)Q4mm zLl4*qxOv6I%1=IMmk6hG(_?c1An|_{B{mODW3BjdXKcxm6Ed8G4Zzf;A5}uB=a7*f^)MK{*Ea9)2 z%&!EeqY+$4_EUJVd{~125up9baC$?+MJ+!?4NHh)_-hRFYtz$y6t`pjlqmYYs^c&F z*RPFFyA+(O{V8PF3BvaK2afkwp40Y zvL{=bvRzSQ-=cmqxBJ|d-}&SFJ#1)YV(N|lPkiWIZ#lF93>E3nM3tTPZ6Hs2mpzE5h0(divH2XbFvwk6qczpAlC zylFv%QJX|Y1$J6@J%KqiUyf>kHx%*L+p>s&<(__HvM^;~Z8UWtErURyNJbFoH${kc z46;B%dUs$pobN_nKDE`;FFqd9&>32Kf7&ZF7d>9XpX7q#Sv6Xf@^ifUr@X%DXlD1o zRvx!FypjM@h~eN(S@q^D^kQYSnr+U1Jt4GrdCYuHEQ$Tx0j2OWBa|4UmJ}yP&tiJZsG3pvI%%EPO-GH51b+so0ST ziAMJ4Ea9Wwi>;*flMX6WE8kg{q=4N1l6}FSUwbh0$H(yN znU=ec+^`uh3U#Hu?#?Jf-^}Sjyaio>yU4w&zA88On)9*Dch{q}%CFD)ih7pTK334I zT6UM(M(`ji!Hc=I8&ht_^KK~M%dc+k6`|iW>Z;q+HXtA-mp$iCTQ)YlL3jzh@#Hh$ z;;c^(iLX9C3AwmFiM@wfdMqgY`9gp1^^{AWUQ~5M|nbF`0}1Cu%r z>tslKfK}e4{l_>NmyTq9!&2j6s3AJ92O8!R)F;CEsE8p;qECGeJ6up;fE&B1Ci{ZW35=~m5Tv+Q5D>|jmF~3Hh>=! z)_CLHRlHKq`Oau8?kk8+r@f&3mOv;C%~DnnZQ1w3_^w=7 z5=pGy!vj;3rnTLrgc~O;jbUjq?7U!mE)!N7Ks~mu{8Ey5y!AYCpamqo#;W)P-CA=>#*ZP7Vt%~;# zeBn-i2a~b1sphvprDbjoYN-)7y>GK&d>vi0xi%NW5(*@{iw&m5x1+anf=(H2A_Y)W z#cS+1bXfK=sX_L#G_ASy&VuG0vpDX_5r>#G4Uv+2cRKC?Jp{|{49rU%p0?eb2Jv9- z+u;Ky&-a;6c&Q9tIes#KA+aY6y*l1$YVdGBG_n(NE?qj7;2hT7aUjw1ud4)U9FqVu zswlQuzsY0kcKm@C4u9?GCBFY$Prby`Fs*B2QVB;_cqWfcrAp%^@*5extbUeQ*r+y` ze|t=dZJEt`2}n?>su{bVYIS0+?MnYEvCQxl_ctozft ze&eh`X}e5~&FnA}N$dj^dq$#ULxcCkEl4-V;MhS<)jVz&!nI?GY6YlpQCFW3-1Bpt zokWEpj=Ku}2&Dn9mK253Ufn7S@BM>^=RJbbPpC=GwgU|xm<{@8Z>GKKtY7n7t{Iv> zoa&lj27h65`hf&h?t;Xr_iw+6Qu`g`^1HSVN(R`i#Oxj^FYSI7^77e{d~4?iD`1F8 zyWlwNEBKwiLwTE)5=BKO&(}pmBHMubURkfDj@)-Wz)5^?8 z!x~YEYFQ~j#kmu**;o&vUQdKAA-!-BFkfMz10z2pqr5_crIqPdV1DI=`#UdTet|w% zKR+KjF$pQCOEGXB1M_}OF)CEX*G5V<31uUfU12o=M!K_NPO7gWQzxH+8H#0Ln6H4# z)X8kWM!x}RPA1hUH|$e~IHsJ<>jC%7_P$>#Ka7t?!oeh<6Z9c;7wfiQn42 zM)|;bu(hIV)B0|fflvcCQtbqO)y)C$z5|Zqtc%6yDc3!Qp4RGQwLv<5JaE*f0V(A3 zp?)h1j*?btD*?Nxcm*Tmw#;F5)19AoLr7cCT`tXT_L7W0t}w3InV3uF5blx!wrh7% zlWWD+F7(}VUXtL~KqZI=)rlRvmz44D?N}E0O#oFlKjLrWYOq=~@h4~o_N#dj1q2j;&%&RyWrKCah##2&7oo^?mJ`PoJYm!wPOkT;ToI2w8UZCKU)X=?P&` z(?U@|7_E|Y^}0alr`Maw(y0%gN1Ii#Oa95AobtKeebLLH23u%xaW~T=b#P8srp{2< z$Pf6qiVC>ad1PfbSd?v(#>toVqmJ%=e_WMdpcb za@j2=tH@i#`yOPpD(O;lB_{I)*&;q!_!fIiBKb!6F>Uw07)?uFJvas1YL8294-Q0- ze4$o0x4^Fp-OyVm$(IJeFEZhL#c^Zc3|~?uQv^eh^&ijQR|A78aONZao#N8dhB&8% zM-ijBVZ-%TQgdFnc4WxA7DkNo`FlK^s5~`XeN994_$Vw{@IszmH7Ynt(Zb>*-^Fg% z2Z(LuxKTAsQC!4=XIv21ldUz3Oy&xY2%W!XkG1Rm->A(4LZU9s1y3Jl{RZ*Bj$<7eh02wL`ZX*9RzX=bi)NHE78Bd dXZl%U=r4(SIf!CMM7Y>MY#=)t+sJ?J_dm`dN8kVe From dc40e08d5e78f63d208962e005749e5fe2f2cbbf Mon Sep 17 00:00:00 2001 From: nikitapaliwal Date: Mon, 23 Jan 2023 12:44:39 +0530 Subject: [PATCH 2/4] Added the case for reading blank cells as empty strings in place of 'NULL' strings. --- .../main/java/io/cdap/plugin/batch/source/ExcelInputReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputReader.java b/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputReader.java index 56804c44d..20ca06e18 100644 --- a/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputReader.java +++ b/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputReader.java @@ -90,7 +90,7 @@ public class ExcelInputReader extends BatchSource Date: Tue, 24 Jan 2023 16:50:59 +0530 Subject: [PATCH 3/4] Added the case for reading blank cells as empty strings in place of 'NULL' strings. --- .../java/io/cdap/plugin/batch/source/ExcelInputFormat.java | 5 ----- .../java/io/cdap/plugin/batch/source/ExcelInputReader.java | 7 ++----- .../io/cdap/plugin/batch/source/ExcelInputReaderTest.java | 3 ++- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputFormat.java b/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputFormat.java index 487e4949e..c8accfb18 100644 --- a/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputFormat.java +++ b/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputFormat.java @@ -199,7 +199,6 @@ public boolean nextKeyValue() throws IOException, InterruptedException { while (cellIterator.hasNext()) { Cell cell = cellIterator.next(); String colName = CellReference.convertNumToColString(cell.getColumnIndex()); - String colValue = ""; switch (cell.getCellType()) { case Cell.CELL_TYPE_STRING: sb.append(colName) @@ -219,10 +218,6 @@ public boolean nextKeyValue() throws IOException, InterruptedException { .append(COLUMN_SEPERATOR).append(cell.getNumericCellValue()).append(CELL_SEPERATOR); } break; - - case Cell.CELL_TYPE_BLANK: - sb.append(colName).append(COLUMN_SEPERATOR).append(colValue).append(CELL_SEPERATOR); - break; } } value = new Text(sb.toString()); diff --git a/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputReader.java b/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputReader.java index 20ca06e18..350fa9cf6 100644 --- a/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputReader.java +++ b/core-plugins/src/main/java/io/cdap/plugin/batch/source/ExcelInputReader.java @@ -201,12 +201,9 @@ public void transform(KeyValue input, Emitter= 1) { + if (columnValue.length > 1) { String name = columnValue[0]; - String value = ""; - if (columnValue.length > 1) { - value = columnValue[1]; - } + String value = columnValue[1]; if (columnMapping.containsKey(name)) { excelColumnValueMap.put(columnMapping.get(name), value); } else { diff --git a/core-plugins/src/test/java/io/cdap/plugin/batch/source/ExcelInputReaderTest.java b/core-plugins/src/test/java/io/cdap/plugin/batch/source/ExcelInputReaderTest.java index 1a81a4bcb..c8549d933 100644 --- a/core-plugins/src/test/java/io/cdap/plugin/batch/source/ExcelInputReaderTest.java +++ b/core-plugins/src/test/java/io/cdap/plugin/batch/source/ExcelInputReaderTest.java @@ -159,7 +159,7 @@ public void testExcelInputReader() throws Exception { .put("skipFirstRow", "false") .put("terminateIfEmptyRow", "false") .put("rowsLimit", "") - .put("outputSchema", "A:string,B:string,C:String,D:String,E:String,F:String") + .put("outputSchema", "A:string,B:string,C:String,D:String,E:String,F:String,G:String") .put("ifErrorRecord", "Ignore error and continue") .put("errorDatasetName", "") .build(); @@ -191,6 +191,7 @@ public void testExcelInputReader() throws Exception { Assert.assertEquals(nameIdMap.get(output.get(1).get("B")), output.get(1).get("A")); Assert.assertEquals(nameIdMap.get(output.get(2).get("B")), output.get(2).get("A")); Assert.assertEquals(nameIdMap.get(output.get(3).get("B")), output.get(3).get("A")); + Assert.assertEquals("", output.get(0).get("G")); Assert.assertEquals("Expected records", 9, output.size()); } From 21ead6befa6a28f733da94b505ded3b3078dc6ed Mon Sep 17 00:00:00 2001 From: nikitapaliwal Date: Tue, 24 Jan 2023 17:03:35 +0530 Subject: [PATCH 4/4] Added the case for reading blank cells as empty strings in place of 'NULL' strings. --- .../batch/source/ExcelInputReaderTest.java | 5 ++--- .../test/resources/civil_test_data_two.xlsx | Bin 7755 -> 7787 bytes 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/core-plugins/src/test/java/io/cdap/plugin/batch/source/ExcelInputReaderTest.java b/core-plugins/src/test/java/io/cdap/plugin/batch/source/ExcelInputReaderTest.java index c8549d933..dc7dcb9e4 100644 --- a/core-plugins/src/test/java/io/cdap/plugin/batch/source/ExcelInputReaderTest.java +++ b/core-plugins/src/test/java/io/cdap/plugin/batch/source/ExcelInputReaderTest.java @@ -159,7 +159,7 @@ public void testExcelInputReader() throws Exception { .put("skipFirstRow", "false") .put("terminateIfEmptyRow", "false") .put("rowsLimit", "") - .put("outputSchema", "A:string,B:string,C:String,D:String,E:String,F:String,G:String") + .put("outputSchema", "A:string,B:string,C:String,D:String,E:String,F:String") .put("ifErrorRecord", "Ignore error and continue") .put("errorDatasetName", "") .build(); @@ -180,7 +180,7 @@ public void testExcelInputReader() throws Exception { nameIdMap.put("john", "3.0"); nameIdMap.put("romy", "1.0"); nameIdMap.put("Paulo", "11.0"); - nameIdMap.put("", "10.0"); + nameIdMap.put("Ruskin", "10.0"); nameIdMap.put("Alan", "8.0"); nameIdMap.put("Bill", "13.0"); nameIdMap.put("Ada", "14.0"); @@ -191,7 +191,6 @@ public void testExcelInputReader() throws Exception { Assert.assertEquals(nameIdMap.get(output.get(1).get("B")), output.get(1).get("A")); Assert.assertEquals(nameIdMap.get(output.get(2).get("B")), output.get(2).get("A")); Assert.assertEquals(nameIdMap.get(output.get(3).get("B")), output.get(3).get("A")); - Assert.assertEquals("", output.get(0).get("G")); Assert.assertEquals("Expected records", 9, output.size()); } diff --git a/core-plugins/src/test/resources/civil_test_data_two.xlsx b/core-plugins/src/test/resources/civil_test_data_two.xlsx index 1dc4d76004db18dbe2261924fb587218c6323dfe..24bafe3e52a35adf5cc88da23308f40d075b9e6a 100644 GIT binary patch delta 2899 zcmZ9O3p~^N8^>qHrVUFZnMMwCx8^pv%;i*M=%iSZ&1r}*S-F;l!pWud%Q=h^ig57P z{wyh`lFJ;orH(SUB6p#f`+s!)ozwZfUeE9M{k>nG@9lX#&-49q%_qzyk2r`zWI-?( z4CJ4PuaLA85fy#`l87z?u-_@(fyoF<4C(*#LYa-dyQm?vS}|oLc*+S;k$9S_Xj8X_ zhxXPVf`+XMKfC zV~iJKvZf{%{6krfZwrxefA;#t+22?--OJBMA}#yxOo~6^XESm2svNj-}@u@q7j zQL&7#<6F!HReH8n)#$&O!DL;ZSehKq^@}k0h39u$$2aL6uW`F)nRjw`O;{3lDXV+9 zaCkLz1>Y1IZSZcopsw2UV(&$zz@_0ckA{arp1%FPFOcj}_N%k%luxi}CZjr=KzzV3 z$Q{1lddBo5L%{evTgbnG4r2)}oMZjDzJM)X!{)R*6G~}h%amDO^zKCq{jfKTrHF|6 zN`XU@@k~C3PJC&?+K_44FgcpxWP{(O)t;II&v`u=Lg?y>@4?J{mZ`=^?V2MbE%8Ts zK4KQjpOM{4q!-DnYvZfuBllP1vryf&mzpNf69Ctlku~MN#9kU&`x3h7Pq3(F)@5CK z+*9kxUky7~`LyqneqEE_#u}q+dZA%BdXn4y$H-uRVQy{c?K?X1?CWS>@*c~+sm6&f zMe{2bvqc!o8NWg4s4f@h<)@gw_Lb|Ab1u$(i!a7->>acp#ZI7U_gT8WC|tmJ?{)EyK9GO6CPuQo# zM(d55$wBqg+1;+M7PS4_-Ohi)MR%v_hhq!h^lO?5{G#X{Ap+4ax2>fP_ZhEBRw#a$ zWzb$GRjqVK2;6+KY`3FaJKb$>a&m?wH zZ>j@EnZGwv9o5g`P5{kJ;%n-|uSnLh;0*2npvfe5Qm?7I7-r@*Z=vHr5tG^_p=Z{t#~ zEIyy3Al9Z4d@#mfTlP-_$>6W(KpP{`TNx_}sfyv?kt8U1-im^TAs{hVm9g6)#<3hb zBuN5nU`@fm5MnX+m9cV=s#p#QNrHpv))ajhLOiBH8LI#>j^ms`k|e<_Yl;aB0gV|} z#wtOo;yB?*k~DbUnnHjfU@>AUSQUuz6;2|Ov>j|W)K$7IZEE|d&3;~bvR>A5*s;+V#AxZM!c^e8DhCswzRl)9q z7{_y(kR%1Lfh`3X6gS<;7*K*`r`190`>zbl*nDEE2v(4*DS(yue&C3>*8fdpFHjG4 zLVXA`7wAeyw?CKe{mfBV3Y3oSkLwoRNNyHRX>OUWnS78po zEL>kGf&yU4P@n^m1{g>Y0V^rOUuJF(;}H_#n>VX+ZDydaysz6t4}WN3t$BHPNLaJh z7E2nhZI*4@?uHG}5eF#j=6K6K14$5Qp$H`W|Hsk7YLpV$I$Ky=X_c}0N&FPoh6}1-D!B%6PISjQ4zFPRX_#gFt z)-GqK$X-4*sgzo}8qVnJb+8VoU|a6}Z-~fEGy45sNVi95&y%_P~iB1>5bk(j%9p?0bG4G`t{s z44yVW7-jHW!Dlh<=4kp!Omf?-f5nT$vp)X0VW#V~y5*LqKbUtLr5~I}UGE)SVGOy9 ztZFRLxz&d*-#wLja;C=226eVNFUV_uS$)^8k_XVv>BI@0!b*6;tsf?a=i)WA44q(M z&;4zC6jEhccl++Hqa57MGmM?3-#Z!Npz%(%h@N5c?59_?pZRi}&{s)^?FJin$?4(v zB}XF?GR7(s9;1$F=#tMK=?lOfzI>fg5dT)=z&JDclh;uy^_j-Qm!qn$jGV6RrPp<6 zr(b4mn{Kr`oyI?Is>k(Sjdgcv0N$yl=RMr3ar!9WE)f{;!QsJJPRGSyc;|-1sH$(D z>UD=El3heocnXl$bhqu!$y2eH3)Vs>FFV@O)G|gF zHalhVi{-oEw*xRrVi{k80}7}XmTe8R&6rA;J$zUe3<3>`eIHYJ7O*qsYpTib(*Js& zvsW11OgWa}Zu(bHtY|AK1g_um6W^DGCBrOMyT;z7sY9`KY~Hnt#T<>~|U*@ByXjzB$f6_pzA(e?M-*F5jiY z5eE^lEa=;H)zz^juk?^`o$A76NdMhW*eU?Ll=f^b;a7!vmvJGf7P@NpcN#6=j*{k9 vC2CPp-0z_P0+B z2pWtci$EYjHb7LG5s_vn1_gp32nYuF6P$5ZXZ|zu?z{87@1A?^neV)FM{S?kD(!QX zhp9mj2m~aYYVttIMNVG&2EF8rfPw~fg;s<2I}q#~zI6*GH?1qXtw-Il>zMiKaHNJ~ z`fa0kW3JfejsAaZnXR`u)gnpyyni*D3k-c~Fn(45wq#BZC7w>QLK>e0LEac|WO9CH zq4TYf!(O=I?q=0JHztytJYBWDK=Su%Vyf*@_i%5>f^STjSD1U8YEMKnfcNYkYseD} z4-JGoxDt8np54|6N)G!lGjD3_!?Wox!$X6KrkrHVBX)j$=RLQ)snKr!prUbyS;I(g z)o3QKpq*DJ8i@O{ou3$GF8t6XJaQE1Nxm9BCkm<*P4g1VysNuw9(x(lPj;O?WqLHZ z!pqj;B_}v2K@V&6Zd8R1*e$XOCI?G0kKd0hDd+6`R7M|~>U=AhpAiYP3F7X@xT$E$ zLLNPD+wn}!nXnYx&U`X&9JaXpfr>p0={{1#6}cMOn7s+5wKy{oiF-g zx-e`eXb2n!^m4aNjY%Z2)s;KEg(|&GlqJ590*2{u7L(t&7t+V!J9o~EjQSZ=Sy-v4 zmqr>MBjBzI%2jHtb#!W&@n5`F(KCMS9M5s5oc~aD{!7GQ>FD|Nh`~0|fPHz3b3-~$ zN=#*4qh}jV)+XL$8jN*-*G{V`(|g{jTuN+>mG$}4cB-8%u0GaQa>~9z9d0Zx zZ`XJ^vofS*P3*+ll=kc$Hp%@9_I&$}$ZS_-z-N)1n@(>iR{|sy(00*2zAcWkMrG+X zKm?4ld@esS3H5DzQhb)QvPwgF3(?%@rGG9Sw8+2B-$23#o9zMj(1mL(XvB8^V*CNi{9^t=5<>9W2V3E8+LCNIM^rnW@Hx9c zXZw22SnYVygceTH4ChQ!K`#8N+-uMtxgJC#9ko+(RtWuTDc%#|8a00O70rzlp9!W$ z^8FRUFe<2r+$AMBfESPlL0y!QcCx3l@KxLF=}+geDS-{+ue&aoPc~swD9*w+7eG4F zNlMZO>rlJFYba0c1#Kk3<%xUl#H|gIEl2`!V22ynmnE@9685EmwJ0Yk2?g#~+zeh{ z`8%$@xmEIZFQM{9OUb7?!-Y?K2{pM_Z=^KHUAl4s=gS9cS6YFf;yB1sTmwRtGr&G2 zs*CSTN2^Ki5hIQ6<+A@2y^~W_lL90pjo2EaCRK!VwpDIx4$Jka^B@-#6(^#niosa5^b$Ab&ia|*JA>i$#6t=~0NEfjY04n+Va74B31I;-TjRsSuKsyEvRfC;CDhi)4^-;~-wk1gc^=Yt zPa4*t?MdQUW$&k!LMStK3(BWfI~P@!q#}B7sgzR`{s(7f<6V|p8h`j?8Dsb5wQGap ztQRVbjB9=#u-&HQN#5;d(EmQ=qkc^4&jS^J>J!xsm^?xAm&I&TPl`iKdD=0|qeo=R z*^OP;m}+%)RR13y;^^ucUz+PK)&7W{BUAHPw*taz?MFQ=gA~FIL!;a(1In{Hl5*`D z*cY;gW&++Dqq}2_4U2BmPk20Pe16?^3|r)Dq9YVtUl%<0=KACiEftcXLL(j%e*z8(0uM?C*ZqK5zoaaP`c#CV^%jVdW!b$|1piON9l=es+ju!nP#9hfYpd@P4o(A4c`y}vKz zJ_AnV2VF4PbI2jrp(o(6TJoydk(j84nRKL@rci6oWyqBmIK6|mVjIJzXBvBlZ)F`A z3&8F@Or5>HX)Zj0qwZ(%YgN|RuP=Jp1s?{Y7W32%-iQ0-UQ;-VJY27NSfg)Z=D}(M zy8uE^PG9T1bqY$&agU{o=&wD0gXh+jYUc2WuLf~PKdzB!Xv6OOC>Rt1FqZN)>g#Hh^Qg5cQvrjRSv2K`F0KUAU(awbe0B-m&T&gx$1EgYgmSer7S$kka zid0DRwe~v>3ar6yTK@5jV6|*)H6f7W(E(vG##?3Mk6U>!9@%a#-AWVo@~