Skip to content

Commit c9a3ae2

Browse files
snappy-sachinsnappy-sachin
committed
Merge branch 'master' into SNAP-2603
2 parents cb653dd + a8d34b9 commit c9a3ae2

21 files changed

Lines changed: 367 additions & 251 deletions

‎core/src/main/scala/org/apache/spark/sql/SnappySession.scala‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2208,11 +2208,7 @@ object CachedKey {
22082208
case a: AttributeReference =>
22092209
AttributeReference(a.name, a.dataType, a.nullable)(exprId = ExprId(-1))
22102210
case a: Alias =>
2211-
val name = if (a.name == Utils.WEIGHTAGE_COLUMN_NAME ||
2212-
a.name.startsWith(Utils.SKIP_ANALYSIS_PREFIX)) {
2213-
a.name
2214-
} else "none"
2215-
Alias(a.child, name)(exprId = ExprId(-1))
2211+
Alias(a.child, a.name)(exprId = ExprId(-1))
22162212
case ae: AggregateExpression => ae.copy(resultId = ExprId(-1))
22172213
case _: ScalarSubquery =>
22182214
throw new IllegalStateException("scalar subquery should not have been present")

‎core/src/test/scala/org/apache/spark/sql/store/PlanCachingTest.scala‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Copyright (c) 2018 SnappyData, Inc. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you
5+
* may not use this file except in compliance with the License. You
6+
* may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13+
* implied. See the License for the specific language governing
14+
* permissions and limitations under the License. See accompanying
15+
* LICENSE file.
16+
*/
17+
package org.apache.spark.sql.store
18+
19+
import io.snappydata.Property.PlanCaching
20+
import io.snappydata.SnappyFunSuite
21+
import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll}
22+
23+
import org.apache.spark.Logging
24+
import org.apache.spark.sql.SnappySession
25+
26+
class PlanCachingTest extends SnappyFunSuite
27+
with Logging
28+
with BeforeAndAfter
29+
with BeforeAndAfterAll{
30+
31+
var planCaching : Boolean = false
32+
33+
override def beforeAll(): Unit = {
34+
planCaching = PlanCaching.get(snc.sessionState.conf)
35+
PlanCaching.set(snc.sessionState.conf, true)
36+
super.beforeAll()
37+
}
38+
39+
override def afterAll(): Unit = {
40+
PlanCaching.set(snc.sessionState.conf, planCaching)
41+
snc.sql("drop table if exists tcol")
42+
super.afterAll()
43+
}
44+
45+
test("SNAP-2716 - Wrong column names in results due to aliases being ignored in plan caching"){
46+
val cacheMap = SnappySession.getPlanCache.asMap()
47+
cacheMap.clear()
48+
snc.sql("create table tcol(col1 int not null)")
49+
snc.sql("select col1, case when col1 < 10 then 'small' else 'big' " +
50+
"end as value_category from tcol")
51+
val dataFrame = snc.sql("select col1, case when col1 < 10 then 'small' else 'big' " +
52+
"end as tmp_value_category from tcol;")
53+
54+
val columns = dataFrame.schema.fields.map(_.name)
55+
56+
assert(columns sameElements Array("COL1", "TMP_VALUE_CATEGORY"))
57+
58+
assert(cacheMap.size() == 2)
59+
60+
snc.sql("select col1, case when col1 < 11 then 'small' else 'big' " +
61+
"end as tmp_value_category from tcol;")
62+
63+
assert(cacheMap.size() == 2)
64+
}
65+
}

‎dtests/src/resources/scripts/ct/ct_create_and_load_colocatedWithEviction_column_table.sql‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,15 @@ CREATE TABLE ORDERS_DETAILS
1717
TRDR_SYS_LOGON_ID VARCHAR(64),SOLICITED_FG VARCHAR(1),RCVD_FROM_FIRMID_CD VARCHAR(20),RCV_DESK VARCHAR(20),
1818
SYS_ACCT_ID_SRC VARCHAR(64) ,CUST_ACCT_MNEMONIC VARCHAR(128),CUST_SLANG VARCHAR(20) ,SYS_ACCT_TYPE VARCHAR(20) ,
1919
CUST_EXCH_ACCT_ID VARCHAR(64) ,SYS_SECURITY_ALT_ID VARCHAR(64) ,TICKER_SYMBOL VARCHAR(32) ,TICKER_SYMBOL_SUFFIX VARCHAR(20) ,
20-
PRODUCT_CAT_CD VARCHAR(20) ,SIDE VARCHAR(20) ,LIMIT_PRICE DECIMAL(38, 18),STOP_PRICE DECIMAL(38, 18),ORDER_QTY DECIMAL(18, 4) ,
21-
TOTAL_EXECUTED_QTY DECIMAL(18, 4) ,AVG_PRICE DECIMAL(38, 18) ,DAY_EXECUTED_QTY DECIMAL(18, 4) ,DAY_AVG_PRICE DECIMAL(38, 18) ,
20+
PRODUCT_CAT_CD VARCHAR(20) ,SIDE VARCHAR(20) ,LIMIT_PRICE DECIMAL(18, 8),STOP_PRICE DECIMAL(18, 8),ORDER_QTY DECIMAL(18, 4) ,
21+
TOTAL_EXECUTED_QTY DECIMAL(18, 4) ,AVG_PRICE DECIMAL(18, 8) ,DAY_EXECUTED_QTY DECIMAL(18, 4) ,DAY_AVG_PRICE DECIMAL(18, 8) ,
2222
REMNG_QTY DECIMAL(18, 4) ,CNCL_QTY DECIMAL(18, 4) ,CNCL_BY_FG VARCHAR(1) ,EXPIRE_TS TIMESTAMP ,EXEC_INSTR VARCHAR(64) ,
2323
TIME_IN_FORCE VARCHAR(20) ,RULE80AF VARCHAR(1) ,DEST_FIRMID_CD VARCHAR(20) ,SENT_TO_CONDUIT VARCHAR(20) ,SENT_TO_MPID VARCHAR(20) ,
2424
RCV_METHOD_CD VARCHAR(20) ,LIMIT_ORDER_DISP_IND VARCHAR(1) ,MERGED_ORDER_FG VARCHAR(1) ,MERGED_TO_ORDER_ID VARCHAR(64) ,
2525
RCV_DEPT_ID VARCHAR(20) ,ROUTE_METHOD_CD VARCHAR(20) ,LOCATE_ID VARCHAR(256) ,LOCATE_TS TIMESTAMP ,LOCATE_OVERRIDE_REASON VARCHAR(2000) ,
2626
LOCATE_BROKER VARCHAR(256) ,ORDER_BRCH_SEQ_TXT VARCHAR(20) ,IGNORE_CD VARCHAR(20) ,CLIENT_ORDER_REFID VARCHAR(64) ,
27-
CLIENT_ORDER_ORIG_REFID VARCHAR(64) ,ORDER_TYPE_CD VARCHAR(20) ,SENT_TO_ORDER_ID VARCHAR(64) ,ASK_PRICE DECIMAL(38, 18) ,
28-
ASK_QTY DECIMAL(18, 4) ,BID_PRICE DECIMAL(38, 18) ,BID_QTY DECIMAL(18, 4) ,REG_NMS_EXCEP_CD VARCHAR(20) ,REG_NMS_EXCEP_TXT VARCHAR(2000) ,
27+
CLIENT_ORDER_ORIG_REFID VARCHAR(64) ,ORDER_TYPE_CD VARCHAR(20) ,SENT_TO_ORDER_ID VARCHAR(64) ,ASK_PRICE DECIMAL(18, 8) ,
28+
ASK_QTY DECIMAL(18, 4) ,BID_PRICE DECIMAL(18, 8) ,BID_QTY DECIMAL(18, 4) ,REG_NMS_EXCEP_CD VARCHAR(20) ,REG_NMS_EXCEP_TXT VARCHAR(2000) ,
2929
REG_NMS_LINK_ID VARCHAR(64) ,REG_NMS_PRINTS VARCHAR(1) ,REG_NMS_STOP_TIME TIMESTAMP ,SENT_TS TIMESTAMP ,RULE92 VARCHAR(1) ,
3030
RULE92_OVERRIDE_TXT VARCHAR(2000) ,RULE92_RATIO DECIMAL(25, 10) ,EXMPT_STGY_BEGIN_TIME TIMESTAMP ,EXMPT_STGY_END_TIME TIMESTAMP ,
3131
EXMPT_STGY_PRICE_INST VARCHAR(2000) ,EXMPT_STGY_QTY DECIMAL(18, 4) ,CAPACITY VARCHAR(20) ,DISCRETION_QTY DECIMAL(18, 4) ,
@@ -39,13 +39,13 @@ CREATE TABLE ORDERS_DETAILS
3939
EXTERNAL_TRDR_ID VARCHAR(64) ,ANONYMOUS_ORDER_FG VARCHAR(1) ,SYS_SECURITY_ALT_SRC VARCHAR(20) ,CURR_CD VARCHAR(20) ,
4040
EVENT_TYPE_CD VARCHAR(20) ,SYS_CLIENT_ACCT_ID VARCHAR(64) ,SYS_FIRM_ACCT_ID VARCHAR(20) ,SYS_TRDR_ID VARCHAR(64) ,DEST_ID INTEGER ,
4141
OPT_PUT_OR_CALL VARCHAR(20) ,SRC_FEED_REF_CD VARCHAR(64) ,DIGEST_KEY VARCHAR(128) ,EFF_TS TIMESTAMP ,ENTRY_TS TIMESTAMP ,
42-
OPT_STRIKE_PRICE DECIMAL(38, 18) ,OPT_MATURITY_DATE VARCHAR(20) ,ORDER_RESTR VARCHAR(4) ,SHORT_SELL_EXEMPT_CD VARCHAR(4) ,
42+
OPT_STRIKE_PRICE DECIMAL(18, 8) ,OPT_MATURITY_DATE VARCHAR(20) ,ORDER_RESTR VARCHAR(4) ,SHORT_SELL_EXEMPT_CD VARCHAR(4) ,
4343
QUOTE_TIME TIMESTAMP ,SLS_CREDIT VARCHAR(20) ,SYS_SECURITY_ID VARCHAR(64) ,SYS_SECURITY_ID_SRC VARCHAR(20) ,SYS_SRC_SYS_ID VARCHAR(20) ,
4444
SYS_ORDER_ID_UNIQUE_SUFFIX VARCHAR(20) ,DEST_ID_SRC VARCHAR(4) ,GLB_ROOT_SRC_SYS_ID VARCHAR(20) ,GLB_ROOT_ORDER_ID_SUFFIX VARCHAR(64) ,
4545
SYS_ROOT_ORDER_ID_SUFFIX VARCHAR(20) ,SYS_PARENT_ORDER_ID_SUFFIX VARCHAR(20) ,CREDIT_BREACH_PERCENT DECIMAL(25, 10) ,
4646
CREDIT_BREACH_OVERRIDE VARCHAR(256) ,INFO_BARRIER_ID VARCHAR(256) ,EXCH_PARTICIPANT_ID VARCHAR(64) ,REJECT_REASON_CD VARCHAR(4) ,
47-
DIRECTED_DEST VARCHAR(20) ,REG_NMS_LINK_TYPE VARCHAR(20) ,CONVER_RATIO DECIMAL(18, 9) ,STOCK_REF_PRICE DECIMAL(38, 18) ,
48-
CB_SWAP_ORDER_FG VARCHAR(1) ,EV DECIMAL(38, 18) ,SYS_DATA_MODIFIED_TS TIMESTAMP ,CMSN_TYPE VARCHAR(20) ,SYS_CREDIT_TRDR_ID VARCHAR(20) ,
47+
DIRECTED_DEST VARCHAR(20) ,REG_NMS_LINK_TYPE VARCHAR(20) ,CONVER_RATIO DECIMAL(18, 9) ,STOCK_REF_PRICE DECIMAL(18, 8) ,
48+
CB_SWAP_ORDER_FG VARCHAR(1) ,EV DECIMAL(18, 8) ,SYS_DATA_MODIFIED_TS TIMESTAMP ,CMSN_TYPE VARCHAR(20) ,SYS_CREDIT_TRDR_ID VARCHAR(20) ,
4949
SYS_ENTRY_USER_ID VARCHAR(20) ,OPEN_CLOSE_CD VARCHAR(20) ,AS_OF_TRD_FG VARCHAR(1) ,HANDLING_INSTR VARCHAR(20) ,SECURITY_DESC VARCHAR(512) ,
5050
MINIMUM_QTY DECIMAL(21, 6) ,CUST_OR_FIRM VARCHAR(20) ,MAXIMUM_SHOW DECIMAL(21, 6) ,SECURITY_SUB_TYPE VARCHAR(20) ,MULTILEG_RPT_TYPE VARCHAR(4) ,
5151
ORDER_ACTION_TYPE VARCHAR(4) ,BARRIER_STYLE VARCHAR(4) ,AUTO_IOI_REF_TYPE VARCHAR(4) ,PEG_OFFSET_VAL DECIMAL(10, 2) ,AUTO_IOI_OFFSET DECIMAL(28, 12) ,
@@ -67,25 +67,25 @@ CREATE TABLE EXEC_DETAILS
6767
(EXEC_DID BIGINT,SYS_EXEC_VER INTEGER,SYS_EXEC_ID VARCHAR(64),TRD_DATE VARCHAR(20),ALT_EXEC_ID VARCHAR(64),SYS_EXEC_STAT VARCHAR(20),
6868
DW_EXEC_STAT VARCHAR(20),ORDER_OWNER_FIRM_ID VARCHAR(20),TRDR_SYS_LOGON_ID VARCHAR(64),CONTRA_BROKER_MNEMONIC VARCHAR(20),SIDE VARCHAR(20),
6969
TICKER_SYMBOL VARCHAR(32),SYS_SECURITY_ALT_ID VARCHAR(64),PRODUCT_CAT_CD VARCHAR(20),LAST_MKT VARCHAR(20),EXECUTED_QTY DECIMAL(18, 4),
70-
EXEC_PRICE DECIMAL( 38, 18),EXEC_PRICE_CURR_CD VARCHAR(20),EXEC_CAPACITY VARCHAR(20),CLIENT_ACCT_ID BIGINT,FIRM_ACCT_ID BIGINT,
70+
EXEC_PRICE DECIMAL( 18, 8),EXEC_PRICE_CURR_CD VARCHAR(20),EXEC_CAPACITY VARCHAR(20),CLIENT_ACCT_ID BIGINT,FIRM_ACCT_ID BIGINT,
7171
AVG_PRICE_ACCT_ID BIGINT,OCEAN_ACCT_ID BIGINT,EXEC_CNTRY_CD VARCHAR(20),CMSN VARCHAR(20),COMMENT_TXT VARCHAR(2000),
7272
ACT_BRCH_SEQ_TXT VARCHAR(20),IGNORE_CD VARCHAR(20),SRC_SYS VARCHAR(20),EXEC_TYPE_CD VARCHAR(20),LIQUIDITY_CD VARCHAR(20),
73-
ASK_PRICE DECIMAL( 38, 18),ASK_QTY DECIMAL(18, 4),TRD_REPORT_ASOF_DATE VARCHAR(20),BID_PRICE DECIMAL( 38, 18),BID_QTY DECIMAL(18, 4),
73+
ASK_PRICE DECIMAL( 18, 8),ASK_QTY DECIMAL(18, 4),TRD_REPORT_ASOF_DATE VARCHAR(20),BID_PRICE DECIMAL( 18, 8),BID_QTY DECIMAL(18, 4),
7474
CROSS_ID VARCHAR(64),NYSE_SUBREPORT_TYPE VARCHAR(20),QUOTE_COORDINATOR VARCHAR(20),QUOTE_TIME TIMESTAMP,REG_NMS_EXCEPT_CD VARCHAR(20),
7575
REG_NMS_EXCEPT_TXT VARCHAR(2000),REG_NMS_LINK_ID VARCHAR(64),REG_NMS_MKT_CENTER_ID VARCHAR(64),REG_NMS_OVERRIDE VARCHAR(20),REG_NMS_PRINTS VARCHAR(1),
7676
EXECUTED_BY VARCHAR(20),TICKER_SYMBOL_SUFFIX VARCHAR(20),PREREGNMS_TRD_MOD1 VARCHAR(1),PREREGNMS_TRD_MOD2 VARCHAR(1),PREREGNMS_TRD_MOD3 VARCHAR(1),
7777
PREREGNMS_TRD_MOD4 VARCHAR(1),NMS_FG VARCHAR(1),GIVEUP_BROKER VARCHAR(20),CHANNEL_NM VARCHAR(128),ORDER_FLOW_ENTRY VARCHAR(20),FLOW_CAT VARCHAR(20),
7878
FLOW_CLASS VARCHAR(20),FLOW_TGT VARCHAR(20),ORDER_FLOW_CHANNEL VARCHAR(20),FLOW_SUBCAT VARCHAR(20),SYS_ACCT_ID_SRC VARCHAR(64),STRTGY_CD VARCHAR(20),
7979
EXECUTING_BROKER_CD VARCHAR(20),LEAF_EXEC_FG VARCHAR(1),RCVD_EXEC_ID VARCHAR(64),RCVD_EXEC_VER INTEGER,ORDER_FLOW_DESK VARCHAR(20),
80-
SYS_ROOT_ORDER_ID VARCHAR(64),SYS_ROOT_ORDER_VER INTEGER,GLB_ROOT_ORDER_ID VARCHAR(64),TOTAL_EXECUTED_QTY DECIMAL(18, 4),AVG_PRICE DECIMAL( 38, 18),
80+
SYS_ROOT_ORDER_ID VARCHAR(64),SYS_ROOT_ORDER_VER INTEGER,GLB_ROOT_ORDER_ID VARCHAR(64),TOTAL_EXECUTED_QTY DECIMAL(18, 4),AVG_PRICE DECIMAL( 18, 8),
8181
DEST_CD VARCHAR(20),CLIENT_ORDER_REFID VARCHAR(64),CLIENT_ORDER_ORIG_REFID VARCHAR(64),CROSS_EXEC_FG VARCHAR(1),OCEAN_PRODUCT_ID BIGINT,
8282
TRDR_ID BIGINT,REF_TIME_ID INTEGER,CREATED_BY VARCHAR(64),CREATED_DATE TIMESTAMP,FIX_EXEC_ID VARCHAR(64),FIX_ORIGINAL_EXEC_ID VARCHAR(64),
8383
RELATED_MKT_CENTER VARCHAR(20),TRANS_TS TIMESTAMP,SYS_SECURITY_ALT_SRC VARCHAR(20),EVENT_TYPE_CD VARCHAR(20),SYS_CLIENT_ACCT_ID VARCHAR(64),
8484
SYS_FIRM_ACCT_ID VARCHAR(20),SYS_AVG_PRICE_ACCT_ID VARCHAR(20),SYS_TRDR_ID VARCHAR(64),ACT_BRCH_SEQ VARCHAR(20),SYS_ORDER_ID VARCHAR(64),
85-
SYS_ORDER_VER INTEGER,SRC_FEED_REF_CD VARCHAR(64),DIGEST_KEY VARCHAR(128),TRUE_LAST_MKT VARCHAR(20),ENTRY_TS TIMESTAMP,OPT_STRIKE_PRICE DECIMAL( 38, 18),
85+
SYS_ORDER_VER INTEGER,SRC_FEED_REF_CD VARCHAR(64),DIGEST_KEY VARCHAR(128),TRUE_LAST_MKT VARCHAR(20),ENTRY_TS TIMESTAMP,OPT_STRIKE_PRICE DECIMAL( 18, 8),
8686
OPT_MATURITY_DATE VARCHAR(20),EXPIRE_TS TIMESTAMP,OPT_PUT_OR_CALL VARCHAR(20),SYS_ORDER_STAT_CD VARCHAR(20),CONTRA_ACCT VARCHAR(64),CONTRA_ACCT_SRC VARCHAR(20),
8787
CONTRA_BROKER_SRC VARCHAR(20),SYS_SECURITY_ID VARCHAR(64),SYS_SECURITY_ID_SRC VARCHAR(20),SYS_SRC_SYS_ID VARCHAR(20),SYS_ORDER_ID_UNIQUE_SUFFIX VARCHAR(20),
88-
DEST VARCHAR(20),DEST_ID_SRC VARCHAR(4),CONVER_RATIO DECIMAL(18, 9),STOCK_REF_PRICE DECIMAL( 38, 18),AS_OF_TRD_FG VARCHAR(1),MULTILEG_RPT_TYPE VARCHAR(4),
88+
DEST VARCHAR(20),DEST_ID_SRC VARCHAR(4),CONVER_RATIO DECIMAL(18, 9),STOCK_REF_PRICE DECIMAL( 18, 8),AS_OF_TRD_FG VARCHAR(1),MULTILEG_RPT_TYPE VARCHAR(4),
8989
REG_NMS_LINK_TYPE VARCHAR(20),EXEC_SUB_TYPE VARCHAR(4),CMSN_TYPE VARCHAR(20),QUOTE_CONDITION_IND VARCHAR(20),TRD_THROUGH_FG VARCHAR(1),
9090
REGNMS_ORDER_LINK_ID VARCHAR(64),REGNMS_ORDER_LINK_TYPE VARCHAR(20),DK_IND VARCHAR(20),NBBO_QUOTE_TIME VARCHAR(20),GLB_ROOT_SRC_SYS_ID VARCHAR(20),
9191
TRD_REPORT_TYPE VARCHAR(20),REPORT_TO_EXCH_FG VARCHAR(1),CMPLN_COMMENT VARCHAR(256),DEAL_TYPE VARCHAR(4),EXEC_COMMENTS VARCHAR(256),
@@ -98,7 +98,7 @@ CREATE TABLE EXEC_DETAILS
9898
CUST_FX_RATE_CCY_FROM VARCHAR(10),SYS_GFCID VARCHAR(20),CONTRA_SIDE VARCHAR(20),OPT_CONTRACT_MULTIPLIER DECIMAL(10, 2),PRIOR_REF_PRICE_TS TIMESTAMP,
9999
SECURITY_SUB_TYPE VARCHAR(20),MSG_DIRECTION VARCHAR(20),LEAF_SYS_EXEC_ID VARCHAR(64),LEAF_SRC_SYS VARCHAR(20),FIX_LAST_MKT VARCHAR(20),
100100
FIX_CONTRA_BROKER_MNEMONIC VARCHAR(20),RIO_MSG_SRC VARCHAR(64),SNAPSHOT_TS TIMESTAMP,EXTERNAL_TRANS_TS TIMESTAMP,PRICE_CATEGORY VARCHAR(32),
101-
UNDERLYING_FX_RATE DECIMAL(36, 8),CONVERSION_RATE DECIMAL(36, 8),TRANS_COMMENT VARCHAR(256),AGGRESSOR_FLAG VARCHAR(1))
101+
UNDERLYING_FX_RATE DECIMAL(18, 8),CONVERSION_RATE DECIMAL(18, 8),TRANS_COMMENT VARCHAR(256),AGGRESSOR_FLAG VARCHAR(1))
102102
USING column OPTIONS (partition_by 'EXEC_DID', redundancy ':redundancy',
103103
COLOCATE_WITH 'ORDERS_DETAILS', EVICTION_BY ':evictionByOption');
104104

0 commit comments

Comments
 (0)