Merge pull request #58 from passren/0.7.1

0.7.1

Author: passren

.github/workflows/run-test.yaml

@@ -35,7 +35,7 @@ jobs:
           pip install boto3
           pip install tenacity
           pip install pyparsing
-          pip install sqlean.py
+          pip install sqlean.py==3.45.1
       - name: black
         run: |
           pip install black

pydynamodb/__init__.py

@@ -6,7 +6,7 @@
 if TYPE_CHECKING:
     from .connection import Connection
 
-__version__: str = "0.7.0"
+__version__: str = "0.7.1"
 
 # Globals https://www.python.org/dev/peps/pep-0249/#globals
 apilevel: str = "2.0"

pydynamodb/superset_dynamodb/querydb.py

@@ -155,18 +155,18 @@ def rollback(self):
 
     def close(self):
         if self.connection is not None:
-            if self.config.purge_enabled:
+            if self.cache_enabled and self.config.purge_enabled:
                 self.purge()
             self.connection.close()
 
     @synchronized
     def purge(self) -> int:
         purged_count = 0
 
-        if not self.cache_enabled:
+        if self.config.purge_time < self.config.expire_time:
             return purged_count
 
-        if self.config.purge_time < self.config.expire_time:
+        if not self.cache_enabled or not self.has_table(QueryDB.CACHE_TABLE):
             return purged_count
 
         _purge_period = datetime.now() - timedelta(seconds=self.config.purge_time)

setup.py

@@ -14,7 +14,7 @@
     "botocore>=1.24.7",
     "tenacity>=4.1.0",
     "pyparsing>=3.0.0",
-    "sqlean.py>=3.45.0",
+    "sqlean.py==3.45.1",
 ]
 
 extras_require = {

tests/test_superset_dml_select.py

@@ -16,6 +16,10 @@ def test_parse_base_select(self):
         assert ret == {
             "Statement": "SELECT col_list[1],col_map.A FROM \"Issues\" WHERE key_partition = 'row_1'"
         }
+        assert parser.parser.inner_columns == None
+        assert parser.parser.inner_exprs == None
+        assert parser.parser.outer_columns == None
+        assert parser.parser.outer_exprs == None
 
     def test_parse_nested_select_case_1(self):
         sql = """
@@ -136,6 +140,11 @@ def test_parse_nested_select_case_4(self):
         """
         parser = SQLParser(sql, parser_class=SupersetSelect)
         ret = parser.transform()
+        assert ret == {'Statement': 'SELECT id FROM "lakefront-ingest-stg-config-table"'}
+        assert parser.parser.inner_columns == None
+        assert parser.parser.inner_exprs == None
+        assert parser.parser.outer_columns == "text_split(id,'.',1)"
+        assert parser.parser.outer_exprs == ""
 
     def test_parse_nested_select_case_5(self):
         sql = """
@@ -147,4 +156,9 @@ def test_parse_nested_select_case_5(self):
         GROUP BY id
         """
         parser = SQLParser(sql, parser_class=SupersetSelect)
-        ret = parser.transform()
+        ret = parser.transform()
+        assert ret == {'Statement': 'SELECT id FROM "lakefront-ingest-stg-config-table"'}
+        assert parser.parser.inner_columns == "text_split(id,'.',1)"
+        assert parser.parser.inner_exprs == ""
+        assert parser.parser.outer_columns == "id"
+        assert parser.parser.outer_exprs == "AS virtual_table\n        GROUP BY id"

tests/test_superset_dynamodb.py

@@ -172,18 +172,16 @@ def test_insert_flat_data(self, cursor):
     def test_execute_select(self, superset_cursor):
         superset_cursor.execute(
             """
-            SELECT col_list, A FROM (
-                SELECT col_list[1] col_list, NUMBER(col_map.A)
-                    FROM %s WHERE key_partition='row_1'
-            )
+            SELECT col_list[1] col_list, NUMBER(col_map.A)
+                FROM %s WHERE key_partition='row_1'
         """
             % TESTCASE04_TABLE
         )
         ret = superset_cursor.fetchall()
         assert len(ret) == 6
         assert [(d[0], d[1]) for d in superset_cursor.description] == [
-            ("col_list", "TEXT"),
-            ("A", "REAL"),
+            ("col_list", "STRING"),
+            ("A", "NUMBER"),
         ]
 
     def test_execute_nested_select(self, superset_cursor):
@@ -325,6 +323,20 @@ def test_sqlean_string_functions(self, superset_cursor):
         assert len(ret) == 2
         assert ret[0] == ("F", "F", "2")
 
+    def test_sqlalchemy_execute_single_select(self, superset_engine):
+        _, conn = superset_engine
+        rows = conn.execute(
+            text(
+                """
+                SELECT col_list[1] col_list_1, NUMBER(col_map.A)
+                FROM %s WHERE key_partition=:pk
+            """
+                % TESTCASE04_TABLE
+            ),
+            {"pk": "row_2"},
+        ).fetchall()
+        assert len(rows) == 8
+
     def test_sqlalchemy_execute_nested_select(self, superset_engine):
         _, conn = superset_engine
         rows = conn.execute(
@@ -395,6 +407,7 @@ def test_cached_querydb_step1(self, superset_engine):
         os.environ["PYDYNAMODB_QUERYDB_LOAD_BATCH_SIZE"] = "20"
         os.environ["PYDYNAMODB_QUERYDB_EXPIRE_TIME"] = "3"
 
+        self.test_sqlalchemy_execute_single_select(superset_engine)
         self.test_sqlalchemy_execute_nested_select(superset_engine)
         self.test_sqlalchemy_execute_flat_data(superset_engine)
         self.test_sqlalchemy_execute_alias_select(superset_engine)
@@ -425,6 +438,7 @@ def test_cached_querydb_step2(self, cursor):
 
     def test_cached_querydb_step3(self, superset_engine):
         # Cache used
+        self.test_sqlalchemy_execute_single_select(superset_engine)
         self.test_sqlalchemy_execute_nested_select(superset_engine)
         self.test_sqlalchemy_execute_alias_select(superset_engine)