chore: BI-0 sync

app/app_configs/api.yaml

@@ -42,6 +42,8 @@ CONNECTOR_AVAILABILITY:
       availability: free
     - conn_type: promql
       availability: free
+    - conn_type: starrocks
+      availability: hidden
   visible_connectors: []
 CORS_ALLOWED_HEADERS:
 - Content-Type

app/dl_control_api/pyproject.toml

@@ -23,6 +23,7 @@ dl-connector-mysql = {path = "../../lib/dl_connector_mysql"}
 dl-connector-oracle = {path = "../../lib/dl_connector_oracle"}
 dl-connector-postgresql = {path = "../../lib/dl_connector_postgresql"}
 dl-connector-promql = {path = "../../lib/dl_connector_promql"}
+dl-connector-starrocks = {path = "../../lib/dl_connector_starrocks"}
 dl-connector-trino = {path = "../../lib/dl_connector_trino"}
 dl-connector-ydb = {path = "../../lib/dl_connector_ydb"}
 dl-constants = {path = "../../lib/dl_constants"}
@@ -50,6 +51,7 @@ DEP002 = [
   "dl-connector-oracle",
   "dl-connector-postgresql",
   "dl-connector-promql",
+  "dl-connector-starrocks",
   "dl-connector-trino",
   "dl-connector-ydb",
   "dl-model-tools",

app/dl_data_api/pyproject.toml

@@ -24,6 +24,7 @@ dl-connector-mysql = {path = "../../lib/dl_connector_mysql"}
 dl-connector-oracle = {path = "../../lib/dl_connector_oracle"}
 dl-connector-postgresql = {path = "../../lib/dl_connector_postgresql"}
 dl-connector-promql = {path = "../../lib/dl_connector_promql"}
+dl-connector-starrocks = {path = "../../lib/dl_connector_starrocks"}
 dl-connector-trino = {path = "../../lib/dl_connector_trino"}
 dl-connector-ydb = {path = "../../lib/dl_connector_ydb"}
 dl-core = {path = "../../lib/dl_core"}
@@ -51,6 +52,7 @@ DEP002 = [
   "dl-connector-oracle",
   "dl-connector-postgresql",
   "dl-connector-promql",
+  "dl-connector-starrocks",
   "dl-connector-trino",
   "dl-connector-ydb",
   "dl-model-tools",

lib/dl_connector_starrocks/dl_connector_starrocks/core/async_adapters_starrocks.py

@@ -18,7 +18,6 @@
 from dl_app_tools.profiling_base import generic_profiler_async
 from dl_constants.types import TBIChunksGen
 from dl_core.connection_executors.adapters.adapter_actions.async_base import AsyncDBVersionAdapterAction
-from dl_core.connection_executors.adapters.adapter_actions.db_version import AsyncDBVersionAdapterActionViaFunctionQuery
 from dl_core.connection_executors.adapters.async_adapters_base import (
     AsyncCache,
     AsyncDirectDBAdapter,
@@ -40,6 +39,7 @@
     ExecutionStepDataChunk,
 )
 from dl_core.connection_executors.models.scoped_rci import DBAdapterScopedRCI
+from dl_core.connection_models.common_models import DBIdent
 from dl_core.connectors.base.error_handling import ETBasedExceptionMaker
 from dl_sqlalchemy_starrocks.base import BIStarRocksDialect
 
@@ -53,6 +53,20 @@
 
 LOGGER = logging.getLogger(__name__)
 
+
+@attr.s(frozen=True)
+class AsyncStarRocksDBVersionAction(AsyncDBVersionAdapterAction):
+    _async_adapter: AsyncDirectDBAdapter = attr.ib(kw_only=True)
+
+    async def run_db_version_action(self, db_ident: DBIdent) -> str | None:
+        query = DBAdapterQuery(sa.select([sa.text("current_version()")]), db_name=db_ident.db_name)
+        result = await self._async_adapter.execute(query)
+        async for chunk in result.raw_chunk_generator:
+            if chunk:
+                return str(chunk[0][0])
+        return None
+
+
 _DBA_ASYNC_STARROCKS_TV = TypeVar("_DBA_ASYNC_STARROCKS_TV", bound="AsyncStarRocksAdapter")
 
 
@@ -79,7 +93,7 @@ class AsyncStarRocksAdapter(
     )
 
     def _make_async_db_version_action(self) -> AsyncDBVersionAdapterAction:
-        return AsyncDBVersionAdapterActionViaFunctionQuery(async_adapter=self)
+        return AsyncStarRocksDBVersionAction(async_adapter=self)
 
     @property
     def _dialect(self) -> sa.engine.default.DefaultDialect:

lib/dl_connector_starrocks/dl_connector_starrocks/formula/connector.py

@@ -1,13 +1,17 @@
-from sqlalchemy.dialects.mysql.base import MySQLDialect as SAMySQLDialect
-
 from dl_formula.connectors.base.connector import FormulaConnector
+from dl_sqlalchemy_starrocks.base import BIStarRocksDialect
 
 from dl_connector_starrocks.formula.constants import StarRocksDialect as StarRocksDialectNS
+from dl_connector_starrocks.formula.definitions.all import DEFINITIONS
+from dl_connector_starrocks.formula.literal import StarRocksLiteralizer
+from dl_connector_starrocks.formula.type_constructor import StarRocksTypeConstructor
 
 
 class StarRocksFormulaConnector(FormulaConnector):
     dialect_ns_cls = StarRocksDialectNS
     dialects = StarRocksDialectNS.STARROCKS
-    default_dialect = StarRocksDialectNS.STARROCKS_3_0
-    op_definitions = ()
-    sa_dialect = SAMySQLDialect()
+    default_dialect = StarRocksDialectNS.STARROCKS_3_2
+    op_definitions = DEFINITIONS
+    literalizer_cls = StarRocksLiteralizer
+    type_constructor_cls = StarRocksTypeConstructor
+    sa_dialect = BIStarRocksDialect()

lib/dl_connector_starrocks/dl_connector_starrocks/formula/constants.py

@@ -9,5 +9,5 @@
 
 
 class StarRocksDialect(DialectNamespace):
-    STARROCKS_3_0 = simple_combo(name=DIALECT_NAME_STARROCKS, version=(3, 0))
-    STARROCKS = STARROCKS_3_0
+    STARROCKS_3_2 = simple_combo(name=DIALECT_NAME_STARROCKS, version=(3, 2))
+    STARROCKS = STARROCKS_3_2

lib/dl_connector_starrocks/dl_connector_starrocks/formula/definitions/all.py

@@ -0,0 +1,32 @@
+from dl_connector_starrocks.formula.definitions.conditional_blocks import DEFINITIONS_COND_BLOCKS
+from dl_connector_starrocks.formula.definitions.functions_aggregation import DEFINITIONS_AGG
+from dl_connector_starrocks.formula.definitions.functions_datetime import DEFINITIONS_DATETIME
+from dl_connector_starrocks.formula.definitions.functions_hash import DEFINITIONS_HASH
+from dl_connector_starrocks.formula.definitions.functions_logical import DEFINITIONS_LOGICAL
+from dl_connector_starrocks.formula.definitions.functions_markup import DEFINITIONS_MARKUP
+from dl_connector_starrocks.formula.definitions.functions_math import DEFINITIONS_MATH
+from dl_connector_starrocks.formula.definitions.functions_native import DEFINITIONS_NATIVE
+from dl_connector_starrocks.formula.definitions.functions_string import DEFINITIONS_STRING
+from dl_connector_starrocks.formula.definitions.functions_type import DEFINITIONS_TYPE
+from dl_connector_starrocks.formula.definitions.functions_window import DEFINITIONS_WINDOW
+from dl_connector_starrocks.formula.definitions.operators_binary import DEFINITIONS_BINARY
+from dl_connector_starrocks.formula.definitions.operators_ternary import DEFINITIONS_TERNARY
+from dl_connector_starrocks.formula.definitions.operators_unary import DEFINITIONS_UNARY
+
+
+DEFINITIONS = [
+    *DEFINITIONS_COND_BLOCKS,
+    *DEFINITIONS_AGG,
+    *DEFINITIONS_DATETIME,
+    *DEFINITIONS_LOGICAL,
+    *DEFINITIONS_MARKUP,
+    *DEFINITIONS_MATH,
+    *DEFINITIONS_NATIVE,
+    *DEFINITIONS_STRING,
+    *DEFINITIONS_TYPE,
+    *DEFINITIONS_WINDOW,
+    *DEFINITIONS_UNARY,
+    *DEFINITIONS_BINARY,
+    *DEFINITIONS_TERNARY,
+    *DEFINITIONS_HASH,
+]

lib/dl_connector_starrocks/dl_connector_starrocks/formula/definitions/conditional_blocks.py

@@ -0,0 +1,22 @@
+import sqlalchemy as sa
+
+from dl_formula.definitions.base import TranslationVariant
+import dl_formula.definitions.conditional_blocks as base
+
+from dl_connector_starrocks.formula.constants import StarRocksDialect as D
+
+
+V = TranslationVariant.make
+
+
+DEFINITIONS_COND_BLOCKS = [
+    # _case_block_
+    base.CaseBlock.for_dialect(D.STARROCKS),
+    # _if_block_
+    base.IfBlock3(
+        variants=[
+            V(D.STARROCKS, sa.func.IF),
+        ]
+    ),
+    base.IfBlockMulti.for_dialect(D.STARROCKS),
+]

lib/dl_connector_starrocks/dl_connector_starrocks/formula/definitions/functions_aggregation.py

@@ -0,0 +1,49 @@
+import sqlalchemy as sa
+
+from dl_formula.definitions.base import TranslationVariant
+import dl_formula.definitions.functions_aggregation as base
+
+from dl_connector_starrocks.formula.constants import StarRocksDialect as D
+
+
+V = TranslationVariant.make
+
+
+DEFINITIONS_AGG = [
+    # any
+    base.AggAny(
+        variants=[
+            V(D.STARROCKS, sa.func.ANY_VALUE),
+        ]
+    ),
+    # avg
+    base.AggAvgFromNumber.for_dialect(D.STARROCKS),
+    # TODO: BI-7171 AggAvgFromDate, AggAvgFromDatetime, AggAvgFromDatetimeTZ
+    # avg_if
+    base.AggAvgIf.for_dialect(D.STARROCKS),
+    # count
+    base.AggCount0.for_dialect(D.STARROCKS),
+    base.AggCount1.for_dialect(D.STARROCKS),
+    # count_if
+    base.AggCountIf.for_dialect(D.STARROCKS),
+    # countd
+    base.AggCountd.for_dialect(D.STARROCKS),
+    # countd_if
+    base.AggCountdIf.for_dialect(D.STARROCKS),
+    # max
+    base.AggMax.for_dialect(D.STARROCKS),
+    # min
+    base.AggMin.for_dialect(D.STARROCKS),
+    # stdev
+    base.AggStdev.for_dialect(D.STARROCKS),
+    # stdevp
+    base.AggStdevp.for_dialect(D.STARROCKS),
+    # sum
+    base.AggSum.for_dialect(D.STARROCKS),
+    # sum_if
+    base.AggSumIf.for_dialect(D.STARROCKS),
+    # var
+    base.AggVar.for_dialect(D.STARROCKS),
+    # varp
+    base.AggVarp.for_dialect(D.STARROCKS),
+]

lib/dl_connector_starrocks/dl_connector_starrocks/formula/definitions/functions_datetime.py

@@ -0,0 +1,84 @@
+import sqlalchemy as sa
+
+from dl_formula.definitions.base import TranslationVariant
+import dl_formula.definitions.functions_datetime as base
+
+from dl_connector_starrocks.formula.constants import StarRocksDialect as D
+
+
+V = TranslationVariant.make
+
+
+DEFINITIONS_DATETIME = [
+    # TODO: BI-7171 dateadd, datepart, datetrunc
+    # day
+    base.FuncDay(
+        variants=[
+            V(D.STARROCKS, sa.func.DAYOFMONTH),
+        ]
+    ),
+    # TODO: BI-7171 dayofweek
+    # genericnow
+    base.FuncGenericNow(
+        variants=[
+            V(D.STARROCKS, sa.func.NOW),
+        ]
+    ),
+    # hour
+    base.FuncHourDate.for_dialect(D.STARROCKS),
+    base.FuncHourDatetime(
+        variants=[
+            V(D.STARROCKS, sa.func.HOUR),
+        ]
+    ),
+    # minute
+    base.FuncMinuteDate.for_dialect(D.STARROCKS),
+    base.FuncMinuteDatetime(
+        variants=[
+            V(D.STARROCKS, sa.func.MINUTE),
+        ]
+    ),
+    # month
+    base.FuncMonth(
+        variants=[
+            V(D.STARROCKS, sa.func.MONTH),
+        ]
+    ),
+    # now
+    base.FuncNow(
+        variants=[
+            V(D.STARROCKS, sa.func.NOW),
+        ]
+    ),
+    # quarter
+    base.FuncQuarter(
+        variants=[
+            V(D.STARROCKS, sa.func.QUARTER),
+        ]
+    ),
+    # second
+    base.FuncSecondDate.for_dialect(D.STARROCKS),
+    base.FuncSecondDatetime(
+        variants=[
+            V(D.STARROCKS, sa.func.SECOND),
+        ]
+    ),
+    # today
+    base.FuncToday(
+        variants=[
+            V(D.STARROCKS, sa.func.CURDATE),
+        ]
+    ),
+    # week
+    base.FuncWeek(
+        variants=[
+            V(D.STARROCKS, sa.func.WEEKOFYEAR),
+        ]
+    ),
+    # year
+    base.FuncYear(
+        variants=[
+            V(D.STARROCKS, sa.func.YEAR),
+        ]
+    ),
+]

lib/dl_connector_starrocks/dl_connector_starrocks/formula/definitions/functions_hash.py

@@ -0,0 +1,30 @@
+import sqlalchemy as sa
+
+from dl_formula.definitions.base import TranslationVariant
+import dl_formula.definitions.functions_hash as base
+from dl_formula.shortcuts import n
+
+from dl_connector_starrocks.formula.constants import StarRocksDialect as D
+
+
+V = TranslationVariant.make
+
+
+DEFINITIONS_HASH = [
+    base.MD5(
+        variants=[
+            V(
+                D.STARROCKS,
+                lambda value: n.func.upper(sa.func.md5(value)),
+            ),
+        ]
+    ),
+    base.SHA256(
+        variants=[
+            V(
+                D.STARROCKS,
+                lambda value: n.func.upper(sa.func.sha2(value, 256)),
+            ),
+        ]
+    ),
+]

lib/dl_connector_starrocks/dl_connector_starrocks/formula/definitions/functions_logical.py

@@ -0,0 +1,37 @@
+import sqlalchemy as sa
+
+from dl_formula.definitions.base import TranslationVariant
+import dl_formula.definitions.functions_logical as base
+
+from dl_connector_starrocks.formula.constants import StarRocksDialect as D
+
+
+V = TranslationVariant.make
+
+
+DEFINITIONS_LOGICAL = [
+    # case
+    base.FuncCase.for_dialect(D.STARROCKS),
+    # if
+    base.FuncIf.for_dialect(D.STARROCKS),
+    # ifnull
+    base.FuncIfnull(
+        variants=[
+            V(D.STARROCKS, sa.func.IFNULL),
+        ]
+    ),
+    # iif
+    base.FuncIif3Legacy.for_dialect(D.STARROCKS),
+    # isnull
+    base.FuncIsnull(
+        variants=[
+            V(D.STARROCKS, sa.func.ISNULL),
+        ]
+    ),
+    # zn
+    base.FuncZn(
+        variants=[
+            V(D.STARROCKS, lambda x: sa.func.IFNULL(x, 0)),
+        ]
+    ),
+]