-
Notifications
You must be signed in to change notification settings - Fork 2.1k
fix(starrocks): query INFORMATION_SCHEMA for table comments #27035
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
878eb74
42e4f35
d6d215d
d68d7e0
b4e92ba
cdaf1b1
d4ec49b
b41c725
cfcb18f
abe12df
d291dce
5b8cd45
8debddc
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -43,6 +43,7 @@ | |
| STARROCKS_PARTITION_DETAILS, | ||
| STARROCKS_SHOW_FULL_COLUMNS, | ||
| ) | ||
| from metadata.ingestion.source.database.starrocks.utils import get_table_comment | ||
|
Check warning on line 46 in ingestion/src/metadata/ingestion/source/database/starrocks/metadata.py
|
||
| from metadata.utils.logger import ingestion_logger | ||
| from metadata.utils.ssl_manager import SSLManager, check_ssl_and_init | ||
|
|
||
|
|
@@ -258,21 +259,16 @@ | |
|
|
||
| return tables | ||
|
|
||
| @staticmethod | ||
| def get_table_description(schema_name: str, table_name: str, inspector: Inspector) -> Optional[str]: | ||
| # pylint: disable=arguments-differ,unused-argument | ||
| def get_table_description(self, schema_name: str, table_name: str, inspector: Inspector) -> Optional[str]: | ||
|
Comment on lines
+262
to
+263
|
||
| description = None | ||
| try: | ||
| table_info: dict = inspector.get_table_comment(table_name, schema_name) | ||
| table_info = get_table_comment(None, self.connection, table_name, schema=schema_name) | ||
|
Check warning on line 266 in ingestion/src/metadata/ingestion/source/database/starrocks/metadata.py
|
||
| except Exception as exc: # pylint: disable=broad-except | ||
| logger.debug(traceback.format_exc()) | ||
| logger.warning(f"Table description error for table [{schema_name}.{table_name}]: {exc}") | ||
| else: | ||
| description = table_info.get("text") | ||
|
|
||
| if description is None: | ||
| return None | ||
| if isinstance(description, (list, tuple)) and len(description) > 0: | ||
| return description[0] | ||
| return description | ||
|
Comment on lines
+262
to
272
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
|
|
||
| def _get_columns(self, table_name, schema=None): | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -14,7 +14,7 @@ | |
| """ | ||
|
|
||
| from unittest import TestCase | ||
| from unittest.mock import patch | ||
| from unittest.mock import MagicMock, patch | ||
|
|
||
| import pytest | ||
| from sqlalchemy import types as sqltypes | ||
|
|
@@ -26,6 +26,7 @@ | |
| StarRocksSource, | ||
| _get_sqlalchemy_type, | ||
| ) | ||
| from metadata.ingestion.source.database.starrocks.utils import get_table_comment | ||
|
|
||
| mock_starrocks_config = { | ||
| "source": { | ||
|
|
@@ -171,3 +172,79 @@ def test_iceberg_relkind_mapping(self): | |
| from metadata.ingestion.source.database.starrocks.metadata import RELKIND_MAP | ||
|
|
||
| assert RELKIND_MAP["ICEBERG"] == TableType.Iceberg | ||
|
|
||
|
|
||
| class TestStarRocksGetTableDescription(TestCase): | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 💡 Quality: Tests use unittest.TestCase; project prefers plain pytestPer the project's testing guidelines, new tests should use Was this helpful? React with 👍 / 👎 | Reply |
||
| """Tests for get_table_description delegating to utils.get_table_comment""" | ||
|
|
||
| @patch("metadata.ingestion.source.database.common_db_source.CommonDbSourceService.test_connection") | ||
| def setUp(self, test_connection): | ||
| test_connection.return_value = False | ||
| self.config = OpenMetadataWorkflowConfig.model_validate(mock_starrocks_config) | ||
| self.source = StarRocksSource.create( | ||
| mock_starrocks_config["source"], | ||
| self.config.workflowConfig.openMetadataServerConfig, | ||
| ) | ||
| thread_id = self.source.context.get_current_thread_id() | ||
| self.source._connection_map[thread_id] = MagicMock() | ||
|
|
||
| @patch("metadata.ingestion.source.database.starrocks.metadata.get_table_comment") | ||
| def test_returns_table_comment(self, mock_get_table_comment): | ||
| mock_get_table_comment.return_value = {"text": "审计日志表"} | ||
|
|
||
| description = self.source.get_table_description( | ||
| schema_name="test_db", | ||
| table_name="audit_tbl", | ||
| inspector=MagicMock(), | ||
| ) | ||
| assert description == "审计日志表" | ||
| mock_get_table_comment.assert_called_once_with(None, self.source.connection, "audit_tbl", schema="test_db") | ||
|
|
||
|
Comment on lines
+191
to
+202
|
||
| @patch("metadata.ingestion.source.database.starrocks.metadata.get_table_comment") | ||
| def test_returns_none_for_empty_comment(self, mock_get_table_comment): | ||
| mock_get_table_comment.return_value = {"text": None} | ||
|
|
||
| description = self.source.get_table_description( | ||
| schema_name="test_db", | ||
| table_name="no_comment_tbl", | ||
| inspector=MagicMock(), | ||
| ) | ||
| assert description is None | ||
|
|
||
| @patch("metadata.ingestion.source.database.starrocks.metadata.get_table_comment") | ||
| def test_returns_none_on_exception(self, mock_get_table_comment): | ||
| mock_get_table_comment.side_effect = Exception("connection error") | ||
|
|
||
| description = self.source.get_table_description( | ||
| schema_name="test_db", | ||
| table_name="error_tbl", | ||
| inspector=MagicMock(), | ||
| ) | ||
| assert description is None | ||
|
Comment on lines
+177
to
+223
|
||
|
|
||
|
|
||
| class TestGetTableComment(TestCase): | ||
| """Tests for utils.get_table_comment row parsing""" | ||
|
|
||
| def _make_connection(self, rows): | ||
| connection = MagicMock() | ||
| connection.info = {} | ||
| result = MagicMock() | ||
| result.mappings.return_value = iter(rows) | ||
| connection.execute.return_value = result | ||
| return connection | ||
|
Comment on lines
+229
to
+235
|
||
|
|
||
| def test_returns_comment_from_row(self): | ||
| row = {"TABLE_COMMENT": "审计日志表"} | ||
| connection = self._make_connection([row]) | ||
|
|
||
| result = get_table_comment(None, connection, "audit_tbl", schema="test_db") | ||
|
|
||
| assert result == {"text": "审计日志表"} | ||
|
|
||
| def test_returns_none_when_no_rows(self): | ||
| connection = self._make_connection([]) | ||
|
|
||
| result = get_table_comment(None, connection, "missing_tbl", schema="test_db") | ||
|
|
||
| assert result == {"text": None} | ||
Uh oh!
There was an error while loading. Please reload this page.