Merge branch 'master' into fix/stale-version-count-bug

Author: LemonCL

.github/workflows/third_party_review.yml

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: Dependency License Review
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+permissions:
+  contents: read
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout Repository'
+        uses: actions/checkout@v5
+      - name: Dependency Review
+        uses: actions/dependency-review-action@v4
+        with:
+          # Possible values: "critical", "high", "moderate", "low"
+          fail-on-severity: moderate
+          # You can only include one of these two options: `allow-licenses` and `deny-licenses`
+          # ([String]). Only allow these licenses (optional)
+          # Possible values: Any SPDX-compliant license identifiers or expressions from https://spdx.org/licenses/
+          allow-licenses:  BSD-2-Clause, BSD-3-Clause, MIT, Apache-2.0, EPL-2.0, MPL-2.0, CC0-1.0
+          #allow-ghsas: GHSA-abcd-1234-5679, GHSA-efgh-1234-5679
+          # ([String]). Block pull requests that introduce vulnerabilities in the scopes that match this list (optional)
+          # Possible values: "development", "runtime", "unknown"
+          fail-on-scopes: development, runtime

be/src/cloud/cloud_rowset_builder.cpp

@@ -31,7 +31,12 @@ CloudRowsetBuilder::CloudRowsetBuilder(CloudStorageEngine& engine, const WriteRe
                                        RuntimeProfile* profile)
         : BaseRowsetBuilder(req, profile), _engine(engine) {}
 
-CloudRowsetBuilder::~CloudRowsetBuilder() = default;
+CloudRowsetBuilder::~CloudRowsetBuilder() {
+    // Clear file cache immediately when load fails
+    if (_is_init && _rowset != nullptr && _rowset->rowset_meta()->rowset_state() == PREPARED) {
+        _rowset->clear_cache();
+    }
+}
 
 Status CloudRowsetBuilder::init() {
     _tablet = DORIS_TRY(_engine.get_tablet(_req.tablet_id));

be/src/cloud/cloud_tablet.cpp

@@ -62,7 +62,6 @@
 #include "olap/txn_manager.h"
 #include "util/debug_points.h"
 #include "util/stack_util.h"
-#include "vec/common/schema_util.h"
 
 namespace doris {
 #include "common/compile_check_begin.h"

be/src/common/consts.h

@@ -30,7 +30,6 @@ const std::string ROW_STORE_COL = "__DORIS_ROW_STORE_COL__";
 const std::string DYNAMIC_COLUMN_NAME = "__DORIS_DYNAMIC_COL__";
 const std::string PARTIAL_UPDATE_AUTO_INC_COL = "__PARTIAL_UPDATE_AUTO_INC_COLUMN__";
 const std::string VIRTUAL_COLUMN_PREFIX = "__DORIS_VIRTUAL_COL__";
-const std::string SPARSE_COLUMN_PATH = "__DORIS_VARIANT_SPARSE__";
 
 /// The maximum precision representable by a 4-byte decimal (Decimal4Value)
 constexpr int MAX_DECIMAL32_PRECISION = 9;

be/src/http/action/stream_load_forward_handler.cpp

@@ -32,12 +32,6 @@ namespace doris {
 #include "common/compile_check_begin.h"
 
 int StreamLoadForwardHandler::on_header(HttpRequest* req) {
-    std::ostringstream headers_info;
-    const auto& headers = req->headers();
-    for (const auto& header : headers) {
-        headers_info << header.first << ":" << header.second << " ";
-    }
-
     std::ostringstream params_info;
     const auto* params = req->params();
     for (const auto& param : *params) {
@@ -46,7 +40,7 @@ int StreamLoadForwardHandler::on_header(HttpRequest* req) {
 
     LOG(INFO) << "StreamLoadForward request started - "
               << "path: " << req->raw_path() << ", remote: " << req->remote_host() << ", headers: ["
-              << headers_info.str() << "]"
+              << req->get_all_headers() << "]"
               << ", params: [" << params_info.str() << "]";
 
     std::shared_ptr<StreamLoadForwardContext> ctx(new StreamLoadForwardContext());

be/src/http/http_request.cpp

@@ -29,6 +29,7 @@
 #include <utility>
 
 #include "http/http_handler.h"
+#include "http/http_headers.h"
 #include "runtime/stream_load/stream_load_context.h"
 #include "util/stack_util.h"
 
@@ -87,7 +88,12 @@ std::string HttpRequest::debug_string() const {
        << "raw_path:" << _raw_path << "\n"
        << "headers: \n";
     for (auto& iter : _headers) {
-        ss << "key=" << iter.first << ", value=" << iter.second << "\n";
+        if (iequal(iter.first, HttpHeaders::AUTHORIZATION) ||
+            iequal(iter.first, HttpHeaders::PROXY_AUTHORIZATION)) {
+            ss << "key=" << iter.first << ", value=***MASKED***\n";
+        } else {
+            ss << "key=" << iter.first << ", value=" << iter.second << "\n";
+        }
     }
     ss << "params: \n";
     for (auto& iter : _params) {
@@ -116,7 +122,13 @@ const std::string& HttpRequest::param(const std::string& key) const {
 std::string HttpRequest::get_all_headers() const {
     std::stringstream headers;
     for (const auto& header : _headers) {
-        headers << header.first << ":" << header.second + ", ";
+        // Mask sensitive headers
+        if (iequal(header.first, HttpHeaders::AUTHORIZATION) ||
+            iequal(header.first, HttpHeaders::PROXY_AUTHORIZATION)) {
+            headers << header.first << ":***MASKED***, ";
+        } else {
+            headers << header.first << ":" << header.second + ", ";
+        }
     }
     return headers.str();
 }

be/src/olap/base_tablet.cpp

@@ -54,7 +54,6 @@
 #include "util/doris_metrics.h"
 #include "util/key_util.h"
 #include "vec/common/assert_cast.h"
-#include "vec/common/schema_util.h"
 #include "vec/data_types/data_type_factory.hpp"
 #include "vec/jsonb/serialize.h"
 

be/src/olap/compaction.cpp

@@ -82,7 +82,7 @@
 #include "util/pretty_printer.h"
 #include "util/time.h"
 #include "util/trace.h"
-#include "vec/common/schema_util.h"
+#include "vec/common/variant_util.h"
 
 using std::vector;
 
@@ -371,7 +371,7 @@ Status CompactionMixin::do_compact_ordered_rowsets() {
     _output_rowset = _output_rs_writer->manual_build(rowset_meta);
 
     // 2. check variant column path stats
-    RETURN_IF_ERROR(vectorized::schema_util::VariantCompactionUtil::check_path_stats(
+    RETURN_IF_ERROR(vectorized::variant_util::VariantCompactionUtil::check_path_stats(
             _input_rowsets, _output_rowset, _tablet));
     return Status::OK();
 }
@@ -422,7 +422,7 @@ Status CompactionMixin::build_basic_info(bool is_ordered_compaction) {
     // for ordered compaction, we don't need to extend the schema for variant columns
     if (_enable_vertical_compact_variant_subcolumns && !is_ordered_compaction) {
         RETURN_IF_ERROR(
-                vectorized::schema_util::VariantCompactionUtil::get_extended_compaction_schema(
+                vectorized::variant_util::VariantCompactionUtil::get_extended_compaction_schema(
                         _input_rowsets, _cur_tablet_schema));
     }
     return Status::OK();
@@ -1424,7 +1424,7 @@ Status Compaction::check_correctness() {
                 _output_rowset->num_rows());
     }
     // 2. check variant column path stats
-    RETURN_IF_ERROR(vectorized::schema_util::VariantCompactionUtil::check_path_stats(
+    RETURN_IF_ERROR(vectorized::variant_util::VariantCompactionUtil::check_path_stats(
             _input_rowsets, _output_rowset, _tablet));
     return Status::OK();
 }
@@ -1488,7 +1488,7 @@ Status CloudCompactionMixin::build_basic_info() {
     // so get_extended_compaction_schema will extended the schema for variant columns
     if (_enable_vertical_compact_variant_subcolumns) {
         RETURN_IF_ERROR(
-                vectorized::schema_util::VariantCompactionUtil::get_extended_compaction_schema(
+                vectorized::variant_util::VariantCompactionUtil::get_extended_compaction_schema(
                         _input_rowsets, _cur_tablet_schema));
     }
     return Status::OK();

be/src/olap/olap_common.h

@@ -448,6 +448,7 @@ struct OlapReaderStatistics {
     int64_t variant_subtree_leaf_iter_count = 0;
     int64_t variant_subtree_hierarchical_iter_count = 0;
     int64_t variant_subtree_sparse_iter_count = 0;
+    int64_t variant_doc_value_column_iter_count = 0;
 };
 
 using ColumnId = uint32_t;

be/src/olap/rowset/beta_rowset_writer.cpp

@@ -59,7 +59,6 @@
 #include "util/stopwatch.hpp"
 #include "util/time.h"
 #include "vec/columns/column.h"
-#include "vec/common/schema_util.h"
 #include "vec/core/block.h"
 #include "vec/data_types/data_type_factory.hpp"