Merge branch 'master' into fix-doris-25672

Author: eldenmoon

.claude/skills/be-code-style/SKILL.md

@@ -14,6 +14,14 @@ Fix C++ code formatting issues in the BE and Cloud modules using the project's c
 - When CI reports clang-format failures
 - When you need to check or fix C++ code style
 
+## Prerequisites
+
+You need to confirm that the major version of the called clang-format is 16. If the current environment's default does not meet this requirement, try the following:
+
+1. If `.vscode/settings.json` exists, use the clang-format.executable item in it.
+2. If it is a worktree directory, use the `.vscode/settings.json` from the main directory.
+3. Check the path to the compiler toolchain by trying to find it from the `PATH` environment variable, the current directory, and the main directory's `custom_env.sh`. Look for a clang-format with the major version number 16 in that path and its parent directory.
+
 ## Procedure
 
 ### Step 1: Auto-fix formatting
@@ -53,6 +61,7 @@ After running `clang-format.sh`, review the changes with `git diff` to verify on
 ## Excluded Directories
 
 The following are excluded from formatting (see `.clang-format-ignore`):
+
 - `be/src/apache-orc/*`, `be/src/clucene/*`, `be/src/gutil/*`
 - `be/src/glibc-compatibility/*`
 - Specific third-party vendored files (mustache, sse2neon, utf8_check)

.github/workflows/comment-to-trigger-teamcity.yml

@@ -133,6 +133,10 @@ jobs:
         reg="run (buildall|compile|p0|p1|feut|beut|cloudut|external|clickbench|cloud_p0|cloud_p1|vault_p0|nonConcurrent|performance|check_coverage)( [1-9]*[0-9]+)*"
         COMMENT_TRIGGER_TYPE="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p | sed 's/\r//g')"
         COMMENT_REPEAT_TIMES="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p | sed 's/\r//g')"
+        if [[ -n "${COMMENT_REPEAT_TIMES}" && ! "${COMMENT_REPEAT_TIMES}" =~ ^[0-9]+$ ]]; then
+          echo "COMMENT_REPEAT_TIMES '${COMMENT_REPEAT_TIMES}' is not a valid number, ignoring."
+          COMMENT_REPEAT_TIMES=""
+        fi
         echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" | tee -a "$GITHUB_OUTPUT"
         echo "COMMENT_REPEAT_TIMES=${COMMENT_REPEAT_TIMES}" | tee -a "$GITHUB_OUTPUT"
 

.github/workflows/license-eyes.yml

@@ -47,6 +47,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
 
       - name: Get changed files
         if: github.event_name == 'pull_request_target'
@@ -90,6 +91,9 @@ jobs:
           CHANGED_FILES: ${{ steps.changed-files.outputs.added_modified }}
         run: |
           python3 - <<'EOF'
+          import sys
+          # Prevent fork-supplied files from shadowing stdlib modules
+          sys.path = [p for p in sys.path if p not in ('', '.')]
           import yaml, os
 
           with open('.licenserc.yaml') as f:

.github/workflows/opencode-review-runner.yml

@@ -32,10 +32,10 @@ jobs:
         with:
           ref: ${{ inputs.head_sha }}
 
-      - name: Install ripgrep
+      - name: Install runner utilities
         run: |
           sudo apt-get update
-          sudo apt-get install -y ripgrep
+          sudo apt-get install -y ripgrep unzip
 
       - name: Install OpenCode
         run: |
@@ -50,24 +50,26 @@ jobs:
           echo "All install attempts failed"
           exit 1
 
+      - name: Install ossutil
+        run: |
+          tmp_dir="$(mktemp -d)"
+          trap 'rm -rf "$tmp_dir"' EXIT
+          curl -fsSL -o "$tmp_dir/ossutil.zip" https://gosspublic.alicdn.com/ossutil/1.7.19/ossutil-v1.7.19-linux-amd64.zip
+          unzip -q "$tmp_dir/ossutil.zip" -d "$tmp_dir"
+          sudo install -m 0755 "$tmp_dir/ossutil-v1.7.19-linux-amd64/ossutil" /usr/local/bin/ossutil
+
       - name: Configure OpenCode auth
+        id: configure-auth
+        env:
+          OSS_AK: ${{ secrets.OSS_AK }}
+          OSS_SK: ${{ secrets.OSS_SK }}
+          OSS_ENDPOINT: oss-cn-hongkong.aliyuncs.com
+          OSS_AUTH_OBJECT: oss://doris-community-ci/auth.json
         run: |
           mkdir -p ~/.local/share/opencode
-          cat > ~/.local/share/opencode/auth.json <<EOF
-          {
-            "openai": {
-              "type": "oauth",
-              "access": "${CODE_REVIEW_ZCLLL_OPENAI_ACCESS_KEY}",
-              "refresh": "${CODE_REVIEW_ZCLLL_OPENAI_REFRESH_KEY}",
-              "expires": 1779122093655,
-              "accountId": "${CODE_REVIEW_ZCLLL_OPENAI_ACCOUNT_ID}"
-            }
-          }
-          EOF
-        env:
-          CODE_REVIEW_ZCLLL_OPENAI_ACCESS_KEY: ${{ secrets.CODE_REVIEW_ZCLLL_OPENAI_ACCESS_KEY }}
-          CODE_REVIEW_ZCLLL_OPENAI_REFRESH_KEY: ${{ secrets.CODE_REVIEW_ZCLLL_OPENAI_REFRESH_KEY }}
-          CODE_REVIEW_ZCLLL_OPENAI_ACCOUNT_ID: ${{ secrets.CODE_REVIEW_ZCLLL_OPENAI_ACCOUNT_ID }}
+          ossutil -i "$OSS_AK" -k "$OSS_SK" -e "$OSS_ENDPOINT" cp -f "$OSS_AUTH_OBJECT" ~/.local/share/opencode/auth.json
+          chmod 600 ~/.local/share/opencode/auth.json
+          test -s ~/.local/share/opencode/auth.json
 
       - name: Prepare review context directory
         run: |
@@ -209,7 +211,10 @@ jobs:
           status=${PIPESTATUS[0]}
           set -e
 
-          last_log_line=$(awk 'NF { line = $0 } END { print line }' "$REVIEW_CONTEXT_DIR/opencode-review.log")
+          last_log_line=$(
+            awk 'NF { line = $0 } END { print line }' "$REVIEW_CONTEXT_DIR/opencode-review.log" \
+              | perl -pe 's/\e\[[0-9;?]*[ -\/]*[@-~]//g'
+          )
 
           failure_reason=""
           if printf '%s\n' "$last_log_line" | rg -q -i '^Error:|SSE read timed out'; then
@@ -227,6 +232,23 @@ jobs:
             exit 1
           fi
 
+      - name: Persist OpenCode auth
+        if: ${{ always() && steps.configure-auth.outcome == 'success' }}
+        env:
+          OSS_AK: ${{ secrets.OSS_AK }}
+          OSS_SK: ${{ secrets.OSS_SK }}
+          OSS_ENDPOINT: oss-cn-hongkong.aliyuncs.com
+          OSS_AUTH_OBJECT: oss://doris-community-ci/auth.json
+        run: |
+          if [ ! -s ~/.local/share/opencode/auth.json ]; then
+            echo "::warning::OpenCode auth file is missing or empty; skip OSS auth persistence."
+            exit 0
+          fi
+
+          if ! ossutil -i "$OSS_AK" -k "$OSS_SK" -e "$OSS_ENDPOINT" cp -f ~/.local/share/opencode/auth.json "$OSS_AUTH_OBJECT"; then
+            echo "::warning::Failed to persist OpenCode auth to OSS; continue because review already finished."
+          fi
+
       - name: Comment PR on review failure
         if: ${{ always() && steps.review.outcome != 'success' }}
         env:

AGENTS.md

@@ -2,6 +2,10 @@
 
 This is the codebase for Apache Doris, an MPP OLAP database. It primarily consists of the Backend module BE (`be/`, execution and storage engine), the Frontend module FE (`fe/`, optimizer and transaction core), and the Cloud module (`cloud/`, storage-compute separation). Your basic development workflow is: modify code, build using standard procedures, add and run tests, and submit relevant changes.
 
+## Security Threat Model
+
+For security scans, vulnerability triage, security reviews, and changes involving authentication, authorization, network boundaries, external catalogs, cloud tenancy, or other security-sensitive behavior, read `threat-model.md` first. Use it to determine in-scope components, trust boundaries, attacker roles, explicit non-goals, and triage classification. Findings that are out of model or by design under `threat-model.md` should be reported as such, not treated as Doris vulnerabilities.
+
 ## When running in a WORKTREE directory
 
 To ensure smooth test execution without interference between worktrees, the first thing to do upon entering a worktree directory is to check if `.worktree_initialized` exists. If not, execute `hooks/setup_worktree.sh`, setting `$ROOT_WORKSPACE_PATH` to the base directory (typically `${DORIS_REPO}`) beforehand. After successful execution, verify that `.worktree_initialized` has been touched and that `thirdparty/installed` dependencies exist correctly. Also check if submodules have been properly initialized; if not, do so manually.
@@ -12,6 +16,8 @@ When working in worktree mode, all operations must be confined to the current wo
 
 Assert correctness only—never use defensive programming with `if` or similar constructs. Any `if` check for errors must have a clearly known inevitable failure path (not speculation). If no such scenario is found, strictly avoid using `if(valid)` checks. However, you may use the `DORIS_CHECK` macro for precondition assertions (if inside performance-sensitive areas like loops, it can only be `DCHECK`). For example, if logically A=true should always imply B=true, then strictly avoid `if (A && B)` and instead use `if (A) { DORIS_CHECK(B); ... }`. In short, the principle is: upon discovering errors or unexpected situations, report errors or crash—never allow the process to continue.
 
+For `PaddedPODArray` and its peripheral packaging types, such as some certain Column, negative alignment allows the use of -1 as a valid index. No additional special handling is needed when the index may be -1.
+
 When adding code, strictly follow existing similar code in similar contexts, including interface usage, error handling, and locking patterns. When adding any code, first try to reference existing functionality. Second, examine the relevant context paragraphs to fully understand the logic.
 
 After adding code, conduct self-review and refactoring attempts to ensure good abstraction and reuse as much as possible.
@@ -35,7 +41,7 @@ When conducting code review (including self-review and review tasks), complete t
 Always use only the `build.sh` script with its correct parameters to build Doris BE and FE. For example, the simplest BE+FE build command is `./build.sh --be --fe`.
 Build type can be set via `BUILD_TYPE` in `custom_env.sh`, but only set it to `RELEASE` when explicitly required for performance testing; otherwise, keep it as `ASAN`.
 You may modify BE and FE ports and network settings in `conf/` before compilation to ensure correctness and avoid conflicts.
-Build artifacts are in the current directory's `output/`. If starting the service, ensure all process artifacts have their conf set with appropriate non-conflicting ports and `priority_networks = 10.16.10.3/24`. Use `--daemon` when starting. Cluster startup is slow; wait at least 30s for success. If still not ready after waiting, continue waiting. If not ready after a long time, check BE and FE logs to investigate.
+Build artifacts are in the current directory's `output/`. If starting the service, ensure all process artifacts have their conf set with appropriate non-conflicting ports and the correct `priority_networks` that match the current network environment, for example `priority_networks = 10.16.10.3/24`. Use `--daemon` when starting. Cluster startup is slow; wait at least 30s for success. If still not ready after waiting, continue waiting. If not ready after a long time, check BE and FE logs to investigate.
 For first-time cluster startup, you may need to manually add the backend.
 
 ## Testing Standards
@@ -47,11 +53,13 @@ You must use the preset scripts in the codebase with their correct parameters to
 Key utility functions in BE code, as well as the core logic of complete features, must have corresponding unit tests. If it is inconvenient to add unit tests, revisit the module design and function decomposition to ensure high cohesion and low coupling.
 
 Added regression tests must comply with the following standards:
+
 1. Use `order_qt` prefix or manually add `order by` to ensure ordered results
 2. For cases expected to error, use the `test{sql,exception}` pattern
 3. After completing tests, do not drop tables; instead drop tables before using them in tests, to preserve the environment for debugging
 4. For ordinary single test tables, do not use `def tableName` form; instead hardcode your table name in all SQL
 5. Except for variables you explicitly need to adjust for testing current functionality, other variables do not need extra setup before testing. For example, nereids optimizer and pipeline engine settings can use default states
+6. For determined expected results, do not using methods like `assert` in test groovy files, but instead generate the `.out` file using `qt_sql` and similar methods.
 
 ## Commit Standards
 
@@ -83,10 +91,12 @@ Problem Summary: <Describe the problem this commit addresses>
 ```
 
 Key rules for commit messages:
+
 1. The title must follow the `[type](module)` format validated by the PR title checker (`.github/workflows/title-checker.yml`). Common types include: `fix`, `feature`, `improvement`, `refactor`, `chore`, `test`, `doc`. Common modules include: `fe`, `be`, `cloud`, `regression`, `build`
 2. The short summary must be concise and written in imperative mood (e.g., `[fix](fe) Fix null pointer in scan node` not `[fix](fe) Fixed null pointer`)
 3. The `Issue Number` field must reference the corresponding GitHub Issue with `close #xxx` syntax when applicable
 4. The `Release note` section must be filled in for any user-visible behavior or feature change; write "None" for internal refactoring or test-only changes
-5. The test section must honestly reflect the testing performed; do not claim tests that were not actually run
+5. The `Problem Summary` section should cover the following content when available: problem reproduction method, root cause in code, end-to-end results/phenomena before and after repair, and the fix. If it's a refactoring, explain the reason. If it's a performance improvement, specify the case and the exact improvement amount. DO NOT mention any specific JIRA numbers. The background of the problem should be fully understandable through this section alone.
+6. The test section must honestly reflect the testing performed; do not claim tests that were not actually run
 
 Files in a git commit should only be related to the current modification task. Environment modifications for running (for example `conf/`, `AGENTS.md`, `hooks/`) must not be `git add`ed. When delivering the final task, ensure all actual code modifications have been committed.

be/benchmark/benchmark_zone_map_index.hpp

@@ -34,7 +34,6 @@
 
 #include "core/data_type/data_type_factory.hpp"
 #include "core/string_ref.h"
-#include "storage/field.h"
 #include "storage/index/zone_map/zone_map_index.h"
 #include "storage/tablet/tablet_schema.h"
 #include "util/slice.h"
@@ -116,9 +115,8 @@ std::unique_ptr<ZoneMapIndexWriter> make_writer() {
         col = make_column(FieldType::OLAP_FIELD_TYPE_VARCHAR, 64, 1);
         dtype = DataTypeFactory::instance().create_data_type(TYPE_VARCHAR, false, 0, 0, 64);
     }
-    std::unique_ptr<StorageField> field(StorageFieldFactory::create(*col));
     std::unique_ptr<ZoneMapIndexWriter> w;
-    (void)ZoneMapIndexWriter::create(dtype, field.get(), w);
+    (void)ZoneMapIndexWriter::create(dtype, col.get(), w);
     return w;
 }
 

be/src/cloud/cloud_rowset_writer.cpp

@@ -54,6 +54,8 @@ Status CloudRowsetWriter::init(const RowsetWriterContext& rowset_writer_context)
     _rowset_meta->set_rowset_id(_context.rowset_id);
     _rowset_meta->set_partition_id(_context.partition_id);
     _rowset_meta->set_tablet_id(_context.tablet_id);
+    _rowset_meta->set_db_id(_context.db_id);
+    _rowset_meta->set_table_id(_context.table_id);
     _rowset_meta->set_index_id(_context.index_id);
     _rowset_meta->set_tablet_schema_hash(_context.tablet_schema_hash);
     _rowset_meta->set_rowset_type(_context.rowset_type);

be/src/cloud/cloud_schema_change_job.cpp

@@ -573,6 +573,10 @@ Status CloudSchemaChangeJob::_convert_historical_rowsets(const SchemaChangeParam
             }
         }
         _new_tablet->add_rowsets(std::move(_output_rowsets), true, wlock, false);
+        // Ensure the real new tablet has a continuous local version graph before it becomes
+        // visible. Later RUNNING-tablet delete bitmap sync depends on capturing all old versions.
+        RETURN_IF_ERROR(_cloud_storage_engine.meta_mgr().fill_version_holes(
+                _new_tablet.get(), _new_tablet->max_version_unlocked(), wlock));
         _new_tablet->set_cumulative_layer_point(_output_cumulative_point);
         _new_tablet->reset_approximate_stats(stats.num_rowsets(), stats.num_segments(),
                                              stats.num_rows(), stats.data_size());

be/src/cloud/cloud_tablets_channel.cpp

@@ -25,6 +25,7 @@
 #include "cloud/config.h"
 #include "load/channel/tablets_channel.h"
 #include "load/delta_writer/delta_writer.h"
+#include "storage/tablet_info.h"
 
 namespace doris {
 
@@ -42,6 +43,9 @@ std::unique_ptr<BaseDeltaWriter> CloudTabletsChannel::create_delta_writer(
 
 Status CloudTabletsChannel::add_batch(const PTabletWriterAddBlockRequest& request,
                                       PTabletWriterAddBlockResult* response) {
+    if (_schema != nullptr && _schema->row_binlog_index_schema() != nullptr) {
+        return Status::NotSupported("cloud mode does not support binlog<row> now");
+    }
     // FIXME(plat1ko): Too many duplicate code with `TabletsChannel`
     SCOPED_TIMER(_add_batch_timer);
     int64_t cur_seq = 0;

be/src/cloud/pb_convert.cpp

@@ -118,11 +118,17 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, const RowsetMetaPB& in)
         out->set_job_id(in.job_id());
     }
     if (in.has_commit_tso()) {
-        out->set_commit_tso(in.commit_tso());
+        out->mutable_commit_tso()->CopyFrom(in.commit_tso());
     }
     if (in.has_is_row_binlog()) {
         out->set_is_row_binlog(in.is_row_binlog());
     }
+    if (in.has_db_id()) {
+        out->set_db_id(in.db_id());
+    }
+    if (in.has_table_id()) {
+        out->set_table_id(in.table_id());
+    }
 }
 
 void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, RowsetMetaPB&& in) {
@@ -208,11 +214,17 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, RowsetMetaPB&& in) {
         out->set_job_id(in.job_id());
     }
     if (in.has_commit_tso()) {
-        out->set_commit_tso(in.commit_tso());
+        out->mutable_commit_tso()->CopyFrom(in.commit_tso());
     }
     if (in.has_is_row_binlog()) {
         out->set_is_row_binlog(in.is_row_binlog());
     }
+    if (in.has_db_id()) {
+        out->set_db_id(in.db_id());
+    }
+    if (in.has_table_id()) {
+        out->set_table_id(in.table_id());
+    }
 }
 
 RowsetMetaPB cloud_rowset_meta_to_doris(const RowsetMetaCloudPB& in) {
@@ -308,11 +320,17 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, const RowsetMetaCloudPB& in)
         out->set_job_id(in.job_id());
     }
     if (in.has_commit_tso()) {
-        out->set_commit_tso(in.commit_tso());
+        out->mutable_commit_tso()->CopyFrom(in.commit_tso());
     }
     if (in.has_is_row_binlog()) {
         out->set_is_row_binlog(in.is_row_binlog());
     }
+    if (in.has_db_id()) {
+        out->set_db_id(in.db_id());
+    }
+    if (in.has_table_id()) {
+        out->set_table_id(in.table_id());
+    }
 }
 
 void cloud_rowset_meta_to_doris(RowsetMetaPB* out, RowsetMetaCloudPB&& in) {
@@ -397,11 +415,17 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, RowsetMetaCloudPB&& in) {
         out->set_job_id(in.job_id());
     }
     if (in.has_commit_tso()) {
-        out->set_commit_tso(in.commit_tso());
+        out->mutable_commit_tso()->CopyFrom(in.commit_tso());
     }
     if (in.has_is_row_binlog()) {
         out->set_is_row_binlog(in.is_row_binlog());
     }
+    if (in.has_db_id()) {
+        out->set_db_id(in.db_id());
+    }
+    if (in.has_table_id()) {
+        out->set_table_id(in.table_id());
+    }
 }
 
 TabletSchemaCloudPB doris_tablet_schema_to_cloud(const TabletSchemaPB& in) {