remove config::load_reader_max_block_bytes

Author: mrhhsg

be/src/common/config.cpp

@@ -1729,10 +1729,6 @@ DEFINE_mBool(enable_auto_clone_on_mow_publish_missing_version, "false");
 // The maximum csv line reader output buffer size
 DEFINE_mInt64(max_csv_line_reader_output_buffer_size, "4294967296");
 
-// The maximum bytes of a single block returned by load file readers (CsvReader, NewJsonReader,
-// ParquetReader, OrcReader). Default is 64MB. Set to 0 to disable the limit.
-DEFINE_mInt64(load_reader_max_block_bytes, "67108864");
-
 // Maximum number of OpenMP threads allowed for concurrent vector index builds.
 // -1 means auto: use 80% of the available CPU cores.
 DEFINE_Int32(omp_threads_limit, "-1");

be/src/common/config.h

@@ -1790,10 +1790,6 @@ DECLARE_String(fuzzy_test_type);
 // The maximum csv line reader output buffer size
 DECLARE_mInt64(max_csv_line_reader_output_buffer_size);
 
-// The maximum bytes of a single block returned by load file readers (CsvReader, NewJsonReader,
-// ParquetReader, OrcReader). Default is 200MB. Set to 0 to disable the limit.
-DECLARE_mInt64(load_reader_max_block_bytes);
-
 // Maximum number of OpenMP threads available for concurrent index builds.
 // -1 means auto: use 80% of detected CPU cores.
 DECLARE_Int32(omp_threads_limit);

be/src/format/orc/vorc_reader.cpp

@@ -2451,14 +2451,6 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
         return Status::OK();
     }
 
-    // Limit memory per batch for load paths: pre-shrink _batch_size using the bytes-per-row
-    // estimate from the previous batch so the current batch stays within load_reader_max_block_bytes
-    // (effective from call 2 onward; first batch is capped on the next call).
-    const int64_t max_block_bytes =
-            (_state != nullptr && _state->query_type() == TQueryType::LOAD &&
-             config::load_reader_max_block_bytes > 0)
-                    ? config::load_reader_max_block_bytes
-                    : 0;
     if (_push_down_agg_type == TPushAggOp::type::COUNT) {
         auto rows = std::min(get_remaining_rows(), (int64_t)_batch_size);
 
@@ -2475,15 +2467,6 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
         return Status::OK();
     }
 
-    if (max_block_bytes > 0 && _load_bytes_per_row > 0 && _row_reader) {
-        size_t new_batch_size = std::max(
-                (size_t)1, (size_t)((int64_t)max_block_bytes / (int64_t)_load_bytes_per_row));
-        if (new_batch_size != _batch_size) {
-            _batch_size = new_batch_size;
-            _batch = _row_reader->createRowBatch(_batch_size);
-        }
-    }
-
     if (!_seek_to_read_one_line()) {
         *eof = true;
         return Status::OK();
@@ -2826,9 +2809,6 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
         *read_rows = block->rows();
     }
 
-    if (max_block_bytes > 0 && *read_rows > 0) {
-        _load_bytes_per_row = block->bytes() / *read_rows;
-    }
     return Status::OK();
 }
 

be/src/format/orc/vorc_reader.h

@@ -694,10 +694,7 @@ class OrcReader : public GenericReader {
     io::FileSystemProperties _system_properties;
     io::FileDescription _file_description;
     size_t _batch_size;
-    // Bytes-per-row estimate from the previous batch, used to pre-shrink _batch_size
-    // before reading so that oversized blocks are prevented from the current call onward.
-    // Zero means no prior data (first batch).
-    size_t _load_bytes_per_row = 0;
+
     int64_t _range_start_offset;
     int64_t _range_size;
     std::string _ctz;

be/src/format/parquet/vparquet_reader.cpp

@@ -727,19 +727,6 @@ Status ParquetReader::get_next_block(Block* block, size_t* read_rows, bool* eof)
         return Status::OK();
     }
 
-    // Limit memory per batch for load paths.
-    // _load_bytes_per_row is updated after each batch so the *next* call pre-shrinks _batch_size
-    // before reading, ensuring the current batch is already within the limit (from call 2 onward).
-    const int64_t max_block_bytes =
-            (_state != nullptr && _state->query_type() == TQueryType::LOAD &&
-             config::load_reader_max_block_bytes > 0)
-                    ? config::load_reader_max_block_bytes
-                    : 0;
-    if (max_block_bytes > 0 && _load_bytes_per_row > 0) {
-        _batch_size = std::max((size_t)1,
-                               (size_t)((int64_t)max_block_bytes / (int64_t)_load_bytes_per_row));
-    }
-
     SCOPED_RAW_TIMER(&_reader_statistics.column_read_time);
     Status batch_st =
             _current_group_reader->next_batch(block, _batch_size, read_rows, &_row_group_eof);
@@ -756,10 +743,6 @@ Status ParquetReader::get_next_block(Block* block, size_t* read_rows, bool* eof)
                                      batch_st.to_string());
     }
 
-    if (max_block_bytes > 0 && *read_rows > 0) {
-        _load_bytes_per_row = block->bytes() / *read_rows;
-    }
-
     if (_row_group_eof) {
         auto column_st = _current_group_reader->merged_column_statistics();
         _column_statistics.merge(column_st);

be/src/format/parquet/vparquet_reader.h

@@ -343,10 +343,7 @@ class ParquetReader : public GenericReader {
 
     // parquet file reader object
     size_t _batch_size;
-    // Bytes-per-row estimate from the previous batch, used to pre-shrink _batch_size
-    // before reading so that oversized blocks are prevented from the current call onward.
-    // Zero means no prior data (first batch).
-    size_t _load_bytes_per_row = 0;
+
     int64_t _range_start_offset;
     int64_t _range_size;
     const cctz::time_zone* _ctz = nullptr;