Skip to content

Commit aa88a9c

Browse files
gavinchougavinchou
authored
[chore](compile) Fix some implicit conversion issues (apache#53331)
be/src/cloud/cloud_internal_service.cpp be/src/cloud/pb_convert.cpp be/src/io/file_factory.cpp be/src/io/fs/hdfs_file_reader.cpp be/src/io/fs/hdfs_file_writer.cpp be/src/io/fs/local_file_writer.cpp be/src/io/fs/s3_file_writer.cpp be/src/io/fs/s3_file_writer.h
1 parent a3da6b6 commit aa88a9c

8 files changed

Lines changed: 50 additions & 28 deletions

‎be/src/cloud/cloud_internal_service.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "io/cache/block_file_cache_factory.h"
2424

2525
namespace doris {
26+
#include "common/compile_check_begin.h"
2627

2728
CloudInternalServiceImpl::CloudInternalServiceImpl(CloudStorageEngine& engine, ExecEnv* exec_env)
2829
: PInternalService(exec_env), _engine(engine) {}
@@ -83,7 +84,7 @@ void CloudInternalServiceImpl::get_file_cache_meta_by_tablet_id(
8384
auto rowsets = tablet->get_snapshot_rowset();
8485
std::for_each(rowsets.cbegin(), rowsets.cend(), [&](const RowsetSharedPtr& rowset) {
8586
std::string rowset_id = rowset->rowset_id().to_string();
86-
for (int64_t segment_id = 0; segment_id < rowset->num_segments(); segment_id++) {
87+
for (int32_t segment_id = 0; segment_id < rowset->num_segments(); segment_id++) {
8788
std::string file_name = fmt::format("{}_{}.dat", rowset_id, segment_id);
8889
auto cache_key = io::BlockFileCache::hash(file_name);
8990
auto* cache = io::FileCacheFactory::instance()->get_by_path(cache_key);
@@ -107,5 +108,6 @@ void CloudInternalServiceImpl::get_file_cache_meta_by_tablet_id(
107108
VLOG_DEBUG << "warm up get meta request=" << request->DebugString()
108109
<< ", response=" << response->DebugString();
109110
}
111+
#include "common/compile_check_end.h"
110112

111113
} // namespace doris

‎be/src/cloud/pb_convert.cpp‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include <utility>
2424

2525
namespace doris::cloud {
26+
#include "common/compile_check_begin.h"
2627

2728
RowsetMetaCloudPB doris_rowset_meta_to_cloud(const RowsetMetaPB& in) {
2829
RowsetMetaCloudPB out;
@@ -546,17 +547,17 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) {
546547
doris_tablet_schema_to_cloud(out->mutable_schema(), std::move(*in.mutable_schema()));
547548
}
548549
if (in.rs_metas_size()) {
549-
size_t rs_metas_size = in.rs_metas_size();
550+
int32_t rs_metas_size = in.rs_metas_size();
550551
out->mutable_rs_metas()->Reserve(rs_metas_size);
551-
for (size_t i = 0; i < rs_metas_size; ++i) {
552+
for (int i = 0; i < rs_metas_size; ++i) {
552553
doris_rowset_meta_to_cloud(out->add_rs_metas(), std::move(*in.mutable_rs_metas(i)));
553554
}
554555
}
555556
// ATTN: inc_rs_metas are deprecated, here is for conversion check
556557
if (in.inc_rs_metas_size()) {
557-
size_t rs_metas_size = in.inc_rs_metas_size();
558+
int rs_metas_size = in.inc_rs_metas_size();
558559
out->mutable_inc_rs_metas()->Reserve(rs_metas_size);
559-
for (size_t i = 0; i < rs_metas_size; ++i) {
560+
for (int i = 0; i < rs_metas_size; ++i) {
560561
doris_rowset_meta_to_cloud(out->add_inc_rs_metas(),
561562
std::move(*in.mutable_inc_rs_metas(i)));
562563
}
@@ -570,9 +571,9 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) {
570571
out->set_preferred_rowset_type(in.preferred_rowset_type());
571572
out->set_tablet_type(in.tablet_type());
572573
if (in.stale_rs_metas_size()) {
573-
size_t rs_metas_size = in.stale_rs_metas_size();
574+
int rs_metas_size = in.stale_rs_metas_size();
574575
out->mutable_stale_rs_metas()->Reserve(rs_metas_size);
575-
for (size_t i = 0; i < rs_metas_size; i++) {
576+
for (int i = 0; i < rs_metas_size; i++) {
576577
doris_rowset_meta_to_cloud(out->add_stale_rs_metas(),
577578
std::move(*in.mutable_stale_rs_metas(i)));
578579
}
@@ -701,17 +702,17 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) {
701702
cloud_tablet_schema_to_doris(out->mutable_schema(), std::move(*in.mutable_schema()));
702703
}
703704
if (in.rs_metas_size()) {
704-
size_t rs_metas_size = in.rs_metas_size();
705+
int rs_metas_size = in.rs_metas_size();
705706
out->mutable_rs_metas()->Reserve(rs_metas_size);
706-
for (size_t i = 0; i < rs_metas_size; ++i) {
707+
for (int i = 0; i < rs_metas_size; ++i) {
707708
cloud_rowset_meta_to_doris(out->add_rs_metas(), std::move(*in.mutable_rs_metas(i)));
708709
}
709710
}
710711
// ATTN: inc_rs_metas are deprecated, here is for conversion check
711712
if (in.inc_rs_metas_size()) {
712-
size_t rs_metas_size = in.inc_rs_metas_size();
713+
int rs_metas_size = in.inc_rs_metas_size();
713714
out->mutable_inc_rs_metas()->Reserve(rs_metas_size);
714-
for (size_t i = 0; i < rs_metas_size; i++) {
715+
for (int i = 0; i < rs_metas_size; i++) {
715716
cloud_rowset_meta_to_doris(out->add_inc_rs_metas(),
716717
std::move(*in.mutable_inc_rs_metas(i)));
717718
}
@@ -725,9 +726,9 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) {
725726
out->set_preferred_rowset_type(in.preferred_rowset_type());
726727
out->set_tablet_type(in.tablet_type());
727728
if (in.stale_rs_metas_size()) {
728-
size_t rs_metas_size = in.stale_rs_metas_size();
729+
int rs_metas_size = in.stale_rs_metas_size();
729730
out->mutable_stale_rs_metas()->Reserve(rs_metas_size);
730-
for (size_t i = 0; i < rs_metas_size; i++) {
731+
for (int i = 0; i < rs_metas_size; i++) {
731732
cloud_rowset_meta_to_doris(out->add_stale_rs_metas(),
732733
std::move(*in.mutable_stale_rs_metas(i)));
733734
}
@@ -760,5 +761,6 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) {
760761
out->set_schema_version(in.schema_version());
761762
}
762763
}
764+
#include "common/compile_check_end.h"
763765

764766
} // namespace doris::cloud

‎be/src/io/file_factory.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include <mutex>
2525
#include <utility>
2626

27+
#include "common/cast_set.h"
2728
#include "common/config.h"
2829
#include "common/status.h"
2930
#include "io/fs/broker_file_system.h"
@@ -51,6 +52,7 @@
5152
#include "util/uid_util.h"
5253

5354
namespace doris {
55+
#include "common/compile_check_begin.h"
5456

5557
constexpr std::string_view RANDOM_CACHE_BASE_PATH = "random";
5658

@@ -87,7 +89,7 @@ int32_t get_broker_index(const std::vector<TNetworkAddress>& brokers, const std:
8789
}
8890

8991
// secondly select broker by hash of file path
90-
auto key = HashUtil::hash(path.data(), path.size(), 0);
92+
auto key = HashUtil::hash(path.data(), cast_set<int32_t>(path.size()), 0);
9193
return key % brokers.size();
9294
}
9395

@@ -274,5 +276,6 @@ Status FileFactory::create_pipe_reader(const TUniqueId& load_id, io::FileReaderS
274276

275277
return Status::OK();
276278
}
279+
#include "common/compile_check_end.h"
277280

278281
} // namespace doris

‎be/src/io/fs/hdfs_file_reader.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
#include "util/doris_metrics.h"
3838

3939
namespace doris::io {
40+
#include "common/compile_check_begin.h"
4041

4142
bvar::Adder<uint64_t> hdfs_bytes_read_total("hdfs_file_reader", "bytes_read");
4243
bvar::LatencyRecorder hdfs_bytes_per_read("hdfs_file_reader", "bytes_per_read"); // also QPS
@@ -138,8 +139,11 @@ Status HdfsFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_r
138139

139140
size_t has_read = 0;
140141
while (has_read < bytes_req) {
142+
int64_t max_to_read = bytes_req - has_read;
143+
tSize to_read = static_cast<tSize>(
144+
std::min(max_to_read, static_cast<int64_t>(std::numeric_limits<tSize>::max())));
141145
tSize loop_read = hdfsPread(_handle->fs(), _handle->file(), offset + has_read,
142-
to + has_read, bytes_req - has_read);
146+
to + has_read, to_read);
143147
{
144148
[[maybe_unused]] Status error_ret;
145149
TEST_INJECTION_POINT_RETURN_WITH_VALUE("HdfsFileReader:read_error", error_ret);
@@ -266,5 +270,6 @@ void HdfsFileReader::_collect_profile_before_close() {
266270
#endif
267271
}
268272
}
273+
#include "common/compile_check_end.h"
269274

270275
} // namespace doris::io

‎be/src/io/fs/hdfs_file_writer.cpp‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
#include "util/jni-util.h"
4646

4747
namespace doris::io {
48+
#include "common/compile_check_begin.h"
4849

4950
bvar::Adder<uint64_t> hdfs_file_writer_total("hdfs_file_writer_total_num");
5051
bvar::Adder<uint64_t> hdfs_bytes_written_total("hdfs_file_writer_bytes_written");
@@ -78,7 +79,7 @@ class HdfsWriteMemUsageRecorder {
7879
HdfsWriteMemUsageRecorder() = default;
7980
~HdfsWriteMemUsageRecorder() = default;
8081
size_t max_usage() const {
81-
return static_cast<size_t>(max_jvm_heap_size() *
82+
return static_cast<size_t>(static_cast<double>(max_jvm_heap_size()) *
8283
config::max_hdfs_wirter_jni_heap_usage_ratio);
8384
}
8485
Status acquire_memory(size_t memory_size, int try_time) {
@@ -90,8 +91,8 @@ class HdfsWriteMemUsageRecorder {
9091
}
9192
auto unit = config::hdfs_jni_write_sleep_milliseconds;
9293
std::default_random_engine rng = make_random_engine();
93-
std::uniform_int_distribution<uint32_t> u(unit, 2 * unit);
94-
std::uniform_int_distribution<uint32_t> u2(2 * unit, 4 * unit);
94+
std::uniform_int_distribution<int64_t> u(unit, 2 * unit);
95+
std::uniform_int_distribution<int64_t> u2(2 * unit, 4 * unit);
9596
auto duration_ms =
9697
try_time < (config::hdfs_jni_write_max_retry_time / 2) ? u(rng) : u2(rng);
9798
std::unique_lock lck {cur_memory_latch};
@@ -364,8 +365,11 @@ Status HdfsFileWriter::append_hdfs_file(std::string_view content) {
364365
{
365366
TEST_INJECTION_POINT_CALLBACK("HdfsFileWriter::append_hdfs_file_delay");
366367
SCOPED_BVAR_LATENCY(hdfs_bvar::hdfs_write_latency);
368+
int64_t max_to_write = content.size();
369+
tSize to_write = static_cast<tSize>(std::min(
370+
max_to_write, static_cast<int64_t>(std::numeric_limits<tSize>::max())));
367371
written_bytes = SYNC_POINT_HOOK_RETURN_VALUE(
368-
hdfsWrite(_hdfs_handler->hdfs_fs, _hdfs_file, content.data(), content.size()),
372+
hdfsWrite(_hdfs_handler->hdfs_fs, _hdfs_file, content.data(), to_write),
369373
"HdfsFileWriter::append_hdfs_file::hdfsWrite", content);
370374
{
371375
TEST_INJECTION_POINT_RETURN_WITH_VALUE(
@@ -470,5 +474,6 @@ Result<FileWriterPtr> HdfsFileWriter::create(Path full_path, std::shared_ptr<Hdf
470474
inflight_hdfs_file_writer << 1;
471475
return std::make_unique<HdfsFileWriter>(std::move(path), handler, hdfs_file, fs_name, opts);
472476
}
477+
#include "common/compile_check_end.h"
473478

474479
} // namespace doris::io

‎be/src/io/fs/local_file_writer.cpp‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
#include <utility>
3131

3232
// IWYU pragma: no_include <opentelemetry/common/threadlocal.h>
33+
#include "common/cast_set.h"
3334
#include "common/compiler_util.h" // IWYU pragma: keep
3435
#include "common/macros.h"
3536
#include "common/status.h"
@@ -43,6 +44,7 @@
4344
#include "util/doris_metrics.h"
4445

4546
namespace doris::io {
47+
#include "common/compile_check_begin.h"
4648
namespace {
4749

4850
Status sync_dir(const io::Path& dirname) {
@@ -143,9 +145,9 @@ Status LocalFileWriter::appendv(const Slice* data, size_t data_cnt) {
143145
// Never request more than IOV_MAX in one request.
144146
size_t iov_count = std::min(data_cnt - completed_iov, static_cast<size_t>(IOV_MAX));
145147
ssize_t res;
146-
RETRY_ON_EINTR(res, SYNC_POINT_HOOK_RETURN_VALUE(
147-
::writev(_fd, iov.data() + completed_iov, iov_count),
148-
"LocalFileWriter::writev", _fd));
148+
RETRY_ON_EINTR(res, SYNC_POINT_HOOK_RETURN_VALUE(::writev(_fd, iov.data() + completed_iov,
149+
cast_set<int32_t>(iov_count)),
150+
"LocalFileWriter::writev", _fd));
149151
DBUG_EXECUTE_IF("LocalFileWriter::appendv.io_error", {
150152
auto sub_path = dp->param<std::string>("sub_path", "");
151153
if ((sub_path.empty() && _path.filename().compare(kTestFilePath)) ||
@@ -243,5 +245,6 @@ Status LocalFileWriter::_close(bool sync) {
243245
TEST_SYNC_POINT_RETURN_WITH_VALUE("LocalFileWriter::close", Status::IOError("inject io error"));
244246
return fd_reclaim_func(Status::OK());
245247
}
248+
#include "common/compile_check_end.h"
246249

247250
} // namespace doris::io

‎be/src/io/fs/s3_file_writer.cpp‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
#include "util/s3_util.h"
4343

4444
namespace doris::io {
45+
#include "common/compile_check_begin.h"
4546

4647
bvar::Adder<uint64_t> s3_file_writer_total("s3_file_writer_total_num");
4748
bvar::Adder<uint64_t> s3_bytes_written_total("s3_file_writer_bytes_written");
@@ -282,7 +283,7 @@ Status S3FileWriter::appendv(const Slice* data, size_t data_cnt) {
282283
return Status::OK();
283284
}
284285

285-
void S3FileWriter::_upload_one_part(int64_t part_num, UploadFileBuffer& buf) {
286+
void S3FileWriter::_upload_one_part(int part_num, UploadFileBuffer& buf) {
286287
VLOG_DEBUG << "upload_one_part " << _obj_storage_path_opts.path.native()
287288
<< " part=" << part_num;
288289
if (buf.is_cancelled()) {
@@ -386,14 +387,14 @@ Status S3FileWriter::_complete() {
386387
}
387388

388389
// check number of parts
389-
int expected_num_parts1 = (_bytes_appended / config::s3_write_buffer_size) +
390-
!!(_bytes_appended % config::s3_write_buffer_size);
391-
int expected_num_parts2 =
390+
int64_t expected_num_parts1 = (_bytes_appended / config::s3_write_buffer_size) +
391+
!!(_bytes_appended % config::s3_write_buffer_size);
392+
int64_t expected_num_parts2 =
392393
(_bytes_appended % config::s3_write_buffer_size) ? _cur_part_num : _cur_part_num - 1;
393394
DCHECK_EQ(expected_num_parts1, expected_num_parts2)
394395
<< " bytes_appended=" << _bytes_appended << " cur_part_num=" << _cur_part_num
395396
<< " s3_write_buffer_size=" << config::s3_write_buffer_size;
396-
if (_failed || _completed_parts.size() != expected_num_parts1 ||
397+
if (_failed || _completed_parts.size() != static_cast<size_t>(expected_num_parts1) ||
397398
expected_num_parts1 != expected_num_parts2) {
398399
_st = Status::InternalError(
399400
"failed to complete multipart upload, error status={} failed={} #complete_parts={} "
@@ -486,5 +487,6 @@ std::string S3FileWriter::_dump_completed_part() const {
486487
}
487488
return ss.str();
488489
}
490+
#include "common/compile_check_end.h"
489491

490492
} // namespace doris::io

‎be/src/io/fs/s3_file_writer.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ class S3FileWriter final : public FileWriter {
8383
Status _create_multi_upload_request();
8484
Status _set_upload_to_remote_less_than_buffer_size();
8585
void _put_object(UploadFileBuffer& buf);
86-
void _upload_one_part(int64_t part_num, UploadFileBuffer& buf);
86+
void _upload_one_part(int part_num, UploadFileBuffer& buf);
8787
bool _complete_part_task_callback(Status s);
8888
Status _build_upload_buffer();
8989

0 commit comments

Comments
 (0)