Skip to content

Commit 3f9419a

Browse files
committed
Added cut off for file drops to avoid large listing.
When there are 1000+ files in the remote but only <10 files are selected, a full listing is not necessary.
1 parent 81c7dcf commit 3f9419a

1 file changed

Lines changed: 34 additions & 7 deletions

File tree

backend/source/backend/session.cpp

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1641,13 +1641,40 @@ void Session::registerRpcSftpExistsBatch()
16411641
Ids::makeChannelId(channelIdString),
16421642
[paths](RpcHelper::RpcOnce&& reply, auto&& channel)
16431643
{
1644-
// Group destinations by parent directory and list each parent once
1645-
// (readdir) rather than issuing one stat round-trip per path. A bulk
1646-
// drop targets a single directory, so this collapses N sequential
1647-
// SFTP round-trips into a single listing — a shallow existence diff,
1648-
// like sync does. A listing failure (e.g. the parent doesn't exist
1649-
// yet) degrades to "nothing exists", matching the prior per-stat
1650-
// fallback.
1644+
// Small drops: a handful of stat round-trips is cheaper than reading
1645+
// a potentially huge target directory in full. Above the threshold the
1646+
// single readdir below wins (one round-trip instead of N).
1647+
constexpr std::size_t listingThreshold = 10;
1648+
if (paths.size() < listingThreshold)
1649+
{
1650+
std::vector<bool> results;
1651+
results.reserve(paths.size());
1652+
for (auto const& path : paths)
1653+
{
1654+
auto fut = channel->stat(Utility::pathFromUtf8(path));
1655+
if (fut.wait_for(futureTimeout) != std::future_status::ready)
1656+
{
1657+
Log::warn(
1658+
"sftp::existsBatch: stat timeout for '{}' (treating as not-exists)", path
1659+
);
1660+
results.push_back(false);
1661+
continue;
1662+
}
1663+
const auto result = fut.get();
1664+
results.push_back(result.has_value());
1665+
}
1666+
Log::info("sftp::existsBatch: probed {} paths via stat", paths.size());
1667+
reply({{"success", true}, {"exists", results}});
1668+
return;
1669+
}
1670+
1671+
// Larger drops: group destinations by parent directory and list each
1672+
// parent once (readdir) rather than issuing one stat round-trip per
1673+
// path. A bulk drop targets a single directory, so this collapses N
1674+
// sequential SFTP round-trips into a single listing — a shallow
1675+
// existence diff, like sync does. A listing failure (e.g. the parent
1676+
// doesn't exist yet) degrades to "nothing exists", matching the
1677+
// per-stat fallback.
16511678
std::unordered_map<std::string, std::optional<std::unordered_set<std::string>>> listingByParent;
16521679

16531680
auto listParent = [&channel](

0 commit comments

Comments
 (0)