Skip to content

refactor: improve build dependency pull #1668

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dengbo11 merged 1 commit into from
Jun 4, 2026
Merged

refactor: improve build dependency pull #1668

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
171 changes: 92 additions & 79 deletions libs/linglong/src/linglong/builder/linglong_builder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,13 +106,16 @@
return LINGLONG_OK;
}

utils::error::Result<void> pullDependency(const package::Reference &ref,
repo::OSTreeRepo &repo,
const std::string &module) noexcept
} // namespace

namespace detail {

utils::error::Result<void> pullResolvedRef(const package::Reference &ref,
repo::OSTreeRepo &repo,
const std::string &module) noexcept
{
LINGLONG_TRACE("pull " + ref.toString());

// 如果依赖已存在，则直接使用
if (repo.getLayerDir(ref, module)) {
return LINGLONG_OK;
}
Expand Down Expand Up @@ -150,9 +153,69 @@
return LINGLONG_OK;
}

} // namespace
utils::error::Result<package::Reference> pullDependency(const std::string &fuzzyRefStr,
repo::OSTreeRepo &repo,
const std::string &module,
bool useRemote) noexcept
{
LINGLONG_TRACE("pull dependency " + fuzzyRefStr);

auto fuzzyRef = package::FuzzyReference::parse(fuzzyRefStr);
if (!fuzzyRef) {
return LINGLONG_ERR("invalid ref " + fuzzyRefStr, fuzzyRef);
}

std::optional<package::Reference> localRef;
auto localResult = repo.clearReference(
*fuzzyRef,
{ .forceRemote = false, .fallbackToRemote = false, .semanticMatching = true });
if (localResult) {
localRef = std::move(localResult).value();
}

if (!useRemote) {

Check warning on line 176 in libs/linglong/src/linglong/builder/linglong_builder.cpp

View workflow job for this annotation

GitHub Actions / cppcheck 

Consider using std::all_of or std::none_of algorithm instead of a raw loop.
if (localRef) {
return std::move(*localRef);
}
return LINGLONG_ERR(fmt::format("failed to get local ref {}", fuzzyRef->toString()),
localResult);

Check warning on line 181 in libs/linglong/src/linglong/builder/linglong_builder.cpp

View workflow job for this annotation

GitHub Actions / cppcheck 

Consider using std::any_of algorithm instead of a raw loop.
}
Comment thread
reddevillg marked this conversation as resolved.

std::optional<package::Reference> remoteRef;
auto remoteResult = repo.clearReference(
*fuzzyRef,
{ .forceRemote = true, .fallbackToRemote = true, .semanticMatching = true });
if (remoteResult) {
remoteRef = std::move(remoteResult).value();
}

bool preferRemote = remoteRef && (!localRef || remoteRef->version > localRef->version);
auto &bestRef = preferRemote ? remoteRef : localRef;

if (!bestRef) {
return LINGLONG_ERR(fmt::format("ref doesn't exist {}", fuzzyRef->toString()));
}
Comment thread
reddevillg marked this conversation as resolved.

auto pullRes = pullResolvedRef(*bestRef, repo, module);
if (pullRes) {
return std::move(*bestRef);
}

if (preferRemote && localRef) {
LogW("failed to pull remote version {}, falling back to local: {}",
bestRef->toString(),
pullRes.error().message());
auto fallbackRes = pullResolvedRef(*localRef, repo, module);
if (fallbackRes) {
return std::move(*localRef);
}
return LINGLONG_ERR("failed to pull local fallback version " + localRef->toString(),
fallbackRes);
}

return LINGLONG_ERR("failed to pull version " + bestRef->toString(), pullRes);
}

namespace detail {
void mergeOutput(const std::vector<std::filesystem::path> &src,
const std::filesystem::path &dest,
const std::vector<std::string> &targets,
Expand Down Expand Up @@ -432,27 +495,9 @@
return LINGLONG_ERR(fuzzyRef);
}

// always try to get newest version from remote
auto ref = repo.clearReference(
*fuzzyRef,
{ .forceRemote = true, .fallbackToRemote = true, .semanticMatching = true });
auto localRef = repo.clearReference(
*fuzzyRef,
{ .forceRemote = false, .fallbackToRemote = false, .semanticMatching = true });
if (localRef) {
if (!ref || localRef->version > ref->version) {
ref = std::move(localRef);
LogD("use local tools {}", ref->toString());
}
}

auto ref = detail::pullDependency(fuzzyRef->toString(), this->repo, "binary", true);
if (!ref) {
return LINGLONG_ERR("failed to find utils " + id, ref);
}

auto res = pullDependency(*ref, this->repo, "binary");
if (!res) {
return LINGLONG_ERR("failed to get utils " + id, res);
return LINGLONG_ERR("failed to get utils " + id, ref);
}

auto layerItem = this->repo.getLayerItem(*ref);
Expand All @@ -464,49 +509,21 @@
// assumes these dependencies are available for the current architecture,
// this requires the same version of `build-utils` to be built for both
// the target and the current architectures.
auto baseRef = clearDependency(info.base, false, true);
auto baseRef = detail::pullDependency(info.base, this->repo, "binary", true);
if (!baseRef) {
return LINGLONG_ERR("base not exist: " + info.base);
}
if (!pullDependency(*baseRef, this->repo, "binary")) {
return LINGLONG_ERR("failed to pull base binary " + info.base);
return LINGLONG_ERR("base not exist: " + info.base, baseRef);
}

if (info.runtime) {
auto runtimeRef = clearDependency(info.runtime.value(), false, true);
auto runtimeRef = detail::pullDependency(info.runtime.value(), this->repo, "binary", true);
if (!runtimeRef) {
return LINGLONG_ERR("runtime not exist: " + info.runtime.value());
}
if (!pullDependency(*runtimeRef, this->repo, "binary")) {
return LINGLONG_ERR("failed to pull runtime binary " + info.runtime.value());
return LINGLONG_ERR("runtime not exist: " + info.runtime.value(), runtimeRef);
}
}

return ref;
}

utils::error::Result<package::Reference> Builder::clearDependency(const std::string &ref,
bool forceRemote,
bool fallbackToRemote) noexcept
{
LINGLONG_TRACE("clear dependency");

auto fuzzyRef = package::FuzzyReference::parse(ref);
if (!fuzzyRef) {
return LINGLONG_ERR("invalid ref " + ref);
}

auto res = repo.clearReference(*fuzzyRef,
{ .forceRemote = forceRemote,
.fallbackToRemote = fallbackToRemote,
.semanticMatching = true });
if (!res) {
return LINGLONG_ERR(fmt::format("ref doesn't exist {}", fuzzyRef->toString()));
}

return res;
}

utils::error::Result<void> Builder::buildStagePullDependency() noexcept
{
LINGLONG_TRACE("build stage pull dependency");
Expand All @@ -520,36 +537,36 @@
.toStdString(),
2);

auto baseRef = clearDependency(this->project->base, !this->buildOptions.skipPullDepend, false);
auto baseRef = detail::pullDependency(this->project->base,
this->repo,
"binary",
!this->buildOptions.skipPullDepend);
if (!baseRef) {
return LINGLONG_ERR("base dependency error", baseRef);
}

std::optional<package::Reference> runtimeRef;
if (this->project->runtime) {
auto ref =
clearDependency(*this->project->runtime, !this->buildOptions.skipPullDepend, false);
auto ref = detail::pullDependency(*this->project->runtime,
this->repo,
"binary",
!this->buildOptions.skipPullDepend);
if (!ref) {
return LINGLONG_ERR("runtime dependency error", ref);
}
runtimeRef = std::move(ref).value();
}

if (!this->buildOptions.skipPullDepend) {
auto ref = pullDependency(*baseRef, this->repo, "binary");
if (!ref.has_value()) {
return LINGLONG_ERR("failed to pull base binary " + baseRef->toString(), ref);
}

printReplacedText(fmt::format("{:<35}{:<15}{:<15}complete\n",
baseRef->id,
baseRef->version.toString(),
"binary"),
2);

ref = pullDependency(*baseRef, this->repo, "develop");
if (!ref.has_value()) {
return LINGLONG_ERR("failed to pull base develop " + baseRef->toString(), ref);
auto res = detail::pullResolvedRef(*baseRef, this->repo, "develop");
if (!res) {
return LINGLONG_ERR("failed to pull base develop " + baseRef->toString(), res);
}

printReplacedText(fmt::format("{:<35}{:<15}{:<15}complete\n",
Expand All @@ -559,20 +576,15 @@
2);

if (runtimeRef) {
ref = pullDependency(*runtimeRef, this->repo, "binary");
if (!ref.has_value()) {
return LINGLONG_ERR("failed to pull runtime binary " + runtimeRef->toString(), ref);
}

printReplacedText(fmt::format("{:<35}{:<15}{:<15}complete\n",
runtimeRef->id,
runtimeRef->version.toString(),
"binary"),
2);
ref = pullDependency(*runtimeRef, this->repo, "develop");
if (!ref.has_value()) {
res = detail::pullResolvedRef(*runtimeRef, this->repo, "develop");
if (!res) {
return LINGLONG_ERR("failed to pull runtime develop " + runtimeRef->toString(),
ref);
res);
}

printReplacedText(fmt::format("{:<35}{:<15}{:<15}complete\n",
Expand Down Expand Up @@ -1558,9 +1570,9 @@
if (!ret) {
return LINGLONG_ERR(ret);
}
}

Check warning on line 1573 in libs/linglong/src/linglong/builder/linglong_builder.cpp

View workflow job for this annotation

GitHub Actions / cppcheck 

Local variable 'ret' shadows outer variable

Check warning on line 1573 in libs/linglong/src/linglong/builder/linglong_builder.cpp

View workflow job for this annotation

GitHub Actions / cppcheck 

Local variable 'ret' shadows outer variable

auto baseRef = clearDependency(this->project->base, false, false);
auto baseRef = detail::pullDependency(this->project->base, this->repo, "binary", false);
if (!baseRef) {
return LINGLONG_ERR(baseRef);
}
Expand All @@ -1580,7 +1592,8 @@
}

if (this->project->runtime) {
auto runtimeRef = clearDependency(this->project->runtime.value(), false, false);
auto runtimeRef =
detail::pullDependency(this->project->runtime.value(), this->repo, "binary", false);
if (!runtimeRef) {
return LINGLONG_ERR(runtimeRef);
}
Expand Down
12 changes: 8 additions & 4 deletions libs/linglong/src/linglong/builder/linglong_builder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,14 @@ void mergeOutput(const std::vector<std::filesystem::path> &src,
const std::filesystem::path &dest,
const std::vector<std::string> &targets,
const std::vector<std::string> &excludes);
}
utils::error::Result<void> pullResolvedRef(const package::Reference &ref,
repo::OSTreeRepo &repo,
const std::string &module) noexcept;
utils::error::Result<package::Reference> pullDependency(const std::string &fuzzyRefStr,
repo::OSTreeRepo &repo,
const std::string &module,
bool useRemote) noexcept;
} // namespace detail

class Builder
{
Expand Down Expand Up @@ -131,9 +138,6 @@ class Builder
void fixLocaltimeInOverlay(std::unique_ptr<utils::OverlayFS> &base);
utils::error::Result<package::Reference>
ensureUtils(const std::string &id, const package::Architecture &arch) noexcept;
utils::error::Result<package::Reference> clearDependency(const std::string &ref,
bool forceRemote,
bool fallbackToRemote) noexcept;
auto generateEntryScript() noexcept -> utils::error::Result<void>;
auto generateBuildDependsScript() noexcept -> utils::error::Result<bool>;
auto generateDependsScript() noexcept -> utils::error::Result<bool>;
Expand Down
8 changes: 4 additions & 4 deletions libs/linglong/src/linglong/repo/ostree_repo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ class OSTreeRepo : public QObject
std::vector<std::filesystem::path> overlays = {},
const std::optional<std::string> &subRef = std::nullopt) noexcept;

[[nodiscard]] utils::error::Result<package::LayerDir>
virtual utils::error::Result<package::LayerDir>
getLayerDir(const package::Reference &ref,
const std::string &module = "binary",
const std::optional<std::string> &subRef = std::nullopt) const noexcept;
Expand All @@ -119,9 +119,9 @@ class OSTreeRepo : public QObject
const std::string &url,
const package::Reference &reference,
const std::string &module = "binary") const noexcept;
[[nodiscard]] utils::error::Result<void> pull(service::Task &taskContext,
const package::ReferenceWithRepo &refRepo,
const std::string &module) noexcept;
[[nodiscard]] virtual utils::error::Result<void> pull(service::Task &taskContext,
const package::ReferenceWithRepo &refRepo,
const std::string &module) noexcept;

[[nodiscard]] virtual utils::error::Result<package::Reference>
clearReference(const package::FuzzyReference &fuzzy,
Expand Down
1 change: 1 addition & 0 deletions libs/linglong/tests/ll-tests/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ pfl_add_executable(
src/common/tempdir.h
src/linglong/builder/config_test.cpp
src/linglong/builder/linglong_builder_test.cpp
src/linglong/builder/pull_dependency_test.cpp
src/linglong/builder/source_fetcher_test.cpp
src/linglong/cli/cli_test.cpp
src/linglong/common/gkeyfile_wrapper_test.cpp
Expand Down
Loading
Loading