diff --git a/.version b/.version index dc148c42f45d..02eb44632167 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.33.3 +2.33.4 diff --git a/doc/manual/rl-next/s3-credential-chain-web-identity.md b/doc/manual/rl-next/s3-credential-chain-web-identity.md new file mode 100644 index 000000000000..4dfece0e3b7d --- /dev/null +++ b/doc/manual/rl-next/s3-credential-chain-web-identity.md @@ -0,0 +1,26 @@ +--- +synopsis: "S3: restore STS WebIdentity and ECS container credential providers" +prs: [15507] +--- + +Nix 2.33 replaced the S3 backend's `aws-sdk-cpp` credential chain with a +custom chain built on `aws-c-auth`. That chain omitted two providers, +breaking S3 binary cache access in container workloads: + +- **STS WebIdentity** (`AWS_WEB_IDENTITY_TOKEN_FILE`, `AWS_ROLE_ARN`, + `AWS_ROLE_SESSION_NAME`) — used by EKS IRSA, GitHub Actions OIDC, and + any `sts:AssumeRoleWithWebIdentity` federation. +- **ECS container metadata** (`AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`, + `AWS_CONTAINER_CREDENTIALS_FULL_URI`) — used by ECS tasks and EKS Pod + Identity. + +The typical symptom was a misleading IMDS error +(`Valid credentials could not be sourced by the IMDS provider`), because +IMDS is the last provider tried after the correct one was skipped. + +Both providers are now part of the chain, ordered to match the +pre-2.33 `DefaultAWSCredentialsProviderChain`: +`Environment → SSO → Profile → STS WebIdentity → (ECS | IMDS)`. +As in both the old and new AWS SDK default chains, ECS and IMDS are +mutually exclusive: when container credential environment variables are +set, IMDS is skipped. diff --git a/maintainers/upload-debug-info-to-sentry.py b/maintainers/upload-debug-info-to-sentry.py index e1c242fd2fee..4e315c65e872 100755 --- a/maintainers/upload-debug-info-to-sentry.py +++ b/maintainers/upload-debug-info-to-sentry.py @@ -130,7 +130,8 @@ def main(): for lib in libs: build_id = get_build_id(lib) if build_id is None: - print(f" {lib} (no build ID)", file=sys.stderr) + print(f" {lib} (no build ID, uploading binary)", file=sys.stderr) + debug_files.append(lib) continue local = find_debug_file_in_dirs(build_id, args.debug_dir) @@ -141,7 +142,8 @@ def main(): debuginfo = fetch_debuginfo(build_id) if debuginfo is None: - print(f" {lib} ({build_id}, no debug info in cache)", file=sys.stderr) + print(f" {lib} ({build_id}): no separate debug info, uploading binary", file=sys.stderr) + debug_files.append(lib) continue print(f" {lib} ({build_id}): member={debuginfo['member']}", file=sys.stderr) nar_path = download_nar(build_id, debuginfo["archive"]) diff --git a/packaging/binary-tarball.nix b/packaging/binary-tarball.nix index 86aae0ac524c..859cb6cd6d6e 100644 --- a/packaging/binary-tarball.nix +++ b/packaging/binary-tarball.nix @@ -74,7 +74,8 @@ runCommand "nix-binary-tarball-${version}" env '' fn=$out/$dir.tar.xz mkdir -p $out/nix-support echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cfJ $fn \ + tar cf - \ + --sort=name \ --owner=0 --group=0 --mode=u+rw,uga+r \ --mtime='1970-01-01' \ --absolute-names \ @@ -90,5 +91,5 @@ runCommand "nix-binary-tarball-${version}" env '' $TMPDIR/install-freebsd-multi-user.sh \ $TMPDIR/install-multi-user \ $TMPDIR/reginfo \ - $(cat ${installerClosureInfo}/store-paths) + $(cat ${installerClosureInfo}/store-paths) | xz --threads=1 > $fn '' diff --git a/packaging/sentry-native.nix b/packaging/sentry-native.nix index 06accdd5b18a..0e21e9be55c2 100644 --- a/packaging/sentry-native.nix +++ b/packaging/sentry-native.nix @@ -49,4 +49,6 @@ stdenv.mkDerivation rec { "out" "dev" ]; + + separateDebugInfo = true; } diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index c3341da73a26..2cdc8e38c234 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -38,7 +38,9 @@ static std::string doRenderMarkdownToTerminal(std::string_view markdown) # endif .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES, .oflags = -# if HAVE_LOWDOWN_1_4 +# if HAVE_LOWDOWN_3 + LOWDOWN_NORELLINK +# elif HAVE_LOWDOWN_1_4 LOWDOWN_TERM_NORELLINK // To render full links while skipping relative ones # else LOWDOWN_TERM_NOLINK diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 087da84f9293..c9d5105693cf 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -44,6 +44,10 @@ configdata.set( 'HAVE_LOWDOWN_1_4', lowdown.version().version_compare('>= 1.4.0').to_int(), ) +configdata.set( + 'HAVE_LOWDOWN_3', + lowdown.version().version_compare('>= 3.0.0').to_int(), +) readline_flavor = get_option('readline-flavor') if readline_flavor == 'editline' diff --git a/src/libstore/aws-creds.cc b/src/libstore/aws-creds.cc index 9b0ddefdca85..1fa84ac0e2cf 100644 --- a/src/libstore/aws-creds.cc +++ b/src/libstore/aws-creds.cc @@ -10,7 +10,7 @@ # include # include -// C library headers for SSO provider support +// C library headers for SSO, STS WebIdentity, and ECS credential providers # include // C library headers for custom logging @@ -170,6 +170,53 @@ static std::shared_ptr createSSOProvider( return createWrappedProvider(aws_credentials_provider_new_sso(allocator, &options), allocator); } +/** + * Create an STS WebIdentity credentials provider using the C library directly. + * This reads AWS_WEB_IDENTITY_TOKEN_FILE, AWS_ROLE_ARN, AWS_ROLE_SESSION_NAME, + * and AWS_REGION from the environment (falling back to the profile config). + * Used by EKS IRSA, GitHub Actions OIDC, and other sts:AssumeRoleWithWebIdentity flows. + * Returns nullptr if the required parameters can't be resolved. + */ +static std::shared_ptr createSTSWebIdentityProvider( + const std::string & profileName, + Aws::Crt::Io::ClientBootstrap * bootstrap, + Aws::Crt::Io::TlsContext * tlsContext, + Aws::Crt::Allocator * allocator = Aws::Crt::ApiAllocator()) +{ + aws_credentials_provider_sts_web_identity_options options; + AWS_ZERO_STRUCT(options); + + options.bootstrap = bootstrap->GetUnderlyingHandle(); + options.tls_ctx = tlsContext ? tlsContext->GetUnderlyingHandle() : nullptr; + if (!profileName.empty()) { + options.profile_name_override = aws_byte_cursor_from_c_str(profileName.c_str()); + } + + return createWrappedProvider(aws_credentials_provider_new_sts_web_identity(allocator, &options), allocator); +} + +/** + * Create an ECS container credentials provider using the C library directly. + * This reads AWS_CONTAINER_CREDENTIALS_RELATIVE_URI or + * AWS_CONTAINER_CREDENTIALS_FULL_URI (plus the optional + * AWS_CONTAINER_AUTHORIZATION_TOKEN / _TOKEN_FILE) from the environment. + * Used by ECS tasks and EKS Pod Identity. + * Returns nullptr if neither URI env var is set. + */ +static std::shared_ptr createECSProvider( + Aws::Crt::Io::ClientBootstrap * bootstrap, + Aws::Crt::Io::TlsContext * tlsContext, + Aws::Crt::Allocator * allocator = Aws::Crt::ApiAllocator()) +{ + aws_credentials_provider_ecs_environment_options options; + AWS_ZERO_STRUCT(options); + + options.bootstrap = bootstrap->GetUnderlyingHandle(); + options.tls_ctx = tlsContext ? tlsContext->GetUnderlyingHandle() : nullptr; + + return createWrappedProvider(aws_credentials_provider_new_ecs_from_environment(allocator, &options), allocator); +} + static AwsCredentials getCredentialsFromProvider(std::shared_ptr provider) { if (!provider || !provider->IsValid()) { @@ -223,13 +270,14 @@ class AwsCredentialProviderImpl : public AwsCredentialProvider // This ensures AWS logs respect Nix's verbosity settings and are formatted consistently. initialiseAwsLogger(); - // Create a shared TLS context for SSO (required for HTTPS connections) + // Create a shared TLS context for SSO, STS WebIdentity, and ECS providers (required for HTTPS) auto allocator = Aws::Crt::ApiAllocator(); auto tlsCtxOptions = Aws::Crt::Io::TlsContextOptions::InitDefaultClient(allocator); tlsContext = std::make_shared(tlsCtxOptions, Aws::Crt::Io::TlsMode::CLIENT, allocator); if (!tlsContext || !*tlsContext) { - warn("failed to create TLS context for AWS SSO; SSO authentication will be unavailable"); + warn( + "failed to create TLS context for AWS credential providers; SSO, STS WebIdentity, and ECS container authentication will be unavailable"); tlsContext = nullptr; } @@ -273,19 +321,20 @@ AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile) debug("[pid=%d] creating new AWS credential provider for profile '%s'", getpid(), profileDisplayName); - // Build a custom credential chain: Environment → SSO → Profile → IMDS + // Build a custom credential chain: Environment → SSO → Profile → STS WebIdentity → ECS → IMDS // This works for both default and named profiles, ensuring consistent behavior // including SSO support and proper TLS context for STS-based role assumption. Aws::Crt::Auth::CredentialsProviderChainConfig chainConfig; auto allocator = Aws::Crt::ApiAllocator(); - auto addProviderToChain = [&](std::string_view name, auto createProvider) { + auto addProviderToChain = [&](std::string_view name, auto createProvider) -> bool { if (auto provider = createProvider()) { chainConfig.Providers.push_back(provider); debug("Added AWS %s Credential Provider to chain for profile '%s'", name, profileDisplayName); - } else { - debug("Skipped AWS %s Credential Provider for profile '%s'", name, profileDisplayName); + return true; } + debug("Skipped AWS %s Credential Provider for profile '%s'", name, profileDisplayName); + return false; }; // 1. Environment variables (highest priority) @@ -311,12 +360,37 @@ AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile) return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(profileConfig, allocator); }); - // 4. IMDS provider (for EC2 instances, lowest priority) - addProviderToChain("IMDS", [&]() { - Aws::Crt::Auth::CredentialsProviderImdsConfig imdsConfig; - imdsConfig.Bootstrap = bootstrap; - return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderImds(imdsConfig, allocator); - }); + // 4. STS WebIdentity (AWS_WEB_IDENTITY_TOKEN_FILE + AWS_ROLE_ARN — EKS IRSA, GitHub Actions OIDC) + // 5. ECS container metadata (AWS_CONTAINER_CREDENTIALS_RELATIVE_URI — ECS tasks, EKS Pod Identity) + // ECS and IMDS are mutually exclusive per both the aws-c-auth default chain and the + // pre-2.33 aws-sdk-cpp DefaultAWSCredentialsProviderChain: when container credential + // env vars are set, IMDS is skipped so a transient ECS endpoint failure can't silently + // fall through to the (typically broader) EC2 instance profile. + bool ecsAdded = false; + if (tlsContext) { + addProviderToChain("STS WebIdentity", [&]() { + return createSTSWebIdentityProvider(profile, bootstrap, tlsContext.get(), allocator); + }); + ecsAdded = + addProviderToChain("ECS", [&]() { return createECSProvider(bootstrap, tlsContext.get(), allocator); }); + } else { + debug( + "Skipped AWS STS WebIdentity and ECS Credential Providers for profile '%s': TLS context unavailable", + profileDisplayName); + } + + // 6. IMDS provider (for EC2 instances, lowest priority) — only if ECS didn't claim the slot + if (!ecsAdded) { + addProviderToChain("IMDS", [&]() { + Aws::Crt::Auth::CredentialsProviderImdsConfig imdsConfig; + imdsConfig.Bootstrap = bootstrap; + return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderImds(imdsConfig, allocator); + }); + } else { + debug( + "Skipped AWS IMDS Credential Provider for profile '%s': ECS provider is active (mutually exclusive)", + profileDisplayName); + } return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChain(chainConfig, allocator); } diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 51bab9953548..fc1dee92eaf7 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -94,6 +94,8 @@ std::shared_ptr RemoteFSAccessor::accessObject(const StorePath & std::optional RemoteFSAccessor::maybeLstat(const CanonPath & path) { + if (path.isRoot()) + return Stat{.type = tDirectory}; auto res = fetch(path); return res.first->maybeLstat(res.second); } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 842381acf66e..7bcf81fbedda 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -150,18 +150,23 @@ class SimpleLogger : public Logger Verbosity verbosity = lvlInfo; -void writeToStderr(std::string_view s) +static void writeFullLogging(Descriptor fd, std::string_view s) { try { - writeFull(getStandardError(), s, false); + writeFull(fd, s, false); } catch (SystemError & e) { - /* Ignore failing writes to stderr. We need to ignore write - errors to ensure that cleanup code that logs to stderr runs - to completion if the other side of stderr has been closed - unexpectedly. */ + /* Ignore failing logging writes. We need to ignore write + errors to ensure that cleanup code that writes logs runs + to completion if the other side of the logging fd has + been closed unexpectedly. */ } } +void writeToStderr(std::string_view s) +{ + writeFullLogging(getStandardError(), s); +} + std::unique_ptr makeSimpleLogger(bool printBuildLogs) { return std::make_unique(printBuildLogs); @@ -245,15 +250,15 @@ struct JSONLogger : Logger void write(const nlohmann::json & json) { - auto line = - (includeNixPrefix ? "@nix " : "") + json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace); + auto line = (includeNixPrefix ? "@nix " : "") + + json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) + "\n"; /* Acquire a lock to prevent log messages from clobbering each other. */ try { auto state(_state.lock()); if (state->enabled) - writeLine(fd, line); + writeFullLogging(fd, line); } catch (...) { bool enabled = false; std::swap(_state.lock()->enabled, enabled); diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc index 2a167b52b3fb..52b9e51a221b 100644 --- a/src/libutil/terminal.cc +++ b/src/libutil/terminal.cc @@ -163,7 +163,7 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int w // Note: this object intentionally leaks to avoid a destructor ordering issue (specifically, ~ProgressBar() calling // getWindowSize() after windowSize has been destroyed). -static auto * const windowSize = new Sync>({0, 0}); +static auto * const windowSize = new Sync>{{0, 0}}; void updateWindowSize() {