Skip to content

Commit d9526df

Browse files
Ericson2314Ericson2314
committed
Merge branch 'master' into ca-resolve-two-phase
2 parents 85307c2 + 03f6fbd commit d9526df

4 files changed

Lines changed: 198 additions & 3 deletions

File tree

‎subprojects/hydra-tests/content-addressed/basic.t‎

Lines changed: 46 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,12 @@ my $project = $db->resultset('Projects')->create({name => "tests", displayname =
2424
my $jobset = createBaseJobset($db, "content-addressed", "content-addressed.nix", $ctx{jobsdir});
2525

2626
ok(evalSucceeds($ctx{context}, $jobset), "Evaluating jobs/content-addressed.nix should exit with return code 0");
27-
is(nrQueuedBuildsForJobset($jobset), 10, "Evaluating jobs/content-addressed.nix should result in 6 builds");
27+
is(nrQueuedBuildsForJobset($jobset), 14, "Evaluating jobs/content-addressed.nix should result in 14 builds");
2828

29-
for my $build (queuedBuildsForJobset($jobset)) {
30-
ok(runBuild($ctx{context}, $build), "Build '".$build->job."' from jobs/content-addressed.nix should exit with code 0");
29+
my @builds = queuedBuildsForJobset($jobset);
30+
ok(runBuilds($ctx{context}, @builds), "Building all jobs from jobs/content-addressed.nix should exit with code 0");
31+
32+
for my $build (@builds) {
3133
my $newbuild = $db->resultset('Builds')->find($build->id);
3234
is($newbuild->finished, 1, "Build '".$build->job."' from jobs/content-addressed.nix should be finished.");
3335
my $expected = $build->job eq "fails" ? 1 : $build->job =~ /with_failed/ ? 6 : $build->job =~ /FailingCA/ ? 2 : 0;
@@ -56,5 +58,46 @@ for my $build (queuedBuildsForJobset($jobset)) {
5658
# XXX: This test seems to not do what it seems to be doing. See documentation: https://metacpan.org/pod/Test2::V0#isnt($got,-$do_not_want,-$name)
5759
isnt(<$ctx{deststoredir}/realisations/*>, "", "The destination store should have the realisations of the built derivations registered");
5860

61+
# Early cutoff: earlyCutoffUpstream1 and earlyCutoffUpstream2 have
62+
# different derivations but produce the same content-addressed output.
63+
# After building earlyCutoffDownstream1, earlyCutoffDownstream2 should
64+
# be cached because its resolved input is identical.
65+
my $upstream1 = $db->resultset('Builds')->find({
66+
jobset_id => $jobset->id,
67+
job => "earlyCutoffUpstream1",
68+
});
69+
my $upstream2 = $db->resultset('Builds')->find({
70+
jobset_id => $jobset->id,
71+
job => "earlyCutoffUpstream2",
72+
});
73+
74+
my $upstream1_out = $upstream1->buildoutputs->find({ name => "out" });
75+
my $upstream2_out = $upstream2->buildoutputs->find({ name => "out" });
76+
is($upstream1_out->path, $upstream2_out->path,
77+
"Both upstream builds should resolve to the same content-addressed output path");
78+
79+
my $downstream1 = $db->resultset('Builds')->find({
80+
jobset_id => $jobset->id,
81+
job => "earlyCutoffDownstream1",
82+
});
83+
my $downstream2 = $db->resultset('Builds')->find({
84+
jobset_id => $jobset->id,
85+
job => "earlyCutoffDownstream2",
86+
});
87+
88+
my $downstream1_out = $downstream1->buildoutputs->find({ name => "out" });
89+
my $downstream2_out = $downstream2->buildoutputs->find({ name => "out" });
90+
is($downstream1_out->path, $downstream2_out->path,
91+
"Both downstream builds should resolve to the same content-addressed output path");
92+
93+
# TODO: Once the queue runner deduplicates steps by resolved derivation
94+
# path (not just original drv path), we should also verify that both
95+
# original steps resolve to steps with the same derivation. (Might even
96+
# be the same step, but that doesn't matter as much).
97+
#
98+
# If there are multiple steps for the single resolved derivation,
99+
# additionally, only one should get built, and the other should be a
100+
# cached successes (as is normal for duplicative build steps).
101+
59102
done_testing;
60103

‎subprojects/hydra-tests/content-addressed/dyn-drv.t‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
use feature 'unicode_strings';
2+
use strict;
3+
use warnings;
4+
use Setup;
5+
use Test2::V0;
6+
7+
# Based on https://github.com/NixOS/nix/blob/14ffc1787182b8702910788aea02bd5804afb32e/tests/functional/dyn-drv/text-hashed-output.nix
8+
#
9+
# A single derivation produces a .drv file as its output; another
10+
# derivation (wrapper) depends on building that dynamically-produced .drv
11+
# and using its output via builtins.outputOf.
12+
13+
my $ctx = test_context(
14+
nix_config => qq|
15+
experimental-features = ca-derivations dynamic-derivations
16+
|,
17+
);
18+
19+
my $db = $ctx->db();
20+
21+
my $jobset = createBaseJobset($db, "dyn-drv", "dyn-drv.nix", $ctx->jobsdir);
22+
23+
ok(evalSucceeds($ctx, $jobset), "Evaluation of dyn-drv.nix should succeed");
24+
is(nrQueuedBuildsForJobset($jobset), 1, "Should queue 1 build (wrapper)");
25+
26+
my @builds = queuedBuildsForJobset($jobset);
27+
ok(runBuilds($ctx, @builds), "All dynamic derivation builds should complete");
28+
29+
# hello and producingDrv are standard CA derivations, so they must succeed.
30+
for my $build (grep { $_->job ne 'wrapper' } @builds) {
31+
$build->discard_changes;
32+
is($build->finished, 1, "Build '" . $build->job . "' should be finished");
33+
is($build->buildstatus, 0, "Build '" . $build->job . "' should succeed");
34+
}
35+
36+
# wrapper is the dynamic derivation consumer.
37+
# It exercises the full resolution path: build producingDrv, discover the .drv
38+
# at its output, resolve via try_resolve + flatten_chain, build the resolved drv.
39+
my ($wrapper) = grep { $_->job eq 'wrapper' } @builds;
40+
ok(defined $wrapper, "wrapper (dynamic derivation consumer) build should exist");
41+
if ($wrapper) {
42+
$wrapper->discard_changes;
43+
is($wrapper->finished, 1, "wrapper should be finished");
44+
is($wrapper->buildstatus, 0, "wrapper should succeed");
45+
46+
# Hydra currently doesn't understand the dynamic derivation structure,
47+
# so it only sees 2 build steps (producingDrv + wrapper itself) rather
48+
# than the full chain (producingDrv + dynamic hello + wrapper).
49+
my $nrSteps = $wrapper->buildsteps->count;
50+
is($nrSteps, 2, "wrapper should have 2 build steps (dynamic structure not yet tracked)");
51+
}
52+
53+
done_testing;

‎subprojects/hydra-tests/jobs/content-addressed.nix‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,4 +64,33 @@ rec {
6464
"lib"
6565
];
6666
};
67+
68+
# Early-cutoff test: two upstream derivations that differ in a dummy
69+
# attribute but produce the same output (an empty directory). Because
70+
# they are content-addressed, they resolve to the same output path.
71+
# Both downstreams depend on a different upstream, but since the
72+
# resolved input is identical, the second downstream should be cached.
73+
earlyCutoffUpstream1 = cfg.mkContentAddressedDerivation {
74+
name = "early-cutoff-upstream";
75+
builder = ./empty-dir-builder.sh;
76+
dummy = "1";
77+
};
78+
79+
earlyCutoffUpstream2 = cfg.mkContentAddressedDerivation {
80+
name = "early-cutoff-upstream";
81+
builder = ./empty-dir-builder.sh;
82+
dummy = "2";
83+
};
84+
85+
earlyCutoffDownstream1 = cfg.mkContentAddressedDerivation {
86+
name = "early-cutoff-downstream";
87+
builder = ./dir-with-file-builder.sh;
88+
FOO = earlyCutoffUpstream1;
89+
};
90+
91+
earlyCutoffDownstream2 = cfg.mkContentAddressedDerivation {
92+
name = "early-cutoff-downstream";
93+
builder = ./dir-with-file-builder.sh;
94+
FOO = earlyCutoffUpstream2;
95+
};
6796
}

‎subprojects/hydra-tests/jobs/dyn-drv.nix‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# Adapted from https://github.com/NixOS/nix/blob/14ffc1787182b8702910788aea02bd5804afb32e/tests/functional/dyn-drv/text-hashed-output.nix
2+
#
3+
# A derivation produces a .drv file as its output; another derivation depends
4+
# on building that dynamically-produced .drv and using its output.
5+
let
6+
cfg = import ./config.nix;
7+
8+
# A CA derivation that writes content based on the GREETING env var.
9+
# The GREETING contains 'X' which producingDrv rewrites to 'Y' via
10+
# tr at build time, so the dynamically-produced .drv differs from
11+
# the statically-known one.
12+
hello = cfg.mkContentAddressedDerivation {
13+
name = "hello";
14+
builder = "/bin/sh";
15+
args = [
16+
"-c"
17+
''
18+
mkdir -p "$out"
19+
echo "greeting while builder: $GREETING" >&2
20+
echo "saving greeting to output: $GREETING" > $out/result
21+
''
22+
];
23+
GREETING = "XXXX derivation";
24+
};
25+
26+
# A CA derivation whose output IS a .drv file.
27+
# Copies hello's .drv then rewrites X→Y so the dynamic derivation
28+
# builds with GREETING="YYYY derivation" instead of "XXXX derivation".
29+
# tr is in coreutils so it's available on PATH.
30+
producingDrv = cfg.mkDerivation {
31+
name = "hello.drv";
32+
builder = "/bin/sh";
33+
args = [
34+
"-c"
35+
''
36+
drv=${builtins.unsafeDiscardOutputDependency hello.drvPath}
37+
echo rewriting "$drv" >&2
38+
tr X Y < "$drv" > "$out"
39+
''
40+
];
41+
__contentAddressed = true;
42+
outputHashMode = "text";
43+
outputHashAlgo = "sha256";
44+
};
45+
46+
in
47+
{
48+
# The actual dynamic derivation consumer: depends on the output of the
49+
# .drv file that producingDrv produces. Nix must:
50+
# 1. Build producingDrv (get the .drv file)
51+
# 2. Discover the .drv at its output
52+
# 3. Build THAT .drv (which runs dyn-drv-builder.sh with GREETING="YYYY derivation")
53+
# 4. Use its output here
54+
wrapper = cfg.mkContentAddressedDerivation {
55+
name = "dyn-drv-wrapper";
56+
builder = "/bin/sh";
57+
args = [
58+
"-c"
59+
''
60+
result=${builtins.outputOf producingDrv.outPath "out"}
61+
# Verify the dynamically-built derivation used the rewritten GREETING
62+
case "$(cat "$result/result")" in
63+
*YYYY*) ;;
64+
*) echo "expected YYYY in result" >&2; exit 1 ;;
65+
esac
66+
cp -r "$result" $out
67+
''
68+
];
69+
};
70+
}

0 commit comments

Comments
 (0)