Skip to content

Commit 0eed540

Browse files
Ericson2314Ericson2314
committed
WIP only use FullInputs for formats
1 parent 7dbf5e7 commit 0eed540

18 files changed

Lines changed: 527 additions & 432 deletions

‎src/libexpr/primops.cc‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1677,16 +1677,26 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
16771677
StorePathSet refs;
16781678
state.store->computeFSClosure(d.drvPath, refs);
16791679
for (auto & j : refs) {
1680-
drv.inputs.srcs.insert(j);
1680+
drv.inputs.insert(SingleDerivedPath::Opaque{j});
16811681
if (j.isDerivation()) {
1682-
drv.inputs.drvs.map[j].value = state.store->readDerivation(j).outputNames();
1682+
for (auto & outputName : state.store->readDerivation(j).outputNames()) {
1683+
drv.inputs.insert(
1684+
SingleDerivedPath::Built{
1685+
.drvPath = makeConstantStorePathRef(j),
1686+
.output = outputName,
1687+
});
1688+
}
16831689
}
16841690
}
16851691
},
16861692
[&](const NixStringContextElem::Built & b) {
1687-
drv.inputs.drvs.ensureSlot(*b.drvPath).value.insert(b.output);
1693+
drv.inputs.insert(
1694+
SingleDerivedPath::Built{
1695+
.drvPath = b.drvPath,
1696+
.output = b.output,
1697+
});
16881698
},
1689-
[&](const NixStringContextElem::Opaque & o) { drv.inputs.srcs.insert(o.path); },
1699+
[&](const NixStringContextElem::Opaque & o) { drv.inputs.insert(SingleDerivedPath::Opaque{o.path}); },
16901700
},
16911701
c.raw);
16921702
}

‎src/libstore-tests/derivation-advanced-attrs.cc‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ class DerivationAdvancedAttrsTest : public JsonCharacterizationTest<Derivation>,
4343
this->readTest(fileName, [&](auto encoded) {
4444
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
4545
auto options = derivationOptionsFromStructuredAttrs(
46-
*this->store, got.inputs.drvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
46+
*this->store, got.inputs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
4747
EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedFeatures);
4848
});
4949
}
@@ -59,7 +59,7 @@ class DerivationAdvancedAttrsTest : public JsonCharacterizationTest<Derivation>,
5959
this->readTest(fileName, [&](auto encoded) {
6060
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
6161
auto options = derivationOptionsFromStructuredAttrs(
62-
*this->store, got.inputs.drvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
62+
*this->store, got.inputs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
6363

6464
EXPECT_EQ(options, expected);
6565
EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedSystemFeatures);
@@ -188,7 +188,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults)
188188
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
189189

190190
auto options = derivationOptionsFromStructuredAttrs(
191-
*this->store, got.inputs.drvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
191+
*this->store, got.inputs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
192192

193193
EXPECT_TRUE(!got.structuredAttrs);
194194

@@ -234,7 +234,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
234234
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
235235

236236
auto options = derivationOptionsFromStructuredAttrs(
237-
*this->store, got.inputs.drvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
237+
*this->store, got.inputs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
238238

239239
EXPECT_TRUE(!got.structuredAttrs);
240240

@@ -330,7 +330,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d
330330
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
331331

332332
auto options = derivationOptionsFromStructuredAttrs(
333-
*this->store, got.inputs.drvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
333+
*this->store, got.inputs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
334334

335335
EXPECT_TRUE(got.structuredAttrs);
336336

@@ -381,7 +381,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
381381
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
382382

383383
auto options = derivationOptionsFromStructuredAttrs(
384-
*this->store, got.inputs.drvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
384+
*this->store, got.inputs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
385385

386386
EXPECT_TRUE(got.structuredAttrs);
387387

‎src/libstore-tests/derivation.cc‎

Lines changed: 56 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -224,24 +224,30 @@ Derivation makeSimpleDrv()
224224
{
225225
Derivation drv;
226226
drv.name = "simple-derivation";
227-
drv.inputs.srcs = {
228-
StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"),
229-
};
230-
drv.inputs.drvs = {
231-
.map =
232-
{
227+
drv.inputs =
228+
FullInputs{
229+
.srcs =
233230
{
234-
StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"),
235-
{
236-
.value =
231+
StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"),
232+
},
233+
.drvs =
234+
{
235+
.map =
236+
{
237237
{
238-
"cat",
239-
"dog",
238+
StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"),
239+
{
240+
.value =
241+
{
242+
"cat",
243+
"dog",
244+
},
245+
},
240246
},
241-
},
247+
},
242248
},
243-
},
244-
};
249+
}
250+
.toSet();
245251
drv.platform = "wasm-sel4";
246252
drv.builder = "foo";
247253
drv.args = {
@@ -270,45 +276,51 @@ Derivation makeDynDepDerivation()
270276
{
271277
Derivation drv;
272278
drv.name = "dyn-dep-derivation";
273-
drv.inputs.srcs = {
274-
StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"},
275-
};
276-
drv.inputs.drvs = {
277-
.map =
278-
{
279+
drv.inputs =
280+
FullInputs{
281+
.srcs =
279282
{
280-
StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"},
281-
DerivedPathMap<StringSet>::ChildNode{
282-
.value =
283-
{
284-
"cat",
285-
"dog",
286-
},
287-
.childMap =
283+
StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"},
284+
},
285+
.drvs =
286+
{
287+
.map =
288+
{
288289
{
289-
{
290-
"cat",
291-
DerivedPathMap<StringSet>::ChildNode{
292-
.value =
290+
StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"},
291+
DerivedPathMap<StringSet>::ChildNode{
292+
.value =
293+
{
294+
"cat",
295+
"dog",
296+
},
297+
.childMap =
298+
{
293299
{
294-
"kitten",
300+
"cat",
301+
DerivedPathMap<StringSet>::ChildNode{
302+
.value =
303+
{
304+
"kitten",
305+
},
306+
},
295307
},
296-
},
297-
},
298-
{
299-
"goose",
300-
DerivedPathMap<StringSet>::ChildNode{
301-
.value =
302308
{
303-
"gosling",
309+
"goose",
310+
DerivedPathMap<StringSet>::ChildNode{
311+
.value =
312+
{
313+
"gosling",
314+
},
315+
},
304316
},
305-
},
317+
},
306318
},
307319
},
308-
},
320+
},
309321
},
310-
},
311-
};
322+
}
323+
.toSet();
312324
drv.platform = "wasm-sel4";
313325
drv.builder = "foo";
314326
drv.args = {

‎src/libstore/build/derivation-building-goal.cc‎

Lines changed: 55 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -108,23 +108,38 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation)
108108
109109
Note that some inputs might not be in the eval store because they
110110
are (resolved) derivation outputs in a resolved derivation. */
111+
112+
std::set<const SingleDerivedPath::Built *> builtInputs;
113+
114+
StorePathSet inputSrcs;
115+
116+
for (auto & input : drv->inputs)
117+
std::visit(overloaded{
118+
[&](const SingleDerivedPath::Opaque & op) {
119+
inputSrcs.insert(op.path);
120+
},
121+
[&](const SingleDerivedPath::Built & built) {
122+
builtInputs.insert(&built);
123+
},
124+
}, input.raw());
125+
111126
if (&worker.evalStore != &worker.store) {
112-
RealisedPath::Set inputSrcs;
113-
for (auto & i : drv->inputs.srcs)
114-
if (worker.evalStore.isValidPath(i))
115-
inputSrcs.insert(i);
116-
copyClosure(worker.evalStore, worker.store, inputSrcs);
127+
StorePathSet evalStorePaths;
128+
for (auto & path : inputSrcs)
129+
if (worker.evalStore.isValidPath(path))
130+
evalStorePaths.insert(path);
131+
copyClosure(worker.evalStore, worker.store, evalStorePaths);
117132
}
118133

119-
for (auto & i : drv->inputs.srcs) {
120-
if (worker.store.isValidPath(i))
134+
for (auto & path : inputSrcs) {
135+
if (worker.store.isValidPath(path))
121136
continue;
122137
if (!settings.useSubstitutes)
123138
throw Error(
124139
"dependency '%s' of '%s' does not exist, and substitution is disabled",
125-
worker.store.printStorePath(i),
140+
worker.store.printStorePath(path),
126141
worker.store.printStorePath(drvPath));
127-
waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(i)));
142+
waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(path)));
128143
}
129144

130145
co_await await(std::move(waitees));
@@ -148,45 +163,47 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation)
148163
/* Determine the full set of input paths. */
149164

150165
if (storeDerivation) {
151-
assert(drv->inputs.drvs.map.empty());
166+
/* For resolved derivations, there should be no Built inputs */
167+
assert(std::ranges::none_of(drv->inputs, [](const auto & input) {
168+
return std::holds_alternative<SingleDerivedPath::Built>(input.raw());
169+
}));
152170
/* Store the resolved derivation, as part of the record of
153171
what we're actually building */
154172
writeDerivation(worker.store, *drv);
155173
}
156174

157-
{
158-
/* If we get this far, we know no dynamic drvs inputs */
159-
160-
for (auto & [depDrvPath, depNode] : drv->inputs.drvs.map) {
161-
for (auto & outputName : depNode.value) {
162-
/* Don't need to worry about `inputGoals`, because
163-
impure derivations are always resolved above. Can
164-
just use DB. This case only happens in the (older)
165-
input addressed and fixed output derivation cases. */
166-
auto outMap = [&] {
167-
for (auto * drvStore : {&worker.evalStore, &worker.store})
168-
if (drvStore->isValidPath(depDrvPath))
169-
return worker.store.queryDerivationOutputMap(depDrvPath, drvStore);
170-
assert(false);
171-
}();
172-
173-
auto outMapPath = outMap.find(outputName);
174-
if (outMapPath == outMap.end()) {
175-
throw Error(
176-
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
177-
worker.store.printStorePath(drvPath),
178-
outputName,
179-
worker.store.printStorePath(depDrvPath));
180-
}
175+
/* Process source inputs */
176+
worker.store.computeFSClosure(inputSrcs, inputPaths);
177+
178+
/* Process derivation inputs */
179+
for (auto * built : builtInputs) {
180+
/* Only handle simple (non-dynamic) derivation inputs */
181+
if (auto * opaque = std::get_if<SingleDerivedPath::Opaque>(&built->drvPath->raw())) {
182+
auto & depDrvPath = opaque->path;
183+
/* Don't need to worry about `inputGoals`, because
184+
impure derivations are always resolved above. Can
185+
just use DB. This case only happens in the (older)
186+
input addressed and fixed output derivation cases. */
187+
auto outMap = [&] {
188+
for (auto * drvStore : {&worker.evalStore, &worker.store})
189+
if (drvStore->isValidPath(depDrvPath))
190+
return worker.store.queryDerivationOutputMap(depDrvPath, drvStore);
191+
assert(false);
192+
}();
181193

182-
worker.store.computeFSClosure(outMapPath->second, inputPaths);
194+
auto outMapPath = outMap.find(built->output);
195+
if (outMapPath == outMap.end()) {
196+
throw Error(
197+
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
198+
worker.store.printStorePath(drvPath),
199+
built->output,
200+
worker.store.printStorePath(depDrvPath));
183201
}
202+
203+
worker.store.computeFSClosure(outMapPath->second, inputPaths);
184204
}
185205
}
186206

187-
/* Second, the input sources. */
188-
worker.store.computeFSClosure(drv->inputs.srcs, inputPaths);
189-
190207
debug("added input paths %s", worker.store.showPaths(inputPaths));
191208

192209
/* Okay, try to build. Note that here we don't wait for a build

‎src/libstore/build/derivation-goal.cc‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
6666

6767
auto drvOptions = [&]() -> DerivationOptions<SingleDerivedPath> {
6868
try {
69-
return derivationOptionsFromStructuredAttrs(
70-
worker.store, drv->inputs.drvs, drv->env, get(drv->structuredAttrs));
69+
return derivationOptionsFromStructuredAttrs(worker.store, drv->inputs, drv->env, get(drv->structuredAttrs));
7170
} catch (Error & e) {
7271
e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
7372
throw;

0 commit comments

Comments
 (0)