Skip to content

Commit 7c47777

Browse files
grahamcgrahamc
authored
Merge pull request #56 from DeterminateSystems/references-without-context
Improve lazy trees backward compatibility
2 parents df93fa8 + 8eee061 commit 7c47777

13 files changed

Lines changed: 177 additions & 26 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,15 @@ jobs:
8989
| ".#hydraJobs.tests." + .')
9090
9191
flake_regressions:
92-
if: github.event_name == 'merge_group'
92+
if: |
93+
github.event_name == 'merge_group'
94+
|| (
95+
github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src'
96+
&& (
97+
(github.event.action == 'labeled' && github.event.label.name == 'flake-regression-test')
98+
|| (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'flake-regression-test'))
99+
)
100+
)
93101
needs: build_x86_64-linux
94102
runs-on: namespace-profile-x86-32cpu-64gb
95103
steps:
@@ -112,7 +120,15 @@ jobs:
112120
- run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH PARALLEL="-P 50%" flake-regressions/eval-all.sh
113121

114122
flake_regressions_lazy:
115-
if: github.event_name == 'merge_group'
123+
if: |
124+
github.event_name == 'merge_group'
125+
|| (
126+
github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src'
127+
&& (
128+
(github.event.action == 'labeled' && github.event.label.name == 'flake-regression-test')
129+
|| (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'flake-regression-test'))
130+
)
131+
)
116132
needs: build_x86_64-linux
117133
runs-on: namespace-profile-x86-32cpu-64gb
118134
steps:

‎src/libexpr-test-support/include/nix/expr/tests/value/context.hh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,11 @@ struct Arbitrary<NixStringContextElem::DrvDeep> {
2323
static Gen<NixStringContextElem::DrvDeep> arbitrary();
2424
};
2525

26+
template<>
27+
struct Arbitrary<NixStringContextElem::Path> {
28+
static Gen<NixStringContextElem::Path> arbitrary();
29+
};
30+
2631
template<>
2732
struct Arbitrary<NixStringContextElem> {
2833
static Gen<NixStringContextElem> arbitrary();

‎src/libexpr-test-support/tests/value/context.cc‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,15 @@ Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arb
1515
});
1616
}
1717

18+
Gen<NixStringContextElem::Path> Arbitrary<NixStringContextElem::Path>::arbitrary()
19+
{
20+
return gen::map(gen::arbitrary<StorePath>(), [](StorePath storePath) {
21+
return NixStringContextElem::Path{
22+
.storePath = storePath,
23+
};
24+
});
25+
}
26+
1827
Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
1928
{
2029
return gen::mapcat(
@@ -30,6 +39,9 @@ Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
3039
case 2:
3140
return gen::map(
3241
gen::arbitrary<NixStringContextElem::Built>(), [](NixStringContextElem a) { return a; });
42+
case 3:
43+
return gen::map(
44+
gen::arbitrary<NixStringContextElem::Path>(), [](NixStringContextElem a) { return a; });
3345
default:
3446
assert(false);
3547
}

‎src/libexpr/eval-cache.cc‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -618,18 +618,21 @@ string_t AttrCursor::getStringWithContext()
618618
if (auto s = std::get_if<string_t>(&cachedValue->second)) {
619619
bool valid = true;
620620
for (auto & c : s->second) {
621-
const StorePath & path = std::visit(overloaded {
622-
[&](const NixStringContextElem::DrvDeep & d) -> const StorePath & {
623-
return d.drvPath;
621+
const StorePath * path = std::visit(overloaded {
622+
[&](const NixStringContextElem::DrvDeep & d) -> const StorePath * {
623+
return &d.drvPath;
624624
},
625-
[&](const NixStringContextElem::Built & b) -> const StorePath & {
626-
return b.drvPath->getBaseStorePath();
625+
[&](const NixStringContextElem::Built & b) -> const StorePath * {
626+
return &b.drvPath->getBaseStorePath();
627627
},
628-
[&](const NixStringContextElem::Opaque & o) -> const StorePath & {
629-
return o.path;
628+
[&](const NixStringContextElem::Opaque & o) -> const StorePath * {
629+
return &o.path;
630+
},
631+
[&](const NixStringContextElem::Path & p) -> const StorePath * {
632+
return nullptr;
630633
},
631634
}, c.raw);
632-
if (!root->state.store->isValidPath(path)) {
635+
if (!path || !root->state.store->isValidPath(*path)) {
633636
valid = false;
634637
break;
635638
}

‎src/libexpr/eval.cc‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -952,8 +952,8 @@ void EvalState::mkPos(Value & v, PosIdx p)
952952
// FIXME: only do this for virtual store paths?
953953
attrs.alloc(sFile).mkString(path->path.abs(),
954954
{
955-
NixStringContextElem::Opaque{
956-
.path = store->toStorePath(path->path.abs()).first
955+
NixStringContextElem::Path{
956+
.storePath = store->toStorePath(path->path.abs()).first
957957
}
958958
});
959959
else
@@ -2078,7 +2078,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
20782078
else if (firstType == nFloat)
20792079
v.mkFloat(nf);
20802080
else if (firstType == nPath) {
2081-
if (!context.empty())
2081+
if (hasContext(context))
20822082
state.error<EvalError>("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow();
20832083
v.mkPath(state.rootPath(CanonPath(str())));
20842084
} else
@@ -2277,7 +2277,10 @@ std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::s
22772277
{
22782278
auto s = forceString(v, pos, errorCtx);
22792279
if (v.context()) {
2280-
error<EvalError>("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow();
2280+
NixStringContext context;
2281+
copyContext(v, context);
2282+
if (hasContext(context))
2283+
error<EvalError>("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow();
22812284
}
22822285
return s;
22832286
}
@@ -2336,7 +2339,16 @@ BackedStringView EvalState::coerceToString(
23362339
v.payload.path.path
23372340
: copyToStore
23382341
? store->printStorePath(copyPathToStore(context, v.path()))
2339-
: std::string(v.path().path.abs());
2342+
: ({
2343+
auto path = v.path();
2344+
if (path.accessor == rootFS && store->isInStore(path.path.abs())) {
2345+
context.insert(
2346+
NixStringContextElem::Path{
2347+
.storePath = store->toStorePath(path.path.abs()).first
2348+
});
2349+
}
2350+
std::string(path.path.abs());
2351+
});
23402352
}
23412353

23422354
if (v.type() == nAttrs) {
@@ -2499,6 +2511,11 @@ std::pair<SingleDerivedPath, std::string_view> EvalState::coerceToSingleDerivedP
24992511
[&](NixStringContextElem::Built && b) -> SingleDerivedPath {
25002512
return std::move(b);
25012513
},
2514+
[&](NixStringContextElem::Path && p) -> SingleDerivedPath {
2515+
error<EvalError>(
2516+
"string '%s' has no context",
2517+
s).withTrace(pos, errorCtx).debugThrow();
2518+
},
25022519
}, ((NixStringContextElem &&) *context.begin()).raw);
25032520
return {
25042521
std::move(derivedPath),

‎src/libexpr/include/nix/expr/value/context.hh‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,35 @@ struct NixStringContextElem {
5454
*/
5555
using Built = SingleDerivedPath::Built;
5656

57+
/**
58+
* A store path that will not result in a store reference when
59+
* used in a derivation or toFile.
60+
*
61+
* When you apply `builtins.toString` to a path value representing
62+
* a path in the Nix store (as is the case with flake inputs),
63+
* historically you got a string without context
64+
* (e.g. `/nix/store/...-source`). This is broken, since it allows
65+
* you to pass a store path to a derivation/toFile without a
66+
* proper store reference. This is especially a problem with lazy
67+
* trees, since the store path is a virtual path that doesn't
68+
* exist.
69+
*
70+
* For backwards compatibility, and to warn users about this
71+
* unsafe use of `toString`, we keep track of such strings as a
72+
* special type of context.
73+
*/
74+
struct Path
75+
{
76+
StorePath storePath;
77+
78+
GENERATE_CMP(Path, me->storePath);
79+
};
80+
5781
using Raw = std::variant<
5882
Opaque,
5983
DrvDeep,
60-
Built
84+
Built,
85+
Path
6186
>;
6287

6388
Raw raw;
@@ -82,4 +107,10 @@ struct NixStringContextElem {
82107

83108
typedef std::set<NixStringContextElem> NixStringContext;
84109

110+
/**
111+
* Returns false if `context` has no elements other than
112+
* `NixStringContextElem::Path`.
113+
*/
114+
bool hasContext(const NixStringContext & context);
115+
85116
}

‎src/libexpr/primops.cc‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,9 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
8989
if (maybePathsOut)
9090
maybePathsOut->emplace(d.drvPath);
9191
},
92+
[&](const NixStringContextElem::Path & p) {
93+
// FIXME: do something?
94+
},
9295
}, c.raw);
9396
}
9497

@@ -1414,6 +1417,8 @@ static void derivationStrictInternal(
14141417
derivation. */
14151418
StringMap rewrites;
14161419

1420+
std::optional<std::string> drvS;
1421+
14171422
for (auto & c : context) {
14181423
std::visit(overloaded {
14191424
/* Since this allows the builder to gain access to every
@@ -1438,6 +1443,17 @@ static void derivationStrictInternal(
14381443
[&](const NixStringContextElem::Opaque & o) {
14391444
drv.inputSrcs.insert(state.devirtualize(o.path, &rewrites));
14401445
},
1446+
[&](const NixStringContextElem::Path & p) {
1447+
if (!drvS) drvS = drv.unparse(*state.store, true);
1448+
if (drvS->find(p.storePath.to_string()) != drvS->npos) {
1449+
auto devirtualized = state.devirtualize(p.storePath, &rewrites);
1450+
warn(
1451+
"Using 'builtins.derivation' to create a derivation named '%s' that references the store path '%s' without a proper context. "
1452+
"The resulting derivation will not have a correct store reference, so this is unreliable and may stop working in the future.",
1453+
drvName,
1454+
state.store->printStorePath(devirtualized));
1455+
}
1456+
},
14411457
}, c.raw);
14421458
}
14431459

@@ -2346,10 +2362,21 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
23462362
std::string contents(state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.toFile"));
23472363

23482364
StorePathSet refs;
2365+
StringMap rewrites;
23492366

23502367
for (auto c : context) {
23512368
if (auto p = std::get_if<NixStringContextElem::Opaque>(&c.raw))
23522369
refs.insert(p->path);
2370+
else if (auto p = std::get_if<NixStringContextElem::Path>(&c.raw)) {
2371+
if (contents.find(p->storePath.to_string()) != contents.npos) {
2372+
auto devirtualized = state.devirtualize(p->storePath, &rewrites);
2373+
warn(
2374+
"Using 'builtins.toFile' to create a file named '%s' that references the store path '%s' without a proper context. "
2375+
"The resulting file will not have a correct store reference, so this is unreliable and may stop working in the future.",
2376+
name,
2377+
state.store->printStorePath(devirtualized));
2378+
}
2379+
}
23532380
else
23542381
state.error<EvalError>(
23552382
"files created by %1% may not reference derivations, but %2% references %3%",
@@ -2359,6 +2386,8 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
23592386
).atPos(pos).debugThrow();
23602387
}
23612388

2389+
contents = rewriteStrings(contents, rewrites);
2390+
23622391
auto storePath = settings.readOnlyMode
23632392
? state.store->makeFixedOutputPathFromCA(name, TextInfo {
23642393
.hash = hashString(HashAlgorithm::SHA256, contents),

‎src/libexpr/primops/context.cc‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,15 @@ namespace nix {
77

88
static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos, Value * * args, Value & v)
99
{
10-
NixStringContext context;
10+
NixStringContext context, filtered;
11+
1112
auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext");
12-
v.mkString(*s);
13+
14+
for (auto & c : context)
15+
if (auto * p = std::get_if<NixStringContextElem::Path>(&c.raw))
16+
filtered.insert(*p);
17+
18+
v.mkString(*s, filtered);
1319
}
1420

1521
static RegisterPrimOp primop_unsafeDiscardStringContext({
@@ -21,12 +27,19 @@ static RegisterPrimOp primop_unsafeDiscardStringContext({
2127
.fun = prim_unsafeDiscardStringContext,
2228
});
2329

30+
bool hasContext(const NixStringContext & context)
31+
{
32+
for (auto & c : context)
33+
if (!std::get_if<NixStringContextElem::Path>(&c.raw))
34+
return true;
35+
return false;
36+
}
2437

2538
static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, Value & v)
2639
{
2740
NixStringContext context;
2841
state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.hasContext");
29-
v.mkBool(!context.empty());
42+
v.mkBool(hasContext(context));
3043
}
3144

3245
static RegisterPrimOp primop_hasContext({
@@ -103,7 +116,7 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V
103116
NixStringContext context;
104117
auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.addDrvOutputDependencies");
105118

106-
auto contextSize = context.size();
119+
auto contextSize = context.size();
107120
if (contextSize != 1) {
108121
state.error<EvalError>(
109122
"context of string '%s' must have exactly one element, but has %d",
@@ -136,6 +149,11 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V
136149
above does not make much sense. */
137150
return std::move(c);
138151
},
152+
[&](const NixStringContextElem::Path & p) -> NixStringContextElem::DrvDeep {
153+
state.error<EvalError>(
154+
"`addDrvOutputDependencies` does not work on a string without context"
155+
).atPos(pos).debugThrow();
156+
},
139157
}, context.begin()->raw) }),
140158
};
141159

@@ -206,6 +224,8 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
206224
[&](NixStringContextElem::Opaque && o) {
207225
contextInfos[std::move(o.path)].path = true;
208226
},
227+
[&](NixStringContextElem::Path && p) {
228+
},
209229
}, ((NixStringContextElem &&) i).raw);
210230
}
211231

‎src/libexpr/print.cc‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,11 @@ class Printer
249249

250250
void printString(Value & v)
251251
{
252-
printLiteralString(output, v.string_view(), options.maxStringLength, options.ansiColors);
252+
NixStringContext context;
253+
copyContext(v, context);
254+
std::ostringstream s;
255+
printLiteralString(s, v.string_view(), options.maxStringLength, options.ansiColors);
256+
output << state.devirtualize(s.str(), context);
253257
}
254258

255259
void printPath(Value & v)

‎src/libexpr/value-to-json.cc‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,10 @@
77
#include <iomanip>
88
#include <nlohmann/json.hpp>
99

10-
1110
namespace nix {
11+
1212
using json = nlohmann::json;
13+
1314
json printValueAsJSON(EvalState & state, bool strict,
1415
Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore)
1516
{
@@ -31,9 +32,7 @@ json printValueAsJSON(EvalState & state, bool strict,
3132

3233
case nString:
3334
copyContext(v, context);
34-
// FIXME: only use the context from `v`.
35-
// FIXME: make devirtualization configurable?
36-
out = state.devirtualize(v.c_str(), context);
35+
out = v.c_str();
3736
break;
3837

3938
case nPath:

0 commit comments

Comments
 (0)