Skip to content

Commit c04c71f

Browse files
wolfvwolfv
committed
add prefix upload
1 parent 0e75b96 commit c04c71f

1 file changed

Lines changed: 143 additions & 51 deletions

File tree

‎vinca/generate_gha.py‎

Lines changed: 143 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,13 @@ def parse_command_line(argv):
7575
help="How many packages to build at most per stage",
7676
)
7777

78+
parser.add_argument(
79+
"--prefix-channel",
80+
dest="prefix_channel",
81+
default=None,
82+
help="Publish to this prefix.dev channel using trusted publishing (replaces anaconda upload)",
83+
)
84+
7885
arguments = parser.parse_args(argv[1:])
7986
config.parsed_args = arguments
8087
return arguments
@@ -250,6 +257,7 @@ def build_unix_pipeline(
250257
outfile="linux.yml",
251258
pipeline_name="build_unix",
252259
target="",
260+
prefix_channel=None,
253261
):
254262
blurb = {"jobs": {}, "name": pipeline_name}
255263

@@ -266,28 +274,63 @@ def build_unix_pipeline(
266274
batch_keys.append(batch_key)
267275

268276
pretty_stage_name = get_stage_name(batch)
269-
azure_template["jobs"][batch_key] = {
277+
278+
build_env = {
279+
"CURRENT_RECIPES": f"{' '.join([pkg for pkg in batch])}",
280+
"BUILD_TARGET": target,
281+
}
282+
if prefix_channel:
283+
build_env["PREFIX_UPLOAD_CHANNEL"] = prefix_channel
284+
else:
285+
build_env["ANACONDA_API_TOKEN"] = "${{ secrets.ANACONDA_API_TOKEN }}"
286+
287+
steps = [
288+
{
289+
"name": "Checkout code",
290+
"uses": "actions/checkout@v6",
291+
},
292+
{
293+
"name": f"Build {' '.join([pkg for pkg in batch])}",
294+
"env": build_env,
295+
"run": script,
296+
},
297+
]
298+
299+
if prefix_channel:
300+
target_glob = f"{target}*" if target else "*"
301+
upload_script = lu(
302+
f"if compgen -G \"$HOME/conda-bld/{target_glob}/*.conda\" > /dev/null; then\n"
303+
f" ~/.pixi/bin/rattler-build upload prefix"
304+
f" --channel {prefix_channel}"
305+
f" --generate-attestation"
306+
f" $HOME/conda-bld/{target_glob}/*.conda\n"
307+
f"else\n"
308+
f" echo \"No packages to upload to prefix.dev\"\n"
309+
f"fi\n"
310+
)
311+
steps.append(
312+
{
313+
"name": "Upload to prefix.dev",
314+
"run": upload_script,
315+
}
316+
)
317+
318+
job = {
270319
"name": pretty_stage_name,
271320
"runs-on": runs_on,
272321
"strategy": {"fail-fast": False},
273322
"needs": prev_batch_keys,
274-
"steps": [
275-
{
276-
"name": "Checkout code",
277-
"uses": "actions/checkout@v6",
278-
},
279-
{
280-
"name": f"Build {' '.join([pkg for pkg in batch])}",
281-
"env": {
282-
"ANACONDA_API_TOKEN": "${{ secrets.ANACONDA_API_TOKEN }}",
283-
"CURRENT_RECIPES": f"{' '.join([pkg for pkg in batch])}",
284-
"BUILD_TARGET": target, # use for cross-compilation
285-
},
286-
"run": script,
287-
},
288-
],
323+
"steps": steps,
289324
}
290325

326+
if prefix_channel:
327+
job["permissions"] = {
328+
"id-token": "write",
329+
"attestations": "write",
330+
}
331+
332+
azure_template["jobs"][batch_key] = job
333+
291334
prev_batch_keys = batch_keys
292335

293336
if len(azure_template.get("jobs", [])) == 0:
@@ -306,6 +349,7 @@ def build_linux_pipeline(
306349
runs_on="ubuntu-latest",
307350
outfile="linux.yml",
308351
pipeline_name="build_linux",
352+
prefix_channel=None,
309353
):
310354
build_unix_pipeline(
311355
stages,
@@ -316,6 +360,7 @@ def build_linux_pipeline(
316360
outfile=outfile,
317361
pipeline_name=pipeline_name,
318362
target="linux-64",
363+
prefix_channel=prefix_channel,
319364
)
320365

321366

@@ -328,6 +373,7 @@ def build_osx_pipeline(
328373
script=azure_unix_script,
329374
target="osx-64",
330375
pipeline_name="build_osx64",
376+
prefix_channel=None,
331377
):
332378
build_unix_pipeline(
333379
stages,
@@ -338,10 +384,13 @@ def build_osx_pipeline(
338384
outfile=outfile,
339385
target=target,
340386
pipeline_name=pipeline_name,
387+
prefix_channel=prefix_channel,
341388
)
342389

343390

344-
def build_win_pipeline(stages, trigger_branch, outfile="win.yml", azure_template=None):
391+
def build_win_pipeline(
392+
stages, trigger_branch, outfile="win.yml", azure_template=None, prefix_channel=None
393+
):
345394
vm_imagename = "windows-2022"
346395
# Build Win pipeline
347396
blurb = {"jobs": {}, "name": "build_win"}
@@ -365,7 +414,61 @@ def build_win_pipeline(stages, trigger_branch, outfile="win.yml", azure_template
365414
batch_keys.append(batch_key)
366415

367416
pretty_stage_name = get_stage_name(batch)
368-
azure_template["jobs"][batch_key] = {
417+
418+
build_env = {
419+
"CURRENT_RECIPES": f"{' '.join([pkg for pkg in batch])}",
420+
"PYTHONUNBUFFERED": 1,
421+
}
422+
if prefix_channel:
423+
build_env["PREFIX_UPLOAD_CHANNEL"] = prefix_channel
424+
else:
425+
build_env["ANACONDA_API_TOKEN"] = "${{ secrets.ANACONDA_API_TOKEN }}"
426+
427+
steps = [
428+
{"name": "Checkout code", "uses": "actions/checkout@v6"},
429+
{
430+
"name": "Setup pixi",
431+
"uses": "prefix-dev/setup-pixi@v0.9.4",
432+
"with": {
433+
"pixi-version": "v0.63.2",
434+
"cache": "true",
435+
},
436+
},
437+
{
438+
"uses": "egor-tensin/cleanup-path@v5",
439+
"with": {
440+
"dirs": "C:\\Program Files\\Git\\usr\\bin;C:\\Program Files\\Git\\bin;C:\\Program Files\\Git\\cmd;C:\\Program Files\\Git\\mingw64\\bin"
441+
},
442+
},
443+
{
444+
"shell": "cmd",
445+
"run": azure_win_preconfig_script,
446+
"name": "conda-forge build setup",
447+
},
448+
{
449+
"shell": "cmd",
450+
"run": script,
451+
"env": build_env,
452+
"name": f"Build {' '.join([pkg for pkg in batch])}",
453+
},
454+
]
455+
456+
if prefix_channel:
457+
upload_script = lu(
458+
f"pixi run rattler-build upload prefix"
459+
f" --channel {prefix_channel}"
460+
f" --generate-attestation"
461+
f" C:\\bld\\win-64\\*.conda\n"
462+
)
463+
steps.append(
464+
{
465+
"shell": "cmd",
466+
"name": "Upload to prefix.dev",
467+
"run": upload_script,
468+
}
469+
)
470+
471+
job = {
369472
"name": pretty_stage_name,
370473
"runs-on": vm_imagename,
371474
"strategy": {"fail-fast": False},
@@ -374,40 +477,17 @@ def build_win_pipeline(stages, trigger_branch, outfile="win.yml", azure_template
374477
"CONDA_BLD_PATH": "C:\\\\bld\\\\",
375478
"VINCA_CUSTOM_CMAKE_BUILD_DIR": "C:\\\\x\\\\",
376479
},
377-
"steps": [
378-
{"name": "Checkout code", "uses": "actions/checkout@v6"},
379-
{
380-
"name": "Setup pixi",
381-
"uses": "prefix-dev/setup-pixi@v0.9.4",
382-
"with": {
383-
"pixi-version": "v0.63.2",
384-
"cache": "true",
385-
},
386-
},
387-
{
388-
"uses": "egor-tensin/cleanup-path@v5",
389-
"with": {
390-
"dirs": "C:\\Program Files\\Git\\usr\\bin;C:\\Program Files\\Git\\bin;C:\\Program Files\\Git\\cmd;C:\\Program Files\\Git\\mingw64\\bin"
391-
},
392-
},
393-
{
394-
"shell": "cmd",
395-
"run": azure_win_preconfig_script,
396-
"name": "conda-forge build setup",
397-
},
398-
{
399-
"shell": "cmd",
400-
"run": script,
401-
"env": {
402-
"ANACONDA_API_TOKEN": "${{ secrets.ANACONDA_API_TOKEN }}",
403-
"CURRENT_RECIPES": f"{' '.join([pkg for pkg in batch])}",
404-
"PYTHONUNBUFFERED": 1,
405-
},
406-
"name": f"Build {' '.join([pkg for pkg in batch])}",
407-
},
408-
],
480+
"steps": steps,
409481
}
410482

483+
if prefix_channel:
484+
job["permissions"] = {
485+
"id-token": "write",
486+
"attestations": "write",
487+
}
488+
489+
azure_template["jobs"][batch_key] = job
490+
411491
prev_batch_keys = batch_keys
412492

413493
if len(azure_template.get("jobs", [])) == 0:
@@ -551,18 +631,22 @@ def main():
551631

552632
fo.write("\n".join(order))
553633

634+
prefix_channel = args.prefix_channel
635+
554636
if args.platform == "linux-64":
555637
build_unix_pipeline(
556638
stages,
557639
args.trigger_branch,
558640
outfile="linux.yml",
559641
pipeline_name="build_linux64",
642+
prefix_channel=prefix_channel,
560643
)
561644

562645
if args.platform == "osx-64":
563646
build_osx_pipeline(
564647
stages,
565648
args.trigger_branch,
649+
prefix_channel=prefix_channel,
566650
)
567651

568652
if args.platform == "osx-arm64":
@@ -574,6 +658,7 @@ def main():
574658
script=azure_unix_script,
575659
target=platform,
576660
pipeline_name="build_osx_arm64",
661+
prefix_channel=prefix_channel,
577662
)
578663

579664
if args.platform == "linux-aarch64":
@@ -585,11 +670,17 @@ def main():
585670
outfile="linux_aarch64.yml",
586671
target=platform,
587672
pipeline_name="build_linux_aarch64",
673+
prefix_channel=prefix_channel,
588674
)
589675

590676
# windows
591677
if args.platform == "win-64":
592-
build_win_pipeline(stages, args.trigger_branch, outfile="win.yml")
678+
build_win_pipeline(
679+
stages,
680+
args.trigger_branch,
681+
outfile="win.yml",
682+
prefix_channel=prefix_channel,
683+
)
593684

594685
if args.platform == "emscripten-wasm32":
595686
build_unix_pipeline(
@@ -598,4 +689,5 @@ def main():
598689
outfile="emscripten_wasm32.yml",
599690
pipeline_name="build_emscripten_wasm32",
600691
target="emscripten-wasm32",
692+
prefix_channel=prefix_channel,
601693
)

0 commit comments

Comments
 (0)