Skip to content

Commit 86040a7

Browse files
autarchautarch
committed
Use a new S3 bucket instead of mciuploads
This is being done for two reasons. First, we want to move projects away from all using the shared `mciuploads` bucket. Second, the credentials to access `mciuploads` are only available for the DB Tools Evergreen project for the `master` branch. When we make a PR off a different branch, we cannot see those credentials. That means those branches cannot interact with S3 in CI, which leads to lots of CI failures. Switching to a new bucket and using `ec2.assume_role` to access that bucket means this works on any branch.
1 parent 8202f34 commit 86040a7

2 files changed

Lines changed: 65 additions & 65 deletions

File tree

‎common.yml‎

Lines changed: 13 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -265,8 +265,7 @@ functions:
265265
# upload individual release artifacts to task page
266266
- command: s3.put
267267
params:
268-
aws_key: ${aws_key}
269-
aws_secret: ${aws_secret}
268+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
270269
local_files_include_filter:
271270
- src/github.com/mongodb/mongo-tools/mongodb-database-tools*.deb
272271
- src/github.com/mongodb/mongo-tools/mongodb-database-tools*.msi
@@ -275,7 +274,7 @@ functions:
275274
- src/github.com/mongodb/mongo-tools/mongodb-database-tools*.zip
276275
remote_file: mongo-tools/pkgs/${build_id}/
277276
content_type: application/octet-stream
278-
bucket: mciuploads
277+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
279278
permissions: public-read
280279
display_name: "Release Artifact - "
281280

@@ -291,25 +290,24 @@ functions:
291290
- mongodb-database-tools*.rpm
292291
- mongodb-database-tools*.tgz
293292
- mongodb-database-tools*.zip
293+
294294
- command: s3.put
295295
params:
296-
aws_key: ${aws_key}
297-
aws_secret: ${aws_secret}
296+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
298297
local_file: src/github.com/mongodb/mongo-tools/upload.tgz
299298
remote_file: mongo-tools/task/dist/${build_id}/all-release-artifacts.tgz
300299
content_type: application/x-gzip
301-
bucket: mciuploads
300+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
302301
permissions: public-read
303302
display_name: All Release Artifacts (.tgz)
304303

305304
"fetch dist release artifacts":
306305
- command: s3.get
307306
params:
308-
aws_key: ${aws_key}
309-
aws_secret: ${aws_secret}
307+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
310308
remote_file: mongo-tools/task/dist/${build_id}/all-release-artifacts.tgz
311309
extract_to: src/github.com/mongodb/mongo-tools/
312-
bucket: mciuploads
310+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
313311

314312
"sign artifacts":
315313
command: shell.exec
@@ -335,10 +333,10 @@ functions:
335333
working_dir: src/github.com/mongodb/mongo-tools
336334
script: |
337335
rm -rf ./mongorestore/testdata/longcollectionname/
336+
338337
- command: s3.put
339338
params:
340-
aws_key: ${aws_key}
341-
aws_secret: ${aws_secret}
339+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
342340
local_files_include_filter:
343341
- src/github.com/mongodb/mongo-tools/mongodb-database-tools*.sig
344342
- src/github.com/mongodb/mongo-tools/mongodb-database-tools*.deb
@@ -347,7 +345,7 @@ functions:
347345
- src/github.com/mongodb/mongo-tools/mongodb-database-tools*.tgz
348346
- src/github.com/mongodb/mongo-tools/mongodb-database-tools*.zip
349347
remote_file: mongo-tools/task/sign/${build_id}/
350-
bucket: mciuploads
348+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
351349
permissions: public-read
352350
content_type: application/octet-stream
353351

@@ -366,15 +364,15 @@ functions:
366364
script: |
367365
${_set_shell_env}
368366
go run release/release.go upload-json
367+
369368
- command: s3.put
370369
params:
371-
aws_key: ${aws_key}
372-
aws_secret: ${aws_secret}
370+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
373371
local_file: src/github.com/mongodb/mongo-tools/release.json
374372
remote_file: mongo-tools/release/${build_id}/
375373
optional: true
376374
content_type: application/json
377-
bucket: mciuploads
375+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
378376
permissions: public-read
379377

380378
"generate full JSON feed":

‎mongodump_passthrough/functions.yml‎

Lines changed: 52 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,7 @@ variables:
6767
f_resmoke_with_binaries_fetch: &f_resmoke_with_binaries_fetch
6868
command: s3.get
6969
params:
70-
aws_key: ${aws_key}
71-
aws_secret: ${aws_secret}
70+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
7271
# Changed for mongodump_passthrough
7372
remote_file: mongo-tools/mongodump_passthrough/${build_variant}/${revision}/resmoke-with-binaries/${build_id}.tgz
7473
bucket: mciuploads
@@ -77,21 +76,21 @@ variables:
7776
f_resmoke_wheelhouse_fetch: &f_resmoke_wheelhouse_fetch
7877
command: s3.get
7978
params:
80-
aws_key: ${aws_key}
81-
aws_secret: ${aws_secret}
79+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
8280
# Changed for mongodump_passthrough
8381
remote_file: mongo-tools/mongodump_passthrough/${build_variant}/${revision}/resmoke-python-wheelhouse/${build_id}.tgz
84-
bucket: mciuploads
82+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
8583
extract_to: "."
8684

8785
f_migration_verifier_binary_fetch: &f_migration_verifier_binary_fetch
8886
command: s3.get
8987
params:
90-
aws_key: ${aws_key}
91-
aws_secret: ${aws_secret}
88+
aws_key: ${AWS_KEY}
89+
aws_secret: ${AWS_SECRET}
90+
aws_session_token: ${AWS_SESSION_TOKEN}
9291
# Changed for mongodump_passthrough
9392
remote_file: mongo-tools/mongodump_passthrough/${mongosync_compile_build_variant}/${revision}/${version_id}/migration_verifier
94-
bucket: mciuploads
93+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
9594
local_file: src/mongosync/migration_verifier
9695

9796
f_make_migration_verifier_binary_executable:
@@ -113,11 +112,12 @@ variables:
113112
f_mongosync_binary_fetch: &f_mongosync_binary_fetch
114113
command: s3.get
115114
params:
116-
aws_key: ${aws_key}
117-
aws_secret: ${aws_secret}
115+
aws_key: ${AWS_KEY}
116+
aws_secret: ${AWS_SECRET}
117+
aws_session_token: ${AWS_SESSION_TOKEN}
118118
# Changed for mongodump_passthrough
119119
remote_file: mongo-tools/mongodump_passthrough/${mongosync_compile_build_variant}/${revision}/${version_id}/${mongosync_binary_folder}/${version_id}.tgz
120-
bucket: mciuploads
120+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
121121
extract_to: "src/mongosync"
122122

123123
f_generate_github_access_token: &f_generate_github_access_token
@@ -355,14 +355,14 @@ functions:
355355
- "./src/**"
356356
- "./.resmoke_mongo_version.yml"
357357
- "./.resmoke_mongo_release_values.yml"
358+
358359
- command: s3.put
359360
params:
360-
aws_key: ${aws_key}
361-
aws_secret: ${aws_secret}
361+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
362362
local_file: resmoke-with-binaries.tgz
363363
# Changed for mongodump_passthrough
364364
remote_file: mongo-tools/mongodump_passthrough/${build_variant}/${revision}/resmoke-with-binaries/${build_id}.tgz
365-
bucket: mciuploads
365+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
366366
permissions: private
367367
visibility: signed
368368
content_type: application/gzip
@@ -375,14 +375,14 @@ functions:
375375
include:
376376
- "./wheelhouse/**"
377377
- "./poetry-requirements.txt"
378+
378379
- command: s3.put
379380
params:
380-
aws_key: ${aws_key}
381-
aws_secret: ${aws_secret}
381+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
382382
local_file: resmoke-python-wheelhouse.tgz
383383
# Changed for mongodump_passthrough
384384
remote_file: mongo-tools/mongodump_passthrough/${build_variant}/${revision}/resmoke-python-wheelhouse/${build_id}.tgz
385-
bucket: mciuploads
385+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
386386
permissions: private
387387
visibility: signed
388388
content_type: application/gzip
@@ -407,16 +407,16 @@ functions:
407407
PATH=$PATH:$HOME:/opt/golang/go1.23/bin
408408
GOROOT=/opt/golang/go1.23
409409
./build.sh
410+
410411
# upload the compiled verifier to S3.
411412
- command: s3.put
412413
params:
413-
aws_key: ${aws_key}
414-
aws_secret: ${aws_secret}
414+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
415415
local_file: src/migration-verifier/migration_verifier
416416
# Changed for mongodump_passthrough
417417
remote_file: mongo-tools/mongodump_passthrough/${build_variant}/${revision}/${version_id}/migration_verifier
418418
content_type: application/x-executable
419-
bucket: mciuploads
419+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
420420
permissions: public-read
421421
display_name: "migration_verifier release artifact (compiled)"
422422

@@ -437,14 +437,14 @@ functions:
437437
source_dir: "src/mongosync"
438438
include:
439439
- "./dist/*"
440+
440441
- command: s3.put
441442
params:
442-
aws_key: ${aws_key}
443-
aws_secret: ${aws_secret}
443+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
444444
local_file: ${mongosync_binary_folder}.tgz
445445
# Changed for mongodump_passthrough
446446
remote_file: mongo-tools/mongodump_passthrough/${build_variant}/${revision}/${version_id}/${mongosync_binary_folder}/${version_id}.tgz
447-
bucket: mciuploads
447+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
448448
permissions: public-read
449449
content_type: application/gzip
450450
display_name: ${mongosync_binary_folder}
@@ -487,20 +487,21 @@ functions:
487487
params:
488488
binary: "./src/mongosync/evergreen/scripts/mongo_coredumps_gather.sh"
489489
add_expansions_to_env: true
490+
490491
- command: archive.targz_pack
491492
params:
492493
target: mongo-coredumps.tgz
493494
source_dir: *_resmoke_dir
494495
include:
495496
- "./**.core"
496497
- "./**.mdmp" # Windows: minidumps
498+
497499
- command: s3.put
498500
params:
499-
aws_key: ${aws_key}
500-
aws_secret: ${aws_secret}
501+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
501502
local_file: mongo-coredumps.tgz
502503
remote_file: mongosync/${build_variant}/${revision}/coredumps/mongo-coredumps-${build_id}-${task_name}-${execution}.tgz
503-
bucket: mciuploads
504+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
504505
permissions: public-read
505506
content_type: application/gzip
506507
display_name: Core Dumps - Execution ${execution}
@@ -520,13 +521,13 @@ functions:
520521
source_dir: "src/mongosync/mongosync-testing-server-logs"
521522
include:
522523
- "./**"
524+
523525
- command: s3.put
524526
params:
525-
aws_key: ${aws_key}
526-
aws_secret: ${aws_secret}
527+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
527528
local_file: server-logs.tgz
528529
remote_file: mongosync/${build_variant}/${revision}/mongo-logs/mongo-logs-${build_id}-${task_name}-${execution}.tgz
529-
bucket: mciuploads
530+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
530531
permissions: public-read
531532
content_type: application/gzip
532533
display_name: Integration Test mongod / mongos logs ${execution}
@@ -539,50 +540,47 @@ functions:
539540
include:
540541
- "out.log"
541542
- "server.log"
543+
542544
- command: s3.put
543545
params:
544-
aws_key: ${aws_key}
545-
aws_secret: ${aws_secret}
546+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
546547
local_file: mongo-orchestration-logs.tgz
547548
remote_file: mongosync/${build_variant}/${revision}/mongo-orchestration-logs/mongo-orchestration-logs.tgz
548-
bucket: mciuploads
549+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
549550
permissions: public-read
550551
content_type: application/gzip
551552
display_name: mongo-orchestration logs
552553
optional: true
553554

554555
- command: s3.put
555556
params:
556-
aws_key: ${aws_key}
557-
aws_secret: ${aws_secret}
557+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
558558
local_files_include_filter:
559559
- src/mongosync/*.suite
560560
remote_file: mongosync/${build_variant}/${revision}/${build_id}/${task_name}/${execution}/go-test-logs/
561-
bucket: mciuploads
561+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
562562
permissions: public-read
563563
content_type: text/plain
564564
display_name: "Output from `go test` command"
565565

566566
- command: s3.put
567567
params:
568-
aws_key: ${aws_key}
569-
aws_secret: ${aws_secret}
568+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
570569
local_file: src/mongosync/server-binaries.tgz
571570
remote_file: mongosync/${build_variant}/${revision}/mongo-binaries/mongo-binaries-${build_id}-${task_name}-${execution}.tgz
572-
bucket: mciuploads
571+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
573572
permissions: public-read
574573
content_type: application/gzip
575574
display_name: Failed Integration Test mongod / mongos binaries ${execution}
576575
optional: true
577576

578577
- command: s3.put
579578
params:
580-
aws_key: ${aws_key}
581-
aws_secret: ${aws_secret}
579+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
582580
local_files_include_filter:
583581
- src/mongosync/mongosync-testing-server-data-files/archive/*.tgz
584582
remote_file: mongosync/${build_variant}/${revision}/${build_id}/${task_name}/datafiles/
585-
bucket: mciuploads
583+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
586584
permissions: public-read
587585
content_type: application/octet-stream
588586
display_name: Failed Integration Test Data Files ${execution}
@@ -593,13 +591,13 @@ functions:
593591
source_dir: "src/mongosync/coverage"
594592
include:
595593
- "./**"
594+
596595
- command: s3.put
597596
params:
598-
aws_key: ${aws_key}
599-
aws_secret: ${aws_secret}
597+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
600598
local_file: coverage.gz
601599
remote_file: mongosync/${build_variant}/${revision}/coverage/${task_name}/coverage.gz
602-
bucket: mciuploads
600+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
603601
permissions: public-read
604602
content_type: application/gzip
605603
display_name: Coverage statistics
@@ -610,13 +608,13 @@ functions:
610608
source_dir: "src/resmoke/tools_output"
611609
include:
612610
- "**"
611+
613612
- command: s3.put
614613
params:
615-
aws_key: ${aws_key}
616-
aws_secret: ${aws_secret}
614+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
617615
local_file: tools_output.tgz
618616
remote_file: mongo-tools/${build_variant}/${revision}/tools_output/${task_name}/${build_id}/tools_output.tgz
619-
bucket: mciuploads
617+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
620618
permissions: public-read
621619
content_type: application/gzip
622620
display_name: Tools Output
@@ -638,36 +636,40 @@ functions:
638636
"run jstestfuzz":
639637
- *f_expansions_write
640638
- *f_generate_github_access_token
639+
641640
- command: github.generate_token
642641
params:
643642
owner: 10gen
644643
repo: qa
645644
expansion_name: generated_token_qa
646645
permissions:
647646
contents: read
647+
648648
- command: subprocess.exec
649649
params:
650650
binary: "./src/mongosync/evergreen/scripts/clone_jstestfuzz_jstest_repos.sh"
651651
add_expansions_to_env: true
652+
652653
- command: subprocess.exec
653654
type: test
654655
params:
655656
binary: "./src/mongosync/evergreen/scripts/run_jstestfuzz.sh"
656657
add_expansions_to_env: true
658+
657659
- command: archive.targz_pack
658660
params:
659661
target: "jstests.tgz"
660662
source_dir: "src/jstestfuzz"
661663
include:
662664
- "out/*.js"
665+
663666
- command: s3.put
664667
params:
665-
aws_key: ${aws_key}
666-
aws_secret: ${aws_secret}
668+
role_arn: arn:aws:iam::391144487543:role/evergreen-project-mongo-tools
667669
local_file: jstests.tgz
668670
# Changed for mongodump_passthrough
669671
remote_file: mongo-tools/mongodump_passthrough/${build_variant}/${revision}/jstestfuzz/${task_id}-${execution}.tgz
670-
bucket: mciuploads
672+
bucket: evergreen-project-mongo-tools-i6qg5nn6nbm
671673
permissions: private
672674
visibility: signed
673675
content_type: application/gzip

0 commit comments

Comments
 (0)