Skip to content

Commit 42d52dd

Browse files
RLIndiaRLIndia
committed
Merge branch 'topic-release' of https://github.com/RLOpenCatalyst/rgdeploy into topic-release
2 parents a988e5c + 76486d8 commit 42d52dd

4 files changed

Lines changed: 82 additions & 87 deletions

File tree

‎README.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,13 +69,13 @@ If an AMI with the pre-requisites has been shared with you, you can skip this se
6969

7070
You can create the AMI with pre-requisites yourself by following these steps:
7171
- Install packer
72-
- Create a Role and attach a policy which permits ECR and EC2 actions and Replace it in builders section
7372
- Export AWS Access Keys and Secret Keys
7473
- export AWS_ACCESS_KEY_ID="your_Access_Key"
7574
- export AWS_SECRET_ACCESS_KEY="your_Secret_Key"
7675
- export AWS_DEFAULT_REGION="Your_Region"
77-
- Download dump.tar.gz to your local rdeploy folder from s3://rg-deployment-docs
78-
- Run packer build package-rg.json.
76+
- Clone this repo on a machine.
77+
- Create a Role and attach a policy which permits ECR and EC2 actions and Replace the "iam-instance_profile" :"<your_rolename>" in builders section which is in the packer-rg.json.
78+
- Run packer build packer-rg.json
7979
- packer build -var 'awsRegion=your_region' -var 'vpcId=your_VPCID' -var 'subnetId=your_SubnetID' packer-rg.json
8080
- At Run time pass VPCID, SubnetID, AWSRegion as variables declared in packer-rg.json
8181
- Note that AMI id from the output

‎dump/studies.json‎

Lines changed: 77 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,18 @@
2121
"sharedTo": [],
2222
"name": "1000 Genomes",
2323
"studyType": "Public",
24-
"resourceDetails": [],
24+
"resourceDetails": [{
25+
"description": "http://www.internationalgenome.org/formats",
26+
"type": "s3",
27+
"arn": "arn:aws:s3:::1000genomes",
28+
"region": "us-east-1"
29+
}],
2530
"moreInformation": "License: Data from the 1000 Genomes Project is now available without embargo, following the final publication from the project. Use of the data should be cited in the usual way, with current details available at http://www.internationalgenome.org/faq/how-do-i-cite-1000-genomes-project",
26-
"createdBy": "",
2731
"__v": 0,
2832
"isDeleted": false,
2933
"repositoryName": "Registry of Open Data on AWS",
3034
"isShared": false,
31-
"isLinked": false,
32-
"accountId": ""
35+
"isLinked": false
3336
}
3437
{
3538
"_id": {
@@ -55,16 +58,19 @@
5558
"sharedTo": [],
5659
"name": "IRS 990 Filings (Spreadsheets)",
5760
"studyType": "Public",
58-
"resourceDetails": [],
61+
"resourceDetails": [{
62+
"description": "Excerpts of electronic Form 990 and 990-EZ filings, converted to spreadsheet form",
63+
"type": "s3",
64+
"arn": "arn:aws:s3:::irs-990-spreadsheets",
65+
"region": "us-east-1"
66+
}],
5967
"moreInformation": "License:Attribution 4.0 International [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/)",
6068
"__v": 0,
6169
"isDeleted": false,
6270
"linkedResources": [],
6371
"isLinked": false,
6472
"repositoryName": "Registry of Open Data on AWS",
65-
"isShared": false,
66-
"createdBy": "",
67-
"accountId": ""
73+
"isShared": false
6874
}
6975
{
7076
"_id": {
@@ -91,15 +97,18 @@
9197
"sharedTo": [],
9298
"name": "ChEMBL",
9399
"studyType": "Public",
94-
"resourceDetails": [],
100+
"resourceDetails" : [{
101+
"description" : "ChEMBL 27",
102+
"type" : "s3",
103+
"arn" : "arn:aws:s3:::aws-roda-hcls-datalake",
104+
"region" : "us-east-1"
105+
}],
95106
"moreInformation": "License:https://github.com/aws-samples/data-lake-as-code/blob/roda/docs/roda_attributions.txt",
96107
"__v": 0,
97108
"isDeleted": false,
98109
"repositoryName": "Registry of Open Data on AWS",
99110
"isShared": false,
100-
"createdBy": "",
101-
"isLinked": false,
102-
"accountId": ""
111+
"isLinked": false
103112
}
104113
{
105114
"_id": {
@@ -127,16 +136,19 @@
127136
"sharedTo": [],
128137
"name": "Open Targets",
129138
"studyType": "Public",
130-
"resourceDetails": [],
139+
"resourceDetails" : [{
140+
"description" : "Open Targets 20.06. Information onOpen Targets releases can be found [here](https://www.targetvalidation.org/downloads/data).",
141+
"type" : "s3",
142+
"arn" : "arn:aws:s3:::aws-roda-hcls-datalake",
143+
"region" : "us-east-1"
144+
}],
131145
"moreInformation": "License:https://github.com/aws-samples/data-lake-as-code/blob/roda/docs/roda_attributions.txt",
132146
"__v": 0,
133147
"isDeleted": false,
134148
"linkedResources": [],
135149
"isLinked": false,
136150
"repositoryName": "Registry of Open Data on AWS",
137-
"isShared": false,
138-
"createdBy": "",
139-
"accountId": ""
151+
"isShared": false
140152
}
141153
{
142154
"_id": {
@@ -165,16 +177,19 @@
165177
"sharedTo": [],
166178
"name": "A2D2: Audi Autonomous Driving Dataset",
167179
"studyType": "Public",
168-
"resourceDetails": [],
180+
"resourceDetails": [{
181+
"description": "http://a2d2.audi",
182+
"type": "s3",
183+
"arn": "arn:aws:s3:::aev-autonomous-driving-dataset",
184+
"region": "eu-central-1"
185+
}],
169186
"moreInformation": "License:https://creativecommons.org/licenses/by-nd/4.0/",
170187
"__v": 0,
171188
"isDeleted": false,
172189
"repositoryName": "Registry of Open Data on AWS",
173190
"linkedResources": [],
174191
"isLinked": false,
175-
"isShared": false,
176-
"createdBy": "",
177-
"accountId": ""
192+
"isShared": false
178193
}
179194
{
180195
"_id": {
@@ -206,15 +221,18 @@
206221
"sharedTo": [],
207222
"name": "Allen Mouse Brain Atlas",
208223
"studyType": "Public",
209-
"resourceDetails": [],
224+
"resourceDetails": [{
225+
"description": "Project data files in a public bucket",
226+
"type": "s3",
227+
"arn": "arn:aws:s3:::allen-mouse-brain-atlas",
228+
"region": "us-west-2"
229+
}]
210230
"moreInformation": "License:http://www.alleninstitute.org/legal/terms-use/",
211231
"__v": 0,
212232
"isDeleted": false,
213233
"repositoryName": "Registry of Open Data on AWS",
214234
"isShared": false,
215-
"createdBy": "",
216-
"isLinked": false,
217-
"accountId": ""
235+
"isLinked": false
218236
}
219237
{
220238
"_id": {
@@ -241,15 +259,18 @@
241259
"sharedTo": [],
242260
"name": "3000 Rice Genomes Project",
243261
"studyType": "Public",
244-
"resourceDetails": [],
262+
"resourceDetails": [{
263+
"description": "http://s3.amazonaws.com/3kricegenome/README-snp_pipeline.txt",
264+
"type": "s3",
265+
"arn": "arn:aws:s3:::3kricegenome",
266+
"region": "us-east-1"
267+
}],
245268
"moreInformation": "License: This data is available for anyone to use under the terms of the Toronto Statement, which is available [here](http://www.nature.com/nature/journal/v461/n7261/box/461168a_BX1.html)",
246269
"__v": 0,
247270
"isDeleted": false,
248271
"repositoryName": "Registry of Open Data on AWS",
249272
"isShared": false,
250-
"createdBy": "",
251-
"isLinked": false,
252-
"accountId": ""
273+
"isLinked": false
253274
}
254275
{
255276
"_id": {
@@ -284,11 +305,14 @@
284305
"sharedTo": [],
285306
"name": "21st Century Daily Global Seamless Remote Sensing Data Cubes (SDCs)",
286307
"studyType": "Public",
287-
"resourceDetails": [],
308+
"resourceDetails": [{
309+
"description": "Daily Seamless Remote Sensing Data Cube (SDC).",
310+
"type": "s3",
311+
"arn": "arn:aws:s3:::sdc-daily-thu",
312+
"region": "us-west-2"
313+
}]
288314
"moreInformation": "License:Open to non-commercial uses.",
289315
"__v": 0,
290-
"createdBy": "",
291-
"accountId": "",
292316
"isDeleted": true,
293317
"repositoryName": "Registry of Open Data on AWS",
294318
"isShared": false,
@@ -320,16 +344,19 @@
320344
"sharedTo": [],
321345
"name": "Africa Soil Information Service (AfSIS) Soil Chemistry",
322346
"studyType": "Public",
323-
"resourceDetails": [],
347+
"resourceDetails": [{
348+
"description": "Paired wet and dry chemistry measurements for georeferenced soils\ncollected by the Africa Soil Information Service (AfSIS), stored\nas CSV and OPUS files.\n",
349+
"type": "s3",
350+
"arn": "arn:aws:s3:::afsis",
351+
"region": "us-east-1"
352+
}],
324353
"moreInformation": "License:ODC Open Database License (\"[ODbL](https://opendatacommons.org/licenses/odbl/summary/index.html)\") version 1.0, with attribution to AfSIS",
325354
"__v": 0,
326355
"isDeleted": false,
327356
"repositoryName": "Registry of Open Data on AWS",
328357
"linkedResources": [],
329358
"isLinked": false,
330-
"isShared": false,
331-
"createdBy": "",
332-
"accountId": ""
359+
"isShared": false
333360
}
334361
{
335362
"_id": {
@@ -354,16 +381,20 @@
354381
"sharedTo": [],
355382
"name": "AgricultureVision",
356383
"studyType": "Public",
357-
"resourceDetails": [],
384+
"resourceDetails": [{
385+
"description": "Terms of use and paper provided. Dataset provided as a series of tar.gz files with data foreach year and an associated json file dscribing the train/validation/test split.",
386+
"type": "s3",
387+
"arn": "arn:aws:s3:::agriculture-vision",
388+
"region": "us-east-1",
389+
"RequesterPays": true
390+
}],
358391
"moreInformation": "License:Provided in the bucket.",
359392
"__v": 0,
360393
"isDeleted": false,
361394
"linkedResources": [],
362395
"isLinked": false,
363396
"repositoryName": "Registry of Open Data on AWS",
364-
"isShared": false,
365-
"createdBy": "",
366-
"accountId": ""
397+
"isShared": false
367398
}
368399
{
369400
"_id": {
@@ -393,54 +424,17 @@
393424
"sharedTo": [],
394425
"name": "Allen Brain Observatory - Visual Coding AWS Public Data Set",
395426
"studyType": "Public",
396-
"resourceDetails": [],
427+
"resourceDetails": [{
428+
"description": "Project data files in a public bucket",
429+
"type": "s3",
430+
"arn": "arn:aws:s3:::allen-brain-observatory",
431+
"region": "us-west-2"
432+
}],
397433
"moreInformation": "License:http://www.alleninstitute.org/legal/terms-use/",
398434
"__v": 0,
399435
"isDeleted": false,
400436
"linkedResources": [],
401437
"isLinked": false,
402438
"repositoryName": "Registry of Open Data on AWS",
403-
"isShared": false,
404-
"createdBy": "",
405-
"accountId": ""
406-
}
407-
{
408-
"_id": {
409-
"$oid": "5ff449546ae1c429994084dd"
410-
},
411-
"description": "his bucket contains multiple datasets (as Quilt packages) created by the\nAllen Institute for Cell Science (AICS). The imaging data in this bucket contains\neither of the following:\n\n1) field of view images from glass plates\n2) cell membrane, DNA, and structure segmentations\n3) cell membrane, DNA and structure contours\n4) machine learning imaging predictions of the previously listed modalities.\n\nIn addition, many of the datasets include CSVs that contain feature sets\nrelated to that data.\n",
412-
"tags": [
413-
"aws-pds",
414-
"life sciences",
415-
"biology",
416-
"cell imaging",
417-
"cell biology",
418-
"microscopy",
419-
"image processing",
420-
"machine learning",
421-
"Homo sapiens"
422-
],
423-
"bookmarkedBy": [],
424-
"projectId": [],
425-
"createdAt": {
426-
"$date": "2021-01-05T09:30:15.208Z"
427-
},
428-
"updatedAt": {
429-
"$date": "2021-01-05T09:30:15.208Z"
430-
},
431-
"levelOfSharing": 0,
432-
"sharedTo": [],
433-
"name": "Allen Cell Imaging Collections",
434-
"studyType": "Public",
435-
"resourceDetails": [],
436-
"moreInformation": "License:https://www.allencell.org/terms-of-use.html",
437-
"__v": 0,
438-
"createdBy": "",
439-
"accountId": "",
440-
"isDeleted": false,
441-
"listOfLinkedProducts": [],
442-
"isLinked": false,
443-
"isShared": false,
444-
"linkedResources": [],
445-
"repositoryName": "Registry of Open Data on AWS"
439+
"isShared": false
446440
}

‎packer-rg.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
},
4545
{
4646
"type": "file",
47-
"source": "dump.tar.gz",
47+
"source": "dump",
4848
"destination": "{{user `RG_SRC`}}/"
4949
},
5050
{

‎provisioners/provision-rg.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,4 @@ sudo cp ${RG_SRC}/nginx.conf ${RG_HOME}
1111
sudo echo -e "RG_HOME='/opt/deploy/sp2'\n" >> sudo /etc/environment
1212
cd ${RG_SRC}
1313
sudo tar -czf config.tar.gz config
14+
sudo zip dump.zip dump/*

0 commit comments

Comments
 (0)