Skip to content

Commit 121d1fe

Browse files
zainasirzainasir
committed
migrate from mskcc enterprise to knowledgesystems
0 parents  commit 121d1fe

21 files changed

Lines changed: 2777 additions & 0 deletions

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
bootstrap
2+
bootstrap.zip
3+
.idea

‎Makefile‎

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
REGION = us-east-1
2+
BROKER_NAME = github-lfs
3+
SNAPSHOT_NAME = github-lfs-snapshot
4+
PROFILE = YOUR_PROFILE_NAME
5+
LFS_BUCKET = YOUR_BUCKET_NAME
6+
SNAPSHOT_BUCKET = YOUR_BUCKET_NAME
7+
SNAPSHOT_PREFIX = public/
8+
EXTRA_SNAPSHOT_BUCKETS =
9+
ACCOUNT_ID = YOUR_ACCOUNT_ID
10+
ROLE_NAME = github-lfs-lambda-role
11+
REPO = YOUR_ORG/YOUR_REPO
12+
REF = master
13+
GITHUB_HOST = github.com
14+
15+
# -----------------------------------------------------------------------
16+
# Init (first time only - sets up Go modules)
17+
# -----------------------------------------------------------------------
18+
19+
init-broker:
20+
cd lfs-broker && go mod init github.com/cbioportal/lfs-broker && \
21+
go get github.com/aws/aws-lambda-go@latest && \
22+
go get github.com/aws/aws-sdk-go-v2@latest && \
23+
go get github.com/aws/aws-sdk-go-v2/config@latest && \
24+
go get github.com/aws/aws-sdk-go-v2/service/s3@latest && \
25+
go get github.com/aws/aws-sdk-go-v2/service/secretsmanager@latest && \
26+
go mod tidy
27+
28+
init-snapshot:
29+
cd lfs-snapshot && go mod init github.com/cbioportal/lfs-snapshot && \
30+
go get github.com/aws/aws-lambda-go@latest && \
31+
go get github.com/aws/aws-sdk-go-v2@latest && \
32+
go get github.com/aws/aws-sdk-go-v2/config@latest && \
33+
go get github.com/aws/aws-sdk-go-v2/service/s3@latest && \
34+
go mod tidy
35+
36+
init: init-broker init-snapshot
37+
38+
# -----------------------------------------------------------------------
39+
# Bucket setup
40+
# -----------------------------------------------------------------------
41+
42+
configure-bucket:
43+
aws s3api put-public-access-block \
44+
--bucket $(LFS_BUCKET) \
45+
--public-access-block-configuration \
46+
"BlockPublicAcls=false,IgnorePublicAcls=false,BlockPublicPolicy=false,RestrictPublicBuckets=false" \
47+
--region $(REGION) \
48+
--profile $(PROFILE)
49+
aws s3api put-bucket-policy \
50+
--bucket $(LFS_BUCKET) \
51+
--policy file://docs/bucket-policy.json \
52+
--region $(REGION) \
53+
--profile $(PROFILE)
54+
55+
# -----------------------------------------------------------------------
56+
# Build
57+
# -----------------------------------------------------------------------
58+
59+
build-broker:
60+
cd lfs-broker && GOOS=linux GOARCH=amd64 go build -o bootstrap main.go && zip function.zip bootstrap
61+
62+
build-snapshot:
63+
cd lfs-snapshot && GOOS=linux GOARCH=amd64 go build -o bootstrap main.go && zip function.zip bootstrap
64+
65+
build: build-broker build-snapshot
66+
67+
# -----------------------------------------------------------------------
68+
# Create (first time setup)
69+
# -----------------------------------------------------------------------
70+
71+
create-broker: build-broker
72+
aws lambda create-function \
73+
--function-name $(BROKER_NAME) \
74+
--runtime provided.al2023 \
75+
--role arn:aws:iam::$(ACCOUNT_ID):role/$(ROLE_NAME) \
76+
--handler bootstrap \
77+
--zip-file fileb://lfs-broker/function.zip \
78+
--environment Variables="{S3_BUCKET=$(LFS_BUCKET),LFS_SECRET_NAME=github-lfs-api-keys}" \
79+
--timeout 30 \
80+
--region $(REGION) \
81+
--profile $(PROFILE)
82+
aws lambda create-function-url-config \
83+
--function-name $(BROKER_NAME) \
84+
--auth-type NONE \
85+
--region $(REGION) \
86+
--profile $(PROFILE)
87+
aws lambda add-permission \
88+
--function-name $(BROKER_NAME) \
89+
--statement-id FunctionURLAllowPublicAccess \
90+
--action lambda:InvokeFunctionUrl \
91+
--principal "*" \
92+
--function-url-auth-type NONE \
93+
--region $(REGION) \
94+
--profile $(PROFILE)
95+
aws lambda add-permission \
96+
--function-name $(BROKER_NAME) \
97+
--statement-id FunctionURLAllowInvoke \
98+
--action lambda:InvokeFunction \
99+
--principal "*" \
100+
--region $(REGION) \
101+
--profile $(PROFILE)
102+
103+
create-snapshot: build-snapshot
104+
aws lambda create-function \
105+
--function-name $(SNAPSHOT_NAME) \
106+
--runtime provided.al2023 \
107+
--role arn:aws:iam::$(ACCOUNT_ID):role/$(ROLE_NAME) \
108+
--handler bootstrap \
109+
--zip-file fileb://lfs-snapshot/function.zip \
110+
--environment Variables="{LFS_BUCKET=$(LFS_BUCKET),SNAPSHOT_BUCKET=$(SNAPSHOT_BUCKET),SNAPSHOT_PREFIX=$(SNAPSHOT_PREFIX),EXTRA_SNAPSHOT_BUCKETS=$(EXTRA_SNAPSHOT_BUCKETS)}" \
111+
--timeout 300 \
112+
--region $(REGION) \
113+
--profile $(PROFILE)
114+
aws lambda create-function-url-config \
115+
--function-name $(SNAPSHOT_NAME) \
116+
--auth-type NONE \
117+
--region $(REGION) \
118+
--profile $(PROFILE)
119+
aws lambda add-permission \
120+
--function-name $(SNAPSHOT_NAME) \
121+
--statement-id FunctionURLAllowPublicAccess \
122+
--action lambda:InvokeFunctionUrl \
123+
--principal "*" \
124+
--function-url-auth-type NONE \
125+
--region $(REGION) \
126+
--profile $(PROFILE)
127+
aws lambda add-permission \
128+
--function-name $(SNAPSHOT_NAME) \
129+
--statement-id FunctionURLAllowInvoke \
130+
--action lambda:InvokeFunction \
131+
--principal "*" \
132+
--region $(REGION) \
133+
--profile $(PROFILE)
134+
135+
create: create-broker create-snapshot
136+
137+
# -----------------------------------------------------------------------
138+
# Deploy (update existing functions)
139+
# -----------------------------------------------------------------------
140+
141+
deploy-broker: build-broker
142+
aws lambda update-function-code \
143+
--function-name $(BROKER_NAME) \
144+
--zip-file fileb://lfs-broker/function.zip \
145+
--region $(REGION) \
146+
--profile $(PROFILE)
147+
148+
deploy-snapshot: build-snapshot
149+
aws lambda update-function-code \
150+
--function-name $(SNAPSHOT_NAME) \
151+
--zip-file fileb://lfs-snapshot/function.zip \
152+
--region $(REGION) \
153+
--profile $(PROFILE)
154+
155+
deploy: deploy-broker deploy-snapshot
156+
157+
# -----------------------------------------------------------------------
158+
# Logs
159+
# -----------------------------------------------------------------------
160+
161+
logs-broker:
162+
aws logs tail /aws/lambda/$(BROKER_NAME) \
163+
--region $(REGION) \
164+
--profile $(PROFILE) \
165+
--follow
166+
167+
logs-snapshot:
168+
aws logs tail /aws/lambda/$(SNAPSHOT_NAME) \
169+
--region $(REGION) \
170+
--profile $(PROFILE) \
171+
--follow
172+
173+
# -----------------------------------------------------------------------
174+
# Info
175+
# -----------------------------------------------------------------------
176+
177+
url-broker:
178+
aws lambda get-function-url-config \
179+
--function-name $(BROKER_NAME) \
180+
--region $(REGION) \
181+
--profile $(PROFILE) \
182+
--query FunctionUrl \
183+
--output text
184+
185+
url-snapshot:
186+
aws lambda get-function-url-config \
187+
--function-name $(SNAPSHOT_NAME) \
188+
--region $(REGION) \
189+
--profile $(PROFILE) \
190+
--query FunctionUrl \
191+
--output text
192+
193+
# -----------------------------------------------------------------------
194+
# Secrets
195+
# -----------------------------------------------------------------------
196+
197+
list-curators:
198+
aws secretsmanager get-secret-value \
199+
--secret-id github-lfs-api-keys \
200+
--region $(REGION) \
201+
--profile $(PROFILE) \
202+
--query SecretString \
203+
--output text | jq 'keys'
204+
205+
# -----------------------------------------------------------------------
206+
# Snapshot
207+
# -----------------------------------------------------------------------
208+
209+
full-snapshot:
210+
$(eval TOKEN := $(shell gh auth token --hostname $(GITHUB_HOST)))
211+
$(eval URL := $(shell make url-snapshot --no-print-directory))
212+
curl -X POST $(URL) \
213+
-H "Content-Type: application/json" \
214+
-d '{"repo":"$(REPO)","ref":"$(REF)","token":"$(TOKEN)","github_host":"$(GITHUB_HOST)"}'
215+
216+
.PHONY: init init-broker init-snapshot \
217+
configure-bucket \
218+
build build-broker build-snapshot \
219+
create create-broker create-snapshot \
220+
deploy deploy-broker deploy-snapshot \
221+
logs-broker logs-snapshot \
222+
url-broker url-snapshot \
223+
list-curators full-snapshot

0 commit comments

Comments
 (0)